This commit is contained in:
Kevin Darby committed 2019-01-02 17:04:10 -05:00
1 parent eb1b42a5e8
commit 458eb2aef1
1283 files changed
+323373

No files matched your search

@@ -0,0 +1,156 @@
/* Copyright 2015 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
#ifndef TENSORFLOW_CORE_KERNELS_ADJUST_CONTRAST_OP_H_
#define TENSORFLOW_CORE_KERNELS_ADJUST_CONTRAST_OP_H_
#include "third_party/eigen3/unsupported/Eigen/CXX11/Tensor"
#include "tensorflow/core/framework/tensor_types.h"
namespace tensorflow {
namespace functor {
// Functor used by AdjustContrastOp to do the computations.
template <typename Device, typename T>
struct AdjustContrast {
void operator()(const Device& d, typename TTypes<T, 4>::ConstTensor input,
typename TTypes<float>::ConstScalar contrast_factor,
typename TTypes<float>::ConstScalar min_value,
typename TTypes<float>::ConstScalar max_value,
typename TTypes<float, 4>::Tensor mean_values,
typename TTypes<float, 4>::Tensor output) {
const int batch = input.dimension(0);
const int height = input.dimension(1);
const int width = input.dimension(2);
const int channels = input.dimension(3);
Eigen::array<int, 4> scalar_broadcast;
scalar_broadcast[0] = batch;
scalar_broadcast[1] = height;
scalar_broadcast[2] = width;
scalar_broadcast[3] = channels;
#if !defined(EIGEN_HAS_INDEX_LIST)
Eigen::array<int, 2> reduction_axis;
reduction_axis[0] = 1;
reduction_axis[1] = 2;
Eigen::array<int, 4> broadcast_dims;
broadcast_dims[0] = 1;
broadcast_dims[1] = height;
broadcast_dims[2] = width;
broadcast_dims[3] = 1;
Eigen::Tensor<int, 4>::Dimensions reshape_dims;
reshape_dims[0] = batch;
reshape_dims[1] = 1;
reshape_dims[2] = 1;
reshape_dims[3] = channels;
#else
Eigen::IndexList<Eigen::type2index<1>, Eigen::type2index<2> >
reduction_axis;
Eigen::IndexList<Eigen::type2index<1>, int, int, Eigen::type2index<1> >
broadcast_dims;
broadcast_dims.set(1, height);
broadcast_dims.set(2, width);
Eigen::IndexList<int, Eigen::type2index<1>, Eigen::type2index<1>, int>
reshape_dims;
reshape_dims.set(0, batch);
reshape_dims.set(3, channels);
#endif
Eigen::Sizes<1, 1, 1, 1> scalar;
float num_reduced_coeffs = height * width;
mean_values.device(d) =
(input.template cast<float>().sum(reduction_axis).eval() /
num_reduced_coeffs)
.reshape(reshape_dims)
.broadcast(broadcast_dims);
auto contrast_factor_tensor =
contrast_factor.reshape(scalar).broadcast(scalar_broadcast);
auto adjusted =
(input.template cast<float>() - mean_values) * contrast_factor_tensor +
mean_values;
auto min_bcast = min_value.reshape(scalar).broadcast(scalar_broadcast);
auto max_bcast = max_value.reshape(scalar).broadcast(scalar_broadcast);
// TODO(wicke): This is rather slow and should be re-written as pure cuda.
output.device(d) = adjusted.cwiseMin(max_bcast).cwiseMax(min_bcast);
}
};
// Functor used by AdjustContrastOpv2 to do the computations.
template <typename Device>
struct AdjustContrastv2 {
void operator()(const Device& d, typename TTypes<float, 4>::ConstTensor input,
typename TTypes<float>::ConstScalar contrast_factor,
typename TTypes<float, 4>::Tensor output) {
const int batch = input.dimension(0);
const int height = input.dimension(1);
const int width = input.dimension(2);
const int channels = input.dimension(3);
Eigen::array<int, 4> scalar_broadcast;
scalar_broadcast[0] = batch;
scalar_broadcast[1] = height;
scalar_broadcast[2] = width;
scalar_broadcast[3] = channels;
#if !defined(EIGEN_HAS_INDEX_LIST)
Eigen::array<int, 2> reduction_axis;
reduction_axis[0] = 0;
reduction_axis[1] = 1;
Eigen::array<int, 4> broadcast_dims;
broadcast_dims[0] = 1;
broadcast_dims[1] = height;
broadcast_dims[2] = width;
broadcast_dims[3] = 1;
Eigen::Tensor<int, 4>::Dimensions reshape_dims;
reshape_dims[0] = batch;
reshape_dims[1] = 1;
reshape_dims[2] = 1;
reshape_dims[3] = channels;
Eigen::array<int, 4> reduced_dims_first;
reduced_dims_first[0] = 1;
reduced_dims_first[1] = 2;
reduced_dims_first[2] = 0;
reduced_dims_first[3] = 3;
#else
Eigen::IndexList<Eigen::type2index<0>, Eigen::type2index<1> >
reduction_axis;
Eigen::IndexList<Eigen::type2index<1>, int, int, Eigen::type2index<1> >
broadcast_dims;
broadcast_dims.set(1, height);
broadcast_dims.set(2, width);
Eigen::IndexList<int, Eigen::type2index<1>, Eigen::type2index<1>, int>
reshape_dims;
reshape_dims.set(0, batch);
reshape_dims.set(3, channels);
Eigen::IndexList<Eigen::type2index<1>, Eigen::type2index<2>,
Eigen::type2index<0>, Eigen::type2index<3> >
reduced_dims_first;
#endif
Eigen::Sizes<1, 1, 1, 1> scalar;
float num_reduced_coeffs = height * width;
output.device(d) =
(input.shuffle(reduced_dims_first).sum(reduction_axis).eval() /
num_reduced_coeffs)
.reshape(reshape_dims)
.broadcast(broadcast_dims);
auto contrast_factor_tensor =
contrast_factor.reshape(scalar).broadcast(scalar_broadcast);
auto adjusted = (input - output) * contrast_factor_tensor;
output.device(d) += adjusted;
}
};
} // namespace functor
} // namespace tensorflow
#endif // TENSORFLOW_CORE_KERNELS_ADJUST_CONTRAST_OP_H_
@@ -0,0 +1,146 @@
/* Copyright 2017 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
#ifndef TENSORFLOW_CORE_KERNELS_ADJUST_HSV_GPU_CU_H_
#define TENSORFLOW_CORE_KERNELS_ADJUST_HSV_GPU_CU_H_
#if GOOGLE_CUDA
#define EIGEN_USE_GPU
#include "tensorflow/core/framework/register_types.h"
#include "tensorflow/core/framework/types.h"
namespace tensorflow {
namespace internal {
typedef struct RgbTuple {
float r;
float g;
float b;
} RgbTuple;
typedef struct HsvTuple {
float h;
float s;
float v;
} HsvTuple;
inline __device__ HsvTuple rgb2hsv_cuda(const float r, const float g,
const float b) {
HsvTuple tuple;
const float M = fmaxf(r, fmaxf(g, b));
const float m = fminf(r, fminf(g, b));
const float chroma = M - m;
float h = 0.0f, s = 0.0f;
// hue
if (chroma > 0.0f) {
if (M == r) {
const float num = (g - b) / chroma;
const float sign = copysignf(1.0f, num);
h = ((sign < 0.0f) * 6.0f + sign * fmodf(sign * num, 6.0f)) / 6.0f;
} else if (M == g) {
h = ((b - r) / chroma + 2.0f) / 6.0f;
} else {
h = ((r - g) / chroma + 4.0f) / 6.0f;
}
} else {
h = 0.0f;
}
// saturation
if (M > 0.0) {
s = chroma / M;
} else {
s = 0.0f;
}
tuple.h = h;
tuple.s = s;
tuple.v = M;
return tuple;
}
inline __device__ RgbTuple hsv2rgb_cuda(const float h, const float s,
const float v) {
RgbTuple tuple;
const float new_h = h * 6.0f;
const float chroma = v * s;
const float x = chroma * (1.0f - fabsf(fmodf(new_h, 2.0f) - 1.0f));
const float new_m = v - chroma;
const bool between_0_and_1 = new_h >= 0.0f && new_h < 1.0f;
const bool between_1_and_2 = new_h >= 1.0f && new_h < 2.0f;
const bool between_2_and_3 = new_h >= 2.0f && new_h < 3.0f;
const bool between_3_and_4 = new_h >= 3.0f && new_h < 4.0f;
const bool between_4_and_5 = new_h >= 4.0f && new_h < 5.0f;
const bool between_5_and_6 = new_h >= 5.0f && new_h < 6.0f;
tuple.r = chroma * (between_0_and_1 || between_5_and_6) +
x * (between_1_and_2 || between_4_and_5) + new_m;
tuple.g = chroma * (between_1_and_2 || between_2_and_3) +
x * (between_0_and_1 || between_3_and_4) + new_m;
tuple.b = chroma * (between_3_and_4 || between_4_and_5) +
x * (between_2_and_3 || between_5_and_6) + new_m;
return tuple;
}
template <bool AdjustHue, bool AdjustSaturation, bool AdjustV>
__global__ void adjust_hsv_nhwc(const int64 number_elements,
const float* const __restrict__ input,
float* const output,
const float* const hue_delta,
const float* const saturation_scale,
const float* const value_scale) {
// multiply by 3 since we're dealing with contiguous RGB bytes for each pixel
// (NHWC)
const int64 idx = (blockDim.x * blockIdx.x + threadIdx.x) * 3;
// bounds check
if (idx > number_elements - 1) {
return;
}
if (!AdjustHue && !AdjustSaturation && !AdjustV) {
output[idx] = input[idx];
output[idx + 1] = input[idx + 1];
output[idx + 2] = input[idx + 2];
return;
}
const HsvTuple hsv = rgb2hsv_cuda(input[idx], input[idx + 1], input[idx + 2]);
float new_h = hsv.h;
float new_s = hsv.s;
float new_v = hsv.v;
// hue adjustment
if (AdjustHue) {
const float delta = *hue_delta;
new_h = fmodf(hsv.h + delta, 1.0f);
if (new_h < 0.0f) {
new_h = fmodf(1.0f + new_h, 1.0f);
}
}
// saturation adjustment
if (AdjustSaturation && saturation_scale != nullptr) {
const float scale = *saturation_scale;
new_s = fminf(1.0f, fmaxf(0.0f, hsv.s * scale));
}
// value adjustment
if (AdjustV && value_scale != nullptr) {
const float scale = *value_scale;
new_v = hsv.v * scale;
}
const RgbTuple rgb = hsv2rgb_cuda(new_h, new_s, new_v);
output[idx] = rgb.r;
output[idx + 1] = rgb.g;
output[idx + 2] = rgb.b;
}
} // namespace internal
} // namespace tensorflow
#endif // GOOGLE_CUDA
#endif // TENSORFLOW_CORE_KERNELS_ADJUST_HSV_GPU_CU_H_
@@ -0,0 +1,40 @@
/* Copyright 2016 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
#ifndef TENSORFLOW_CORE_KERNELS_ADJUST_HUE_OP_H_
#define TENSORFLOW_CORE_KERNELS_ADJUST_HUE_OP_H_
#if GOOGLE_CUDA
#define EIGEN_USE_GPU
#include "third_party/eigen3/unsupported/Eigen/CXX11/Tensor"
#include "tensorflow/core/framework/types.h"
namespace tensorflow {
typedef Eigen::GpuDevice GPUDevice;
namespace functor {
struct AdjustHueGPU {
void operator()(GPUDevice* device, const int64 number_of_elements,
const float* const input, const float* const delta,
float* const output);
};
} // namespace functor
} // namespace tensorflow
#endif // GOOGLE_CUDA
#endif // TENSORFLOW_CORE_KERNELS_ADJUST_HUE_OP_H_
@@ -0,0 +1,40 @@
/* Copyright 2017 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
#ifndef TENSORFLOW_CORE_KERNELS_ADJUST_SATURATION_OP_H_
#define TENSORFLOW_CORE_KERNELS_ADJUST_SATURATION_OP_H_
#if GOOGLE_CUDA
#define EIGEN_USE_GPU
#include "third_party/eigen3/unsupported/Eigen/CXX11/Tensor"
#include "tensorflow/core/framework/types.h"
namespace tensorflow {
typedef Eigen::GpuDevice GPUDevice;
namespace functor {
struct AdjustSaturationGPU {
void operator()(GPUDevice* device, const int64 number_of_elements,
const float* const input, const float* const scale,
float* const output);
};
} // namespace functor
} // namespace tensorflow
#endif // GOOGLE_CUDA
#endif // TENSORFLOW_CORE_KERNELS_ADJUST_SATURATION_OP_H_
@@ -0,0 +1,226 @@
/* Copyright 2015 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
#ifndef TENSORFLOW_CORE_KERNELS_AGGREGATE_OPS_H_
#define TENSORFLOW_CORE_KERNELS_AGGREGATE_OPS_H_
// Functor definitions for Aggregate ops, must be compilable by nvcc.
#include "third_party/eigen3/unsupported/Eigen/CXX11/Tensor"
#include "tensorflow/core/framework/tensor_types.h"
namespace tensorflow {
namespace functor {
template <typename Device, typename T>
struct Add2Functor {
void operator()(const Device& d, typename TTypes<T>::Flat out,
typename TTypes<T>::ConstFlat in1,
typename TTypes<T>::ConstFlat in2);
};
template <typename Device, typename T>
struct Add2EigenImpl {
static void Compute(const Device& d, typename TTypes<T>::Flat out,
typename TTypes<T>::ConstFlat in1,
typename TTypes<T>::ConstFlat in2) {
out.device(d) = in1 + in2;
}
};
template <typename Device, typename T>
struct Add3Functor {
void operator()(const Device& d, typename TTypes<T>::Flat out,
typename TTypes<T>::ConstFlat in1,
typename TTypes<T>::ConstFlat in2,
typename TTypes<T>::ConstFlat in3);
};
template <typename Device, typename T>
struct Add3EigenImpl {
static void Compute(const Device& d, typename TTypes<T>::Flat out,
typename TTypes<T>::ConstFlat in1,
typename TTypes<T>::ConstFlat in2,
typename TTypes<T>::ConstFlat in3) {
out.device(d) = in1 + in2 + in3;
}
};
template <typename Device, typename T>
struct Add4Functor {
void operator()(const Device& d, typename TTypes<T>::Flat out,
typename TTypes<T>::ConstFlat in1,
typename TTypes<T>::ConstFlat in2,
typename TTypes<T>::ConstFlat in3,
typename TTypes<T>::ConstFlat in4);
};
template <typename Device, typename T>
struct Add4EigenImpl {
static void Compute(const Device& d, typename TTypes<T>::Flat out,
typename TTypes<T>::ConstFlat in1,
typename TTypes<T>::ConstFlat in2,
typename TTypes<T>::ConstFlat in3,
typename TTypes<T>::ConstFlat in4) {
out.device(d) = in1 + in2 + in3 + in4;
}
};
template <typename Device, typename T>
struct Add5Functor {
void operator()(const Device& d, typename TTypes<T>::Flat out,
typename TTypes<T>::ConstFlat in1,
typename TTypes<T>::ConstFlat in2,
typename TTypes<T>::ConstFlat in3,
typename TTypes<T>::ConstFlat in4,
typename TTypes<T>::ConstFlat in5);
};
template <typename Device, typename T>
struct Add5EigenImpl {
static void Compute(const Device& d, typename TTypes<T>::Flat out,
typename TTypes<T>::ConstFlat in1,
typename TTypes<T>::ConstFlat in2,
typename TTypes<T>::ConstFlat in3,
typename TTypes<T>::ConstFlat in4,
typename TTypes<T>::ConstFlat in5) {
out.device(d) = in1 + in2 + in3 + in4 + in5;
}
};
template <typename Device, typename T>
struct Add6Functor {
void operator()(const Device& d, typename TTypes<T>::Flat out,
typename TTypes<T>::ConstFlat in1,
typename TTypes<T>::ConstFlat in2,
typename TTypes<T>::ConstFlat in3,
typename TTypes<T>::ConstFlat in4,
typename TTypes<T>::ConstFlat in5,
typename TTypes<T>::ConstFlat in6);
};
template <typename Device, typename T>
struct Add6EigenImpl {
static void Compute(const Device& d, typename TTypes<T>::Flat out,
typename TTypes<T>::ConstFlat in1,
typename TTypes<T>::ConstFlat in2,
typename TTypes<T>::ConstFlat in3,
typename TTypes<T>::ConstFlat in4,
typename TTypes<T>::ConstFlat in5,
typename TTypes<T>::ConstFlat in6) {
out.device(d) = in1 + in2 + in3 + in4 + in5 + in6;
}
};
template <typename Device, typename T>
struct Add7Functor {
void operator()(const Device& d, typename TTypes<T>::Flat out,
typename TTypes<T>::ConstFlat in1,
typename TTypes<T>::ConstFlat in2,
typename TTypes<T>::ConstFlat in3,
typename TTypes<T>::ConstFlat in4,
typename TTypes<T>::ConstFlat in5,
typename TTypes<T>::ConstFlat in6,
typename TTypes<T>::ConstFlat in7);
};
template <typename Device, typename T>
struct Add7EigenImpl {
static void Compute(const Device& d, typename TTypes<T>::Flat out,
typename TTypes<T>::ConstFlat in1,
typename TTypes<T>::ConstFlat in2,
typename TTypes<T>::ConstFlat in3,
typename TTypes<T>::ConstFlat in4,
typename TTypes<T>::ConstFlat in5,
typename TTypes<T>::ConstFlat in6,
typename TTypes<T>::ConstFlat in7) {
out.device(d) = in1 + in2 + in3 + in4 + in5 + in6 + in7;
}
};
template <typename Device, typename T>
struct Add8Functor {
void operator()(
const Device& d, typename TTypes<T>::Flat out,
typename TTypes<T>::ConstFlat in1, typename TTypes<T>::ConstFlat in2,
typename TTypes<T>::ConstFlat in3, typename TTypes<T>::ConstFlat in4,
typename TTypes<T>::ConstFlat in5, typename TTypes<T>::ConstFlat in6,
typename TTypes<T>::ConstFlat in7, typename TTypes<T>::ConstFlat in8);
};
template <typename Device, typename T>
struct Add8EigenImpl {
static void Compute(
const Device& d, typename TTypes<T>::Flat out,
typename TTypes<T>::ConstFlat in1, typename TTypes<T>::ConstFlat in2,
typename TTypes<T>::ConstFlat in3, typename TTypes<T>::ConstFlat in4,
typename TTypes<T>::ConstFlat in5, typename TTypes<T>::ConstFlat in6,
typename TTypes<T>::ConstFlat in7, typename TTypes<T>::ConstFlat in8) {
out.device(d) = in1 + in2 + in3 + in4 + in5 + in6 + in7 + in8;
}
};
// Add8p is like Add8 except the underlying implementation should +=
// rather than assign to the output.
template <typename Device, typename T>
struct Add8pFunctor {
void operator()(
const Device& d, typename TTypes<T>::Flat out,
typename TTypes<T>::ConstFlat in1, typename TTypes<T>::ConstFlat in2,
typename TTypes<T>::ConstFlat in3, typename TTypes<T>::ConstFlat in4,
typename TTypes<T>::ConstFlat in5, typename TTypes<T>::ConstFlat in6,
typename TTypes<T>::ConstFlat in7, typename TTypes<T>::ConstFlat in8);
};
template <typename Device, typename T>
struct Add8pEigenImpl {
static void Compute(
const Device& d, typename TTypes<T>::Flat out,
typename TTypes<T>::ConstFlat in1, typename TTypes<T>::ConstFlat in2,
typename TTypes<T>::ConstFlat in3, typename TTypes<T>::ConstFlat in4,
typename TTypes<T>::ConstFlat in5, typename TTypes<T>::ConstFlat in6,
typename TTypes<T>::ConstFlat in7, typename TTypes<T>::ConstFlat in8) {
out.device(d) += in1 + in2 + in3 + in4 + in5 + in6 + in7 + in8;
}
};
template <typename Device, typename T>
struct Add9Functor {
void operator()(
const Device& d, typename TTypes<T>::Flat out,
typename TTypes<T>::ConstFlat in1, typename TTypes<T>::ConstFlat in2,
typename TTypes<T>::ConstFlat in3, typename TTypes<T>::ConstFlat in4,
typename TTypes<T>::ConstFlat in5, typename TTypes<T>::ConstFlat in6,
typename TTypes<T>::ConstFlat in7, typename TTypes<T>::ConstFlat in8,
typename TTypes<T>::ConstFlat in9);
};
template <typename Device, typename T>
struct Add9EigenImpl {
static void Compute(
const Device& d, typename TTypes<T>::Flat out,
typename TTypes<T>::ConstFlat in1, typename TTypes<T>::ConstFlat in2,
typename TTypes<T>::ConstFlat in3, typename TTypes<T>::ConstFlat in4,
typename TTypes<T>::ConstFlat in5, typename TTypes<T>::ConstFlat in6,
typename TTypes<T>::ConstFlat in7, typename TTypes<T>::ConstFlat in8,
typename TTypes<T>::ConstFlat in9) {
out.device(d) = in1 + in2 + in3 + in4 + in5 + in6 + in7 + in8 + in9;
}
};
} // namespace functor
} // namespace tensorflow
#endif // TENSORFLOW_CORE_KERNELS_AGGREGATE_OPS_H_
@@ -0,0 +1,253 @@
/* Copyright 2015 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
#ifndef TENSORFLOW_CORE_KERNELS_AGGREGATE_OPS_CPU_H_
#define TENSORFLOW_CORE_KERNELS_AGGREGATE_OPS_CPU_H_
#include "third_party/eigen3/unsupported/Eigen/CXX11/Tensor"
#include "tensorflow/core/framework/tensor_types.h"
#include "tensorflow/core/kernels/aggregate_ops.h"
typedef Eigen::ThreadPoolDevice CPUDevice;
#ifdef TENSORFLOW_USE_SYCL
typedef Eigen::SyclDevice SYCLDevice;
#endif // TENSORFLOW_USE_SYCL
namespace tensorflow {
// Partial specializations for a CPUDevice, that uses the Eigen implementation
// from AddNEigenImpl.
namespace functor {
template <typename T>
struct Add2Functor<CPUDevice, T> {
void operator()(const CPUDevice& d, typename TTypes<T>::Flat out,
typename TTypes<T>::ConstFlat in1,
typename TTypes<T>::ConstFlat in2) {
Add2EigenImpl<CPUDevice, T>::Compute(d, out, in1, in2);
}
};
template <typename T>
struct Add3Functor<CPUDevice, T> {
void operator()(const CPUDevice& d, typename TTypes<T>::Flat out,
typename TTypes<T>::ConstFlat in1,
typename TTypes<T>::ConstFlat in2,
typename TTypes<T>::ConstFlat in3) {
Add3EigenImpl<CPUDevice, T>::Compute(d, out, in1, in2, in3);
}
};
template <typename T>
struct Add4Functor<CPUDevice, T> {
void operator()(const CPUDevice& d, typename TTypes<T>::Flat out,
typename TTypes<T>::ConstFlat in1,
typename TTypes<T>::ConstFlat in2,
typename TTypes<T>::ConstFlat in3,
typename TTypes<T>::ConstFlat in4) {
Add4EigenImpl<CPUDevice, T>::Compute(d, out, in1, in2, in3, in4);
}
};
template <typename T>
struct Add5Functor<CPUDevice, T> {
void operator()(const CPUDevice& d, typename TTypes<T>::Flat out,
typename TTypes<T>::ConstFlat in1,
typename TTypes<T>::ConstFlat in2,
typename TTypes<T>::ConstFlat in3,
typename TTypes<T>::ConstFlat in4,
typename TTypes<T>::ConstFlat in5) {
Add5EigenImpl<CPUDevice, T>::Compute(d, out, in1, in2, in3, in4, in5);
}
};
template <typename T>
struct Add6Functor<CPUDevice, T> {
void operator()(const CPUDevice& d, typename TTypes<T>::Flat out,
typename TTypes<T>::ConstFlat in1,
typename TTypes<T>::ConstFlat in2,
typename TTypes<T>::ConstFlat in3,
typename TTypes<T>::ConstFlat in4,
typename TTypes<T>::ConstFlat in5,
typename TTypes<T>::ConstFlat in6) {
Add6EigenImpl<CPUDevice, T>::Compute(d, out, in1, in2, in3, in4, in5, in6);
}
};
template <typename T>
struct Add7Functor<CPUDevice, T> {
void operator()(const CPUDevice& d, typename TTypes<T>::Flat out,
typename TTypes<T>::ConstFlat in1,
typename TTypes<T>::ConstFlat in2,
typename TTypes<T>::ConstFlat in3,
typename TTypes<T>::ConstFlat in4,
typename TTypes<T>::ConstFlat in5,
typename TTypes<T>::ConstFlat in6,
typename TTypes<T>::ConstFlat in7) {
Add7EigenImpl<CPUDevice, T>::Compute(d, out, in1, in2, in3, in4, in5, in6,
in7);
}
};
template <typename T>
struct Add8Functor<CPUDevice, T> {
void operator()(
const CPUDevice& d, typename TTypes<T>::Flat out,
typename TTypes<T>::ConstFlat in1, typename TTypes<T>::ConstFlat in2,
typename TTypes<T>::ConstFlat in3, typename TTypes<T>::ConstFlat in4,
typename TTypes<T>::ConstFlat in5, typename TTypes<T>::ConstFlat in6,
typename TTypes<T>::ConstFlat in7, typename TTypes<T>::ConstFlat in8) {
Add8EigenImpl<CPUDevice, T>::Compute(d, out, in1, in2, in3, in4, in5, in6,
in7, in8);
}
};
template <typename T>
struct Add8pFunctor<CPUDevice, T> {
void operator()(
const CPUDevice& d, typename TTypes<T>::Flat out,
typename TTypes<T>::ConstFlat in1, typename TTypes<T>::ConstFlat in2,
typename TTypes<T>::ConstFlat in3, typename TTypes<T>::ConstFlat in4,
typename TTypes<T>::ConstFlat in5, typename TTypes<T>::ConstFlat in6,
typename TTypes<T>::ConstFlat in7, typename TTypes<T>::ConstFlat in8) {
Add8pEigenImpl<CPUDevice, T>::Compute(d, out, in1, in2, in3, in4, in5, in6,
in7, in8);
}
};
template <typename T>
struct Add9Functor<CPUDevice, T> {
void operator()(
const CPUDevice& d, typename TTypes<T>::Flat out,
typename TTypes<T>::ConstFlat in1, typename TTypes<T>::ConstFlat in2,
typename TTypes<T>::ConstFlat in3, typename TTypes<T>::ConstFlat in4,
typename TTypes<T>::ConstFlat in5, typename TTypes<T>::ConstFlat in6,
typename TTypes<T>::ConstFlat in7, typename TTypes<T>::ConstFlat in8,
typename TTypes<T>::ConstFlat in9) {
Add9EigenImpl<CPUDevice, T>::Compute(d, out, in1, in2, in3, in4, in5, in6,
in7, in8, in9);
}
};
#ifdef TENSORFLOW_USE_SYCL
// Partial specializations for a SYCLDevice, that uses the Eigen implementation
// from AddNEigenImpl.
template <typename T>
struct Add2Functor<SYCLDevice, T> {
void operator()(const SYCLDevice& d, typename TTypes<T>::Flat out,
typename TTypes<T>::ConstFlat in1,
typename TTypes<T>::ConstFlat in2) {
Add2EigenImpl<SYCLDevice, T>::Compute(d, out, in1, in2);
}
};
template <typename T>
struct Add3Functor<SYCLDevice, T> {
void operator()(const SYCLDevice& d, typename TTypes<T>::Flat out,
typename TTypes<T>::ConstFlat in1,
typename TTypes<T>::ConstFlat in2,
typename TTypes<T>::ConstFlat in3) {
Add3EigenImpl<SYCLDevice, T>::Compute(d, out, in1, in2, in3);
}
};
template <typename T>
struct Add4Functor<SYCLDevice, T> {
void operator()(const SYCLDevice& d, typename TTypes<T>::Flat out,
typename TTypes<T>::ConstFlat in1,
typename TTypes<T>::ConstFlat in2,
typename TTypes<T>::ConstFlat in3,
typename TTypes<T>::ConstFlat in4) {
Add4EigenImpl<SYCLDevice, T>::Compute(d, out, in1, in2, in3, in4);
}
};
template <typename T>
struct Add5Functor<SYCLDevice, T> {
void operator()(const SYCLDevice& d, typename TTypes<T>::Flat out,
typename TTypes<T>::ConstFlat in1,
typename TTypes<T>::ConstFlat in2,
typename TTypes<T>::ConstFlat in3,
typename TTypes<T>::ConstFlat in4,
typename TTypes<T>::ConstFlat in5) {
Add5EigenImpl<SYCLDevice, T>::Compute(d, out, in1, in2, in3, in4, in5);
}
};
template <typename T>
struct Add6Functor<SYCLDevice, T> {
void operator()(const SYCLDevice& d, typename TTypes<T>::Flat out,
typename TTypes<T>::ConstFlat in1,
typename TTypes<T>::ConstFlat in2,
typename TTypes<T>::ConstFlat in3,
typename TTypes<T>::ConstFlat in4,
typename TTypes<T>::ConstFlat in5,
typename TTypes<T>::ConstFlat in6) {
Add6EigenImpl<SYCLDevice, T>::Compute(d, out, in1, in2, in3, in4, in5, in6);
}
};
template <typename T>
struct Add7Functor<SYCLDevice, T> {
void operator()(const SYCLDevice& d, typename TTypes<T>::Flat out,
typename TTypes<T>::ConstFlat in1,
typename TTypes<T>::ConstFlat in2,
typename TTypes<T>::ConstFlat in3,
typename TTypes<T>::ConstFlat in4,
typename TTypes<T>::ConstFlat in5,
typename TTypes<T>::ConstFlat in6,
typename TTypes<T>::ConstFlat in7) {
Add7EigenImpl<SYCLDevice, T>::Compute(d, out, in1, in2, in3, in4, in5, in6,
in7);
}
};
template <typename T>
struct Add8Functor<SYCLDevice, T> {
void operator()(
const SYCLDevice& d, typename TTypes<T>::Flat out,
typename TTypes<T>::ConstFlat in1, typename TTypes<T>::ConstFlat in2,
typename TTypes<T>::ConstFlat in3, typename TTypes<T>::ConstFlat in4,
typename TTypes<T>::ConstFlat in5, typename TTypes<T>::ConstFlat in6,
typename TTypes<T>::ConstFlat in7, typename TTypes<T>::ConstFlat in8) {
Add8EigenImpl<SYCLDevice, T>::Compute(d, out, in1, in2, in3, in4, in5, in6,
in7, in8);
}
};
template <typename T>
struct Add8pFunctor<SYCLDevice, T> {
void operator()(
const SYCLDevice& d, typename TTypes<T>::Flat out,
typename TTypes<T>::ConstFlat in1, typename TTypes<T>::ConstFlat in2,
typename TTypes<T>::ConstFlat in3, typename TTypes<T>::ConstFlat in4,
typename TTypes<T>::ConstFlat in5, typename TTypes<T>::ConstFlat in6,
typename TTypes<T>::ConstFlat in7, typename TTypes<T>::ConstFlat in8) {
Add8pEigenImpl<SYCLDevice, T>::Compute(d, out, in1, in2, in3, in4, in5, in6,
in7, in8);
}
};
template <typename T>
struct Add9Functor<SYCLDevice, T> {
void operator()(
const SYCLDevice& d, typename TTypes<T>::Flat out,
typename TTypes<T>::ConstFlat in1, typename TTypes<T>::ConstFlat in2,
typename TTypes<T>::ConstFlat in3, typename TTypes<T>::ConstFlat in4,
typename TTypes<T>::ConstFlat in5, typename TTypes<T>::ConstFlat in6,
typename TTypes<T>::ConstFlat in7, typename TTypes<T>::ConstFlat in8,
typename TTypes<T>::ConstFlat in9) {
Add9EigenImpl<SYCLDevice, T>::Compute(d, out, in1, in2, in3, in4, in5, in6,
in7, in8, in9);
}
};
#endif // TENSORFLOW_USE_SYCL
} // namespace functor
} // namespace tensorflow
#endif // TENSORFLOW_CORE_KERNELS_AGGREGATE_OPS_CPU_H_
@@ -0,0 +1,68 @@
/* Copyright 2015 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
#ifndef TENSORFLOW_CORE_KERNELS_ARGMAX_OP_H_
#define TENSORFLOW_CORE_KERNELS_ARGMAX_OP_H_
// Generator definition for ArgMaxOp, must be compilable by nvcc.
#include "third_party/eigen3/unsupported/Eigen/CXX11/Tensor"
#include "tensorflow/core/framework/tensor_types.h"
#include "tensorflow/core/platform/types.h"
namespace tensorflow {
namespace functor {
template <typename Device, typename T, typename Tout>
struct ArgMax {
#define DECLARE_COMPUTE_SPEC(Dims) \
EIGEN_ALWAYS_INLINE static void Reduce##Dims( \
const Device& d, typename TTypes<T, Dims>::ConstTensor input, \
const int32 dimension, typename TTypes<Tout, Dims - 1>::Tensor output) { \
output.device(d) = input.argmax(dimension).template cast<Tout>(); \
}
DECLARE_COMPUTE_SPEC(1);
DECLARE_COMPUTE_SPEC(2);
DECLARE_COMPUTE_SPEC(3);
DECLARE_COMPUTE_SPEC(4);
DECLARE_COMPUTE_SPEC(5);
#undef DECLARE_COMPUTE_SPEC
};
template <typename Device, typename T, typename Tout>
struct ArgMin {
#define DECLARE_COMPUTE_SPEC(Dims) \
EIGEN_ALWAYS_INLINE static void Reduce##Dims( \
const Device& d, typename TTypes<T, Dims>::ConstTensor input, \
const int32 dimension, typename TTypes<Tout, Dims - 1>::Tensor output) { \
output.device(d) = input.argmin(dimension).template cast<Tout>(); \
}
DECLARE_COMPUTE_SPEC(1);
DECLARE_COMPUTE_SPEC(2);
DECLARE_COMPUTE_SPEC(3);
DECLARE_COMPUTE_SPEC(4);
DECLARE_COMPUTE_SPEC(5);
#undef DECLARE_COMPUTE_SPEC
};
} // namespace functor
} // namespace tensorflow
#endif // TENSORFLOW_CORE_KERNELS_ARGMAX_OP_H_
@@ -0,0 +1,146 @@
/* Copyright 2015 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
#ifndef TENSORFLOW_CORE_KERNELS_ASSIGN_OP_H_
#define TENSORFLOW_CORE_KERNELS_ASSIGN_OP_H_
#define EIGEN_USE_THREADS
#include "third_party/eigen3/unsupported/Eigen/CXX11/Tensor"
#include "tensorflow/core/framework/op_kernel.h"
#include "tensorflow/core/framework/tensor_types.h"
namespace tensorflow {
// TODO(jeff): Get rid of use_exclusive_lock_ option
// Computes *input[0] = input[1]
class AssignOp : public OpKernel {
public:
explicit AssignOp(OpKernelConstruction* context) : OpKernel(context) {
OP_REQUIRES_OK(context,
context->GetAttr("use_locking", &use_exclusive_lock_));
OP_REQUIRES_OK(context,
context->GetAttr("validate_shape", &validate_shape_));
OP_REQUIRES(context, IsRefType(context->input_type(0)),
errors::InvalidArgument("lhs input needs to be a ref type"));
if (!context
->GetAttr("_grappler_relax_allocator_constraints",
&relax_constraints_)
.ok()) {
relax_constraints_ = false;
}
}
void Compute(OpKernelContext* context) override {
const Tensor& rhs = context->input(1);
// We always return the input ref.
context->forward_ref_input_to_ref_output(0, 0);
// We can't always know how this value will be used downstream, so make
// conservative assumptions in specifying constraints on the memory
// allocation attributes, unless the Grappler graph analysis determined that
// it was safe not to.
AllocatorAttributes attr;
if (!relax_constraints_) {
attr.set_gpu_compatible(true);
attr.set_nic_compatible(true);
}
{
mutex_lock l(*context->input_ref_mutex(0));
const Tensor& old_lhs = context->mutable_input(0, /* lock_held */ true);
const bool same_shape = old_lhs.shape().IsSameSize(rhs.shape());
if (validate_shape_) {
OP_REQUIRES(context, same_shape,
errors::InvalidArgument(
"Assign requires shapes of both tensors to match. "
"lhs shape= ",
old_lhs.shape().DebugString(),
" rhs shape= ", rhs.shape().DebugString()));
}
// In the code below we try to minimize the amount of memory allocation
// and copying by trying the following two shortcuts:
// 1. If the lhs is initialized and has the same number of elements as
// the rhs we can avoid a memory allocation.
// 2. If we can reuse the rhs buffer we avoid both a memory allocation
// and copying.
// 1. Try to copy into an existing buffer.
if (old_lhs.IsInitialized() &&
old_lhs.shape().num_elements() == rhs.shape().num_elements()) {
// The existing lhs tensor has already been initialized and the right
// hand side can fit in the underlying buffer.
Tensor reshaped_old_lhs;
if (same_shape) {
reshaped_old_lhs = old_lhs;
} else {
CHECK(reshaped_old_lhs.CopyFrom(old_lhs, rhs.shape()));
context->replace_ref_input(0, reshaped_old_lhs,
/* lock_held */ true);
}
if (use_exclusive_lock_) {
Copy(context, &reshaped_old_lhs, rhs);
return;
}
} else {
// 2. Try to reuse the rhs.
std::unique_ptr<Tensor> input_alias = context->forward_input(
1, OpKernelContext::Params::kNoReservation /*output_index*/,
rhs.dtype(), rhs.shape(), DEVICE_MEMORY, attr);
if (input_alias != nullptr) {
// Update the ref to point to the new buffer.
context->replace_ref_input(0, *input_alias, /* lock_held */ true);
return;
}
// Otherwise, create a new persistent tensor whose shape matches the
// right hand side, hand off to lhs and copy the rhs into it.
PersistentTensor copy;
Tensor* copyTensor = nullptr;
OP_REQUIRES_OK(
context, context->allocate_persistent(old_lhs.dtype(), rhs.shape(),
&copy, &copyTensor, attr));
// We track memory of variables in variable ops instead of in this
// assign op.
context->clear_recorded_memory();
context->replace_ref_input(0, *copyTensor, /* lock_held */ true);
if (use_exclusive_lock_) {
Copy(context, copyTensor, rhs);
return;
}
}
}
// The tensor has already been initialized and the right hand side
// matches the left hand side's shape. We have been told to do the
// copy outside the lock.
Tensor old_unlocked_lhs = context->mutable_input(0, /* lock_held */ false);
Copy(context, &old_unlocked_lhs, rhs);
}
virtual void Copy(OpKernelContext* context, Tensor* lhs,
const Tensor& rhs) = 0;
bool use_exclusive_lock_;
bool validate_shape_;
bool relax_constraints_;
};
} // end namespace tensorflow
#endif // TENSORFLOW_CORE_KERNELS_ASSIGN_OP_H_
@@ -0,0 +1,79 @@
/* Copyright 2015 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
#ifndef TENSORFLOW_CORE_KERNELS_AVGPOOLING_OP_H_
#define TENSORFLOW_CORE_KERNELS_AVGPOOLING_OP_H_
// Functor definition for AvgPoolingOp, must be compilable by nvcc.
#include "tensorflow/core/framework/tensor_types.h"
#include "tensorflow/core/kernels/eigen_pooling.h"
#include "tensorflow/core/platform/types.h"
namespace tensorflow {
namespace functor {
template <typename Device, typename T>
struct SpatialAvgPooling {
void operator()(const Device& d, typename TTypes<T, 4>::Tensor output,
typename TTypes<T, 4>::ConstTensor input, int window_rows,
int window_cols, int row_stride, int col_stride,
const Eigen::PaddingType& padding) {
if (Eigen::internal::is_same<Device, Eigen::GpuDevice>::value) {
// Use 32bit indexing to speed up the computations
To32Bit(output).swap_layout().device(d) = Eigen::SpatialAvgPooling(
To32Bit(input).swap_layout(), window_cols, window_rows, col_stride,
row_stride, padding);
} else {
// Because we swap the layout, we swap the row/cols as well
output.swap_layout().device(d) = Eigen::SpatialAvgPooling(
input.swap_layout(), window_cols, window_rows, col_stride, row_stride,
padding);
}
}
};
} // namespace functor
typedef Eigen::GpuDevice GPUDevice;
// Launch a custom GPU kernels from Yanqing for the avgpooling backward
// operation that works NHWC data formats. Arguments:
// top_diff: backprop to the output of the pooling layer
// num: number of input batches
// height: input height
// width: input width
// channels: number of input channels
// pooled_height: the height of the output to the pooling layer
// pooled_width: the width of the output to the pooling layer
// kernel_h: the height of the pooling kernel
// kernel_w: the width of the pooling kernel
// stride_h: the height of the vertical stride
// stride_w: the width of the horizontal stride
// pad_t: padding size to the top side
// pad_l: padding size to the left side
// bottom_diff: backprop to the input of the pooling layer.
template <typename T>
bool RunAvePoolBackwardNHWC(const T* const top_diff, const int num,
const int height, const int width,
const int channels, const int pooled_height,
const int pooled_width, const int kernel_h,
const int kernel_w, const int stride_h,
const int stride_w, const int pad_t,
const int pad_l, T* const bottom_diff,
const GPUDevice& d);
} // namespace tensorflow
#endif // TENSORFLOW_CORE_KERNELS_AVGPOOLING_OP_H_
@@ -0,0 +1,620 @@
/* Copyright 2015 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
// See docs in ../ops/math_ops.cc.
#ifndef TENSORFLOW_CORE_KERNELS_BATCH_MATMUL_OP_IMPL_H_
#define TENSORFLOW_CORE_KERNELS_BATCH_MATMUL_OP_IMPL_H_
#define EIGEN_USE_THREADS
#include <vector>
#include "third_party/eigen3/unsupported/Eigen/CXX11/Tensor"
#include "tensorflow/core/framework/op.h"
#include "tensorflow/core/framework/op_kernel.h"
#include "tensorflow/core/framework/register_types.h"
#include "tensorflow/core/framework/tensor.h"
#include "tensorflow/core/framework/tensor_shape.h"
#include "tensorflow/core/framework/type_traits.h"
#include "tensorflow/core/framework/types.h"
#include "tensorflow/core/kernels/fill_functor.h"
#include "tensorflow/core/platform/logging.h"
#include "tensorflow/core/platform/types.h"
#include "tensorflow/core/util/work_sharder.h"
#if GOOGLE_CUDA
#include "tensorflow/core/platform/stream_executor.h"
#endif // GOOGLE_CUDA
namespace tensorflow {
typedef Eigen::ThreadPoolDevice CPUDevice;
typedef Eigen::GpuDevice GPUDevice;
#ifdef TENSORFLOW_USE_SYCL
typedef Eigen::SyclDevice SYCLDevice;
#endif // TENSORFLOW_USE_SYCL
namespace {
Eigen::IndexPair<Eigen::DenseIndex> ContractionDims(bool adj_x, bool adj_y) {
if (!adj_x) {
if (!adj_y) {
return Eigen::IndexPair<Eigen::DenseIndex>(1, 0);
} else {
return Eigen::IndexPair<Eigen::DenseIndex>(1, 1);
}
} else {
if (!adj_y) {
return Eigen::IndexPair<Eigen::DenseIndex>(0, 0);
} else {
return Eigen::IndexPair<Eigen::DenseIndex>(0, 1);
}
}
}
// Parallel batch matmul kernel based on the multi-threaded tensor contraction
// in Eigen.
template <typename Scalar, bool IsComplex = true>
struct ParallelMatMulKernel {
static void Conjugate(const OpKernelContext* context, Tensor* out) {
const Eigen::ThreadPoolDevice d = context->eigen_cpu_device();
auto z = out->tensor<Scalar, 3>();
z.device(d) = z.conjugate();
}
static void Run(const OpKernelContext* context, const Tensor& in_x,
const Tensor in_y, bool adj_x, bool adj_y, Tensor* out,
int start, int limit) {
static_assert(IsComplex, "Complex type expected.");
auto Tx = in_x.tensor<Scalar, 3>();
auto Ty = in_y.tensor<Scalar, 3>();
auto Tz = out->tensor<Scalar, 3>();
// We use the identities
// conj(a) * conj(b) = conj(a * b)
// conj(a) * b = conj(a * conj(b))
// to halve the number of cases. The final conjugation of the result is
// done at the end of LaunchBatchMatMul<CPUDevice, Scalar>::Launch().
Eigen::array<Eigen::IndexPair<Eigen::DenseIndex>, 1> contract_pairs;
contract_pairs[0] = ContractionDims(adj_x, adj_y);
const Eigen::ThreadPoolDevice d = context->eigen_cpu_device();
for (int i = start; i < limit; ++i) {
auto x = Tx.template chip<0>(i);
auto z = Tz.template chip<0>(i);
if (adj_x != adj_y) {
auto y = Ty.template chip<0>(i).conjugate();
z.device(d) = x.contract(y, contract_pairs);
} else {
auto y = Ty.template chip<0>(i);
z.device(d) = x.contract(y, contract_pairs);
}
}
}
};
// The Eigen contraction kernel used here is very large and slow to compile,
// so we partially specialize ParallelMatMulKernel for real types to avoid all
// but one of the instantiations.
template <typename Scalar>
struct ParallelMatMulKernel<Scalar, false> {
static void Conjugate(const OpKernelContext* context, Tensor* out) {}
static void Run(const OpKernelContext* context, const Tensor& in_x,
const Tensor& in_y, bool adj_x, bool adj_y, Tensor* out,
int start, int limit) {
auto Tx = in_x.tensor<Scalar, 3>();
auto Ty = in_y.tensor<Scalar, 3>();
auto Tz = out->tensor<Scalar, 3>();
Eigen::array<Eigen::IndexPair<Eigen::DenseIndex>, 1> contract_pairs;
contract_pairs[0] = ContractionDims(adj_x, adj_y);
const Eigen::ThreadPoolDevice d = context->eigen_cpu_device();
for (int i = start; i < limit; ++i) {
auto x = Tx.template chip<0>(i);
auto y = Ty.template chip<0>(i);
auto z = Tz.template chip<0>(i);
z.device(d) = x.contract(y, contract_pairs);
}
}
};
// TODO(rmlarsen): Get rid of this when we have upstreamed improvements
// for matrix*vector and vector*matrix to Eigen's general matrix product.
template <typename Tx, typename Ty, typename Tz>
static void Multiply(bool adj_x, bool adj_y, Tx x, Ty y, Tz z) {
if (!adj_x) {
if (!adj_y) {
z.noalias() = x * y;
} else {
z.noalias() = x * y.adjoint();
}
} else {
if (!adj_y) {
z.noalias() = x.adjoint() * y;
} else {
z.noalias() = x.adjoint() * y.adjoint();
}
}
}
// Sequential batch matmul kernel that calls the regular Eigen matmul.
// We prefer this over the tensor contraction because it performs
// better on vector-matrix and matrix-vector products.
template <typename Scalar>
struct SequentialMatMulKernel {
using Matrix =
Eigen::Matrix<Scalar, Eigen::Dynamic, Eigen::Dynamic, Eigen::RowMajor>;
using ConstMatrixMap = Eigen::Map<const Matrix>;
using MatrixMap = Eigen::Map<Matrix>;
static ConstMatrixMap ConstTensorSliceToEigenMatrix(const Tensor& t,
int slice) {
return ConstMatrixMap(
t.flat<Scalar>().data() + slice * t.dim_size(1) * t.dim_size(2),
t.dim_size(1), t.dim_size(2));
}
static MatrixMap TensorSliceToEigenMatrix(Tensor* t, int slice) {
return MatrixMap(
t->flat<Scalar>().data() + slice * t->dim_size(1) * t->dim_size(2),
t->dim_size(1), t->dim_size(2));
}
static void Run(const Tensor& in_x, const Tensor& in_y, bool adj_x,
bool adj_y, Tensor* out, int start, int limit) {
for (int i = start; i < limit; ++i) {
auto x = ConstTensorSliceToEigenMatrix(in_x, i);
auto y = ConstTensorSliceToEigenMatrix(in_y, i);
auto z = TensorSliceToEigenMatrix(out, i);
// TODO(rmlarsen): Get rid of the special casing here when we have
// upstreamed improvements for matrix*vector and vector*matrix to
// Eigen's general matrix product.
if (!adj_x && x.rows() == 1) {
Multiply(adj_x, adj_y, x.row(0), y, z);
} else if (adj_x && x.cols() == 1) {
Multiply(adj_x, adj_y, x.col(0), y, z);
} else if (!adj_y && y.cols() == 1) {
Multiply(adj_x, adj_y, x, y.col(0), z);
} else if (adj_y && y.rows() == 1) {
Multiply(adj_x, adj_y, x, y.row(0), z);
} else {
Multiply(adj_x, adj_y, x, y, z);
}
}
}
};
} // namespace
template <typename Device, typename Scalar>
struct LaunchBatchMatMul;
template <typename Scalar>
struct LaunchBatchMatMul<CPUDevice, Scalar> {
static void Launch(OpKernelContext* context, const Tensor& in_x,
const Tensor& in_y, bool adj_x, bool adj_y, Tensor* out) {
typedef ParallelMatMulKernel<Scalar, Eigen::NumTraits<Scalar>::IsComplex>
ParallelMatMulKernel;
bool conjugate_result = false;
// Number of matrix multiplies i.e. size of the batch.
const int64 batch_size = in_x.dim_size(0);
const int64 cost_per_unit =
in_x.dim_size(1) * in_x.dim_size(2) * out->dim_size(2);
const int64 small_dim = std::min(
std::min(in_x.dim_size(1), in_x.dim_size(2)), out->dim_size(2));
const int64 kMaxCostOuterParallelism = 128 * 128 * 256; // heuristic.
auto worker_threads = *(context->device()->tensorflow_cpu_worker_threads());
if (small_dim > 1 &&
(batch_size == 1 || cost_per_unit > kMaxCostOuterParallelism)) {
// Parallelize over inner dims.
// For large matrix products it is counter-productive to parallelize
// over the batch dimension.
ParallelMatMulKernel::Run(context, in_x, in_y, adj_x, adj_y, out, 0,
batch_size);
conjugate_result = adj_x;
} else {
// Parallelize over outer dims. For small matrices and large batches, it
// is counter-productive to parallelize the inner matrix multiplies.
Shard(worker_threads.num_threads, worker_threads.workers, batch_size,
cost_per_unit,
[&in_x, &in_y, adj_x, adj_y, out](int start, int limit) {
SequentialMatMulKernel<Scalar>::Run(in_x, in_y, adj_x, adj_y, out,
start, limit);
});
}
if (conjugate_result) {
// We used one of the identities
// conj(a) * conj(b) = conj(a * b)
// conj(a) * b = conj(a * conj(b))
// above, we need to conjugate the final output. This is a
// no-op for non-complex types.
ParallelMatMulKernel::Conjugate(context, out);
}
}
};
#if GOOGLE_CUDA
namespace {
template <typename T>
se::DeviceMemory<T> AsDeviceMemory(const T* cuda_memory) {
se::DeviceMemoryBase wrapped(const_cast<T*>(cuda_memory));
se::DeviceMemory<T> typed(wrapped);
return typed;
}
class CublasScratchAllocator : public se::ScratchAllocator {
public:
using Stream = se::Stream;
using DeviceMemoryBytes = se::DeviceMemory<uint8>;
CublasScratchAllocator(OpKernelContext* context) : context_(context) {}
int64 GetMemoryLimitInBytes(Stream* stream) override { return -1; }
se::port::StatusOr<DeviceMemoryBytes> AllocateBytes(
Stream* stream, int64 byte_size) override {
Tensor temporary_memory;
Status allocation_status(context_->allocate_temp(
DT_UINT8, TensorShape({byte_size}), &temporary_memory));
if (!allocation_status.ok()) {
return se::port::StatusOr<DeviceMemoryBytes>(
DeviceMemoryBytes::MakeFromByteSize(nullptr, 0));
}
// Hold the reference of the allocated tensors until the end of the
// allocator.
allocated_tensors_.push_back(temporary_memory);
return se::port::StatusOr<DeviceMemoryBytes>(
DeviceMemoryBytes::MakeFromByteSize(
temporary_memory.flat<uint8>().data(),
temporary_memory.flat<uint8>().size()));
}
private:
OpKernelContext* context_;
std::vector<Tensor> allocated_tensors_;
};
} // namespace
template <typename Scalar>
struct LaunchBatchMatMul<GPUDevice, Scalar> {
static void Launch(OpKernelContext* context, const Tensor& in_x,
const Tensor& in_y, bool adj_x, bool adj_y, Tensor* out) {
constexpr se::blas::Transpose kTranspose =
is_complex<Scalar>::value ? se::blas::Transpose::kConjugateTranspose
: se::blas::Transpose::kTranspose;
se::blas::Transpose trans[] = {se::blas::Transpose::kNoTranspose,
kTranspose};
const uint64 m = in_x.dim_size(adj_x ? 2 : 1);
const uint64 k = in_x.dim_size(adj_x ? 1 : 2);
const uint64 n = in_y.dim_size(adj_y ? 1 : 2);
const uint64 batch_size = in_x.dim_size(0);
auto blas_transpose_a = trans[adj_x];
auto blas_transpose_b = trans[adj_y];
auto* stream = context->op_device_context()->stream();
OP_REQUIRES(context, stream, errors::Internal("No GPU stream available."));
typedef se::DeviceMemory<Scalar> DeviceMemoryType;
std::vector<DeviceMemoryType> a_device_memory;
std::vector<DeviceMemoryType> b_device_memory;
std::vector<DeviceMemoryType> c_device_memory;
std::vector<DeviceMemoryType*> a_ptrs;
std::vector<DeviceMemoryType*> b_ptrs;
std::vector<DeviceMemoryType*> c_ptrs;
a_device_memory.reserve(batch_size);
b_device_memory.reserve(batch_size);
c_device_memory.reserve(batch_size);
a_ptrs.reserve(batch_size);
b_ptrs.reserve(batch_size);
c_ptrs.reserve(batch_size);
auto* a_base_ptr = in_x.template flat<Scalar>().data();
auto* b_base_ptr = in_y.template flat<Scalar>().data();
auto* c_base_ptr = out->template flat<Scalar>().data();
for (int64 i = 0; i < batch_size; ++i) {
a_device_memory.push_back(AsDeviceMemory(a_base_ptr + i * m * k));
b_device_memory.push_back(AsDeviceMemory(b_base_ptr + i * k * n));
c_device_memory.push_back(AsDeviceMemory(c_base_ptr + i * m * n));
a_ptrs.push_back(&a_device_memory.back());
b_ptrs.push_back(&b_device_memory.back());
c_ptrs.push_back(&c_device_memory.back());
}
typedef Scalar Coefficient;
// Cublas does
// C = A x B
// where A, B and C are assumed to be in column major.
// We want the output to be in row-major, so we can compute
// C' = B' x A', where ' stands for transpose (not adjoint).
// TODO(yangzihao): Choose the best of the three strategies using autotune.
if (batch_size == 1) {
// This is a regular matrix*matrix or matrix*vector multiply. Avoid the
// overhead of the scratch allocator and the batch interface.
if (n == 1 &&
blas_transpose_b != se::blas::Transpose::kConjugateTranspose &&
blas_transpose_a != se::blas::Transpose::kConjugateTranspose) {
// This is a matrix*vector multiply so use GEMV to compute A * b.
// Here we are multiplying in the natural order, so we have to flip
// the transposition flag to compensate for the tensor being stored
// row-major. Since GEMV doesn't provide a way to just conjugate an
// argument, we have to defer those cases to GEMM below.
auto gemv_trans_a = blas_transpose_a == se::blas::Transpose::kTranspose
? se::blas::Transpose::kNoTranspose
: se::blas::Transpose::kTranspose;
bool blas_launch_status =
stream
->ThenBlasGemv(gemv_trans_a, adj_x ? m : k, adj_x ? k : m,
static_cast<Coefficient>(1.0), *(a_ptrs[0]),
adj_x ? m : k, *(b_ptrs[0]), 1,
static_cast<Coefficient>(0.0), c_ptrs[0], 1)
.ok();
if (!blas_launch_status) {
context->SetStatus(errors::Internal(
"Blas xGEMV launch failed : a.shape=", in_x.shape().DebugString(),
", b.shape=", in_y.shape().DebugString(), ", m=", m, ", n=", n,
", k=", k));
}
} else {
bool blas_launch_status =
stream
->ThenBlasGemm(blas_transpose_b, blas_transpose_a, n, m, k,
static_cast<Coefficient>(1.0), *(b_ptrs[0]),
adj_y ? k : n, *(a_ptrs[0]), adj_x ? m : k,
static_cast<Coefficient>(0.0), c_ptrs[0], n)
.ok();
if (!blas_launch_status) {
context->SetStatus(errors::Internal(
"Blas xGEMM launch failed : a.shape=", in_x.shape().DebugString(),
", b.shape=", in_y.shape().DebugString(), ", m=", m, ", n=", n,
", k=", k));
}
}
} else {
CublasScratchAllocator scratch_allocator(context);
bool blas_launch_status =
stream
->ThenBlasGemmBatchedWithScratch(
blas_transpose_b, blas_transpose_a, n, m, k,
static_cast<Coefficient>(1.0), b_ptrs, adj_y ? k : n, a_ptrs,
adj_x ? m : k, static_cast<Coefficient>(0.0), c_ptrs, n,
batch_size, &scratch_allocator)
.ok();
if (!blas_launch_status) {
context->SetStatus(errors::Internal(
"Blas xGEMMBatched launch failed : a.shape=",
in_x.shape().DebugString(),
", b.shape=", in_y.shape().DebugString(), ", m=", m, ", n=", n,
", k=", k, ", batch_size=", batch_size));
}
}
}
};
template <>
struct LaunchBatchMatMul<GPUDevice, Eigen::half> {
static void Launch(OpKernelContext* context, const Tensor& in_x,
const Tensor& in_y, bool adj_x, bool adj_y, Tensor* out) {
typedef Eigen::half Scalar;
constexpr perftools::gputools::blas::Transpose kTranspose =
is_complex<Scalar>::value
? perftools::gputools::blas::Transpose::kConjugateTranspose
: perftools::gputools::blas::Transpose::kTranspose;
perftools::gputools::blas::Transpose trans[] = {
perftools::gputools::blas::Transpose::kNoTranspose, kTranspose};
const uint64 m = in_x.dim_size(adj_x ? 2 : 1);
const uint64 k = in_x.dim_size(adj_x ? 1 : 2);
const uint64 n = in_y.dim_size(adj_y ? 1 : 2);
const uint64 batch_size = in_x.dim_size(0);
auto blas_transpose_a = trans[adj_x];
auto blas_transpose_b = trans[adj_y];
auto* stream = context->op_device_context()->stream();
OP_REQUIRES(context, stream, errors::Internal("No GPU stream available."));
typedef perftools::gputools::DeviceMemory<Scalar> DeviceMemoryType;
std::vector<DeviceMemoryType> a_device_memory;
std::vector<DeviceMemoryType> b_device_memory;
std::vector<DeviceMemoryType> c_device_memory;
std::vector<DeviceMemoryType*> a_ptrs;
std::vector<DeviceMemoryType*> b_ptrs;
std::vector<DeviceMemoryType*> c_ptrs;
a_device_memory.reserve(batch_size);
b_device_memory.reserve(batch_size);
c_device_memory.reserve(batch_size);
a_ptrs.reserve(batch_size);
b_ptrs.reserve(batch_size);
c_ptrs.reserve(batch_size);
auto* a_base_ptr = in_x.template flat<Scalar>().data();
auto* b_base_ptr = in_y.template flat<Scalar>().data();
auto* c_base_ptr = out->template flat<Scalar>().data();
for (int64 i = 0; i < batch_size; ++i) {
a_device_memory.push_back(AsDeviceMemory(a_base_ptr + i * m * k));
b_device_memory.push_back(AsDeviceMemory(b_base_ptr + i * k * n));
c_device_memory.push_back(AsDeviceMemory(c_base_ptr + i * m * n));
a_ptrs.push_back(&a_device_memory.back());
b_ptrs.push_back(&b_device_memory.back());
c_ptrs.push_back(&c_device_memory.back());
}
typedef float Coefficient;
// Cublas does
// C = A x B
// where A, B and C are assumed to be in column major.
// We want the output to be in row-major, so we can compute
// C' = B' x A', where ' stands for transpose (not adjoint).
// TODO(yangzihao): Choose the best of the three strategies using autotune.
if (batch_size == 1) {
// This is a regular matrix*matrix or matrix*vector multiply. Avoid the
// overhead of the scratch allocator and the batch interface.
// TODO(benbarsdell): Use fp16 Gemv if it becomes supported by CUBLAS
bool blas_launch_status =
stream
->ThenBlasGemm(blas_transpose_b, blas_transpose_a, n, m, k,
static_cast<Coefficient>(1.0), *(b_ptrs[0]),
adj_y ? k : n, *(a_ptrs[0]), adj_x ? m : k,
static_cast<Coefficient>(0.0), c_ptrs[0], n)
.ok();
if (!blas_launch_status) {
context->SetStatus(errors::Internal(
"Blas xGEMM launch failed : a.shape=", in_x.shape().DebugString(),
", b.shape=", in_y.shape().DebugString(), ", m=", m, ", n=", n,
", k=", k));
}
} else {
CublasScratchAllocator scratch_allocator(context);
bool blas_launch_status =
stream
->ThenBlasGemmBatchedWithScratch(
blas_transpose_b, blas_transpose_a, n, m, k,
static_cast<Coefficient>(1.0), b_ptrs, adj_y ? k : n, a_ptrs,
adj_x ? m : k, static_cast<Coefficient>(0.0), c_ptrs, n,
batch_size, &scratch_allocator)
.ok();
if (!blas_launch_status) {
context->SetStatus(
errors::Internal("Blas xGEMMBatched launch failed : a.shape=",
in_x.shape().DebugString(), ", b.shape=",
in_y.shape().DebugString(), ", m=", m, ", n=", n,
", k=", k, ", batch_size=", batch_size));
}
}
}
};
#endif // GOOGLE_CUDA
#ifdef TENSORFLOW_USE_SYCL
template <typename Scalar>
struct ParallelMatMulKernelSYCL {
static void Run(const OpKernelContext* context, const Tensor& in_x,
const Tensor& in_y, bool adj_x, bool adj_y, Tensor* out,
int start, int limit) {
auto Tx = in_x.tensor<Scalar, 3>();
auto Ty = in_y.tensor<Scalar, 3>();
auto Tz = out->tensor<Scalar, 3>();
Eigen::array<Eigen::IndexPair<Eigen::DenseIndex>, 1> contract_pairs;
contract_pairs[0] = ContractionDims(adj_x, adj_y);
auto d = context->eigen_sycl_device();
for (int i = start; i < limit; ++i) {
auto x = Tx.template chip<0>(i);
auto y = Ty.template chip<0>(i);
auto z = Tz.template chip<0>(i);
z.device(d) = x.contract(y, contract_pairs);
}
}
};
template <typename Scalar>
struct LaunchBatchMatMul<SYCLDevice, Scalar> {
static void Launch(OpKernelContext* context, const Tensor& in_x,
const Tensor& in_y, bool adj_x, bool adj_y, Tensor* out) {
// Number of matrix multiplies i.e. size of the batch.
const int64 batch_size = in_x.dim_size(0);
ParallelMatMulKernelSYCL<Scalar>::Run(context, in_x, in_y, adj_x, adj_y,
out, 0, batch_size);
}
};
#endif // TENSORFLOW_USE_SYCL
template <typename Device, typename Scalar>
class BatchMatMul : public OpKernel {
public:
explicit BatchMatMul(OpKernelConstruction* context) : OpKernel(context) {
OP_REQUIRES_OK(context, context->GetAttr("adj_x", &adj_x_));
OP_REQUIRES_OK(context, context->GetAttr("adj_y", &adj_y_));
}
virtual ~BatchMatMul() {}
void Compute(OpKernelContext* ctx) override {
const Tensor& in0 = ctx->input(0);
const Tensor& in1 = ctx->input(1);
OP_REQUIRES(ctx, in0.dims() == in1.dims(),
errors::InvalidArgument("In[0] and In[1] has different ndims: ",
in0.shape().DebugString(), " vs. ",
in1.shape().DebugString()));
const int ndims = in0.dims();
OP_REQUIRES(
ctx, ndims >= 2,
errors::InvalidArgument("In[0] and In[1] ndims must be >= 2: ", ndims));
TensorShape out_shape;
for (int i = 0; i < ndims - 2; ++i) {
OP_REQUIRES(ctx, in0.dim_size(i) == in1.dim_size(i),
errors::InvalidArgument(
"In[0].dim(", i, ") and In[1].dim(", i,
") must be the same: ", in0.shape().DebugString(), " vs ",
in1.shape().DebugString()));
out_shape.AddDim(in0.dim_size(i));
}
auto n = (ndims == 2) ? 1 : out_shape.num_elements();
auto d0 = in0.dim_size(ndims - 2);
auto d1 = in0.dim_size(ndims - 1);
Tensor in0_reshaped;
CHECK(in0_reshaped.CopyFrom(in0, TensorShape({n, d0, d1})));
auto d2 = in1.dim_size(ndims - 2);
auto d3 = in1.dim_size(ndims - 1);
Tensor in1_reshaped;
CHECK(in1_reshaped.CopyFrom(in1, TensorShape({n, d2, d3})));
if (adj_x_) std::swap(d0, d1);
if (adj_y_) std::swap(d2, d3);
OP_REQUIRES(ctx, d1 == d2,
errors::InvalidArgument(
"In[0] mismatch In[1] shape: ", d1, " vs. ", d2, ": ",
in0.shape().DebugString(), " ", in1.shape().DebugString(),
" ", adj_x_, " ", adj_y_));
out_shape.AddDim(d0);
out_shape.AddDim(d3);
Tensor* out = nullptr;
OP_REQUIRES_OK(ctx, ctx->allocate_output(0, out_shape, &out));
if (out->NumElements() == 0) {
return;
}
if (in0.NumElements() == 0 || in1.NumElements() == 0) {
functor::SetZeroFunctor<Device, Scalar> f;
f(ctx->eigen_device<Device>(), out->flat<Scalar>());
return;
}
Tensor out_reshaped;
CHECK(out_reshaped.CopyFrom(*out, TensorShape({n, d0, d3})));
LaunchBatchMatMul<Device, Scalar>::Launch(ctx, in0_reshaped, in1_reshaped,
adj_x_, adj_y_, &out_reshaped);
}
private:
bool adj_x_;
bool adj_y_;
};
#define REGISTER_BATCH_MATMUL_CPU(TYPE) \
REGISTER_KERNEL_BUILDER( \
Name("BatchMatMul").Device(DEVICE_CPU).TypeConstraint<TYPE>("T"), \
BatchMatMul<CPUDevice, TYPE>)
#define REGISTER_BATCH_MATMUL_GPU(TYPE) \
REGISTER_KERNEL_BUILDER( \
Name("BatchMatMul").Device(DEVICE_GPU).TypeConstraint<TYPE>("T"), \
BatchMatMul<GPUDevice, TYPE>)
#ifdef TENSORFLOW_USE_SYCL
#define REGISTER_BATCH_MATMUL_SYCL(TYPE) \
REGISTER_KERNEL_BUILDER( \
Name("BatchMatMul").Device(DEVICE_SYCL).TypeConstraint<TYPE>("T"), \
BatchMatMul<SYCLDevice, TYPE>)
#endif // TENSORFLOW_USE_SYCL
} // end namespace tensorflow
#endif // TENSORFLOW_CORE_KERNELS_BATCH_MATMUL_OP_IMPL_H_
@@ -0,0 +1,156 @@
/* Copyright 2015 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
#ifndef TENSORFLOW_CORE_KERNELS_BATCH_NORM_OP_H_
#define TENSORFLOW_CORE_KERNELS_BATCH_NORM_OP_H_
// Functor definition for BatchNormOp, must be compilable by nvcc.
#include "third_party/eigen3/unsupported/Eigen/CXX11/Tensor"
#include "tensorflow/core/framework/tensor_types.h"
namespace tensorflow {
namespace functor {
// Functor used by BatchNormOp to do the computations.
template <typename Device, typename T>
struct BatchNorm {
void operator()(const Device& d, typename TTypes<T, 4>::ConstTensor input,
typename TTypes<T>::ConstVec mean,
typename TTypes<T>::ConstVec var,
typename TTypes<T>::ConstVec beta,
typename TTypes<T>::ConstVec gamma, T variance_epsilon,
bool scale_after_normalization,
typename TTypes<T, 4>::Tensor output) {
const int depth = mean.dimension(0);
const int rest_size = input.size() / depth;
Eigen::DSizes<int, 2> rest_by_depth(rest_size, depth);
#if !defined(EIGEN_HAS_INDEX_LIST)
Eigen::DSizes<int, 2> rest_by_one(rest_size, 1);
Eigen::DSizes<int, 2> one_by_depth(1, depth);
Eigen::DSizes<int, 2> depth_by_one(depth, 1);
#else
Eigen::IndexList<int, Eigen::type2index<1> > rest_by_one;
rest_by_one.set(0, rest_size);
Eigen::IndexList<Eigen::type2index<1>, int> one_by_depth;
one_by_depth.set(1, depth);
Eigen::IndexList<int, Eigen::type2index<1> > depth_by_one;
depth_by_one.set(0, depth);
#endif
if (scale_after_normalization) {
output.reshape(rest_by_depth).device(d) =
(input.reshape(rest_by_depth) -
mean.reshape(one_by_depth).broadcast(rest_by_one)) *
((var + var.constant(variance_epsilon)).rsqrt() * gamma)
.eval()
.reshape(one_by_depth)
.broadcast(rest_by_one) +
beta.reshape(one_by_depth).broadcast(rest_by_one);
} else {
output.reshape(rest_by_depth).device(d) =
(input.reshape(rest_by_depth) -
mean.reshape(one_by_depth).broadcast(rest_by_one)) *
((var + var.constant(variance_epsilon)).rsqrt())
.eval()
.reshape(one_by_depth)
.broadcast(rest_by_one) +
beta.reshape(one_by_depth).broadcast(rest_by_one);
}
}
};
template <typename Device, typename T>
struct BatchNormGrad {
void operator()(const Device& d, typename TTypes<T, 4>::ConstTensor input,
typename TTypes<T>::ConstVec mean,
typename TTypes<T>::ConstVec var,
typename TTypes<T>::ConstVec gamma,
typename TTypes<T, 4>::ConstTensor out_backprop,
T variance_epsilon, bool scale_after_normalization,
typename TTypes<T, 4>::Tensor dx, typename TTypes<T>::Vec dm,
typename TTypes<T>::Vec dv, typename TTypes<T>::Vec db,
typename TTypes<T>::Vec dg, typename TTypes<T>::Vec scratch1,
typename TTypes<T>::Vec scratch2) {
const int depth = mean.dimension(0);
const int rest_size = input.size() / depth;
typedef typename TTypes<T>::ConstVec::Index Index;
Eigen::DSizes<Index, 2> rest_by_depth(rest_size, depth);
#if !defined(EIGEN_HAS_INDEX_LIST)
Eigen::DSizes<Index, 2> rest_by_one(rest_size, 1);
Eigen::DSizes<Index, 2> one_by_depth(1, depth);
Eigen::array<Index, 1> reduction_axis;
reduction_axis[0] = 0; // Reduces on first dimension.
#else
Eigen::IndexList<Index, Eigen::type2index<1> > rest_by_one;
rest_by_one.set(0, rest_size);
Eigen::IndexList<Eigen::type2index<1>, Index> one_by_depth;
one_by_depth.set(1, depth);
Eigen::IndexList<Eigen::type2index<0> > reduction_axis;
#endif
// db = out_backprop
//
// dg = out_backprop * ((x - m) * rsqrt(v + epsilon))
//
// dv = sum_over_rest(out_backprop * gamma * (x - m)) *
// (-1/2) * (v + epsilon) ^ (-3/2)
//
// dm = sum_over_rest(out_backprop * gamma) * (-1 / rsqrt(v + epsilon))
//
// dx = out_backprop * (gamma * rsqrt(v + epsilon))
db.device(d) = out_backprop.reshape(rest_by_depth).sum(reduction_axis);
// scratch1 = rsqrt(v + epsilon)
scratch1.device(d) = (var + var.constant(variance_epsilon)).rsqrt();
// scratch2 = sum_over_rest(out_backprop * (x - m))
scratch2.device(d) = (out_backprop.reshape(rest_by_depth) *
(input.reshape(rest_by_depth) -
mean.reshape(one_by_depth).broadcast(rest_by_one)))
.sum(reduction_axis);
if (scale_after_normalization) {
dx.reshape(rest_by_depth).device(d) =
out_backprop.reshape(rest_by_depth) * ((scratch1 * gamma)
.eval()
.reshape(one_by_depth)
.broadcast(rest_by_one));
dm.device(d) = -db * (scratch1 * gamma).eval();
dg.device(d) = scratch2 * scratch1;
} else {
dx.reshape(rest_by_depth).device(d) =
out_backprop.reshape(rest_by_depth) *
scratch1.reshape(one_by_depth).broadcast(rest_by_one);
dm.device(d) = -db * scratch1;
dg.device(d) = dg.constant(static_cast<T>(0.0)); // Gamma is not learned.
}
// scratch1 = - 1/2 * (var + epsilon) ^ (-3/2)
scratch1.device(d) = scratch1 * scratch1.constant(static_cast<T>(-0.5f)) /
(var + var.constant(variance_epsilon));
if (scale_after_normalization) {
dv.device(d) = scratch2 * (scratch1 * gamma).eval();
} else {
dv.device(d) = scratch2 * scratch1;
}
}
};
} // namespace functor
} // namespace tensorflow
#endif // TENSORFLOW_CORE_KERNELS_BATCH_NORM_OP_H_
@@ -0,0 +1,281 @@
/* Copyright 2016 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
// Abstractions for processing small tasks in a batched fashion, to reduce
// processing times and costs that can be amortized across multiple tasks.
//
// The core class is BatchScheduler, which groups tasks into batches.
//
// BatchScheduler encapsulates logic for aggregating multiple tasks into a
// batch, and kicking off processing of a batch on a thread pool it manages.
//
// This file defines an abstract BatchScheduler class.
#ifndef TENSORFLOW_CORE_KERNELS_BATCHING_UTIL_BATCH_SCHEDULER_H_
#define TENSORFLOW_CORE_KERNELS_BATCHING_UTIL_BATCH_SCHEDULER_H_
#include <stddef.h>
#include <algorithm>
#include <functional>
#include <memory>
#include <utility>
#include <vector>
#include "tensorflow/core/lib/core/notification.h"
#include "tensorflow/core/lib/core/status.h"
#include "tensorflow/core/platform/logging.h"
#include "tensorflow/core/platform/macros.h"
#include "tensorflow/core/platform/mutex.h"
#include "tensorflow/core/platform/thread_annotations.h"
#include "tensorflow/core/platform/types.h"
namespace tensorflow {
namespace serving {
// The abstract superclass for a unit of work to be done as part of a batch.
//
// An implementing subclass typically contains (or points to):
// (a) input data;
// (b) a thread-safe completion signal (e.g. a Notification);
// (c) a place to store the outcome (success, or some error), upon completion;
// (d) a place to store the output data, upon success.
//
// Items (b), (c) and (d) are typically non-owned pointers to data homed
// elsewhere, because a task's ownership gets transferred to a BatchScheduler
// (see below) and it may be deleted as soon as it is done executing.
class BatchTask {
public:
virtual ~BatchTask() = default;
// Returns the size of the task, in terms of how much it contributes to the
// size of a batch. (A batch's size is the sum of its task sizes.)
virtual size_t size() const = 0;
};
// A thread-safe collection of BatchTasks, to be executed together in some
// fashion.
//
// At a given time, a batch is either "open" or "closed": an open batch can
// accept new tasks; a closed one cannot. A batch is monotonic: initially it is
// open and tasks can be added to it; then it is closed and its set of tasks
// remains fixed for the remainder of its life. A closed batch cannot be re-
// opened. Tasks can never be removed from a batch.
//
// Type parameter TaskType must be a subclass of BatchTask.
template <typename TaskType>
class Batch {
public:
Batch() = default;
virtual ~Batch(); // Blocks until the batch is closed.
// Appends 'task' to the batch. After calling AddTask(), the newly-added task
// can be accessed via task(num_tasks()-1) or mutable_task(num_tasks()-1).
// Dies if the batch is closed.
void AddTask(std::unique_ptr<TaskType> task);
// Removes the most recently added task. Returns nullptr if the batch is
// empty.
std::unique_ptr<TaskType> RemoveTask();
// Returns the number of tasks in the batch.
int num_tasks() const;
// Returns true iff the batch contains 0 tasks.
bool empty() const;
// Returns a reference to the ith task (in terms of insertion order).
const TaskType& task(int i) const;
// Returns a pointer to the ith task (in terms of insertion order).
TaskType* mutable_task(int i);
// Returns the sum of the task sizes.
size_t size() const;
// Returns true iff the batch is currently closed.
bool IsClosed() const;
// Blocks until the batch is closed.
void WaitUntilClosed() const;
// Marks the batch as closed. Dies if called more than once.
void Close();
private:
mutable mutex mu_;
// The tasks in the batch.
std::vector<std::unique_ptr<TaskType>> tasks_ GUARDED_BY(mu_);
// The sum of the sizes of the tasks in 'tasks_'.
size_t size_ GUARDED_BY(mu_) = 0;
// Whether the batch has been closed.
Notification closed_;
TF_DISALLOW_COPY_AND_ASSIGN(Batch);
};
// An abstract batch scheduler class. Collects individual tasks into batches,
// and processes each batch on a pool of "batch threads" that it manages. The
// actual logic for processing a batch is accomplished via a callback.
//
// Type parameter TaskType must be a subclass of BatchTask.
template <typename TaskType>
class BatchScheduler {
public:
virtual ~BatchScheduler() = default;
// Submits a task to be processed as part of a batch.
//
// Ownership of '*task' is transferred to the callee iff the method returns
// Status::OK. In that case, '*task' is left as nullptr. Otherwise, '*task' is
// left as-is.
//
// If no batch processing capacity is available to process this task at the
// present time, and any task queue maintained by the implementing subclass is
// full, this method returns an UNAVAILABLE error code. The client may retry
// later.
//
// Other problems, such as the task size being larger than the maximum batch
// size, yield other, permanent error types.
//
// In all cases, this method returns "quickly" without blocking for any
// substantial amount of time. If the method returns Status::OK, the task is
// processed asynchronously, and any errors that occur during the processing
// of the batch that includes the task can be reported to 'task'.
virtual Status Schedule(std::unique_ptr<TaskType>* task) = 0;
// Returns the number of tasks that have been scheduled (i.e. accepted by
// Schedule()), but have yet to be handed to a thread for execution as part of
// a batch. Note that this returns the number of tasks, not the aggregate task
// size (so if there is one task of size 3 and one task of size 5, this method
// returns 2 rather than 8).
virtual size_t NumEnqueuedTasks() const = 0;
// Returns a guaranteed number of size 1 tasks that can be Schedule()d without
// getting an UNAVAILABLE error. In a typical implementation, returns the
// available space on a queue.
//
// There are two important caveats:
// 1. The guarantee does not extend to varying-size tasks due to possible
// internal fragmentation of batches.
// 2. The guarantee only holds in a single-thread environment or critical
// section, i.e. if an intervening thread cannot call Schedule().
//
// This method is useful for monitoring, or for guaranteeing a future slot in
// the schedule (but being mindful about the caveats listed above).
virtual size_t SchedulingCapacity() const = 0;
// Returns the maximum allowed size of tasks submitted to the scheduler. (This
// is typically equal to a configured maximum batch size.)
virtual size_t max_task_size() const = 0;
};
//////////
// Implementation details follow. API users need not read.
template <typename TaskType>
Batch<TaskType>::~Batch() {
WaitUntilClosed();
}
template <typename TaskType>
void Batch<TaskType>::AddTask(std::unique_ptr<TaskType> task) {
DCHECK(!IsClosed());
{
mutex_lock l(mu_);
size_ += task->size();
tasks_.push_back(std::move(task));
}
}
template <typename TaskType>
std::unique_ptr<TaskType> Batch<TaskType>::RemoveTask() {
{
mutex_lock l(mu_);
if (tasks_.empty()) {
return nullptr;
}
std::unique_ptr<TaskType> task = std::move(tasks_.back());
size_ -= task->size();
tasks_.pop_back();
return task;
}
}
template <typename TaskType>
int Batch<TaskType>::num_tasks() const {
{
mutex_lock l(mu_);
return tasks_.size();
}
}
template <typename TaskType>
bool Batch<TaskType>::empty() const {
{
mutex_lock l(mu_);
return tasks_.empty();
}
}
template <typename TaskType>
const TaskType& Batch<TaskType>::task(int i) const {
DCHECK_GE(i, 0);
{
mutex_lock l(mu_);
DCHECK_LT(i, tasks_.size());
return *tasks_[i].get();
}
}
template <typename TaskType>
TaskType* Batch<TaskType>::mutable_task(int i) {
DCHECK_GE(i, 0);
{
mutex_lock l(mu_);
DCHECK_LT(i, tasks_.size());
return tasks_[i].get();
}
}
template <typename TaskType>
size_t Batch<TaskType>::size() const {
{
mutex_lock l(mu_);
return size_;
}
}
template <typename TaskType>
bool Batch<TaskType>::IsClosed() const {
return const_cast<Notification*>(&closed_)->HasBeenNotified();
}
template <typename TaskType>
void Batch<TaskType>::WaitUntilClosed() const {
const_cast<Notification*>(&closed_)->WaitForNotification();
}
template <typename TaskType>
void Batch<TaskType>::Close() {
closed_.Notify();
}
} // namespace serving
} // namespace tensorflow
#endif // TENSORFLOW_CORE_KERNELS_BATCHING_UTIL_BATCH_SCHEDULER_H_
@@ -0,0 +1,134 @@
/* Copyright 2016 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
// PeriodicFunction will periodically call the given function with a specified
// period in a background thread. After Start() returns, the thread is
// guaranteed to have started. The destruction of the class causes the
// background thread to be destroyed as well. Start() should not be called more
// than once.
//
// PeriodicFunction runs the function as soon as any previous run both is
// complete and was started more than "interval_micros" earlier. Thus, runs are
// both serialized, and normally have a period of "interval_micros" if no run
// exceeds the time.
//
// Note that, if the function takes longer than two interval_micross to finish,
// then PeriodicFunction will "skip" at least one call to the function. For
// instance, if the period is 50ms and the function starts runs at time 0 for
// 150ms, then the function will immediately start executing again at time 150,
// but there will be no function runs corresponding to times 50 or 100. This is
// especially important to remember when using an environment with a simulated
// clock: advancing simulated time atomically over N interval_micross will not
// cause the function to be called N times.
//
// This object is thread-safe.
//
// Example:
//
// class Foo {
// public:
// Foo() : periodic_function_([this]() { Bar(); },
// 1000 /* 1000us == 1ms*/) {
// }
//
// private:
// void Bar() { ... }
//
// PeriodicFunction periodic_function_;
// };
#ifndef TENSORFLOW_CORE_KERNELS_BATCHING_UTIL_PERIODIC_FUNCTION_H_
#define TENSORFLOW_CORE_KERNELS_BATCHING_UTIL_PERIODIC_FUNCTION_H_
#include "tensorflow/core/kernels/batching_util/periodic_function.h"
#include <functional>
#include <memory>
#include <string>
#include "tensorflow/core/lib/core/notification.h"
#include "tensorflow/core/platform/env.h"
#include "tensorflow/core/platform/macros.h"
#include "tensorflow/core/platform/mutex.h"
#include "tensorflow/core/platform/thread_annotations.h"
#include "tensorflow/core/platform/types.h"
namespace tensorflow {
namespace serving {
namespace internal {
class PeriodicFunctionTestAccess;
}
class PeriodicFunction {
public:
// Provides the ability to customize several aspects of the PeriodicFunction.
// Passed to constructor of PeriodicFunction.
struct Options {
Options() {}
// Any standard thread options, such as stack size, should
// be passed via "thread_options".
ThreadOptions thread_options;
// Specifies the thread name prefix (see the description in class
// Thread).
string thread_name_prefix = "periodic_function";
// The environment to use. Does not take ownership, but must remain alive
// for as long as the PeriodicFunction exists.
Env* env = Env::Default();
// Specifies the length of sleep before the first invocation of the
// function.
// This can be used for adding a random jitter to avoid synchronous behavior
// across multiple periodic functions.
int64 startup_delay_micros = 0;
};
// Also starts the background thread which will be calling the function.
PeriodicFunction(const std::function<void()>& function, int64 interval_micros,
const Options& options = Options());
~PeriodicFunction();
private:
friend class internal::PeriodicFunctionTestAccess;
// Notifies the background thread to stop.
void NotifyStop();
// (Blocking.) Loops forever calling "function_" every "interval_micros_".
void RunLoop(int64 start) LOCKS_EXCLUDED(mutex_);
const std::function<void()> function_; // Actual client function
const int64 interval_micros_; // Interval between calls.
const Options options_;
// Protects state below.
mutable mutex mutex_;
// Used to notify the thread to stop.
Notification stop_thread_;
// Thread for running "function_"
std::unique_ptr<Thread> thread_ GUARDED_BY(mutex_) = nullptr;
TF_DISALLOW_COPY_AND_ASSIGN(PeriodicFunction);
};
} // namespace serving
} // namespace tensorflow
#endif // TENSORFLOW_CORE_KERNELS_BATCHING_UTIL_PERIODIC_FUNCTION_H_
@@ -0,0 +1,706 @@
/* Copyright 2016 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
#ifndef TENSORFLOW_CORE_KERNELS_BATCHING_UTIL_SHARED_BATCH_SCHEDULER_H_
#define TENSORFLOW_CORE_KERNELS_BATCHING_UTIL_SHARED_BATCH_SCHEDULER_H_
#include <stddef.h>
#include <deque>
#include <functional>
#include <list>
#include <memory>
#include <string>
#include <utility>
#include <vector>
#include "tensorflow/core/kernels/batching_util/batch_scheduler.h"
#include "tensorflow/core/kernels/batching_util/periodic_function.h"
#include "tensorflow/core/lib/core/errors.h"
#include "tensorflow/core/lib/core/status.h"
#include "tensorflow/core/lib/strings/strcat.h"
#include "tensorflow/core/platform/byte_order.h"
#include "tensorflow/core/platform/cpu_info.h"
#include "tensorflow/core/platform/env.h"
#include "tensorflow/core/platform/thread_annotations.h"
#include "tensorflow/core/platform/types.h"
namespace tensorflow {
namespace serving {
namespace internal {
template <typename TaskType>
class Queue;
} // namespace internal
} // namespace serving
} // namespace tensorflow
namespace tensorflow {
namespace serving {
// A batch scheduler for server instances that service multiple request types
// (e.g. multiple machine-learned models, or multiple versions of a model served
// concurrently), or even multiple distinct tasks for a given request. The
// scheduler multiplexes batches of different kinds of tasks onto a fixed-size
// thread pool (each batch contains tasks of a single type), in a carefully
// controlled manner. A common configuration is to set the number of threads
// equal to the number of hardware accelerator units, in which case the
// scheduler takes care of multiplexing the task types onto the shared hardware,
// in a manner that is both fair and efficient.
//
// Semantically, SharedBatchScheduler behaves like having N instances of
// BasicBatchScheduler (see basic_batch_scheduler.h), one per task type. The
// difference is that under the covers there is a single shared thread pool,
// instead of N independent ones, with their sharing deliberately coordinated.
//
// SharedBatchScheduler does not implement the BatchScheduler API; rather, it
// presents an abstraction of "queues", where each queue corresponds to one type
// of task. Tasks submitted to a given queue are placed in their own batches,
// and cannot be mixed with other tasks. Queues can be added and deleted
// dynamically, to accommodate e.g. versions of a model being brought up and
// down over the lifetime of a server.
//
// The batch thread pool round-robins through the queues, running one batch
// from a queue and then moving to the next queue. Each queue behaves like a
// BasicBatchScheduler instance, in the sense that it has maximum batch size and
// timeout parameters, which govern when a batch is eligible to be processed.
//
// Each queue is independently configured with a maximum size (in terms of the
// maximum number of batches worth of enqueued tasks). For online serving, it is
// recommended that the queue sizes be configured such that the sum of the sizes
// of the active queues roughly equal the number of batch threads. (The idea is
// that if all threads become available at roughly the same time, there will be
// enough enqueued work for them to take on, but no more.)
//
// If queue sizes are configured in the manner suggested above, the maximum time
// a task can spend in a queue before being placed in a batch and assigned to a
// thread for processing, is the greater of:
// - the maximum time to process one batch of tasks from any active queue
// - the configured timeout parameter for the task's queue (which can be 0)
//
// For bulk processing jobs and throughput-oriented benchmarks, you may want to
// set the maximum queue size to a large value.
//
// TODO(b/26539183): Support queue servicing policies other than round-robin.
// E.g. let each queue specify a "share" (an int >= 1), so e.g. with queues A
// and B having shares 1 and 2 respectively, the servicing pattern is ABBABB...
//
//
// PERFORMANCE TUNING: See README.md.
//
template <typename TaskType>
class SharedBatchScheduler
: public std::enable_shared_from_this<SharedBatchScheduler<TaskType>> {
public:
// TODO(b/25089730): Tune defaults based on best practices as they develop.
struct Options {
// The name to use for the pool of batch threads.
string thread_pool_name = {"batch_threads"};
// The number of threads to use to process batches.
// Must be >= 1, and should be tuned carefully.
int num_batch_threads = port::NumSchedulableCPUs();
// The environment to use.
// (Typically only overridden by test code.)
Env* env = Env::Default();
};
// Ownership is shared between the caller of Create() and any queues created
// via AddQueue().
static Status Create(
const Options& options,
std::shared_ptr<SharedBatchScheduler<TaskType>>* scheduler);
~SharedBatchScheduler();
// Adds a queue to which tasks may be submitted. The returned queue implements
// the BatchScheduler API. Each queue has its own set of scheduling options,
// and its own callback to process batches of tasks submitted to the queue.
//
// The returned queue's destructor blocks until all tasks submitted to it have
// been processed.
struct QueueOptions {
// The maximum size of each batch.
//
// The scheduler may form batches of any size between 1 and this number
// (inclusive). If there is a need to quantize the batch sizes, i.e. only
// submit batches whose size is in a small set of allowed sizes, that can be
// done by adding padding in the process-batch callback.
size_t max_batch_size = 1000;
// If a task has been enqueued for this amount of time (in microseconds),
// and a thread is available, the scheduler will immediately form a batch
// from enqueued tasks and assign the batch to the thread for processing,
// even if the batch's size is below 'max_batch_size'.
//
// This parameter offers a way to bound queue latency, so that a task isn't
// stuck in the queue indefinitely waiting for enough tasks to arrive to
// make a full batch. (The latency bound is given in the class documentation
// above.)
//
// The goal is to smooth out batch sizes under low request rates, and thus
// avoid latency spikes.
int64 batch_timeout_micros = 0;
// The maximum allowable number of enqueued (accepted by Schedule() but
// not yet being processed on a batch thread) tasks in terms of batches.
// If this limit is reached, Schedule() will return an UNAVAILABLE error.
// See the class documentation above for guidelines on how to tune this
// parameter.
size_t max_enqueued_batches = 10;
};
Status AddQueue(const QueueOptions& options,
std::function<void(std::unique_ptr<Batch<TaskType>>)>
process_batch_callback,
std::unique_ptr<BatchScheduler<TaskType>>* queue);
private:
explicit SharedBatchScheduler(const Options& options);
// The code executed in 'batch_threads_'. Obtains a batch to process from the
// queue pointed to by 'next_queue_to_schedule_', and processes it. If that
// queue declines to provide a batch to process, moves onto the next queue. If
// no queues provide a batch to process, just sleeps briefly and exits.
void ThreadLogic();
const Options options_;
mutex mu_;
// A list of queues. (We use std::list instead of std::vector to ensure that
// iterators are not invalidated by adding/removing elements. It also offers
// efficient removal of elements from the middle.)
using QueueList = std::list<std::unique_ptr<internal::Queue<TaskType>>>;
// All "active" queues, i.e. ones that either:
// - have not been removed, or
// - have been removed but are not yet empty.
QueueList queues_ GUARDED_BY(mu_);
// An iterator over 'queues_', pointing to the queue from which the next
// available batch thread should grab work.
typename QueueList::iterator next_queue_to_schedule_ GUARDED_BY(mu_);
// Used by idle batch threads to wait for work to enter the system. Notified
// whenever a batch becomes schedulable.
condition_variable schedulable_batch_cv_;
// Threads that process batches obtained from the queues.
std::vector<std::unique_ptr<PeriodicFunction>> batch_threads_;
TF_DISALLOW_COPY_AND_ASSIGN(SharedBatchScheduler);
};
//////////
// Implementation details follow. API users need not read.
namespace internal {
// A task queue for SharedBatchScheduler. Accepts tasks and accumulates them
// into batches, and dispenses those batches to be processed via a "pull"
// interface. The queue's behavior is governed by maximum batch size, timeout
// and maximum queue length parameters; see their documentation in
// SharedBatchScheduler.
//
// The queue is implemented as a deque of batches, with these invariants:
// - The number of batches is between 1 and 'options_.max_enqueued_batches'.
// - The back-most batch is open; the rest are closed.
//
// Submitted tasks are added to the open batch. If that batch doesn't have room
// but the queue isn't full, then that batch is closed and a new open batch is
// started.
//
// Batch pull requests are handled by dequeuing the front-most batch if it is
// closed. If the front-most batch is open (i.e. the queue contains only one
// batch) and has reached the timeout, it is immediately closed and returned;
// otherwise no batch is returned for the request.
template <typename TaskType>
class Queue {
public:
using ProcessBatchCallback =
std::function<void(std::unique_ptr<Batch<TaskType>>)>;
using SchedulableBatchCallback = std::function<void()>;
Queue(const typename SharedBatchScheduler<TaskType>::QueueOptions& options,
Env* env, ProcessBatchCallback process_batch_callback,
SchedulableBatchCallback schdulable_batch_callback);
// Illegal to destruct unless the queue is empty.
~Queue();
// Submits a task to the queue, with the same semantics as
// BatchScheduler::Schedule().
Status Schedule(std::unique_ptr<TaskType>* task);
// Returns the number of enqueued tasks, with the same semantics as
// BatchScheduler::NumEnqueuedTasks().
size_t NumEnqueuedTasks() const;
// Returns the queue capacity, with the same semantics as
// BatchScheduler::SchedulingCapacity().
size_t SchedulingCapacity() const;
// Returns the maximum allowed size of tasks submitted to the queue.
size_t max_task_size() const { return options_.max_batch_size; }
// Called by a thread that is ready to process a batch, to request one from
// this queue. Either returns a batch that is ready to be processed, or
// nullptr if the queue declines to schedule a batch at this time. If it
// returns a batch, the batch is guaranteed to be closed.
std::unique_ptr<Batch<TaskType>> ScheduleBatch();
// Processes a batch that has been returned earlier by ScheduleBatch().
void ProcessBatch(std::unique_ptr<Batch<TaskType>> batch);
// Determines whether the queue is empty, i.e. has no tasks waiting or being
// processed.
bool IsEmpty() const;
// Marks the queue closed, and waits until it is empty.
void CloseAndWaitUntilEmpty();
bool closed() const {
mutex_lock l(mu_);
return closed_;
}
private:
// Same as IsEmpty(), but assumes the caller already holds a lock on 'mu_'.
bool IsEmptyInternal() const EXCLUSIVE_LOCKS_REQUIRED(mu_);
// Closes the open batch residing at the back of 'batches_', and inserts a
// fresh open batch behind it.
void StartNewBatch() EXCLUSIVE_LOCKS_REQUIRED(mu_);
// Determines whether the open batch residing at the back of 'batches_' is
// currently schedulable.
bool IsOpenBatchSchedulable() const EXCLUSIVE_LOCKS_REQUIRED(mu_);
const typename SharedBatchScheduler<TaskType>::QueueOptions options_;
// The environment to use.
Env* env_;
// A callback invoked to processes a batch of work units. Always invoked from
// a batch thread.
ProcessBatchCallback process_batch_callback_;
// A callback invoked to notify the scheduler that a new batch has become
// schedulable.
SchedulableBatchCallback schedulable_batch_callback_;
mutable mutex mu_;
// Whether this queue can accept new tasks. This variable is monotonic: it
// starts as false, and then at some point gets set to true and remains true
// for the duration of this object's life.
bool closed_ GUARDED_BY(mu_) = false;
// The enqueued batches. See the invariants in the class comments above.
std::deque<std::unique_ptr<Batch<TaskType>>> batches_ GUARDED_BY(mu_);
// The time at which the first task was added to the open (back-most) batch
// in 'batches_'. Valid iff that batch contains at least one task.
uint64 open_batch_start_time_micros_ GUARDED_BY(mu_);
// Whether this queue contains a batch that is eligible to be scheduled. Used
// to keep track of when to call 'schedulable_batch_callback_'.
bool schedulable_batch_ GUARDED_BY(mu_) = false;
// The number of batches currently being processed by batch threads.
// Incremented in ScheduleBatch() and decremented in ProcessBatch().
int num_batches_being_processed_ GUARDED_BY(mu_) = 0;
// Used by CloseAndWaitUntilEmpty() to wait until the queue is empty, for the
// case in which the queue is not empty when CloseAndWaitUntilEmpty() starts.
// When ProcessBatch() dequeues the last batch and makes the queue empty, if
// 'empty_notification_' is non-null it calls 'empty_notification_->Notify()'.
Notification* empty_notification_ GUARDED_BY(mu_) = nullptr;
TF_DISALLOW_COPY_AND_ASSIGN(Queue);
};
// A RAII-style object that points to a Queue and implements
// the BatchScheduler API. To be handed out to clients who call AddQueue().
template <typename TaskType>
class QueueHandle : public BatchScheduler<TaskType> {
public:
QueueHandle(std::shared_ptr<SharedBatchScheduler<TaskType>> scheduler,
Queue<TaskType>* queue);
~QueueHandle() override;
Status Schedule(std::unique_ptr<TaskType>* task) override;
size_t NumEnqueuedTasks() const override;
size_t SchedulingCapacity() const override;
size_t max_task_size() const override { return queue_->max_task_size(); }
private:
// The scheduler that owns 'queue_'.
std::shared_ptr<SharedBatchScheduler<TaskType>> scheduler_;
// The queue this handle wraps. Owned by 'scheduler_', which keeps it alive at
// least until this class's destructor closes it.
Queue<TaskType>* queue_;
TF_DISALLOW_COPY_AND_ASSIGN(QueueHandle);
};
} // namespace internal
template <typename TaskType>
Status SharedBatchScheduler<TaskType>::Create(
const Options& options,
std::shared_ptr<SharedBatchScheduler<TaskType>>* scheduler) {
if (options.num_batch_threads < 1) {
return errors::InvalidArgument("num_batch_threads must be positive; was ",
options.num_batch_threads);
}
scheduler->reset(new SharedBatchScheduler<TaskType>(options));
return Status::OK();
}
template <typename TaskType>
SharedBatchScheduler<TaskType>::~SharedBatchScheduler() {
// Wait until the batch threads finish clearing out and deleting the closed
// queues.
for (;;) {
{
mutex_lock l(mu_);
if (queues_.empty()) {
break;
}
}
const int64 kSleepTimeMicros = 100;
options_.env->SleepForMicroseconds(kSleepTimeMicros);
}
// Delete the batch threads before allowing state the threads may access (e.g.
// 'mu_') to be deleted.
batch_threads_.clear();
}
template <typename TaskType>
Status SharedBatchScheduler<TaskType>::AddQueue(
const QueueOptions& options,
std::function<void(std::unique_ptr<Batch<TaskType>>)>
process_batch_callback,
std::unique_ptr<BatchScheduler<TaskType>>* queue) {
if (options.max_batch_size == 0) {
return errors::InvalidArgument("max_batch_size must be positive; was ",
options.max_batch_size);
}
if (options.batch_timeout_micros < 0) {
return errors::InvalidArgument(
"batch_timeout_micros must be non-negative; was ",
options.batch_timeout_micros);
}
if (options.max_enqueued_batches < 0) {
return errors::InvalidArgument(
"max_enqueued_batches must be non-negative; was ",
options.max_enqueued_batches);
}
auto schedulable_batch_callback = [this] {
mutex_lock l(mu_);
schedulable_batch_cv_.notify_one();
};
auto internal_queue =
std::unique_ptr<internal::Queue<TaskType>>(new internal::Queue<TaskType>(
options, options_.env, process_batch_callback,
schedulable_batch_callback));
auto handle = std::unique_ptr<BatchScheduler<TaskType>>(
new internal::QueueHandle<TaskType>(this->shared_from_this(),
internal_queue.get()));
{
mutex_lock l(mu_);
queues_.push_back(std::move(internal_queue));
if (next_queue_to_schedule_ == queues_.end()) {
next_queue_to_schedule_ = queues_.begin();
}
}
*queue = std::move(handle);
return Status::OK();
}
template <typename TaskType>
SharedBatchScheduler<TaskType>::SharedBatchScheduler(const Options& options)
: options_(options), next_queue_to_schedule_(queues_.end()) {
// Kick off the batch threads.
PeriodicFunction::Options periodic_fn_options;
periodic_fn_options.thread_name_prefix =
strings::StrCat(options.thread_pool_name, "_");
for (int i = 0; i < options.num_batch_threads; ++i) {
std::unique_ptr<PeriodicFunction> thread(new PeriodicFunction(
[this] { this->ThreadLogic(); },
0 /* function invocation interval time */, periodic_fn_options));
batch_threads_.push_back(std::move(thread));
}
}
template <typename TaskType>
void SharedBatchScheduler<TaskType>::ThreadLogic() {
// A batch to process next (or nullptr if no work to do).
std::unique_ptr<Batch<TaskType>> batch_to_process;
// The queue with which 'batch_to_process' is associated.
internal::Queue<TaskType>* queue_for_batch = nullptr;
{
mutex_lock l(mu_);
const int num_queues = queues_.size();
for (int num_queues_tried = 0;
batch_to_process == nullptr && num_queues_tried < num_queues;
++num_queues_tried) {
DCHECK(next_queue_to_schedule_ != queues_.end());
// If a closed queue responds to ScheduleBatch() with nullptr, the queue
// will never yield any further batches so we can drop it. To avoid a
// race, we take a snapshot of the queue's closedness state *before*
// calling ScheduleBatch().
const bool queue_closed = (*next_queue_to_schedule_)->closed();
// Ask '*next_queue_to_schedule_' if it wants us to process a batch.
batch_to_process = (*next_queue_to_schedule_)->ScheduleBatch();
if (batch_to_process != nullptr) {
queue_for_batch = next_queue_to_schedule_->get();
}
// Advance 'next_queue_to_schedule_'.
if (queue_closed && (*next_queue_to_schedule_)->IsEmpty() &&
batch_to_process == nullptr) {
// We've encountered a closed queue with no work to do. Drop it.
DCHECK_NE(queue_for_batch, next_queue_to_schedule_->get());
next_queue_to_schedule_ = queues_.erase(next_queue_to_schedule_);
} else {
++next_queue_to_schedule_;
}
if (next_queue_to_schedule_ == queues_.end() && !queues_.empty()) {
// We've hit the end. Wrap to the first queue.
next_queue_to_schedule_ = queues_.begin();
}
}
if (batch_to_process == nullptr) {
// We couldn't find any work to do. Wait until a new batch becomes
// schedulable, or some time has elapsed, before checking again.
const int64 kTimeoutMillis = 1; // The smallest accepted granule of time.
WaitForMilliseconds(&l, &schedulable_batch_cv_, kTimeoutMillis);
return;
}
}
queue_for_batch->ProcessBatch(std::move(batch_to_process));
}
namespace internal {
template <typename TaskType>
Queue<TaskType>::Queue(
const typename SharedBatchScheduler<TaskType>::QueueOptions& options,
Env* env, ProcessBatchCallback process_batch_callback,
SchedulableBatchCallback schedulable_batch_callback)
: options_(options),
env_(env),
process_batch_callback_(process_batch_callback),
schedulable_batch_callback_(schedulable_batch_callback) {
// Create an initial, open batch.
batches_.emplace_back(new Batch<TaskType>);
}
template <typename TaskType>
Queue<TaskType>::~Queue() {
mutex_lock l(mu_);
DCHECK(IsEmptyInternal());
// Close the (empty) open batch, so its destructor doesn't block.
batches_.back()->Close();
}
template <typename TaskType>
Status Queue<TaskType>::Schedule(std::unique_ptr<TaskType>* task) {
if ((*task)->size() > options_.max_batch_size) {
return errors::InvalidArgument("Task size ", (*task)->size(),
" is larger than maximum batch size ",
options_.max_batch_size);
}
bool notify_of_schedulable_batch = false;
{
mutex_lock l(mu_);
DCHECK(!closed_);
if (batches_.back()->size() + (*task)->size() > options_.max_batch_size) {
if (batches_.size() >= options_.max_enqueued_batches) {
return errors::Unavailable(
"The batch scheduling queue to which this task was submitted is "
"full");
}
StartNewBatch();
}
if (batches_.back()->empty()) {
open_batch_start_time_micros_ = env_->NowMicros();
}
batches_.back()->AddTask(std::move(*task));
if (!schedulable_batch_) {
if (batches_.size() > 1 || IsOpenBatchSchedulable()) {
schedulable_batch_ = true;
notify_of_schedulable_batch = true;
}
}
}
if (notify_of_schedulable_batch) {
schedulable_batch_callback_();
}
return Status::OK();
}
template <typename TaskType>
size_t Queue<TaskType>::NumEnqueuedTasks() const {
mutex_lock l(mu_);
size_t num_enqueued_tasks = 0;
for (const auto& batch : batches_) {
num_enqueued_tasks += batch->num_tasks();
}
return num_enqueued_tasks;
}
template <typename TaskType>
size_t Queue<TaskType>::SchedulingCapacity() const {
mutex_lock l(mu_);
const int num_new_batches_schedulable =
options_.max_enqueued_batches - batches_.size();
const int open_batch_capacity =
options_.max_batch_size - batches_.back()->size();
return (num_new_batches_schedulable * options_.max_batch_size) +
open_batch_capacity;
}
template <typename TaskType>
std::unique_ptr<Batch<TaskType>> Queue<TaskType>::ScheduleBatch() {
// The batch to schedule, which we may populate below. (If left as nullptr,
// that means we are electing not to schedule a batch at this time.)
std::unique_ptr<Batch<TaskType>> batch_to_schedule;
{
mutex_lock l(mu_);
// Consider closing the open batch at this time, to schedule it.
if (batches_.size() == 1 && IsOpenBatchSchedulable()) {
StartNewBatch();
}
if (batches_.size() >= 2) {
// There is at least one closed batch that is ready to be scheduled.
++num_batches_being_processed_;
batch_to_schedule = std::move(batches_.front());
batches_.pop_front();
} else {
schedulable_batch_ = false;
}
}
return batch_to_schedule;
}
template <typename TaskType>
void Queue<TaskType>::ProcessBatch(std::unique_ptr<Batch<TaskType>> batch) {
process_batch_callback_(std::move(batch));
{
mutex_lock l(mu_);
--num_batches_being_processed_;
if (empty_notification_ != nullptr && IsEmptyInternal()) {
empty_notification_->Notify();
}
}
}
template <typename TaskType>
bool Queue<TaskType>::IsEmpty() const {
mutex_lock l(mu_);
return IsEmptyInternal();
}
template <typename TaskType>
void Queue<TaskType>::CloseAndWaitUntilEmpty() {
Notification empty;
{
mutex_lock l(mu_);
closed_ = true;
if (IsEmptyInternal()) {
empty.Notify();
} else {
// Arrange for ProcessBatch() to notify when the queue becomes empty.
empty_notification_ = &empty;
}
}
empty.WaitForNotification();
}
template <typename TaskType>
bool Queue<TaskType>::IsEmptyInternal() const {
return num_batches_being_processed_ == 0 && batches_.size() == 1 &&
batches_.back()->empty();
}
template <typename TaskType>
void Queue<TaskType>::StartNewBatch() {
batches_.back()->Close();
batches_.emplace_back(new Batch<TaskType>);
}
template <typename TaskType>
bool Queue<TaskType>::IsOpenBatchSchedulable() const {
Batch<TaskType>* open_batch = batches_.back().get();
if (open_batch->empty()) {
return false;
}
return closed_ || open_batch->size() >= options_.max_batch_size ||
env_->NowMicros() >=
open_batch_start_time_micros_ + options_.batch_timeout_micros;
}
template <typename TaskType>
QueueHandle<TaskType>::QueueHandle(
std::shared_ptr<SharedBatchScheduler<TaskType>> scheduler,
Queue<TaskType>* queue)
: scheduler_(scheduler), queue_(queue) {}
template <typename TaskType>
QueueHandle<TaskType>::~QueueHandle() {
queue_->CloseAndWaitUntilEmpty();
}
template <typename TaskType>
Status QueueHandle<TaskType>::Schedule(std::unique_ptr<TaskType>* task) {
return queue_->Schedule(task);
}
template <typename TaskType>
size_t QueueHandle<TaskType>::NumEnqueuedTasks() const {
return queue_->NumEnqueuedTasks();
}
template <typename TaskType>
size_t QueueHandle<TaskType>::SchedulingCapacity() const {
return queue_->SchedulingCapacity();
}
} // namespace internal
} // namespace serving
} // namespace tensorflow
#endif // TENSORFLOW_CORE_KERNELS_BATCHING_UTIL_SHARED_BATCH_SCHEDULER_H_
@@ -0,0 +1,51 @@
/* Copyright 2016 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
#ifndef TENSORFLOW_CORE_KERNELS_BETAINC_OP_H_
#define TENSORFLOW_CORE_KERNELS_BETAINC_OP_H_
// Functor definition for BetaincOp, must be compilable by nvcc.
#include "third_party/eigen3/unsupported/Eigen/CXX11/Tensor"
#include "tensorflow/core/framework/tensor_types.h"
namespace tensorflow {
namespace functor {
// Functor used by BetaincOp to do the computations.
template <typename Device, typename T, int NDIM>
struct Betainc {
void operator()(const Device& d, typename TTypes<T, NDIM>::ConstTensor a,
typename TTypes<T, NDIM>::ConstTensor b,
typename TTypes<T, NDIM>::ConstTensor x,
typename TTypes<T, NDIM>::Tensor output) {
output.device(d) = Eigen::betainc(a, b, x);
}
void BCast(const Device& d, typename TTypes<T, NDIM>::ConstTensor a,
const typename Eigen::array<Eigen::DenseIndex, NDIM>& bcast_a,
typename TTypes<T, NDIM>::ConstTensor b,
const typename Eigen::array<Eigen::DenseIndex, NDIM>& bcast_b,
typename TTypes<T, NDIM>::ConstTensor x,
const typename Eigen::array<Eigen::DenseIndex, NDIM>& bcast_x,
typename TTypes<T, NDIM>::Tensor output) {
output.device(d) = Eigen::betainc(
a.broadcast(bcast_a), b.broadcast(bcast_b), x.broadcast(bcast_x));
}
};
} // namespace functor
} // namespace tensorflow
#endif // TENSORFLOW_CORE_KERNELS_BETAINC_OP_H_
@@ -0,0 +1,55 @@
/* Copyright 2015 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
#ifndef TENSORFLOW_CORE_KERNELS_BIAS_OP_H_
#define TENSORFLOW_CORE_KERNELS_BIAS_OP_H_
// Functor definition for BiasOp, must be compilable by nvcc.
#include "third_party/eigen3/unsupported/Eigen/CXX11/Tensor"
#include "tensorflow/core/framework/tensor_types.h"
namespace tensorflow {
namespace functor {
// Functor used by BiasOp to do the computations.
template <typename Device, typename T, int Dims>
struct Bias {
// Add "bias" to "input", broadcasting it on all dimensions but the last one.
void operator()(const Device& d, typename TTypes<T, Dims>::ConstTensor input,
typename TTypes<T>::ConstVec bias,
typename TTypes<T, Dims>::Tensor output) {
if (input.size() >= INT_MAX) {
const int64_t bias_size = bias.dimension(0);
const int64_t rest_size = input.size() / bias_size;
Eigen::DSizes<int64_t, 1> one_d(input.size());
Eigen::DSizes<int64_t, 1> bcast(rest_size);
output.reshape(one_d).device(d) =
input.reshape(one_d) + bias.broadcast(bcast).reshape(one_d);
} else {
const int bias_size = bias.dimension(0);
const int rest_size = input.size() / bias_size;
Eigen::DSizes<int, 1> one_d(input.size());
Eigen::DSizes<int, 1> bcast(rest_size);
To32Bit(output).reshape(one_d).device(d) =
To32Bit(input).reshape(one_d) +
To32Bit(bias).broadcast(bcast).reshape(one_d);
}
}
};
} // namespace functor
} // namespace tensorflow
#endif // TENSORFLOW_CORE_KERNELS_BIAS_OP_H_
@@ -0,0 +1,80 @@
/* Copyright 2015 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
#ifndef TENSORFLOW_CORE_KERNELS_BIAS_OP_GPU_H_
#define TENSORFLOW_CORE_KERNELS_BIAS_OP_GPU_H_
#define EIGEN_USE_GPU
#include "third_party/eigen3/unsupported/Eigen/CXX11/Tensor"
#include "tensorflow/core/framework/op_kernel.h"
#include "tensorflow/core/framework/tensor_types.h"
#include "tensorflow/core/kernels/gpu_utils.h"
#include "tensorflow/core/util/tensor_format.h"
namespace tensorflow {
typedef Eigen::GpuDevice GPUDevice;
template <typename T>
struct BiasGPU {
static void compute(const GPUDevice& d, const T* input, const T* bias,
T* output, int32 batch, int32 height, int32 width,
int32 channel, TensorFormat data_format);
};
template <typename T>
struct BiasGradGPU {
static void compute(const GPUDevice& device, const T* output_backprop,
T* bias_backprop, int32 batch, int32 height, int32 width,
int32 channel, TensorFormat data_format);
static void DoRowReduction(OpKernelContext* context, T* output,
const T* input, int rows, int cols);
static void DoColReduction(OpKernelContext* context, T* output,
const T* input, int rows, int cols);
};
enum class BiasAddGradGPUMode {
kInvalid = 0,
kNative = 1,
kReduction = 2,
};
// Describe the BiasGradGPU result from a perf experiment.
//
// Arguments:
// algorithm: returns the method to use for bias add grad.
// elapsed_time; returns the measured elapsed time in microseconds.
class BiasGradGPUProfileResult {
public:
bool is_valid() const {
return (algorithm_ != BiasAddGradGPUMode::kInvalid &&
elapsed_time_ != std::numeric_limits<float>::max());
}
BiasAddGradGPUMode algorithm() const { return algorithm_; }
void set_algorithm(BiasAddGradGPUMode val) { algorithm_ = val; }
uint64 elapsed_time() const { return elapsed_time_; }
void set_elapsed_time(uint64 val) { elapsed_time_ = val; }
private:
BiasAddGradGPUMode algorithm_ = BiasAddGradGPUMode::kInvalid;
uint64 elapsed_time_ = std::numeric_limits<uint64>::max();
};
} // namespace tensorflow
#endif // TENSORFLOW_CORE_KERNELS_BIAS_OP_GPU_H_
@@ -0,0 +1,41 @@
/* Copyright 2015 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
#ifndef TENSORFLOW_CORE_KERNELS_BINCOUNT_OP_H_
#define TENSORFLOW_CORE_KERNELS_BINCOUNT_OP_H_
#include "third_party/eigen3/unsupported/Eigen/CXX11/Tensor"
#include "tensorflow/core/framework/op_kernel.h"
#include "tensorflow/core/framework/tensor_types.h"
#include "tensorflow/core/framework/types.h"
#include "tensorflow/core/lib/core/errors.h"
namespace tensorflow {
namespace functor {
template <typename Device, typename T>
struct BincountFunctor {
static Status Compute(OpKernelContext* context,
const typename TTypes<int32, 1>::ConstTensor& arr,
const typename TTypes<T, 1>::ConstTensor& weights,
typename TTypes<T, 1>::Tensor& output);
};
} // end namespace functor
} // end namespace tensorflow
#endif // TENSORFLOW_CORE_KERNELS_BINCOUNT_OP_H_
@@ -0,0 +1,30 @@
/* Copyright 2015 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
// See docs in ../ops/array_ops.cc.
#ifndef TENSORFLOW_CORE_KERNELS_BITCAST_OP_H_
#define TENSORFLOW_CORE_KERNELS_BITCAST_OP_H_
#include <string.h> // for memcpy
#include "tensorflow/core/framework/op_kernel.h"
#include "tensorflow/core/framework/register_types.h"
#include "tensorflow/core/framework/tensor.h"
#include "tensorflow/core/framework/tensor_shape.h"
#include "tensorflow/core/framework/types.h"
#include "tensorflow/core/lib/core/casts.h"
#endif // TENSORFLOW_CORE_KERNELS_BITCAST_OP_H_
File diff suppressed because it is too large. Load diff
@@ -0,0 +1,96 @@
// Copyright 2017 The TensorFlow Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// =============================================================================
#ifndef TENSORFLOW_CORE_KERNELS_BOOSTED_TREES_QUANTILES_QUANTILE_STREAM_RESOURCE_H_
#define TENSORFLOW_CORE_KERNELS_BOOSTED_TREES_QUANTILES_QUANTILE_STREAM_RESOURCE_H_
#include <vector>
#include "tensorflow/core/framework/resource_mgr.h"
#include "tensorflow/core/kernels/boosted_trees/quantiles/weighted_quantiles_stream.h"
#include "tensorflow/core/platform/macros.h"
#include "tensorflow/core/platform/mutex.h"
namespace tensorflow {
using QuantileStream =
boosted_trees::quantiles::WeightedQuantilesStream<float, float>;
// Quantile Stream Resource for a list of streams sharing the same number of
// quantiles, maximum elements, and epsilon.
class BoostedTreesQuantileStreamResource : public ResourceBase {
public:
BoostedTreesQuantileStreamResource(const float epsilon,
const int64 max_elements,
const int64 num_streams)
: are_buckets_ready_(false),
epsilon_(epsilon),
num_streams_(num_streams),
max_elements_(max_elements) {
streams_.reserve(num_streams_);
boundaries_.reserve(num_streams_);
for (int64 idx = 0; idx < num_streams; ++idx) {
streams_.push_back(QuantileStream(epsilon, max_elements));
boundaries_.push_back(std::vector<float>());
}
}
string DebugString() override { return "QuantileStreamResource"; }
tensorflow::mutex* mutex() { return &mu_; }
QuantileStream* stream(const int64 index) { return &streams_[index]; }
const std::vector<float>& boundaries(const int64 index) {
return boundaries_[index];
}
void set_boundaries(const std::vector<float>& boundaries, const int64 index) {
boundaries_[index] = boundaries;
}
float epsilon() const { return epsilon_; }
int64 num_streams() const { return num_streams_; }
bool are_buckets_ready() const { return are_buckets_ready_; }
void set_buckets_ready(const bool are_buckets_ready) {
are_buckets_ready_ = are_buckets_ready;
}
private:
~BoostedTreesQuantileStreamResource() override {}
// Mutex for the whole resource.
tensorflow::mutex mu_;
// Quantile streams.
std::vector<QuantileStream> streams_;
// Stores the boundaries. Same size as streams_.
std::vector<std::vector<float>> boundaries_;
// Whether boundaries are created. Initially boundaries are empty until
// set_boundaries are called.
bool are_buckets_ready_;
const float epsilon_;
const int64 num_streams_;
// An upper-bound for the number of elements.
int64 max_elements_;
TF_DISALLOW_COPY_AND_ASSIGN(BoostedTreesQuantileStreamResource);
};
} // namespace tensorflow
#endif // TENSORFLOW_CORE_KERNELS_BOOSTED_TREES_QUANTILES_QUANTILE_STREAM_RESOURCE_H_
@@ -0,0 +1,132 @@
// Copyright 2018 The TensorFlow Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// =============================================================================
#ifndef TENSORFLOW_CORE_KERNELS_BOOSTED_TREES_QUANTILES_WEIGHTED_QUANTILES_BUFFER_H_
#define TENSORFLOW_CORE_KERNELS_BOOSTED_TREES_QUANTILES_WEIGHTED_QUANTILES_BUFFER_H_
#include <algorithm>
#include <unordered_map>
#include <vector>
#include "tensorflow/core/platform/logging.h"
#include "tensorflow/core/platform/types.h"
namespace tensorflow {
namespace boosted_trees {
namespace quantiles {
// Buffering container ideally suited for scenarios where we need
// to sort and dedupe/compact fixed chunks of a stream of weighted elements.
template <typename ValueType, typename WeightType,
typename CompareFn = std::less<ValueType>>
class WeightedQuantilesBuffer {
public:
struct BufferEntry {
BufferEntry(ValueType v, WeightType w)
: value(std::move(v)), weight(std::move(w)) {}
BufferEntry() : value(), weight(0) {}
bool operator<(const BufferEntry& other) const {
return kCompFn(value, other.value);
}
bool operator==(const BufferEntry& other) const {
return value == other.value && weight == other.weight;
}
friend std::ostream& operator<<(std::ostream& strm,
const BufferEntry& entry) {
return strm << "{" << entry.value << ", " << entry.weight << "}";
}
ValueType value;
WeightType weight;
};
explicit WeightedQuantilesBuffer(int64 block_size, int64 max_elements)
: max_size_(std::min(block_size << 1, max_elements)) {
QCHECK(max_size_ > 0) << "Invalid buffer specification: (" << block_size
<< ", " << max_elements << ")";
vec_.reserve(max_size_);
}
// Disallow copying as it's semantically non-sensical in the Squawd algorithm
// but enable move semantics.
WeightedQuantilesBuffer(const WeightedQuantilesBuffer& other) = delete;
WeightedQuantilesBuffer& operator=(const WeightedQuantilesBuffer&) = delete;
WeightedQuantilesBuffer(WeightedQuantilesBuffer&& other) = default;
WeightedQuantilesBuffer& operator=(WeightedQuantilesBuffer&& other) = default;
// Push entry to buffer and maintain a compact representation within
// pre-defined size limit.
void PushEntry(ValueType value, WeightType weight) {
// Callers are expected to act on a full compacted buffer after the
// PushEntry call returns.
QCHECK(!IsFull()) << "Buffer already full: " << max_size_;
// Ignore zero and negative weight entries.
if (weight <= 0) {
return;
}
// Push back the entry to the buffer.
vec_.push_back(BufferEntry(std::move(value), std::move(weight)));
}
// Returns a sorted vector view of the base buffer and clears the buffer.
// Callers should minimize how often this is called, ideally only right after
// the buffer becomes full.
std::vector<BufferEntry> GenerateEntryList() {
std::vector<BufferEntry> ret;
if (vec_.size() == 0) {
return ret;
}
ret.swap(vec_);
vec_.reserve(max_size_);
std::sort(ret.begin(), ret.end());
size_t num_entries = 0;
for (size_t i = 1; i < ret.size(); ++i) {
if (ret[i].value != ret[i - 1].value) {
BufferEntry tmp = ret[i];
++num_entries;
ret[num_entries] = tmp;
} else {
ret[num_entries].weight += ret[i].weight;
}
}
ret.resize(num_entries + 1);
return ret;
}
int64 Size() const { return vec_.size(); }
bool IsFull() const { return vec_.size() >= max_size_; }
void Clear() { vec_.clear(); }
private:
using BufferVector = typename std::vector<BufferEntry>;
// Comparison function.
static constexpr decltype(CompareFn()) kCompFn = CompareFn();
// Base buffer.
size_t max_size_;
BufferVector vec_;
};
template <typename ValueType, typename WeightType, typename CompareFn>
constexpr decltype(CompareFn())
WeightedQuantilesBuffer<ValueType, WeightType, CompareFn>::kCompFn;
} // namespace quantiles
} // namespace boosted_trees
} // namespace tensorflow
#endif // TENSORFLOW_CORE_KERNELS_BOOSTED_TREES_QUANTILES_WEIGHTED_QUANTILES_BUFFER_H_
@@ -0,0 +1,330 @@
// Copyright 2018 The TensorFlow Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// =============================================================================
#ifndef TENSORFLOW_CORE_KERNELS_BOOSTED_TREES_QUANTILES_WEIGHTED_QUANTILES_STREAM_H_
#define TENSORFLOW_CORE_KERNELS_BOOSTED_TREES_QUANTILES_WEIGHTED_QUANTILES_STREAM_H_
#include <cmath>
#include <memory>
#include <vector>
#include "tensorflow/core/kernels/boosted_trees/quantiles/weighted_quantiles_buffer.h"
#include "tensorflow/core/kernels/boosted_trees/quantiles/weighted_quantiles_summary.h"
#include "tensorflow/core/platform/types.h"
namespace tensorflow {
namespace boosted_trees {
namespace quantiles {
// Class to compute approximate quantiles with error bound guarantees for
// weighted data sets.
// This implementation is an adaptation of techniques from the following papers:
// * (2001) Space-efficient online computation of quantile summaries.
// * (2004) Power-conserving computation of order-statistics over
// sensor networks.
// * (2007) A fast algorithm for approximate quantiles in high speed
// data streams.
// * (2016) XGBoost: A Scalable Tree Boosting System.
//
// The key ideas at play are the following:
// - Maintain an in-memory multi-level quantile summary in a way to guarantee
// a maximum approximation error of eps * W per bucket where W is the total
// weight across all points in the input dataset.
// - Two base operations are defined: MERGE and COMPRESS. MERGE combines two
// summaries guaranteeing a epsNew = max(eps1, eps2). COMPRESS compresses
// a summary to b + 1 elements guaranteeing epsNew = epsOld + 1/b.
// - b * sizeof(summary entry) must ideally be small enough to fit in an
// average CPU L2 cache.
// - To distribute this algorithm with maintaining error bounds, we need
// the worker-computed summaries to have no more than eps / h error
// where h is the height of the distributed computation graph which
// is 2 for an MR with no combiner.
//
// We mainly want to max out IO bw by ensuring we're not compute-bound and
// using a reasonable amount of RAM.
//
// Complexity:
// Compute: O(n * log(1/eps * log(eps * n))).
// Memory: O(1/eps * log^2(eps * n)) <- for one worker streaming through the
// entire dataset.
// An epsilon value of zero would make the algorithm extremely inefficent and
// therefore, is disallowed.
template <typename ValueType, typename WeightType,
typename CompareFn = std::less<ValueType>>
class WeightedQuantilesStream {
public:
using Buffer = WeightedQuantilesBuffer<ValueType, WeightType, CompareFn>;
using BufferEntry = typename Buffer::BufferEntry;
using Summary = WeightedQuantilesSummary<ValueType, WeightType, CompareFn>;
using SummaryEntry = typename Summary::SummaryEntry;
explicit WeightedQuantilesStream(double eps, int64 max_elements)
: eps_(eps), buffer_(1LL, 2LL), finalized_(false) {
// See the class documentation. An epsilon value of zero could cause
// perfoamance issues.
QCHECK(eps > 0) << "An epsilon value of zero is not allowed.";
std::tie(max_levels_, block_size_) = GetQuantileSpecs(eps, max_elements);
buffer_ = Buffer(block_size_, max_elements);
summary_levels_.reserve(max_levels_);
}
// Disallow copy and assign but enable move semantics for the stream.
WeightedQuantilesStream(const WeightedQuantilesStream& other) = delete;
WeightedQuantilesStream& operator=(const WeightedQuantilesStream&) = delete;
WeightedQuantilesStream(WeightedQuantilesStream&& other) = default;
WeightedQuantilesStream& operator=(WeightedQuantilesStream&& other) = default;
// Pushes one entry while maintaining approximation error invariants.
void PushEntry(const ValueType& value, const WeightType& weight) {
// Validate state.
QCHECK(!finalized_) << "Finalize() already called.";
// Push element to base buffer.
buffer_.PushEntry(value, weight);
// When compacted buffer is full we need to compress
// and push weighted quantile summary up the level chain.
if (buffer_.IsFull()) {
PushBuffer(buffer_);
}
}
// Pushes full buffer while maintaining approximation error invariants.
void PushBuffer(Buffer& buffer) {
// Validate state.
QCHECK(!finalized_) << "Finalize() already called.";
// Create local compressed summary and propagate.
local_summary_.BuildFromBufferEntries(buffer.GenerateEntryList());
local_summary_.Compress(block_size_, eps_);
PropagateLocalSummary();
}
// Pushes full summary while maintaining approximation error invariants.
void PushSummary(const std::vector<SummaryEntry>& summary) {
// Validate state.
QCHECK(!finalized_) << "Finalize() already called.";
// Create local compressed summary and propagate.
local_summary_.BuildFromSummaryEntries(summary);
local_summary_.Compress(block_size_, eps_);
PropagateLocalSummary();
}
// Flushes approximator and finalizes state.
void Finalize() {
// Validate state.
QCHECK(!finalized_) << "Finalize() may only be called once.";
// Flush any remaining buffer elements.
PushBuffer(buffer_);
// Create final merged summary.
local_summary_.Clear();
for (auto& summary : summary_levels_) {
local_summary_.Merge(summary);
summary.Clear();
}
summary_levels_.clear();
summary_levels_.shrink_to_fit();
finalized_ = true;
}
// Generates requested number of quantiles after finalizing stream.
// The returned quantiles can be queried using std::lower_bound to get
// the bucket for a given value.
std::vector<ValueType> GenerateQuantiles(int64 num_quantiles) const {
// Validate state.
QCHECK(finalized_)
<< "Finalize() must be called before generating quantiles.";
return local_summary_.GenerateQuantiles(num_quantiles);
}
// Generates requested number of boundaries after finalizing stream.
// The returned boundaries can be queried using std::lower_bound to get
// the bucket for a given value.
// The boundaries, while still guaranteeing approximation bounds, don't
// necessarily represent the actual quantiles of the distribution.
// Boundaries are preferable over quantiles when the caller is less
// interested in the actual quantiles distribution and more interested in
// getting a representative sample of boundary values.
std::vector<ValueType> GenerateBoundaries(int64 num_boundaries) const {
// Validate state.
QCHECK(finalized_)
<< "Finalize() must be called before generating boundaries.";
return local_summary_.GenerateBoundaries(num_boundaries);
}
// Calculates approximation error for the specified level.
// If the passed level is negative, the approximation error for the entire
// summary is returned. Note that after Finalize is called, only the overall
// error is available.
WeightType ApproximationError(int64 level = -1) const {
if (finalized_) {
QCHECK(level <= 0) << "Only overall error is available after Finalize()";
return local_summary_.ApproximationError();
}
if (summary_levels_.empty()) {
// No error even if base buffer isn't empty.
return 0;
}
// If level is negative, we get the approximation error
// for the top-most level which is the max approximation error
// in all summaries by construction.
if (level < 0) {
level = summary_levels_.size() - 1;
}
QCHECK(level < summary_levels_.size()) << "Invalid level.";
return summary_levels_[level].ApproximationError();
}
size_t MaxDepth() const { return summary_levels_.size(); }
// Generates requested number of quantiles after finalizing stream.
const Summary& GetFinalSummary() const {
// Validate state.
QCHECK(finalized_)
<< "Finalize() must be called before requesting final summary.";
return local_summary_;
}
// Helper method which, given the desired approximation error
// and an upper bound on the number of elements, computes the optimal
// number of levels and block size and returns them in the tuple.
static std::tuple<int64, int64> GetQuantileSpecs(double eps,
int64 max_elements);
// Serializes the internal state of the stream.
std::vector<Summary> SerializeInternalSummaries() const {
// The buffer should be empty for serialize to work.
QCHECK_EQ(buffer_.Size(), 0);
std::vector<Summary> result;
result.reserve(summary_levels_.size() + 1);
for (const Summary& summary : summary_levels_) {
result.push_back(summary);
}
result.push_back(local_summary_);
return result;
}
// Resets the state of the stream with a serialized state.
void DeserializeInternalSummaries(const std::vector<Summary>& summaries) {
// Clear the state before deserializing.
buffer_.Clear();
summary_levels_.clear();
local_summary_.Clear();
QCHECK_GT(max_levels_, summaries.size() - 1);
for (int i = 0; i < summaries.size() - 1; ++i) {
summary_levels_.push_back(summaries[i]);
}
local_summary_ = summaries[summaries.size() - 1];
}
private:
// Propagates local summary through summary levels while maintaining
// approximation error invariants.
void PropagateLocalSummary() {
// Validate state.
QCHECK(!finalized_) << "Finalize() already called.";
// No-op if there's nothing to add.
if (local_summary_.Size() <= 0) {
return;
}
// Propagate summary through levels.
size_t level = 0;
for (bool settled = false; !settled; ++level) {
// Ensure we have enough depth.
if (summary_levels_.size() <= level) {
summary_levels_.emplace_back();
}
// Merge summaries.
Summary& current_summary = summary_levels_[level];
local_summary_.Merge(current_summary);
// Check if we need to compress and propagate summary higher.
if (current_summary.Size() == 0 ||
local_summary_.Size() <= block_size_ + 1) {
current_summary = std::move(local_summary_);
settled = true;
} else {
// Compress, empty current level and propagate.
local_summary_.Compress(block_size_, eps_);
current_summary.Clear();
}
}
}
// Desired approximation precision.
double eps_;
// Maximum number of levels.
int64 max_levels_;
// Max block size per level.
int64 block_size_;
// Base buffer.
Buffer buffer_;
// Local summary used to minimize memory allocation and cache misses.
// After the stream is finalized, this summary holds the final quantile
// estimates.
Summary local_summary_;
// Summary levels;
std::vector<Summary> summary_levels_;
// Flag indicating whether the stream is finalized.
bool finalized_;
};
template <typename ValueType, typename WeightType, typename CompareFn>
inline std::tuple<int64, int64>
WeightedQuantilesStream<ValueType, WeightType, CompareFn>::GetQuantileSpecs(
double eps, int64 max_elements) {
int64 max_level = 1LL;
int64 block_size = 2LL;
QCHECK(eps >= 0 && eps < 1);
QCHECK_GT(max_elements, 0);
if (eps <= std::numeric_limits<double>::epsilon()) {
// Exact quantile computation at the expense of RAM.
max_level = 1;
block_size = std::max(max_elements, int64{2});
} else {
// The bottom-most level will become full at most
// (max_elements / block_size) times, the level above will become full
// (max_elements / 2 * block_size) times and generally level l becomes
// full (max_elements / 2^l * block_size) times until the last
// level max_level becomes full at most once meaning when the inequality
// (2^max_level * block_size >= max_elements) is satisfied.
// In what follows, we jointly solve for max_level and block_size by
// gradually increasing the level until the inequality above is satisfied.
// We could alternatively set max_level = ceil(log2(eps * max_elements));
// and block_size = ceil(max_level / eps) + 1 but that tends to give more
// pessimistic bounds and wastes RAM needlessly.
for (max_level = 1, block_size = 2;
(1LL << max_level) * block_size < max_elements; ++max_level) {
// Update upper bound on block size at current level, we always
// increase the estimate by 2 to hold the min/max elements seen so far.
block_size = static_cast<size_t>(ceil(max_level / eps)) + 1;
}
}
return std::make_tuple(max_level, std::max(block_size, int64{2}));
}
} // namespace quantiles
} // namespace boosted_trees
} // namespace tensorflow
#endif // TENSORFLOW_CORE_KERNELS_BOOSTED_TREES_QUANTILES_WEIGHTED_QUANTILES_STREAM_H_
@@ -0,0 +1,344 @@
// Copyright 2018 The TensorFlow Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// =============================================================================
#ifndef TENSORFLOW_CORE_KERNELS_BOOSTED_TREES_QUANTILES_WEIGHTED_QUANTILES_SUMMARY_H_
#define TENSORFLOW_CORE_KERNELS_BOOSTED_TREES_QUANTILES_WEIGHTED_QUANTILES_SUMMARY_H_
#include <cstring>
#include <vector>
#include "tensorflow/core/kernels/boosted_trees/quantiles/weighted_quantiles_buffer.h"
namespace tensorflow {
namespace boosted_trees {
namespace quantiles {
// Summary holding a sorted block of entries with upper bound guarantees
// over the approximation error.
template <typename ValueType, typename WeightType,
typename CompareFn = std::less<ValueType>>
class WeightedQuantilesSummary {
public:
using Buffer = WeightedQuantilesBuffer<ValueType, WeightType, CompareFn>;
using BufferEntry = typename Buffer::BufferEntry;
struct SummaryEntry {
SummaryEntry(const ValueType& v, const WeightType& w, const WeightType& min,
const WeightType& max) {
// Explicitly initialize all of memory (including padding from memory
// alignment) to allow the struct to be msan-resistant "plain old data".
//
// POD = http://en.cppreference.com/w/cpp/concept/PODType
memset(this, 0, sizeof(*this));
value = v;
weight = w;
min_rank = min;
max_rank = max;
}
SummaryEntry() {
memset(this, 0, sizeof(*this));
value = ValueType();
weight = 0;
min_rank = 0;
max_rank = 0;
}
bool operator==(const SummaryEntry& other) const {
return value == other.value && weight == other.weight &&
min_rank == other.min_rank && max_rank == other.max_rank;
}
friend std::ostream& operator<<(std::ostream& strm,
const SummaryEntry& entry) {
return strm << "{" << entry.value << ", " << entry.weight << ", "
<< entry.min_rank << ", " << entry.max_rank << "}";
}
// Max rank estimate for previous smaller value.
WeightType PrevMaxRank() const { return max_rank - weight; }
// Min rank estimate for next larger value.
WeightType NextMinRank() const { return min_rank + weight; }
ValueType value;
WeightType weight;
WeightType min_rank;
WeightType max_rank;
};
// Re-construct summary from the specified buffer.
void BuildFromBufferEntries(const std::vector<BufferEntry>& buffer_entries) {
entries_.clear();
entries_.reserve(buffer_entries.size());
WeightType cumulative_weight = 0;
for (const auto& entry : buffer_entries) {
WeightType current_weight = entry.weight;
entries_.emplace_back(entry.value, entry.weight, cumulative_weight,
cumulative_weight + current_weight);
cumulative_weight += current_weight;
}
}
// Re-construct summary from the specified summary entries.
void BuildFromSummaryEntries(
const std::vector<SummaryEntry>& summary_entries) {
entries_.clear();
entries_.reserve(summary_entries.size());
entries_.insert(entries_.begin(), summary_entries.begin(),
summary_entries.end());
}
// Merges two summaries through an algorithm that's derived from MergeSort
// for summary entries while guaranteeing that the max approximation error
// of the final merged summary is no greater than the approximation errors
// of each individual summary.
// For example consider summaries where each entry is of the form
// (element, weight, min rank, max rank):
// summary entries 1: (1, 3, 0, 3), (4, 2, 3, 5)
// summary entries 2: (3, 1, 0, 1), (4, 1, 1, 2)
// merged: (1, 3, 0, 3), (3, 1, 3, 4), (4, 3, 4, 7).
void Merge(const WeightedQuantilesSummary& other_summary) {
// Make sure we have something to merge.
const auto& other_entries = other_summary.entries_;
if (other_entries.empty()) {
return;
}
if (entries_.empty()) {
BuildFromSummaryEntries(other_summary.entries_);
return;
}
// Move current entries to make room for a new buffer.
std::vector<SummaryEntry> base_entries(std::move(entries_));
entries_.clear();
entries_.reserve(base_entries.size() + other_entries.size());
// Merge entries maintaining ranks. The idea is to stack values
// in order which we can do in linear time as the two summaries are
// already sorted. We keep track of the next lower rank from either
// summary and update it as we pop elements from the summaries.
// We handle the special case when the next two elements from either
// summary are equal, in which case we just merge the two elements
// and simultaneously update both ranks.
auto it1 = base_entries.cbegin();
auto it2 = other_entries.cbegin();
WeightType next_min_rank1 = 0;
WeightType next_min_rank2 = 0;
while (it1 != base_entries.cend() && it2 != other_entries.cend()) {
if (kCompFn(it1->value, it2->value)) { // value1 < value2
// Take value1 and use the last added value2 to compute
// the min rank and the current value2 to compute the max rank.
entries_.emplace_back(it1->value, it1->weight,
it1->min_rank + next_min_rank2,
it1->max_rank + it2->PrevMaxRank());
// Update next min rank 1.
next_min_rank1 = it1->NextMinRank();
++it1;
} else if (kCompFn(it2->value, it1->value)) { // value1 > value2
// Take value2 and use the last added value1 to compute
// the min rank and the current value1 to compute the max rank.
entries_.emplace_back(it2->value, it2->weight,
it2->min_rank + next_min_rank1,
it2->max_rank + it1->PrevMaxRank());
// Update next min rank 2.
next_min_rank2 = it2->NextMinRank();
++it2;
} else { // value1 == value2
// Straight additive merger of the two entries into one.
entries_.emplace_back(it1->value, it1->weight + it2->weight,
it1->min_rank + it2->min_rank,
it1->max_rank + it2->max_rank);
// Update next min ranks for both.
next_min_rank1 = it1->NextMinRank();
next_min_rank2 = it2->NextMinRank();
++it1;
++it2;
}
}
// Fill in any residual.
while (it1 != base_entries.cend()) {
entries_.emplace_back(it1->value, it1->weight,
it1->min_rank + next_min_rank2,
it1->max_rank + other_entries.back().max_rank);
++it1;
}
while (it2 != other_entries.cend()) {
entries_.emplace_back(it2->value, it2->weight,
it2->min_rank + next_min_rank1,
it2->max_rank + base_entries.back().max_rank);
++it2;
}
}
// Compresses buffer into desired size. The size specification is
// considered a hint as we always keep the first and last elements and
// maintain strict approximation error bounds.
// The approximation error delta is taken as the max of either the requested
// min error or 1 / size_hint.
// After compression, the approximation error is guaranteed to increase
// by no more than that error delta.
// This algorithm is linear in the original size of the summary and is
// designed to be cache-friendly.
void Compress(int64 size_hint, double min_eps = 0) {
// No-op if we're already within the size requirement.
size_hint = std::max(size_hint, int64{2});
if (entries_.size() <= size_hint) {
return;
}
// First compute the max error bound delta resulting from this compression.
double eps_delta = TotalWeight() * std::max(1.0 / size_hint, min_eps);
// Compress elements ensuring approximation bounds and elements diversity
// are both maintained.
int64 add_accumulator = 0, add_step = entries_.size();
auto write_it = entries_.begin() + 1, last_it = write_it;
for (auto read_it = entries_.begin(); read_it + 1 != entries_.end();) {
auto next_it = read_it + 1;
while (next_it != entries_.end() && add_accumulator < add_step &&
next_it->PrevMaxRank() - read_it->NextMinRank() <= eps_delta) {
add_accumulator += size_hint;
++next_it;
}
if (read_it == next_it - 1) {
++read_it;
} else {
read_it = next_it - 1;
}
(*write_it++) = (*read_it);
last_it = read_it;
add_accumulator -= add_step;
}
// Write last element and resize.
if (last_it + 1 != entries_.end()) {
(*write_it++) = entries_.back();
}
entries_.resize(write_it - entries_.begin());
}
// To construct the boundaries we first run a soft compress over a copy
// of the summary and retrieve the values.
// The resulting boundaries are guaranteed to both contain at least
// num_boundaries unique elements and maintain approximation bounds.
std::vector<ValueType> GenerateBoundaries(int64 num_boundaries) const {
std::vector<ValueType> output;
if (entries_.empty()) {
return output;
}
// Generate soft compressed summary.
WeightedQuantilesSummary<ValueType, WeightType, CompareFn>
compressed_summary;
compressed_summary.BuildFromSummaryEntries(entries_);
// Set an epsilon for compression that's at most 1.0 / num_boundaries
// more than epsilon of original our summary since the compression operation
// adds ~1.0/num_boundaries to final approximation error.
float compression_eps = ApproximationError() + (1.0 / num_boundaries);
compressed_summary.Compress(num_boundaries, compression_eps);
// Return boundaries.
output.reserve(compressed_summary.entries_.size());
for (const auto& entry : compressed_summary.entries_) {
output.push_back(entry.value);
}
return output;
}
// To construct the desired n-quantiles we repetitively query n ranks from the
// original summary. The following algorithm is an efficient cache-friendly
// O(n) implementation of that idea which avoids the cost of the repetitive
// full rank queries O(nlogn).
std::vector<ValueType> GenerateQuantiles(int64 num_quantiles) const {
std::vector<ValueType> output;
if (entries_.empty()) {
return output;
}
num_quantiles = std::max(num_quantiles, int64{2});
output.reserve(num_quantiles + 1);
// Make successive rank queries to get boundaries.
// We always keep the first (min) and last (max) entries.
for (size_t cur_idx = 0, rank = 0; rank <= num_quantiles; ++rank) {
// This step boils down to finding the next element sub-range defined by
// r = (rmax[i + 1] + rmin[i + 1]) / 2 where the desired rank d < r.
WeightType d_2 = 2 * (rank * entries_.back().max_rank / num_quantiles);
size_t next_idx = cur_idx + 1;
while (next_idx < entries_.size() &&
d_2 >= entries_[next_idx].min_rank + entries_[next_idx].max_rank) {
++next_idx;
}
cur_idx = next_idx - 1;
// Determine insertion order.
if (next_idx == entries_.size() ||
d_2 < entries_[cur_idx].NextMinRank() +
entries_[next_idx].PrevMaxRank()) {
output.push_back(entries_[cur_idx].value);
} else {
output.push_back(entries_[next_idx].value);
}
}
return output;
}
// Calculates current approximation error which should always be <= eps.
double ApproximationError() const {
if (entries_.empty()) {
return 0;
}
WeightType max_gap = 0;
for (auto it = entries_.cbegin() + 1; it < entries_.end(); ++it) {
max_gap = std::max(max_gap,
std::max(it->max_rank - it->min_rank - it->weight,
it->PrevMaxRank() - (it - 1)->NextMinRank()));
}
return static_cast<double>(max_gap) / TotalWeight();
}
ValueType MinValue() const {
return !entries_.empty() ? entries_.front().value
: std::numeric_limits<ValueType>::max();
}
ValueType MaxValue() const {
return !entries_.empty() ? entries_.back().value
: std::numeric_limits<ValueType>::max();
}
WeightType TotalWeight() const {
return !entries_.empty() ? entries_.back().max_rank : 0;
}
int64 Size() const { return entries_.size(); }
void Clear() { entries_.clear(); }
const std::vector<SummaryEntry>& GetEntryList() const { return entries_; }
private:
// Comparison function.
static constexpr decltype(CompareFn()) kCompFn = CompareFn();
// Summary entries.
std::vector<SummaryEntry> entries_;
};
template <typename ValueType, typename WeightType, typename CompareFn>
constexpr decltype(CompareFn())
WeightedQuantilesSummary<ValueType, WeightType, CompareFn>::kCompFn;
} // namespace quantiles
} // namespace boosted_trees
} // namespace tensorflow
#endif // TENSORFLOW_CORE_KERNELS_BOOSTED_TREES_QUANTILES_WEIGHTED_QUANTILES_SUMMARY_H_
@@ -0,0 +1,173 @@
/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
#ifndef TENSORFLOW_CORE_KERNELS_BOOSTED_TREES_RESOURCES_H_
#define TENSORFLOW_CORE_KERNELS_BOOSTED_TREES_RESOURCES_H_
#include "tensorflow/core/framework/resource_mgr.h"
#include "tensorflow/core/platform/mutex.h"
#include "tensorflow/core/platform/protobuf.h"
namespace tensorflow {
// Forward declaration for proto class TreeEnsemble
namespace boosted_trees {
class TreeEnsemble;
} // namespace boosted_trees
// A StampedResource is a resource that has a stamp token associated with it.
// Before reading from or applying updates to the resource, the stamp should
// be checked to verify that the update is not stale.
class StampedResource : public ResourceBase {
public:
StampedResource() : stamp_(-1) {}
bool is_stamp_valid(int64 stamp) const { return stamp_ == stamp; }
int64 stamp() const { return stamp_; }
void set_stamp(int64 stamp) { stamp_ = stamp; }
private:
int64 stamp_;
};
// Keep a tree ensemble in memory for efficient evaluation and mutation.
class BoostedTreesEnsembleResource : public StampedResource {
public:
BoostedTreesEnsembleResource();
string DebugString() override;
bool InitFromSerialized(const string& serialized, const int64 stamp_token);
string SerializeAsString() const;
int32 num_trees() const;
// Find the next node to which the example (specified by index_in_batch)
// traverses down from the current node indicated by tree_id and node_id.
// Args:
// tree_id: the index of the tree in the ensemble.
// node_id: the index of the node within the tree.
// index_in_batch: the index of the example within the batch (relevant to
// the index of the row to read in each bucketized_features).
// bucketized_features: vector of feature Vectors.
int32 next_node(
const int32 tree_id, const int32 node_id, const int32 index_in_batch,
const std::vector<TTypes<int32>::ConstVec>& bucketized_features) const;
float node_value(const int32 tree_id, const int32 node_id) const;
void set_node_value(const int32 tree_id, const int32 node_id,
const float logits);
int32 GetNumLayersGrown(const int32 tree_id) const;
void SetNumLayersGrown(const int32 tree_id, int32 new_num_layers) const;
void UpdateLastLayerNodesRange(const int32 node_range_start,
int32 node_range_end) const;
void GetLastLayerNodesRange(int32* node_range_start,
int32* node_range_end) const;
int64 GetNumNodes(const int32 tree_id);
void UpdateGrowingMetadata() const;
int32 GetNumLayersAttempted();
bool is_leaf(const int32 tree_id, const int32 node_id) const;
int32 feature_id(const int32 tree_id, const int32 node_id) const;
int32 bucket_threshold(const int32 tree_id, const int32 node_id) const;
int32 left_id(const int32 tree_id, const int32 node_id) const;
int32 right_id(const int32 tree_id, const int32 node_id) const;
// Add a tree to the ensemble and returns a new tree_id.
int32 AddNewTree(const float weight);
// Adds new tree with one node to the ensemble and sets node's value to logits
int32 AddNewTreeWithLogits(const float weight, const float logits);
// Grows the tree by adding a split and leaves.
void AddBucketizedSplitNode(const int32 tree_id, const int32 node_id,
const int32 feature_id, const int32 threshold,
const float gain, const float left_contrib,
const float right_contrib, int32* left_node_id,
int32* right_node_id);
// Retrieves tree weights and returns as a vector.
// It involves a copy, so should be called only sparingly (like once per
// iteration, not per example).
std::vector<float> GetTreeWeights() const;
float GetTreeWeight(const int32 tree_id) const;
float IsTreeFinalized(const int32 tree_id) const;
float IsTreePostPruned(const int32 tree_id) const;
void SetIsFinalized(const int32 tree_id, const bool is_finalized);
// Sets the weight of i'th tree.
void SetTreeWeight(const int32 tree_id, const float weight);
// Resets the resource and frees the protos in arena.
// Caller needs to hold the mutex lock while calling this.
virtual void Reset();
void PostPruneTree(const int32 current_tree);
// For a given node, returns the id in a pruned tree, as well as correction
// to the cached prediction that should be applied. If tree was not
// post-pruned, current_node_id will be equal to initial_node_id and logit
// update will be equal to zero.
void GetPostPruneCorrection(const int32 tree_id, const int32 initial_node_id,
int32* current_node_id,
float* logit_update) const;
mutex* get_mutex() { return &mu_; }
private:
// Helper method to check whether a node is a terminal node in that it
// only has leaf nodes as children.
bool IsTerminalSplitNode(const int32 tree_id, const int32 node_id) const;
// For each pruned node, finds the leaf where it finally ended up and
// calculates the total update from that pruned node prediction.
void CalculateParentAndLogitUpdate(
const int32 start_node_id,
const std::vector<std::pair<int32, float>>& nodes_change,
int32* parent_id, float* change) const;
// Helper method to collect the information to be used to prune some nodes in
// the tree.
void RecursivelyDoPostPrunePreparation(
const int32 tree_id, const int32 node_id,
std::vector<int32>* nodes_to_delete,
std::vector<std::pair<int32, float>>* nodes_meta);
protected:
protobuf::Arena arena_;
mutex mu_;
boosted_trees::TreeEnsemble* tree_ensemble_;
};
} // namespace tensorflow
#endif // TENSORFLOW_CORE_KERNELS_BOOSTED_TREES_RESOURCES_H_
@@ -0,0 +1,69 @@
/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
#ifndef TENSORFLOW_CORE_KERNELS_BOOSTED_TREES_TREE_HELPER_H_
#define TENSORFLOW_CORE_KERNELS_BOOSTED_TREES_TREE_HELPER_H_
#include <cmath>
namespace tensorflow {
static bool GainsAreEqual(const float g1, const float g2) {
const float kTolerance = 1e-15;
return std::abs(g1 - g2) < kTolerance;
}
static bool GainIsLarger(const float g1, const float g2) {
const float kTolerance = 1e-15;
return g1 - g2 >= kTolerance;
}
static void CalculateWeightsAndGains(const float g, const float h,
const float l1, const float l2,
float* weight, float* gain) {
const float kEps = 1e-15;
// The formula for weight is -(g+l1*sgn(w))/(H+l2), for gain it is
// (g+l1*sgn(w))^2/(h+l2).
// This is because for each leaf we optimize
// 1/2(h+l2)*w^2+g*w+l1*abs(w)
float g_with_l1 = g;
// Apply L1 regularization.
// 1) Assume w>0 => w=-(g+l1)/(h+l2)=> g+l1 < 0 => g < -l1
// 2) Assume w<0 => w=-(g-l1)/(h+l2)=> g-l1 > 0 => g > l1
// For g from (-l1, l1), thus there is no solution => set to 0.
if (l1 > 0) {
if (g > l1) {
g_with_l1 -= l1;
} else if (g < -l1) {
g_with_l1 += l1;
} else {
*weight = 0.0;
*gain = 0.0;
return;
}
}
// Apply L2 regularization.
if (h + l2 <= kEps) {
// Avoid division by 0 or infinitesimal.
*weight = 0;
*gain = 0;
} else {
*weight = -g_with_l1 / (h + l2);
*gain = -g_with_l1 * (*weight);
}
}
} // namespace tensorflow
#endif // TENSORFLOW_CORE_KERNELS_BOOSTED_TREES_TREE_HELPER_H_
@@ -0,0 +1,54 @@
/* Copyright 2015 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
#ifndef TENSORFLOW_CORE_KERNELS_BOUNDS_CHECK_H_
#define TENSORFLOW_CORE_KERNELS_BOUNDS_CHECK_H_
#include <type_traits>
#include "third_party/eigen3/Eigen/Core"
#include "tensorflow/core/platform/macros.h"
namespace tensorflow {
// Check that 0 <= index < limit using a single comparison, assuming
// that 0 <= limit if Index is signed. Intended for use in performance
// critical contexts where 0 <= index < limit is almost always true.
template <typename Ta, typename Tb>
EIGEN_ALWAYS_INLINE EIGEN_DEVICE_FUNC bool FastBoundsCheck(const Ta index,
const Tb limit) {
static_assert(std::is_integral<Ta>::value && std::is_integral<Tb>::value,
"FastBoundsCheck can only be used on integer types.");
typedef typename std::make_unsigned<decltype(index + limit)>::type UIndex;
return TF_PREDICT_TRUE(static_cast<UIndex>(index) <
static_cast<UIndex>(limit));
}
namespace internal {
// Ensure that the compiler cannot elide a copy into a local, for
// bounds checking on source tensors that might be updated asynchronously.
// This function may only be used on primitive integral types (int32, int64,
// etc). It does not guarantee any atomicity or barriers.
template <typename T>
EIGEN_ALWAYS_INLINE EIGEN_DEVICE_FUNC const T SubtleMustCopy(const T &x) {
static_assert(std::is_integral<T>::value,
"SubtleMustCopy can only be used on integer types.");
auto *to_x = reinterpret_cast<const volatile T *>(&x);
return *to_x;
}
} // namespace internal
} // namespace tensorflow
#endif // TENSORFLOW_CORE_KERNELS_BOUNDS_CHECK_H_
@@ -0,0 +1,242 @@
/* Copyright 2015 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
#ifndef TENSORFLOW_CORE_KERNELS_BROADCAST_TO_OP_H_
#define TENSORFLOW_CORE_KERNELS_BROADCAST_TO_OP_H_
#include "tensorflow/core/framework/op_kernel.h"
#include "tensorflow/core/framework/tensor.h"
#include "tensorflow/core/framework/tensor_shape.h"
#include "tensorflow/core/framework/tensor_types.h"
#include "tensorflow/core/framework/types.h"
#include "third_party/eigen3/unsupported/Eigen/CXX11/Tensor"
namespace tensorflow {
namespace functor {
template <typename Device, typename T>
struct BroadcastTo {
void operator()(const Device &d, OpKernelContext *ctx, Tensor &output_tensor,
const TensorShape &output_shape, const Tensor &input_tensor,
const TensorShape &input_shape) {
#define BROADCAST_SHAPE(broadcast, reshape, NDIMS, input_shape, output_shape) \
for (int i = 0; i < NDIMS; i++) { \
if (reshape[i] != broadcast[i]) { \
OP_REQUIRES(ctx, \
((reshape[i] != 0) && (broadcast[i] % reshape[i] == 0)), \
errors::InvalidArgument("invalid shape to broadcast from ", \
input_shape.DebugString(), " to ", \
output_shape.DebugString())); \
broadcast[i] = broadcast[i] / reshape[i]; \
} else { \
broadcast[i] = 1; \
} \
}
if (output_shape.num_elements() == 0) {
return;
}
if (output_shape == input_shape) {
output_tensor.flat<T>().device(d) = input_tensor.flat<T>();
return;
}
switch (output_shape.dims()) {
case 0: {
if (input_shape.dims() > 0) {
ctx->CtxFailure(errors::InvalidArgument(
"invalid shape to broadcast from ", input_shape.DebugString(),
" to ", output_shape.DebugString()));
break;
}
output_tensor.scalar<T>().device(d) = input_tensor.scalar<T>();
break;
}
case 1: {
auto reshape = AsEigenDSizesWithPrefix<1>(input_shape);
auto broadcast = output_shape.AsEigenDSizes<1>();
BROADCAST_SHAPE(broadcast, reshape, 1, input_shape, output_shape);
auto output = output_tensor.tensor<T, 1>();
switch (input_shape.dims()) {
case 0: {
output.device(d) = output.constant(input_tensor.scalar<T>()());
} break;
case 1: {
auto input = input_tensor.tensor<T, 1>();
output.device(d) = input.broadcast(broadcast);
} break;
default:
ctx->CtxFailure(errors::InvalidArgument(
"invalid shape to broadcast from ", input_shape.DebugString(),
" to ", output_shape.DebugString()));
break;
}
} break;
case 2: {
auto reshape = AsEigenDSizesWithPrefix<2>(input_shape);
auto broadcast = output_shape.AsEigenDSizes<2>();
BROADCAST_SHAPE(broadcast, reshape, 2, input_shape, output_shape);
auto output = output_tensor.tensor<T, 2>();
switch (input_shape.dims()) {
case 0: {
output.device(d) = output.constant(input_tensor.scalar<T>()());
} break;
case 1: {
auto input = input_tensor.tensor<T, 1>();
output.device(d) = input.reshape(reshape).broadcast(broadcast);
} break;
case 2: {
auto input = input_tensor.tensor<T, 2>();
output.device(d) = input.broadcast(broadcast);
} break;
default:
ctx->CtxFailure(errors::InvalidArgument(
"invalid shape to broadcast from ", input_shape.DebugString(),
" to ", output_shape.DebugString()));
break;
}
} break;
case 3: {
auto reshape = AsEigenDSizesWithPrefix<3>(input_shape);
auto broadcast = output_shape.AsEigenDSizes<3>();
BROADCAST_SHAPE(broadcast, reshape, 3, input_shape, output_shape);
auto output = output_tensor.tensor<T, 3>();
switch (input_shape.dims()) {
case 0: {
output.device(d) = output.constant(input_tensor.scalar<T>()());
} break;
case 1: {
auto input = input_tensor.tensor<T, 1>();
output.device(d) = input.reshape(reshape).broadcast(broadcast);
} break;
case 2: {
auto input = input_tensor.tensor<T, 2>();
output.device(d) = input.reshape(reshape).broadcast(broadcast);
} break;
case 3: {
auto input = input_tensor.tensor<T, 3>();
output.device(d) = input.broadcast(broadcast);
} break;
default:
ctx->CtxFailure(errors::InvalidArgument(
"invalid shape to broadcast from ", input_shape.DebugString(),
" to ", output_shape.DebugString()));
break;
}
} break;
case 4: {
auto reshape = AsEigenDSizesWithPrefix<4>(input_shape);
auto broadcast = output_shape.AsEigenDSizes<4>();
BROADCAST_SHAPE(broadcast, reshape, 4, input_shape, output_shape);
auto output = output_tensor.tensor<T, 4>();
switch (input_shape.dims()) {
case 0: {
output.device(d) = output.constant(input_tensor.scalar<T>()());
} break;
case 1: {
auto input = input_tensor.tensor<T, 1>();
output.device(d) = input.reshape(reshape).broadcast(broadcast);
} break;
case 2: {
auto input = input_tensor.tensor<T, 2>();
output.device(d) = input.reshape(reshape).broadcast(broadcast);
} break;
case 3: {
auto input = input_tensor.tensor<T, 3>();
output.device(d) = input.reshape(reshape).broadcast(broadcast);
} break;
case 4: {
auto input = input_tensor.tensor<T, 4>();
output.device(d) = input.broadcast(broadcast);
} break;
default:
ctx->CtxFailure(errors::InvalidArgument(
"invalid shape to broadcast from ", input_shape.DebugString(),
" to ", output_shape.DebugString()));
break;
}
} break;
case 5: {
auto reshape = AsEigenDSizesWithPrefix<5>(input_shape);
auto broadcast = output_shape.AsEigenDSizes<5>();
BROADCAST_SHAPE(broadcast, reshape, 5, input_shape, output_shape);
auto output = output_tensor.tensor<T, 5>();
switch (input_shape.dims()) {
case 0: {
output.device(d) = output.constant(input_tensor.scalar<T>()());
} break;
case 1: {
auto input = input_tensor.tensor<T, 1>();
output.device(d) = input.reshape(reshape).broadcast(broadcast);
} break;
case 2: {
auto input = input_tensor.tensor<T, 2>();
output.device(d) = input.reshape(reshape).broadcast(broadcast);
} break;
case 3: {
auto input = input_tensor.tensor<T, 3>();
output.device(d) = input.reshape(reshape).broadcast(broadcast);
} break;
case 4: {
auto input = input_tensor.tensor<T, 4>();
output.device(d) = input.reshape(reshape).broadcast(broadcast);
} break;
case 5: {
auto input = input_tensor.tensor<T, 5>();
output.device(d) = input.broadcast(broadcast);
} break;
default:
ctx->CtxFailure(errors::InvalidArgument(
"invalid shape to broadcast from ", input_shape.DebugString(),
" to ", output_shape.DebugString()));
break;
}
} break;
default:
ctx->CtxFailure(errors::InvalidArgument(
"invalid shape to broadcast from ", input_shape.DebugString(),
" to ", output_shape.DebugString()));
break;
}
}
private:
template <int NDIMS>
Eigen::DSizes<Eigen::DenseIndex, NDIMS> AsEigenDSizesWithPrefix(
const TensorShape &shape) const {
Eigen::DSizes<Eigen::DenseIndex, NDIMS> dsizes;
for (int d = 0; d < NDIMS - shape.dims(); d++) {
dsizes[d] = 1;
}
for (int d = NDIMS - shape.dims(); d < NDIMS; d++) {
dsizes[d] = shape.dim_size(d - (NDIMS - shape.dims()));
}
return dsizes;
}
};
} // namespace functor
} // namespace tensorflow
#endif // TENSORFLOW_CORE_KERNELS_BROADCAST_TO_OP_H_
@@ -0,0 +1,41 @@
/* Copyright 2015 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
#ifndef TENSORFLOW_CORE_KERNELS_BUCKETIZE_OP_H_
#define TENSORFLOW_CORE_KERNELS_BUCKETIZE_OP_H_
#include <vector>
#include "third_party/eigen3/unsupported/Eigen/CXX11/Tensor"
#include "tensorflow/core/framework/op_kernel.h"
#include "tensorflow/core/framework/tensor_types.h"
#include "tensorflow/core/framework/types.h"
#include "tensorflow/core/lib/core/errors.h"
namespace tensorflow {
namespace functor {
template <typename Device, typename T>
struct BucketizeFunctor {
static Status Compute(OpKernelContext* context,
const typename TTypes<T, 1>::ConstTensor& input,
const std::vector<float>& boundaries_vector,
typename TTypes<int32, 1>::Tensor& output);
};
} // namespace functor
} // namespace tensorflow
#endif // TENSORFLOW_CORE_KERNELS_BUCKETIZE_OP_H_
@@ -0,0 +1,326 @@
/* Copyright 2015 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
#ifndef TENSORFLOW_CORE_KERNELS_CAST_OP_H_
#define TENSORFLOW_CORE_KERNELS_CAST_OP_H_
#include "third_party/eigen3/unsupported/Eigen/CXX11/Tensor"
#include "tensorflow/core/framework/bfloat16.h"
#include "tensorflow/core/framework/op_kernel.h"
#include "tensorflow/core/framework/tensor_types.h"
#include "tensorflow/core/framework/types.h"
#include "tensorflow/core/platform/byte_order.h"
#include "tensorflow/core/platform/types.h"
// Note that the GPU cast functor templates need to be instantiated unlike the
// CPU ones, and hence their specializations are different than that for CPUs.
#ifdef SPECIALIZE_FOR_GPUS
#define SPECIALIZE_CAST(DEVICE, OUT_TYPE, IN_OUT) \
template <typename Device> \
struct CastFunctor<Device, OUT_TYPE, IN_OUT> { \
void operator()(const Device& d, \
typename TTypes<OUT_TYPE>::Flat out_tensor, \
typename TTypes<IN_OUT>::ConstFlat in_tensor, \
bool truncate = false) { \
if (truncate) { \
out_tensor.device(d) = \
in_tensor.unaryExpr(LSBZeroSetter<IN_OUT, OUT_TYPE>()) \
.template cast<OUT_TYPE>(); \
} else { \
out_tensor.device(d) = in_tensor.template cast<OUT_TYPE>(); \
} \
} \
}; \
template struct CastFunctor<DEVICE, OUT_TYPE, IN_OUT>;
#else
#define SPECIALIZE_CAST(DEVICE, OUT_TYPE, IN_OUT) \
template <> \
struct CastFunctor<DEVICE, OUT_TYPE, IN_OUT> { \
void operator()(const DEVICE& d, \
typename TTypes<OUT_TYPE>::Flat out_tensor, \
typename TTypes<IN_OUT>::ConstFlat in_tensor, \
bool truncate = false) { \
if (truncate) { \
out_tensor.device(d) = \
in_tensor.unaryExpr(LSBZeroSetter<IN_OUT, OUT_TYPE>()) \
.template cast<OUT_TYPE>(); \
} else { \
out_tensor.device(d) = in_tensor.template cast<OUT_TYPE>(); \
} \
} \
};
#endif
#define CAST_FUNCTORS(devname) \
SPECIALIZE_CAST(devname, float, double) \
SPECIALIZE_CAST(devname, float, std::complex<double>) \
SPECIALIZE_CAST(devname, std::complex<float>, std::complex<double>) \
SPECIALIZE_CAST(devname, std::complex<float>, double) \
SPECIALIZE_CAST(devname, Eigen::half, double) \
SPECIALIZE_CAST(devname, Eigen::half, float) \
SPECIALIZE_CAST(devname, Eigen::half, std::complex<double>) \
SPECIALIZE_CAST(devname, Eigen::half, std::complex<float>) \
SPECIALIZE_CAST(devname, bfloat16, float) \
template <typename OUT_TYPE, typename IN_OUT> \
struct CastFunctor<devname, OUT_TYPE, IN_OUT> { \
void operator()(const devname& d, \
typename TTypes<OUT_TYPE>::Flat out_tensor, \
typename TTypes<IN_OUT>::ConstFlat in_tensor, \
bool truncate = false) { \
out_tensor.device(d) = in_tensor.template cast<OUT_TYPE>(); \
} \
};
namespace tensorflow {
typedef std::function<void(OpKernelContext*, const Tensor&, Tensor*,
bool trunc)>
CastFunctorType;
// Common base class of Cast kernels
class CastOpBase : public OpKernel {
public:
explicit CastOpBase(OpKernelConstruction* ctx);
void Compute(OpKernelContext* ctx) override;
protected:
DataType src_dtype_;
DataType dst_dtype_;
DataType external_src_dtype_;
DataType external_dst_dtype_;
bool use_truncation_;
CastFunctorType work_ = nullptr;
Status Unimplemented();
TF_DISALLOW_COPY_AND_ASSIGN(CastOpBase);
};
// CPU implementation of Cast
class CpuCastOp : public CastOpBase {
public:
explicit CpuCastOp(OpKernelConstruction* ctx);
private:
Status Prepare();
};
namespace functor {
template <typename I>
constexpr int MantissaWidth() {
return std::numeric_limits<I>::digits;
}
template <>
constexpr int MantissaWidth<Eigen::half>() {
// Remember, there's 1 hidden bit
return 10 + 1;
}
template <>
constexpr int MantissaWidth<bfloat16>() {
// Remember, there's 1 hidden bit
return 7 + 1;
}
template <typename Device, typename Tout, typename Tin>
void Cast(const Device& d, typename TTypes<Tout>::Flat o,
typename TTypes<Tin>::ConstFlat i) {
o.device(d) = i.template cast<Tout>();
}
template <typename Device, typename Tout, typename Tin>
struct CastFunctor {
void operator()(const Device& d, typename TTypes<Tout>::Flat o,
typename TTypes<Tin>::ConstFlat i, bool truncate = false);
};
// Only enable LSBZeroSetterHelper for 64 and 32 bit input data types.
// Specialize for others if needed in future.
template <typename I>
typename std::enable_if<sizeof(I) == 8, void>::type EIGEN_DEVICE_FUNC
EIGEN_STRONG_INLINE static LSBZeroSetterHelper(I& t, int n) {
// Only zero the bits for non-NaNs.
// For NaNs, let the non-truncation version handle it.
if (!std::isnan(t)) {
uint64_t* p = reinterpret_cast<uint64_t*>(&t);
*p &= (0xFFFFFFFFFFFFFFFF << n);
}
}
template <typename I>
typename std::enable_if<sizeof(I) == 4, void>::type EIGEN_DEVICE_FUNC
EIGEN_STRONG_INLINE static LSBZeroSetterHelper(I& t, int n) {
// Only zero the bits for non-NaNs.
// For NaNs, let the non-truncation version handle it.
if (!std::isnan(t)) {
uint32_t* p = reinterpret_cast<uint32_t*>(&t);
*p &= (0xFFFFFFFF << n);
}
}
// Set n least significant bits to 0
template <typename I, typename O>
struct LSBZeroSetter {
EIGEN_EMPTY_STRUCT_CTOR(LSBZeroSetter)
EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const I operator()(const I& a) const {
constexpr int bits = MantissaWidth<I>() - MantissaWidth<O>();
static_assert(
bits > 0,
"The output type must have fewer mantissa bits than the input type\n");
I t = a;
LSBZeroSetterHelper(t, bits);
return t;
}
};
template <typename I, typename O>
struct LSBZeroSetter<std::complex<I>, std::complex<O>> {
EIGEN_EMPTY_STRUCT_CTOR(LSBZeroSetter)
EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const std::complex<I> operator()(
const std::complex<I>& a) const {
constexpr int bits = MantissaWidth<I>() - MantissaWidth<O>();
static_assert(
bits > 0,
"The output type must have fewer mantissa bits than the input type\n");
I re = std::real(a);
I img = std::imag(a);
LSBZeroSetterHelper(re, bits);
LSBZeroSetterHelper(img, bits);
std::complex<I> toReturn(re, img);
return toReturn;
}
};
template <typename I, typename O>
struct LSBZeroSetter<std::complex<I>, O> {
EIGEN_EMPTY_STRUCT_CTOR(LSBZeroSetter)
// Sets the 16 LSBits of the float to 0
EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const std::complex<I> operator()(
const std::complex<I>& a) const {
constexpr int bits = MantissaWidth<I>() - MantissaWidth<O>();
static_assert(
bits > 0,
"The output type must have fewer mantissa bits than the input type\n");
I re = std::real(a);
I img = std::imag(a);
LSBZeroSetterHelper(re, bits);
LSBZeroSetterHelper(img, bits);
std::complex<I> toReturn(re, img);
return toReturn;
}
};
} // end namespace functor
} // end namespace tensorflow
namespace Eigen {
namespace internal {
// Eigen can't convert to/from complex numbers, because it is limited to cases
// that can be static_casted. But numpy is able to cast to/from complex, which
// we want to replicate. So we add specializations for complex here.
template <typename From, typename To>
struct scalar_cast_op<std::complex<From>, To> {
EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE To
operator()(const std::complex<From>& a) const {
// Replicate numpy behavior of returning just the real part
return static_cast<To>(a.real());
}
};
template <typename From, typename To>
struct scalar_cast_op<From, std::complex<To>> {
EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE std::complex<To> operator()(
const From& a) const {
// Replicate numpy behavior of setting the imaginary part to 0
return std::complex<To>(static_cast<To>(a), To(0));
}
};
template <typename From, typename To>
struct scalar_cast_op<std::complex<From>, std::complex<To>> {
EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE std::complex<To> operator()(
const std::complex<From>& a) const {
return std::complex<To>(static_cast<To>(a.real()),
static_cast<To>(a.imag()));
}
};
template <typename From, typename To>
struct functor_traits_complex_impl {
enum { Cost = NumTraits<To>::AddCost, PacketAccess = false };
};
template <typename From, typename To>
struct functor_traits<scalar_cast_op<std::complex<From>, To>>
: functor_traits_complex_impl<std::complex<From>, To> {};
template <typename From, typename To>
struct functor_traits<scalar_cast_op<From, std::complex<To>>>
: functor_traits_complex_impl<From, std::complex<To>> {};
// Needed to avoid ambiguous partial specialization
template <typename From, typename To>
struct functor_traits<scalar_cast_op<std::complex<From>, std::complex<To>>>
: functor_traits_complex_impl<std::complex<From>, std::complex<To>> {};
// Specialized cast op impls for bfloat16.
template <>
struct scalar_cast_op<::tensorflow::bfloat16, float> {
EIGEN_EMPTY_STRUCT_CTOR(scalar_cast_op)
typedef float result_type;
EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE float operator()(
const ::tensorflow::bfloat16& a) const {
float ret;
uint16_t* p = reinterpret_cast<uint16_t*>(&ret);
#if __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__
p[0] = a.value;
p[1] = 0;
#else
static_assert(::tensorflow::port::kLittleEndian,
"Not a little endian system!");
p[0] = 0;
p[1] = a.value;
#endif
return ret;
}
};
template <>
struct functor_traits<scalar_cast_op<::tensorflow::bfloat16, float>> {
enum { Cost = NumTraits<float>::AddCost, PacketAccess = false };
};
template <>
struct scalar_cast_op<float, ::tensorflow::bfloat16> {
EIGEN_EMPTY_STRUCT_CTOR(scalar_cast_op)
typedef ::tensorflow::bfloat16 result_type;
EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const ::tensorflow::bfloat16 operator()(
const float a) const {
return ::tensorflow::bfloat16(a);
}
};
template <>
struct functor_traits<scalar_cast_op<float, ::tensorflow::bfloat16>> {
enum { Cost = NumTraits<float>::AddCost, PacketAccess = false };
};
} // namespace internal
} // namespace Eigen
#endif // TENSORFLOW_CORE_KERNELS_CAST_OP_H_
@@ -0,0 +1,160 @@
/* Copyright 2016 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
#ifndef TENSORFLOW_CORE_KERNELS_CAST_OP_IMPL_H_
#define TENSORFLOW_CORE_KERNELS_CAST_OP_IMPL_H_
#define EIGEN_USE_THREADS
#include "tensorflow/core/framework/op_kernel.h"
#include "tensorflow/core/kernels/cast_op.h"
namespace tensorflow {
namespace functor {
CAST_FUNCTORS(Eigen::ThreadPoolDevice);
#ifdef TENSORFLOW_USE_SYCL
CAST_FUNCTORS(Eigen::SyclDevice);
#endif // TENSORFLOW_USE_SYCL
} // namespace functor
#define CURRY_TYPES3_NO_HALF(FN, arg0, arg1) \
FN(arg0, arg1, bool); \
FN(arg0, arg1, uint8); \
FN(arg0, arg1, uint16); \
FN(arg0, arg1, uint32); \
FN(arg0, arg1, uint64); \
FN(arg0, arg1, int8); \
FN(arg0, arg1, int16); \
FN(arg0, arg1, int32); \
FN(arg0, arg1, int64); \
FN(arg0, arg1, float); \
FN(arg0, arg1, double); \
FN(arg0, arg1, std::complex<float>); \
FN(arg0, arg1, std::complex<double>)
#define CURRY_TYPES3_NO_BF16(FN, arg0, arg1) \
CURRY_TYPES3_NO_HALF(FN, arg0, arg1) \
FN(arg0, arg1, Eigen::half);
#define CURRY_TYPES3(FN, arg0, arg1) \
CURRY_TYPES3_NO_BF16(FN, arg0, arg1) \
FN(arg0, arg1, bfloat16);
#define CAST_CASE(DEVICE, IN, OUT) \
if (DataTypeToEnum<OUT>::value == dst_dtype) { \
return [](OpKernelContext* ctx, const Tensor& inp, Tensor* out, \
bool truncate) { \
functor::CastFunctor<DEVICE, OUT, IN> func; \
func(ctx->eigen_device<DEVICE>(), out->flat<OUT>(), inp.flat<IN>(), \
truncate); \
}; \
}
// The functions below are implemented in the cast_op_impl_*.cc files.
CastFunctorType GetCpuCastFromBool(DataType dst_dtype);
CastFunctorType GetCpuCastFromUint8(DataType dst_dtype);
CastFunctorType GetCpuCastFromUint16(DataType dst_dtype);
CastFunctorType GetCpuCastFromInt8(DataType dst_dtype);
CastFunctorType GetCpuCastFromUint32(DataType dst_dtype);
CastFunctorType GetCpuCastFromUint64(DataType dst_dtype);
CastFunctorType GetCpuCastFromInt8(DataType dst_dtype);
CastFunctorType GetCpuCastFromInt16(DataType dst_dtype);
CastFunctorType GetCpuCastFromInt32(DataType dst_dtype);
CastFunctorType GetCpuCastFromInt64(DataType dst_dtype);
CastFunctorType GetCpuCastFromHalf(DataType dst_dtype);
CastFunctorType GetCpuCastFromFloat(DataType dst_dtype);
CastFunctorType GetCpuCastFromDouble(DataType dst_dtype);
CastFunctorType GetCpuCastFromComplex64(DataType dst_dtype);
CastFunctorType GetCpuCastFromComplex128(DataType dst_dtype);
CastFunctorType GetCpuCastFromBfloat(DataType dst_dtype);
#if GOOGLE_CUDA
// Same, for GPU.
CastFunctorType GetGpuCastFromBool(DataType dst_dtype);
CastFunctorType GetGpuCastFromUint8(DataType dst_dtype);
CastFunctorType GetGpuCastFromUint16(DataType dst_dtype);
CastFunctorType GetGpuCastFromInt8(DataType dst_dtype);
CastFunctorType GetGpuCastFromUint32(DataType dst_dtype);
CastFunctorType GetGpuCastFromUint64(DataType dst_dtype);
CastFunctorType GetGpuCastFromInt16(DataType dst_dtype);
CastFunctorType GetGpuCastFromInt32(DataType dst_dtype);
CastFunctorType GetGpuCastFromInt64(DataType dst_dtype);
CastFunctorType GetGpuCastFromHalf(DataType dst_dtype);
CastFunctorType GetGpuCastFromFloat(DataType dst_dtype);
CastFunctorType GetGpuCastFromDouble(DataType dst_dtype);
CastFunctorType GetGpuCastFromComplex64(DataType dst_dtype);
CastFunctorType GetGpuCastFromComplex128(DataType dst_dtype);
CastFunctorType GetGpuCastFromBfloat(DataType dst_dtype);
#endif // GOOGLE_CUDA
#ifdef TENSORFLOW_USE_SYCL
CastFunctorType GetSyclCastFromBool(DataType dst_dtype);
CastFunctorType GetSyclCastFromUint8(DataType dst_dtype);
CastFunctorType GetSyclCastFromUint16(DataType dst_dtype);
CastFunctorType GetSyclCastFromUint32(DataType dst_dtype);
CastFunctorType GetSyclCastFromUint64(DataType dst_dtype);
CastFunctorType GetSyclCastFromInt16(DataType dst_dtype);
CastFunctorType GetSyclCastFromInt32(DataType dst_dtype);
CastFunctorType GetSyclCastFromInt64(DataType dst_dtype);
CastFunctorType GetSyclCastFromFloat(DataType dst_dtype);
CastFunctorType GetSyclCastFromDouble(DataType dst_dtype);
#endif // TENSORFLOW_USE_SYCL
} // namespace tensorflow
#endif // TENSORFLOW_CORE_KERNELS_CAST_OP_IMPL_H_
@@ -0,0 +1,94 @@
/* Copyright 2015 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
#ifndef TENSORFLOW_CORE_KERNELS_COLORSPACE_OP_H_
#define TENSORFLOW_CORE_KERNELS_COLORSPACE_OP_H_
#include "third_party/eigen3/unsupported/Eigen/CXX11/Tensor"
#include "tensorflow/core/framework/tensor_shape.h"
#include "tensorflow/core/framework/tensor_types.h"
namespace tensorflow {
namespace functor {
template <typename Device, typename T>
struct RGBToHSV {
void operator()(const Device &d,
typename TTypes<T, 2>::ConstTensor input_data,
typename TTypes<T, 1>::Tensor range,
typename TTypes<T, 2>::Tensor output_data) {
auto H = output_data.template chip<1>(0);
auto S = output_data.template chip<1>(1);
auto V = output_data.template chip<1>(2);
auto R = input_data.template chip<1>(0);
auto G = input_data.template chip<1>(1);
auto B = input_data.template chip<1>(2);
#if !defined(EIGEN_HAS_INDEX_LIST)
Eigen::array<int, 1> channel_axis{{1}};
#else
Eigen::IndexList<Eigen::type2index<1> > channel_axis;
#endif
V.device(d) = input_data.maximum(channel_axis);
range.device(d) = V - input_data.minimum(channel_axis);
S.device(d) = (V > T(0)).select(range / V, V.constant(T(0)));
auto norm = range.inverse() * (T(1) / T(6));
// TODO(wicke): all these assignments are only necessary because a combined
// expression is larger than kernel parameter space. A custom kernel is
// probably in order.
H.device(d) = (R == V).select(
norm * (G - B), (G == V).select(norm * (B - R) + T(2) / T(6),
norm * (R - G) + T(4) / T(6)));
H.device(d) = (range > T(0)).select(H, H.constant(T(0)));
H.device(d) = (H < T(0)).select(H + T(1), H);
}
};
template <typename Device, typename T>
struct HSVToRGB {
void operator()(const Device &d,
typename TTypes<T, 2>::ConstTensor input_data,
typename TTypes<T, 2>::Tensor output_data) {
auto H = input_data.template chip<1>(0);
auto S = input_data.template chip<1>(1);
auto V = input_data.template chip<1>(2);
// TODO(wicke): compute only the fractional part of H for robustness
auto dh = H * T(6);
auto dr = ((dh - T(3)).abs() - T(1)).cwiseMax(T(0)).cwiseMin(T(1));
auto dg = (-(dh - T(2)).abs() + T(2)).cwiseMax(T(0)).cwiseMin(T(1));
auto db = (-(dh - T(4)).abs() + T(2)).cwiseMax(T(0)).cwiseMin(T(1));
auto one_s = -S + T(1);
auto R = output_data.template chip<1>(0);
auto G = output_data.template chip<1>(1);
auto B = output_data.template chip<1>(2);
R.device(d) = (one_s + S * dr) * V;
G.device(d) = (one_s + S * dg) * V;
B.device(d) = (one_s + S * db) * V;
}
};
} // namespace functor
} // namespace tensorflow
#endif // TENSORFLOW_CORE_KERNELS_COLORSPACE_OP_H_
@@ -0,0 +1,42 @@
/* Copyright 2017 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
#ifndef TENSORFLOW_CORE_KERNELS_COMPARE_AND_BITPACK_OP_H_
#define TENSORFLOW_CORE_KERNELS_COMPARE_AND_BITPACK_OP_H_
#include "third_party/eigen3/unsupported/Eigen/CXX11/Tensor"
#include "tensorflow/core/framework/op_kernel.h"
#include "tensorflow/core/framework/tensor_types.h"
#include "tensorflow/core/platform/types.h"
namespace tensorflow {
namespace functor {
typedef Eigen::ThreadPoolDevice CPUDevice;
typedef Eigen::GpuDevice GPUDevice;
template <typename Device, typename T>
struct CompareAndBitpack {
void operator()(OpKernelContext* c, typename TTypes<T>::ConstMatrix input,
typename TTypes<T>::ConstScalar threshold,
TTypes<uint8>::Matrix output);
};
} // namespace functor
} // namespace tensorflow
#endif // TENSORFLOW_CORE_KERNELS_COMPARE_AND_BITPACK_OP_H_
@@ -0,0 +1,69 @@
/* Copyright 2015 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
#ifndef TENSORFLOW_CORE_KERNELS_CONCAT_LIB_H_
#define TENSORFLOW_CORE_KERNELS_CONCAT_LIB_H_
#include <vector>
#include "third_party/eigen3/unsupported/Eigen/CXX11/Tensor"
#include "tensorflow/core/framework/device_base.h"
namespace tensorflow {
// Functors to concatenate tensors. These always take a rank-2 tensor (i.e a
// matrix) and concatenate it along the axis 1 ("putting them next to each
// other" as opposed to "putting them on top of one another").
//
// Any concatenation of n-dimensional tensors across any axis can be reduced to
// a concatenation of two-dimensional tensors across the axis 1 by first
// partitioning the axes of the original tensors into those less than the axis
// to be concatenated across and the rest. Then reshape the tensors into a
// two-dimensional tensor by collapsing these two sets of axes and concatenate
// the resulting matrices across the axis 1, finally reshaping the result to
// have the proper shape.
//
// So, for example, when stacking N tensors, reshape each to have shape
// {1, Numelements} and reshape the result matrix to have shape
// {1, N * NumElements} before passing it to this functor.
// Assumes all inputs are nonempty
template <typename T>
void ConcatCPU(
DeviceBase* d,
const std::vector<std::unique_ptr<typename TTypes<T, 2>::ConstMatrix>>&
inputs,
typename TTypes<T, 2>::Matrix* output);
#if GOOGLE_CUDA
template <typename T>
void ConcatGPU(
OpKernelContext* c,
const std::vector<std::unique_ptr<typename TTypes<T, 2>::ConstMatrix>>&
inputs_flat,
Tensor* output, typename TTypes<T, 2>::Tensor* output_flat);
#endif // GOOGLE_CUDA
#ifdef TENSORFLOW_USE_SYCL
template <typename T>
void ConcatSYCL(
const Eigen::SyclDevice& d,
const std::vector<std::unique_ptr<typename TTypes<T, 2>::ConstMatrix>>&
inputs,
typename TTypes<T, 2>::Matrix* output);
#endif // TENSORFLOW_USE_SYCL
} // namespace tensorflow
#endif // TENSORFLOW_CORE_KERNELS_CONCAT_LIB_H_
@@ -0,0 +1,169 @@
/* Copyright 2015 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
#ifndef TENSORFLOW_CORE_KERNELS_CONCAT_LIB_CPU_H_
#define TENSORFLOW_CORE_KERNELS_CONCAT_LIB_CPU_H_
#define EIGEN_USE_THREADS
#include <vector>
#include "tensorflow/core/framework/register_types.h"
#include "tensorflow/core/kernels/concat_lib.h"
#include "tensorflow/core/util/work_sharder.h"
namespace tensorflow {
// ElementCopier must be a struct with a single Copy function, which is passed
// the output pointer, input pointer, input index, and number of elements to
// copy from input to output.
template <typename T, typename ElementCopier>
void ConcatCPUImpl(
DeviceBase* d,
const std::vector<std::unique_ptr<typename TTypes<T, 2>::ConstMatrix>>&
inputs,
int64 cost_per_unit, ElementCopier copier,
typename TTypes<T, 2>::Matrix* output) {
size_t num_inputs = inputs.size();
std::vector<ptrdiff_t> sizes;
sizes.reserve(num_inputs);
int64 row_size = 0;
for (const auto& input : inputs) {
sizes.push_back(input->dimension(1));
row_size += sizes.back();
}
auto worker_threads = d->tensorflow_cpu_worker_threads();
int num_threads = std::min(4, worker_threads->num_threads);
// strings define a different amount of work (generally much more) compared
// with standard POD, so we parallelize differently.
if (!std::is_same<T, string>::value) {
num_threads =
static_cast<int>(std::min<int64>(num_threads, output->size() / 4096));
}
// Single threaded mode.
// TODO(dga): Deduplicate this code w.r.t. sharded code below.
if (num_threads == 0) {
T* out = &(*output)(0, 0);
std::vector<const T*> inp;
inp.reserve(num_inputs);
for (const auto& input : inputs) {
inp.push_back(&(*input)(0, 0));
}
const int64 dim0 = output->dimension(0);
for (int64 i = 0; i < dim0; ++i) {
for (int64 j = 0; j < num_inputs; ++j) {
auto size = sizes[j];
copier.Copy(out, inp[j], j, size);
out += size;
inp[j] += size;
}
}
return;
}
// Sharded mode.
auto work = [&row_size, &sizes, &inputs, &output, &copier, &num_inputs](
int64 start, int64 end) {
int64 skipped_rows = start / row_size;
T* out = output->data() + skipped_rows * row_size;
T* out_start = output->data() + start;
T* out_end = output->data() + end;
// Handle partial row at start
if (out < out_start) {
for (size_t j = 0; j < num_inputs; ++j) {
ptrdiff_t size = sizes[j];
ptrdiff_t offset = out_start - out;
if (size <= offset) {
out += size;
continue;
}
const T* inp = &(*inputs[j])(skipped_rows, 0);
if (offset > 0) {
out += offset;
inp += offset;
size -= offset;
}
size = std::min(size, out_end - out);
if (size <= 0) break;
copier.Copy(out, inp, j, size);
out += size;
}
++skipped_rows;
}
if (out == out_end) return;
CHECK(out >= out_start);
CHECK(out < out_end);
// Copy remaining data.
std::vector<const T*> inp;
inp.reserve(num_inputs);
for (const auto& input : inputs) {
inp.push_back(&(*input)(skipped_rows, 0));
}
const int64 dim0 = output->dimension(0);
for (int64 i = skipped_rows; i < dim0; ++i) {
for (int64 j = 0; j < num_inputs; ++j) {
ptrdiff_t size = std::min(sizes[j], out_end - out);
copier.Copy(out, inp[j], j, size);
out += size;
inp[j] += size;
if (out == out_end) return;
}
}
};
Shard(worker_threads->num_threads, worker_threads->workers, output->size(),
cost_per_unit, work);
}
#ifdef TENSORFLOW_USE_SYCL
template <typename T, typename ElementCopier>
void ConcatSYCLImpl(
const Eigen::SyclDevice& d,
const std::vector<std::unique_ptr<typename TTypes<T, 2>::ConstMatrix>>&
inputs,
int64 cost_per_unit, ElementCopier copier,
typename TTypes<T, 2>::Matrix* output) {
size_t num_inputs = inputs.size();
std::vector<ptrdiff_t> sizes;
sizes.reserve(num_inputs);
int64 row_size = 0;
for (const auto& input : inputs) {
sizes.push_back(input->dimension(1));
row_size += sizes.back();
}
T* out = &(*output)(0, 0);
std::vector<const T*> inp;
inp.reserve(num_inputs);
for (const auto& input : inputs) {
inp.push_back(&(*input)(0, 0));
}
const int64 dim0 = output->dimension(0);
for (int64 i = 0; i < dim0; ++i) {
for (int64 j = 0; j < num_inputs; ++j) {
auto size = sizes[j];
d.memcpy(out, inp[j], size * sizeof(T));
out += size;
inp[j] += size;
}
}
}
#endif // TENSORFLOW_USE_SYCL
} // namespace tensorflow
#endif // TENSORFLOW_CORE_KERNELS_CONCAT_LIB_CPU_H_
@@ -0,0 +1,138 @@
/* Copyright 2015 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
#ifndef TENSORFLOW_CORE_KERNELS_CONDITIONAL_ACCUMULATOR_H_
#define TENSORFLOW_CORE_KERNELS_CONDITIONAL_ACCUMULATOR_H_
#include "tensorflow/core/kernels/fill_functor.h"
#include "tensorflow/core/kernels/typed_conditional_accumulator_base.h"
namespace tensorflow {
/**
* An aggregation object for adding dense gradients.
*
* The two main methods of this class are TryApplyGrad and TryTakeGrad.
*
* TryApplyGrad tries add a gradient to the accumulator. The attempt is
* successful if local_step >= global_step, i.e., if the gradient is not stale,
* having been computed using up-to-date information. Otherwise, the gradient is
* silently dropped.
*
* TryTakeGrad logs an attempt to read the average gradient. The attempt is
* blocked until the number of gradients accumulated (via TryApplyGrad) is equal
* or exceeds the number requested by TryTakeGrad.
* Once this condition is satisfied, the following actions are taken:
* (1) the value of the average gradient is returned
* (2) the count of accumulated gradients is reset to 0
* (3) the internal global_step value (current_global_step_) is incremented by 1
*
* ConditionalAccumulator is the datatype-dependent templated sub-class of
* ConditionalAccumulatorBase. It implements the virtual arithmetic methods that
* are used by for aggregating, averaging, allocating, returning dense Tensors.
*/
template <typename Device, typename T>
class ConditionalAccumulator
: public TypedConditionalAccumulatorBase<const Tensor> {
public:
// Args:
// dtype: The datatype of the gradients to be accumulated.
// shape: The shape of the accumulated gradients.
// name: A name to use for the ConditionalAccumulator.
// reduction_type: The reduction type, i.e., MEAN or SUM
ConditionalAccumulator(const DataType& dtype, const PartialTensorShape& shape,
const string& name, const string& reduction_type)
: TypedConditionalAccumulatorBase<const Tensor>(dtype, shape, name,
reduction_type) {}
~ConditionalAccumulator() override{};
protected:
// accum_grad is the tensor that holds the aggregate gradient.
// It is initialized the first time ApplyGrad is called.
Tensor* accum_grad_ = nullptr;
PersistentTensor accum_grad_persistent_;
functor::SetZeroFunctor<Device, T> set_zero_functor_;
Status ValidateShape(const Tensor* tensor)
EXCLUSIVE_LOCKS_REQUIRED(this->mu_) {
// Must be compatible with accumulated gradient if available
if (counter_ > 0) {
if (!accum_grad_->shape().IsSameSize(tensor->shape())) {
return errors::InvalidArgument("Shape mismatch: expected ",
accum_grad_->shape().DebugString(),
", got ", tensor->shape().DebugString());
}
}
// Must also be compatible with given shape
if (!shape_.IsCompatibleWith(tensor->shape())) {
return errors::InvalidArgument("Shape mismatch: expected ",
shape_.DebugString(), ", got ",
tensor->shape().DebugString());
}
return Status::OK();
}
void AllocateAndAssignToAccumGradFunction(OpKernelContext* ctx,
const Tensor* grad) override {
// TODO(b/32704451): Don't just ignore the ::tensorflow::Status object!
ctx->allocate_persistent(dtype_, grad->shape(), &accum_grad_persistent_,
&accum_grad_)
.IgnoreError();
accum_grad_->flat<T>().device(ctx->template eigen_device<Device>()) =
grad->flat<T>();
}
void AddToAccumGradFunction(OpKernelContext* ctx,
const Tensor* grad) override {
accum_grad_->flat<T>().device(ctx->template eigen_device<Device>()) +=
grad->flat<T>();
}
void DivideAccumGradByCounter(OpKernelContext* ctx) override
EXCLUSIVE_LOCKS_REQUIRED(this->mu_) {
Tensor c(DataTypeToEnum<T>::value, {});
c.scalar<T>()() = TypeConverter<T, int>::ConvertUToT(this->counter_);
this->accum_grad_->template flat<T>().device(
ctx->template eigen_device<Device>()) =
this->accum_grad_->template flat<T>() / c.scalar<T>()();
}
bool SetOutput(OpKernelContext* ctx) override {
ctx->set_output(0, *accum_grad_);
return true;
}
bool GetAndValidateTensorInputForApplyGrad(OpKernelContext* ctx,
const Tensor** tensor) override
EXCLUSIVE_LOCKS_REQUIRED(this->mu_) {
// Get input gradient tensor
const Tensor* grad_tensor;
OP_REQUIRES_OK_BOOLEAN(ctx, ctx->input("gradient", &grad_tensor));
*tensor = grad_tensor;
OP_REQUIRES_OK_BOOLEAN(ctx, this->ValidateShape(*tensor));
return true;
}
void CleanUpGradTensor(const Tensor* tensor) override {
// do nothing
}
TF_DISALLOW_COPY_AND_ASSIGN(ConditionalAccumulator);
};
} // namespace tensorflow
#endif // TENSORFLOW_CORE_KERNELS_CONDITIONAL_ACCUMULATOR_H_
@@ -0,0 +1,203 @@
/* Copyright 2015 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
#ifndef TENSORFLOW_CORE_KERNELS_CONDITIONAL_ACCUMULATOR_BASE_H_
#define TENSORFLOW_CORE_KERNELS_CONDITIONAL_ACCUMULATOR_BASE_H_
#include <deque>
#include "third_party/eigen3/unsupported/Eigen/CXX11/Tensor"
#include "tensorflow/core/framework/numeric_op.h"
#include "tensorflow/core/framework/op_kernel.h"
#include "tensorflow/core/framework/resource_mgr.h"
namespace tensorflow {
/**
* ConditionalAccumulator/ConditionalAccumulatorBase implements an aggregation
* object for adding gradients.
* The two main methods of this class are TryApplyGrad and TryTakeGrad.
*
* TryApplyGrad tries add a gradient to the accumulator. The attempt is
* successful if local_step >= global_step, i.e., if the gradient is not stale,
* having been computed using up-to-date information. Otherwise, the gradient is
* silently dropped.
*
* TryTakeGrad logs an attempt to read the average gradient. The attempt is
* blocked until the number of gradients accumulated (via TryApplyGrad) is equal
* or exceeds the number requested by TryTakeGrad.
* Once this condition is satisfied, the following actions are taken:
* (1) the value of the average gradient is returned
* (2) the count of accumulated gradients is reset to 0
* (3) the internal global_step value (current_global_step_) is incremented by 1
*/
class ConditionalAccumulatorBase : public ResourceBase {
public:
// Args:
// dtype: The datatype of the gradients to be accumulated.
// shape: The shape of the accumulated gradients.
// name: A name to use for the ConditionalAccumulator.
ConditionalAccumulatorBase(const DataType& dtype,
const PartialTensorShape& shape,
const string& name, const string& reduction_type);
typedef AsyncOpKernel::DoneCallback DoneCallback;
virtual void TryApplyGrad(int64 local_step, OpKernelContext* ctx) = 0;
void TryTakeGrad(int num_required, OpKernelContext* ctx,
DoneCallback callback);
// Accessor methods
uint32 num_accumulated() {
mutex_lock lock(mu_);
return counter_;
}
const DataType& dtype() const { return dtype_; }
string DebugString() override { return "A conditional accumulator"; }
// SetGlobalStep is a modifier method for current_global_step.
// It returns an InvalidArgument error if the new_global_step is less than
// current_global_step.
Status SetGlobalStep(int64 new_global_step);
Status MatchesNodeDef(const NodeDef& node_def);
protected:
// Virtual methods to be implemented by sub-classes for different datatypes.
// Implements arithmetic operations specific to datatype.
virtual void DivideAccumGradByCounter(OpKernelContext* ctx)
EXCLUSIVE_LOCKS_REQUIRED(mu_) = 0;
virtual bool SetOutput(OpKernelContext* ctx) = 0;
enum RunResult { kNoProgress, kComplete };
// Helper struct holding information about a TakeGrad attempt
struct Attempt;
typedef std::function<RunResult(Attempt*)> RunCallback;
struct Attempt {
int elements_requested;
DoneCallback done_callback; // must be run outside mu_
OpKernelContext* context;
CancellationManager* cancellation_manager; // not owned
CancellationToken cancellation_token;
RunCallback run_callback; // must be run while holding mu_
bool is_cancelled;
Attempt(int elements_requested, DoneCallback done_callback,
OpKernelContext* context, CancellationManager* cancellation_manager,
CancellationToken cancellation_token, RunCallback run_callback)
: elements_requested(elements_requested),
done_callback(std::move(done_callback)),
context(context),
cancellation_manager(cancellation_manager),
cancellation_token(cancellation_token),
run_callback(std::move(run_callback)),
is_cancelled(false) {}
};
// Helper struct for deregistration of a cancellation token and executing a
// DoneCallback after a TakeGrad attempt is complete.
struct CleanUp {
CleanUp(DoneCallback&& f, CancellationToken ct, CancellationManager* cm)
: finished(f), to_deregister(ct), cm(cm) {}
DoneCallback finished;
CancellationToken to_deregister;
CancellationManager* cm;
};
// Fields
const DataType dtype_;
const PartialTensorShape shape_;
const string name_;
const string reduction_type_;
mutex mu_;
int counter_ GUARDED_BY(mu_);
int64 current_global_step_ GUARDED_BY(mu_);
std::deque<Attempt> takegrad_attempts_ GUARDED_BY(mu_);
// Methods
// Helper function for creating cancellation callback
void Cancel(CancellationManager* cancellation_manager,
CancellationToken token);
// Helper functions to process TakeGrad attempts.
// FlushUnlocked is called at the end of each TryApplyGrad and TryTakeGrad
// calls to try to clear the TakeGrad attempts. This in turn calls
// TryAttemptLocked, which then executes the RunCallback of the logged
// attempts.
// Both functions are modeled after core/kernels/queue_base.
// Note: ApplyGrad attempts never block -- unlike in a queue with limited
// capacity, we can always add the newest gradient to our accumulator
// (if it is not stale) or drop it silently (if it is stale).
void FlushUnlocked();
bool TryAttemptLocked(std::vector<CleanUp>* clean_up)
EXCLUSIVE_LOCKS_REQUIRED(mu_);
// Helper methods
// void DeepCopy(Tensor* dst);
bool TakeGradLockedHelper(OpKernelContext* ctx, DoneCallback callback)
EXCLUSIVE_LOCKS_REQUIRED(mu_);
};
/*
* Modifications to convenience macros defined in core/framework/op_kernel.h.
* The below macros return a boolean if the test fails, so that the calling
* function can get an indication that a failure has occurred.
*/
#define OP_REQUIRES_BOOLEAN(CTX, EXP, STATUS) \
do { \
if (!TF_PREDICT_TRUE(EXP)) { \
(CTX)->CtxFailure(__FILE__, __LINE__, (STATUS)); \
return false; \
} \
} while (0)
#define OP_REQUIRES_OK_BOOLEAN(CTX, STATUS) \
do { \
::tensorflow::Status _s(STATUS); \
if (!TF_PREDICT_TRUE(_s.ok())) { \
(CTX)->CtxFailureWithWarning(__FILE__, __LINE__, _s); \
return false; \
} \
} while (0)
/*
* Convenience classes for helping to convert between numeric types.
* The specialization for Eigen::half here simplifies specialization of
* ConditionalAccumulator classes later.
*/
template <typename T, typename U>
class TypeConverter {
public:
static T ConvertUToT(U c) { return c; /* implicit conversion */ }
};
template <typename U>
class TypeConverter<Eigen::half, U> {
public:
static Eigen::half ConvertUToT(U c) {
return Eigen::half_impl::float_to_half_rtne(c);
}
};
} // namespace tensorflow
#endif // TENSORFLOW_CORE_KERNELS_CONDITIONAL_ACCUMULATOR_BASE_H_
@@ -0,0 +1,240 @@
/* Copyright 2015 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
#ifndef TENSORFLOW_CORE_KERNELS_CONDITIONAL_ACCUMULATOR_BASE_OP_H_
#define TENSORFLOW_CORE_KERNELS_CONDITIONAL_ACCUMULATOR_BASE_OP_H_
#define EIGEN_USE_THREADS
#include "tensorflow/core/kernels/conditional_accumulator_base.h"
#include "tensorflow/core/framework/op_kernel.h"
#include "tensorflow/core/framework/register_types.h"
#include "tensorflow/core/framework/resource_mgr.h"
#include "tensorflow/core/framework/tensor.h"
#include "tensorflow/core/framework/tensor_shape.h"
#include "tensorflow/core/framework/types.h"
#include "tensorflow/core/lib/core/errors.h"
#include "tensorflow/core/platform/macros.h"
#include "tensorflow/core/platform/mutex.h"
#include "tensorflow/core/platform/thread_annotations.h"
#include "tensorflow/core/platform/types.h"
typedef Eigen::ThreadPoolDevice CPUDevice;
typedef std::function<void()> DoneCallback;
namespace tensorflow {
/**
* Defines a ConditionalAccumulatorBaseOp, which constructs a
* ConditionalAccumulatorBase (via sub-class's Creator) and returns its handle.
*/
class ConditionalAccumulatorBaseOp : public OpKernel {
public:
explicit ConditionalAccumulatorBaseOp(OpKernelConstruction* context)
: OpKernel(context), accumulator_handle_set_(false) {
OP_REQUIRES_OK(context,
context->allocate_persistent(DT_STRING, TensorShape({2}),
&accumulator_handle_, nullptr));
OP_REQUIRES_OK(context, context->GetAttr("shape", &shape_));
OP_REQUIRES_OK(context, context->GetAttr("dtype", &dtype_));
OP_REQUIRES_OK(context,
context->GetAttr("reduction_type", &reduction_type_));
}
void Compute(OpKernelContext* ctx) override {
mutex_lock l(mu_);
if (!accumulator_handle_set_) {
OP_REQUIRES_OK(ctx, SetAccumulatorHandle(ctx));
}
ctx->set_output_ref(0, &mu_, accumulator_handle_.AccessTensor(ctx));
}
protected:
~ConditionalAccumulatorBaseOp() override {
// If the accumulator object was not shared, delete it.
if (accumulator_handle_set_ && cinfo_.resource_is_private_to_kernel()) {
TF_CHECK_OK((cinfo_.resource_manager()
->template Delete<ConditionalAccumulatorBase>(
cinfo_.container(), cinfo_.name())));
}
}
protected:
typedef std::function<Status(ConditionalAccumulatorBase**)> Creator;
// Subclasses must override this
virtual Creator GetCreator() const = 0;
// Variables required to construct ConditionalAccumulator
DataType dtype_;
PartialTensorShape shape_;
ContainerInfo cinfo_;
string reduction_type_;
private:
Status SetAccumulatorHandle(OpKernelContext* ctx)
EXCLUSIVE_LOCKS_REQUIRED(mu_) {
TF_RETURN_IF_ERROR(cinfo_.Init(ctx->resource_manager(), def()));
// Check input signature
DataTypeVector expected_inputs = {};
TF_RETURN_IF_ERROR(ctx->MatchSignature(expected_inputs, {DT_STRING_REF}));
Creator creator = GetCreator();
ConditionalAccumulatorBase* accumulator;
TF_RETURN_IF_ERROR(
(cinfo_.resource_manager()
->template LookupOrCreate<ConditionalAccumulatorBase>(
cinfo_.container(), cinfo_.name(), &accumulator, creator)));
core::ScopedUnref unref_me(accumulator);
// Verify that the shared accumulator is compatible
// with the requested arguments.
TF_RETURN_IF_ERROR(accumulator->MatchesNodeDef(def()));
auto h = accumulator_handle_.AccessTensor(ctx)->template flat<string>();
h(0) = cinfo_.container();
h(1) = cinfo_.name();
accumulator_handle_set_ = true;
return Status::OK();
}
mutex mu_;
PersistentTensor accumulator_handle_ GUARDED_BY(mu_);
bool accumulator_handle_set_ GUARDED_BY(mu_);
};
/**
* General OpKernel for ConditionalAccumulatorBase-related ops.
*/
class ConditionalAccumulatorBaseAsyncOpKernel : public AsyncOpKernel {
public:
explicit ConditionalAccumulatorBaseAsyncOpKernel(
OpKernelConstruction* context)
: AsyncOpKernel(context) {}
void ComputeAsync(OpKernelContext* ctx, DoneCallback callback) final {
ConditionalAccumulatorBase* accumulator;
OP_REQUIRES_OK_ASYNC(
ctx, GetResourceFromContext(ctx, "handle", &accumulator), callback);
ComputeAsync(ctx, accumulator, [callback, accumulator]() {
accumulator->Unref();
callback();
});
}
protected:
virtual void ComputeAsync(OpKernelContext* ctx,
ConditionalAccumulatorBase* accumulator,
DoneCallback callback) = 0;
};
/**
* General OpKernel for ConditionalAccumulatorBase-related ops.
*/
class ConditionalAccumulatorBaseSyncOpKernel : public OpKernel {
public:
explicit ConditionalAccumulatorBaseSyncOpKernel(OpKernelConstruction* context)
: OpKernel(context) {}
void Compute(OpKernelContext* ctx) final {
ConditionalAccumulatorBase* accumulator;
OP_REQUIRES_OK(ctx, GetResourceFromContext(ctx, "handle", &accumulator));
Compute(ctx, accumulator);
accumulator->Unref();
}
protected:
virtual void Compute(OpKernelContext* ctx,
ConditionalAccumulatorBase* accumulator) = 0;
};
/**
* Defines a AccumulateGradientOp, the execution of which adds a gradient to the
* given ConditionalAccumulator.
*/
class ConditionalAccumulatorBaseApplyGradientOp
: public ConditionalAccumulatorBaseSyncOpKernel {
public:
explicit ConditionalAccumulatorBaseApplyGradientOp(
OpKernelConstruction* context)
: ConditionalAccumulatorBaseSyncOpKernel(context) {}
protected:
virtual void CheckSignature(OpKernelContext* ctx,
ConditionalAccumulatorBase* accumulator) = 0;
void Compute(OpKernelContext* ctx,
ConditionalAccumulatorBase* accumulator) override {
// Check input signature
CheckSignature(ctx, accumulator);
// Get input local_step
const Tensor* local_step_tensor;
OP_REQUIRES_OK(ctx, ctx->input("local_step", &local_step_tensor));
if (!TensorShapeUtils::IsScalar(local_step_tensor->shape())) {
ctx->CtxFailureWithWarning(errors::InvalidArgument(
"Argument local_step must be scalar, but had bad shape ",
local_step_tensor->shape().DebugString()));
}
// Actually try to apply gradient now
accumulator->TryApplyGrad(local_step_tensor->scalar<int64>()(), ctx);
}
};
/**
* Defines a TakeAccumulatedGradientOp, the execution of which adds a gradient
* to the given ConditionalAccumulator.
*/
class ConditionalAccumulatorBaseTakeGradientOp
: public ConditionalAccumulatorBaseAsyncOpKernel {
public:
explicit ConditionalAccumulatorBaseTakeGradientOp(
OpKernelConstruction* context)
: ConditionalAccumulatorBaseAsyncOpKernel(context) {}
protected:
virtual void CheckSignature(OpKernelContext* ctx,
ConditionalAccumulatorBase* accumulator,
DoneCallback callback) = 0;
void ComputeAsync(OpKernelContext* ctx,
ConditionalAccumulatorBase* accumulator,
DoneCallback callback) override {
// Check signature
CheckSignature(ctx, accumulator, callback);
// Get input num_required
const Tensor* num_required_tensor;
OP_REQUIRES_OK_ASYNC(ctx, ctx->input("num_required", &num_required_tensor),
callback);
if (!TensorShapeUtils::IsScalar(num_required_tensor->shape())) {
ctx->CtxFailureWithWarning(errors::InvalidArgument(
"Argument num_required must be scalar, but had bad shape ",
num_required_tensor->shape().DebugString()));
callback();
}
// Actually try to take gradient now
accumulator->TryTakeGrad(num_required_tensor->scalar<int32>()(), ctx,
callback);
}
};
} // namespace tensorflow
#endif // TENSORFLOW_CORE_KERNELS_CONDITIONAL_ACCUMULATOR_BASE_OP_H_
@@ -0,0 +1,50 @@
/* Copyright 2015 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
#ifndef TENSORFLOW_CORE_KERNELS_CONSTANT_OP_H_
#define TENSORFLOW_CORE_KERNELS_CONSTANT_OP_H_
#include "third_party/eigen3/unsupported/Eigen/CXX11/Tensor"
#include "tensorflow/core/framework/op_kernel.h"
#include "tensorflow/core/framework/tensor_types.h"
#include "tensorflow/core/platform/macros.h"
namespace tensorflow {
// ConstantOp returns a tensor specified by ConstantOpDef.
class ConstantOp : public OpKernel {
public:
explicit ConstantOp(OpKernelConstruction* ctx);
void Compute(OpKernelContext* ctx) override;
bool IsExpensive() override { return false; }
~ConstantOp() override;
private:
Tensor tensor_;
TF_DISALLOW_COPY_AND_ASSIGN(ConstantOp);
};
class PlaceholderOp : public OpKernel {
public:
explicit PlaceholderOp(OpKernelConstruction* ctx);
void Compute(OpKernelContext* ctx) override;
private:
PartialTensorShape expected_shape_;
};
} // namespace tensorflow
#endif // TENSORFLOW_CORE_KERNELS_CONSTANT_OP_H_
@@ -0,0 +1,118 @@
/* Copyright 2015 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
#ifndef TENSORFLOW_CORE_KERNELS_CONTROL_FLOW_OPS_H_
#define TENSORFLOW_CORE_KERNELS_CONTROL_FLOW_OPS_H_
#include "tensorflow/core/framework/op_kernel.h"
namespace tensorflow {
// A ControlTriggerOp is similar to a NoOp. However, it always treats the input
// control edges as Live edges. Its primary use so far is in the scheduling of
// recvs, where we add ControlTrigger nodes and use them to trigger recvs. We
// allow ControlTrigger nodes to be enabled by dead nodes.
class ControlTriggerOp : public OpKernel {
public:
explicit ControlTriggerOp(OpKernelConstruction* context)
: OpKernel(context) {}
void Compute(OpKernelContext* context) override {}
bool IsExpensive() override { return false; }
};
// A switch op has two inputs and two outputs. It forwards the value of
// Input:0 to the output specified by input:1. Input:1 is a boolean tensor.
// Input:0 is forwarded to output:0 if input:1 is false, otherwise to
// output:1.
class SwitchOp : public OpKernel {
public:
explicit SwitchOp(OpKernelConstruction* context) : OpKernel(context) {}
void Compute(OpKernelContext* context) override;
bool IsExpensive() override { return false; }
~SwitchOp() override {}
TF_DISALLOW_COPY_AND_ASSIGN(SwitchOp);
};
// A merge op has n inputs and two outputs. It forwards the value of the
// first input that becomes available to its first output, and the
// index of the first input to its second output.
class MergeOp : public OpKernel {
public:
explicit MergeOp(OpKernelConstruction* context);
void Compute(OpKernelContext* context) override;
bool IsExpensive() override { return false; }
~MergeOp() override {}
TF_DISALLOW_COPY_AND_ASSIGN(MergeOp);
};
// An enter op has one input and one output. It creates or finds
// the child frame that is uniquely identified by the frame_name,
// and makes its input available to the child frame.
class EnterOp : public OpKernel {
public:
explicit EnterOp(OpKernelConstruction* context) : OpKernel(context) {}
void Compute(OpKernelContext* context) override;
bool IsExpensive() override { return false; }
~EnterOp() override {}
TF_DISALLOW_COPY_AND_ASSIGN(EnterOp);
};
// An exit op has one input and one output. It exits the current
// frame to its parent frame, and makes its input available to the
// parent frame.
class ExitOp : public OpKernel {
public:
explicit ExitOp(OpKernelConstruction* context) : OpKernel(context) {}
void Compute(OpKernelContext* context) override;
bool IsExpensive() override { return false; }
~ExitOp() override {}
TF_DISALLOW_COPY_AND_ASSIGN(ExitOp);
};
// A next_iteration op has one input and one output. It makes its input
// available to the next iteration.
class NextIterationOp : public OpKernel {
public:
explicit NextIterationOp(OpKernelConstruction* context) : OpKernel(context) {}
void Compute(OpKernelContext* context) override;
bool IsExpensive() override { return false; }
~NextIterationOp() override {}
TF_DISALLOW_COPY_AND_ASSIGN(NextIterationOp);
};
// A LoopCond op has one input and one output. The input is a boolean
// scalar representing the taken branches of the "pivot" Switch that
// determines loop termination. As a contract, any high-level front-end
// should always use port '0' of the "pivot" switches for loop exit.
class LoopCondOp : public OpKernel {
public:
explicit LoopCondOp(OpKernelConstruction* context);
~LoopCondOp() override;
void Compute(OpKernelContext* context) override;
bool IsExpensive() override;
TF_DISALLOW_COPY_AND_ASSIGN(LoopCondOp);
};
} // namespace tensorflow
#endif // TENSORFLOW_CORE_KERNELS_CONTROL_FLOW_OPS_H_
@@ -0,0 +1,316 @@
/* Copyright 2015 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
#ifndef TENSORFLOW_CORE_KERNELS_CONV_2D_H_
#define TENSORFLOW_CORE_KERNELS_CONV_2D_H_
#include "third_party/eigen3/unsupported/Eigen/CXX11/Tensor"
#include "tensorflow/core/framework/tensor_types.h"
#include "tensorflow/core/kernels/eigen_backward_spatial_convolutions.h"
#include "tensorflow/core/kernels/eigen_spatial_convolutions.h"
#include "tensorflow/core/util/tensor_format.h"
namespace tensorflow {
namespace functor {
// TODO(yangke): revisit these operations and in particular, see if we can
// combine all of them into just one operation without causing nvcc to
// timeout.
template <typename Device, typename T, int Dims, typename IndexType>
struct ShuffleAndReverse {
void operator()(const Device& d,
typename TTypes<T, Dims, IndexType>::ConstTensor input,
const Eigen::DSizes<IndexType, Dims>& order,
const Eigen::array<bool, Dims>& reverse_dims,
typename TTypes<T, Dims, IndexType>::Tensor output) {
output.device(d) = input.shuffle(order).reverse(reverse_dims);
}
};
template <typename Device, typename T, int Dims, typename IndexType>
struct InflatePadAndShuffle {
void operator()(
const Device& d, typename TTypes<T, Dims, IndexType>::ConstTensor input,
const Eigen::DSizes<IndexType, Dims>& strides,
const Eigen::array<Eigen::IndexPair<IndexType>, Dims>& pad_dims,
const Eigen::DSizes<IndexType, Dims>& order,
typename TTypes<T, Dims, IndexType>::Tensor output) {
output.device(d) = input.inflate(strides).pad(pad_dims).shuffle(order);
}
};
template <typename Device, typename Input, typename Filter, typename Output>
void SpatialConvolutionFunc(const Device& d, Output output, Input input,
Filter filter, int row_stride, int col_stride,
int row_dilation, int col_dilation,
const Eigen::PaddingType& padding) {
// Need to swap row/col when calling Eigen.
output.device(d) =
Eigen::SpatialConvolution(input, filter, col_stride, row_stride, padding,
col_dilation, row_dilation);
}
template <typename Device, typename T>
struct SpatialConvolution {
void operator()(const Device& d, typename TTypes<T, 4>::Tensor output,
typename TTypes<T, 4>::ConstTensor input,
typename TTypes<T, 4>::ConstTensor filter, int row_stride,
int col_stride, int row_dilation, int col_dilation,
const Eigen::PaddingType& padding) {
SpatialConvolutionFunc(d, output, input, filter, row_stride, col_stride,
row_dilation, col_dilation, padding);
}
};
template <typename Device>
struct SpatialConvolution<Device, Eigen::half> {
void operator()(const Device& d,
typename TTypes<Eigen::half, 4>::Tensor output,
typename TTypes<Eigen::half, 4>::ConstTensor input,
typename TTypes<Eigen::half, 4>::ConstTensor filter,
int row_stride, int col_stride, int row_dilation,
int col_dilation, const Eigen::PaddingType& padding) {
output.device(d) =
Eigen::SpatialConvolution(input.cast<float>(), filter.cast<float>(),
col_stride, row_stride, padding, col_dilation,
row_dilation)
.cast<Eigen::half>();
}
};
template <typename Device, typename T>
struct SpatialConvolutionBackwardInput {
void operator()(const Device& d, typename TTypes<T, 4>::Tensor input_backward,
typename TTypes<T, 4>::ConstTensor kernel,
typename TTypes<T, 4>::ConstTensor output_backward,
int row_stride, int col_stride, int row_dilation,
int col_dilation) {
// Need to swap row/col when calling Eigen.
input_backward.device(d) = Eigen::SpatialConvolutionBackwardInput(
kernel, output_backward, input_backward.dimension(2),
input_backward.dimension(1), col_stride, row_stride, col_dilation,
row_dilation);
}
};
template <typename Device, typename T>
struct SpatialConvolutionBackwardFilter {
void operator()(const Device& d,
typename TTypes<T, 4>::Tensor kernel_backward,
typename TTypes<T, 4>::ConstTensor input,
typename TTypes<T, 4>::ConstTensor output_backward,
int row_stride, int col_stride, int row_dilation,
int col_dilation) {
// Need to swap row/col when calling Eigen.
kernel_backward.device(d) = Eigen::SpatialConvolutionBackwardKernel(
input, output_backward, kernel_backward.dimension(1),
kernel_backward.dimension(0), col_stride, row_stride, col_dilation,
row_dilation);
}
};
// TODO(vrv): Figure out how to use the MatMulFunctor in matmul_op.h.
// My initial attempt to do this compiled but failed in the pytest
// due to a swigdeps error.
template <typename Device, typename T>
struct MatMulConvFunctor {
// Computes on device "d": out = in0 * in1, where * is matrix
// multiplication.
void operator()(
const Device& d, typename TTypes<T, 2>::Tensor out,
typename TTypes<T, 2>::ConstTensor in0,
typename TTypes<T, 2>::ConstTensor in1,
const Eigen::array<Eigen::IndexPair<Eigen::DenseIndex>, 1>& dim_pair) {
out.device(d) = in0.contract(in1, dim_pair);
}
};
// Shuffles a filter tensor from TensorFlow format HWIO to dst_filter_format.
//
// Note: Currently OIHW is the only supported destination format. Support for
// OHWI format will be added in a follow-up change.
template <typename Device, typename T, typename IndexType, int NDIMS>
struct TransformFilter {
void operator()(const Device& d, FilterTensorFormat dst_filter_format,
typename TTypes<T, NDIMS, IndexType>::ConstTensor in,
typename TTypes<T, NDIMS, IndexType>::Tensor out) {
// Merge the spatial dimensions together to speed up the shuffle operation.
Eigen::DSizes<IndexType, 3> merged_dims;
merged_dims[0] = in.dimension(0); // spatial dimensions
for (int i = 1; i < NDIMS - 2; ++i) {
merged_dims[0] *= in.dimension(i);
}
merged_dims[1] = in.dimension(NDIMS - 2); // input filters
merged_dims[2] = in.dimension(NDIMS - 1); // output filters
CHECK(dst_filter_format == FORMAT_OIHW)
<< "Unsupported destination filter format: "
<< ToString(dst_filter_format);
// Source filter format is FORMAT_HWIO and spatial dimensions HW are merged
// in the beginning.
Eigen::DSizes<IndexType, 3> shuffling_perm =
Eigen::DSizes<IndexType, 3>(2, 1, 0);
Eigen::DSizes<IndexType, NDIMS> expanded_dims;
int out_index = 0;
for (int merged_dim = 0; merged_dim < merged_dims.rank(); ++merged_dim) {
if (shuffling_perm[merged_dim] == 0) {
for (int spatial_dim = 0; spatial_dim < NDIMS - 2; ++spatial_dim) {
expanded_dims[out_index++] = in.dimension(spatial_dim);
}
} else {
constexpr int kLastSpatialDim = NDIMS - 3;
expanded_dims[out_index++] =
in.dimension(kLastSpatialDim + shuffling_perm[merged_dim]);
}
}
out.device(d) =
in.reshape(merged_dims).shuffle(shuffling_perm).reshape(expanded_dims);
}
};
template <typename Device, typename T, typename IndexType>
struct TransformDepth {
void operator()(const Device& d,
typename TTypes<T, 4, IndexType>::ConstTensor in,
const Eigen::DSizes<IndexType, 4>& shuffle,
typename TTypes<T, 4, IndexType>::Tensor out) {
Eigen::DSizes<IndexType, 3> merged_dims;
Eigen::DSizes<IndexType, 4> expanded_dims;
Eigen::DSizes<IndexType, 3> new_shuffle;
// Merge dimensions that won't be shuffled together to speed things up.
if (shuffle[1] == 2 && shuffle[2] == 3) {
merged_dims[0] = in.dimension(0);
merged_dims[1] = in.dimension(1);
merged_dims[2] = in.dimension(2) * in.dimension(3);
new_shuffle[0] = shuffle[0];
new_shuffle[1] = 2;
new_shuffle[2] = shuffle[3];
expanded_dims[0] = in.dimension(shuffle[0]);
expanded_dims[1] = in.dimension(2);
expanded_dims[2] = in.dimension(3);
expanded_dims[3] = in.dimension(shuffle[3]);
} else if (shuffle[0] == 2 && shuffle[1] == 3) {
merged_dims[0] = in.dimension(0);
merged_dims[1] = in.dimension(1);
merged_dims[2] = in.dimension(2) * in.dimension(3);
new_shuffle[0] = 2;
new_shuffle[1] = shuffle[2];
new_shuffle[2] = shuffle[3];
expanded_dims[0] = in.dimension(2);
expanded_dims[1] = in.dimension(3);
expanded_dims[2] = in.dimension(shuffle[2]);
expanded_dims[3] = in.dimension(shuffle[3]);
} else if (shuffle[0] == 0 && shuffle[1] == 3 && shuffle[2] == 1 &&
shuffle[3] == 2) {
merged_dims[0] = in.dimension(0);
merged_dims[1] = in.dimension(1) * in.dimension(2);
merged_dims[2] = in.dimension(3);
new_shuffle[0] = 0;
new_shuffle[1] = 2;
new_shuffle[2] = 1;
expanded_dims[0] = in.dimension(0);
expanded_dims[1] = in.dimension(3);
expanded_dims[2] = in.dimension(1);
expanded_dims[3] = in.dimension(2);
} else {
assert(false && "unexpected shuffle");
}
out.device(d) =
in.reshape(merged_dims).shuffle(new_shuffle).reshape(expanded_dims);
}
};
template <typename Device, typename T, typename IndexType, int NDIMS>
struct PadInput {
void operator()(const Device& d,
typename TTypes<T, NDIMS, IndexType>::ConstTensor in,
const std::array<int, NDIMS - 2>& padding_left,
const std::array<int, NDIMS - 2>& padding_right,
typename TTypes<T, NDIMS, IndexType>::Tensor out,
TensorFormat format) {
Eigen::array<Eigen::IndexPair<IndexType>, NDIMS> padding;
padding[GetTensorDimIndex<NDIMS - 2>(format, 'N')] = {0, 0};
for (int i = 0; i < NDIMS - 2; ++i) {
padding[GetTensorDimIndex<NDIMS - 2>(format, '0' + i)] = {
padding_left[i], padding_right[i]};
}
padding[GetTensorDimIndex<NDIMS - 2>(format, 'C')] = {0, 0};
out.device(d) = in.pad(padding);
}
};
// Converts a tensor from:
// [batch, <spatial>, filters]
// to:
// [batch, filters, <spatial>]
template <typename Device, typename T, int NDIMS>
struct NHWCToNCHW {
void operator()(const Device& d, typename TTypes<T, NDIMS>::ConstTensor in,
typename TTypes<T, NDIMS>::Tensor out);
};
// Converts a tensor from:
// [batch, filters, <spatial>]
// to:
// [batch, <spatial>, filters]
template <typename Device, typename T, int NDIMS>
struct NCHWToNHWC {
void operator()(const Device& d, typename TTypes<T, NDIMS>::ConstTensor in,
typename TTypes<T, NDIMS>::Tensor out);
};
// Converts a tensor from:
// [dim0, dim1, dim2]
// to:
// [dim0, dim2, dim1]
template <typename Device, typename T, bool conjugate = false>
struct SwapDimension1And2InTensor3 {
void operator()(const Device& d, const T* in,
const gtl::ArraySlice<int64>& input_dims, T* out);
};
// Converts a tensor from:
// [dim0, dim1, dim2]
// to:
// [dim2, dim1, dim0]
template <typename Device, typename T, bool conjugate = false>
struct SwapDimension0And2InTensor3 {
void operator()(const Device& d, const T* in,
const gtl::ArraySlice<int64>& input_dims, T* out);
};
// Transforms back filter from OIHW to HWOI format to reverse effect of
// TransformFilter above.
// TODO(hinsu): Support reverse transformation from filter format OHWI as well.
template <typename Device, typename T, int NDIMS>
struct ReverseTransformFilter {
void operator()(const Device& d, typename TTypes<T, NDIMS>::ConstTensor in,
typename TTypes<T, NDIMS>::Tensor out);
};
} // namespace functor
template <class T>
class ConvAlgorithmMap;
template <>
class ConvAlgorithmMap<Eigen::ThreadPoolDevice> {};
} // namespace tensorflow
#endif // TENSORFLOW_CORE_KERNELS_CONV_2D_H_
@@ -0,0 +1,91 @@
/* Copyright 2016 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
// Functors for 3d convolution.
#ifndef TENSORFLOW_CORE_KERNELS_CONV_3D_H_
#define TENSORFLOW_CORE_KERNELS_CONV_3D_H_
#include "tensorflow/core/framework/tensor_types.h"
#include "tensorflow/core/kernels/eigen_backward_cuboid_convolutions.h"
#include "tensorflow/core/kernels/eigen_cuboid_convolution.h"
namespace tensorflow {
namespace functor {
// Applies a 3D convolution to a batch of multi-channel volumes.
template <typename Device, typename T>
struct CuboidConvolution;
// Backward input pass for the cuboid convolution.
template <typename Device, typename T>
struct CuboidConvolutionBackwardInput;
// Backward filter pass for the cuboid convolution.
template <typename Device, typename T>
struct CuboidConvolutionBackwardFilter;
typedef Eigen::ThreadPoolDevice CPUDevice;
template <typename T>
struct CuboidConvolution<CPUDevice, T> {
void operator()(const CPUDevice& d, typename TTypes<T, 5>::Tensor output,
typename TTypes<T, 5>::ConstTensor input,
typename TTypes<T, 5>::ConstTensor filter, int stride_planes,
int stride_rows, int stride_cols,
const Eigen::PaddingType& padding) {
output.device(d) = Eigen::CuboidConvolution(
input, filter, stride_planes, stride_rows, stride_cols, padding);
}
};
template <typename T>
struct CuboidConvolutionBackwardInput<CPUDevice, T> {
void operator()(const CPUDevice& d,
typename TTypes<T, 5>::Tensor input_backward,
typename TTypes<T, 5>::ConstTensor filter,
typename TTypes<T, 5>::ConstTensor output_backward,
int stride_planes, int stride_rows, int stride_cols) {
// Need to swap the order of plane/row/col strides when calling Eigen.
input_backward.device(d) = Eigen::CuboidConvolutionBackwardInput(
filter, output_backward,
input_backward.dimension(3), // input_planes
input_backward.dimension(2), // input_rows
input_backward.dimension(1), // input_cols
stride_cols, stride_rows, stride_planes);
}
};
template <typename T>
struct CuboidConvolutionBackwardFilter<CPUDevice, T> {
void operator()(const CPUDevice& d,
typename TTypes<T, 5>::Tensor filter_backward,
typename TTypes<T, 5>::ConstTensor input,
typename TTypes<T, 5>::ConstTensor output_backward,
int stride_planes, int stride_rows, int stride_cols) {
// Need to swap the order of plane/row/col strides when calling Eigen.
filter_backward.device(d) = Eigen::CuboidConvolutionBackwardKernel(
input, output_backward,
filter_backward.dimension(2), // kernel_planes
filter_backward.dimension(1), // kernel_rows
filter_backward.dimension(0), // kernel_cols
stride_cols, stride_rows, stride_planes);
}
};
} // namespace functor
} // namespace tensorflow
#endif // TENSORFLOW_CORE_KERNELS_CONV_3D_H_
@@ -0,0 +1,269 @@
/* Copyright 2016 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
// This is the common header for the input and filter backprop kernels.
//
// The operation to compute Conv2D gradients.
//
// To compute the gradients for Conv2D, we need three input tensors:
// input, filter, and backprop for output.
// And we need to compute two backprops: one for input and one for filter. We
// compute them in two different kernels.
//
// Both backprops can be computed as straightforward conv2d.
//
// Consider a case where the input is 3x3 and the filter is 2x1:
//
// INPUT = [ A B C ]
// [ D E F ]
// [ G H I ]
//
// where each "A", "B", etc is batch x in_depth
//
// FILTER = [ X Y ]
//
// where both "X" and "Y" are in_depth x out_depth
//
// With VALID padding, the output is 3x2:
//
// OUTPUT = [ a b ]
// [ c d ]
// [ e f ]
//
// where each "a", "b", etc is batch x out_depth
//
// So we have:
//
// a = A * X + B * Y
// b = B * X + C * Y
// c = D * X + E * Y
// d = E * X + F * Y
// e = G * X + H * Y
// f = H * X + I * Y
//
// So when we have backprops for the outputs (we denote them by
// a', b', ... ):
//
// The backprops for the input are:
//
// A' = a' * X^t
// B' = a' * Y^t + b' * X^t
// C' = b' * Y^t
// ...
//
// This is essentially computing a 2d conv of
//
// INPUT = [ 0 a' b' 0 ]
// [ 0 c' d' 0 ]
// [ 0 e' f' 0 ]
// and
//
// FILTER = [ Y^t X^t ]
//
// The backprops for the filter are:
//
// X' = A^t * a' + B^t * b' + D^t * c' + E^t * d' + G^t * e' + H^t * f'
// Y' = B^t * a' + C^t * b' + E^t + c' + F^t * d' + H^t * e' + I^t * f'
//
// This is essentially computing a 2d conv of
//
// INPUT = [ A^t B^t C^t ]
// [ D^t E^t F^t ]
// [ G^t H^t I^t ]
//
// and
//
// FILTER = [ a' b' ]
// [ c' d' ]
// [ e' f' ]
//
//
//////////////////////////////////////////////////////////
//
// With stride more than one, it's a bit more complicated (we will need to
// create holes to the backprop).
//
// Consider the case where
//
// INPUT = [ A B C D E ]
// [ F G H I J ]
// [ K L M N O ]
// and
//
// FILTER = [ X Y Z ]
//
// with stride 2.
//
// The output will be
//
// OUTPUT = [ a b ]
// [ c d ]
//
// where:
//
// a = A * X + B * Y + C * Z
// b = C * X + D * Y + E * Z
// c = K * X + L * Y + M * Z
// d = M * X + N * Y + O * Z
//
//
// To compute the backprop for INPUT, we need to convolve
//
// INPUT = [ 0 0 a' 0 b' 0 0 ]
// [ 0 0 0 0 0 0 0 ]
// [ 0 0 c' 0 d' 0 0 ]
//
// (notice the holes in INPUT)
//
// and
//
// FILTER = [ Z^t Y^t X^t ]
//
// with stride 1.
//
// To compute the backprop for FILTER, we need to convolve
//
// INPUT = [ A^t B^t C^t D^t E^t ]
// [ F^t G^t H^t I^t J^t ]
// [ K^t L^t M^t N^t O^t ]
// and
//
// FILTER = [ a' 0 b' ]
// [ 0 0 0 ]
// [ c' 0 d' ]
//
// (notice the holes in FILTER)
//
//
// with stride 1
//
//////////////////////////////////////////////////////////
//
//
// The case for SAME padding is in fact very similar to VALID -- we just
// need to pad the input tensor a bit when computing the filter_backprop.
#ifndef TENSORFLOW_CORE_KERNELS_CONV_GRAD_OPS_H_
#define TENSORFLOW_CORE_KERNELS_CONV_GRAD_OPS_H_
#include <vector>
#include "tensorflow/core/framework/tensor_shape.h"
#include "tensorflow/core/lib/core/stringpiece.h"
#include "tensorflow/core/util/padding.h"
#include "tensorflow/core/util/tensor_format.h"
namespace tensorflow {
// Forward declaration.
class OpKernelContext;
template <typename Device, typename T>
struct LaunchConv2DBackpropInputOp {
void operator()(OpKernelContext* ctx, bool use_cudnn, bool cudnn_use_autotune,
const Tensor& out_backprop, const Tensor& filter,
int row_dilation, int col_dilation, int row_stride,
int col_stride, const Padding& padding, Tensor* in_backprop,
TensorFormat data_format);
};
template <typename Device, typename T>
struct LaunchConv2DBackpropFilterOp {
void operator()(OpKernelContext* ctx, bool use_cudnn, bool cudnn_use_autotune,
const Tensor& out_backprop, const Tensor& input,
int row_dilation, int col_dilation, int row_stride,
int col_stride, const Padding& padding,
Tensor* filter_backprop, TensorFormat data_format);
};
#ifdef GOOGLE_CUDA
template <typename T>
struct LaunchConv2DBackpropInputOp<Eigen::GpuDevice, T> {
void operator()(OpKernelContext* ctx, bool use_cudnn, bool cudnn_use_autotune,
const Tensor& input, const Tensor& filter, int row_dilation,
int col_dilation, int row_stride, int col_stride,
const Padding& padding, Tensor* output,
TensorFormat data_format);
};
template <typename T>
struct LaunchConv2DBackpropFilterOp<Eigen::GpuDevice, T> {
void operator()(OpKernelContext* ctx, bool use_cudnn, bool cudnn_use_autotune,
const Tensor& out_backprop, const Tensor& input,
int row_dilation, int col_dilation, int row_stride,
int col_stride, const Padding& padding,
Tensor* filter_backprop, TensorFormat data_format);
};
#endif // GOOGLE_CUDA
// Information about a single spatial dimension for a convolution
// backpropagation.
struct ConvBackpropSpatialDimension {
int64 input_size;
int64 filter_size;
int64 output_size;
int64 stride;
int64 dilation;
int64 expanded_output_size;
// Number of padding elements to be added before/after this dimension of
// the input when computing Conv?DBackpropInput.
int64 pad_before, pad_after;
};
// Computed dimensions for a backwards convolution.
struct ConvBackpropDimensions {
// Information about each spatial dimension.
gtl::InlinedVector<ConvBackpropSpatialDimension, 3> spatial_dims;
// Batch size.
int64 batch_size;
// Input and output feature depth.
int64 in_depth, out_depth;
// Convenience access methods for spatial dimensions properties.
int64 input_size(int dim) const { return spatial_dims[dim].input_size; }
int64 filter_size(int dim) const { return spatial_dims[dim].filter_size; }
int64 output_size(int dim) const { return spatial_dims[dim].output_size; }
int64 stride(int dim) const { return spatial_dims[dim].stride; }
int64 dilation(int dim) const { return spatial_dims[dim].dilation; }
// Compute padding for the given spatial dimension.
int SpatialPadding(const Padding& padding, int dim) const;
};
// Common code between implementations of Conv?DBackpropInput and
// Conv?DBackpropFilter. Verifies that the dimensions all match, and computes
// sizes/padding for the spatial dimensions.
Status ConvBackpropComputeDimensions(StringPiece label, int num_spatial_dims,
const TensorShape& input_shape,
const TensorShape& filter_shape,
const TensorShape& out_backprop_shape,
const std::vector<int32>& strides,
Padding padding, TensorFormat data_format,
ConvBackpropDimensions* dims);
// The V2 version computes the same outputs with arbitrary dilation rate.
// TODO(b/67112639): Merge V2 versions and the original versions eventually.
Status ConvBackpropComputeDimensionsV2(
StringPiece label, int num_spatial_dims, const TensorShape& input_shape,
const TensorShape& filter_shape, const TensorShape& out_backprop_shape,
const gtl::ArraySlice<int32>& dilations, const std::vector<int32>& strides,
Padding padding, TensorFormat data_format, ConvBackpropDimensions* dims);
} // namespace tensorflow
#endif // TENSORFLOW_CORE_KERNELS_CONV_GRAD_OPS_H_
@@ -0,0 +1,115 @@
/* Copyright 2016 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
#ifndef TENSORFLOW_CORE_KERNELS_CONV_OPS_H_
#define TENSORFLOW_CORE_KERNELS_CONV_OPS_H_
#include "third_party/eigen3/unsupported/Eigen/CXX11/Tensor"
#include "tensorflow/core/framework/resource_mgr.h"
#include "tensorflow/core/platform/mem.h"
#include "tensorflow/core/util/tensor_format.h"
#if GOOGLE_CUDA
#include "tensorflow/core/kernels/conv_ops_gpu.h"
#include "tensorflow/core/platform/stream_executor.h"
#endif // GOOGLE_CUDA
namespace tensorflow {
// Forward declaration.
class OpKernelContext;
template <typename Device, typename T>
struct LaunchConv2DOp {
void operator()(OpKernelContext* ctx, bool use_cudnn, bool cudnn_use_autotune,
const Tensor& input, const Tensor& filter, int row_dilation,
int col_dilation, int row_stride, int col_stride,
const Padding& padding, Tensor* output,
TensorFormat data_format);
};
#ifdef GOOGLE_CUDA
template <typename T>
struct LaunchConv2DOp<Eigen::GpuDevice, T> {
void operator()(OpKernelContext* ctx, bool use_cudnn, bool cudnn_use_autotune,
const Tensor& input, const Tensor& filter, int row_dilation,
int col_dilation, int row_stride, int col_stride,
const Padding& padding, Tensor* output,
TensorFormat data_format);
};
#endif // GOOGLE_CUDA
// Used to keep track of persistent memory buffers used within the op.
// It uses malloc and free to avoid the time cost of initializing the memory.
template <class T, size_t size>
struct Im2ColBufferResource : public ResourceBase {
Im2ColBufferResource<T, size>() {
data = static_cast<T*>(port::Malloc(size * sizeof(T)));
}
~Im2ColBufferResource<T, size>() { port::Free(data); }
// This mutex ensures that only a single operation at a time is able to use
// the buffer memory held by this resource.
mutex mu;
T* data;
string DebugString() { return "Im2ColBufferResource"; }
};
// Convolution parameters specified by Op attributes.
struct Conv2DParameters {
std::vector<int32> dilations;
std::vector<int32> strides;
Padding padding;
TensorFormat data_format;
};
// Convolution dimensions inferred from parameters, input and filter tensors.
struct Conv2DDimensions {
int batch;
int input_rows;
int input_cols;
int in_depth;
int filter_rows;
int filter_cols;
int patch_depth;
int out_depth;
int stride_rows;
int stride_cols;
int dilation_rows;
int dilation_cols;
int64 out_rows;
int64 out_cols;
int64 pad_rows;
int64 pad_cols;
};
// Initializes and validates Conv2D parameters configured by OpKernel
// attributes.
Status InitConv2DParameters(const OpKernelConstruction* context,
Conv2DParameters* params);
// Computes and validates convolutions dimensions from Conv2D parameters. If
// parameters are valid, dimensions will be updated with derived convolution
// dimensions, otherwise error will be returned.
Status ComputeConv2DDimension(const Conv2DParameters& params,
const Tensor& input, const Tensor& filter,
Conv2DDimensions* dimensions);
} // namespace tensorflow
#endif // TENSORFLOW_CORE_KERNELS_CONV_OPS_H_
@@ -0,0 +1,205 @@
/* Copyright 2015 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
#ifndef TENSORFLOW_CORE_KERNELS_CONV_OPS_GPU_H_
#define TENSORFLOW_CORE_KERNELS_CONV_OPS_GPU_H_
#if GOOGLE_CUDA
#include <tuple>
#include <unordered_map>
#include "tensorflow/core/framework/op_kernel.h"
#include "tensorflow/core/kernels/gpu_utils.h"
#include "tensorflow/core/lib/gtl/inlined_vector.h"
#include "tensorflow/core/lib/hash/hash.h"
namespace tensorflow {
// Get the Cudnn workspace limit from the environment variable, which is in MB.
// Return the workspace memory limit in bytes. If no value is set, return the
// default value.
int64 GetCudnnWorkspaceLimit(const string& envvar_in_mb,
int64 default_value_in_bytes);
// A class to provide scratch-space allocator for Stream-Executor Cudnn
// callback. TensorFlow is responsible for releasing the temporary buffers after
// the kernel finishes.
class CudnnScratchAllocator : public se::ScratchAllocator {
public:
virtual ~CudnnScratchAllocator() {}
CudnnScratchAllocator(int64 memory_limit, OpKernelContext* context)
: memory_limit_(memory_limit), total_byte_size_(0), context_(context) {}
int64 GetMemoryLimitInBytes(se::Stream* stream) override {
return memory_limit_;
}
se::port::StatusOr<se::DeviceMemory<uint8>> AllocateBytes(
se::Stream* stream, int64 byte_size) override {
Tensor temporary_memory;
if (byte_size < 0) {
return se::port::Status{se::port::error::INVALID_ARGUMENT,
"Requested negative byte size!"};
}
if (byte_size > memory_limit_) {
return se::port::StatusOr<se::DeviceMemory<uint8>>();
}
AllocationAttributes allocation_attr;
allocation_attr.no_retry_on_failure = true;
Status allocation_status(context_->allocate_temp(
DT_UINT8, TensorShape({byte_size}), &temporary_memory,
AllocatorAttributes(), allocation_attr));
if (!allocation_status.ok()) {
return se::port::StatusOr<se::DeviceMemory<uint8>>();
}
// Hold the reference of the allocated tensors until the end of the
// allocator.
allocated_tensors_.push_back(temporary_memory);
total_byte_size_ += byte_size;
return se::port::StatusOr<se::DeviceMemory<uint8>>(
AsDeviceMemory(temporary_memory.flat<uint8>().data(),
temporary_memory.flat<uint8>().size()));
}
int64 TotalByteSize() { return total_byte_size_; }
private:
int64 memory_limit_;
int64 total_byte_size_;
OpKernelContext* context_;
std::vector<Tensor> allocated_tensors_;
};
// Encapsulate all the shape information that is used in both forward and
// backward conv operations.
class ConvParameters {
public:
using SpatialArray = gtl::InlinedVector<int64, 3>;
ConvParameters(int64 batch, int64 in_depths, const SpatialArray& in,
TensorFormat data_format, int64 out_depths,
const SpatialArray& filter, const SpatialArray& dilation,
const SpatialArray& stride, const SpatialArray& padding,
DataType dtype, int device_id)
: batch_(batch),
in_depths_(in_depths),
out_depths_(out_depths),
in_(in),
data_format_(data_format),
filter_(filter),
dilation_(dilation),
stride_(stride),
padding_(padding),
dtype_(dtype),
device_id_(device_id) {
hash_code_ = batch;
hash_code_ = Hash64Combine(hash_code_, in_depths);
for (int64 val : in) hash_code_ = Hash64Combine(hash_code_, val);
hash_code_ = Hash64Combine(hash_code_, data_format);
hash_code_ = Hash64Combine(hash_code_, out_depths);
for (int64 val : filter) hash_code_ = Hash64Combine(hash_code_, val);
for (int64 val : dilation) hash_code_ = Hash64Combine(hash_code_, val);
for (int64 val : stride) hash_code_ = Hash64Combine(hash_code_, val);
for (int64 val : padding) hash_code_ = Hash64Combine(hash_code_, val);
hash_code_ = Hash64Combine(hash_code_, dtype);
hash_code_ = Hash64Combine(hash_code_, device_id);
}
bool operator==(const ConvParameters& other) const {
return this->get_data_as_tuple() == other.get_data_as_tuple();
}
bool operator!=(const ConvParameters& other) const {
return !(*this == other);
}
uint64 hash() const { return hash_code_; }
string ToString() const {
// clang-format off
return strings::StrCat(
batch_, ", ", in_depths_, ", ",
"(", str_util::Join(in_, ", "), "), ",
::tensorflow::ToString(data_format_), ", ",
out_depths_, ", ",
"(", str_util::Join(filter_, ", "), "), ",
"(", str_util::Join(dilation_, ", "), "), ",
"(", str_util::Join(stride_, ", "), "), ",
"(", str_util::Join(padding_, ", "), "), ",
dtype_, ", ",
device_id_);
// clang-format on
}
// The purpose of this function is to disable winograd nonfused conv algorithm
// for certain input parameters so as to avoid a bug in cuDNNv5 and cuDNNv6.
template <typename T>
bool ShouldIncludeWinogradNonfusedAlgo(
se::StreamExecutor* stream_exec) const {
auto* dnn_support = stream_exec->AsDnn();
if (!dnn_support) {
return false;
}
// Skip this check for cuDNN 7 and newer.
auto version = dnn_support->GetVersion();
if (version.ok() && version.ValueOrDie().major_version() >= 7) {
return true;
}
return ShouldIncludeWinogradNonfusedAlgoPreCudnn7<T>();
}
protected:
using ParameterDataType =
std::tuple<int64, int64, SpatialArray, TensorFormat, int64, SpatialArray,
SpatialArray, SpatialArray, SpatialArray, DataType, int>;
ParameterDataType get_data_as_tuple() const {
return std::make_tuple(batch_, in_depths_, in_, data_format_, out_depths_,
filter_, dilation_, stride_, padding_, dtype_,
device_id_);
}
uint64 hash_code_;
private:
friend struct ConvParametersPeer; // For testing purposes.
template <typename T>
bool ShouldIncludeWinogradNonfusedAlgoPreCudnn7() const {
int64 total_size = 16 * std::ceil(batch_ / 16.0) *
std::max(in_depths_, out_depths_) * in_[0] * in_[1] *
sizeof(T);
int64 threshold = 1LL << 31;
if (total_size >= threshold) {
return false;
} else {
return true;
}
}
int64 batch_;
int64 in_depths_;
int64 out_depths_;
SpatialArray in_;
TensorFormat data_format_;
SpatialArray filter_;
SpatialArray dilation_;
SpatialArray stride_;
SpatialArray padding_;
DataType dtype_;
int device_id_;
};
typedef Eigen::GpuDevice GPUDevice;
} // namespace tensorflow
#endif // GOOGLE_CUDA
#endif // TENSORFLOW_CORE_KERNELS_CONV_OPS_GPU_H_
@@ -0,0 +1,71 @@
/* Copyright 2016 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
#ifndef TENSORFLOW_CORE_KERNELS_CROP_AND_RESIZE_OP_H_
#define TENSORFLOW_CORE_KERNELS_CROP_AND_RESIZE_OP_H_
#include "third_party/eigen3/unsupported/Eigen/CXX11/Tensor"
#include "tensorflow/core/framework/numeric_types.h"
#include "tensorflow/core/framework/op_kernel.h"
#include "tensorflow/core/framework/tensor_types.h"
namespace tensorflow {
namespace functor {
template <typename Device, typename T>
struct CropAndResize {
// We assume that the tensor sizes are correct.
bool operator()(const OpKernelContext* context,
typename TTypes<T, 4>::ConstTensor image,
typename TTypes<float, 2>::ConstTensor boxes,
typename TTypes<int32, 1>::ConstTensor box_ind,
string method_name, float extrapolation_value,
typename TTypes<float, 4>::Tensor crops);
};
template <typename Device, typename T>
struct CropAndResizeBackpropImage {
// We assume that the tensor sizes are correct.
bool operator()(const Device& d, typename TTypes<float, 4>::ConstTensor grads,
typename TTypes<float, 2>::ConstTensor boxes,
typename TTypes<int32, 1>::ConstTensor box_ind,
typename TTypes<T, 4>::Tensor grads_image,
const string& method_name);
};
template <typename Device, typename T>
struct CropAndResizeBackpropBoxes {
// We assume that the tensor sizes are correct.
bool operator()(const Device& d, typename TTypes<float, 4>::ConstTensor grads,
typename TTypes<T, 4>::ConstTensor image,
typename TTypes<float, 2>::ConstTensor boxes,
typename TTypes<int32, 1>::ConstTensor box_ind,
typename TTypes<float, 2>::Tensor grads_boxes);
};
template <typename Device>
struct CheckValidBoxIndexHelper {
// Checks if all values in box_index are in [0, batch).
void operator()(const Device& d,
typename TTypes<int32, 1>::ConstTensor box_index, int batch,
typename TTypes<bool, 0>::Tensor isvalid) {
isvalid.device(d) = ((box_index >= 0) && (box_index < batch)).all();
}
};
} // namespace functor
} // namespace tensorflow
#endif // TENSORFLOW_CORE_KERNELS_CROP_AND_RESIZE_OP_H_
@@ -0,0 +1,54 @@
/* Copyright 2015 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
#ifndef TENSORFLOW_CORE_KERNELS_CROSS_OP_H_
#define TENSORFLOW_CORE_KERNELS_CROSS_OP_H_
#include "third_party/eigen3/unsupported/Eigen/CXX11/Tensor"
#include "tensorflow/core/framework/tensor_shape.h"
#include "tensorflow/core/framework/tensor_types.h"
namespace tensorflow {
namespace functor {
template <typename Device, typename Type>
struct Cross {
void operator()(const Device &d,
typename TTypes<Type, 2>::ConstTensor in0_data,
typename TTypes<Type, 2>::ConstTensor in1_data,
typename TTypes<Type, 2>::Tensor output_data) {
auto s1 = output_data.template chip<1>(0);
auto s2 = output_data.template chip<1>(1);
auto s3 = output_data.template chip<1>(2);
auto u1 = in0_data.template chip<1>(0);
auto u2 = in0_data.template chip<1>(1);
auto u3 = in0_data.template chip<1>(2);
auto v1 = in1_data.template chip<1>(0);
auto v2 = in1_data.template chip<1>(1);
auto v3 = in1_data.template chip<1>(2);
s1.device(d) = u2 * v3 - u3 * v2;
s2.device(d) = u3 * v1 - u1 * v3;
s3.device(d) = u1 * v2 - u2 * v1;
}
};
} // namespace functor
} // namespace tensorflow
#endif // TENSORFLOW_CORE_KERNELS_CROSS_OP_H_
@@ -0,0 +1,120 @@
/* Copyright 2016 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
#ifndef TENSORFLOW_CORE_KERNELS_CUDA_DEVICE_ARRAY_H_
#define TENSORFLOW_CORE_KERNELS_CUDA_DEVICE_ARRAY_H_
#if GOOGLE_CUDA
#include "tensorflow/core/common_runtime/gpu/gpu_event_mgr.h"
#include "tensorflow/core/framework/op_kernel.h"
#include "tensorflow/core/kernels/cuda_device_array_gpu.h"
namespace tensorflow {
// Create an array of value on the host, to be sent to kernel using
// CudaDeviceArrayStruct.
//
// Usage:
// int size = ...;
// CudaDeviceArrayOnHost ptrs(context, size);
// OP_REQUIRES_OK(ptrs.Init());
// for (int i = 0; i < size; ++i) {
// ptrs.Set(i, ...);
// }
// OP_REQUIRES_OK(ptrs.Finalize());
// launchKernel(..., ptrs.data, ...);
//
// ValueType must be memcopyable.
template <typename ValueType, int MaxInlineValues = 8>
class CudaDeviceArrayOnHost {
public:
CudaDeviceArrayOnHost(OpKernelContext* context, int32 size)
: context_(context),
total_bytes_(static_cast<int64>(size) * sizeof(ValueType)) {
data_.size = size;
}
Status Init() {
if (inlined()) {
values_ = data_.inline_values;
return Status::OK();
}
// Out-of-line: allocate data that will be memcopied.
AllocatorAttributes attr;
attr.set_on_host(true);
attr.set_gpu_compatible(true);
TF_RETURN_IF_ERROR(
context_->allocate_temp(DT_INT8, TensorShape{total_bytes_},
&out_of_line_values_on_host_, attr));
values_ = reinterpret_cast<ValueType*>(
out_of_line_values_on_host_.flat<int8>().data());
return Status::OK();
}
void Set(int index, ValueType val) {
DCHECK(values_); // ensure Init was called.
DCHECK_LT(index, data_.size);
*(values_ + index) = val;
}
Status Finalize() {
if (inlined()) {
return Status::OK();
}
// Out-of-line - copy pointers to device.
auto stream = context_->op_device_context()->stream();
TensorReference tensor_ref(out_of_line_values_on_host_);
TF_RETURN_IF_ERROR(context_->allocate_temp(
DT_INT8, TensorShape{total_bytes_}, &out_of_line_values_on_gpu_));
se::DeviceMemoryBase output_values_base{
out_of_line_values_on_gpu_.flat<int8>().data(),
static_cast<uint64>(total_bytes_)};
stream->ThenMemcpy(&output_values_base,
out_of_line_values_on_host_.flat<int8>().data(),
total_bytes_);
context_->device()->tensorflow_gpu_device_info()->event_mgr->ThenExecute(
stream, [tensor_ref]() { tensor_ref.Unref(); });
data_.out_of_line_values = reinterpret_cast<ValueType*>(
out_of_line_values_on_gpu_.flat<int8>().data());
return Status::OK();
}
const CudaDeviceArrayStruct<ValueType, MaxInlineValues>& data() const {
// Ensure Finalize is called.
DCHECK(inlined() || out_of_line_values_on_gpu_.IsInitialized());
return data_;
}
private:
bool inlined() const { return data_.size <= MaxInlineValues; }
OpKernelContext* const context_;
const int64 total_bytes_; // total size of all pointers.
ValueType* values_ = nullptr;
CudaDeviceArrayStruct<ValueType, MaxInlineValues> data_;
Tensor out_of_line_values_on_host_;
Tensor out_of_line_values_on_gpu_;
TF_DISALLOW_COPY_AND_ASSIGN(CudaDeviceArrayOnHost);
};
} // namespace tensorflow
#endif // GOOGLE_CUDA
#endif // TENSORFLOW_CORE_KERNELS_CUDA_DEVICE_ARRAY_H_
@@ -0,0 +1,50 @@
/* Copyright 2016 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
// Contains structs and functions to be included in device code.
#ifndef TENSORFLOW_CORE_KERNELS_CUDA_DEVICE_ARRAY_GPU_H_
#define TENSORFLOW_CORE_KERNELS_CUDA_DEVICE_ARRAY_GPU_H_
#if GOOGLE_CUDA
namespace tensorflow {
static constexpr int kMaxInlineCudaPointers = 8;
// To decode on the device side, use GetCudaDeviceArrayOnDevice.
// To encode on the host side, use CudaDeviceArrayOnHost.
template <typename ValueType, int MaxInlineValues = 8>
struct CudaDeviceArrayStruct {
int32 size;
// used if size <= MaxInlineValues;
ValueType inline_values[MaxInlineValues];
ValueType* out_of_line_values = nullptr; // used if size > MaxInlineValues;
};
template <typename ValueType, int MaxInlineValues = 8>
EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE ValueType* GetCudaDeviceArrayOnDevice(
CudaDeviceArrayStruct<ValueType, MaxInlineValues>* data) {
if (data->size <= MaxInlineValues) {
return data->inline_values;
} else {
return data->out_of_line_values;
}
}
} // namespace tensorflow
#endif // GOOGLE_CUDA
#endif // TENSORFLOW_CORE_KERNELS_CUDA_DEVICE_ARRAY_GPU_H_
@@ -0,0 +1,70 @@
/* Copyright 2016 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
// Helper functions to run 3d pooling on GPU using CuDNN.
#ifndef TENSORFLOW_CORE_KERNELS_CUDNN_POOLING_GPU_H_
#define TENSORFLOW_CORE_KERNELS_CUDNN_POOLING_GPU_H_
#include <array>
#include "tensorflow/core/framework/op_kernel.h"
#if GOOGLE_CUDA
#include "tensorflow/core/platform/stream_executor.h"
#endif
#include "tensorflow/core/util/padding.h"
namespace tensorflow {
#if GOOGLE_CUDA
// Runs (avg/max)pooling on GPU.
// Dimension order for all array arguments is: x, y, z.
template <typename T>
class DnnPooling3dOp {
public:
static void Compute(OpKernelContext* context,
se::dnn::PoolingMode pooling_mode,
const std::array<int64, 3>& size,
const std::array<int64, 3>& stride,
const std::array<int64, 3>& padding,
TensorFormat data_format, const Tensor& tensor_in,
Tensor* output);
};
// Computes the gradient of (avg/max)pooling on GPU.
// Dimension order for all array arguments is: x, y, z.
template <typename T>
class DnnPooling3dGradOp {
public:
static void Compute(OpKernelContext* context,
se::dnn::PoolingMode pooling_mode,
const std::array<int64, 3>& window,
const std::array<int64, 3>& stride,
const std::array<int64, 3>& padding,
const std::array<int64, 3>& output_size,
TensorFormat data_format, const Tensor& out_backprop,
const TensorShape& tensor_in_shape,
const Tensor* tensor_in, const Tensor* tensor_out,
Tensor* input_backprop);
};
#endif
} // namespace tensorflow
#endif // TENSORFLOW_CORE_KERNELS_CUDNN_POOLING_GPU_H_
@@ -0,0 +1,61 @@
/* Copyright 2015 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
#ifndef TENSORFLOW_CORE_KERNELS_CWISE_OP_CLIP_H_
#define TENSORFLOW_CORE_KERNELS_CWISE_OP_CLIP_H_
#include "tensorflow/core/kernels/cwise_ops_common.h"
namespace tensorflow {
namespace functor {
// Unary functor for clip [Tensor, Scalar, Scalar]
template <typename Device, typename T>
struct UnaryClipOp {
void operator()(const Device &d, typename TTypes<T>::ConstFlat &in0_flat,
typename TTypes<T>::ConstFlat &in1_flat,
typename TTypes<T>::ConstFlat &in2_flat,
typename TTypes<T>::Flat &out_flat) const;
};
// Binary functor for clip [Tensor, Scalar, Tensor]
template <typename Device, typename T>
struct BinaryRightClipOp {
void operator()(const Device &d, typename TTypes<T>::ConstFlat &in0_flat,
typename TTypes<T>::ConstFlat &in1_flat,
typename TTypes<T>::ConstFlat &in2_flat,
typename TTypes<T>::Flat &out_flat) const;
};
// Binary functor for clip [Tensor, Tensor, Scalar]
template <typename Device, typename T>
struct BinaryLeftClipOp {
void operator()(const Device &d, typename TTypes<T>::ConstFlat &in0_flat,
typename TTypes<T>::ConstFlat &in1_flat,
typename TTypes<T>::ConstFlat &in2_flat,
typename TTypes<T>::Flat &out_flat) const;
};
// Ternary functor for clip [Tensor, Tensor, Tensor]
template <typename Device, typename T>
struct TernaryClipOp {
void operator()(const Device &d, typename TTypes<T>::ConstFlat &in0_flat,
typename TTypes<T>::ConstFlat &in1_flat,
typename TTypes<T>::ConstFlat &in2_flat,
typename TTypes<T>::Flat &out_flat) const;
};
} // namespace functor
} // namespace tensorflow
#endif // TENSORFLOW_CORE_KERNELS_CWISE_OP_CLIP_H_
File diff suppressed because it is too large. Load diff
@@ -0,0 +1,605 @@
/* Copyright 2015 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
#ifndef TENSORFLOW_CORE_KERNELS_CWISE_OPS_COMMON_H_
#define TENSORFLOW_CORE_KERNELS_CWISE_OPS_COMMON_H_
// See docs in ../ops/math_ops.cc.
#define EIGEN_USE_THREADS
#include "tensorflow/core/lib/bfloat16/bfloat16.h"
#ifdef TENSORFLOW_USE_SYCL
#include "tensorflow/core/kernels/cwise_ops_sycl_common.h"
#endif
#include "tensorflow/core/kernels/cwise_ops.h"
#include "tensorflow/core/kernels/cwise_ops_gradients.h"
#include "tensorflow/core/framework/op.h"
#include "tensorflow/core/framework/op_kernel.h"
#include "tensorflow/core/framework/tensor_types.h"
#include "tensorflow/core/framework/variant_op_registry.h"
#include "tensorflow/core/platform/logging.h"
#include "tensorflow/core/util/bcast.h"
namespace tensorflow {
typedef Eigen::ThreadPoolDevice CPUDevice;
typedef Eigen::GpuDevice GPUDevice;
#ifdef TENSORFLOW_USE_SYCL
typedef Eigen::SyclDevice SYCLDevice;
#endif
class BinaryOpShared : public OpKernel {
public:
explicit BinaryOpShared(OpKernelConstruction* ctx, DataType out, DataType in);
protected:
struct BinaryOpState {
// Sets up bcast with the shape of in0 and in1, ensures that the bcast
// is valid, and if so, set out, either by allocating a new buffer using
// ctx->output(...) or by creating an alias for an owned input buffer for
// in-place computation.
// Caller must check ctx->status() upon return for non-ok status.
// If ctx->status().ok() is true, then out is guaranteed to be allocated.
BinaryOpState(OpKernelContext* ctx);
const Tensor& in0;
const Tensor& in1;
BCast bcast;
Tensor* out = nullptr;
int64 out_num_elements;
int64 in0_num_elements;
int64 in1_num_elements;
int ndims;
};
void SetUnimplementedError(OpKernelContext* ctx);
void SetComputeError(OpKernelContext* ctx);
};
// Coefficient-wise binary operations:
// Device: E.g., CPUDevice, GPUDevice.
// Functor: defined in cwise_ops.h. E.g., functor::add.
template <typename Device, typename Functor>
class BinaryOp : public BinaryOpShared {
public:
typedef typename Functor::in_type Tin; // Input scalar data type.
typedef typename Functor::out_type Tout; // Output scalar data type.
explicit BinaryOp(OpKernelConstruction* ctx)
: BinaryOpShared(ctx, DataTypeToEnum<Tout>::v(),
DataTypeToEnum<Tin>::v()) {}
void Compute(OpKernelContext* ctx) override {
// 'state': Shared helper not dependent on T to reduce code size
BinaryOpState state(ctx);
if (!ctx->status().ok()) return;
Tensor* out = state.out;
BCast* bcast = &state.bcast;
auto& in0 = state.in0;
auto& in1 = state.in1;
if (state.out_num_elements == 0) {
return;
}
const int ndims = state.ndims;
const Device& eigen_device = ctx->eigen_device<Device>();
bool error = false;
bool* const error_ptr = Functor::has_errors ? &error : nullptr;
if (ndims <= 1) {
auto out_flat = out->flat<Tout>();
if (state.in1_num_elements == 1) {
// tensor op scalar
functor::BinaryFunctor<Device, Functor, 1>().Right(
eigen_device, out_flat, in0.template flat<Tin>(),
in1.template scalar<Tin>(), error_ptr);
} else if (state.in0_num_elements == 1) {
// scalar op tensor
functor::BinaryFunctor<Device, Functor, 1>().Left(
eigen_device, out_flat, in0.template scalar<Tin>(),
in1.template flat<Tin>(), error_ptr);
} else {
functor::BinaryFunctor<Device, Functor, 1>()(
eigen_device, out_flat, in0.template flat<Tin>(),
in1.template flat<Tin>(), error_ptr);
}
} else if (ndims == 2) {
functor::BinaryFunctor<Device, Functor, 2>().BCast(
eigen_device, out->shaped<Tout, 2>(bcast->result_shape()),
in0.template shaped<Tin, 2>(bcast->x_reshape()),
BCast::ToIndexArray<2>(bcast->x_bcast()),
in1.template shaped<Tin, 2>(bcast->y_reshape()),
BCast::ToIndexArray<2>(bcast->y_bcast()), error_ptr);
} else if (ndims == 3) {
functor::BinaryFunctor<Device, Functor, 3>().BCast(
eigen_device, out->shaped<Tout, 3>(bcast->result_shape()),
in0.template shaped<Tin, 3>(bcast->x_reshape()),
BCast::ToIndexArray<3>(bcast->x_bcast()),
in1.template shaped<Tin, 3>(bcast->y_reshape()),
BCast::ToIndexArray<3>(bcast->y_bcast()), error_ptr);
} else if (ndims == 4) {
functor::BinaryFunctor<Device, Functor, 4>().BCast(
eigen_device, out->shaped<Tout, 4>(bcast->result_shape()),
in0.template shaped<Tin, 4>(bcast->x_reshape()),
BCast::ToIndexArray<4>(bcast->x_bcast()),
in1.template shaped<Tin, 4>(bcast->y_reshape()),
BCast::ToIndexArray<4>(bcast->y_bcast()), error_ptr);
} else if (ndims == 5) {
functor::BinaryFunctor<Device, Functor, 5>().BCast(
eigen_device, out->shaped<Tout, 5>(bcast->result_shape()),
in0.template shaped<Tin, 5>(bcast->x_reshape()),
BCast::ToIndexArray<5>(bcast->x_bcast()),
in1.template shaped<Tin, 5>(bcast->y_reshape()),
BCast::ToIndexArray<5>(bcast->y_bcast()), error_ptr);
} else {
SetUnimplementedError(ctx);
}
if (Functor::has_errors && error) {
SetComputeError(ctx);
}
}
};
template <typename Device, typename T>
class ApproximateEqualOp : public OpKernel {
public:
explicit ApproximateEqualOp(OpKernelConstruction* context)
: OpKernel(context) {
float tolerance;
OP_REQUIRES_OK(context, context->GetAttr("tolerance", &tolerance));
tolerance_ = T(tolerance);
}
void Compute(OpKernelContext* context) override {
const Tensor& x_input = context->input(0);
const Tensor& y_input = context->input(1);
OP_REQUIRES(
context, x_input.shape() == y_input.shape(),
errors::InvalidArgument("x and y must be of the same shape. ",
"x shape: ", x_input.shape().DebugString(),
". y shape: ", y_input.shape().DebugString()));
Tensor* z_output = nullptr;
OP_REQUIRES_OK(context,
context->allocate_output(0, x_input.shape(), &z_output));
const Device& d = context->eigen_device<Device>();
typename TTypes<T>::ConstFlat x(x_input.flat<T>());
typename TTypes<T>::ConstFlat y(y_input.flat<T>());
typename TTypes<bool>::Flat z(z_output->flat<bool>());
functor::ApproximateEqual<Device, T>()(d, x, y, tolerance_, z);
}
private:
T tolerance_;
};
// Basic coefficient-wise binary operations that are known to not require
// any broadcasting. This is the case for example of the gradients of
// unary operations.
// Device: E.g., CPUDevice, GPUDevice.
// Functor: defined above. E.g., functor::tanh_grad.
template <typename Device, typename Functor>
class SimpleBinaryOp : public OpKernel {
public:
typedef typename Functor::in_type Tin; // Input scalar data type.
typedef typename Functor::out_type Tout; // Output scalar data type.
explicit SimpleBinaryOp(OpKernelConstruction* ctx) : OpKernel(ctx) {}
void Compute(OpKernelContext* ctx) override {
const Tensor& in0 = ctx->input(0);
const Tensor& in1 = ctx->input(1);
auto in0_flat = in0.flat<Tin>();
auto in1_flat = in1.flat<Tin>();
const Device& eigen_device = ctx->eigen_device<Device>();
Tensor* out = nullptr;
if (std::is_same<Tin, Tout>::value) {
OP_REQUIRES_OK(ctx, ctx->forward_input_or_allocate_output(
{0, 1}, 0, in0.shape(), &out));
} else {
OP_REQUIRES_OK(ctx, ctx->allocate_output(0, in0.shape(), &out));
}
auto out_flat = out->flat<Tout>();
functor::SimpleBinaryFunctor<Device, Functor>()(eigen_device, out_flat,
in0_flat, in1_flat);
}
};
// Coefficient-wise unary operations:
// Device: E.g., CPUDevice, GPUDevice.
// Functor: defined in cwise_ops.h. E.g., functor::sqrt.
template <typename Device, typename Functor>
class UnaryOp : public OpKernel {
public:
typedef typename Functor::in_type Tin; // Input scalar data type.
typedef typename Functor::out_type Tout; // Output scalar data type.
// Tin may be different from Tout. E.g., abs: complex64 -> float
explicit UnaryOp(OpKernelConstruction* ctx) : OpKernel(ctx) {
auto in = DataTypeToEnum<Tin>::v();
auto out = DataTypeToEnum<Tout>::v();
OP_REQUIRES_OK(ctx, ctx->MatchSignature({in}, {out}));
}
void Compute(OpKernelContext* ctx) override {
const Tensor& inp = ctx->input(0);
Tensor* out = nullptr;
if (std::is_same<Tin, Tout>::value) {
OP_REQUIRES_OK(ctx, ctx->forward_input_or_allocate_output(
{0}, 0, inp.shape(), &out));
} else {
OP_REQUIRES_OK(ctx, ctx->allocate_output(0, inp.shape(), &out));
}
functor::UnaryFunctor<Device, Functor>()(
ctx->eigen_device<Device>(), out->flat<Tout>(), inp.flat<Tin>());
}
};
template <typename Device, VariantUnaryOp OpEnum>
class UnaryVariantOp : public OpKernel {
public:
explicit UnaryVariantOp(OpKernelConstruction* ctx) : OpKernel(ctx) {}
void Compute(OpKernelContext* ctx) override {
const Tensor& inp = ctx->input(0);
OP_REQUIRES(
ctx, TensorShapeUtils::IsScalar(inp.shape()),
errors::InvalidArgument("Non-scalar variants are not supported."));
const Variant& v = inp.scalar<Variant>()();
Variant v_out;
OP_REQUIRES_OK(ctx, UnaryOpVariant<Device>(ctx, OpEnum, v, &v_out));
Tensor out(cpu_allocator(), DT_VARIANT, TensorShape());
out.scalar<Variant>()() = std::move(v_out);
ctx->set_output(0, std::move(out));
}
};
namespace functor {
template <typename D, typename Out, typename Rhs>
void Assign(const D& d, Out out, Rhs rhs) {
out.device(d) = rhs;
}
// Partial specialization of BinaryFunctor<Device=CPUDevice, Functor, NDIMS>
// for functors with with no error checking.
template <typename Functor, int NDIMS>
struct BinaryFunctor<CPUDevice, Functor, NDIMS, false> {
void operator()(const CPUDevice& d, typename Functor::tout_type out,
typename Functor::tin_type in0,
typename Functor::tin_type in1, bool* error) {
Assign(d, out, in0.binaryExpr(in1, typename Functor::func()));
}
void Left(const CPUDevice& d, typename Functor::tout_type out,
typename Functor::tscalar_type scalar,
typename Functor::tin_type in, bool* error) {
typedef typename Functor::out_type Tout;
typedef typename Functor::in_type Tin;
typedef typename Functor::func Binary;
typedef typename Eigen::internal::scalar_left<Tout, Tin, Binary> Unary;
Assign(d, out, in.unaryExpr(Unary(scalar.data())));
}
void Right(const CPUDevice& d, typename Functor::tout_type out,
typename Functor::tin_type in,
typename Functor::tscalar_type scalar, bool* error) {
typedef typename Functor::out_type Tout;
typedef typename Functor::in_type Tin;
typedef typename Functor::func Binary;
typedef typename Eigen::internal::scalar_right<Tout, Tin, Binary> Unary;
Assign(d, out, in.unaryExpr(Unary(scalar.data())));
}
void BCast(const CPUDevice& dev,
typename TTypes<typename Functor::out_type, NDIMS>::Tensor out,
typename TTypes<typename Functor::in_type, NDIMS>::ConstTensor in0,
typename Eigen::array<Eigen::DenseIndex, NDIMS> bcast0,
typename TTypes<typename Functor::in_type, NDIMS>::ConstTensor in1,
typename Eigen::array<Eigen::DenseIndex, NDIMS> bcast1,
bool* error) {
typename Functor::func func;
if (AllOne<NDIMS>(bcast0) && AllOne<NDIMS>(bcast1)) {
Assign(dev, out, in0.binaryExpr(in1, func));
} else if (AllOne<NDIMS>(bcast0)) {
auto rhs = in1.broadcast(bcast1);
Assign(dev, out, in0.binaryExpr(rhs, func));
} else if (AllOne<NDIMS>(bcast1)) {
auto lhs = in0.broadcast(bcast0);
Assign(dev, out, lhs.binaryExpr(in1, func));
} else {
auto lhs = in0.broadcast(bcast0);
auto rhs = in1.broadcast(bcast1);
Assign(dev, out, lhs.binaryExpr(rhs, func));
}
}
};
// Partial specialization of BinaryFunctor<Device=CPUDevice, Functor, 2>
// for functors with with no error checking.
template <typename Functor>
struct BinaryFunctor<CPUDevice, Functor, 2, false> {
enum { NDIMS = 2 };
void operator()(const CPUDevice& d, typename Functor::tout_type out,
typename Functor::tin_type in0,
typename Functor::tin_type in1, bool* error) {
Assign(d, out, in0.binaryExpr(in1, typename Functor::func()));
}
void Left(const CPUDevice& d, typename Functor::tout_type out,
typename Functor::tscalar_type scalar,
typename Functor::tin_type in, bool* error) {
typedef typename Functor::out_type Tout;
typedef typename Functor::in_type Tin;
typedef typename Functor::func Binary;
typedef typename Eigen::internal::scalar_left<Tout, Tin, Binary> Unary;
Assign(d, out, in.unaryExpr(Unary(scalar.data())));
}
void Right(const CPUDevice& d, typename Functor::tout_type out,
typename Functor::tin_type in,
typename Functor::tscalar_type scalar, bool* error) {
typedef typename Functor::out_type Tout;
typedef typename Functor::in_type Tin;
typedef typename Functor::func Binary;
typedef typename Eigen::internal::scalar_right<Tout, Tin, Binary> Unary;
Assign(d, out, in.unaryExpr(Unary(scalar.data())));
}
#if !defined(EIGEN_HAS_INDEX_LIST)
inline Eigen::DSizes<int, 2> NByOne(int n) {
return Eigen::DSizes<int, 2>(n, 1);
}
inline Eigen::DSizes<int, 2> OneByM(int m) {
return Eigen::DSizes<int, 2>(1, m);
}
#else
inline Eigen::IndexList<int, Eigen::type2index<1>> NByOne(int n) {
Eigen::IndexList<int, Eigen::type2index<1>> ret;
ret.set(0, n);
return ret;
}
inline Eigen::IndexList<Eigen::type2index<1>, int> OneByM(int m) {
Eigen::IndexList<Eigen::type2index<1>, int> ret;
ret.set(1, m);
return ret;
}
#endif
void BCast(const CPUDevice& dev,
typename TTypes<typename Functor::out_type, NDIMS>::Tensor out,
typename TTypes<typename Functor::in_type, NDIMS>::ConstTensor in0,
typename Eigen::array<Eigen::DenseIndex, NDIMS> bcast0,
typename TTypes<typename Functor::in_type, NDIMS>::ConstTensor in1,
typename Eigen::array<Eigen::DenseIndex, NDIMS> bcast1,
bool* error) {
typedef typename Functor::in_type T;
typename Functor::func func;
if (Functor::use_bcast_optimization && use_bcast_optimization<T>::value) {
// Optimize for speed by using Eigen::type2index and avoid
// .broadcast() when we know its a no-op.
//
// Here, we need to handle 6 cases depending on how many "1"
// exist in in0 and in1's shapes (4 numbers in total). It's not
// possible that two shapes have more than 2 1s because those
// are simplified to NDIMS==1 case.
//
// Because this optimization increases the binary size for each
// Functor (+, -, *, /, <, <=, etc.), type and ndim combination.
// we only apply such optimization for selected ops/types/ndims.
//
// Because NDIMS, Functor::use_broadcast_optimization and
// use_broadcast_optimization<T> are compile-time constant, gcc
// does a decent job avoiding generating code when conditions
// are not met.
const int a = in0.dimension(0); // in0 is shape [a, b]
const int b = in0.dimension(1);
const int c = in1.dimension(0); // in1 is shape [c, d]
const int d = in1.dimension(1);
if ((a == 1) && (d == 1)) {
auto lhs = in0.reshape(OneByM(b)).broadcast(NByOne(c));
auto rhs = in1.reshape(NByOne(c)).broadcast(OneByM(b));
Assign(dev, out, lhs.binaryExpr(rhs, func));
return;
}
if ((b == 1) && (c == 1)) {
auto lhs = in0.reshape(NByOne(a)).broadcast(OneByM(d));
auto rhs = in1.reshape(OneByM(d)).broadcast(NByOne(a));
Assign(dev, out, lhs.binaryExpr(rhs, func));
return;
}
if (a == 1) {
auto lhs = in0.reshape(OneByM(b)).broadcast(NByOne(c));
auto rhs = in1;
Assign(dev, out, lhs.binaryExpr(rhs, func));
return;
}
if (b == 1) {
auto lhs = in0.reshape(NByOne(a)).broadcast(OneByM(d));
auto rhs = in1;
Assign(dev, out, lhs.binaryExpr(rhs, func));
return;
}
if (c == 1) {
auto lhs = in0;
auto rhs = in1.reshape(OneByM(d)).broadcast(NByOne(a));
Assign(dev, out, lhs.binaryExpr(rhs, func));
return;
}
if (d == 1) {
auto lhs = in0;
auto rhs = in1.reshape(NByOne(c)).broadcast(OneByM(b));
Assign(dev, out, lhs.binaryExpr(rhs, func));
return;
}
const bool bcast0_all_one = AllOne<NDIMS>(bcast0);
const bool bcast1_all_one = AllOne<NDIMS>(bcast1);
if (bcast0_all_one && !bcast1_all_one) {
auto lhs = in0; // No need to do broadcast for in0
auto rhs = in1.broadcast(bcast1);
Assign(dev, out, lhs.binaryExpr(rhs, func));
return;
}
if (!bcast0_all_one && bcast1_all_one) {
auto lhs = in0.broadcast(bcast0);
auto rhs = in1; // No need to do broadcast for in1
Assign(dev, out, lhs.binaryExpr(rhs, func));
return;
}
}
// Fallback path. Always works and probably slower.
auto lhs = in0.broadcast(bcast0);
auto rhs = in1.broadcast(bcast1);
Assign(dev, out, lhs.binaryExpr(rhs, func));
}
};
// Version of BinaryFunctor with error handling.
template <typename Functor, int NDIMS>
struct BinaryFunctor<CPUDevice, Functor, NDIMS, true> {
void operator()(const CPUDevice& d, typename Functor::tout_type out,
typename Functor::tin_type in0,
typename Functor::tin_type in1, bool* error) {
Assign(d, out, in0.binaryExpr(in1, typename Functor::func(error)));
}
void Left(const CPUDevice& d, typename Functor::tout_type out,
typename Functor::tscalar_type scalar,
typename Functor::tin_type in, bool* error) {
typedef typename Functor::out_type Tout;
typedef typename Functor::in_type Tin;
typedef typename Functor::func Binary;
typedef typename Eigen::internal::scalar_left<Tout, Tin, Binary> Unary;
Assign(d, out, in.unaryExpr(Unary(scalar.data(), error)));
}
void Right(const CPUDevice& d, typename Functor::tout_type out,
typename Functor::tin_type in,
typename Functor::tscalar_type scalar, bool* error) {
typedef typename Functor::out_type Tout;
typedef typename Functor::in_type Tin;
typedef typename Functor::func Binary;
typedef typename Eigen::internal::scalar_right<Tout, Tin, Binary> Unary;
Assign(d, out, in.unaryExpr(Unary(scalar.data(), error)));
}
void BCast(const CPUDevice& dev,
typename TTypes<typename Functor::out_type, NDIMS>::Tensor out,
typename TTypes<typename Functor::in_type, NDIMS>::ConstTensor in0,
typename Eigen::array<Eigen::DenseIndex, NDIMS> bcast0,
typename TTypes<typename Functor::in_type, NDIMS>::ConstTensor in1,
typename Eigen::array<Eigen::DenseIndex, NDIMS> bcast1,
bool* error) {
typename Functor::func func(error);
auto lhs = in0.broadcast(bcast0);
auto rhs = in1.broadcast(bcast1);
Assign(dev, out, lhs.binaryExpr(rhs, func));
}
};
// Partial specialization of UnaryFunctor<Device=CPUDevice, Functor>.
template <typename Functor>
struct UnaryFunctor<CPUDevice, Functor> {
void operator()(const CPUDevice& d, typename Functor::tout_type out,
typename Functor::tin_type in) {
Assign(d, out, in.unaryExpr(typename Functor::func()));
}
};
// Partial specialization of ApproximateEqual<Device=CPUDevice, T>.
template <typename T>
struct ApproximateEqual<CPUDevice, T> {
void operator()(const CPUDevice& d, typename TTypes<T>::ConstFlat x,
typename TTypes<T>::ConstFlat y, T tolerance,
typename TTypes<bool>::Flat z) {
auto diff = x - y;
z.device(d) = diff.abs() <= tolerance;
}
};
} // end namespace functor
#define REGISTER(OP, D, N, F, T) \
REGISTER_KERNEL_BUILDER(Name(N).Device(DEVICE_##D).TypeConstraint<T>("T"), \
OP<D##Device, F<T>>);
#define REGISTER_VARIANT(OP, D, N, ENUM) \
REGISTER_KERNEL_BUILDER( \
Name(N).Device(DEVICE_##D).TypeConstraint<Variant>("T"), \
OP<D##Device, ENUM>);
// Macros to register kernels for multiple types (T0, T1, etc.) on
// device type "D" (CPU or GPU) for operation "N" (e.g., sqrt) using
// the functor "F" (e.g., functor::sqrt).
#if defined(__ANDROID_TYPES_SLIM__)
// Note that __ANDROID_TYPES_SLIM__ is also checked in the cwise_ops*.cc files.
// Normally Android TensorFlow is built with a reduced number of types (float).
// Override on the command-line using "--copt=-D__ANDROID_TYPES_FULL__"
// to generate a library with full type support with a consequent increase in
// code size.
#define REGISTER2(OP, D, N, F, T0, T1) REGISTER(OP, D, N, F, T0)
#define REGISTER3(OP, D, N, F, T0, T1, T2) REGISTER(OP, D, N, F, T0)
#define REGISTER4(OP, D, N, F, T0, T1, T2, T3) REGISTER(OP, D, N, F, T0)
#define REGISTER5(OP, D, N, F, T0, T1, T2, T3, T4) REGISTER(OP, D, N, F, T0)
#define REGISTER6(OP, D, N, F, T0, T1, T2, T3, T4, T5) REGISTER(OP, D, N, F, T0)
#define REGISTER7(OP, D, N, F, T0, T1, T2, T3, T4, T5, T6) \
REGISTER(OP, D, N, F, T0)
#define REGISTER8(OP, D, N, F, T0, T1, T2, T3, T4, T5, T6, T7) \
REGISTER(OP, D, N, F, T0)
#define REGISTER9(OP, D, N, F, T0, T1, T2, T3, T4, T5, T6, T7, T8) \
REGISTER(OP, D, N, F, T0)
#else // !defined(__ANDROID_TYPES_SLIM__)
#define REGISTER2(OP, D, N, F, T0, T1) \
REGISTER(OP, D, N, F, T0) \
REGISTER(OP, D, N, F, T1)
#define REGISTER3(OP, D, N, F, T0, T1, T2) \
REGISTER2(OP, D, N, F, T0, T1) \
REGISTER(OP, D, N, F, T2)
#define REGISTER4(OP, D, N, F, T0, T1, T2, T3) \
REGISTER2(OP, D, N, F, T0, T1) \
REGISTER2(OP, D, N, F, T2, T3)
#define REGISTER5(OP, D, N, F, T0, T1, T2, T3, T4) \
REGISTER3(OP, D, N, F, T0, T1, T2) \
REGISTER2(OP, D, N, F, T3, T4)
#define REGISTER6(OP, D, N, F, T0, T1, T2, T3, T4, T5) \
REGISTER3(OP, D, N, F, T0, T1, T2) \
REGISTER3(OP, D, N, F, T3, T4, T5)
#define REGISTER7(OP, D, N, F, T0, T1, T2, T3, T4, T5, T6) \
REGISTER4(OP, D, N, F, T0, T1, T2, T3) \
REGISTER3(OP, D, N, F, T4, T5, T6)
#define REGISTER8(OP, D, N, F, T0, T1, T2, T3, T4, T5, T6, T7) \
REGISTER4(OP, D, N, F, T0, T1, T2, T3) \
REGISTER4(OP, D, N, F, T4, T5, T6, T7)
#define REGISTER9(OP, D, N, F, T0, T1, T2, T3, T4, T5, T6, T7, T8) \
REGISTER5(OP, D, N, F, T0, T1, T2, T3, T4) \
REGISTER4(OP, D, N, F, T5, T6, T7, T8)
// Instead of adding REGISTER10, etc., shard the .cc files - see
// cwise_op_equal_to_*.cc for an example.
#endif // defined(__ANDROID_TYPES_SLIM__)
} // end namespace tensorflow
#endif // TENSORFLOW_CORE_KERNELS_CWISE_OPS_COMMON_H_
@@ -0,0 +1,211 @@
/* Copyright 2015 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
#ifndef TENSORFLOW_CORE_KERNELS_CWISE_OPS_GRADIENTS_H_
#define TENSORFLOW_CORE_KERNELS_CWISE_OPS_GRADIENTS_H_
#define EIGEN_USE_THREADS
#include "third_party/eigen3/unsupported/Eigen/CXX11/Tensor"
#include "tensorflow/core/kernels/cwise_ops.h"
namespace Eigen {
namespace internal {
// Gradient for the tanh function
template <typename T>
struct scalar_tanh_gradient_op {
EIGEN_EMPTY_STRUCT_CTOR(scalar_tanh_gradient_op)
EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const T
operator()(const T& output, const T& output_gradient) const {
return output_gradient * (T(1) - output * output);
}
template <typename Packet>
EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const Packet
packetOp(const Packet& output, const Packet& output_gradient) const {
return pmul(output_gradient,
psub(pset1<Packet>(T(1)), pmul(output, output)));
}
};
template <typename T>
struct functor_traits<scalar_tanh_gradient_op<T>> {
enum {
Cost = NumTraits<T>::AddCost + 2 * NumTraits<T>::MulCost,
PacketAccess = packet_traits<T>::HasSub && packet_traits<T>::HasMul,
};
};
// Gradient for the sigmoid function
template <typename T>
struct scalar_sigmoid_gradient_op {
EIGEN_EMPTY_STRUCT_CTOR(scalar_sigmoid_gradient_op)
EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const T
operator()(const T& output, const T& output_gradient) const {
return output_gradient * output * (T(1) - output);
}
template <typename Packet>
EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const Packet
packetOp(const Packet& output, const Packet& output_gradient) const {
return pmul(output_gradient,
pmul(output, psub(pset1<Packet>(T(1)), output)));
}
};
template <typename T>
struct functor_traits<scalar_sigmoid_gradient_op<T>> {
enum {
Cost = NumTraits<T>::AddCost + 2 * NumTraits<T>::MulCost,
PacketAccess = packet_traits<T>::HasSub && packet_traits<T>::HasMul,
};
};
// Gradient for the inverse function
template <typename T>
struct scalar_inverse_gradient_op {
EIGEN_EMPTY_STRUCT_CTOR(scalar_inverse_gradient_op)
EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const T
operator()(const T& output, const T& output_gradient) const {
const T out_conj = numext::conj(output);
return -output_gradient * out_conj * out_conj;
}
template <typename Packet>
EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const Packet
packetOp(const Packet& output, const Packet& output_gradient) const {
const Packet out_conj = pconj(output);
return pnegate(pmul(output_gradient, pmul(out_conj, out_conj)));
}
};
template <typename T>
struct functor_traits<scalar_inverse_gradient_op<T>> {
enum {
Cost = NumTraits<T>::AddCost + 2 * NumTraits<T>::MulCost,
PacketAccess = packet_traits<T>::HasMul,
};
};
// Gradient for the sqrt function
template <typename T>
struct scalar_sqrt_gradient_op {
EIGEN_EMPTY_STRUCT_CTOR(scalar_sqrt_gradient_op)
EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const T
operator()(const T& output, const T& output_gradient) const {
const T out_conj = numext::conj(output);
return static_cast<T>(0.5) * output_gradient / out_conj;
}
template <typename Packet>
EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const Packet
packetOp(const Packet& output, const Packet& output_gradient) const {
const Packet const_half = pset1<Packet>(static_cast<T>(0.5));
const Packet out_conj = pconj(output);
return pdiv(pmul(const_half, output_gradient), out_conj);
}
};
template <typename T>
struct functor_traits<scalar_sqrt_gradient_op<T>> {
enum {
PacketAccess = packet_traits<T>::HasMul & packet_traits<T>::HasDiv,
Cost = NumTraits<T>::MulCost + scalar_div_cost<T, PacketAccess>::value,
};
};
// Gradient for the rsqrt function
template <typename T>
struct scalar_rsqrt_gradient_op {
EIGEN_EMPTY_STRUCT_CTOR(scalar_rsqrt_gradient_op)
EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const T
operator()(const T& output, const T& output_gradient) const {
const T out_conj = numext::conj(output);
return static_cast<T>(-0.5) * (output_gradient * out_conj) *
(out_conj * out_conj);
}
template <typename Packet>
EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const Packet
packetOp(const Packet& output, const Packet& output_gradient) const {
const Packet const_half = pset1<Packet>(static_cast<T>(-0.5));
const Packet out_conj = pconj(output);
return pmul(const_half, pmul(pmul(output_gradient, out_conj),
pmul(out_conj, out_conj)));
}
};
template <typename T>
struct functor_traits<scalar_rsqrt_gradient_op<T>> {
enum {
Cost = 4 * NumTraits<T>::MulCost,
PacketAccess = packet_traits<T>::HasMul,
};
};
} // end namespace internal
} // end namespace Eigen
namespace tensorflow {
namespace functor {
template <typename Device, typename Functor>
struct SimpleBinaryFunctor {
void operator()(const Device& d, typename Functor::tout_type out,
typename Functor::tin_type in0,
typename Functor::tin_type in1);
};
// Partial specialization of BinaryFunctor for CPU devices
typedef Eigen::ThreadPoolDevice CPUDevice;
template <typename Functor>
struct SimpleBinaryFunctor<CPUDevice, Functor> {
void operator()(const CPUDevice& d, typename Functor::tout_type out,
typename Functor::tin_type in0,
typename Functor::tin_type in1) {
out.device(d) = in0.binaryExpr(in1, typename Functor::func());
}
};
#ifdef TENSORFLOW_USE_SYCL
// Partial specialization of BinaryFunctor for SYCL devices
typedef Eigen::SyclDevice SYCLDevice;
template <typename Functor>
struct SimpleBinaryFunctor<SYCLDevice, Functor> {
void operator()(const SYCLDevice& d, typename Functor::tout_type out,
typename Functor::tin_type in0,
typename Functor::tin_type in1) {
out.device(d) = in0.binaryExpr(in1, typename Functor::func());
}
};
#endif // TENSORFLOW_USE_SYCL
template <typename T>
struct tanh_grad : base<T, Eigen::internal::scalar_tanh_gradient_op<T>> {};
template <typename T>
struct sigmoid_grad : base<T, Eigen::internal::scalar_sigmoid_gradient_op<T>> {
};
template <typename T>
struct inverse_grad : base<T, Eigen::internal::scalar_inverse_gradient_op<T>> {
};
template <typename T>
struct sqrt_grad : base<T, Eigen::internal::scalar_sqrt_gradient_op<T>> {};
template <typename T>
struct rsqrt_grad : base<T, Eigen::internal::scalar_rsqrt_gradient_op<T>> {};
template <typename T>
struct igamma_grad_a : base<T, Eigen::internal::scalar_igamma_der_a_op<T>> {};
} // end namespace functor
} // end namespace tensorflow
#endif // TENSORFLOW_CORE_KERNELS_CWISE_OPS_GRADIENTS_H_
@@ -0,0 +1,163 @@
/* Copyright 2016 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
#if !TENSORFLOW_USE_SYCL
#error This file must only be included when building TensorFlow with SYCL support
#endif
#ifndef TENSORFLOW_CORE_KERNELS_CWISE_OPS_SYCL_COMMON_H_
#define TENSORFLOW_CORE_KERNELS_CWISE_OPS_SYCL_COMMON_H_
#include "third_party/eigen3/unsupported/Eigen/CXX11/Tensor"
#include "tensorflow/core/framework/register_types.h"
#include "tensorflow/core/kernels/cwise_ops.h"
#include "tensorflow/core/platform/types.h"
namespace tensorflow {
namespace functor {
typedef Eigen::SyclDevice SYCLDevice;
template <typename OUT, typename RHS>
void Assign(const SYCLDevice& d, OUT out, RHS rhs) {
out.device(d) = rhs;
}
// Partial specialization of UnaryFunctor<Device=SYCLDevice, Functor>.
template <typename Functor>
struct UnaryFunctor<SYCLDevice, Functor> {
void operator()(const SYCLDevice& d, typename Functor::tout_type out,
typename Functor::tin_type in) {
To32Bit(out).device(d) = To32Bit(in).unaryExpr(typename Functor::func());
}
};
// Partial specialization of BinaryFunctor<Device=SYCLDevice, Functor>.
template <typename Functor, int NDIMS, bool has_errors>
struct BinaryFunctor<SYCLDevice, Functor, NDIMS, has_errors> {
void operator()(const SYCLDevice& d, typename Functor::tout_type out,
typename Functor::tin_type in0,
typename Functor::tin_type in1, bool* error) {
To32Bit(out).device(d) =
To32Bit(in0).binaryExpr(To32Bit(in1), typename Functor::func());
}
void Left(const SYCLDevice& d, typename Functor::tout_type out,
typename Functor::tscalar_type scalar,
typename Functor::tin_type in, bool* error) {
typedef typename Functor::func Binary;
constexpr int NumDims = Functor::tin_type::NumDimensions;
static_assert(NumDims == 1, "Unexpected size");
Eigen::Sizes<1> scalar_dim;
out.device(d) = scalar.reshape(scalar_dim)
.broadcast(in.dimensions())
.binaryExpr(in, Binary());
}
void Right(const SYCLDevice& d, typename Functor::tout_type out,
typename Functor::tin_type in,
typename Functor::tscalar_type scalar, bool* error) {
typedef typename Functor::func Binary;
constexpr int NumDims = Functor::tin_type::NumDimensions;
static_assert(NumDims == 1, "Unexpected size");
Eigen::Sizes<1> scalar_dim;
out.device(d) = in.binaryExpr(
scalar.reshape(scalar_dim).broadcast(in.dimensions()), Binary());
}
void BCast(const SYCLDevice& d,
typename TTypes<typename Functor::out_type, NDIMS>::Tensor out,
typename TTypes<typename Functor::in_type, NDIMS>::ConstTensor in0,
typename Eigen::array<Eigen::DenseIndex, NDIMS> bcast0,
typename TTypes<typename Functor::in_type, NDIMS>::ConstTensor in1,
typename Eigen::array<Eigen::DenseIndex, NDIMS> bcast1,
bool* error) {
typedef typename Functor::in_type T;
typename Functor::func func;
if ((NDIMS == 2) && Functor::use_bcast_optimization &&
use_bcast_optimization<T>::value) {
const bool bcast0_all_one = AllOne<NDIMS>(bcast0);
const bool bcast1_all_one = AllOne<NDIMS>(bcast1);
if (bcast0_all_one && !bcast1_all_one) {
To32Bit(out).device(d) =
To32Bit(in0).binaryExpr(To32Bit(in1).broadcast(bcast1), func);
return;
}
if (!bcast0_all_one && bcast1_all_one) {
To32Bit(out).device(d) =
To32Bit(in0).broadcast(bcast0).binaryExpr(To32Bit(in1), func);
return;
}
}
To32Bit(out).device(d) = To32Bit(in0).broadcast(bcast0).binaryExpr(
To32Bit(in1).broadcast(bcast1), func);
}
};
// Macros to explicitly instantiate kernels on GPU for multiple types
// (T0, T1, etc.) for UnaryFunctor (e.g., functor::sqrt).
#define DEFINE_UNARY1(F, T) template struct UnaryFunctor<SYCLDevice, F<T> >
#define DEFINE_UNARY2(F, T0, T1) \
DEFINE_UNARY1(F, T0); \
DEFINE_UNARY1(F, T1)
#define DEFINE_UNARY3(F, T0, T1, T2) \
DEFINE_UNARY2(F, T0, T1); \
DEFINE_UNARY1(F, T2)
#define DEFINE_UNARY4(F, T0, T1, T2, T3) \
DEFINE_UNARY2(F, T0, T1); \
DEFINE_UNARY2(F, T2, T3)
#define DEFINE_UNARY5(F, T0, T1, T2, T3, T4) \
DEFINE_UNARY2(F, T0, T1); \
DEFINE_UNARY3(F, T2, T3, T4)
// Macros to explicitly instantiate kernels on GPU for multiple types
// (T0, T1, etc.) for BinaryFunctor.
#define DEFINE_BINARY1(F, T) \
template struct BinaryFunctor<SYCLDevice, F<T>, 1>; \
template struct BinaryFunctor<SYCLDevice, F<T>, 2>; \
template struct BinaryFunctor<SYCLDevice, F<T>, 3>
#define DEFINE_BINARY2(F, T0, T1) \
DEFINE_BINARY1(F, T0); \
DEFINE_BINARY1(F, T1)
#define DEFINE_BINARY3(F, T0, T1, T2) \
DEFINE_BINARY2(F, T0, T1); \
DEFINE_BINARY1(F, T2)
#define DEFINE_BINARY4(F, T0, T1, T2, T3) \
DEFINE_BINARY2(F, T0, T1); \
DEFINE_BINARY2(F, T2, T3)
#define DEFINE_BINARY5(F, T0, T1, T2, T3, T4) \
DEFINE_BINARY2(F, T0, T1); \
DEFINE_BINARY3(F, T2, T3, T4)
#define DEFINE_BINARY6(F, T0, T1, T2, T3, T4, T5) \
DEFINE_BINARY3(F, T0, T1, T2); \
DEFINE_BINARY3(F, T3, T4, T5)
#define DEFINE_BINARY7(F, T0, T1, T2, T3, T4, T5, T6) \
DEFINE_BINARY3(F, T0, T1, T2); \
DEFINE_BINARY4(F, T3, T4, T5, T6)
#define DEFINE_BINARY8(F, T0, T1, T2, T3, T4, T5, T6, T7) \
DEFINE_BINARY4(F, T0, T1, T2, T3); \
DEFINE_BINARY4(F, T4, T5, T6, T7)
#define DEFINE_BINARY9(F, T0, T1, T2, T3, T4, T5, T6, T7, T8) \
DEFINE_BINARY4(F, T0, T1, T2, T3); \
DEFINE_BINARY5(F, T4, T5, T6, T7, T8)
#define DEFINE_BINARY10(F, T0, T1, T2, T3, T4, T5, T6, T7, T8, T9) \
DEFINE_BINARY5(F, T0, T1, T2, T3, T4); \
DEFINE_BINARY5(F, T5, T6, T7, T8, T9)
} // end namespace functor
} // end namespace tensorflow
#endif // TENSORFLOW_CORE_KERNELS_CWISE_OPS_SYCL_COMMON_H_
@@ -0,0 +1,147 @@
/* Copyright 2017 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
#ifndef TENSORFLOW_CORE_KERNELS_DATA_CAPTURED_FUNCTION_H_
#define TENSORFLOW_CORE_KERNELS_DATA_CAPTURED_FUNCTION_H_
#include <memory>
#include <vector>
#include "tensorflow/core/framework/function.h"
#include "tensorflow/core/framework/tensor.h"
#include "tensorflow/core/kernels/data/dataset.h"
#include "tensorflow/core/lib/core/status.h"
#include "tensorflow/core/lib/gtl/array_slice.h"
#include "tensorflow/core/lib/random/random.h"
#include "tensorflow/core/platform/macros.h"
namespace tensorflow {
class Device;
class OpKernelContext;
class ResourceMgr;
namespace data {
// A `CapturedFunction` encapsulates a TensorFlow function and all of
// the runtime support required to execute it.
//
// The `Dataset`-related classes use `CapturedFunction` to execute
// TensorFlow functions outside a the normal `OpKernel::Compute()`
// context.
class CapturedFunction {
public:
// Creates a new instance using a list of named attributes, fetching captured
// inputs from a context argument.
static Status Create(const NameAttrList& func, OpKernelContext* ctx,
const string& argument,
std::unique_ptr<CapturedFunction>* out_function);
// Creates a new instance using a list of named attributes, fetching captured
// inputs from a context argument.
//
// If `use_inter_op_parallelism` is false, the runtime may use an executor
// that is optimized for small functions.
static Status Create(const NameAttrList& func, OpKernelContext* ctx,
const string& argument, bool use_inter_op_parallelism,
std::unique_ptr<CapturedFunction>* out_function);
~CapturedFunction();
// Runs the "Captured function" using the given FLR and caches the lib and
// handle generated during instantiation. If Run is called with a different
// lib afterwards, generates an error. This method takes ownership of the
// tensors in `args`, in order to be able to deallocate them as early as
// possible. Use `RunWithBorrowedArgs()` if the caller needs to retain
// ownership of the `args`.
Status Run(IteratorContext* ctx, std::vector<Tensor>&& args,
std::vector<Tensor>* rets);
// Synchronously runs the captured function on the given `args`, and stores
// the results in `*rets`. Prefer to use `Run()` or `RunAsync()` when
// possible.
Status RunWithBorrowedArgs(IteratorContext* ctx,
const std::vector<Tensor>& args,
std::vector<Tensor>* rets);
// Explicitly instantiate this function for use in the given
// context. This method, and the context-less overload
// `RunInstantiated()` below can be useful for calling a captured
// function in cases where an `IteratorContext*` is not available
// (such as a destructor).
Status Instantiate(IteratorContext* ctx);
// Synchronously runs the captured function on the given `args`, and stores
// the results in `*rets`. Prefer to use `Run()` or `RunAsync()` when
// possible.
//
// REQUIRES: `this->Instantiate()` must have been called before this method.
Status RunInstantiated(const std::vector<Tensor>& args,
std::vector<Tensor>* rets);
// Asynchronously runs the captured function on the given `args`, stores
// the results in `*rets`, and calls the given `done` callback when the
// function returns. This method takes ownership of the tensors in `args`,
// in order to be able to deallocate them as early as possible.
void RunAsync(IteratorContext* ctx, std::vector<Tensor>&& args,
std::vector<Tensor>* rets,
FunctionLibraryRuntime::DoneCallback done,
const string& prefix);
// Returns the named list of function arguments.
const NameAttrList& func() { return func_; }
// Returns that additional captured inputs that will be passed to the function
// when `Run*()` is called.
const std::vector<Tensor>& captured_inputs() { return captured_inputs_; }
// Returns a step ID for use when running a `CapturedFunction`.
static int64 generate_step_id() {
// Choose a step ID that is guaranteed not to clash with any
// Session-generated step ID. DirectSession only generates
// non-negative step IDs (contiguous, starting from 0), and
// MasterSession generates 56-bit random step IDs whose MSB is
// always 0, so a negative random step ID should suffice.
return -std::abs(static_cast<int64>(random::New64()));
}
private:
CapturedFunction(const NameAttrList& func,
std::vector<Tensor> captured_inputs,
bool use_inter_op_parallelism);
Status GetHandle(IteratorContext* ctx,
FunctionLibraryRuntime::Handle* out_handle);
mutex mu_;
const NameAttrList func_;
FunctionLibraryRuntime* lib_ GUARDED_BY(mu_);
FunctionLibraryRuntime::Handle f_handle_ GUARDED_BY(mu_);
const std::vector<Tensor> captured_inputs_;
DataTypeSlice ret_types_;
std::function<void(std::function<void()>)> captured_runner_ = nullptr;
const bool use_inter_op_parallelism_;
TF_DISALLOW_COPY_AND_ASSIGN(CapturedFunction);
};
} // namespace data
// TODO(b/114112161): Remove these aliases when all users have moved over to the
// `tensorflow::data` namespace.
using data::CapturedFunction;
} // namespace tensorflow
#endif // TENSORFLOW_CORE_KERNELS_DATA_CAPTURED_FUNCTION_H_
@@ -0,0 +1,20 @@
/* Copyright 2017 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
#ifndef TENSORFLOW_CORE_KERNELS_DATA_DATASET_H_
#define TENSORFLOW_CORE_KERNELS_DATA_DATASET_H_
#include "tensorflow/core/framework/dataset.h"
#endif // TENSORFLOW_CORE_KERNELS_DATA_DATASET_H_
@@ -0,0 +1,43 @@
/* Copyright 2017 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
#ifndef TENSORFLOW_CORE_KERNELS_DATA_DATASET_UTILS_H_
#define TENSORFLOW_CORE_KERNELS_DATA_DATASET_UTILS_H_
#include "tensorflow/core/framework/tensor.h"
#include "tensorflow/core/kernels/data/captured_function.h"
#include "tensorflow/core/kernels/data/dataset.h"
namespace tensorflow {
namespace data {
Status MakeIteratorFromInputElement(
IteratorContext* ctx, const std::vector<Tensor>& input_element,
int64 thread_index, CapturedFunction* captured_func, StringPiece prefix,
std::unique_ptr<IteratorBase>* out_iterator);
// Returns Status::OK() if `expected` and `received` types match,
// errors::InvalidArgument otherwise.
Status VerifyTypesMatch(const DataTypeVector& expected,
const DataTypeVector& received);
// Returns Status::OK() if `expected` and `received` shapes are compatible,
// errors::InvalidArgument otherwise.
Status VerifyShapesCompatible(const std::vector<PartialTensorShape>& expected,
const std::vector<PartialTensorShape>& received);
} // namespace data
} // namespace tensorflow
#endif // TENSORFLOW_CORE_KERNELS_DATA_DATASET_UTILS_H_
@@ -0,0 +1,119 @@
/* Copyright 2017 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
#ifndef TENSORFLOW_CORE_KERNELS_DATA_EXPERIMENTAL_INDEXED_DATASET_H_
#define TENSORFLOW_CORE_KERNELS_DATA_EXPERIMENTAL_INDEXED_DATASET_H_
#include "tensorflow/core/framework/dataset.h"
#include "tensorflow/core/framework/op_kernel.h"
namespace tensorflow {
namespace data {
// TODO(saeta): Urgh, this is ugly.
class MaterializedIndexedDataset {
public:
virtual ~MaterializedIndexedDataset() = default;
// Retrieve the element at a given index. The output tensors are stored in
// out_tensors.
//
// If `index` is greater than `Size()`, tensorflow::errors::OutOfRangeError is
// returned.
//
// Get is thread-safe.
virtual Status Get(IteratorContext&& ctx, uint64 index,
std::vector<Tensor>* out_tensors) const = 0;
// Size determines the number of elements in this IndexedDataset.
//
// Size is thread-safe.
virtual Status Size(uint64* size) const = 0;
// Returns a vector of DataType values, representing the respective
// element types of each tuple component in the outputs of this dataset.
virtual const DataTypeVector& output_dtypes() const = 0;
// Returns a vector of tensor shapes, representing the respective
// (and possibly partially defined) shapes of each tuple component
// in the outputs of this dataset.
virtual const std::vector<PartialTensorShape>& output_shapes() const = 0;
};
// IndexedDataset represents a dataset that supports random access in addition
// to iterator-based sequential access.
//
// Note: IndexedDatasets are HIGHLY experimental at this time. Expect
// significant (backwards incompatible) changes!
class IndexedDataset : public DatasetBase {
public:
IndexedDataset(DatasetContext&& ctx) : DatasetBase(std::move(ctx)) {}
// Materialize (if necessary) the dataset, and return a pointer.
// TODO(saeta): Add in `IteratorContext* ctx` when materializing.
virtual Status MaterializeDataset(
std::shared_ptr<MaterializedIndexedDataset>* materialized) = 0;
};
// IndexedDatasetOpKernel abstracts away interfacing IndexedDatasets with the
// rest of the TensorFlow runtime.
//
// Most IndexedDataset's will be private members of classes inheriting from this
// class.
class IndexedDatasetOpKernel : public OpKernel {
public:
IndexedDatasetOpKernel(OpKernelConstruction* ctx) : OpKernel(ctx) {}
void Compute(OpKernelContext* ctx) final;
protected:
// Subclasses should implement this method. It will be called during Compute
// execution.
virtual void MakeIndexedDataset(OpKernelContext* ctx,
IndexedDataset** output) = 0;
template <typename T>
Status ParseScalarArgument(OpKernelContext* ctx,
const StringPiece& argument_name, T* output) {
const Tensor* argument_t;
TF_RETURN_IF_ERROR(ctx->input(argument_name, &argument_t));
if (!TensorShapeUtils::IsScalar(argument_t->shape())) {
return errors::InvalidArgument(argument_name, " must be a scalar");
}
*output = argument_t->scalar<T>()();
return Status::OK();
}
};
// Validates and extracts an `IndexedDataset` object from `tensor`.
//
// `tensor` must have been written by a call to
// `StoreIndexedDatasetInVariantTensor`
//
// The retrieved pointer isa borrowed reference to the dataset, which is owned
// by the tensor. The consumer must either acquire its own reference to the
// dataset by calling `(*out_dataset)->Ref()`, or ensure that `tensor` is not
// destroyed or mutated while the retrieved pointer is in use.
Status GetIndexedDatasetFromVariantTensor(const Tensor& tensor,
IndexedDataset** out_dataset);
// Stores an `IndexedDataset` object in `tensor.`
//
// The ownership of `dataset` is transferred to `tensor`.
Status StoreIndexedDatasetInVariantTensor(IndexedDataset* dataset,
Tensor* tensor);
} // namespace data
} // namespace tensorflow
#endif // TENSORFLOW_CORE_KERNELS_DATA_EXPERIMENTAL_INDEXED_DATASET_H_
@@ -0,0 +1,42 @@
/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
#ifndef TENSORFLOW_CORE_KERNELS_DATA_GENERATOR_DATASET_OP_H_
#define TENSORFLOW_CORE_KERNELS_DATA_GENERATOR_DATASET_OP_H_
#include "tensorflow/core/framework/dataset.h"
namespace tensorflow {
namespace data {
class GeneratorDatasetOp : public DatasetOpKernel {
public:
explicit GeneratorDatasetOp(OpKernelConstruction* ctx);
void MakeDataset(OpKernelContext* ctx, DatasetBase** output) override;
private:
class Dataset;
DataTypeVector output_types_;
std::vector<PartialTensorShape> output_shapes_;
NameAttrList init_func_;
NameAttrList next_func_;
NameAttrList finalize_func_;
};
} // namespace data
} // namespace tensorflow
#endif // TENSORFLOW_CORE_KERNELS_DATA_GENERATOR_DATASET_OP_H_
@@ -0,0 +1,149 @@
/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
#ifndef TENSORFLOW_CORE_KERNELS_DATA_ITERATOR_OPS_H_
#define TENSORFLOW_CORE_KERNELS_DATA_ITERATOR_OPS_H_
#include "tensorflow/core/common_runtime/function.h"
#include "tensorflow/core/framework/dataset.h"
#include "tensorflow/core/framework/op_kernel.h"
#include "tensorflow/core/kernels/ops_util.h"
namespace tensorflow {
namespace data {
class IteratorResource;
class IteratorHandleOp : public OpKernel {
public:
explicit IteratorHandleOp(OpKernelConstruction* ctx);
// The resource is deleted from the resource manager only when it is private
// to kernel. Ideally the resource should be deleted when it is no longer held
// by anyone, but it would break backward compatibility.
~IteratorHandleOp() override;
void Compute(OpKernelContext* context) override LOCKS_EXCLUDED(mu_);
private:
// During the first Compute(), resource is either created or looked up using
// shared_name. In the latter case, the resource found should be verified if
// it is compatible with this op's configuration. The verification may fail in
// cases such as two graphs asking queues of the same shared name to have
// inconsistent capacities.
Status VerifyResource(IteratorResource* resource);
template <typename To, typename From> // use like this: down_cast<T*>(foo);
static inline To down_cast(From* f) { // so we only accept pointers
static_assert(
(std::is_base_of<From, typename std::remove_pointer<To>::type>::value),
"target type not derived from source type");
// We skip the assert and hence the dynamic_cast if RTTI is disabled.
#if !defined(__GNUC__) || defined(__GXX_RTTI)
// Uses RTTI in dbg and fastbuild. asserts are disabled in opt builds.
assert(f == nullptr || dynamic_cast<To>(f) != nullptr);
#endif // !defined(__GNUC__) || defined(__GXX_RTTI)
return static_cast<To>(f);
}
FunctionLibraryRuntime* CreatePrivateFLR(
OpKernelContext* ctx, std::unique_ptr<DeviceMgr>* device_mgr,
std::unique_ptr<FunctionLibraryDefinition>* flib_def,
std::unique_ptr<ProcessFunctionLibraryRuntime>* pflr);
mutex mu_;
ContainerInfo cinfo_; // Written once under mu_ then constant afterwards.
IteratorResource* resource_ GUARDED_BY(mu_) = nullptr;
DataTypeVector output_dtypes_;
std::vector<PartialTensorShape> output_shapes_;
const int graph_def_version_;
string name_;
};
// Like IteratorHandleOp, but creates handles which are never shared, and does
// not hold a reference to these handles. The latter is important for eager
// execution, since OpKernel instances generally live as long as the program
// running them.
class AnonymousIteratorHandleOp : public OpKernel {
public:
explicit AnonymousIteratorHandleOp(OpKernelConstruction* context);
void Compute(OpKernelContext* context) override;
private:
// Coordinates Iterator unique name creation across AnonymousIteratorHandleOp
// instances.
static mutex static_resource_lookup_mutex_;
// current_id_ is just a hint for creating unique names. If it turns out
// there's a collision (e.g. because another AnonymousIteratorHandleOp
// instance is generating handles) we'll just skip that id.
static int64 current_id_ GUARDED_BY(static_resource_lookup_mutex_);
DataTypeVector output_dtypes_;
std::vector<PartialTensorShape> output_shapes_;
const int graph_def_version_;
};
class MakeIteratorOp : public OpKernel {
public:
explicit MakeIteratorOp(OpKernelConstruction* ctx) : OpKernel(ctx) {}
void Compute(OpKernelContext* ctx) override;
};
class IteratorGetNextOp : public AsyncOpKernel {
public:
explicit IteratorGetNextOp(OpKernelConstruction* ctx)
: AsyncOpKernel(ctx),
background_worker_(ctx->env(),
strings::StrCat("iterator_get_next_thread_",
SanitizeThreadSuffix(name()))) {}
void ComputeAsync(OpKernelContext* ctx, DoneCallback done) override;
private:
BackgroundWorker background_worker_;
};
class IteratorGetNextSyncOp : public OpKernel {
public:
explicit IteratorGetNextSyncOp(OpKernelConstruction* ctx) : OpKernel(ctx) {}
void Compute(OpKernelContext* ctx) override;
};
class IteratorToStringHandleOp : public OpKernel {
public:
explicit IteratorToStringHandleOp(OpKernelConstruction* ctx)
: OpKernel(ctx) {}
void Compute(OpKernelContext* ctx) override;
};
class IteratorFromStringHandleOp : public OpKernel {
public:
explicit IteratorFromStringHandleOp(OpKernelConstruction* ctx);
void Compute(OpKernelContext* ctx) override;
private:
DataTypeVector output_dtypes_;
std::vector<PartialTensorShape> output_shapes_;
};
} // namespace data
} // namespace tensorflow
#endif // TENSORFLOW_CORE_KERNELS_DATA_ITERATOR_OPS_H_
@@ -0,0 +1,38 @@
/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
#ifndef TENSORFLOW_CORE_KERNELS_DATA_OPTIONAL_OPS_H_
#define TENSORFLOW_CORE_KERNELS_DATA_OPTIONAL_OPS_H_
#include <vector>
#include "tensorflow/core/framework/tensor.h"
#include "tensorflow/core/framework/variant_tensor_data.h"
namespace tensorflow {
namespace data {
// Stores a DT_VARIANT value representing an Optional with the given value
// in the `output_index`^th output of the given kernel execution context.
Status WriteOptionalWithValueToOutput(OpKernelContext* ctx, int output_index,
std::vector<Tensor> value);
// Stores a DT_VARIANT value representing an Optional with no value
// in the `output_index`^th output of the given kernel execution context.
Status WriteOptionalNoneToOutput(OpKernelContext* ctx, int output_index);
} // namespace data
} // namespace tensorflow
#endif // TENSORFLOW_CORE_KERNELS_DATA_OPTIONAL_OPS_H_
@@ -0,0 +1,54 @@
/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
#ifndef TENSORFLOW_CORE_KERNELS_DATA_PARALLEL_MAP_ITERATOR_H_
#define TENSORFLOW_CORE_KERNELS_DATA_PARALLEL_MAP_ITERATOR_H_
#include <memory>
#include "tensorflow/core/framework/dataset.h"
namespace tensorflow {
namespace data {
// A function that transforms elements of one dataset into another
// asynchronously. The arguments are:
// 1. An `IteratorContext*` for the context in which the function should
// execute.
// 2. A `std::vector<Tensor>` containing the input element.
// 3. A `std::vector<Tensor>*` to which the function will write the result.
// 4. A `StatusCallback` that should be invoked when the function is complete.
using ParallelMapIteratorFunction =
std::function<void(IteratorContext*, std::vector<Tensor>,
std::vector<Tensor>*, StatusCallback)>;
// Returns a new iterator that applies `map_func` to the elements of
// `input_dataset` using the given degree of parallelism. `init_func` (if
// specified) will be executed when the iterator is initialized (see
// `IteratorBase::Initialize()`) and enables the user to specify error checking
// logic that can fail early.
std::unique_ptr<IteratorBase> NewParallelMapIterator(
const DatasetBaseIterator::BaseParams& params,
const DatasetBase* input_dataset,
std::function<Status(IteratorContext*)> init_func,
ParallelMapIteratorFunction map_func, int32 num_parallel_calls);
std::unique_ptr<IteratorBase> NewParallelMapIterator(
const DatasetBaseIterator::BaseParams& params,
const DatasetBase* input_dataset, ParallelMapIteratorFunction map_func,
int32 num_parallel_calls);
} // namespace data
} // namespace tensorflow
#endif // TENSORFLOW_CORE_KERNELS_DATA_PARALLEL_MAP_ITERATOR_H_
@@ -0,0 +1,73 @@
/* Copyright 2017 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
#ifndef TENSORFLOW_CORE_KERNELS_DATA_PREFETCH_AUTOTUNER_H_
#define TENSORFLOW_CORE_KERNELS_DATA_PREFETCH_AUTOTUNER_H_
#include "tensorflow/core/platform/types.h"
namespace tensorflow {
namespace data {
// PrefetchAutotuner dynamically adjusts the buffer size of a prefetch iterator.
//
// PrefetchAutotuner attempts to find the minimum buffer size such that there is
// always at least 1 element in the prefetch queue every time the downstream
// iterator calls GetNext().
//
// One common failure mode of input pipelines is being throughput bound. No
// amount of prefetching can address that performance mode. In order to guard
// against this condition, PrefetchAutotuner will only increase the buffer_limit
// if the prefetching thread is able to successfully fill the buffer at its
// current size.
//
// Note: in the current implementation, we never decrease the buffer_limit().
// This should change in the future!
//
// PrefetchAutotuner is NOT thread safe.
class PrefetchAutotuner {
public:
static const int64 kAutoTune = -1;
explicit PrefetchAutotuner(int64 initial_buffer_size);
int64 buffer_limit() const { return buffer_limit_; }
void RecordConsumption(size_t current_buffer_size);
void RecordEmpty() { RecordConsumption(0); }
private:
// PrefetchAutotuner operates as a state machine.
enum class Mode {
// Disables the autotuning.
kDisabled,
// We have increased the size of the buffer, and will transition to
// kDownswing if we successfully fill the buffer.
kUpswing,
// We have successfully filled a buffer of this size. If we ever block the
// downstream iterator, we should increase the buffer size.
kDownswing,
};
int64 buffer_limit_;
Mode mode_ = Mode::kDisabled;
};
} // namespace data
} // namespace tensorflow
#endif // TENSORFLOW_CORE_KERNELS_DATA_PREFETCH_AUTOTUNER_H_
@@ -0,0 +1,41 @@
/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
#ifndef TENSORFLOW_CORE_KERNELS_DATA_PREFETCH_DATASET_OP_H_
#define TENSORFLOW_CORE_KERNELS_DATA_PREFETCH_DATASET_OP_H_
#include "tensorflow/core/kernels/data/dataset.h"
#include "tensorflow/core/kernels/data/prefetch_autotuner.h"
namespace tensorflow {
namespace data {
class PrefetchDatasetOp : public UnaryDatasetOpKernel {
public:
explicit PrefetchDatasetOp(OpKernelConstruction* ctx)
: UnaryDatasetOpKernel(ctx) {}
protected:
void MakeDataset(OpKernelContext* ctx, DatasetBase* input,
DatasetBase** output) override;
private:
class Dataset;
};
} // namespace data
} // namespace tensorflow
#endif // TENSORFLOW_CORE_KERNELS_DATA_PREFETCH_DATASET_OP_H_
@@ -0,0 +1,62 @@
/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
#ifndef TENSORFLOW_CORE_KERNELS_DATA_SINGLE_THREADED_EXECUTOR_H_
#define TENSORFLOW_CORE_KERNELS_DATA_SINGLE_THREADED_EXECUTOR_H_
#include "tensorflow/core/common_runtime/executor.h"
namespace tensorflow {
namespace data {
// Creates a new `Executor` for executing `graph` synchronously on the caller
// thread.
//
// NOTE(mrry): The returned executor is optimized to impose low overhead on
// graphs that perform a small amount of work (e.g. <15us of work per graph on
// present architectures). It eschews concurrency, because issuing work to
// multiple threads can dominate the cost of executing small ops synchronously,
// and because contention in the executor data structures can reduce throughput
// (in terms of ops executed per unit time).
//
// However, the current implementation has the following limitations:
//
// 1. Reference-typed tensors are not supported and will not be supported in
// future.
// 2. Graphs with control flow (containing "Switch" and "Merge" nodes) are not
// currently supported. The current plan is to extend support to "functional"
// control flow after the TensorFlow APIs transition to building graphs in
// that form (e.g. `tf.cond_v2()`).
// 3. Partitioned graphs (containing "_Recv" nodes) are not currently supported.
// The present implementation executes kernels one at a time in topological
// order, and cannot currently distinguish between disconnected subgraphs
// that are logically connected by subgraphs on a different device.
// 4. Memory logging is not currently supported.
// 5. Allocation forwarding is not currently supported.
// 6. Non-default device contexts are not currently supported. In effect, this
// limits the executor to CPU devices.
// 7. Ops that rely on `OpKernelContext::slice_reader_cache()` being non-null
// are not currently supported.
//
// The single-threaded executor is primarily suitable for executing simple
// TensorFlow functions, such as one might find in a `tf.data` pipeline.
Status NewSingleThreadedExecutor(const LocalExecutorParams& params,
std::unique_ptr<const Graph> graph,
Executor** executor);
} // namespace data
} // namespace tensorflow
#endif // TENSORFLOW_CORE_KERNELS_DATA_SINGLE_THREADED_EXECUTOR_H_
@@ -0,0 +1,41 @@
/* Copyright 2017 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
#ifndef TENSORFLOW_CORE_KERNELS_DATA_SQL_DRIVER_MANAGER_H_
#define TENSORFLOW_CORE_KERNELS_DATA_SQL_DRIVER_MANAGER_H_
#include "tensorflow/core/kernels/data/sql/query_connection.h"
namespace tensorflow {
namespace data {
namespace sql {
// A factory class for creating `QueryConnection` instances.
class DriverManager {
public:
// A factory method for creating `QueryConnection` instances.
//
// `driver_name` is the database type (e.g. 'sqlite'). `driver_name`
// corresponds to a `QueryConnection` subclass. For example, if `driver_name`
// == `sqlite`, then `CreateQueryConnection` will create a
// `SqliteQueryConnection` instance.
static std::unique_ptr<QueryConnection> CreateQueryConnection(
const string& driver_name);
};
} // namespace sql
} // namespace data
} // namespace tensorflow
#endif // TENSORFLOW_CORE_KERNELS_DATA_SQL_DRIVER_MANAGER_H_
@@ -0,0 +1,70 @@
/* Copyright 2017 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
#ifndef TENSORFLOW_CORE_KERNELS_DATA_SQL_QUERY_CONNECTION_H_
#define TENSORFLOW_CORE_KERNELS_DATA_SQL_QUERY_CONNECTION_H_
#include "tensorflow/core/framework/tensor.h"
namespace tensorflow {
namespace data {
class IteratorContext;
namespace sql {
// This interface allows a user to connect to a database, execute a query, and
// iterate over the result set, putting the results into an output tensor.
// A subclass implementation is required for each type of database
// (e.g. sqlite3, mysql, etc.)
//
// Presently, a `QueryConnection` instance can only handle one query at a time.
// In a future extension, this class may be refactored so that it creates
// instances of a new class (named, say, `Statement`) which could have a
// one-to-one correspondence with queries. This would make `QueryConnection`
// more consistent with `Connection` classes of other database APIs.
// `QueryConnection` would then be renamed simply `Connection`.
//
// This class is not thread safe. Access to it is guarded by a mutex in
// `SqlDatasetOp::Dataset::Iterator`.
class QueryConnection {
public:
virtual ~QueryConnection() {}
// Opens a connection to the database named by `data_source_name`. Prepares to
// execute `query` against the database.
//
// The client must call `Close()` to release the connection resources, even
// if `Open()` fails. `Close()` must be called before making another call
// to `Open()`.
virtual Status Open(const string& data_source_name, const string& query,
const DataTypeVector& output_types) = 0;
// Closes an opened connection.
virtual Status Close() = 0;
// Retrieves the next row of the result set of the query from the most recent
// call to `Open()`.
//
// If such a row exists, then the row will be stored in `*out_tensors`, and
// `false` will be stored in `*end_of_sequence`.
//
// If there are no more rows in the result set, then instead `true` will be
// stored in `*end_of_sequence`, and the content of `*out_tensors` will be
// undefined.
virtual Status GetNext(IteratorContext* ctx, std::vector<Tensor>* out_tensors,
bool* end_of_sequence) = 0;
};
} // namespace sql
} // namespace data
} // namespace tensorflow
#endif // TENSORFLOW_CORE_KERNELS_DATA_SQL_QUERY_CONNECTION_H_
@@ -0,0 +1,56 @@
/* Copyright 2017 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
#ifndef TENSORFLOW_CORE_KERNELS_DATA_SQL_SQLITE_QUERY_CONNECTION_H_
#define TENSORFLOW_CORE_KERNELS_DATA_SQL_SQLITE_QUERY_CONNECTION_H_
#include <memory>
#include "tensorflow/core/kernels/data/sql/query_connection.h"
#include "tensorflow/core/lib/db/sqlite.h"
#include "tensorflow/core/platform/types.h"
namespace tensorflow {
namespace data {
namespace sql {
class SqliteQueryConnection : public QueryConnection {
public:
SqliteQueryConnection();
~SqliteQueryConnection() override;
Status Open(const string& data_source_name, const string& query,
const DataTypeVector& output_types) override;
Status Close() override;
Status GetNext(IteratorContext* ctx, std::vector<Tensor>* out_tensors,
bool* end_of_sequence) override;
private:
// Prepares the query string `query_`.
Status PrepareQuery();
// Fills `tensor` with the column_index_th element of the current row of
// `stmt_`.
void FillTensorWithResultSetEntry(const DataType& data_type, int column_index,
Tensor* tensor);
Sqlite* db_ = nullptr;
SqliteStatement stmt_;
int column_count_ = 0;
string query_;
DataTypeVector output_types_;
};
} // namespace sql
} // namespace data
} // namespace tensorflow
#endif // TENSORFLOW_CORE_KERNELS_DATA_SQL_SQLITE_QUERY_CONNECTION_H_
@@ -0,0 +1,50 @@
/* Copyright 2017 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
#ifndef TENSORFLOW_CORE_KERNELS_DATA_WINDOW_DATASET_H_
#define TENSORFLOW_CORE_KERNELS_DATA_WINDOW_DATASET_H_
#include <vector>
#include "tensorflow/core/framework/partial_tensor_shape.h"
#include "tensorflow/core/framework/tensor.h"
#include "tensorflow/core/framework/types.h"
#include "tensorflow/core/kernels/data/dataset.h"
namespace tensorflow {
namespace data {
// Creates a dataset representing an eagerly-collected window of elements.
//
// The `elements` argument defines the elements of the resulting
// dataset, which is stored in `out_dataset`.
//
// This dataset is constructed internally for use in datasets that
// build nested dataset expressions (e.g. the reducer function for
// GroupByWindowDataset). It efficiently supports multiple iterators on
// the same window without recomputation.
//
// REQUIRES: `output_types` must match the types of the respective
// element components in `elements`.
// REQUIRES: `output_shapes` must be compatible with the shapes of the
// respective element components in `elements`.a
Status NewWindowDataset(std::vector<std::vector<Tensor>> elements,
DataTypeVector output_types,
std::vector<PartialTensorShape> output_shapes,
DatasetBase** out_dataset);
} // namespace data
} // namespace tensorflow
#endif // TENSORFLOW_CORE_KERNELS_DATA_WINDOW_DATASET_H_
@@ -0,0 +1,86 @@
/* Copyright 2015 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
#ifndef TENSORFLOW_CORE_KERNELS_DATA_FORMAT_OPS_H_
#define TENSORFLOW_CORE_KERNELS_DATA_FORMAT_OPS_H_
// Functor definition for data format dim mapping ops, must be compilable
// by nvcc.
#include "third_party/eigen3/unsupported/Eigen/CXX11/Tensor"
#include "tensorflow/core/framework/tensor_types.h"
namespace tensorflow {
namespace functor {
// Functor used by DataFormatDimMapOP to do the computations.
template <typename Device, typename T>
struct DataFormatDimMap {
void operator()(const Device& d, typename TTypes<T>::ConstFlat x,
typename TTypes<T>::Flat y, const TTypes<int>::Vec dst) {
auto zero = x.constant(0);
auto one = x.constant(1);
auto two = x.constant(2);
auto f_zero = x.constant(dst(0));
auto f_one = x.constant(dst(1));
auto f_two = x.constant(dst(2));
auto f_three = x.constant(dst(3));
auto four = x.constant(4);
auto x_mod = (x + four) % 4;
auto is_zero = (x_mod == zero);
auto is_one = (x_mod == one);
auto is_two = (x_mod == two);
y.device(d) = is_zero.select(
f_zero, is_one.select(f_one, is_two.select(f_two, f_three)));
}
};
template <typename T>
struct VecPermute {
VecPermute(const Eigen::DSizes<Eigen::DenseIndex, 8>& dst) : dst_(dst) {}
Eigen::DSizes<Eigen::DenseIndex, 1> dimensions(
typename TTypes<T>::ConstFlat input) const {
Eigen::DSizes<Eigen::DenseIndex, 1> result;
result[0] = input.dimension(0);
return result;
}
template <typename Output, typename Device>
void eval(typename TTypes<T>::ConstFlat input, Output& output,
const Device& d) const {
for (int i = 0; i < input.size(); ++i) {
output.template chip<0>(dst_[i]).device(d) = input.template chip<0>(i);
}
}
private:
Eigen::DSizes<Eigen::DenseIndex, 8> dst_;
};
// Functor used by DataFormatVecPermuteOp to do the computations.
template <typename Device, typename T>
struct DataFormatVecPermute {
void operator()(const Device& d, typename TTypes<T>::ConstFlat x,
typename TTypes<T>::Flat y,
const Eigen::DSizes<Eigen::DenseIndex, 8>& dst) {
y.device(d) = x.customOp(VecPermute<T>(dst));
}
};
} // namespace functor
} // namespace tensorflow
#endif // TENSORFLOW_CORE_KERNELS_DATA_FORMAT_OPS_H_
@@ -0,0 +1,117 @@
/* Copyright 2016 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
#ifndef TENSORFLOW_CORE_KERNELS_DEEP_CONV2D_H_
#define TENSORFLOW_CORE_KERNELS_DEEP_CONV2D_H_
#include "tensorflow/core/framework/types.h"
namespace tensorflow {
class OpKernelContext;
// DeepConv2D is a Conv2D implementation specialized for deep (i.e. large
// in_depth * out_depth product) convolutions (see deep_conv2d.cc for details).
// DeepConv2DTransform is an interface for implementing transforms for
// DeepConv2D. Implementations must specify transform matrices and
// input/output/filter shapes. DeepConv2d computes:
//
// y = C[Ad * Bg]
//
// C: output transform matrix
// A: input data transform matrix
// B: filter transform matrix
// d: vectorized 2D data tile
// g: vectorized 2D filter tile
// y: vectorized 2D output tile
template <typename T>
class DeepConv2DTransform {
public:
virtual ~DeepConv2DTransform() {}
virtual void GetFilterTransformMatrix(const int64 rows, const int64 cols,
T* transform_matrix) const = 0;
virtual void GetInputTransformMatrix(const int64 rows, const int64 cols,
T* transform_matrix) const = 0;
virtual void GetOutputTransformMatrix(const int64 rows, const int64 cols,
T* transform_matrix) const = 0;
struct Shape {
Shape(int64 r, int64 c) : rows(r), cols(c) {}
int64 rows;
int64 cols;
};
virtual const Shape& filter_shape() const = 0;
virtual const Shape& input_shape() const = 0;
virtual const Shape& output_shape() const = 0;
};
// Conv2D arguments used by DeepConv2D implementation.
struct Conv2DArgs {
// Input layer dimensions
int batch;
int in_rows;
int in_cols;
int in_depth;
int filter_rows;
int filter_cols;
int pad_rows;
int pad_cols;
// Output layer dimensions
int out_rows;
int out_cols;
int out_depth;
Conv2DArgs()
: batch(0),
in_rows(0),
in_cols(0),
in_depth(0),
filter_rows(0),
filter_cols(0),
pad_rows(0),
pad_cols(0),
out_rows(0),
out_cols(0),
out_depth(0) {}
};
// Returns true if convolution operation specified by function arguments
// can use DeepConv2D implementation, and false otherwise.
// May return false based on parameters, cost, or whether feature is disabled.
bool CanUseDeepConv2D(int stride_rows, int stride_cols, int filter_rows,
int filter_cols, int in_depth, int out_depth,
int out_rows, int out_cols);
namespace functor {
// Calls DeepConv2D implementation (see deep_conv2d.cc for details).
template <typename Device, typename T>
struct DeepConv2D {
void operator()(OpKernelContext* ctx, const Conv2DArgs& args, const T* input,
const T* filter, T* output);
};
} // namespace functor
} // namespace tensorflow
#endif // TENSORFLOW_CORE_KERNELS_DEEP_CONV2D_H_
@@ -0,0 +1,108 @@
/* Copyright 2015 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
#ifndef TENSORFLOW_CORE_KERNELS_DENSE_UPDATE_FUNCTOR_H_
#define TENSORFLOW_CORE_KERNELS_DENSE_UPDATE_FUNCTOR_H_
#define EIGEN_USE_THREADS
#include "third_party/eigen3/unsupported/Eigen/CXX11/Tensor"
#include "tensorflow/core/framework/op_kernel.h"
#include "tensorflow/core/framework/tensor_types.h"
namespace tensorflow {
typedef Eigen::ThreadPoolDevice CPUDevice;
typedef Eigen::GpuDevice GPUDevice;
#ifdef TENSORFLOW_USE_SYCL
typedef Eigen::SyclDevice SYCLDevice;
#endif // TENSORFLOW_USE_SYCL
enum DenseUpdateType { ADD, SUB, ASSIGN };
namespace functor {
template <typename Device, typename T, DenseUpdateType OP>
struct DenseUpdate {
void operator()(const Device& d, typename TTypes<T>::Flat params,
typename TTypes<T>::ConstFlat update);
};
template <typename T>
struct DenseUpdate<CPUDevice, T, ADD> {
void operator()(const CPUDevice& d, typename TTypes<T>::Flat params,
typename TTypes<T>::ConstFlat update) {
params.device(d) += update;
}
};
template <typename T>
struct DenseUpdate<CPUDevice, T, SUB> {
void operator()(const CPUDevice& d, typename TTypes<T>::Flat params,
typename TTypes<T>::ConstFlat update) {
params.device(d) -= update;
}
};
template <typename T>
struct DenseUpdate<CPUDevice, T, ASSIGN> {
void operator()(const CPUDevice& d, typename TTypes<T>::Flat params,
typename TTypes<T>::ConstFlat update) {
params.device(d) = update;
}
};
#ifdef TENSORFLOW_USE_SYCL
template <typename T>
struct DenseUpdate<SYCLDevice, T, ADD> {
void operator()(const SYCLDevice& d, typename TTypes<T>::Flat params,
typename TTypes<T>::ConstFlat update) {
params.device(d) += update;
}
};
template <typename T>
struct DenseUpdate<SYCLDevice, T, SUB> {
void operator()(const SYCLDevice& d, typename TTypes<T>::Flat params,
typename TTypes<T>::ConstFlat update) {
params.device(d) -= update;
}
};
template <typename T>
struct DenseUpdate<SYCLDevice, T, ASSIGN> {
void operator()(const SYCLDevice& d, typename TTypes<T>::Flat params,
typename TTypes<T>::ConstFlat update) {
params.device(d) = update;
}
};
#endif // TENSORFLOW_USE_SYCL
} // end namespace functor
template <typename Device>
Status VariantCopyFn(OpKernelContext* context, const Tensor& from, Tensor* to);
template <>
Status VariantCopyFn<CPUDevice>(OpKernelContext* context, const Tensor& from,
Tensor* to);
template <>
Status VariantCopyFn<GPUDevice>(OpKernelContext* context, const Tensor& from,
Tensor* to);
} // end namespace tensorflow
#endif // TENSORFLOW_CORE_KERNELS_DENSE_UPDATE_FUNCTOR_H_
@@ -0,0 +1,56 @@
/* Copyright 2015 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
#ifndef TENSORFLOW_CORE_KERNELS_DEPTHTOSPACE_OP_H_
#define TENSORFLOW_CORE_KERNELS_DEPTHTOSPACE_OP_H_
#include "third_party/eigen3/unsupported/Eigen/CXX11/Tensor"
#include "tensorflow/core/framework/tensor_types.h"
#include "tensorflow/core/util/tensor_format.h"
namespace tensorflow {
namespace functor {
// Functor used by DepthToSpaceOp to do the computations.
// Implements a family of Depth to Space transforms for a 4D 'input' tensor
// to a 4D 'output' tensor, both tensors use type 'T' and layout 'data_format'.
// These transforms multiply the vertical and horizontal image sizes by
// 'block_size', and divide the depth dimension by (block_size * block_size)
// which must divide evenly.
// Each pixel in the input image is converted to a square block of pixels in
// the output image. The Y, X coordinates within each block comes from the
// high component of the input depth (channel) index.
// e.g. for data_format = NHWC:
// Each element in the input tensor can be specified via 6 coordinates,
// ordered by decreasing memory layout significance as:
// n,iY,iX,bY,bX,oC (where n=batch index, iX, iY means X or Y coordinates
// within the input image, bX, bY means coordinates
// within the output block, oC means output channel).
// The output would be a transpose to the following layout:
// n,iY,bY,iX,bX,oC
template <typename Device, typename T, TensorFormat data_format>
struct DepthToSpaceOpFunctor {
void operator()(const Device& d, typename TTypes<T, 4>::ConstTensor input,
int block_size, typename TTypes<T, 4>::Tensor output);
// This 5-D version is to support NCHW_VECT_C.
void operator()(const Device& d, typename TTypes<T, 5>::ConstTensor input,
int block_size, typename TTypes<T, 5>::Tensor output);
};
} // namespace functor
} // namespace tensorflow
#endif // TENSORFLOW_CORE_KERNELS_DEPTHTOSPACE_OP_H_
@@ -0,0 +1,287 @@
/* Copyright 2015 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
#ifndef TENSORFLOW_CORE_KERNELS_DEPTHWISE_CONV_OP_H_
#define TENSORFLOW_CORE_KERNELS_DEPTHWISE_CONV_OP_H_
#include "third_party/eigen3/unsupported/Eigen/CXX11/Tensor"
#include "tensorflow/core/framework/types.h"
#include "tensorflow/core/util/tensor_format.h"
namespace tensorflow {
struct DepthwiseArgs {
// Input layer dimensions
int batch;
int in_rows;
int in_cols;
int in_depth;
int filter_rows;
int filter_cols;
int depth_multiplier;
int stride;
int pad_rows;
int pad_cols;
// Output layer dimensions
int out_rows;
int out_cols;
int out_depth;
DepthwiseArgs()
: batch(0),
in_rows(0),
in_cols(0),
in_depth(0),
filter_rows(0),
filter_cols(0),
depth_multiplier(0),
stride(0),
pad_rows(0),
pad_cols(0),
out_rows(0),
out_cols(0),
out_depth(0) {}
};
// Forward declaration.
class OpKernelContext;
template <typename Device, typename T>
struct LaunchDepthwiseConvOp {
void operator()(OpKernelContext* ctx, const DepthwiseArgs& args,
const T* input, const T* filter, T* output,
TensorFormat data_format);
};
template <typename Device, typename T>
struct LaunchDepthwiseConvBackpropInputOp {
void operator()(OpKernelContext* ctx, const DepthwiseArgs& args,
const T* out_backprop, const T* filter, T* in_backprop,
TensorFormat data_format);
};
template <typename Device, typename T>
struct LaunchDepthwiseConvBackpropFilterOp {
void operator()(OpKernelContext* ctx, const DepthwiseArgs& args,
const T* out_backprop, const T* input, T* filter_backprop,
TensorFormat data_format);
};
#if GOOGLE_CUDA
template <typename T>
struct LaunchDepthwiseConvOp<Eigen::GpuDevice, T> {
void operator()(OpKernelContext* ctx, const DepthwiseArgs& args,
const T* input, const T* filter, T* output,
TensorFormat data_format);
};
template <typename T>
struct LaunchDepthwiseConvBackpropInputOp<Eigen::GpuDevice, T> {
void operator()(class OpKernelContext* ctx, const DepthwiseArgs& args,
const T* out_backprop, const T* filter, T* in_backprop,
TensorFormat data_format);
};
template <typename T>
struct LaunchDepthwiseConvBackpropFilterOp<Eigen::GpuDevice, T> {
void operator()(class OpKernelContext* ctx, const DepthwiseArgs& args,
const T* out_backprop, const T* input, T* filter_backprop,
TensorFormat data_format);
};
#endif
} // namespace tensorflow
namespace tensorflow {
namespace functor {
// Pads 'filter' to vector-register boundary along its inner dimension:
// filter_inner_dim_size = in_depth * depth_multiplier
// Requires 'filter' to have the following storage order:
// [filter_rows, filter_cols, in_depth, depth_multiplier]
// Returns zero-padded filter in 'padded_filter'.
//
// EX:
// in_depth = 3, depth_multiplier = 2, filter [2, 2], register_width = 4
// So we have a total of 3 * 2 = 6 filters, each of spatial size 2 x 2.
//
// filter [rows, cols, in_depth, depth_multiplier]
// [u0, v0, w0, x0] [y0, z0, u1, v1] [w1, x1, y1, z1]
// [u2, v2, w2, x2] [y2, z2, u3, v3] [w3, x3, y3, z3]
//
// padded_filter [rows, cols, in_depth, depth_multiplier]
// [u0, v0, w0, x0] [y0, z0, 0, 0] [u1, v1, w1, x1] [y1, z1, 0, 0]
// [u2, v2, w2, x2] [y2, z2, 0, 0] [u3, v3, w3, x3] [y3, z3, 0, 0]
template <typename T>
struct DepthwiseFilterPadOp {
void operator()(const DepthwiseArgs& args, const T* filter,
T* padded_filter) {
typedef typename Eigen::internal::packet_traits<T>::type Packet;
static const int64 kPacketSize = (sizeof(Packet) / sizeof(T));
// Calculate vectorized and scalar lengths of filter's inner dimension.
const int64 filter_inner_dim_size = args.out_depth;
const int64 vectorized_size =
(filter_inner_dim_size / kPacketSize) * kPacketSize;
const int64 scalar_size = filter_inner_dim_size - vectorized_size;
// Calculate required padding and padded output buffer stride.
const int64 pad_size = scalar_size > 0 ? kPacketSize - scalar_size : 0;
const int64 padded_filter_stride = vectorized_size + kPacketSize;
const int64 filter_spatial_size = args.filter_rows * args.filter_cols;
for (int64 i = 0; i < filter_spatial_size; ++i) {
const int64 input_base = i * filter_inner_dim_size;
const int64 output_base = i * padded_filter_stride;
// Write vectorized length of filter's inner dimension to output.
for (int64 j = 0; j < vectorized_size; j += kPacketSize) {
const auto v = Eigen::internal::ploadu<Packet>(filter + input_base + j);
Eigen::internal::pstoreu<T>(padded_filter + output_base + j, v);
}
// Write scalar length of filter's inner dimension to output.
for (int64 j = 0; j < scalar_size; ++j) {
padded_filter[output_base + vectorized_size + j] =
filter[input_base + vectorized_size + j];
}
// Pad the remainder of output to vector-register boundary.
for (int64 j = 0; j < pad_size; ++j) {
padded_filter[output_base + vectorized_size + scalar_size + j] =
static_cast<T>(0);
}
}
}
};
// Copies data from local region in 'input' specified by 'out_r' and 'out_'c'
// to 'input_buffer'. The copied data is replicated by factor
// 'args.depth_mulitplier', and padded to vector register-width boundaries so
// that it is aligned for efficient traversal and vector multiply-add by the
// depthwise kernel.
//
// EX:
// in_depth = 3, depth_multiplier = 2, filter [2, 2], register_width = 4
//
// input: [batch, in_rows, in_cols, in_depth]
//
// [a0, a1, a2, b0, b1, b2, ..., e0, e1, e2, f0, f1, f2, ...]
//
// input_buffer (register boundaries shown):
// [a0, a0, a1, a1] [a2, a2, 0, 0] in_row = 0, in_col = 0
// [b0, b0, b1, b1] [b2, b2, 0, 0] in_row = 0, in_col = 1
// [e0, e0, e1, e1] [e2, e2, 0, 0] in_row = 1, in_col = 0
// [f0, f0, f1, f1] [f2, f2, 0, 0] in_row = 1, in_col = 1
//
// Returns replicated and padded data from specified input region in
// 'input_buffer'.
template <typename T>
struct DepthwiseInputCopyOp {
void operator()(const DepthwiseArgs& args,
const int64 padded_filter_inner_dim_size, const int64 out_r,
const int64 out_c, const T* input, T* input_buffer) {
typedef typename Eigen::internal::packet_traits<T>::type Packet;
static const int64 kPacketSize = (sizeof(Packet) / sizeof(T));
// Calculate vectorized and scalar (residual) lengths for 'in_depth'.
const int64 input_vectorized_size =
(args.in_depth / kPacketSize) * kPacketSize;
const int64 input_scalar_size = args.in_depth % kPacketSize;
// Calculate vectorized and scalar (residual) lengths for
// 'depth_multiplier'. This is used to efficiently replicate data for
// when 'depth_multiplier' > kPacketSize.
const int64 dm_vectorized_size =
(args.depth_multiplier / kPacketSize) * kPacketSize;
const int64 dm_scalar_size = args.depth_multiplier % kPacketSize;
// Calculate output padding length.
const int64 output_scalar_size = args.out_depth % kPacketSize;
const int64 output_pad_size =
output_scalar_size > 0 ? kPacketSize - output_scalar_size : 0;
const int64 replicated_packet_size = kPacketSize * args.depth_multiplier;
// Iterate through all rows x cols reading 'in_depth' from 'input' and
// replicating by 'depth_multiplier' into 'input_buffer' (otherwise
// zero-padding input buffer as needed).
auto* in_buf = input_buffer;
const int64 in_r_start = out_r * args.stride - args.pad_rows;
const int64 in_c_start = out_c * args.stride - args.pad_cols;
for (int64 f_r = 0; f_r < args.filter_rows; ++f_r) {
const int64 in_r = in_r_start + f_r;
for (int64 f_c = 0; f_c < args.filter_cols; ++f_c) {
const int64 in_c = in_c_start + f_c;
if (in_r >= 0 && in_r < args.in_rows && in_c >= 0 &&
in_c < args.in_cols) {
auto* in = input + (in_r * args.in_cols + in_c) * args.in_depth;
// Copy vectorized portion of inner dimension.
for (int64 d = 0; d < input_vectorized_size; d += kPacketSize) {
auto v = Eigen::internal::ploadu<Packet>(in + d);
for (int dm = 0; dm < args.depth_multiplier; ++dm) {
Eigen::internal::pscatter<T, Packet>(in_buf + dm, v,
args.depth_multiplier);
}
in_buf += replicated_packet_size;
}
// Copy scalar portion of inner dimension.
for (int64 d = 0; d < input_scalar_size; ++d) {
T v = in[input_vectorized_size + d];
const int64 base = d * args.depth_multiplier;
if (dm_vectorized_size > 0) {
// Copy vectorized portion of replicated output.
// This branch is only taken if 'args.depth_multiplier' is
// vectorizable (i.e. args.depth_multiplier >= register width).
auto p = Eigen::internal::pset1<Packet>(v);
for (int64 dm = 0; dm < dm_vectorized_size; dm += kPacketSize) {
Eigen::internal::pstoreu<T>(in_buf + base + dm, p);
}
// Copy scalar portion of replicated output.
for (int64 dm = 0; dm < dm_scalar_size; ++dm) {
in_buf[base + dm_vectorized_size + dm] = v;
}
} else {
// Depth multiplier is less than one packet: scalar copy.
for (int dm = 0; dm < args.depth_multiplier; ++dm) {
in_buf[base + dm] = v;
}
}
}
in_buf += input_scalar_size * args.depth_multiplier;
// Pad the remainder of the output to vector register boundary.
for (int64 d = 0; d < output_pad_size; ++d) {
in_buf[d] = static_cast<T>(0);
}
in_buf += output_pad_size;
} else {
// Zero pad.
memset(in_buf, 0, sizeof(T) * padded_filter_inner_dim_size);
in_buf += padded_filter_inner_dim_size;
}
}
}
}
};
} // namespace functor
} // namespace tensorflow
#endif // TENSORFLOW_CORE_KERNELS_DEPTHWISE_CONV_OP_H_
@@ -0,0 +1,47 @@
/* Copyright 2017 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
#ifndef TENSORFLOW_CORE_KERNELS_DETERMINANT_OP_H_
#define TENSORFLOW_CORE_KERNELS_DETERMINANT_OP_H_
#include "tensorflow/core/framework/tensor_types.h"
namespace tensorflow {
namespace functor {
// Helper functor to compute Determinant from a partially pivoted LU
// factorization.
template <typename Device, typename Scalar>
struct DeterminantFromPivotedLUFunctor {
void operator()(const Device& device,
typename TTypes<Scalar, 3>::ConstTensor lu_factor,
const int* pivots, typename TTypes<Scalar, 1>::Tensor output,
int* info);
};
// Helper functor to compute sign and log of the absolute value of the
// determinant from a partially pivoted LU factorization.
template <typename Device, typename Scalar>
struct LogDeterminantFromPivotedLUFunctor {
void operator()(const Device& device,
typename TTypes<Scalar, 3>::ConstTensor lu_factor,
const int* pivots, typename TTypes<Scalar, 1>::Tensor sign,
typename TTypes<Scalar, 1>::Tensor log_abs_det);
};
} // namespace functor
} // namespace tensorflow
#endif // TENSORFLOW_CORE_KERNELS_DETERMINANT_OP_H_
@@ -0,0 +1,43 @@
/* Copyright 2017 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
#ifndef TENSORFLOW_CORE_KERNELS_DIAG_OP_H_
#define TENSORFLOW_CORE_KERNELS_DIAG_OP_H_
#include "tensorflow/core/framework/op_kernel.h"
#include "tensorflow/core/framework/tensor_types.h"
#include "tensorflow/core/platform/types.h"
namespace tensorflow {
namespace functor {
template <typename Device, typename T>
struct DiagFunctor {
Status operator()(OpKernelContext* context, const int64 size, const T* in,
T* out);
};
template <typename Device, typename T>
struct DiagPartFunctor {
Status operator()(OpKernelContext* context, const int64 size, const T* in,
T* out);
};
} // namespace functor
} // namespace tensorflow
#endif // TENSORFLOW_CORE_KERNELS_DIAG_OP_H_
@@ -0,0 +1,66 @@
/* Copyright 2016 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
#ifndef TENSORFLOW_CORE_KERNELS_DILATION_OPS_H_
#define TENSORFLOW_CORE_KERNELS_DILATION_OPS_H_
#include "third_party/eigen3/unsupported/Eigen/CXX11/Tensor"
#include "tensorflow/core/framework/tensor_types.h"
#include "tensorflow/core/platform/types.h"
namespace tensorflow {
namespace functor {
template <typename Device, typename T>
struct Dilation {
// We assume that the tensor sizes are correct.
void operator()(const Device& d, typename TTypes<T, 4>::ConstTensor input,
typename TTypes<T, 3>::ConstTensor filter, int stride_rows,
int stride_cols, int rate_rows, int rate_cols, int pad_top,
int pad_left, typename TTypes<T, 4>::Tensor output);
};
template <typename Device, typename T>
struct DilationBackpropInput {
// We assume that the tensor sizes are correct.
// To avoid storing the argmax values during forward computation, we recompute
// the argmax during backward computation, which is the reason why we provide
// filter as argument to the backward computation routine.
void operator()(const Device& d, typename TTypes<T, 4>::ConstTensor input,
typename TTypes<T, 3>::ConstTensor filter,
typename TTypes<T, 4>::ConstTensor out_backprop,
int stride_rows, int stride_cols, int rate_rows,
int rate_cols, int pad_top, int pad_left,
typename TTypes<T, 4>::Tensor in_backprop);
};
template <typename Device, typename T>
struct DilationBackpropFilter {
// We assume that the tensor sizes are correct.
// To avoid storing the argmax values during forward computation, we recompute
// the argmax during backward computation, which is the reason why we provide
// filter as argument to the backward computation routine.
void operator()(const Device& d, typename TTypes<T, 4>::ConstTensor input,
typename TTypes<T, 3>::ConstTensor filter,
typename TTypes<T, 4>::ConstTensor out_backprop,
int stride_rows, int stride_cols, int rate_rows,
int rate_cols, int pad_top, int pad_left,
typename TTypes<T, 3>::Tensor filter_backprop);
};
} // namespace functor
} // namespace tensorflow
#endif // TENSORFLOW_CORE_KERNELS_DILATION_OPS_H_
@@ -0,0 +1,125 @@
/* Copyright 2015 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
#ifndef TENSORFLOW_CORE_KERNELS_EIGEN_ACTIVATIONS_H_
#define TENSORFLOW_CORE_KERNELS_EIGEN_ACTIVATIONS_H_
#include "third_party/eigen3/unsupported/Eigen/CXX11/Tensor"
namespace Eigen {
/** scalar_sigmoid_fast_derivative_op
* \ingroup CXX11_NeuralNetworks_Module
* \brief Template functor to compute the fast derivative of a sigmoid
*
* Input should be the backpropagated gradient.
*
* \sa class CwiseUnaryOp, Cwise::sigmoid_fast_derivative()
*/
template <typename T>
struct scalar_sigmoid_fast_derivative_op {
EIGEN_EMPTY_STRUCT_CTOR(scalar_sigmoid_fast_derivative_op)
EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE T operator()(const T& y) const {
const T one = T(1);
return (one - y) * y;
}
template <typename Packet>
inline Packet packetOp(const Packet& y) const {
const Packet one = internal::pset1<Packet>(1);
return internal::pmul(internal::psub(one, y), y);
}
};
namespace internal {
template <typename T>
struct functor_traits<scalar_sigmoid_fast_derivative_op<T> > {
enum {
Cost = NumTraits<T>::AddCost * 2 + NumTraits<T>::MulCost,
PacketAccess = packet_traits<T>::HasAdd && packet_traits<T>::HasMul &&
packet_traits<T>::HasNegate
};
};
} // namespace internal
/** scalar_tanh_fast_derivative_op
* \ingroup CXX11_NeuralNetworks_Module
* \brief Template functor to compute the fast derivative of a tanh
*
* Input should be the backpropagated gradient.
*
* \sa class CwiseUnaryOp, Cwise::tanh_fast_derivative()
*/
template <typename T>
struct scalar_tanh_fast_derivative_op {
EIGEN_EMPTY_STRUCT_CTOR(scalar_tanh_fast_derivative_op)
EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE T operator()(const T& y) const {
const T one = T(1);
return one - (y * y);
}
template <typename Packet>
inline Packet packetOp(const Packet& y) const {
const Packet one = internal::pset1<Packet>(1);
return internal::psub(one, internal::pmul(y, y));
}
};
namespace internal {
template <typename T>
struct functor_traits<scalar_tanh_fast_derivative_op<T> > {
enum {
Cost = NumTraits<T>::AddCost * 2 + NumTraits<T>::MulCost * 1,
PacketAccess = packet_traits<T>::HasAdd && packet_traits<T>::HasMul &&
packet_traits<T>::HasNegate
};
};
} // namespace internal
/**
* \ingroup CXX11_NeuralNetworks_Module
* \brief Template functor to clip the magnitude of the first scalar.
*
* \sa class CwiseBinaryOp, MatrixBase::Clip
*/
template <typename Scalar>
struct scalar_clip_op {
EIGEN_EMPTY_STRUCT_CTOR(scalar_clip_op)
EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const Scalar
operator()(const Scalar& a, const Scalar& b) const {
return numext::mini(numext::maxi(a, -b), b);
}
template <typename Packet>
EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const Packet
packetOp(const Packet& a, const Packet& b) const {
return internal::pmin(internal::pmax(a, internal::pnegate(b)), b);
}
};
namespace internal {
template <typename Scalar>
struct functor_traits<scalar_clip_op<Scalar> > {
enum {
Cost = NumTraits<Scalar>::AddCost * 3,
PacketAccess = packet_traits<Scalar>::HasMax &&
packet_traits<Scalar>::HasMin &&
packet_traits<Scalar>::HasNegate
};
};
} // namespace internal
} // end namespace Eigen
#endif // TENSORFLOW_CORE_KERNELS_EIGEN_ACTIVATIONS_H_
@@ -0,0 +1,255 @@
/* Copyright 2015 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
#ifndef TENSORFLOW_CORE_KERNELS_EIGEN_ATTENTION_H_
#define TENSORFLOW_CORE_KERNELS_EIGEN_ATTENTION_H_
#include "third_party/eigen3/unsupported/Eigen/CXX11/Tensor"
namespace Eigen {
/** ExtractGlimpses
* \ingroup CXX11_NeuralNetworks_Module
*
* \brief Extract glimpses from an input tensor.
*
* The input parameter is expected to be a col-major tensor with a rank of 4
* (depth, x, y, and batch). The width and height parameters specify the
* extension of the returned glimpses. The offsets parameter specifies the x, y
* locations of the center of the glimpses relative to the center of the input
* image. The vector is expected to contain one IndexPair for each image in the
* batch dimension. The normalized boolean indicates if incoming coordinates are
* normalized so that 0.0 and 1.0 correspond to the minimum and maximum of each
* height and width dimension. The centered boolean indicates if incoming
* coordinates are centered relative to the image, in which case -1.0 and 1.0
* correspond to minimum and maximum of each dimension while 0.0 corresponds to
* the center.
*
* The result can be assigned to a tensor of rank equal to that of the input.
* The result will be laid out in col-major order (depth, x, y, batch). The
* dimensions of the result will be equal to the dimensions of the input except
* for width and height which will be equal to the requested glimpse size.
*/
namespace {
template <typename Index>
struct GlimpseExtractionOp {
GlimpseExtractionOp(const Index width, const Index height,
const std::vector<IndexPair<float> >& offsets,
const bool normalized, const bool centered,
const bool uniform_noise)
: width_(width),
height_(height),
offsets_(offsets),
normalized_(normalized),
centered_(centered),
uniform_noise_(uniform_noise) {}
template <typename Input>
DSizes<Index, 4> dimensions(const Input& input) const {
typedef typename internal::traits<Input>::Index IndexType;
typedef TensorRef<Tensor<typename internal::traits<Input>::Scalar, 4,
internal::traits<Input>::Layout, IndexType> >
Ref;
Ref in(input);
DSizes<Index, 4> dims = in.dimensions();
dims[0] = in.dimension(0);
dims[1] = width_;
dims[2] = height_;
dims[3] = in.dimension(3);
return dims;
}
template <typename Input, typename Output, typename Device>
EIGEN_DEVICE_FUNC void eval(const Input& input, Output& output,
const Device& device) const {
typedef typename internal::traits<Input>::Index IndexType;
typedef TensorRef<Tensor<typename internal::traits<Input>::Scalar, 4,
internal::traits<Input>::Layout, IndexType> >
Ref;
Ref in(input);
const Index num_channels = in.dimension(0);
const Index input_width = in.dimension(1);
const Index input_height = in.dimension(2);
const Index batch_size = in.dimension(3);
eigen_assert(input_width > 0);
eigen_assert(input_height > 0);
internal::NormalRandomGenerator<float> gen;
internal::UniformRandomGenerator<float> unigen;
for (Index i = 0; i < batch_size; ++i) {
float x = offsets_[i].first, y = offsets_[i].second;
// Un-normalize coordinates back to pixel space if normalized.
if (normalized_) {
x *= input_width;
y *= input_height;
}
// Un-center if coordinates are centered on the image center.
if (centered_) {
x /= 2.0f;
y /= 2.0f;
x += input_width / 2.0f;
y += input_height / 2.0f;
}
// Remove half of the glimpse window.
x -= width_ / 2.0f;
y -= height_ / 2.0f;
const Index offset_x = (Index)x;
const Index offset_y = (Index)y;
Index glimpse_width = width_;
Index glimpse_height = height_;
bool partial_overlap = false;
DSizes<Index, 3> slice_offset(0, offset_x, offset_y);
DSizes<Index, 3> slice_extent(num_channels, width_, height_);
DSizes<Index, 3> base_offset(0, 0, 0);
if (offset_x < 0) {
slice_offset[1] = 0;
glimpse_width = (std::max<Index>)(0, width_ + offset_x);
slice_extent[1] = glimpse_width;
base_offset[1] = width_ - glimpse_width;
partial_overlap = true;
} else if (offset_x + width_ >= input_width) {
glimpse_width = (std::max<Index>)(0, input_width - offset_x);
slice_extent[1] = glimpse_width;
partial_overlap = true;
}
if (offset_y < 0) {
slice_offset[2] = 0;
glimpse_height = (std::max<Index>)(0, height_ + offset_y);
slice_extent[2] = glimpse_height;
base_offset[2] = height_ - glimpse_height;
partial_overlap = true;
} else if (offset_y + height_ >= input_height) {
glimpse_height = (std::max<Index>)(0, input_height - offset_y);
slice_extent[2] = glimpse_height;
partial_overlap = true;
}
slice_extent[1] = std::min<Index>(input_width, slice_extent[1]);
slice_extent[2] = std::min<Index>(input_height, slice_extent[2]);
if (partial_overlap) {
if (uniform_noise_) {
// Initialize the glimpse with uniform noise.
typedef typename internal::remove_const<
typename internal::traits<Input>::Scalar>::type Scalar;
TensorFixedSize<Scalar, Sizes<> > mini;
mini.device(device) = input.template chip<3>(i).minimum();
TensorFixedSize<float, Sizes<> > range;
range.device(device) = (input.template chip<3>(i).maximum() - mini)
.template cast<float>();
DSizes<Index, 3> glimpse_size(num_channels, width_, height_);
TensorMap<Tensor<float, 3> > tmp(NULL, glimpse_size);
output.template chip<3>(i).device(device) =
mini.reshape(Sizes<1, 1, 1>()).broadcast(glimpse_size) +
(tmp.random(unigen) *
range.reshape(Sizes<1, 1, 1>()).broadcast(glimpse_size))
.template cast<Scalar>();
} else {
// Initialize the glimpse with white noise: compute the mean and sigma
// of each channel, and use them to shape the gaussian.
DSizes<Index, 2> glimpse_size(width_, height_);
DSizes<Index, 2> input_size(input_width, input_height);
typedef typename internal::remove_const<
typename internal::traits<Input>::Scalar>::type Scalar;
for (int j = 0; j < num_channels; ++j) {
TensorFixedSize<Scalar, Sizes<> > mean;
mean.device(device) = input.template chip<3>(i)
.template chip<0>(j)
.template cast<float>()
.mean();
TensorFixedSize<float, Sizes<> > sigma;
sigma.device(device) =
(input.template chip<3>(i)
.template chip<0>(j)
.template cast<float>() -
mean.reshape(Sizes<1, 1>()).broadcast(input_size))
.square()
.mean()
.sqrt();
TensorFixedSize<Scalar, Sizes<> > mini;
mini.device(device) =
input.template chip<3>(i).template chip<0>(j).minimum();
TensorFixedSize<float, Sizes<> > maxi;
maxi.device(device) =
input.template chip<3>(i).template chip<0>(j).maximum();
TensorMap<Tensor<float, 2> > tmp(NULL, glimpse_size);
output.template chip<3>(i).template chip<0>(j).device(device) =
(mean.reshape(Sizes<1, 1>()).broadcast(glimpse_size) +
(tmp.random(gen) *
sigma.reshape(Sizes<1, 1>()).broadcast(glimpse_size))
.template cast<Scalar>())
.cwiseMin(
maxi.reshape(Sizes<1, 1>()).broadcast(glimpse_size))
.cwiseMax(
mini.reshape(Sizes<1, 1>()).broadcast(glimpse_size));
}
}
// Copy the part of the glimpse that cover the input image if any.
if (glimpse_width == 0 || glimpse_height == 0) {
continue;
}
output.template chip<3>(i)
.slice(base_offset, slice_extent)
.device(device) =
input.template chip<3>(i).slice(slice_offset, slice_extent);
} else {
output.template chip<3>(i).device(device) =
input.template chip<3>(i).slice(slice_offset, slice_extent);
}
}
}
private:
const Index width_;
const Index height_;
const std::vector<IndexPair<float> > offsets_;
const bool normalized_;
const bool centered_;
const bool uniform_noise_;
};
} // namespace
template <typename Input>
EIGEN_ALWAYS_INLINE static const TensorCustomUnaryOp<
const GlimpseExtractionOp<typename internal::traits<Input>::Index>,
const Input>
ExtractGlimpses(const Input& input,
const typename internal::traits<Input>::Index width,
const typename internal::traits<Input>::Index height,
const std::vector<IndexPair<float> >& offsets,
const bool normalized = true, const bool centered = true,
const bool uniform_noise = true) {
EIGEN_STATIC_ASSERT(internal::traits<Input>::Layout == ColMajor,
YOU_MADE_A_PROGRAMMING_MISTAKE);
EIGEN_STATIC_ASSERT(internal::traits<Input>::NumDimensions == 4,
YOU_MADE_A_PROGRAMMING_MISTAKE);
typedef typename internal::traits<Input>::Index Index;
const GlimpseExtractionOp<Index> op(width, height, offsets, normalized,
centered, uniform_noise);
return input.customOp(op);
}
} // end namespace Eigen
#endif // TENSORFLOW_CORE_KERNELS_EIGEN_ATTENTION_H_
@@ -0,0 +1,610 @@
/* Copyright 2015 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
#ifndef TENSORFLOW_CORE_KERNELS_EIGEN_BACKWARD_CUBOID_CONVOLUTIONS_H_
#define TENSORFLOW_CORE_KERNELS_EIGEN_BACKWARD_CUBOID_CONVOLUTIONS_H_
#include "third_party/eigen3/unsupported/Eigen/CXX11/Tensor"
#include "tensorflow/core/kernels/eigen_volume_patch.h"
namespace Eigen {
/** CuboidConvolutionBackwardInput
* \ingroup CXX11_NeuralNetworks_Module
*
* \brief Computes the backprop for the input of a 3D convolution.
*
* The output_backward parameter is expected to be a tensor with a rank of 4 or
* more (channels, depth, height, width, and optionally others)
* The kernel parameter is expected to be a 5D tensor (filters, channels,
* kernel_depth, kernel_height, kernel_width)
* output_backward and kernel have to be in the same layout.
*
* The dimensions of the result will be filters, depth, height, width (and
* others if applicable).
*
* It is possible to swap the order of the depth, width and height dimensions
* provided that the same order is used in the input, the kernel, and the
* output.
*
* All dimension orders above are given for col-major, and should be reversed
* for row-major.
*/
template <typename OutputBackward, typename Kernel>
EIGEN_ALWAYS_INLINE static const typename internal::conditional<
internal::traits<OutputBackward>::Layout == ColMajor,
TensorReshapingOp<
const DSizes<typename internal::traits<OutputBackward>::Index,
internal::traits<OutputBackward>::NumDimensions>,
const TensorContractionOp<
const array<
IndexPair<typename internal::traits<OutputBackward>::Index>, 1>,
const Eigen::TensorForcedEvalOp<const TensorReshapingOp<
const DSizes<typename internal::traits<OutputBackward>::Index,
2>,
const TensorShufflingOp<
const array<
typename internal::traits<OutputBackward>::Index, 5>,
const TensorReverseOp<const Eigen::array<bool, 5>,
const Kernel>>>>,
const TensorReshapingOp<
const DSizes<typename internal::traits<OutputBackward>::Index,
2>,
const TensorVolumePatchOp<Dynamic, Dynamic, Dynamic,
const OutputBackward>>>>,
TensorReshapingOp<
const DSizes<typename internal::traits<OutputBackward>::Index,
internal::traits<OutputBackward>::NumDimensions>,
const TensorContractionOp<
const array<
IndexPair<typename internal::traits<OutputBackward>::Index>, 1>,
const TensorReshapingOp<
const DSizes<typename internal::traits<OutputBackward>::Index,
2>,
const TensorVolumePatchOp<Dynamic, Dynamic, Dynamic,
const OutputBackward>>,
const Eigen::TensorForcedEvalOp<const TensorReshapingOp<
const DSizes<typename internal::traits<OutputBackward>::Index,
2>,
const TensorShufflingOp<
const array<
typename internal::traits<OutputBackward>::Index, 5>,
const TensorReverseOp<const Eigen::array<bool, 5>,
const Kernel>>>>>>>::type
CuboidConvolutionBackwardInput(
const Kernel& kernel, const OutputBackward& output_backward,
typename internal::traits<OutputBackward>::Index inputPlanes,
typename internal::traits<OutputBackward>::Index inputRows,
typename internal::traits<OutputBackward>::Index inputCols,
const DenseIndex plane_stride = 1, const DenseIndex row_stride = 1,
const DenseIndex col_stride = 1) {
typedef typename internal::traits<OutputBackward>::Index TensorIndex;
const TensorRef<const Tensor<typename internal::traits<Kernel>::Scalar,
internal::traits<Kernel>::NumDimensions,
internal::traits<Kernel>::Layout, TensorIndex>>
kern(kernel);
const TensorRef<
const Tensor<typename internal::traits<OutputBackward>::Scalar,
internal::traits<OutputBackward>::NumDimensions,
internal::traits<OutputBackward>::Layout, TensorIndex>>
out(output_backward);
EIGEN_STATIC_ASSERT(internal::traits<Kernel>::Layout ==
internal::traits<OutputBackward>::Layout,
YOU_MADE_A_PROGRAMMING_MISTAKE);
static const bool isColMajor =
(internal::traits<OutputBackward>::Layout == ColMajor);
static const int NumDims = internal::traits<OutputBackward>::NumDimensions;
// Number of filters to apply. This is the same as the output depth of the
// result
const TensorIndex kernelFilters =
isColMajor ? kern.dimensions()[0] : kern.dimensions()[4];
// Number of channels. This is the same as the input depth.
const TensorIndex kernelChannels =
isColMajor ? kern.dimensions()[1] : kern.dimensions()[3];
const TensorIndex kernelPlanes =
isColMajor ? kern.dimensions()[2] : kern.dimensions()[2];
const TensorIndex kernelRows =
isColMajor ? kern.dimensions()[3] : kern.dimensions()[1];
const TensorIndex kernelCols =
isColMajor ? kern.dimensions()[4] : kern.dimensions()[0];
const TensorIndex outputPlanes =
isColMajor ? out.dimensions()[1] : out.dimensions()[NumDims - 2];
const TensorIndex outputRows =
isColMajor ? out.dimensions()[2] : out.dimensions()[NumDims - 3];
const TensorIndex outputCols =
isColMajor ? out.dimensions()[3] : out.dimensions()[NumDims - 4];
// TODO(ezhulenev): Add support for inflated strides. Without inflated strides
// effective kernel planes/rows/cols are always the same as the kernel itself
// (see eigen_spatial_convolutions for details).
const TensorIndex kernelPlanesEff = kernelPlanes;
const TensorIndex kernelRowsEff = kernelRows;
const TensorIndex kernelColsEff = kernelCols;
// Computing the forward padding.
const TensorIndex forward_pad_top_z = numext::maxi<Index>(
0,
((outputPlanes - 1) * plane_stride + kernelPlanesEff - inputPlanes) / 2);
const TensorIndex forward_pad_top = numext::maxi<Index>(
0, ((outputRows - 1) * row_stride + kernelRowsEff - inputRows) / 2);
const TensorIndex forward_pad_left = numext::maxi<Index>(
0, ((outputCols - 1) * col_stride + kernelColsEff - inputCols) / 2);
const TensorIndex padding_top_z = kernelPlanesEff - 1 - forward_pad_top_z;
const TensorIndex padding_top = kernelRowsEff - 1 - forward_pad_top;
const TensorIndex padding_left = kernelColsEff - 1 - forward_pad_left;
const TensorIndex padding_bottom_z = inputPlanes -
(outputPlanes - 1) * plane_stride - 2 -
padding_top_z + kernelPlanesEff;
const TensorIndex padding_bottom = inputRows - (outputRows - 1) * row_stride -
2 - padding_top + kernelRowsEff;
const TensorIndex padding_right = inputCols - (outputCols - 1) * col_stride -
2 - padding_left + kernelColsEff;
eigen_assert(padding_top_z >= 0);
eigen_assert(padding_top >= 0);
eigen_assert(padding_left >= 0);
eigen_assert(padding_bottom_z >= 0);
eigen_assert(padding_bottom >= 0);
eigen_assert(padding_right >= 0);
// The kernel has dimensions :
// filters x channels x patch_planes x patch_rows x patch_cols.
// We need to reverse the kernel along the spatial dimensions.
Eigen::array<bool, 5> kernel_reverse;
if (isColMajor) {
kernel_reverse[0] = false;
kernel_reverse[1] = false;
kernel_reverse[2] = true;
kernel_reverse[3] = true;
kernel_reverse[4] = true;
} else {
kernel_reverse[0] = true;
kernel_reverse[1] = true;
kernel_reverse[2] = true;
kernel_reverse[3] = false;
kernel_reverse[4] = false;
}
// Reorder the dimensions to:
// filters x patch_planes x patch_rows x patch_cols x channels
array<TensorIndex, 5> kernel_shuffle;
if (isColMajor) {
// From: filters x channels x planes x rows x cols
// To: filters x planes x rows x cols x channels
kernel_shuffle[0] = 0;
kernel_shuffle[1] = 2;
kernel_shuffle[2] = 3;
kernel_shuffle[3] = 4;
kernel_shuffle[4] = 1;
} else {
// From: cols x rows x planes x channels x filters
// To: channels x cols x rows x planes x filters
kernel_shuffle[0] = 3;
kernel_shuffle[1] = 0;
kernel_shuffle[2] = 1;
kernel_shuffle[3] = 2;
kernel_shuffle[4] = 4;
}
// Collapse the dims
DSizes<TensorIndex, 2> kernel_dims;
if (isColMajor) {
kernel_dims[0] = kernelFilters * kernelPlanes * kernelRows * kernelCols;
kernel_dims[1] = kernelChannels;
} else {
kernel_dims[1] = kernelFilters * kernelPlanes * kernelRows * kernelCols;
kernel_dims[0] = kernelChannels;
}
// The output_backward has dimensions out_depth X out_planes X out_rows X
// out_cols X OTHERS
// When we extract the image patches from output_backward, it will have
// dimensions:
// out_depth X (patch_planes * patch_rows * patch_cols) X (input_planes *
// input_rows * input_cols * OTHERS)
DSizes<TensorIndex, 2> pre_contract_dims;
if (isColMajor) {
pre_contract_dims[0] =
kernelFilters * kernelPlanes * kernelRows * kernelCols;
pre_contract_dims[1] = inputPlanes * inputRows * inputCols;
for (int i = 4; i < NumDims; ++i) {
pre_contract_dims[1] *= out.dimension(i);
}
} else {
pre_contract_dims[1] =
kernelFilters * kernelPlanes * kernelRows * kernelCols;
pre_contract_dims[0] = inputPlanes * inputRows * inputCols;
for (int i = 0; i < NumDims - 4; ++i) {
pre_contract_dims[0] *= out.dimension(i);
}
}
// We will contract along the collapsed dimension that contains the
// kernelFilters, kernelPlanes, kernelRows and kernelCols.
array<IndexPair<TensorIndex>, 1> contract_dims;
if (isColMajor) {
// col-major: kernel.contract(output.patches)
contract_dims[0] = IndexPair<TensorIndex>(0, 0);
} else {
// row-major: output.patches.contract(kernel)
contract_dims[0] = IndexPair<TensorIndex>(1, 1);
}
// Post contraction, the dimensions of the input_backprop is
// channels X input_planes X input_rows X input_cols X OTHERS
DSizes<TensorIndex, NumDims> post_contract_dims;
if (isColMajor) {
post_contract_dims[0] = kernelChannels;
post_contract_dims[1] = inputPlanes;
post_contract_dims[2] = inputRows;
post_contract_dims[3] = inputCols;
for (int i = 4; i < NumDims; ++i) {
post_contract_dims[i] = out.dimension(i);
}
} else {
post_contract_dims[NumDims - 1] = kernelChannels;
post_contract_dims[NumDims - 2] = inputPlanes;
post_contract_dims[NumDims - 3] = inputRows;
post_contract_dims[NumDims - 4] = inputCols;
for (int i = 0; i < NumDims - 4; ++i) {
post_contract_dims[i] = out.dimension(i);
}
}
return choose(
Cond<internal::traits<OutputBackward>::Layout == ColMajor>(),
kernel.reverse(kernel_reverse)
.shuffle(kernel_shuffle)
.reshape(kernel_dims)
.eval()
.contract(output_backward
.extract_volume_patches(
kernelPlanes, kernelRows, kernelCols, 1, 1, 1,
plane_stride, row_stride, col_stride, padding_top_z,
padding_bottom_z, padding_top, padding_bottom,
padding_left, padding_right)
.reshape(pre_contract_dims),
contract_dims)
.reshape(post_contract_dims),
output_backward
.extract_volume_patches(kernelPlanes, kernelRows, kernelCols, 1, 1, 1,
plane_stride, row_stride, col_stride,
padding_top_z, padding_bottom_z, padding_top,
padding_bottom, padding_left, padding_right)
.reshape(pre_contract_dims)
.contract(kernel.reverse(kernel_reverse)
.shuffle(kernel_shuffle)
.reshape(kernel_dims)
.eval(),
contract_dims)
.reshape(post_contract_dims));
}
/** CuboidConvolutionBackwardKernel
* \ingroup CXX11_NeuralNetworks_Module
*
* \brief Computes the backprop for the filter of a 3D convolution.
*
* The output_backward parameter is expected to be a tensor with a rank of 4 or
* more (channels, depth, height, width, and optionally others)
* The kernel parameter is expected to be a 4D tensor (filters, channels,
* kernel_depth, kernel_height, kernel_width)
* output_backward and kernel have to be in the same layout.
*
* The dimensions of the result will be filters, depth, height, width (and
* others if applicable).
*
* It is possible to swap the order of the depth, width and height dimensions
* provided that the same order is used in the input, the kernel, and the
* output.
*
* All dimension orders above are given for col-major, and should be reversed
* for row-major.
*/
template <typename OutputBackward, typename Input>
EIGEN_ALWAYS_INLINE static const typename internal::conditional<
internal::traits<Input>::Layout == ColMajor,
const TensorReverseOp<
const Eigen::array<typename internal::traits<Input>::Index,
internal::traits<Input>::NumDimensions>,
const Eigen::TensorShufflingOp<
const Eigen::array<typename internal::traits<Input>::Index,
internal::traits<Input>::NumDimensions>,
const Eigen::TensorReshapingOp<
const Eigen::DSizes<typename internal::traits<Input>::Index,
internal::traits<Input>::NumDimensions>,
const TensorContractionOp<
const array<
IndexPair<typename internal::traits<Input>::Index>, 1>,
const Eigen::TensorForcedEvalOp<const TensorReshapingOp<
const DSizes<typename internal::traits<Input>::Index,
2>,
const Eigen::TensorShufflingOp<
const Eigen::array<
typename internal::traits<Input>::Index,
internal::traits<Input>::NumDimensions>,
const OutputBackward>>>,
const TensorReshapingOp<
const DSizes<typename internal::traits<Input>::Index,
2>,
const TensorVolumePatchOp<
Dynamic, Dynamic, Dynamic,
const Eigen::TensorForcedEvalOp<
const Eigen::TensorShufflingOp<
const Eigen::array<
typename internal::traits<Input>::Index,
internal::traits<Input>::NumDimensions>,
const Input>>>>>>>>,
const TensorReverseOp<
const Eigen::array<typename internal::traits<Input>::Index,
internal::traits<Input>::NumDimensions>,
const Eigen::TensorShufflingOp<
const Eigen::array<typename internal::traits<Input>::Index,
internal::traits<Input>::NumDimensions>,
const Eigen::TensorReshapingOp<
const Eigen::DSizes<typename internal::traits<Input>::Index,
internal::traits<Input>::NumDimensions>,
const TensorContractionOp<
const array<
IndexPair<typename internal::traits<Input>::Index>, 1>,
const TensorReshapingOp<
const DSizes<typename internal::traits<Input>::Index,
2>,
const TensorVolumePatchOp<
Dynamic, Dynamic, Dynamic,
const Eigen::TensorForcedEvalOp<
const Eigen::TensorShufflingOp<
const Eigen::array<
typename internal::traits<Input>::Index,
internal::traits<Input>::NumDimensions>,
const Input>>>>,
const Eigen::TensorForcedEvalOp<const TensorReshapingOp<
const DSizes<typename internal::traits<Input>::Index,
2>,
const Eigen::TensorShufflingOp<
const Eigen::array<
typename internal::traits<Input>::Index,
internal::traits<Input>::NumDimensions>,
const OutputBackward>>>>>>>>::type
CuboidConvolutionBackwardKernel(
const Input& input, const OutputBackward& output_backward,
typename internal::traits<Input>::Index kernelPlanes,
typename internal::traits<Input>::Index kernelRows,
typename internal::traits<Input>::Index kernelCols,
const DenseIndex stridePlanes = 1, const DenseIndex strideRows = 1,
const DenseIndex strideCols = 1) {
typedef typename internal::traits<Input>::Index TensorIndex;
TensorRef<Tensor<typename internal::traits<Input>::Scalar,
internal::traits<Input>::NumDimensions,
internal::traits<Input>::Layout, TensorIndex>>
in(input);
TensorRef<Tensor<typename internal::traits<OutputBackward>::Scalar,
internal::traits<OutputBackward>::NumDimensions,
internal::traits<OutputBackward>::Layout, TensorIndex>>
out(output_backward);
EIGEN_STATIC_ASSERT(internal::traits<Input>::Layout ==
internal::traits<OutputBackward>::Layout,
YOU_MADE_A_PROGRAMMING_MISTAKE);
static const bool isColMajor = (internal::traits<Input>::Layout == ColMajor);
static const int NumDims = internal::traits<Input>::NumDimensions;
EIGEN_STATIC_ASSERT(internal::traits<Input>::NumDimensions ==
internal::traits<OutputBackward>::NumDimensions,
YOU_MADE_A_PROGRAMMING_MISTAKE);
// We do not support higher dimensional backward convolutions, or convolutions
// without batch dimension.
// TODO(ezhulenev): Relax this constraint, and turn on tests without batch
// dimension in eigen_backward_cuboid_convolutions_test.cc.
EIGEN_STATIC_ASSERT(internal::traits<Input>::NumDimensions == 5,
YOU_MADE_A_PROGRAMMING_MISTAKE);
const TensorIndex inputPlanes =
isColMajor ? in.dimension(1) : in.dimension(NumDims - 2);
const TensorIndex inputRows =
isColMajor ? in.dimension(2) : in.dimension(NumDims - 3);
const TensorIndex inputCols =
isColMajor ? in.dimension(3) : in.dimension(NumDims - 4);
const TensorIndex outputPlanes =
isColMajor ? out.dimension(1) : out.dimension(NumDims - 2);
const TensorIndex outputRows =
isColMajor ? out.dimension(2) : out.dimension(NumDims - 3);
const TensorIndex outputCols =
isColMajor ? out.dimension(3) : out.dimension(NumDims - 4);
// Number of filters. This is the same as the output depth.
const TensorIndex kernelFilters =
isColMajor ? out.dimension(0) : out.dimension(NumDims - 1);
// Number of channels. This is the same as the input depth.
const TensorIndex kernelChannels =
isColMajor ? in.dimension(0) : in.dimension(NumDims - 1);
// Number of batches in the input tensor.
const TensorIndex batch =
isColMajor ? in.dimension(4) : in.dimension(NumDims - 5);
// TODO(ezhulenev): Add support for inflated strides. Without inflated strides
// effective kernel planes/rows/cols are always the same as the kernel itself
// (see eigen_spatial_convolutions for details).
const TensorIndex kernelPlanesEff = kernelPlanes;
const TensorIndex kernelRowsEff = kernelRows;
const TensorIndex kernelColsEff = kernelCols;
// Compute forward padding from input and output_backward dimensions.
const TensorIndex padPlanes = numext::maxi<Index>(
0, (outputPlanes - 1) * stridePlanes + kernelPlanesEff - inputPlanes);
const TensorIndex padRows = numext::maxi<Index>(
0, (outputRows - 1) * strideRows + kernelRowsEff - inputRows);
const TensorIndex padCols = numext::maxi<Index>(
0, (outputCols - 1) * strideCols + kernelColsEff - inputCols);
const TensorIndex padding_top_z = padPlanes / 2;
const TensorIndex padding_top = padRows / 2;
const TensorIndex padding_left = padCols / 2;
// Compute paddings for output_backward before extracting patches.
const auto expanded_out_planes = (outputPlanes - 1) * stridePlanes + 1;
const auto expanded_out_rows = (outputRows - 1) * strideRows + 1;
const auto expanded_out_cols = (outputCols - 1) * strideCols + 1;
const auto padded_out_planes = inputPlanes + kernelPlanes - 1;
const auto padded_out_rows = inputRows + kernelRows - 1;
const auto padded_out_cols = inputCols + kernelCols - 1;
const auto top_pad_planes = kernelPlanes - 1 - padding_top_z;
const auto top_pad_rows = kernelRows - 1 - padding_top;
const auto left_pad_cols = kernelCols - 1 - padding_left;
const auto bottom_pad_planes =
padded_out_planes - expanded_out_planes - top_pad_planes;
const auto bottom_pad_rows =
padded_out_rows - expanded_out_rows - top_pad_rows;
const auto right_pad_cols =
padded_out_cols - expanded_out_cols - left_pad_cols;
// Reorder output_backward dimensions.
array<TensorIndex, 5> output_backward_shuffle;
if (isColMajor) {
// From: [out_depth, out_planes, out_rows, out_cols, batch]
// To: [batch, out_planes, out_rows, out_cols, out_depth]
output_backward_shuffle = {4, 1, 2, 3, 0};
} else {
// From: [batch, out_cols, out_rows, out_planes, out_depth]
// To: [out_depth, out_cols, out_rows, out_planes, batch]
output_backward_shuffle = {4, 1, 2, 3, 0};
}
// Reorder input dimensions.
array<TensorIndex, 5> input_shuffle;
if (isColMajor) {
// From: [in_depth, in_planes, in_rows, in_cols, batch]
// To: [in_depth, batch, in_planes, in_rows, in_cols]
input_shuffle = {0, 4, 1, 2, 3};
} else {
// From: [batch, in_cols, in_rows, in_planes, in_depth]
// To: [in_cols, in_rows, in_planes, batch, in_depth]
input_shuffle = {1, 2, 3, 0, 4};
}
// Input is playing the role of a "kernel" in this convolution.
DSizes<TensorIndex, 2> input_dims;
if (isColMajor) {
input_dims[0] = kernelChannels;
input_dims[1] = batch * inputPlanes * inputRows * inputCols;
} else {
input_dims[1] = kernelChannels;
input_dims[0] = inputCols * inputRows * inputPlanes * batch;
}
// Molds the output of the patch extraction result into a 2D tensor:
// - the first dimension (dims[0]): the patch values to be multiplied with the
// kernels
// - the second dimension (dims[1]): everything else
DSizes<TensorIndex, 2> pre_contract_dims;
if (isColMajor) {
pre_contract_dims[0] = batch * inputPlanes * inputRows * inputCols;
pre_contract_dims[1] =
kernelPlanes * kernelRows * kernelCols * kernelFilters;
} else {
pre_contract_dims[1] = inputCols * inputRows * inputPlanes * batch;
pre_contract_dims[0] =
kernelFilters * kernelCols * kernelRows * kernelPlanes;
}
// We will contract along the collapsed dimension that contains the
// batch, inputPlanes, inputRows and inputCols.
array<IndexPair<TensorIndex>, 1> contract_dims;
contract_dims[0] = IndexPair<TensorIndex>(1, 0);
// Dimensions after contraction.
DSizes<TensorIndex, NumDims> post_contract_dims;
if (isColMajor) {
post_contract_dims[0] = kernelChannels;
post_contract_dims[1] = kernelPlanes;
post_contract_dims[2] = kernelRows;
post_contract_dims[3] = kernelCols;
post_contract_dims[4] = kernelFilters;
} else {
post_contract_dims[0] = kernelFilters;
post_contract_dims[1] = kernelCols;
post_contract_dims[2] = kernelRows;
post_contract_dims[3] = kernelPlanes;
post_contract_dims[4] = kernelChannels;
}
// Reorder output of contraction to valid filter shape.
array<TensorIndex, 5> kernel_shuffle;
if (isColMajor) {
// From: [in_depth, kernel_planes, kernel_rows, kernel_cols, out_depth]
// To: [out_depth, in_depth, kernel_planes, kernel_rows, kernel_cols]
kernel_shuffle = {4, 0, 1, 2, 3};
} else {
// From: [out_depth, kernel_cols, kernel_rows, kernel_planes, in_depth]
// To: [kernel_cols, kernel_rows, kernel_planes, in_depth, out_depth]
kernel_shuffle = {1, 2, 3, 4, 0};
}
// Reverse kernel backprop dimensions.
array<TensorIndex, 5> kernel_reverse;
if (isColMajor) {
kernel_reverse = {false, false, true, true, true};
} else {
kernel_reverse = {true, true, true, false, false};
}
// Create convolution input (aka source of patches) from output backward
// tensor by shuffling dimensions.
const auto the_input =
output_backward.shuffle(output_backward_shuffle).eval();
// Create convolution kernel (aka filter) from input by shuffling and
// reshaping.
const auto the_kernel =
input.shuffle(input_shuffle).reshape(input_dims).eval();
return choose(Cond<internal::traits<Input>::Layout == ColMajor>(),
the_kernel.contract(
the_input
.extract_volume_patches(
inputPlanes, inputRows, inputCols, 1, 1, 1,
stridePlanes, strideRows, strideCols,
top_pad_planes, bottom_pad_planes, top_pad_rows,
bottom_pad_rows, left_pad_cols, right_pad_cols)
.reshape(pre_contract_dims),
contract_dims),
the_input
.extract_volume_patches(
inputPlanes, inputRows, inputCols, 1, 1, 1,
stridePlanes, strideRows, strideCols, top_pad_planes,
bottom_pad_planes, top_pad_rows, bottom_pad_rows,
left_pad_cols, right_pad_cols)
.reshape(pre_contract_dims)
.contract(the_kernel, contract_dims))
.reshape(post_contract_dims)
.shuffle(kernel_shuffle)
.reverse(kernel_reverse);
}
} // end namespace Eigen
#endif // TENSORFLOW_CORE_KERNELS_EIGEN_BACKWARD_CUBOID_CONVOLUTIONS_H_
@@ -0,0 +1,502 @@
/* Copyright 2015 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
#ifndef TENSORFLOW_CORE_KERNELS_EIGEN_BACKWARD_SPATIAL_CONVOLUTIONS_H_
#define TENSORFLOW_CORE_KERNELS_EIGEN_BACKWARD_SPATIAL_CONVOLUTIONS_H_
#include "third_party/eigen3/unsupported/Eigen/CXX11/Tensor"
namespace Eigen {
/** SpatialConvolutionBackwardInput
* \ingroup CXX11_NeuralNetworks_Module
*
* \brief Computes the backprop for the input of a 2D convolution.
*
* The output_backward parameter is expected to be a tensor with a rank of 3 or
* more (channels, height, width, and optionally others)
* The kernel parameter is expected to be a 4D tensor (filters, channels,
* kernel_height, kernel_width)
* The output_backward and the kernel must both be in col-major layout. The
* result will also be in col-major layout.
*
* If row_in_stride, col_in_stride > 1, then applies convolution with holes
* (aka atrous convolution), sampling every row_in_stride, col_in_stride input
* pixels.
*
* The result can be assigned to a tensor of rank equal to the rank of the
* output_backward. The dimensions of the result will be filters, height, width
* (and others if applicable).
*
* It is possible to swap the order of the width and height dimensions provided
* that the same order is used in the input, the kernel, and the output.
*
*/
#ifdef EIGEN_HAS_INDEX_LIST
typedef IndexList<type2index<0>, type2index<0>, type2index<1>, type2index<1> >
ReverseColMajor;
typedef IndexList<type2index<1>, type2index<1>, type2index<0>, type2index<0> >
ReverseRowMajor;
#else
typedef array<bool, 4> ReverseColMajor;
typedef array<bool, 4> ReverseRowMajor;
#endif
template <typename OutputBackward, typename Kernel>
EIGEN_ALWAYS_INLINE static const typename internal::conditional<
internal::traits<OutputBackward>::Layout == ColMajor,
TensorReshapingOp<
const DSizes<typename internal::traits<OutputBackward>::Index,
internal::traits<OutputBackward>::NumDimensions>,
const TensorContractionOp<
const array<
IndexPair<typename internal::traits<OutputBackward>::Index>, 1>,
const Eigen::TensorForcedEvalOp<const TensorReshapingOp<
const DSizes<typename internal::traits<OutputBackward>::Index,
2>,
const TensorShufflingOp<
const array<
typename internal::traits<OutputBackward>::Index, 4>,
const TensorReverseOp<const ReverseColMajor,
const Kernel> > > >,
const TensorReshapingOp<
const DSizes<typename internal::traits<OutputBackward>::Index,
2>,
const TensorImagePatchOp<Dynamic, Dynamic,
const OutputBackward> > > >,
TensorReshapingOp<
const DSizes<typename internal::traits<OutputBackward>::Index,
internal::traits<OutputBackward>::NumDimensions>,
const TensorContractionOp<
const array<
IndexPair<typename internal::traits<OutputBackward>::Index>, 1>,
const TensorReshapingOp<
const DSizes<typename internal::traits<OutputBackward>::Index,
2>,
const TensorImagePatchOp<Dynamic, Dynamic,
const OutputBackward> >,
const Eigen::TensorForcedEvalOp<const TensorReshapingOp<
const DSizes<typename internal::traits<OutputBackward>::Index,
2>,
const TensorShufflingOp<
const array<
typename internal::traits<OutputBackward>::Index, 4>,
const TensorReverseOp<const ReverseRowMajor,
const Kernel> > > > > > >::type
SpatialConvolutionBackwardInput(
const Kernel& kernel, const OutputBackward& output_backward,
typename internal::traits<OutputBackward>::Index inputRows,
typename internal::traits<OutputBackward>::Index inputCols,
const DenseIndex row_stride = 1, const DenseIndex col_stride = 1,
const DenseIndex row_in_stride = 1, const DenseIndex col_in_stride = 1) {
typedef typename internal::traits<OutputBackward>::Index TensorIndex;
typedef typename internal::traits<OutputBackward>::Scalar OutScalar;
TensorRef<Tensor<typename internal::traits<Kernel>::Scalar,
internal::traits<Kernel>::NumDimensions,
internal::traits<Kernel>::Layout, TensorIndex> >
kern(kernel);
TensorRef<Tensor<OutScalar, internal::traits<OutputBackward>::NumDimensions,
internal::traits<OutputBackward>::Layout, TensorIndex> >
out(output_backward);
EIGEN_STATIC_ASSERT(internal::traits<Kernel>::Layout ==
internal::traits<OutputBackward>::Layout,
YOU_MADE_A_PROGRAMMING_MISTAKE);
static const bool isColMajor =
(internal::traits<OutputBackward>::Layout == ColMajor);
static const int NumDims = internal::traits<OutputBackward>::NumDimensions;
// Number of filters to apply. This is the same as the output depth of the
// result
const TensorIndex kernelFilters =
isColMajor ? kern.dimensions()[0] : kern.dimensions()[3];
// Number of channels. This is the same as the input depth.
const TensorIndex kernelChannels =
isColMajor ? kern.dimensions()[1] : kern.dimensions()[2];
const TensorIndex kernelRows =
isColMajor ? kern.dimensions()[2] : kern.dimensions()[1];
const TensorIndex kernelCols =
isColMajor ? kern.dimensions()[3] : kern.dimensions()[0];
// This is the effective kernel size, taking into account the (*_in_stride -
// 1) zero-values
// inserted between consecutive kernel elements in atrous convolution
const TensorIndex kernelRowsEff =
kernelRows + (kernelRows - 1) * (row_in_stride - 1);
const TensorIndex kernelColsEff =
kernelCols + (kernelCols - 1) * (col_in_stride - 1);
const TensorIndex outputRows = isColMajor
? output_backward.dimension(1)
: output_backward.dimension(NumDims - 2);
const TensorIndex outputCols = isColMajor
? output_backward.dimension(2)
: output_backward.dimension(NumDims - 3);
// Computing the forward padding
const TensorIndex forward_pad_top = numext::maxi<Index>(
0, ((outputRows - 1) * row_stride + kernelRowsEff - inputRows) / 2);
const TensorIndex forward_pad_left = numext::maxi<Index>(
0, ((outputCols - 1) * col_stride + kernelColsEff - inputCols) / 2);
const TensorIndex padding_top = kernelRowsEff - 1 - forward_pad_top;
const TensorIndex padding_left = kernelColsEff - 1 - forward_pad_left;
const TensorIndex padding_bottom = inputRows - (outputRows - 1) * row_stride -
2 - padding_top + kernelRowsEff;
const TensorIndex padding_right = inputCols - (outputCols - 1) * col_stride -
2 - padding_left + kernelColsEff;
eigen_assert(padding_top >= 0);
eigen_assert(padding_left >= 0);
eigen_assert(padding_bottom >= 0);
eigen_assert(padding_right >= 0);
// The kernel has dimensions filters X channels X patch_rows X patch_cols
// We need to reverse the kernel along dimensions corresponding to rows and
// cols.
// TODO(yangke): we can make things slightly faster by collapsing the
// dimensions
// where we don't reverse. Try that once we have a faster compiler.
typedef typename internal::conditional<isColMajor, ReverseColMajor,
ReverseRowMajor>::type Reverse;
Reverse kernel_reverse;
#ifndef EIGEN_HAS_INDEX_LIST
if (isColMajor) {
kernel_reverse[0] = false;
kernel_reverse[1] = false;
kernel_reverse[2] = true;
kernel_reverse[3] = true;
} else {
kernel_reverse[0] = true;
kernel_reverse[1] = true;
kernel_reverse[2] = false;
kernel_reverse[3] = false;
}
#endif
// Reorder the dimensions to:
// filters x patch_rows x patch_cols x channels
array<TensorIndex, 4> kernel_shuffle;
if (isColMajor) {
// From: filters x channels x rows x cols
// To: filters x rows x cols x channels
kernel_shuffle[0] = 0;
kernel_shuffle[1] = 2;
kernel_shuffle[2] = 3;
kernel_shuffle[3] = 1;
} else {
// From: cols x rows x channels x filters
// To: channels x cols x rows x filters
kernel_shuffle[0] = 2;
kernel_shuffle[1] = 0;
kernel_shuffle[2] = 1;
kernel_shuffle[3] = 3;
}
// Collapse the dims
DSizes<TensorIndex, 2> kernel_dims;
if (isColMajor) {
kernel_dims[0] = kernelFilters * kernelRows * kernelCols;
kernel_dims[1] = kernelChannels;
} else {
kernel_dims[1] = kernelFilters * kernelRows * kernelCols;
kernel_dims[0] = kernelChannels;
}
// The output_backward has dimensions out_depth X out_rows X out_cols X OTHERS
// When we extract the image patches from output_backward, it will have
// dimensions
// out_depth X (patch_rows * patch_cols) X (input_rows * input_cols *
// OTHERS)
DSizes<TensorIndex, 2> pre_contract_dims;
if (isColMajor) {
pre_contract_dims[0] = kernelFilters * kernelRows * kernelCols;
pre_contract_dims[1] = inputRows * inputCols;
for (int i = 3; i < NumDims; ++i) {
pre_contract_dims[1] *= out.dimension(i);
}
} else {
pre_contract_dims[1] = kernelFilters * kernelRows * kernelCols;
pre_contract_dims[0] = inputRows * inputCols;
for (int i = 0; i < NumDims - 3; ++i) {
pre_contract_dims[0] *= out.dimension(i);
}
}
// We will contract along the collapsed dimension that contains the
// kernelFilters, the kernelRows and the kernelCols.
array<IndexPair<TensorIndex>, 1> contract_dims;
if (isColMajor) {
// col-major: kernel.contract(output.patches)
contract_dims[0] = IndexPair<TensorIndex>(0, 0);
} else {
// row-major: output.patches.contract(kernel)
contract_dims[0] = IndexPair<TensorIndex>(1, 1);
}
// Post contraction, the dimensions of the input_backprop is
// channels X input_rows X input_cols X OTHERS
DSizes<TensorIndex, NumDims> post_contract_dims;
if (isColMajor) {
post_contract_dims[0] = kernelChannels;
post_contract_dims[1] = inputRows;
post_contract_dims[2] = inputCols;
for (int i = 3; i < NumDims; ++i) {
post_contract_dims[i] = out.dimension(i);
}
} else {
post_contract_dims[NumDims - 1] = kernelChannels;
post_contract_dims[NumDims - 2] = inputRows;
post_contract_dims[NumDims - 3] = inputCols;
for (int i = 0; i < NumDims - 3; ++i) {
post_contract_dims[i] = out.dimension(i);
}
}
return choose(
Cond<internal::traits<OutputBackward>::Layout == ColMajor>(),
kernel.reverse(kernel_reverse)
.shuffle(kernel_shuffle)
.reshape(kernel_dims)
.eval()
.contract(
output_backward
.extract_image_patches(
kernelRows, kernelCols, 1, 1, row_in_stride,
col_in_stride, row_stride, col_stride, padding_top,
padding_bottom, padding_left, padding_right, OutScalar(0))
.reshape(pre_contract_dims),
contract_dims)
.reshape(post_contract_dims),
output_backward
.extract_image_patches(kernelRows, kernelCols, 1, 1, row_in_stride,
col_in_stride, row_stride, col_stride,
padding_top, padding_bottom, padding_left,
padding_right, OutScalar(0))
.reshape(pre_contract_dims)
.contract(kernel.reverse(kernel_reverse)
.shuffle(kernel_shuffle)
.reshape(kernel_dims)
.eval(),
contract_dims)
.reshape(post_contract_dims));
}
/** SpatialConvolutionBackwardKernel
* \ingroup CXX11_NeuralNetworks_Module
*
* \brief Computes the backprop for the filter of a 2D convolution.
*
* The output_backward parameter is expected to be a tensor with a rank of 3 or
* more (channels, height, width, and optionally others)
* The kernel parameter is expected to be a 4D tensor (filters, channels,
* kernel_height, kernel_width)
* The output_backward and the kernel must both be in col-major layout. The
* result will also be in col-major layout.
*
* If row_in_stride, col_stride > 1, then applies convolution with holes (aka
* atrous convolution), sampling every row_in_stride, col_in_stride input
* pixels.
*
* The result can be assigned to a tensor of rank equal to the rank of the
* output_backward. The dimensions of the result will be filters, height, width
* (and others if applicable).
*
* It is possible to swap the order of the width and height dimensions provided
* that the same order is used in the input, the kernel, and the output.
*
*/
template <typename OutputBackward, typename Input>
EIGEN_ALWAYS_INLINE static const typename internal::conditional<
internal::traits<OutputBackward>::Layout == ColMajor,
TensorReshapingOp<
const DSizes<typename internal::traits<Input>::Index, 4>,
const TensorContractionOp<
const array<IndexPair<typename internal::traits<Input>::Index>, 1>,
const TensorReshapingOp<
const DSizes<typename internal::traits<Input>::Index, 2>,
const OutputBackward>,
const TensorReshapingOp<
const DSizes<typename internal::traits<Input>::Index, 2>,
const TensorImagePatchOp<Dynamic, Dynamic, const Input> > > >,
TensorReshapingOp<
const DSizes<typename internal::traits<Input>::Index, 4>,
const TensorContractionOp<
const array<IndexPair<typename internal::traits<Input>::Index>, 1>,
const TensorReshapingOp<
const DSizes<typename internal::traits<Input>::Index, 2>,
const TensorImagePatchOp<Dynamic, Dynamic, const Input> >,
const TensorReshapingOp<
const DSizes<typename internal::traits<Input>::Index, 2>,
const OutputBackward> > > >::type
SpatialConvolutionBackwardKernel(
const Input& input, const OutputBackward& output_backward,
typename internal::traits<Input>::Index kernelRows,
typename internal::traits<Input>::Index kernelCols,
const DenseIndex row_stride = 1, const DenseIndex col_stride = 1,
const DenseIndex row_in_stride = 1, const DenseIndex col_in_stride = 1) {
typedef typename internal::traits<Input>::Index TensorIndex;
typedef typename internal::traits<OutputBackward>::Scalar OutScalar;
TensorRef<Tensor<typename internal::traits<Input>::Scalar,
internal::traits<Input>::NumDimensions,
internal::traits<Input>::Layout, TensorIndex> >
in(input);
TensorRef<Tensor<OutScalar, internal::traits<OutputBackward>::NumDimensions,
internal::traits<OutputBackward>::Layout, TensorIndex> >
out(output_backward);
EIGEN_STATIC_ASSERT(internal::traits<Input>::Layout ==
internal::traits<OutputBackward>::Layout,
YOU_MADE_A_PROGRAMMING_MISTAKE);
// stride and in_stride cannot both be larger than 1
eigen_assert(!(row_stride > 1 && row_in_stride > 1) &&
!(col_stride > 1 && col_in_stride > 1));
static const bool isColMajor = (internal::traits<Input>::Layout == ColMajor);
static const int NumDims = internal::traits<Input>::NumDimensions;
EIGEN_STATIC_ASSERT(internal::traits<Input>::NumDimensions ==
internal::traits<OutputBackward>::NumDimensions,
YOU_MADE_A_PROGRAMMING_MISTAKE);
const TensorIndex inputRows =
isColMajor ? in.dimension(1) : in.dimension(NumDims - 2);
const TensorIndex inputCols =
isColMajor ? in.dimension(2) : in.dimension(NumDims - 3);
const TensorIndex outputRows = isColMajor
? output_backward.dimension(1)
: output_backward.dimension(NumDims - 2);
const TensorIndex outputCols = isColMajor
? output_backward.dimension(2)
: output_backward.dimension(NumDims - 3);
// Number of filters to apply. This is the same as the output depth of the
// result
const TensorIndex kernelFilters =
isColMajor ? out.dimensions()[0] : out.dimensions()[NumDims - 1];
// Number of channels. This is the same as the input depth.
const TensorIndex kernelChannels =
isColMajor ? in.dimensions()[0] : in.dimensions()[NumDims - 1];
// This is the effective kernel size, taking into account the (*_in_stride -
// 1) zero-values
// inserted between consecutive kernel elements in atrous convolution
const TensorIndex kernelRowsEff =
kernelRows + (kernelRows - 1) * (row_in_stride - 1);
const TensorIndex kernelColsEff =
kernelCols + (kernelCols - 1) * (col_in_stride - 1);
// Computing the forward padding
const TensorIndex padRows = numext::maxi<Index>(
0, (outputRows - 1) * row_stride + kernelRowsEff - inputRows);
const TensorIndex padCols = numext::maxi<Index>(
0, (outputCols - 1) * col_stride + kernelColsEff - inputCols);
const TensorIndex padding_top = padRows / 2;
const TensorIndex padding_bottom = padRows - padding_top;
const TensorIndex padding_left = padCols / 2;
const TensorIndex padding_right = padCols - padding_left;
// Reshaped out
DSizes<TensorIndex, 2> output_dims;
if (isColMajor) {
output_dims[0] = kernelFilters;
output_dims[1] = outputRows * outputCols;
for (int i = 3; i < NumDims; ++i) {
output_dims[1] *= out.dimension(i);
}
} else {
output_dims[1] = kernelFilters;
output_dims[0] = outputCols * outputRows;
for (int i = 0; i < NumDims - 3; ++i) {
output_dims[0] *= out.dimension(i);
}
}
// Reshaped extract_image_patches(in)
DSizes<TensorIndex, 2> pre_contract_dims;
if (isColMajor) {
pre_contract_dims[0] = kernelChannels * kernelRows * kernelCols;
pre_contract_dims[1] = outputRows * outputCols;
for (int i = 3; i < NumDims; ++i) {
pre_contract_dims[1] *= in.dimension(i);
}
eigen_assert(output_dims[1] == pre_contract_dims[1]);
} else {
pre_contract_dims[1] = kernelCols * kernelRows * kernelChannels;
pre_contract_dims[0] = outputRows * outputCols;
for (int i = 0; i < NumDims - 3; ++i) {
pre_contract_dims[0] *= in.dimension(i);
}
eigen_assert(output_dims[0] == pre_contract_dims[0]);
}
// We will contract along the collapsed dimension that contains the
// outputCols, outputRows and OTHERS.
array<IndexPair<TensorIndex>, 1> contract_dims;
if (isColMajor) {
// col-major: output_backward.contract(input.patches)
contract_dims[0] = IndexPair<TensorIndex>(1, 1);
} else {
// row-major: input.patches.contract(output_backward)
contract_dims[0] = IndexPair<TensorIndex>(0, 0);
}
// After the contraction, the kernel will have the desired shape
// out_depth X in_shape X kernel_rows X kernel_cols
DSizes<TensorIndex, 4> kernel_dims;
if (isColMajor) {
kernel_dims[0] = kernelFilters;
kernel_dims[1] = kernelChannels;
kernel_dims[2] = kernelRows;
kernel_dims[3] = kernelCols;
} else {
kernel_dims[3] = kernelFilters;
kernel_dims[2] = kernelChannels;
kernel_dims[1] = kernelRows;
kernel_dims[0] = kernelCols;
}
return choose(
Cond<internal::traits<Input>::Layout == ColMajor>(),
output_backward.reshape(output_dims)
.contract(
input
.extract_image_patches(
kernelRows, kernelCols, row_stride, col_stride,
row_in_stride, col_in_stride, 1, 1, padding_top,
padding_bottom, padding_left, padding_right, OutScalar(0))
.reshape(pre_contract_dims),
contract_dims)
.reshape(kernel_dims),
input
.extract_image_patches(kernelRows, kernelCols, row_stride, col_stride,
row_in_stride, col_in_stride, 1, 1,
padding_top, padding_bottom, padding_left,
padding_right, OutScalar(0))
.reshape(pre_contract_dims)
.contract(output_backward.reshape(output_dims), contract_dims)
.reshape(kernel_dims));
}
} // end namespace Eigen
#endif // TENSORFLOW_CORE_KERNELS_EIGEN_BACKWARD_SPATIAL_CONVOLUTIONS_H_
File diff suppressed because it is too large. Load diff
@@ -0,0 +1,619 @@
/* Copyright 2015 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
#ifndef TENSORFLOW_CORE_KERNELS_EIGEN_POOLING_H_
#define TENSORFLOW_CORE_KERNELS_EIGEN_POOLING_H_
#include "third_party/eigen3/unsupported/Eigen/CXX11/Tensor"
#include "tensorflow/core/kernels/eigen_volume_patch.h"
namespace Eigen {
/** SpatialMaxPooling
* \ingroup CXX11_NeuralNetworks_Module
*
* \brief Applies a max-pooling over a multichannel input image.
*
* The input parameter is expected to be a with a rank of 4 (channels, height,
* width, others in col-major, and the reverse of that in row-major).
*
* The result can be assigned to a tensor of rank equal to the rank of the
* input. The dimensions of the result will be channels, height, width, and
* others (in col-major, and the reverse of that if the input was row-major).
*
* The order of the width and height dimensions can be swapped if needed.
*
*/
#if !defined(EIGEN_HAS_INDEX_LIST)
template <typename Input>
EIGEN_ALWAYS_INLINE static const TensorReshapingOp<
const Eigen::DSizes<typename internal::traits<Input>::Index,
internal::traits<Input>::NumDimensions>,
const TensorReductionOp<
internal::MaxReducer<typename internal::remove_const<
typename internal::traits<Input>::Scalar>::type>,
const Eigen::array<int, 2>,
const TensorImagePatchOp<Dynamic, Dynamic, const Input> > >
#else
template <typename Input>
EIGEN_ALWAYS_INLINE static const TensorReshapingOp<
const Eigen::DSizes<typename internal::traits<Input>::Index,
internal::traits<Input>::NumDimensions>,
const TensorReductionOp<
internal::MaxReducer<typename internal::remove_const<
typename internal::traits<Input>::Scalar>::type>,
typename internal::conditional<
internal::traits<Input>::Layout == ColMajor,
const Eigen::IndexList<Eigen::type2index<1>, Eigen::type2index<2> >,
const Eigen::IndexList<Eigen::type2index<2>,
Eigen::type2index<3> > >::type,
const TensorImagePatchOp<Dynamic, Dynamic, const Input> > >
#endif
SpatialMaxPooling(const Input& input, DenseIndex patchRows,
DenseIndex patchCols, DenseIndex strideRows,
DenseIndex strideCols, const PaddingType padding_type,
DenseIndex in_strideRows = 1, DenseIndex in_strideCols = 1) {
EIGEN_STATIC_ASSERT(internal::traits<Input>::NumDimensions == 4,
YOU_MADE_A_PROGRAMMING_MISTAKE);
typedef typename internal::traits<Input>::Index TensorIndex;
TensorRef<Tensor<typename internal::traits<Input>::Scalar,
internal::traits<Input>::NumDimensions,
internal::traits<Input>::Layout, TensorIndex> >
in(input);
const DenseIndex patchRowsEff =
patchRows + (patchRows - 1) * (in_strideRows - 1);
const DenseIndex patchColsEff =
patchCols + (patchCols - 1) * (in_strideCols - 1);
static const bool isColMajor = (internal::traits<Input>::Layout == ColMajor);
static const int idxRows = isColMajor ? 1 : 2;
static const int idxCols = isColMajor ? 2 : 1;
// Molds the output of the reduction into the shape expected by the user.
// (assuming col-major):
// - 1st dim: channels
// - 2nd dim: output height
// - 3rd dim: output width
// - 4th dim and beyond: everything else including batch size
Eigen::DSizes<TensorIndex, internal::traits<Input>::NumDimensions>
post_reduce_dims;
post_reduce_dims[0] = in.dimension(0);
if (padding_type == PADDING_VALID) {
post_reduce_dims[idxRows] = Eigen::divup(
static_cast<DenseIndex>(in.dimension(idxRows)) - patchRowsEff + 1,
strideRows);
post_reduce_dims[idxCols] = Eigen::divup(
static_cast<DenseIndex>(in.dimension(idxCols)) - patchColsEff + 1,
strideCols);
} else {
post_reduce_dims[idxRows] = Eigen::divup(
static_cast<DenseIndex>(in.dimension(idxRows)), strideRows);
post_reduce_dims[idxCols] = Eigen::divup(
static_cast<DenseIndex>(in.dimension(idxCols)), strideCols);
}
post_reduce_dims[3] = in.dimension(3);
#if !defined(EIGEN_HAS_INDEX_LIST)
// nvcc doesn't support cxx11
Eigen::array<int, 2> reduction_dims;
if (isColMajor) {
reduction_dims[0] = 1;
reduction_dims[1] = 2;
} else {
reduction_dims[0] = 2;
reduction_dims[1] = 3;
}
#else
// Take advantage of cxx11 to give the compiler information it can use to
// optimize the code.
typename internal::conditional<
internal::traits<Input>::Layout == ColMajor,
const Eigen::IndexList<Eigen::type2index<1>, Eigen::type2index<2> >,
const Eigen::IndexList<Eigen::type2index<2>,
Eigen::type2index<3> > >::type reduction_dims;
#endif
return input
.extract_image_patches(
patchRows, patchCols, strideRows, strideCols, in_strideRows,
in_strideCols, padding_type,
-Eigen::NumTraits<typename internal::remove_const<
typename internal::traits<Input>::Scalar>::type>::highest())
.maximum(reduction_dims)
.reshape(post_reduce_dims);
}
/** CuboidMaxPooling
* \ingroup CXX11_NeuralNetworks_Module
*
* \brief Applies a max-pooling over a multichannel input volume.
*
* The input parameter is expected to be a tensor with a rank of 5 (channels,
* depth, height, width, others in col-major, and the reverse of that in
* row-major).
*
* The result can be assigned to a tensor of rank equal to the rank of the
* input. The dimensions of the result will be channels, depth, height, width,
* and others (in col-major, and the reverse of that if the input was
* row-major).
*
* The order of the depth, width and height dimensions can be swapped if
* needed.
*
*/
#if !defined(EIGEN_HAS_INDEX_LIST)
template <typename Input>
EIGEN_ALWAYS_INLINE static const TensorReshapingOp<
const Eigen::DSizes<DenseIndex, internal::traits<Input>::NumDimensions>,
const TensorReductionOp<
internal::MaxReducer<float>, const Eigen::array<int, 1>,
const TensorReshapingOp<
const Eigen::DSizes<DenseIndex, 3>,
const TensorVolumePatchOp<Dynamic, Dynamic, Dynamic,
const Input> > > >
#else
template <typename Input>
EIGEN_ALWAYS_INLINE static const TensorReshapingOp<
const Eigen::DSizes<DenseIndex, internal::traits<Input>::NumDimensions>,
const TensorReductionOp<
internal::MaxReducer<float>,
const Eigen::IndexList<Eigen::type2index<1> >,
const TensorReshapingOp<
const Eigen::DSizes<DenseIndex, 3>,
const TensorVolumePatchOp<Dynamic, Dynamic, Dynamic,
const Input> > > >
#endif
CuboidMaxPooling(const Input& input, DenseIndex patchPlanes,
DenseIndex patchRows, DenseIndex patchCols,
DenseIndex stridePlanes, DenseIndex strideRows,
DenseIndex strideCols, const PaddingType padding_type) {
EIGEN_STATIC_ASSERT(internal::traits<Input>::NumDimensions == 5,
YOU_MADE_A_PROGRAMMING_MISTAKE);
static const bool isColMajor = (internal::traits<Input>::Layout == ColMajor);
typedef typename internal::traits<Input>::Index TensorIndex;
TensorRef<Tensor<typename internal::traits<Input>::Scalar,
internal::traits<Input>::NumDimensions,
internal::traits<Input>::Layout, TensorIndex> >
in(input);
static const int idxPlanes = isColMajor ? 1 : 3;
static const int idxRows = 2;
static const int idxCols = isColMajor ? 3 : 1;
// Molds the output of the reduction into the shape expected by the used
// (assuming col-major):
// - 1st dim: channels
// - 2nd dim: output depth
// - 3rd dim: output height
// - 4th dim: output width
// - 5th dim and beyond: everything else including batch size
Eigen::DSizes<DenseIndex, internal::traits<Input>::NumDimensions>
post_reduce_dims;
post_reduce_dims[0] = in.dimension(0);
if (padding_type == PADDING_VALID) {
post_reduce_dims[idxPlanes] = Eigen::divup(
static_cast<DenseIndex>(in.dimension(idxPlanes)) - patchPlanes + 1,
stridePlanes);
post_reduce_dims[idxRows] = Eigen::divup(
static_cast<DenseIndex>(in.dimension(idxRows)) - patchRows + 1,
strideRows);
post_reduce_dims[idxCols] = Eigen::divup(
static_cast<DenseIndex>(in.dimension(idxCols)) - patchCols + 1,
strideCols);
} else {
post_reduce_dims[idxPlanes] = Eigen::divup(
static_cast<DenseIndex>(in.dimension(idxPlanes)), stridePlanes);
post_reduce_dims[idxRows] = Eigen::divup(
static_cast<DenseIndex>(in.dimension(idxRows)), strideRows);
post_reduce_dims[idxCols] = Eigen::divup(
static_cast<DenseIndex>(in.dimension(idxCols)), strideCols);
}
post_reduce_dims[4] = in.dimension(4);
Eigen::DSizes<DenseIndex, 3> pre_reduce_dims;
pre_reduce_dims[1] = patchRows * patchCols * patchPlanes;
if (isColMajor) {
pre_reduce_dims[0] = post_reduce_dims[0];
pre_reduce_dims[2] = post_reduce_dims[1] * post_reduce_dims[2] *
post_reduce_dims[3] * post_reduce_dims[4];
} else {
pre_reduce_dims[0] = post_reduce_dims[0] * post_reduce_dims[1] *
post_reduce_dims[2] * post_reduce_dims[3];
pre_reduce_dims[2] = post_reduce_dims[4];
}
#if !defined(EIGEN_HAS_INDEX_LIST)
// nvcc doesn't support cxx11
Eigen::array<int, 1> reduction_dims;
reduction_dims[0] = 1;
#else
// Take advantage of cxx11 to give the compiler information it can use to
// optimize the code.
Eigen::IndexList<Eigen::type2index<1> > reduction_dims;
#endif
return input
.extract_volume_patches(patchPlanes, patchRows, patchCols, stridePlanes,
strideRows, strideCols, padding_type,
-Eigen::NumTraits<float>::highest())
.reshape(pre_reduce_dims)
.maximum(reduction_dims)
.reshape(post_reduce_dims);
}
/** SpatialAvgPooling
* \ingroup CXX11_NeuralNetworks_Module
*
* \brief Applies an average pooling over a multichannel input image.
*
* The input parameter is expected to be a tensor with a rank of 4 (channels,
* height, width, others in col-major, and the reverse of that in row-major).
*
* The result can be assigned to a tensor of rank equal to the rank of the
* input. The dimensions of the result will be channels, height, width, and
* others (in col-major, and the reverse of that if the input was row-major).
*
* The order of the width and height dimensions can be swapped if needed.
*
*/
namespace internal {
template <typename T>
struct AvgPoolMeanReducer {
#if (EIGEN_ARCH_i386 || EIGEN_ARCH_x86_64) && !defined(__CUDACC__)
// We only support packet access for floats.
static const bool PacketAccess = internal::is_same<T, float>::value;
#else
static const bool PacketAccess = false;
#endif
static const bool IsStateful = true;
EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE AvgPoolMeanReducer() : scalarCount_(0) {
typedef typename packet_traits<T>::type Packet;
packetCount_ = pset1<Packet>(T(0.0));
}
EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void reduce(const T t, T* accum) {
if (t != -Eigen::NumTraits<T>::highest()) {
(*accum) = (*accum) + t;
scalarCount_++;
}
}
EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE T initialize() const {
return static_cast<T>(0);
}
EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE T finalize(const T accum) const {
eigen_assert(scalarCount_ > 0);
return accum / T(scalarCount_);
}
#if (EIGEN_ARCH_i386 || EIGEN_ARCH_x86_64) && !defined(__CUDACC__)
#ifdef EIGEN_VECTORIZE_AVX512
#define pequal(a, b) \
_mm512_castsi512_ps( \
_mm512_maskz_set1_epi32(_mm512_cmp_ps_mask(a, b, _CMP_EQ_UQ), -1))
// The ternarylogic function immediate determines the values in the result
// In the case below, 0xd8 implies (false_mask) ? (b) : (a)
// For details, refer to the vpternlogd instruction table at
// http://www.intel.com/content/dam/www/public/us/en/documents/manuals/64-ia-32-architectures-software-developer-vol-2c-manual.pdf
#define psel(a, b, false_mask) \
_mm512_castsi512_ps(_mm512_ternarylogic_epi32( \
_mm512_castps_si512(a), _mm512_castps_si512(b), \
_mm512_castps_si512(false_mask), 0xd8))
#elif defined EIGEN_VECTORIZE_AVX
#define pequal(a, b) _mm256_cmp_ps(a, b, _CMP_EQ_UQ)
#define psel(a, b, false_mask) _mm256_blendv_ps(a, b, false_mask)
#else
#define pequal(a, b) _mm_cmpeq_ps(a, b)
#define psel(a, b, false_mask) \
_mm_or_ps(_mm_andnot_ps(false_mask, a), _mm_and_ps(false_mask, b))
#endif
template <typename Packet>
EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void reducePacket(const Packet& p,
Packet* accum) {
reducePacketWithType(static_cast<T>(0), p, accum);
}
template <typename Packet>
EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void reducePacketWithType(
T, const Packet& p, Packet* accum) {
Packet skip_mask =
pequal(p, pset1<Packet>(-Eigen::NumTraits<T>::highest()));
(*accum) = padd<Packet>(*accum, psel(p, pset1<Packet>(0), skip_mask));
packetCount_ = padd<Packet>(
packetCount_, psel(pset1<Packet>(1), pset1<Packet>(0), skip_mask));
}
template <typename Packet>
EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet initializePacket() const {
return pset1<Packet>(0);
}
template <typename Packet>
EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet
finalizePacket(const Packet& vaccum) const {
return pdiv(vaccum, packetCount_);
}
template <typename Packet>
EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE T
finalizeBoth(const T saccum, const Packet& vaccum) const {
return (saccum + predux(vaccum)) / (scalarCount_ + predux(packetCount_));
}
#endif
protected:
typedef typename packet_traits<T>::type Packet;
int scalarCount_;
Packet packetCount_;
};
template <typename Device>
struct reducer_traits<AvgPoolMeanReducer<float>, Device> {
enum {
Cost = 1,
#if (EIGEN_ARCH_i386 || EIGEN_ARCH_x86_64) && !defined(__CUDACC__)
// We only support packet access for floats.
PacketAccess = true,
#else
PacketAccess = false,
#endif
IsStateful = true,
IsExactlyAssociative = false
};
};
template <>
struct reducer_traits<AvgPoolMeanReducer<float>, GpuDevice> {
enum {
Cost = 1,
PacketAccess = false,
IsStateful = true,
IsExactlyAssociative = false
};
};
} // namespace internal
#if !defined(EIGEN_HAS_INDEX_LIST)
template <typename Input>
EIGEN_ALWAYS_INLINE static const TensorReshapingOp<
const Eigen::DSizes<typename internal::traits<Input>::Index,
internal::traits<Input>::NumDimensions>,
const TensorReductionOp<
internal::AvgPoolMeanReducer<typename internal::remove_const<
typename internal::traits<Input>::Scalar>::type>,
const Eigen::array<int, 2>,
const TensorImagePatchOp<Dynamic, Dynamic, const Input> > >
#else
template <typename Input>
EIGEN_ALWAYS_INLINE static const TensorReshapingOp<
const Eigen::DSizes<typename internal::traits<Input>::Index,
internal::traits<Input>::NumDimensions>,
const TensorReductionOp<
internal::AvgPoolMeanReducer<typename internal::remove_const<
typename internal::traits<Input>::Scalar>::type>,
typename internal::conditional<
internal::traits<Input>::Layout == ColMajor,
const Eigen::IndexList<Eigen::type2index<1>, Eigen::type2index<2> >,
const Eigen::IndexList<Eigen::type2index<2>,
Eigen::type2index<3> > >::type,
const TensorImagePatchOp<Dynamic, Dynamic, const Input> > >
#endif
SpatialAvgPooling(const Input& input, DenseIndex patchRows,
DenseIndex patchCols, DenseIndex strideRows,
DenseIndex strideCols, const PaddingType padding_type,
DenseIndex in_strideRows = 1, DenseIndex in_strideCols = 1) {
EIGEN_STATIC_ASSERT(internal::traits<Input>::NumDimensions == 4,
YOU_MADE_A_PROGRAMMING_MISTAKE);
typedef typename internal::traits<Input>::Index TensorIndex;
TensorRef<Tensor<typename internal::traits<Input>::Scalar,
internal::traits<Input>::NumDimensions,
internal::traits<Input>::Layout, TensorIndex> >
in(input);
const DenseIndex patchRowsEff =
patchRows + (patchRows - 1) * (in_strideRows - 1);
const DenseIndex patchColsEff =
patchCols + (patchCols - 1) * (in_strideCols - 1);
static const bool isColMajor = (internal::traits<Input>::Layout == ColMajor);
static const int idxRows = isColMajor ? 1 : 2;
static const int idxCols = isColMajor ? 2 : 1;
// Molds the output of the reduction into the shape expected by the user.
// (assuming col-major):
// - 1st dim: channels
// - 2nd dim: output height
// - 3rd dim: output width
// - 4th dim and beyond: everything else including batch size
Eigen::DSizes<TensorIndex, internal::traits<Input>::NumDimensions>
post_reduce_dims;
post_reduce_dims[0] = in.dimension(0);
if (padding_type == PADDING_VALID) {
post_reduce_dims[idxRows] = Eigen::divup(
static_cast<DenseIndex>(in.dimension(idxRows)) - patchRowsEff + 1,
strideRows);
post_reduce_dims[idxCols] = Eigen::divup(
static_cast<DenseIndex>(in.dimension(idxCols)) - patchColsEff + 1,
strideCols);
} else {
post_reduce_dims[idxRows] = Eigen::divup(
static_cast<DenseIndex>(in.dimension(idxRows)), strideRows);
post_reduce_dims[idxCols] = Eigen::divup(
static_cast<DenseIndex>(in.dimension(idxCols)), strideCols);
}
post_reduce_dims[3] = in.dimension(3);
typedef typename internal::remove_const<
typename internal::traits<Input>::Scalar>::type CoeffReturnType;
internal::AvgPoolMeanReducer<CoeffReturnType> mean_with_nan;
#if !defined(EIGEN_HAS_INDEX_LIST)
// nvcc doesn't support cxx11
Eigen::array<int, 2> reduction_dims;
if (isColMajor) {
reduction_dims[0] = 1;
reduction_dims[1] = 2;
} else {
reduction_dims[0] = 2;
reduction_dims[1] = 3;
}
#else
// Take advantage of cxx11 to give the compiler information it can use to
// optimize the code.
typename internal::conditional<
internal::traits<Input>::Layout == ColMajor,
const Eigen::IndexList<Eigen::type2index<1>, Eigen::type2index<2> >,
const Eigen::IndexList<Eigen::type2index<2>,
Eigen::type2index<3> > >::type reduction_dims;
#endif
return input
.extract_image_patches(patchRows, patchCols, strideRows, strideCols,
in_strideRows, in_strideCols, padding_type,
-Eigen::NumTraits<CoeffReturnType>::highest())
.reduce(reduction_dims, mean_with_nan)
.reshape(post_reduce_dims);
}
/** CuboidAvgPooling
* \ingroup CXX11_NeuralNetworks_Module
*
* \brief Applies an average pooling over a multichannel input volume.
*
* The input parameter is expected to be a tensor with a rank of 5 (channels,
* depth, height, width, others, and the reverse of that in row-major).
*
* The result can be assigned to a tensor of rank equal to the rank of the
* input. The dimensions of the result will be channels, depth, width, and
* others (in col-major, and the reverse of that if the input was row-major).
*
* The order of the depth, width and height dimensions can be swapped if
* needed.
*
*/
#if !defined(EIGEN_HAS_INDEX_LIST)
template <typename Input>
EIGEN_ALWAYS_INLINE static const TensorReshapingOp<
const Eigen::DSizes<DenseIndex, internal::traits<Input>::NumDimensions>,
const TensorReductionOp<
internal::AvgPoolMeanReducer<float>, const Eigen::array<int, 1>,
const TensorReshapingOp<
const Eigen::DSizes<DenseIndex, 3>,
const TensorVolumePatchOp<Dynamic, Dynamic, Dynamic,
const Input> > > >
#else
template <typename Input>
EIGEN_ALWAYS_INLINE static const TensorReshapingOp<
const Eigen::DSizes<DenseIndex, internal::traits<Input>::NumDimensions>,
const TensorReductionOp<
internal::AvgPoolMeanReducer<float>,
const Eigen::IndexList<Eigen::type2index<1> >,
const TensorReshapingOp<
const Eigen::DSizes<DenseIndex, 3>,
const TensorVolumePatchOp<Dynamic, Dynamic, Dynamic,
const Input> > > >
#endif
CuboidAvgPooling(const Input& input, DenseIndex patchPlanes,
DenseIndex patchRows, DenseIndex patchCols,
DenseIndex stridePlanes, DenseIndex strideRows,
DenseIndex strideCols, const PaddingType padding_type) {
EIGEN_STATIC_ASSERT(internal::traits<Input>::NumDimensions == 5,
YOU_MADE_A_PROGRAMMING_MISTAKE);
static const bool isColMajor = (internal::traits<Input>::Layout == ColMajor);
typedef typename internal::traits<Input>::Index TensorIndex;
TensorRef<Tensor<typename internal::traits<Input>::Scalar,
internal::traits<Input>::NumDimensions,
internal::traits<Input>::Layout, TensorIndex> >
in(input);
static const int idxPlanes = isColMajor ? 1 : 3;
static const int idxRows = 2;
static const int idxCols = isColMajor ? 3 : 1;
// Molds the output of the reduction into the shape expected by the used
// (assuming col-major):
// - 1st dim: channels
// - 2nd dim: outupt depth
// - 3rd dim: output height
// - 4th dim: output width
// - 5th dim and beyond: everything else including batch size
Eigen::DSizes<DenseIndex, internal::traits<Input>::NumDimensions>
post_reduce_dims;
post_reduce_dims[0] = in.dimension(0);
if (padding_type == PADDING_VALID) {
post_reduce_dims[idxPlanes] = Eigen::divup(
static_cast<DenseIndex>(in.dimension(idxPlanes)) - patchPlanes + 1,
stridePlanes);
post_reduce_dims[idxRows] = Eigen::divup(
static_cast<DenseIndex>(in.dimension(idxRows)) - patchRows + 1,
strideRows);
post_reduce_dims[idxCols] = Eigen::divup(
static_cast<DenseIndex>(in.dimension(idxCols)) - patchCols + 1,
strideCols);
} else {
post_reduce_dims[idxPlanes] = Eigen::divup(
static_cast<DenseIndex>(in.dimension(idxPlanes)), stridePlanes);
post_reduce_dims[idxRows] = Eigen::divup(
static_cast<DenseIndex>(in.dimension(idxRows)), strideRows);
post_reduce_dims[idxCols] = Eigen::divup(
static_cast<DenseIndex>(in.dimension(idxCols)), strideCols);
}
post_reduce_dims[4] = in.dimension(4);
Eigen::DSizes<DenseIndex, 3> pre_reduce_dims;
pre_reduce_dims[1] = patchRows * patchCols * patchPlanes;
if (isColMajor) {
pre_reduce_dims[0] = post_reduce_dims[0];
pre_reduce_dims[2] = post_reduce_dims[1] * post_reduce_dims[2] *
post_reduce_dims[3] * post_reduce_dims[4];
} else {
pre_reduce_dims[0] = post_reduce_dims[0] * post_reduce_dims[1] *
post_reduce_dims[2] * post_reduce_dims[3];
pre_reduce_dims[2] = post_reduce_dims[4];
}
typedef typename internal::remove_const<
typename internal::traits<Input>::Scalar>::type CoeffReturnType;
internal::AvgPoolMeanReducer<CoeffReturnType> mean_with_nan;
#if !defined(EIGEN_HAS_INDEX_LIST)
// nvcc doesn't support cxx11
Eigen::array<int, 1> reduction_dims;
reduction_dims[0] = 1;
#else
// Take advantage of cxx11 to give the compiler information it can use to
// optimize the code.
Eigen::IndexList<Eigen::type2index<1> > reduction_dims;
#endif
return input
.extract_volume_patches(patchPlanes, patchRows, patchCols, stridePlanes,
strideRows, strideCols, padding_type,
-Eigen::NumTraits<float>::highest())
.reshape(pre_reduce_dims)
.reduce(reduction_dims, mean_with_nan)
.reshape(post_reduce_dims);
}
} // end namespace Eigen
#endif // TENSORFLOW_CORE_KERNELS_EIGEN_POOLING_H_
@@ -0,0 +1,99 @@
/* Copyright 2015 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
#ifndef TENSORFLOW_CORE_KERNELS_EIGEN_SOFTMAX_H_
#define TENSORFLOW_CORE_KERNELS_EIGEN_SOFTMAX_H_
#include "third_party/eigen3/unsupported/Eigen/CXX11/Tensor"
namespace Eigen {
/** SoftMax
* \ingroup CXX11_NeuralNetworks_Module
*
* \brief Applies a softmax
*
* The input parameter is expected to be a col-major tensor with a rank of 2
* (depth and other).
*
* The result can be assigned to a tensor of rank and dimensions equal to that
* of the input. The result will be laid out in col-major order.
*
*/
namespace {
struct SoftmaxOp {
SoftmaxOp(const float beta) : beta_(beta) {}
template <typename Input>
typename Input::Dimensions dimensions(const Input& input) const {
return input.dimensions();
}
template <typename Input, typename Output, typename Device>
void eval(const Input& input, Output& output, const Device& device) const {
#if !defined(EIGEN_HAS_INDEX_LIST)
// nvcc doesn't support cxx11
Eigen::array<typename internal::traits<Input>::Index, 1> depth_dim;
depth_dim[0] = 0;
Eigen::array<typename internal::traits<Input>::Index, 2> bcast;
bcast[0] = dimensions(input)[0];
bcast[1] = 1;
DSizes<typename internal::traits<Input>::Index, 2> dims2d;
dims2d[0] = 1;
dims2d[1] = dimensions(input)[1];
#else
// Take advantage of cxx11 to give the compiler information it can use to
// optimize the code.
Eigen::IndexList<Eigen::type2index<0> > depth_dim;
Eigen::IndexList<int, Eigen::type2index<1> > bcast;
bcast.set(0, dimensions(input)[0]);
Eigen::IndexList<Eigen::type2index<1>,
typename internal::traits<Input>::Index>
dims2d;
dims2d.set(1, dimensions(input)[1]);
#endif
output.device(device) =
((input -
input.maximum(depth_dim).eval().reshape(dims2d).broadcast(bcast)) *
beta_)
.exp();
output.device(device) =
output /
(output.sum(depth_dim).eval().reshape(dims2d).broadcast(bcast));
}
private:
const float beta_;
};
} // namespace
template <typename Input>
EIGEN_ALWAYS_INLINE static const TensorCustomUnaryOp<const SoftmaxOp,
const Input>
SoftMax(const Input& input, const float beta) {
EIGEN_STATIC_ASSERT(internal::traits<Input>::Layout == ColMajor,
YOU_MADE_A_PROGRAMMING_MISTAKE);
EIGEN_STATIC_ASSERT(internal::traits<Input>::NumDimensions == 2,
YOU_MADE_A_PROGRAMMING_MISTAKE);
const SoftmaxOp op(beta);
return input.customOp(op);
}
} // end namespace Eigen
#endif // TENSORFLOW_CORE_KERNELS_EIGEN_SOFTMAX_H_
File diff suppressed because it is too large. Load diff
@@ -0,0 +1,657 @@
/* Copyright 2017 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
#ifndef TENSORFLOW_CORE_KERNELS_EIGEN_VOLUME_PATCH_H_
#define TENSORFLOW_CORE_KERNELS_EIGEN_VOLUME_PATCH_H_
#include "third_party/eigen3/unsupported/Eigen/CXX11/Tensor"
namespace Eigen {
// Changes the interpretation of padding in TensorVolumePatchOp to be compatible
// with the rest of TensorFlow (odd padding is split so that more padding is put
// on the right end of the tensor).
template <DenseIndex Planes, DenseIndex Rows, DenseIndex Cols, typename ArgType,
typename Device>
struct CustomTensorEvaluator {
typedef TensorVolumePatchOp<Planes, Rows, Cols, ArgType> XprType;
typedef typename XprType::Index Index;
static const int NumInputDims = internal::array_size<
typename TensorEvaluator<ArgType, Device>::Dimensions>::value;
static const int NumDims = NumInputDims + 1;
typedef DSizes<Index, NumDims> Dimensions;
typedef
typename internal::remove_const<typename XprType::Scalar>::type Scalar;
typedef typename XprType::CoeffReturnType CoeffReturnType;
typedef typename PacketType<CoeffReturnType, Device>::type PacketReturnType;
static const Index PacketSize =
internal::unpacket_traits<PacketReturnType>::size;
enum {
IsAligned = false,
PacketAccess = TensorEvaluator<ArgType, Device>::PacketAccess,
BlockAccess = false,
PreferBlockAccess = false,
Layout = TensorEvaluator<ArgType, Device>::Layout,
CoordAccess = NumDims == 6,
RawAccess = false
};
EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE
CustomTensorEvaluator(const XprType& op, const Device& device)
: m_impl(op.expression(), device) {
EIGEN_STATIC_ASSERT(NumDims >= 5, YOU_MADE_A_PROGRAMMING_MISTAKE);
m_paddingValue = op.padding_value();
const typename TensorEvaluator<ArgType, Device>::Dimensions& input_dims =
m_impl.dimensions();
// Cache a few variables.
if (static_cast<int>(Layout) == static_cast<int>(ColMajor)) {
m_inputDepth = input_dims[0];
m_inputPlanes = input_dims[1];
m_inputRows = input_dims[2];
m_inputCols = input_dims[3];
} else {
m_inputDepth = input_dims[NumInputDims - 1];
m_inputPlanes = input_dims[NumInputDims - 2];
m_inputRows = input_dims[NumInputDims - 3];
m_inputCols = input_dims[NumInputDims - 4];
}
m_plane_strides = op.plane_strides();
m_row_strides = op.row_strides();
m_col_strides = op.col_strides();
// Input strides and effective input/patch size
m_in_plane_strides = op.in_plane_strides();
m_in_row_strides = op.in_row_strides();
m_in_col_strides = op.in_col_strides();
m_plane_inflate_strides = op.plane_inflate_strides();
m_row_inflate_strides = op.row_inflate_strides();
m_col_inflate_strides = op.col_inflate_strides();
// The "effective" spatial size after inflating data with zeros.
m_input_planes_eff = (m_inputPlanes - 1) * m_plane_inflate_strides + 1;
m_input_rows_eff = (m_inputRows - 1) * m_row_inflate_strides + 1;
m_input_cols_eff = (m_inputCols - 1) * m_col_inflate_strides + 1;
m_patch_planes_eff =
op.patch_planes() + (op.patch_planes() - 1) * (m_in_plane_strides - 1);
m_patch_rows_eff =
op.patch_rows() + (op.patch_rows() - 1) * (m_in_row_strides - 1);
m_patch_cols_eff =
op.patch_cols() + (op.patch_cols() - 1) * (m_in_col_strides - 1);
if (op.padding_explicit()) {
m_outputPlanes = Eigen::divup(
m_input_planes_eff +
static_cast<Index>(op.padding_top_z() + op.padding_bottom_z()) -
m_patch_planes_eff + 1,
m_plane_strides);
m_outputRows = Eigen::divup(
m_input_rows_eff +
static_cast<Index>(op.padding_top() + op.padding_bottom()) -
m_patch_rows_eff + 1,
m_row_strides);
m_outputCols = Eigen::divup(
m_input_cols_eff +
static_cast<Index>(op.padding_left() + op.padding_right()) -
m_patch_cols_eff + 1,
m_col_strides);
m_planePaddingTop = op.padding_top_z();
m_rowPaddingTop = op.padding_top();
m_colPaddingLeft = op.padding_left();
} else {
// Computing padding from the type
switch (op.padding_type()) {
case PADDING_VALID:
m_outputPlanes = Eigen::divup(
m_input_planes_eff - m_patch_planes_eff + 1, m_plane_strides);
m_outputRows = Eigen::divup(m_input_rows_eff - m_patch_rows_eff + 1,
m_row_strides);
m_outputCols = Eigen::divup(m_input_cols_eff - m_patch_cols_eff + 1,
m_col_strides);
m_planePaddingTop = 0;
m_rowPaddingTop = 0;
m_colPaddingLeft = 0;
break;
case PADDING_SAME: {
m_outputPlanes = Eigen::divup(m_input_planes_eff, m_plane_strides);
m_outputRows = Eigen::divup(m_input_rows_eff, m_row_strides);
m_outputCols = Eigen::divup(m_input_cols_eff, m_col_strides);
const Index dz = numext::maxi<DenseIndex>(
0, (m_outputPlanes - 1) * m_plane_strides + m_patch_planes_eff -
m_input_planes_eff);
const Index dy = numext::maxi<DenseIndex>(
0, (m_outputRows - 1) * m_row_strides + m_patch_rows_eff -
m_input_rows_eff);
const Index dx = numext::maxi<DenseIndex>(
0, (m_outputCols - 1) * m_col_strides + m_patch_cols_eff -
m_input_cols_eff);
m_planePaddingTop = dz / 2;
m_rowPaddingTop = dy / 2;
m_colPaddingLeft = dx / 2;
break;
}
default:
eigen_assert(false && "unexpected padding");
}
}
eigen_assert(m_outputRows > 0);
eigen_assert(m_outputCols > 0);
eigen_assert(m_outputPlanes > 0);
// Dimensions for result of extraction.
if (static_cast<int>(Layout) == static_cast<int>(ColMajor)) {
// ColMajor
// 0: depth
// 1: patch_planes
// 2: patch_rows
// 3: patch_cols
// 4: number of patches
// 5 and beyond: anything else (such as batch).
m_dimensions[0] = input_dims[0];
m_dimensions[1] = op.patch_planes();
m_dimensions[2] = op.patch_rows();
m_dimensions[3] = op.patch_cols();
m_dimensions[4] = m_outputPlanes * m_outputRows * m_outputCols;
for (int i = 5; i < NumDims; ++i) {
m_dimensions[i] = input_dims[i - 1];
}
} else {
// RowMajor
// NumDims-1: depth
// NumDims-2: patch_planes
// NumDims-3: patch_rows
// NumDims-4: patch_cols
// NumDims-5: number of patches
// NumDims-6 and beyond: anything else (such as batch).
m_dimensions[NumDims - 1] = input_dims[NumInputDims - 1];
m_dimensions[NumDims - 2] = op.patch_planes();
m_dimensions[NumDims - 3] = op.patch_rows();
m_dimensions[NumDims - 4] = op.patch_cols();
m_dimensions[NumDims - 5] = m_outputPlanes * m_outputRows * m_outputCols;
for (int i = NumDims - 6; i >= 0; --i) {
m_dimensions[i] = input_dims[i];
}
}
// Strides for the output tensor.
if (static_cast<int>(Layout) == static_cast<int>(ColMajor)) {
m_rowStride = m_dimensions[1];
m_colStride = m_dimensions[2] * m_rowStride;
m_patchStride = m_colStride * m_dimensions[3] * m_dimensions[0];
m_otherStride = m_patchStride * m_dimensions[4];
} else {
m_rowStride = m_dimensions[NumDims - 2];
m_colStride = m_dimensions[NumDims - 3] * m_rowStride;
m_patchStride =
m_colStride * m_dimensions[NumDims - 4] * m_dimensions[NumDims - 1];
m_otherStride = m_patchStride * m_dimensions[NumDims - 5];
}
// Strides for navigating through the input tensor.
m_planeInputStride = m_inputDepth;
m_rowInputStride = m_inputDepth * m_inputPlanes;
m_colInputStride = m_inputDepth * m_inputRows * m_inputPlanes;
m_otherInputStride =
m_inputDepth * m_inputRows * m_inputCols * m_inputPlanes;
m_outputPlanesRows = m_outputPlanes * m_outputRows;
// Fast representations of different variables.
m_fastOtherStride = internal::TensorIntDivisor<Index>(m_otherStride);
m_fastPatchStride = internal::TensorIntDivisor<Index>(m_patchStride);
m_fastColStride = internal::TensorIntDivisor<Index>(m_colStride);
m_fastRowStride = internal::TensorIntDivisor<Index>(m_rowStride);
m_fastInputRowStride =
internal::TensorIntDivisor<Index>(m_row_inflate_strides);
m_fastInputColStride =
internal::TensorIntDivisor<Index>(m_col_inflate_strides);
m_fastInputPlaneStride =
internal::TensorIntDivisor<Index>(m_plane_inflate_strides);
m_fastInputColsEff = internal::TensorIntDivisor<Index>(m_input_cols_eff);
m_fastOutputPlanes = internal::TensorIntDivisor<Index>(m_outputPlanes);
m_fastOutputPlanesRows =
internal::TensorIntDivisor<Index>(m_outputPlanesRows);
if (static_cast<int>(Layout) == static_cast<int>(ColMajor)) {
m_fastOutputDepth = internal::TensorIntDivisor<Index>(m_dimensions[0]);
} else {
m_fastOutputDepth =
internal::TensorIntDivisor<Index>(m_dimensions[NumDims - 1]);
}
}
EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const Dimensions& dimensions() const {
return m_dimensions;
}
EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE bool evalSubExprsIfNeeded(
Scalar* /*data*/) {
m_impl.evalSubExprsIfNeeded(NULL);
return true;
}
EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void cleanup() { m_impl.cleanup(); }
EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE CoeffReturnType
coeff(Index index) const {
// Patch index corresponding to the passed in index.
const Index patchIndex = index / m_fastPatchStride;
// Spatial offset within the patch. This has to be translated into 3D
// coordinates within the patch.
const Index patchOffset =
(index - patchIndex * m_patchStride) / m_fastOutputDepth;
// Batch, etc.
const Index otherIndex = (NumDims == 5) ? 0 : index / m_fastOtherStride;
const Index patch3DIndex =
(NumDims == 5)
? patchIndex
: (index - otherIndex * m_otherStride) / m_fastPatchStride;
// Calculate column index in the input original tensor.
const Index colIndex = patch3DIndex / m_fastOutputPlanesRows;
const Index colOffset = patchOffset / m_fastColStride;
const Index inputCol = colIndex * m_col_strides +
colOffset * m_in_col_strides - m_colPaddingLeft;
const Index origInputCol =
(m_col_inflate_strides == 1)
? inputCol
: ((inputCol >= 0) ? (inputCol / m_fastInputColStride) : 0);
if (inputCol < 0 || inputCol >= m_input_cols_eff ||
((m_col_inflate_strides != 1) &&
(inputCol != origInputCol * m_col_inflate_strides))) {
return Scalar(m_paddingValue);
}
// Calculate row index in the original input tensor.
const Index rowIndex =
(patch3DIndex - colIndex * m_outputPlanesRows) / m_fastOutputPlanes;
const Index rowOffset =
(patchOffset - colOffset * m_colStride) / m_fastRowStride;
const Index inputRow = rowIndex * m_row_strides +
rowOffset * m_in_row_strides - m_rowPaddingTop;
const Index origInputRow =
(m_row_inflate_strides == 1)
? inputRow
: ((inputRow >= 0) ? (inputRow / m_fastInputRowStride) : 0);
if (inputRow < 0 || inputRow >= m_input_rows_eff ||
((m_row_inflate_strides != 1) &&
(inputRow != origInputRow * m_row_inflate_strides))) {
return Scalar(m_paddingValue);
}
// Calculate plane index in the original input tensor.
const Index planeIndex =
(patch3DIndex - m_outputPlanes * (colIndex * m_outputRows + rowIndex));
const Index planeOffset =
patchOffset - colOffset * m_colStride - rowOffset * m_rowStride;
const Index inputPlane = planeIndex * m_plane_strides +
planeOffset * m_in_plane_strides -
m_planePaddingTop;
const Index origInputPlane =
(m_plane_inflate_strides == 1)
? inputPlane
: ((inputPlane >= 0) ? (inputPlane / m_fastInputPlaneStride) : 0);
if (inputPlane < 0 || inputPlane >= m_input_planes_eff ||
((m_plane_inflate_strides != 1) &&
(inputPlane != origInputPlane * m_plane_inflate_strides))) {
return Scalar(m_paddingValue);
}
const int depth_index =
static_cast<int>(Layout) == static_cast<int>(ColMajor) ? 0
: NumDims - 1;
const Index depth =
index - (index / m_fastOutputDepth) * m_dimensions[depth_index];
const Index inputIndex = depth + origInputRow * m_rowInputStride +
origInputCol * m_colInputStride +
origInputPlane * m_planeInputStride +
otherIndex * m_otherInputStride;
return m_impl.coeff(inputIndex);
}
template <int LoadMode>
EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE PacketReturnType
packet(Index index) const {
EIGEN_STATIC_ASSERT(PacketSize > 1, YOU_MADE_A_PROGRAMMING_MISTAKE)
eigen_assert(index + PacketSize - 1 < dimensions().TotalSize());
if (m_in_row_strides != 1 || m_in_col_strides != 1 ||
m_row_inflate_strides != 1 || m_col_inflate_strides != 1 ||
m_in_plane_strides != 1 || m_plane_inflate_strides != 1) {
return packetWithPossibleZero(index);
}
const Index indices[2] = {index, index + PacketSize - 1};
const Index patchIndex = indices[0] / m_fastPatchStride;
if (patchIndex != indices[1] / m_fastPatchStride) {
return packetWithPossibleZero(index);
}
const Index otherIndex =
(NumDims == 5) ? 0 : indices[0] / m_fastOtherStride;
eigen_assert(otherIndex == indices[1] / m_fastOtherStride);
// Find the offset of the element wrt the location of the first element.
const Index patchOffsets[2] = {
(indices[0] - patchIndex * m_patchStride) / m_fastOutputDepth,
(indices[1] - patchIndex * m_patchStride) / m_fastOutputDepth};
const Index patch3DIndex =
(NumDims == 5)
? patchIndex
: (indices[0] - otherIndex * m_otherStride) / m_fastPatchStride;
eigen_assert(patch3DIndex ==
(indices[1] - otherIndex * m_otherStride) / m_fastPatchStride);
const Index colIndex = patch3DIndex / m_fastOutputPlanesRows;
const Index colOffsets[2] = {patchOffsets[0] / m_fastColStride,
patchOffsets[1] / m_fastColStride};
// Calculate col indices in the original input tensor.
const Index inputCols[2] = {
colIndex * m_col_strides + colOffsets[0] - m_colPaddingLeft,
colIndex * m_col_strides + colOffsets[1] - m_colPaddingLeft};
if (inputCols[1] < 0 || inputCols[0] >= m_inputCols) {
return internal::pset1<PacketReturnType>(Scalar(m_paddingValue));
}
if (inputCols[0] != inputCols[1]) {
return packetWithPossibleZero(index);
}
const Index rowIndex =
(patch3DIndex - colIndex * m_outputPlanesRows) / m_fastOutputPlanes;
const Index rowOffsets[2] = {
(patchOffsets[0] - colOffsets[0] * m_colStride) / m_fastRowStride,
(patchOffsets[1] - colOffsets[1] * m_colStride) / m_fastRowStride};
eigen_assert(rowOffsets[0] <= rowOffsets[1]);
// Calculate col indices in the original input tensor.
const Index inputRows[2] = {
rowIndex * m_row_strides + rowOffsets[0] - m_rowPaddingTop,
rowIndex * m_row_strides + rowOffsets[1] - m_rowPaddingTop};
if (inputRows[1] < 0 || inputRows[0] >= m_inputRows) {
return internal::pset1<PacketReturnType>(Scalar(m_paddingValue));
}
if (inputRows[0] != inputRows[1]) {
return packetWithPossibleZero(index);
}
const Index planeIndex =
(patch3DIndex - m_outputPlanes * (colIndex * m_outputRows + rowIndex));
const Index planeOffsets[2] = {
patchOffsets[0] - colOffsets[0] * m_colStride -
rowOffsets[0] * m_rowStride,
patchOffsets[1] - colOffsets[1] * m_colStride -
rowOffsets[1] * m_rowStride};
eigen_assert(planeOffsets[0] <= planeOffsets[1]);
const Index inputPlanes[2] = {
planeIndex * m_plane_strides + planeOffsets[0] - m_planePaddingTop,
planeIndex * m_plane_strides + planeOffsets[1] - m_planePaddingTop};
if (inputPlanes[1] < 0 || inputPlanes[0] >= m_inputPlanes) {
return internal::pset1<PacketReturnType>(Scalar(m_paddingValue));
}
if (inputPlanes[0] >= 0 && inputPlanes[1] < m_inputPlanes) {
// no padding
const int depth_index =
static_cast<int>(Layout) == static_cast<int>(ColMajor) ? 0
: NumDims - 1;
const Index depth =
index - (index / m_fastOutputDepth) * m_dimensions[depth_index];
const Index inputIndex = depth + inputRows[0] * m_rowInputStride +
inputCols[0] * m_colInputStride +
m_planeInputStride * inputPlanes[0] +
otherIndex * m_otherInputStride;
return m_impl.template packet<Unaligned>(inputIndex);
}
return packetWithPossibleZero(index);
}
EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE TensorOpCost
costPerCoeff(bool vectorized) const {
const double compute_cost = 10 * TensorOpCost::DivCost<Index>() +
21 * TensorOpCost::MulCost<Index>() +
8 * TensorOpCost::AddCost<Index>();
return TensorOpCost(0, 0, compute_cost, vectorized, PacketSize);
}
EIGEN_DEVICE_FUNC Scalar* data() const { return NULL; }
const TensorEvaluator<ArgType, Device>& impl() const { return m_impl; }
Index planePaddingTop() const { return m_planePaddingTop; }
Index rowPaddingTop() const { return m_rowPaddingTop; }
Index colPaddingLeft() const { return m_colPaddingLeft; }
Index outputPlanes() const { return m_outputPlanes; }
Index outputRows() const { return m_outputRows; }
Index outputCols() const { return m_outputCols; }
Index userPlaneStride() const { return m_plane_strides; }
Index userRowStride() const { return m_row_strides; }
Index userColStride() const { return m_col_strides; }
Index userInPlaneStride() const { return m_in_plane_strides; }
Index userInRowStride() const { return m_in_row_strides; }
Index userInColStride() const { return m_in_col_strides; }
Index planeInflateStride() const { return m_plane_inflate_strides; }
Index rowInflateStride() const { return m_row_inflate_strides; }
Index colInflateStride() const { return m_col_inflate_strides; }
EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE CoeffReturnType
coeff(const array<Index, NumDims>& coords) const {
// ColMajor
// 0: depth, 1: patch_planes, 2: patch_rows, 3: patch_cols, 4: number of
// patches, 5: batches
// RowMajor
// 0: batches, 1: number of patches, 2: patch_cols , 3: patch_rows, 4:
// patch_planes, 5: depth
const Index patch3DIndex =
coords[static_cast<int>(Layout) == static_cast<int>(ColMajor) ? 4 : 1];
const Index colOffset =
coords[static_cast<int>(Layout) == static_cast<int>(ColMajor) ? 3 : 2];
const Index rowOffset =
coords[static_cast<int>(Layout) == static_cast<int>(ColMajor) ? 2 : 3];
const Index planeOffset =
coords[static_cast<int>(Layout) == static_cast<int>(ColMajor) ? 1 : 4];
array<Index, NumDims - 1> inputCoords;
const Index colIndex = patch3DIndex / m_fastOutputPlanesRows;
const Index inputCol = colIndex * m_col_strides +
colOffset * m_in_col_strides - m_colPaddingLeft;
const Index origInputCol =
(m_col_inflate_strides == 1)
? inputCol
: ((inputCol >= 0) ? (inputCol / m_fastInputColStride) : 0);
if (inputCol < 0 || inputCol >= m_input_cols_eff ||
((m_col_inflate_strides != 1) &&
(inputCol != origInputCol * m_col_inflate_strides))) {
return Scalar(m_paddingValue);
}
const Index rowIndex =
(patch3DIndex - colIndex * m_outputPlanesRows) / m_fastOutputPlanes;
const Index inputRow = rowIndex * m_row_strides +
rowOffset * m_in_row_strides - m_rowPaddingTop;
const Index origInputRow =
(m_row_inflate_strides == 1)
? inputRow
: ((inputRow >= 0) ? (inputRow / m_fastInputRowStride) : 0);
if (inputRow < 0 || inputRow >= m_input_rows_eff ||
((m_row_inflate_strides != 1) &&
(inputRow != origInputRow * m_row_inflate_strides))) {
return Scalar(m_paddingValue);
}
const Index planeIndex =
patch3DIndex - colIndex * m_outputPlanesRows - rowIndex * m_outputRows;
const Index inputPlane = planeIndex * m_plane_strides +
planeOffset * m_in_plane_strides -
m_planePaddingTop;
const Index origInputPlane =
(m_plane_inflate_strides == 1)
? inputPlane
: ((inputPlane >= 0) ? (inputPlane / m_fastInputPlaneStride) : 0);
if (inputPlane < 0 || inputPlane >= m_input_planes_eff ||
((m_plane_inflate_strides != 1) &&
(inputPlane != origInputPlane * m_plane_inflate_strides))) {
return Scalar(m_paddingValue);
}
if (static_cast<int>(Layout) == static_cast<int>(ColMajor)) {
inputCoords[0] = coords[0]; // depth
inputCoords[1] = origInputPlane;
inputCoords[2] = origInputRow;
inputCoords[3] = origInputCol;
inputCoords[4] = coords[5]; // batch
} else {
inputCoords[4] = coords[5]; // depth
inputCoords[3] = origInputPlane;
inputCoords[2] = origInputRow;
inputCoords[1] = origInputCol;
inputCoords[0] = coords[0]; // batch
}
if (TensorEvaluator<ArgType, Device>::CoordAccess) {
return m_impl.coeff(inputCoords);
} else {
Index inputIndex;
if (static_cast<int>(Layout) == static_cast<int>(ColMajor)) {
inputIndex = inputCoords[4] * m_otherInputStride +
inputCoords[3] * m_colInputStride +
inputCoords[2] * m_rowInputStride +
inputCoords[1] * m_planeInputStride + inputCoords[0];
} else {
inputIndex = inputCoords[0] * m_otherInputStride +
inputCoords[1] * m_colInputStride +
inputCoords[2] * m_rowInputStride +
inputCoords[3] * m_planeInputStride + inputCoords[4];
}
return m_impl.coeff(inputIndex);
}
}
protected:
EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE PacketReturnType
packetWithPossibleZero(Index index) const {
EIGEN_ALIGN_MAX typename internal::remove_const<CoeffReturnType>::type
values[PacketSize];
for (int i = 0; i < PacketSize; ++i) {
values[i] = coeff(index + i);
}
PacketReturnType rslt = internal::pload<PacketReturnType>(values);
return rslt;
}
Dimensions m_dimensions;
// Parameters passed to the costructor.
Index m_plane_strides;
Index m_row_strides;
Index m_col_strides;
Index m_outputPlanes;
Index m_outputRows;
Index m_outputCols;
Index m_planePaddingTop;
Index m_rowPaddingTop;
Index m_colPaddingLeft;
Index m_in_plane_strides;
Index m_in_row_strides;
Index m_in_col_strides;
Index m_plane_inflate_strides;
Index m_row_inflate_strides;
Index m_col_inflate_strides;
// Cached input size.
Index m_inputDepth;
Index m_inputPlanes;
Index m_inputRows;
Index m_inputCols;
// Other cached variables.
Index m_outputPlanesRows;
// Effective input/patch post-inflation size.
Index m_input_planes_eff;
Index m_input_rows_eff;
Index m_input_cols_eff;
Index m_patch_planes_eff;
Index m_patch_rows_eff;
Index m_patch_cols_eff;
// Strides for the output tensor.
Index m_otherStride;
Index m_patchStride;
Index m_rowStride;
Index m_colStride;
// Strides for the input tensor.
Index m_planeInputStride;
Index m_rowInputStride;
Index m_colInputStride;
Index m_otherInputStride;
internal::TensorIntDivisor<Index> m_fastOtherStride;
internal::TensorIntDivisor<Index> m_fastPatchStride;
internal::TensorIntDivisor<Index> m_fastColStride;
internal::TensorIntDivisor<Index> m_fastRowStride;
internal::TensorIntDivisor<Index> m_fastInputPlaneStride;
internal::TensorIntDivisor<Index> m_fastInputRowStride;
internal::TensorIntDivisor<Index> m_fastInputColStride;
internal::TensorIntDivisor<Index> m_fastInputColsEff;
internal::TensorIntDivisor<Index> m_fastOutputPlanesRows;
internal::TensorIntDivisor<Index> m_fastOutputPlanes;
internal::TensorIntDivisor<Index> m_fastOutputDepth;
Scalar m_paddingValue;
TensorEvaluator<ArgType, Device> m_impl;
};
// Override the default TensorEvaluator for TensorVolumePatchOp for CPU.
#define OVERRIDE_EVALUATOR(Device) \
template <DenseIndex Planes, DenseIndex Rows, DenseIndex Cols, \
typename ArgType> \
struct TensorEvaluator< \
const TensorVolumePatchOp<Planes, Rows, Cols, ArgType>, Device> \
: public CustomTensorEvaluator<Planes, Rows, Cols, ArgType, Device> { \
EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE TensorEvaluator( \
const typename CustomTensorEvaluator<Planes, Rows, Cols, ArgType, \
Device>::XprType& op, \
const Device& device) \
: CustomTensorEvaluator<Planes, Rows, Cols, ArgType, Device>( \
op, device) {} \
};
OVERRIDE_EVALUATOR(Eigen::ThreadPoolDevice);
OVERRIDE_EVALUATOR(Eigen::DefaultDevice);
#undef OVERRIDE_EVALUATOR
}; // namespace Eigen
#endif // TENSORFLOW_CORE_KERNELS_EIGEN_VOLUME_PATCH_H_
@@ -0,0 +1,56 @@
/* Copyright 2016 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
#ifndef TENSORFLOW_CORE_KERNELS_EXTRACT_IMAGE_PATCHES_OP_H_
#define TENSORFLOW_CORE_KERNELS_EXTRACT_IMAGE_PATCHES_OP_H_
#include "third_party/eigen3/unsupported/Eigen/CXX11/Tensor"
#include "tensorflow/core/framework/tensor_shape.h"
#include "tensorflow/core/framework/tensor_types.h"
namespace tensorflow {
namespace functor {
template <typename Device, typename T>
struct ExtractImagePatchesForward {
void operator()(const Device& d, typename TTypes<T, 4>::ConstTensor input,
int patch_rows, int patch_cols, int stride_rows,
int stride_cols, int rate_rows, int rate_cols,
const Eigen::PaddingType& padding,
typename TTypes<T, 4>::Tensor output) {
// Need to swap row/col when calling Eigen, because our data is in
// NHWC format while Eigen assumes NWHC format.
const int64 N = std::max(input.size(), output.size());
if (N <= std::numeric_limits<Index32>::max()) {
auto output_32bit = To32Bit(output);
output_32bit.device(d) =
To32Bit(input)
.extract_image_patches(patch_cols, patch_rows, stride_cols,
stride_rows, rate_cols, rate_rows, padding)
.reshape(output_32bit.dimensions());
} else {
output.device(d) =
input
.extract_image_patches(patch_cols, patch_rows, stride_cols,
stride_rows, rate_cols, rate_rows, padding)
.reshape(output.dimensions());
}
}
};
} // namespace functor
} // namespace tensorflow
#endif // TENSORFLOW_CORE_KERNELS_EXTRACT_IMAGE_PATCHES_OP_H_
@@ -0,0 +1,58 @@
/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
#ifndef TENSORFLOW_KERNELS_EXTRACT_VOLUME_PATCHES_OP_H_
#define TENSORFLOW_KERNELS_EXTRACT_VOLUME_PATCHES_OP_H_
#include "tensorflow/core/framework/tensor_shape.h"
#include "tensorflow/core/framework/tensor_types.h"
#include "tensorflow/core/kernels/eigen_volume_patch.h"
#include "third_party/eigen3/unsupported/Eigen/CXX11/Tensor"
namespace tensorflow {
namespace functor {
template <typename Device, typename T>
struct ExtractVolumePatchesForward {
void operator()(const Device& d, typename TTypes<T, 5>::ConstTensor input,
int patch_planes, int patch_rows, int patch_cols,
int stride_planes, int stride_rows, int stride_cols,
/* int rate_planes, int rate_rows, int rate_cols, */
const Eigen::PaddingType& padding,
typename TTypes<T, 5>::Tensor output) {
const int64 N = std::max(input.size(), output.size());
if (N <= std::numeric_limits<Index32>::max()) {
auto output_32bit = To32Bit(output);
output_32bit.device(d) =
To32Bit(input)
.extract_volume_patches(patch_cols, patch_rows, patch_planes,
stride_cols, stride_rows, stride_planes,
padding)
.reshape(output_32bit.dimensions());
} else {
output.device(d) =
input
.extract_volume_patches(patch_cols, patch_rows, patch_planes,
stride_cols, stride_rows, stride_planes,
padding)
.reshape(output.dimensions());
}
}
};
} // namespace functor
} // namespace tensorflow
#endif // TENSORFLOW_KERNELS_EXTRACT_VOLUME_PATCHES_OP_H_
@@ -0,0 +1,280 @@
/* Copyright 2016 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
#ifndef TENSORFLOW_CORE_KERNELS_FAKE_QUANT_OPS_FUNCTOR_H_
#define TENSORFLOW_CORE_KERNELS_FAKE_QUANT_OPS_FUNCTOR_H_
#include <tuple>
#define EIGEN_STACK_ALLOCATION_LIMIT 0
#define EIGEN_USE_THREADS
#include "third_party/eigen3/unsupported/Eigen/CXX11/Tensor"
#include "tensorflow/core/framework/tensor_types.h"
#include "tensorflow/core/platform/types.h"
EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE float StdRound(float input) {
// On Android, std::round() isn't present, just round().
#if defined(__ANDROID__)
return round(input);
#else
return std::round(input);
#endif
}
namespace tensorflow {
// Gymnastics with nudged zero point is to ensure that real zero maps to
// an integer, which is required for e.g. zero-padding in convolutional layers.
// Outputs nudged_min, nudged_max, nudged_scale.
EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE void Nudge(
const float min, const float max, const int quant_min, const int quant_max,
float* nudged_min, float* nudged_max, float* scale) {
const float quant_min_float = static_cast<float>(quant_min);
const float quant_max_float = static_cast<float>(quant_max);
*scale = (max - min) / (quant_max_float - quant_min_float);
const float zero_point_from_min = quant_min_float - min / *scale;
const uint16 nudged_zero_point = [zero_point_from_min, quant_min,
quant_min_float, quant_max,
quant_max_float] {
if (zero_point_from_min < quant_min_float) {
return static_cast<uint16>(quant_min);
}
if (zero_point_from_min > quant_max_float) {
return static_cast<uint16>(quant_max);
}
return static_cast<uint16>(StdRound(zero_point_from_min));
}();
*nudged_min = (quant_min_float - nudged_zero_point) * (*scale);
*nudged_max = (quant_max_float - nudged_zero_point) * (*scale);
}
template <typename T>
using ConstScalar = typename tensorflow::TTypes<T>::ConstScalar;
template <typename T>
using Scalar = typename tensorflow::TTypes<T>::Scalar;
template <typename T>
using ConstVec = typename tensorflow::TTypes<T>::ConstVec;
template <typename T>
using Vec = typename tensorflow::TTypes<T>::Vec;
template <typename T>
using ConstFlat = typename tensorflow::TTypes<T>::ConstFlat;
template <typename T>
using Flat = typename tensorflow::TTypes<T>::Flat;
// Functor called by FakeQuantWithMinMaxArgsOp to do the work. Compiles both
// for CPU and GPU.
template <typename Device>
struct FakeQuantWithMinMaxArgsFunctor {
void operator()(const Device& d, ConstFlat<float> inputs, const float min,
const float max, const int quant_min, const int quant_max,
Flat<float> outputs) {
eigen_assert(min <= 0.0f && "min should be <= 0.0");
eigen_assert(max >= 0.0f && "max should be >= 0.0");
eigen_assert(min < max && "min should be < max");
float nudged_min, nudged_max, nudged_scale;
Nudge(min, max, quant_min, quant_max, &nudged_min, &nudged_max,
&nudged_scale);
const float inv_nudged_scale = 1.0f / nudged_scale;
auto clamped = inputs.cwiseMin(nudged_max).cwiseMax(nudged_min);
auto clamped_shifted = clamped - nudged_min;
outputs.device(d) =
(clamped_shifted * inv_nudged_scale + 0.5f).floor() * nudged_scale +
nudged_min;
}
};
// Functor called by FakeQuantWithMinMaxArgsGradientOp to do the work. Compiles
// both for CPU and GPU.
template <typename Device>
struct FakeQuantWithMinMaxArgsGradientFunctor {
void operator()(const Device& d, ConstFlat<float> gradients,
ConstFlat<float> inputs, const float min, const float max,
const int quant_min, const int quant_max,
Flat<float> backprops) {
eigen_assert(min <= 0.0f && "min should be <= 0.0");
eigen_assert(max >= 0.0f && "max should be >= 0.0");
eigen_assert(min < max && "min should be < max");
float nudged_min, nudged_max, nudged_scale;
Nudge(min, max, quant_min, quant_max, &nudged_min, &nudged_max,
&nudged_scale);
auto between_nudged_min_max =
(inputs >= nudged_min && inputs <= nudged_max)
.select(inputs.constant(1.0f), inputs.constant(0.0f));
backprops.device(d) = gradients * between_nudged_min_max;
}
};
// Functor called by FakeQuantWithMinMaxVarsOp to do the work. Compiles both
// for CPU and GPU.
template <typename Device>
struct FakeQuantWithMinMaxVarsFunctor {
void operator()(const Device& d, ConstFlat<float> inputs,
ConstScalar<float> min, ConstScalar<float> max,
const int quant_min, const int quant_max,
Flat<float> outputs) {
const float min_val = min();
const float max_val = max();
// If min and max are both zero, we should just return zero.
if (min_val == 0.0f && max_val == 0.0f) {
outputs.device(d) = outputs.constant(0.0f);
return;
}
float nudged_min, nudged_max, nudged_scale;
Nudge(min_val, max_val, quant_min, quant_max, &nudged_min, &nudged_max,
&nudged_scale);
const auto nudged_scale_repl = inputs.constant(nudged_scale);
const auto clamped = inputs.cwiseMin(nudged_max).cwiseMax(nudged_min);
const auto clamped_shifted = clamped - nudged_min;
outputs.device(d) = (clamped_shifted / nudged_scale_repl + 0.5f).floor() *
nudged_scale_repl +
nudged_min;
}
};
// Functor called by FakeQuantWithMinMaxVarsGradientOp to do the work. Compiles
// both for CPU and GPU.
template <typename Device>
struct FakeQuantWithMinMaxVarsGradientFunctor {
void operator()(const Device& d, ConstFlat<float> gradients,
ConstFlat<float> inputs, ConstScalar<float> min,
ConstScalar<float> max, const int quant_min,
const int quant_max, Flat<float> backprops_wrt_input,
Scalar<float> backprop_wrt_min,
Scalar<float> backprop_wrt_max) {
const float min_val = min();
const float max_val = max();
// If min and max are both zero, we propagate everything to inputs.
if (min_val == 0.0f && max_val == 0.0f) {
backprops_wrt_input.device(d) = gradients;
backprop_wrt_min.device(d) = backprop_wrt_min.constant(0.0f);
backprop_wrt_max.device(d) = backprop_wrt_max.constant(0.0f);
return;
}
float nudged_min, nudged_max, nudged_scale;
Nudge(min_val, max_val, quant_min, quant_max, &nudged_min, &nudged_max,
&nudged_scale);
const auto between_min_max =
(inputs >= nudged_min && inputs <= nudged_max)
.select(inputs.constant(1.0f), inputs.constant(0.0f));
backprops_wrt_input.device(d) = gradients * between_min_max;
const auto below_min =
(inputs < nudged_min)
.select(inputs.constant(1.0f), inputs.constant(0.0f));
backprop_wrt_min.device(d) = (gradients * below_min).sum();
const auto above_max =
(inputs > nudged_max)
.select(inputs.constant(1.0f), inputs.constant(0.0f));
backprop_wrt_max.device(d) = (gradients * above_max).sum();
}
};
using Index = typename tensorflow::TTypes<float>::ConstTensor::Index;
// Functor called by FakeQuantWithMinMaxVarsPerChannelOp to do the work.
// Compiles both for CPU and GPU.
//
// Already verified: inputs, outputs are of shape [b, d], min, max are of shape
// [d].
template <typename Device>
struct FakeQuantWithMinMaxVarsPerChannelFunctor {
void operator()(const Device& d, TTypes<float>::ConstMatrix inputs,
ConstVec<float> min, ConstVec<float> max, const int quant_min,
const int quant_max, TTypes<float>::Matrix outputs) {
for (Index i = 0; i < min.size(); ++i) {
const float min_val = min(i);
const float max_val = max(i);
// If min and max are both zero, we should just return zero.
if (min_val == 0.0f && max_val == 0.0f) {
auto chip = outputs.chip<1>(i);
chip.device(d) = chip.constant(0.0f);
continue;
}
float nudged_min, nudged_max, nudged_scale;
Nudge(min_val, max_val, quant_min, quant_max, &nudged_min, &nudged_max,
&nudged_scale);
const auto clamped =
inputs.chip<1>(i).cwiseMin(nudged_max).cwiseMax(nudged_min);
const auto clamped_shifted = clamped - nudged_min;
outputs.chip<1>(i).device(d) =
(clamped_shifted / nudged_scale + 0.5f).floor() * nudged_scale +
nudged_min;
}
}
};
// Functor called by FakeQuantWithMinMaxVarsPerChannelGradientOp to do the work.
// Compiles both for CPU and GPU.
//
// Already verified: gradients, inputs, backprops_wrt_input are of shape [b, d],
// min, max, backprop_wrt_min, backprop_wrt_max are of shape [d].
template <typename Device>
struct FakeQuantWithMinMaxVarsPerChannelGradientFunctor {
void operator()(const Device& d, TTypes<float>::ConstMatrix gradients,
TTypes<float>::ConstMatrix inputs, ConstVec<float> min,
ConstVec<float> max, const int quant_min, const int quant_max,
TTypes<float>::Matrix backprops_wrt_input,
Vec<float> backprop_wrt_min, Vec<float> backprop_wrt_max) {
for (Index i = 0; i < min.size(); ++i) {
const float min_val = min(i);
const float max_val = max(i);
const auto gradients_chip = gradients.chip<1>(i);
const auto inputs_chip = inputs.chip<1>(i);
// If min and max are both zero, we propagate everything to inputs.
if (min_val == 0.0f && max_val == 0.0f) {
backprops_wrt_input.chip<1>(i).device(d) = gradients_chip;
auto min_chip = backprop_wrt_min.chip<0>(i);
auto max_chip = backprop_wrt_max.chip<0>(i);
min_chip.device(d) = min_chip.constant(0.0f);
max_chip.device(d) = max_chip.constant(0.0f);
continue;
}
float nudged_min, nudged_max, nudged_scale;
Nudge(min_val, max_val, quant_min, quant_max, &nudged_min, &nudged_max,
&nudged_scale);
const auto between_min_max =
(inputs_chip >= nudged_min && inputs_chip <= nudged_max)
.select(inputs_chip.constant(1.0f), inputs_chip.constant(0.0f));
backprops_wrt_input.chip<1>(i).device(d) =
gradients_chip * between_min_max;
const auto below_min =
(inputs_chip < nudged_min)
.select(inputs_chip.constant(1.0f), inputs_chip.constant(0.0f));
Eigen::DSizes<Index, 1> reduce(0);
backprop_wrt_min.chip<0>(i).device(d) =
(gradients_chip * below_min).sum(reduce);
const auto above_max =
(inputs_chip > nudged_max)
.select(inputs_chip.constant(1.0f), inputs_chip.constant(0.0f));
backprop_wrt_max.chip<0>(i).device(d) =
(gradients_chip * above_max).sum(reduce);
}
}
};
} // namespace tensorflow
#endif // TENSORFLOW_CORE_KERNELS_FAKE_QUANT_OPS_FUNCTOR_H_
@@ -0,0 +1,91 @@
/* Copyright 2015 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
#ifndef TENSORFLOW_CORE_KERNELS_FIFO_QUEUE_H_
#define TENSORFLOW_CORE_KERNELS_FIFO_QUEUE_H_
#include <deque>
#include <vector>
#include "tensorflow/core/framework/op_kernel.h"
#include "tensorflow/core/framework/tensor.h"
#include "tensorflow/core/framework/tensor_shape.h"
#include "tensorflow/core/framework/types.h"
#include "tensorflow/core/kernels/queue_op.h"
#include "tensorflow/core/kernels/typed_queue.h"
#include "tensorflow/core/platform/macros.h"
#include "tensorflow/core/platform/mutex.h"
#include "tensorflow/core/platform/types.h"
namespace tensorflow {
class FIFOQueue : public TypedQueue<std::deque<PersistentTensor> > {
public:
FIFOQueue(int32 capacity, const DataTypeVector& component_dtypes,
const std::vector<TensorShape>& component_shapes,
const string& name);
// Implementations of QueueInterface methods --------------------------------
void TryEnqueue(const Tuple& tuple, OpKernelContext* ctx,
DoneCallback callback) override;
void TryEnqueueMany(const Tuple& tuple, OpKernelContext* ctx,
DoneCallback callback) override;
void TryDequeue(OpKernelContext* ctx, CallbackWithTuple callback) override;
void TryDequeueMany(int num_elements, OpKernelContext* ctx,
bool allow_small_batch,
CallbackWithTuple callback) override;
Status MatchesNodeDef(const NodeDef& node_def) override;
int32 size() override {
mutex_lock lock(mu_);
return queues_[0].size();
}
protected:
~FIFOQueue() override {}
// Helper for dequeuing a single element from queues_.
void DequeueLocked(OpKernelContext* ctx, Tuple* tuple)
EXCLUSIVE_LOCKS_REQUIRED(mu_);
static Status GetElementComponentFromBatch(const Tuple& tuple, int64 index,
int component,
OpKernelContext* ctx,
PersistentTensor* out_element);
private:
TF_DISALLOW_COPY_AND_ASSIGN(FIFOQueue);
};
// Defines a FIFOQueueOp, which produces a Queue (specifically, one
// backed by FIFOQueue) that persists across different graph
// executions, and sessions. Running this op produces a single-element
// tensor of handles to Queues in the corresponding device.
class FIFOQueueOp : public TypedQueueOp {
public:
explicit FIFOQueueOp(OpKernelConstruction* context);
private:
Status CreateResource(QueueInterface** ret) override
EXCLUSIVE_LOCKS_REQUIRED(mu_);
std::vector<TensorShape> component_shapes_;
TF_DISALLOW_COPY_AND_ASSIGN(FIFOQueueOp);
};
} // namespace tensorflow
#endif // TENSORFLOW_CORE_KERNELS_FIFO_QUEUE_H_
@@ -0,0 +1,92 @@
/* Copyright 2015 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
#ifndef TENSORFLOW_CORE_KERNELS_FILL_FUNCTOR_H_
#define TENSORFLOW_CORE_KERNELS_FILL_FUNCTOR_H_
#define EIGEN_USE_THREADS
#include "third_party/eigen3/unsupported/Eigen/CXX11/Tensor"
#include "tensorflow/core/framework/tensor_types.h"
#include "tensorflow/core/framework/types.h"
namespace tensorflow {
namespace functor {
template <typename Device, typename T>
struct FillFunctor {
// Computes on device "d": out = out.constant(in(0)),
void operator()(const Device& d, typename TTypes<T>::Flat out,
typename TTypes<T>::ConstScalar in);
};
template <typename Device, typename T>
struct SetZeroFunctor {
// Computes on device "d": out = out.setZero(),
void operator()(const Device& d, typename TTypes<T>::Flat out);
};
// Partial specialization of SetZeroFunctor<Device=Eigen::ThreadPoolDevice, T>.
template <typename T>
struct SetZeroFunctor<Eigen::ThreadPoolDevice, T> {
void operator()(const Eigen::ThreadPoolDevice& d,
typename TTypes<T>::Flat out);
};
#ifdef TENSORFLOW_USE_SYCL
// Partial specialization of SetZeroFunctor<Device=Eigen::SyclDevice, T>.
template <typename T>
struct SetZeroFunctor<Eigen::SyclDevice, T> {
void operator()(const Eigen::SyclDevice& d, typename TTypes<T>::Flat out);
};
#endif // TENSORFLOW_USE_SYCL
template <>
struct SetZeroFunctor<Eigen::ThreadPoolDevice, string> {
void operator()(const Eigen::ThreadPoolDevice& d,
typename TTypes<string>::Flat out);
};
template <typename Device, typename T>
struct SetOneFunctor {
// Computes on device "d": out = out.setOne(),
void operator()(const Device& d, typename TTypes<T>::Flat out);
};
// Partial specialization of SetOneFunctor<Device=Eigen::ThreadPoolDevice, T>.
template <typename T>
struct SetOneFunctor<Eigen::ThreadPoolDevice, T> {
void operator()(const Eigen::ThreadPoolDevice& d,
typename TTypes<T>::Flat out);
};
#ifdef TENSORFLOW_USE_SYCL
// Partial specialization of SetOneFunctor<Device=Eigen::SyclDevice, T>.
template <typename T>
struct SetOneFunctor<Eigen::SyclDevice, T> {
void operator()(const Eigen::SyclDevice& d, typename TTypes<T>::Flat out);
};
#endif // TENSORFLOW_USE_SYCL
template <>
struct SetOneFunctor<Eigen::ThreadPoolDevice, string> {
void operator()(const Eigen::ThreadPoolDevice& d,
typename TTypes<string>::Flat out);
};
} // namespace functor
} // namespace tensorflow
#endif // TENSORFLOW_CORE_KERNELS_FILL_FUNCTOR_H_
@@ -0,0 +1,78 @@
/* Copyright 2016 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
#ifndef TENSORFLOW_CORE_KERNELS_FRACTIONAL_POOL_COMMON_H_
#define TENSORFLOW_CORE_KERNELS_FRACTIONAL_POOL_COMMON_H_
#include <algorithm>
#include <vector>
#include "tensorflow/core/util/guarded_philox_random.h"
namespace tensorflow {
// Shuffle a container randomly, copied from random_shuffle_op.cc
template <class Iter, class Random>
static inline void RandomShuffle(Iter first, Iter last, const Random& uniform) {
if (first == last) {
return;
}
const auto stop = last - 1;
for (auto i = first; i != stop; ++i) {
using std::iter_swap;
iter_swap(i, i + uniform(last - i));
}
}
// Generate pooling sequence for fractional pooling along one dimension.
//
// Regular max/avg pooling can be viewed as a special case, in which given the
// * input_length: e.g. 10
// * output_length: e.g. 5
// it will generate pooling sequence as
// diff sequence: [2, 2, 2, 2, 2]
// or as
// cumulative sequence: [0, 2, 4, 6, 8, 10]
//
// In the case of fractional pooling, input_length is not an integer multiple of
// output_length, randomness plays a role when generating pooling sequence.
// There are two type of randomness (random vs pseudo-random) defined in paper:
// http://arxiv.org/abs/1412.6071
// You can check the paper for the difference between these two types.
//
// In summary, the generated diff sequence satisfy the following properties for
// both types of randomness:
// * length(generated_diff_pooling_sequence) = output_length
// * sum(generated_diff_pooling_sequence) = input_length
// * Let's define floor(input_length / output_length) = K, then
// K <= generated_diff_pooling_sequence[i] <= K+1
// For example, when input_length = 10, output_length = 6, the following are
// valid pooling sequence:
// * [1, 2, 2, 1, 2, 2]
// * [1, 1, 2, 2, 2, 2]
// [1, 3, 2, 2, 2, 2] is not valid.
//
// Args:
// input_length: See above explanation
// output_length: See above explanation
// generator: Parallel version of random number generator
// pseudo_random: Whether or not use pseudo-random
// Returns:
// pooling_sequence: This is the cumulative pooling sequence.
std::vector<int64> GeneratePoolingSequence(int input_length, int output_length,
GuardedPhiloxRandom* generator,
bool pseudo_random);
} // namespace tensorflow
#endif // TENSORFLOW_CORE_KERNELS_FRACTIONAL_POOL_COMMON_H_
@@ -0,0 +1,79 @@
/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
#ifndef TENSORFLOW_CORE_KERNELS_FUNCTION_OPS_H_
#define TENSORFLOW_CORE_KERNELS_FUNCTION_OPS_H_
#include "tensorflow/core/framework/function.h"
#include "tensorflow/core/framework/op_kernel.h"
namespace tensorflow {
static const char* const kArgOp = FunctionLibraryDefinition::kArgOp;
static const char* const kRetOp = FunctionLibraryDefinition::kRetOp;
class ArgOp : public OpKernel {
public:
explicit ArgOp(OpKernelConstruction* ctx);
void Compute(OpKernelContext* ctx) override;
bool IsExpensive() override { return false; }
private:
int index_;
DataType dtype_;
TF_DISALLOW_COPY_AND_ASSIGN(ArgOp);
};
class RetvalOp : public OpKernel {
public:
explicit RetvalOp(OpKernelConstruction* ctx);
void Compute(OpKernelContext* ctx) override;
bool IsExpensive() override { return false; }
private:
int index_;
DataType dtype_;
TF_DISALLOW_COPY_AND_ASSIGN(RetvalOp);
};
class RemoteCallOp : public AsyncOpKernel {
public:
explicit RemoteCallOp(OpKernelConstruction* ctx);
~RemoteCallOp() override {}
void ComputeAsync(OpKernelContext* ctx, DoneCallback done) override;
private:
NameAttrList func_;
DataTypeVector input_dtypes_;
DataTypeVector output_dtypes_;
mutex mu_;
typedef std::pair<string, FunctionLibraryRuntime*> FunctionTarget;
std::map<FunctionTarget, FunctionLibraryRuntime::Handle> handle_cache_
GUARDED_BY(mu_);
TF_DISALLOW_COPY_AND_ASSIGN(RemoteCallOp);
};
} // namespace tensorflow
#endif // TENSORFLOW_CORE_KERNELS_FUNCTION_OPS_H_
@@ -0,0 +1,131 @@
/* Copyright 2016 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
#ifndef TENSORFLOW_CORE_KERNELS_FUSED_BATCH_NORM_OP_H_
#define TENSORFLOW_CORE_KERNELS_FUSED_BATCH_NORM_OP_H_
#include "third_party/eigen3/unsupported/Eigen/CXX11/Tensor"
#include "tensorflow/core/framework/tensor.h"
#include "tensorflow/core/framework/tensor_types.h"
namespace tensorflow {
namespace functor {
#if GOOGLE_CUDA
// There is a behavior difference between cuDNN v4 and v5 with regard to the
// scaling factor for function cudnnBatchNormalizationForwardInference.
// This function corrects the scaling factor if cuDNN v4 is used, so that
// this behavior inconsistency is hidden from TensorFlow users.
// Details: in cuDNN v4, y = bnScale * (x - mean) * variance + bnBias;
// in v5, y = bnScale * (x - mean) / sqrt(variance + epsilon) + bnBias
// The template is instantiated with T as float in batch_norm_ops.cu.cc; for
// other types, the instantiation needs to be added accordingly.
template <class T>
struct VarianceToInvVariance {
void operator()(const Eigen::GpuDevice& d, const T* variance, double epsilon,
int channels, T* inv_variance);
};
// This function converts the inverted variance of the cuDNN forward training
// output to variance for TensorFlow to calculate the running variance.
// The template is instantiated with T as float in batch_norm_ops.cu.cc; for
// other types, the instantiation needs to be added accordingly.
template <class T>
struct InvVarianceToVariance {
void operator()(const Eigen::GpuDevice& d, double epsilon, int sample_size,
int channels, T* variance);
};
// This function sets a GPU tensor to NaNs.
template <class T>
struct SetNanFunctor {
void operator()(const Eigen::GpuDevice& d, typename TTypes<T>::Flat out);
};
#endif // GOOGLE_CUDA
// Functor used by FusedBatchNormGradOp to do the computations when
// is_training=False. Both CPU and GPU will use this functor.
template <typename Device, typename T, typename U>
struct FusedBatchNormFreezeGrad {
void operator()(const Device& d, const Tensor& y_backprop_input,
const Tensor& x_input, const Tensor& scale_input,
const Tensor& pop_mean_input,
const Tensor& pop_variance_input, U epsilon,
Tensor* x_backprop_output, Tensor* scale_backprop_output,
Tensor* offset_backprop_output,
typename TTypes<U>::Vec scratch1,
typename TTypes<U>::Vec scratch2) {
typename TTypes<T, 4>::ConstTensor y_backprop(
y_backprop_input.tensor<T, 4>());
typename TTypes<T, 4>::ConstTensor input(x_input.tensor<T, 4>());
typename TTypes<U>::ConstVec scale(scale_input.vec<U>());
typename TTypes<U>::ConstVec pop_mean(pop_mean_input.vec<U>());
typename TTypes<U>::ConstVec pop_var(pop_variance_input.vec<U>());
typename TTypes<T, 4>::Tensor x_backprop(x_backprop_output->tensor<T, 4>());
typename TTypes<U>::Vec scale_backprop(scale_backprop_output->vec<U>());
typename TTypes<U>::Vec offset_backprop(offset_backprop_output->vec<U>());
const int depth = pop_mean.dimension(0);
const int rest_size = input.size() / depth;
Eigen::DSizes<Eigen::Index, 2> rest_by_depth(rest_size, depth);
#if !defined(EIGEN_HAS_INDEX_LIST)
Eigen::DSizes<Eigen::Index, 2> one_by_depth(1, depth);
Eigen::array<int, 1> reduction_axis{0};
Eigen::array<int, 2> rest_by_one({rest_size, 1});
#else
Eigen::IndexList<Eigen::type2index<1>, Eigen::Index> one_by_depth;
one_by_depth.set(1, depth);
Eigen::IndexList<Eigen::type2index<0> > reduction_axis;
Eigen::IndexList<Eigen::Index, Eigen::type2index<1> > rest_by_one;
rest_by_one.set(0, rest_size);
#endif
// offset_backprop = sum(y_backprop)
// scale_backprop = y_backprop * ((x - pop_mean) * rsqrt(pop_var + epsilon))
// x_backprop = y_backprop * (scale * rsqrt(pop_var + epsilon))
auto y_backprop_rest_by_depth =
y_backprop.reshape(rest_by_depth).template cast<U>();
auto input_rest_by_depth = input.reshape(rest_by_depth).template cast<U>();
offset_backprop.device(d) = y_backprop_rest_by_depth.sum(reduction_axis);
// scratch1 = rsqrt(pop_var + epsilon)
scratch1.device(d) = (pop_var + pop_var.constant(epsilon)).rsqrt();
// scratch2 = sum(y_backprop * (x - mean))
scratch2.device(d) =
(y_backprop_rest_by_depth *
(input_rest_by_depth -
pop_mean.reshape(one_by_depth).broadcast(rest_by_one)))
.sum(reduction_axis);
x_backprop.reshape(rest_by_depth).device(d) =
(y_backprop_rest_by_depth * ((scratch1 * scale)
.eval()
.reshape(one_by_depth)
.broadcast(rest_by_one)))
.template cast<T>();
scale_backprop.device(d) = scratch2 * scratch1;
}
};
} // namespace functor
} // namespace tensorflow
#endif // TENSORFLOW_CORE_KERNELS_FUSED_BATCH_NORM_OP_H_
@@ -0,0 +1,180 @@
/* Copyright 2016 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
#ifndef TENSORFLOW_CORE_KERNELS_GATHER_FUNCTOR_H_
#define TENSORFLOW_CORE_KERNELS_GATHER_FUNCTOR_H_
#include "third_party/eigen3/unsupported/Eigen/CXX11/Tensor"
#include "tensorflow/core/framework/op_kernel.h"
#include "tensorflow/core/framework/tensor_types.h"
#include "tensorflow/core/framework/type_traits.h"
#include "tensorflow/core/framework/variant.h"
#include "tensorflow/core/kernels/bounds_check.h"
#include "tensorflow/core/platform/prefetch.h"
#include "tensorflow/core/platform/types.h"
#include "tensorflow/core/util/work_sharder.h"
namespace tensorflow {
typedef Eigen::ThreadPoolDevice CPUDevice;
typedef Eigen::GpuDevice GPUDevice;
namespace functor {
// Helper method to copy using memcpy.
template <typename T, typename Index, typename SliceIndex,
SliceIndex static_slice_elems>
SliceIndex HandleCopies(OpKernelContext* ctx,
typename TTypes<T, 3>::ConstTensor params,
typename TTypes<Index>::ConstFlat indices,
SliceIndex slice_elems,
typename TTypes<T, 3>::Tensor out) {
const SliceIndex indices_size = static_cast<SliceIndex>(indices.dimension(0));
const SliceIndex batch_size = static_cast<SliceIndex>(params.dimension(0));
const Index limit = static_cast<Index>(params.dimension(1));
T* out_base = &out(0, 0, 0);
const T* params_base = &params(0, 0, 0);
if (static_slice_elems >= 0) {
// Give compiler static knowledge of the number of elements/bytes
slice_elems = static_slice_elems;
}
// Compute slice_bytes here so that static knowledge is available
const size_t slice_bytes = slice_elems * sizeof(T);
auto* worker_threads = ctx->device()->tensorflow_cpu_worker_threads();
mutex mu;
// Store the value of invalidate index for printing error information, it's a
// shared variable.
SliceIndex result = -1;
auto work = [&](int64 start, int64 end) {
SliceIndex batch_idx = static_cast<SliceIndex>(start / indices_size);
SliceIndex indices_idx = static_cast<SliceIndex>(start % indices_size);
SliceIndex batch_idx_end = static_cast<SliceIndex>(end / indices_size);
SliceIndex indices_idx_end = static_cast<SliceIndex>(end % indices_size);
while ((batch_idx < batch_idx_end) ||
(batch_idx == batch_idx_end && indices_idx < indices_idx_end)) {
SliceIndex i_next = indices_idx + 1;
SliceIndex b_next = batch_idx + 1;
if ((batch_idx == batch_idx_end && i_next < indices_idx_end) ||
(i_next < indices_size)) {
port::prefetch<port::PREFETCH_HINT_T0>(
&params(batch_idx, indices(i_next), 0));
port::prefetch<port::PREFETCH_HINT_T0>(&out(batch_idx, i_next, 0));
b_next = batch_idx;
} else if (b_next <= batch_idx_end) {
port::prefetch<port::PREFETCH_HINT_T0>(&params(b_next, indices(0), 0));
port::prefetch<port::PREFETCH_HINT_T0>(&out(b_next, 0, 0));
i_next = 0;
}
const Index index = internal::SubtleMustCopy(indices(indices_idx));
if (!FastBoundsCheck(index, limit)) {
mutex_lock l(mu);
result = indices_idx;
return;
}
// Copy using memcpy if possible, otherwise an Eigen loop
// TODO(cwhipkey): avoid linking to framework to get Allocator (to improve
// ahead-of-time compilation binary size).
if (is_simple_type<T>::value) {
// Avoid auto-promotion to Index from SliceIndex by casting.
memcpy(
out_base + (batch_idx * indices_size + indices_idx) * slice_elems,
params_base + (batch_idx * static_cast<SliceIndex>(limit) +
static_cast<SliceIndex>(index)) *
slice_elems,
slice_bytes);
} else {
// For non-"simple" types (e.g. strings).
out.template chip<1>(indices_idx) = params.template chip<1>(index);
}
indices_idx = i_next;
batch_idx = b_next;
}
};
Shard(worker_threads->num_threads, worker_threads->workers,
batch_size * indices_size, slice_elems * sizeof(T), work);
return result;
}
template <typename T, typename Index>
struct GatherFunctorCPU {
int64 operator()(OpKernelContext* ctx,
typename TTypes<T, 3>::ConstTensor params,
typename TTypes<Index>::ConstFlat indices,
typename TTypes<T, 3>::Tensor out) {
const int64 N = indices.size();
const int64 slice_size = out.dimension(2);
int64 bad_i;
bool use_large = (slice_size > std::numeric_limits<int32>::max() ||
params.size() > std::numeric_limits<int32>::max() ||
N > std::numeric_limits<int32>::max());
#define CALL(elems) \
do { \
if (use_large) { \
bad_i = HandleCopies<T, Index, int64, elems>(ctx, params, indices, \
slice_size, out); \
} else { \
const int32 small_slice = static_cast<int32>(slice_size); \
bad_i = HandleCopies<T, Index, int32, elems>(ctx, params, indices, \
small_slice, out); \
} \
} while (0)
if (slice_size == 10)
CALL(10);
else if (slice_size == 20)
CALL(20);
else
CALL(-1);
#undef CALL
return bad_i;
}
};
template <typename Device, typename T, typename Index>
struct GatherFunctor {
int64 operator()(OpKernelContext* ctx,
typename TTypes<T, 3>::ConstTensor params,
typename TTypes<Index>::ConstFlat indices,
typename TTypes<T, 3>::Tensor out);
};
template <typename T, typename Index>
struct GatherFunctor<CPUDevice, T, Index> {
int64 operator()(OpKernelContext* ctx,
typename TTypes<T, 3>::ConstTensor params,
typename TTypes<Index>::ConstFlat indices,
typename TTypes<T, 3>::Tensor out) {
return GatherFunctorCPU<T, Index>()(ctx, params, indices, out);
}
};
template <typename Index>
struct GatherFunctor<GPUDevice, Variant, Index> {
int64 operator()(OpKernelContext* ctx,
typename TTypes<Variant, 3>::ConstTensor params,
typename TTypes<Index>::ConstFlat indices,
typename TTypes<Variant, 3>::Tensor out) {
return GatherFunctorCPU<Variant, Index>()(ctx, params, indices, out);
}
};
} // namespace functor
} // namespace tensorflow
#endif // TENSORFLOW_CORE_KERNELS_GATHER_FUNCTOR_H_
@@ -0,0 +1,50 @@
/* Copyright 2016 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
#ifndef TENSORFLOW_CORE_KERNELS_GATHER_ND_OP_H_
#define TENSORFLOW_CORE_KERNELS_GATHER_ND_OP_H_
// Functor definition for GatherOp, must be compilable by nvcc.
#include "third_party/eigen3/unsupported/Eigen/CXX11/Tensor"
#include "tensorflow/core/framework/tensor_types.h"
#include "tensorflow/core/kernels/bounds_check.h"
#include "tensorflow/core/platform/types.h"
namespace tensorflow {
class OpKernelContext;
class Status;
class Tensor;
namespace functor {
template <typename Device, typename T, typename Index, int IXDIM>
struct GatherNdSlice {
// Performs a slice gather op on (Tparams, Tindices), writing to Tout.
// Returns an index to Tindices if the value at that index is out of range.
// Returns -1 if all values of Tindices are in range.
Index operator()(const Device& d, const Index slice_size,
typename TTypes<int32>::Scalar Tscratch,
typename TTypes<T, IXDIM + 1>::ConstTensor Tparams,
typename TTypes<Index>::ConstMatrix Tindices,
typename TTypes<T>::Matrix Tout);
};
template <typename Device, typename T, typename Index>
Status DoGatherNd(OpKernelContext* c, const Tensor& params,
const Tensor& indices, Tensor* out);
} // namespace functor
} // namespace tensorflow
#endif // TENSORFLOW_CORE_KERNELS_GATHER_ND_OP_H_
@@ -0,0 +1,160 @@
/* Copyright 2016 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
#ifndef TENSORFLOW_CORE_KERNELS_GATHER_ND_OP_CPU_IMPL_H_
#define TENSORFLOW_CORE_KERNELS_GATHER_ND_OP_CPU_IMPL_H_
// Specialization of GatherNdSlice to CPU
#define EIGEN_USE_THREADS
#include <atomic>
#include "tensorflow/core/framework/op_kernel.h"
#include "tensorflow/core/framework/register_types.h"
#include "tensorflow/core/framework/tensor.h"
#include "tensorflow/core/kernels/bounds_check.h"
#include "tensorflow/core/kernels/gather_nd_op.h"
#include "tensorflow/core/platform/logging.h"
#include "tensorflow/core/platform/mem.h"
#include "tensorflow/core/platform/types.h"
#include "tensorflow/core/util/util.h"
namespace tensorflow {
typedef Eigen::ThreadPoolDevice CPUDevice;
namespace generator {
template <typename T, typename Index, int IXDIM>
class GatherNdSliceGenerator {
public:
EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE GatherNdSliceGenerator(
const Index slice_size, typename TTypes<Index>::ConstMatrix Tindices,
typename TTypes<T, IXDIM + 1>::ConstTensor Tparams,
typename TTypes<T>::Matrix Tout, std::atomic<Index>* error_loc)
: slice_size_(slice_size),
Tindices_(Tindices),
Tparams_(Tparams),
Tout_(Tout),
error_loc_(error_loc) {}
EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE bool GenerateIndices(
const Index loc, Eigen::array<Eigen::DenseIndex, IXDIM + 1>* ix) const {
(*ix)[IXDIM] = 0;
bool out_of_bounds = false;
for (int i = 0; i < IXDIM; ++i) {
const Index ix_i = internal::SubtleMustCopy(Tindices_(loc, i));
(*ix)[i] = ix_i;
out_of_bounds |= !FastBoundsCheck(ix_i, Tparams_.dimension(i));
}
return out_of_bounds;
}
EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE int32
operator()(const Eigen::array<Eigen::DenseIndex, 1>& loc_array) const {
const Index loc = loc_array[0];
Eigen::array<Eigen::DenseIndex, IXDIM + 1> ix;
Eigen::array<Eigen::DenseIndex, 2> ix_out;
ix_out[0] = loc;
ix_out[1] = 0;
const bool out_of_bounds = GenerateIndices(loc, &ix);
if (TF_PREDICT_FALSE(out_of_bounds)) {
error_loc_->store(loc);
std::fill_n(&Tout_(ix_out), slice_size_, T());
} else {
std::copy_n(&Tparams_(ix), slice_size_, &Tout_(ix_out));
}
return static_cast<int32>(0); // Return something...
}
private:
const Index slice_size_;
const typename TTypes<Index>::ConstMatrix Tindices_;
const typename TTypes<T, IXDIM + 1>::ConstTensor Tparams_;
mutable typename TTypes<T>::Matrix Tout_;
std::atomic<Index>* error_loc_;
};
} // namespace generator
namespace functor {
template <typename T, typename Index, int IXDIM>
struct GatherNdSlice<CPUDevice, T, Index, IXDIM> {
Index operator()(const CPUDevice& d, const Index slice_size,
typename TTypes<int32>::Scalar Tscratch,
typename TTypes<T, IXDIM + 1>::ConstTensor Tparams,
typename TTypes<Index>::ConstMatrix Tindices,
typename TTypes<T>::Matrix Tout) {
std::atomic<Index> error_loc(-1);
const Eigen::DenseIndex batch_size = Tindices.dimension(0);
#if !defined(EIGEN_HAS_INDEX_LIST)
Eigen::Tensor<Eigen::DenseIndex, 1>::Dimensions reshape_dims{{ 1 }};
Eigen::array<Eigen::DenseIndex, 1> broadcast_dims{{ batch_size }};
#else
Eigen::IndexList<Eigen::type2index<1> > reshape_dims;
Eigen::IndexList<Eigen::DenseIndex> broadcast_dims;
broadcast_dims.set(0, batch_size);
#endif
generator::GatherNdSliceGenerator<T, Index, IXDIM> gather_nd_generator(
slice_size, Tindices, Tparams, Tout, &error_loc);
#if defined(INTEL_MKL) && defined(ENABLE_MKL)
// Eigen implementation below is not highly performant. gather_nd_generator
// does not seem to be called in parallel, leading to very poor performance.
// Additionally, since it uses scalar (Tscratch) to invoke 'generate', it
// needs to go through redundant operations like 'reshape', 'broadcast' and
// 'sum'. OpenMP loop below essentially does same thing as Eigen code, but
// is considerably more efficient.
#pragma omp parallel for
for (Eigen::DenseIndex i = 0; i < batch_size; i++) {
const Eigen::array<Eigen::DenseIndex, 1> loc{i};
gather_nd_generator(loc);
}
#else // INTEL_MKL && ENABLE_MKL
Tscratch.device(d) = Tscratch.reshape(reshape_dims)
.broadcast(broadcast_dims)
.generate(gather_nd_generator)
.sum();
#endif // INTEL_MKL && ENABLE_MKL
// error_loc() returns -1 if there's no out-of-bounds index,
// otherwise it returns the location of an OOB index in Tindices.
return error_loc.load();
}
};
#define REGISTER_GATHER_ND_FULL(T, Index) \
template Index GatherNdSlice<CPUDevice, T, Index, CPU_PROVIDED_IXDIM>:: \
operator()(const CPUDevice& d, const Index slice_size, \
typename TTypes<int32>::Scalar Tscratch, \
typename TTypes<T, CPU_PROVIDED_IXDIM + 1>::ConstTensor Tparams, \
typename TTypes<Index>::ConstMatrix Tindices, \
typename TTypes<T>::Matrix Tout);
#define REGISTER_GATHER_ND_CPU(type) \
REGISTER_GATHER_ND_FULL(type, int32); \
REGISTER_GATHER_ND_FULL(type, int64)
TF_CALL_ALL_TYPES(REGISTER_GATHER_ND_CPU);
} // namespace functor
} // namespace tensorflow
#endif // TENSORFLOW_CORE_KERNELS_GATHER_ND_OP_CPU_IMPL_H_
@@ -0,0 +1,123 @@
/* Copyright 2016 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
// This is a set of different implementations for the basic matrix by matrix
// multiply function, commonly known as GEMM after the BLAS library's naming.
// Having a standard interface enables us to swap out implementations on
// different platforms, to make sure we're using the optimal version. They are
// implemented as C++ template functors, so they're easy to swap into all of the
// different kernels that use them.
#if !defined(EIGEN_USE_THREADS)
#error "EIGEN_USE_THREADS must be enabled by all .cc files including this."
#endif // EIGEN_USE_THREADS
#ifndef TENSORFLOW_CORE_KERNELS_GEMM_FUNCTORS_H_
#define TENSORFLOW_CORE_KERNELS_GEMM_FUNCTORS_H_
#include <string.h>
#include <map>
#include <vector>
#include "tensorflow/core/common_runtime/threadpool_device.h"
#include "tensorflow/core/framework/op_kernel.h"
#include "tensorflow/core/framework/tensor.h"
#include "tensorflow/core/framework/tensor_types.h"
// Apple provides an optimized BLAS library that is better than Eigen for their
// devices, so use that if possible.
#if defined(__APPLE__) && defined(USE_GEMM_FOR_CONV)
#include <Accelerate/Accelerate.h>
#define USE_CBLAS_GEMM
#endif // __APPLE__
// Older Raspberry Pi systems don't have NEON SIMD acceleration, so Eigen falls
// back to scalar code, but OpenBLAS has much faster support so prefer that.
#if defined(RASPBERRY_PI) && defined(USE_GEMM_FOR_CONV) && defined(USE_OPENBLAS)
#include <cblas.h>
#define USE_CBLAS_GEMM
#endif
// A readable but slow implementation of matrix multiplication, useful for
// debugging and understanding the algorithm. Use instead of FastGemmFunctor in
// the Im2ColConvFunctor template definition inside the op registration to
// enable. Assumes row-major ordering of the values in memory.
template <class T1, class T2, class T3>
class ReferenceGemmFunctor {
public:
void operator()(tensorflow::OpKernelContext* ctx, size_t m, size_t n,
size_t k, const T1* a, size_t lda, const T2* b, size_t ldb,
T3* c, size_t ldc) {
const size_t a_i_stride = lda;
const size_t a_l_stride = 1;
const size_t b_j_stride = 1;
const size_t b_l_stride = ldb;
const size_t c_i_stride = ldc;
const size_t c_j_stride = 1;
size_t i, j, l;
for (j = 0; j < n; j++) {
for (i = 0; i < m; i++) {
T3 total(0);
for (l = 0; l < k; l++) {
const size_t a_index = ((i * a_i_stride) + (l * a_l_stride));
const T1 a_value = a[a_index];
const size_t b_index = ((j * b_j_stride) + (l * b_l_stride));
const T2 b_value = b[b_index];
total += (a_value * b_value);
}
const size_t c_index = ((i * c_i_stride) + (j * c_j_stride));
c[c_index] = total;
}
}
}
};
// Uses the optimized EigenTensor library to implement the matrix multiplication
// required by the Im2ColConvFunctor class. We supply the two input and one
// output types so that the accumulator can potentially be higher-precision than
// the inputs, even though we don't currently take advantage of this.
template <class T1, class T2, class T3>
class FastGemmFunctor {
public:
void operator()(tensorflow::OpKernelContext* ctx, size_t m, size_t n,
size_t k, const T1* a, size_t lda, const T2* b, size_t ldb,
T3* c, size_t ldc) {
typename tensorflow::TTypes<const T1>::Matrix a_matrix(a, m, k);
typename tensorflow::TTypes<const T2>::Matrix b_matrix(b, k, n);
typename tensorflow::TTypes<T3>::Matrix c_matrix(c, m, n);
Eigen::array<Eigen::IndexPair<Eigen::DenseIndex>, 1> dim_pair;
dim_pair[0].first = 1;
dim_pair[0].second = 0;
c_matrix.device(ctx->eigen_device<Eigen::ThreadPoolDevice>()) =
a_matrix.contract(b_matrix, dim_pair);
}
};
// If we have a fast CBLAS library, use its implementation through a wrapper.
#if defined(USE_CBLAS_GEMM)
template <>
class FastGemmFunctor<float, float, float> {
public:
void operator()(tensorflow::OpKernelContext* ctx, size_t m, size_t n,
size_t k, const float* a, size_t lda, const float* b,
size_t ldb, float* c, size_t ldc) {
cblas_sgemm(CblasRowMajor, CblasNoTrans, CblasNoTrans, m, n, k, 1.0f, a,
lda, b, ldb, 0.0f, c, ldc);
}
};
#endif // USE_CBLAS_GEMM
#endif // TENSORFLOW_CORE_KERNELS_GEMM_FUNCTORS_H_
@@ -0,0 +1,163 @@
/* Copyright 2015 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
#ifndef TENSORFLOW_CORE_KERNELS_GPU_UTILS_H_
#define TENSORFLOW_CORE_KERNELS_GPU_UTILS_H_
#if GOOGLE_CUDA
#include <unordered_map>
#include "tensorflow/core/lib/strings/str_util.h"
#include "tensorflow/core/lib/strings/strcat.h"
#include "tensorflow/core/lib/strings/stringprintf.h"
#include "tensorflow/core/platform/logging.h"
#include "tensorflow/core/platform/stream_executor.h"
namespace tensorflow {
template <typename T>
inline se::DeviceMemory<T> AsDeviceMemory(const T* cuda_memory, uint64 size) {
se::DeviceMemoryBase wrapped(const_cast<T*>(cuda_memory), size * sizeof(T));
se::DeviceMemory<T> typed(wrapped);
return typed;
}
// A helper class that looks up the best autotuned config from parameters.
// Due to the noisy nature of autotune, especially with multiple devices, it
// only accepts a config if its margin exceeds a threshold.
// For the same shape configs, if a new best config matches the previous best,
// they get promoted; otherwise, the winner gets demoted. This process stops
// when the winner's score exceeds the threshold.
// In a bad case when two configs are very close to each other and flips
// back and forth randomly, the expected number of experiments before autotune
// settles is O(threshold ^ 2). So we recommend that number of warmup runs
// for any benchmarks.
template <typename Parameters, typename Config>
class AutoTuneMap {
public:
bool Find(const Parameters& params, Config* config) const {
mutex_lock lock(mu_);
auto iter = params_config_map_.find(params);
if (iter == params_config_map_.end() ||
(iter->second.score < min_score_threshold_ &&
iter->second.count <= max_autotune_count_)) {
return false;
}
*config = iter->second.config;
return true;
}
void Insert(const Parameters& params, const Config& config) {
mutex_lock lock(mu_);
auto iter = params_config_map_.find(params);
int new_score = 0;
if (iter == params_config_map_.end()) {
// Create a new entry if params is new.
VLOG(1) << GetActionSummary("creates", params, config);
params_config_map_.insert(
std::make_pair(params, ValueType{config, 1, 1}));
new_score = 1;
} else if (iter->second.score < min_score_threshold_ &&
iter->second.count <= max_autotune_count_) {
DCHECK_GT(iter->second.score, 0);
if (iter->second.config != config) {
// If it is different from the current winner, demotes the winner.
VLOG(1) << GetActionSummary("demotes", params, config);
new_score = --iter->second.score;
++iter->second.count;
if (new_score <= 0) {
VLOG(1) << GetActionSummary("erases", params, config);
params_config_map_.erase(iter);
}
} else {
// If it is the same as the current winner, promotes the winner.
VLOG(1) << GetActionSummary("promotes", params, config);
new_score = ++iter->second.score;
++iter->second.count;
}
}
if (new_score >= min_score_threshold_) {
VLOG(1) << GetActionSummary("accepts", params, config);
}
}
private:
AutoTuneMap(const string& name) : name_(name) {
min_score_threshold_ = 1;
int min_warmup_iterations = 10;
const char* threshold_str = getenv("TF_AUTOTUNE_THRESHOLD");
if (threshold_str != nullptr) {
strings::safe_strto32(threshold_str, &min_score_threshold_);
}
const char* min_warmup_iteration_str =
getenv("TF_AUTOTUNE_MIN_WARMUP_ITERATIONS");
if (min_warmup_iteration_str != nullptr) {
strings::safe_strto32(min_warmup_iteration_str, &min_warmup_iterations);
}
min_score_threshold_ = std::max(min_score_threshold_, 1);
max_autotune_count_ = std::max(
5 * min_score_threshold_ * min_score_threshold_, min_warmup_iterations);
}
template <class Group, class Params, class Cfg>
friend class AutoTuneSingleton;
struct Hasher {
std::size_t operator()(const Parameters& parameter) const {
return parameter.hash();
}
};
string GetActionSummary(StringPiece action, const Parameters& params,
const Config& config) {
return strings::Printf("autotune_map %s %s: %s -> (%s)", name_.c_str(),
string(action).c_str(), params.ToString().c_str(),
config.ToString().c_str());
}
mutable mutex mu_;
struct ValueType {
Config config;
int32 score;
int32 count;
};
std::unordered_map<Parameters, ValueType, Hasher> params_config_map_
GUARDED_BY(mu_);
string name_;
int32 min_score_threshold_;
int32 max_autotune_count_;
TF_DISALLOW_COPY_AND_ASSIGN(AutoTuneMap);
};
// A Singleton helper that manages the global autotune results by groups.
// The caller specified arbitrary Group type that can distinguish between
// different autotune results, even if their Parameters and Configs are the
// same.
template <class Group, typename Parameters, typename Config>
class AutoTuneSingleton {
public:
typedef AutoTuneMap<Parameters, Config> AutoTuneType;
static AutoTuneType* GetInstance() {
static AutoTuneType* instance = new AutoTuneType(Group::name());
return instance;
}
};
} // namespace tensorflow
#endif // GOOGLE_CUDA
#endif // TENSORFLOW_CORE_KERNELS_GPU_UTILS_H_
Loaded 100 of 1283 files, more files were not shown because too many files have changed in this diff. Show more