tf
This commit is contained in:
1 parent
eb1b42a5e8
commit
458eb2aef1
1283 files changed
+323373
No files matched your search
@@ -0,0 +1,156 @@
|
||||
/* Copyright 2015 The TensorFlow Authors. All Rights Reserved.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
==============================================================================*/
|
||||
|
||||
#ifndef TENSORFLOW_CORE_KERNELS_ADJUST_CONTRAST_OP_H_
|
||||
#define TENSORFLOW_CORE_KERNELS_ADJUST_CONTRAST_OP_H_
|
||||
#include "third_party/eigen3/unsupported/Eigen/CXX11/Tensor"
|
||||
#include "tensorflow/core/framework/tensor_types.h"
|
||||
|
||||
namespace tensorflow {
|
||||
namespace functor {
|
||||
|
||||
// Functor used by AdjustContrastOp to do the computations.
|
||||
template <typename Device, typename T>
|
||||
struct AdjustContrast {
|
||||
void operator()(const Device& d, typename TTypes<T, 4>::ConstTensor input,
|
||||
typename TTypes<float>::ConstScalar contrast_factor,
|
||||
typename TTypes<float>::ConstScalar min_value,
|
||||
typename TTypes<float>::ConstScalar max_value,
|
||||
typename TTypes<float, 4>::Tensor mean_values,
|
||||
typename TTypes<float, 4>::Tensor output) {
|
||||
const int batch = input.dimension(0);
|
||||
const int height = input.dimension(1);
|
||||
const int width = input.dimension(2);
|
||||
const int channels = input.dimension(3);
|
||||
|
||||
Eigen::array<int, 4> scalar_broadcast;
|
||||
scalar_broadcast[0] = batch;
|
||||
scalar_broadcast[1] = height;
|
||||
scalar_broadcast[2] = width;
|
||||
scalar_broadcast[3] = channels;
|
||||
#if !defined(EIGEN_HAS_INDEX_LIST)
|
||||
Eigen::array<int, 2> reduction_axis;
|
||||
reduction_axis[0] = 1;
|
||||
reduction_axis[1] = 2;
|
||||
Eigen::array<int, 4> broadcast_dims;
|
||||
broadcast_dims[0] = 1;
|
||||
broadcast_dims[1] = height;
|
||||
broadcast_dims[2] = width;
|
||||
broadcast_dims[3] = 1;
|
||||
Eigen::Tensor<int, 4>::Dimensions reshape_dims;
|
||||
reshape_dims[0] = batch;
|
||||
reshape_dims[1] = 1;
|
||||
reshape_dims[2] = 1;
|
||||
reshape_dims[3] = channels;
|
||||
#else
|
||||
Eigen::IndexList<Eigen::type2index<1>, Eigen::type2index<2> >
|
||||
reduction_axis;
|
||||
Eigen::IndexList<Eigen::type2index<1>, int, int, Eigen::type2index<1> >
|
||||
broadcast_dims;
|
||||
broadcast_dims.set(1, height);
|
||||
broadcast_dims.set(2, width);
|
||||
Eigen::IndexList<int, Eigen::type2index<1>, Eigen::type2index<1>, int>
|
||||
reshape_dims;
|
||||
reshape_dims.set(0, batch);
|
||||
reshape_dims.set(3, channels);
|
||||
#endif
|
||||
Eigen::Sizes<1, 1, 1, 1> scalar;
|
||||
float num_reduced_coeffs = height * width;
|
||||
mean_values.device(d) =
|
||||
(input.template cast<float>().sum(reduction_axis).eval() /
|
||||
num_reduced_coeffs)
|
||||
.reshape(reshape_dims)
|
||||
.broadcast(broadcast_dims);
|
||||
|
||||
auto contrast_factor_tensor =
|
||||
contrast_factor.reshape(scalar).broadcast(scalar_broadcast);
|
||||
auto adjusted =
|
||||
(input.template cast<float>() - mean_values) * contrast_factor_tensor +
|
||||
mean_values;
|
||||
auto min_bcast = min_value.reshape(scalar).broadcast(scalar_broadcast);
|
||||
auto max_bcast = max_value.reshape(scalar).broadcast(scalar_broadcast);
|
||||
// TODO(wicke): This is rather slow and should be re-written as pure cuda.
|
||||
output.device(d) = adjusted.cwiseMin(max_bcast).cwiseMax(min_bcast);
|
||||
}
|
||||
};
|
||||
|
||||
// Functor used by AdjustContrastOpv2 to do the computations.
|
||||
template <typename Device>
|
||||
struct AdjustContrastv2 {
|
||||
void operator()(const Device& d, typename TTypes<float, 4>::ConstTensor input,
|
||||
typename TTypes<float>::ConstScalar contrast_factor,
|
||||
typename TTypes<float, 4>::Tensor output) {
|
||||
const int batch = input.dimension(0);
|
||||
const int height = input.dimension(1);
|
||||
const int width = input.dimension(2);
|
||||
const int channels = input.dimension(3);
|
||||
|
||||
Eigen::array<int, 4> scalar_broadcast;
|
||||
scalar_broadcast[0] = batch;
|
||||
scalar_broadcast[1] = height;
|
||||
scalar_broadcast[2] = width;
|
||||
scalar_broadcast[3] = channels;
|
||||
#if !defined(EIGEN_HAS_INDEX_LIST)
|
||||
Eigen::array<int, 2> reduction_axis;
|
||||
reduction_axis[0] = 0;
|
||||
reduction_axis[1] = 1;
|
||||
Eigen::array<int, 4> broadcast_dims;
|
||||
broadcast_dims[0] = 1;
|
||||
broadcast_dims[1] = height;
|
||||
broadcast_dims[2] = width;
|
||||
broadcast_dims[3] = 1;
|
||||
Eigen::Tensor<int, 4>::Dimensions reshape_dims;
|
||||
reshape_dims[0] = batch;
|
||||
reshape_dims[1] = 1;
|
||||
reshape_dims[2] = 1;
|
||||
reshape_dims[3] = channels;
|
||||
Eigen::array<int, 4> reduced_dims_first;
|
||||
reduced_dims_first[0] = 1;
|
||||
reduced_dims_first[1] = 2;
|
||||
reduced_dims_first[2] = 0;
|
||||
reduced_dims_first[3] = 3;
|
||||
#else
|
||||
Eigen::IndexList<Eigen::type2index<0>, Eigen::type2index<1> >
|
||||
reduction_axis;
|
||||
Eigen::IndexList<Eigen::type2index<1>, int, int, Eigen::type2index<1> >
|
||||
broadcast_dims;
|
||||
broadcast_dims.set(1, height);
|
||||
broadcast_dims.set(2, width);
|
||||
Eigen::IndexList<int, Eigen::type2index<1>, Eigen::type2index<1>, int>
|
||||
reshape_dims;
|
||||
reshape_dims.set(0, batch);
|
||||
reshape_dims.set(3, channels);
|
||||
Eigen::IndexList<Eigen::type2index<1>, Eigen::type2index<2>,
|
||||
Eigen::type2index<0>, Eigen::type2index<3> >
|
||||
reduced_dims_first;
|
||||
#endif
|
||||
Eigen::Sizes<1, 1, 1, 1> scalar;
|
||||
float num_reduced_coeffs = height * width;
|
||||
output.device(d) =
|
||||
(input.shuffle(reduced_dims_first).sum(reduction_axis).eval() /
|
||||
num_reduced_coeffs)
|
||||
.reshape(reshape_dims)
|
||||
.broadcast(broadcast_dims);
|
||||
auto contrast_factor_tensor =
|
||||
contrast_factor.reshape(scalar).broadcast(scalar_broadcast);
|
||||
auto adjusted = (input - output) * contrast_factor_tensor;
|
||||
output.device(d) += adjusted;
|
||||
}
|
||||
};
|
||||
|
||||
} // namespace functor
|
||||
} // namespace tensorflow
|
||||
|
||||
#endif // TENSORFLOW_CORE_KERNELS_ADJUST_CONTRAST_OP_H_
|
||||
@@ -0,0 +1,146 @@
|
||||
/* Copyright 2017 The TensorFlow Authors. All Rights Reserved.
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
==============================================================================*/
|
||||
#ifndef TENSORFLOW_CORE_KERNELS_ADJUST_HSV_GPU_CU_H_
|
||||
#define TENSORFLOW_CORE_KERNELS_ADJUST_HSV_GPU_CU_H_
|
||||
|
||||
#if GOOGLE_CUDA
|
||||
|
||||
#define EIGEN_USE_GPU
|
||||
|
||||
#include "tensorflow/core/framework/register_types.h"
|
||||
#include "tensorflow/core/framework/types.h"
|
||||
|
||||
namespace tensorflow {
|
||||
namespace internal {
|
||||
|
||||
typedef struct RgbTuple {
|
||||
float r;
|
||||
float g;
|
||||
float b;
|
||||
} RgbTuple;
|
||||
|
||||
typedef struct HsvTuple {
|
||||
float h;
|
||||
float s;
|
||||
float v;
|
||||
} HsvTuple;
|
||||
|
||||
inline __device__ HsvTuple rgb2hsv_cuda(const float r, const float g,
|
||||
const float b) {
|
||||
HsvTuple tuple;
|
||||
const float M = fmaxf(r, fmaxf(g, b));
|
||||
const float m = fminf(r, fminf(g, b));
|
||||
const float chroma = M - m;
|
||||
float h = 0.0f, s = 0.0f;
|
||||
// hue
|
||||
if (chroma > 0.0f) {
|
||||
if (M == r) {
|
||||
const float num = (g - b) / chroma;
|
||||
const float sign = copysignf(1.0f, num);
|
||||
h = ((sign < 0.0f) * 6.0f + sign * fmodf(sign * num, 6.0f)) / 6.0f;
|
||||
} else if (M == g) {
|
||||
h = ((b - r) / chroma + 2.0f) / 6.0f;
|
||||
} else {
|
||||
h = ((r - g) / chroma + 4.0f) / 6.0f;
|
||||
}
|
||||
} else {
|
||||
h = 0.0f;
|
||||
}
|
||||
// saturation
|
||||
if (M > 0.0) {
|
||||
s = chroma / M;
|
||||
} else {
|
||||
s = 0.0f;
|
||||
}
|
||||
tuple.h = h;
|
||||
tuple.s = s;
|
||||
tuple.v = M;
|
||||
return tuple;
|
||||
}
|
||||
|
||||
inline __device__ RgbTuple hsv2rgb_cuda(const float h, const float s,
|
||||
const float v) {
|
||||
RgbTuple tuple;
|
||||
const float new_h = h * 6.0f;
|
||||
const float chroma = v * s;
|
||||
const float x = chroma * (1.0f - fabsf(fmodf(new_h, 2.0f) - 1.0f));
|
||||
const float new_m = v - chroma;
|
||||
const bool between_0_and_1 = new_h >= 0.0f && new_h < 1.0f;
|
||||
const bool between_1_and_2 = new_h >= 1.0f && new_h < 2.0f;
|
||||
const bool between_2_and_3 = new_h >= 2.0f && new_h < 3.0f;
|
||||
const bool between_3_and_4 = new_h >= 3.0f && new_h < 4.0f;
|
||||
const bool between_4_and_5 = new_h >= 4.0f && new_h < 5.0f;
|
||||
const bool between_5_and_6 = new_h >= 5.0f && new_h < 6.0f;
|
||||
tuple.r = chroma * (between_0_and_1 || between_5_and_6) +
|
||||
x * (between_1_and_2 || between_4_and_5) + new_m;
|
||||
tuple.g = chroma * (between_1_and_2 || between_2_and_3) +
|
||||
x * (between_0_and_1 || between_3_and_4) + new_m;
|
||||
tuple.b = chroma * (between_3_and_4 || between_4_and_5) +
|
||||
x * (between_2_and_3 || between_5_and_6) + new_m;
|
||||
return tuple;
|
||||
}
|
||||
|
||||
template <bool AdjustHue, bool AdjustSaturation, bool AdjustV>
|
||||
__global__ void adjust_hsv_nhwc(const int64 number_elements,
|
||||
const float* const __restrict__ input,
|
||||
float* const output,
|
||||
const float* const hue_delta,
|
||||
const float* const saturation_scale,
|
||||
const float* const value_scale) {
|
||||
// multiply by 3 since we're dealing with contiguous RGB bytes for each pixel
|
||||
// (NHWC)
|
||||
const int64 idx = (blockDim.x * blockIdx.x + threadIdx.x) * 3;
|
||||
// bounds check
|
||||
if (idx > number_elements - 1) {
|
||||
return;
|
||||
}
|
||||
if (!AdjustHue && !AdjustSaturation && !AdjustV) {
|
||||
output[idx] = input[idx];
|
||||
output[idx + 1] = input[idx + 1];
|
||||
output[idx + 2] = input[idx + 2];
|
||||
return;
|
||||
}
|
||||
const HsvTuple hsv = rgb2hsv_cuda(input[idx], input[idx + 1], input[idx + 2]);
|
||||
float new_h = hsv.h;
|
||||
float new_s = hsv.s;
|
||||
float new_v = hsv.v;
|
||||
// hue adjustment
|
||||
if (AdjustHue) {
|
||||
const float delta = *hue_delta;
|
||||
new_h = fmodf(hsv.h + delta, 1.0f);
|
||||
if (new_h < 0.0f) {
|
||||
new_h = fmodf(1.0f + new_h, 1.0f);
|
||||
}
|
||||
}
|
||||
// saturation adjustment
|
||||
if (AdjustSaturation && saturation_scale != nullptr) {
|
||||
const float scale = *saturation_scale;
|
||||
new_s = fminf(1.0f, fmaxf(0.0f, hsv.s * scale));
|
||||
}
|
||||
// value adjustment
|
||||
if (AdjustV && value_scale != nullptr) {
|
||||
const float scale = *value_scale;
|
||||
new_v = hsv.v * scale;
|
||||
}
|
||||
const RgbTuple rgb = hsv2rgb_cuda(new_h, new_s, new_v);
|
||||
output[idx] = rgb.r;
|
||||
output[idx + 1] = rgb.g;
|
||||
output[idx + 2] = rgb.b;
|
||||
}
|
||||
|
||||
} // namespace internal
|
||||
} // namespace tensorflow
|
||||
|
||||
#endif // GOOGLE_CUDA
|
||||
#endif // TENSORFLOW_CORE_KERNELS_ADJUST_HSV_GPU_CU_H_
|
||||
@@ -0,0 +1,40 @@
|
||||
/* Copyright 2016 The TensorFlow Authors. All Rights Reserved.
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
==============================================================================*/
|
||||
#ifndef TENSORFLOW_CORE_KERNELS_ADJUST_HUE_OP_H_
|
||||
#define TENSORFLOW_CORE_KERNELS_ADJUST_HUE_OP_H_
|
||||
|
||||
#if GOOGLE_CUDA
|
||||
#define EIGEN_USE_GPU
|
||||
|
||||
#include "third_party/eigen3/unsupported/Eigen/CXX11/Tensor"
|
||||
|
||||
#include "tensorflow/core/framework/types.h"
|
||||
|
||||
namespace tensorflow {
|
||||
|
||||
typedef Eigen::GpuDevice GPUDevice;
|
||||
|
||||
namespace functor {
|
||||
|
||||
struct AdjustHueGPU {
|
||||
void operator()(GPUDevice* device, const int64 number_of_elements,
|
||||
const float* const input, const float* const delta,
|
||||
float* const output);
|
||||
};
|
||||
|
||||
} // namespace functor
|
||||
} // namespace tensorflow
|
||||
|
||||
#endif // GOOGLE_CUDA
|
||||
#endif // TENSORFLOW_CORE_KERNELS_ADJUST_HUE_OP_H_
|
||||
@@ -0,0 +1,40 @@
|
||||
/* Copyright 2017 The TensorFlow Authors. All Rights Reserved.
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
==============================================================================*/
|
||||
#ifndef TENSORFLOW_CORE_KERNELS_ADJUST_SATURATION_OP_H_
|
||||
#define TENSORFLOW_CORE_KERNELS_ADJUST_SATURATION_OP_H_
|
||||
|
||||
#if GOOGLE_CUDA
|
||||
#define EIGEN_USE_GPU
|
||||
|
||||
#include "third_party/eigen3/unsupported/Eigen/CXX11/Tensor"
|
||||
|
||||
#include "tensorflow/core/framework/types.h"
|
||||
|
||||
namespace tensorflow {
|
||||
|
||||
typedef Eigen::GpuDevice GPUDevice;
|
||||
|
||||
namespace functor {
|
||||
|
||||
struct AdjustSaturationGPU {
|
||||
void operator()(GPUDevice* device, const int64 number_of_elements,
|
||||
const float* const input, const float* const scale,
|
||||
float* const output);
|
||||
};
|
||||
|
||||
} // namespace functor
|
||||
} // namespace tensorflow
|
||||
|
||||
#endif // GOOGLE_CUDA
|
||||
#endif // TENSORFLOW_CORE_KERNELS_ADJUST_SATURATION_OP_H_
|
||||
@@ -0,0 +1,226 @@
|
||||
/* Copyright 2015 The TensorFlow Authors. All Rights Reserved.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
==============================================================================*/
|
||||
|
||||
#ifndef TENSORFLOW_CORE_KERNELS_AGGREGATE_OPS_H_
|
||||
#define TENSORFLOW_CORE_KERNELS_AGGREGATE_OPS_H_
|
||||
|
||||
// Functor definitions for Aggregate ops, must be compilable by nvcc.
|
||||
|
||||
#include "third_party/eigen3/unsupported/Eigen/CXX11/Tensor"
|
||||
#include "tensorflow/core/framework/tensor_types.h"
|
||||
|
||||
namespace tensorflow {
|
||||
namespace functor {
|
||||
|
||||
template <typename Device, typename T>
|
||||
struct Add2Functor {
|
||||
void operator()(const Device& d, typename TTypes<T>::Flat out,
|
||||
typename TTypes<T>::ConstFlat in1,
|
||||
typename TTypes<T>::ConstFlat in2);
|
||||
};
|
||||
|
||||
template <typename Device, typename T>
|
||||
struct Add2EigenImpl {
|
||||
static void Compute(const Device& d, typename TTypes<T>::Flat out,
|
||||
typename TTypes<T>::ConstFlat in1,
|
||||
typename TTypes<T>::ConstFlat in2) {
|
||||
out.device(d) = in1 + in2;
|
||||
}
|
||||
};
|
||||
|
||||
template <typename Device, typename T>
|
||||
struct Add3Functor {
|
||||
void operator()(const Device& d, typename TTypes<T>::Flat out,
|
||||
typename TTypes<T>::ConstFlat in1,
|
||||
typename TTypes<T>::ConstFlat in2,
|
||||
typename TTypes<T>::ConstFlat in3);
|
||||
};
|
||||
|
||||
template <typename Device, typename T>
|
||||
struct Add3EigenImpl {
|
||||
static void Compute(const Device& d, typename TTypes<T>::Flat out,
|
||||
typename TTypes<T>::ConstFlat in1,
|
||||
typename TTypes<T>::ConstFlat in2,
|
||||
typename TTypes<T>::ConstFlat in3) {
|
||||
out.device(d) = in1 + in2 + in3;
|
||||
}
|
||||
};
|
||||
|
||||
template <typename Device, typename T>
|
||||
struct Add4Functor {
|
||||
void operator()(const Device& d, typename TTypes<T>::Flat out,
|
||||
typename TTypes<T>::ConstFlat in1,
|
||||
typename TTypes<T>::ConstFlat in2,
|
||||
typename TTypes<T>::ConstFlat in3,
|
||||
typename TTypes<T>::ConstFlat in4);
|
||||
};
|
||||
|
||||
template <typename Device, typename T>
|
||||
struct Add4EigenImpl {
|
||||
static void Compute(const Device& d, typename TTypes<T>::Flat out,
|
||||
typename TTypes<T>::ConstFlat in1,
|
||||
typename TTypes<T>::ConstFlat in2,
|
||||
typename TTypes<T>::ConstFlat in3,
|
||||
typename TTypes<T>::ConstFlat in4) {
|
||||
out.device(d) = in1 + in2 + in3 + in4;
|
||||
}
|
||||
};
|
||||
|
||||
template <typename Device, typename T>
|
||||
struct Add5Functor {
|
||||
void operator()(const Device& d, typename TTypes<T>::Flat out,
|
||||
typename TTypes<T>::ConstFlat in1,
|
||||
typename TTypes<T>::ConstFlat in2,
|
||||
typename TTypes<T>::ConstFlat in3,
|
||||
typename TTypes<T>::ConstFlat in4,
|
||||
typename TTypes<T>::ConstFlat in5);
|
||||
};
|
||||
|
||||
template <typename Device, typename T>
|
||||
struct Add5EigenImpl {
|
||||
static void Compute(const Device& d, typename TTypes<T>::Flat out,
|
||||
typename TTypes<T>::ConstFlat in1,
|
||||
typename TTypes<T>::ConstFlat in2,
|
||||
typename TTypes<T>::ConstFlat in3,
|
||||
typename TTypes<T>::ConstFlat in4,
|
||||
typename TTypes<T>::ConstFlat in5) {
|
||||
out.device(d) = in1 + in2 + in3 + in4 + in5;
|
||||
}
|
||||
};
|
||||
|
||||
template <typename Device, typename T>
|
||||
struct Add6Functor {
|
||||
void operator()(const Device& d, typename TTypes<T>::Flat out,
|
||||
typename TTypes<T>::ConstFlat in1,
|
||||
typename TTypes<T>::ConstFlat in2,
|
||||
typename TTypes<T>::ConstFlat in3,
|
||||
typename TTypes<T>::ConstFlat in4,
|
||||
typename TTypes<T>::ConstFlat in5,
|
||||
typename TTypes<T>::ConstFlat in6);
|
||||
};
|
||||
|
||||
template <typename Device, typename T>
|
||||
struct Add6EigenImpl {
|
||||
static void Compute(const Device& d, typename TTypes<T>::Flat out,
|
||||
typename TTypes<T>::ConstFlat in1,
|
||||
typename TTypes<T>::ConstFlat in2,
|
||||
typename TTypes<T>::ConstFlat in3,
|
||||
typename TTypes<T>::ConstFlat in4,
|
||||
typename TTypes<T>::ConstFlat in5,
|
||||
typename TTypes<T>::ConstFlat in6) {
|
||||
out.device(d) = in1 + in2 + in3 + in4 + in5 + in6;
|
||||
}
|
||||
};
|
||||
|
||||
template <typename Device, typename T>
|
||||
struct Add7Functor {
|
||||
void operator()(const Device& d, typename TTypes<T>::Flat out,
|
||||
typename TTypes<T>::ConstFlat in1,
|
||||
typename TTypes<T>::ConstFlat in2,
|
||||
typename TTypes<T>::ConstFlat in3,
|
||||
typename TTypes<T>::ConstFlat in4,
|
||||
typename TTypes<T>::ConstFlat in5,
|
||||
typename TTypes<T>::ConstFlat in6,
|
||||
typename TTypes<T>::ConstFlat in7);
|
||||
};
|
||||
|
||||
template <typename Device, typename T>
|
||||
struct Add7EigenImpl {
|
||||
static void Compute(const Device& d, typename TTypes<T>::Flat out,
|
||||
typename TTypes<T>::ConstFlat in1,
|
||||
typename TTypes<T>::ConstFlat in2,
|
||||
typename TTypes<T>::ConstFlat in3,
|
||||
typename TTypes<T>::ConstFlat in4,
|
||||
typename TTypes<T>::ConstFlat in5,
|
||||
typename TTypes<T>::ConstFlat in6,
|
||||
typename TTypes<T>::ConstFlat in7) {
|
||||
out.device(d) = in1 + in2 + in3 + in4 + in5 + in6 + in7;
|
||||
}
|
||||
};
|
||||
|
||||
template <typename Device, typename T>
|
||||
struct Add8Functor {
|
||||
void operator()(
|
||||
const Device& d, typename TTypes<T>::Flat out,
|
||||
typename TTypes<T>::ConstFlat in1, typename TTypes<T>::ConstFlat in2,
|
||||
typename TTypes<T>::ConstFlat in3, typename TTypes<T>::ConstFlat in4,
|
||||
typename TTypes<T>::ConstFlat in5, typename TTypes<T>::ConstFlat in6,
|
||||
typename TTypes<T>::ConstFlat in7, typename TTypes<T>::ConstFlat in8);
|
||||
};
|
||||
|
||||
template <typename Device, typename T>
|
||||
struct Add8EigenImpl {
|
||||
static void Compute(
|
||||
const Device& d, typename TTypes<T>::Flat out,
|
||||
typename TTypes<T>::ConstFlat in1, typename TTypes<T>::ConstFlat in2,
|
||||
typename TTypes<T>::ConstFlat in3, typename TTypes<T>::ConstFlat in4,
|
||||
typename TTypes<T>::ConstFlat in5, typename TTypes<T>::ConstFlat in6,
|
||||
typename TTypes<T>::ConstFlat in7, typename TTypes<T>::ConstFlat in8) {
|
||||
out.device(d) = in1 + in2 + in3 + in4 + in5 + in6 + in7 + in8;
|
||||
}
|
||||
};
|
||||
|
||||
// Add8p is like Add8 except the underlying implementation should +=
|
||||
// rather than assign to the output.
|
||||
template <typename Device, typename T>
|
||||
struct Add8pFunctor {
|
||||
void operator()(
|
||||
const Device& d, typename TTypes<T>::Flat out,
|
||||
typename TTypes<T>::ConstFlat in1, typename TTypes<T>::ConstFlat in2,
|
||||
typename TTypes<T>::ConstFlat in3, typename TTypes<T>::ConstFlat in4,
|
||||
typename TTypes<T>::ConstFlat in5, typename TTypes<T>::ConstFlat in6,
|
||||
typename TTypes<T>::ConstFlat in7, typename TTypes<T>::ConstFlat in8);
|
||||
};
|
||||
|
||||
template <typename Device, typename T>
|
||||
struct Add8pEigenImpl {
|
||||
static void Compute(
|
||||
const Device& d, typename TTypes<T>::Flat out,
|
||||
typename TTypes<T>::ConstFlat in1, typename TTypes<T>::ConstFlat in2,
|
||||
typename TTypes<T>::ConstFlat in3, typename TTypes<T>::ConstFlat in4,
|
||||
typename TTypes<T>::ConstFlat in5, typename TTypes<T>::ConstFlat in6,
|
||||
typename TTypes<T>::ConstFlat in7, typename TTypes<T>::ConstFlat in8) {
|
||||
out.device(d) += in1 + in2 + in3 + in4 + in5 + in6 + in7 + in8;
|
||||
}
|
||||
};
|
||||
|
||||
template <typename Device, typename T>
|
||||
struct Add9Functor {
|
||||
void operator()(
|
||||
const Device& d, typename TTypes<T>::Flat out,
|
||||
typename TTypes<T>::ConstFlat in1, typename TTypes<T>::ConstFlat in2,
|
||||
typename TTypes<T>::ConstFlat in3, typename TTypes<T>::ConstFlat in4,
|
||||
typename TTypes<T>::ConstFlat in5, typename TTypes<T>::ConstFlat in6,
|
||||
typename TTypes<T>::ConstFlat in7, typename TTypes<T>::ConstFlat in8,
|
||||
typename TTypes<T>::ConstFlat in9);
|
||||
};
|
||||
|
||||
template <typename Device, typename T>
|
||||
struct Add9EigenImpl {
|
||||
static void Compute(
|
||||
const Device& d, typename TTypes<T>::Flat out,
|
||||
typename TTypes<T>::ConstFlat in1, typename TTypes<T>::ConstFlat in2,
|
||||
typename TTypes<T>::ConstFlat in3, typename TTypes<T>::ConstFlat in4,
|
||||
typename TTypes<T>::ConstFlat in5, typename TTypes<T>::ConstFlat in6,
|
||||
typename TTypes<T>::ConstFlat in7, typename TTypes<T>::ConstFlat in8,
|
||||
typename TTypes<T>::ConstFlat in9) {
|
||||
out.device(d) = in1 + in2 + in3 + in4 + in5 + in6 + in7 + in8 + in9;
|
||||
}
|
||||
};
|
||||
|
||||
} // namespace functor
|
||||
} // namespace tensorflow
|
||||
|
||||
#endif // TENSORFLOW_CORE_KERNELS_AGGREGATE_OPS_H_
|
||||
@@ -0,0 +1,253 @@
|
||||
/* Copyright 2015 The TensorFlow Authors. All Rights Reserved.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
==============================================================================*/
|
||||
|
||||
#ifndef TENSORFLOW_CORE_KERNELS_AGGREGATE_OPS_CPU_H_
|
||||
#define TENSORFLOW_CORE_KERNELS_AGGREGATE_OPS_CPU_H_
|
||||
|
||||
#include "third_party/eigen3/unsupported/Eigen/CXX11/Tensor"
|
||||
#include "tensorflow/core/framework/tensor_types.h"
|
||||
|
||||
#include "tensorflow/core/kernels/aggregate_ops.h"
|
||||
|
||||
typedef Eigen::ThreadPoolDevice CPUDevice;
|
||||
|
||||
#ifdef TENSORFLOW_USE_SYCL
|
||||
typedef Eigen::SyclDevice SYCLDevice;
|
||||
#endif // TENSORFLOW_USE_SYCL
|
||||
|
||||
namespace tensorflow {
|
||||
|
||||
// Partial specializations for a CPUDevice, that uses the Eigen implementation
|
||||
// from AddNEigenImpl.
|
||||
namespace functor {
|
||||
template <typename T>
|
||||
struct Add2Functor<CPUDevice, T> {
|
||||
void operator()(const CPUDevice& d, typename TTypes<T>::Flat out,
|
||||
typename TTypes<T>::ConstFlat in1,
|
||||
typename TTypes<T>::ConstFlat in2) {
|
||||
Add2EigenImpl<CPUDevice, T>::Compute(d, out, in1, in2);
|
||||
}
|
||||
};
|
||||
template <typename T>
|
||||
struct Add3Functor<CPUDevice, T> {
|
||||
void operator()(const CPUDevice& d, typename TTypes<T>::Flat out,
|
||||
typename TTypes<T>::ConstFlat in1,
|
||||
typename TTypes<T>::ConstFlat in2,
|
||||
typename TTypes<T>::ConstFlat in3) {
|
||||
Add3EigenImpl<CPUDevice, T>::Compute(d, out, in1, in2, in3);
|
||||
}
|
||||
};
|
||||
template <typename T>
|
||||
struct Add4Functor<CPUDevice, T> {
|
||||
void operator()(const CPUDevice& d, typename TTypes<T>::Flat out,
|
||||
typename TTypes<T>::ConstFlat in1,
|
||||
typename TTypes<T>::ConstFlat in2,
|
||||
typename TTypes<T>::ConstFlat in3,
|
||||
typename TTypes<T>::ConstFlat in4) {
|
||||
Add4EigenImpl<CPUDevice, T>::Compute(d, out, in1, in2, in3, in4);
|
||||
}
|
||||
};
|
||||
template <typename T>
|
||||
struct Add5Functor<CPUDevice, T> {
|
||||
void operator()(const CPUDevice& d, typename TTypes<T>::Flat out,
|
||||
typename TTypes<T>::ConstFlat in1,
|
||||
typename TTypes<T>::ConstFlat in2,
|
||||
typename TTypes<T>::ConstFlat in3,
|
||||
typename TTypes<T>::ConstFlat in4,
|
||||
typename TTypes<T>::ConstFlat in5) {
|
||||
Add5EigenImpl<CPUDevice, T>::Compute(d, out, in1, in2, in3, in4, in5);
|
||||
}
|
||||
};
|
||||
template <typename T>
|
||||
struct Add6Functor<CPUDevice, T> {
|
||||
void operator()(const CPUDevice& d, typename TTypes<T>::Flat out,
|
||||
typename TTypes<T>::ConstFlat in1,
|
||||
typename TTypes<T>::ConstFlat in2,
|
||||
typename TTypes<T>::ConstFlat in3,
|
||||
typename TTypes<T>::ConstFlat in4,
|
||||
typename TTypes<T>::ConstFlat in5,
|
||||
typename TTypes<T>::ConstFlat in6) {
|
||||
Add6EigenImpl<CPUDevice, T>::Compute(d, out, in1, in2, in3, in4, in5, in6);
|
||||
}
|
||||
};
|
||||
template <typename T>
|
||||
struct Add7Functor<CPUDevice, T> {
|
||||
void operator()(const CPUDevice& d, typename TTypes<T>::Flat out,
|
||||
typename TTypes<T>::ConstFlat in1,
|
||||
typename TTypes<T>::ConstFlat in2,
|
||||
typename TTypes<T>::ConstFlat in3,
|
||||
typename TTypes<T>::ConstFlat in4,
|
||||
typename TTypes<T>::ConstFlat in5,
|
||||
typename TTypes<T>::ConstFlat in6,
|
||||
typename TTypes<T>::ConstFlat in7) {
|
||||
Add7EigenImpl<CPUDevice, T>::Compute(d, out, in1, in2, in3, in4, in5, in6,
|
||||
in7);
|
||||
}
|
||||
};
|
||||
|
||||
template <typename T>
|
||||
struct Add8Functor<CPUDevice, T> {
|
||||
void operator()(
|
||||
const CPUDevice& d, typename TTypes<T>::Flat out,
|
||||
typename TTypes<T>::ConstFlat in1, typename TTypes<T>::ConstFlat in2,
|
||||
typename TTypes<T>::ConstFlat in3, typename TTypes<T>::ConstFlat in4,
|
||||
typename TTypes<T>::ConstFlat in5, typename TTypes<T>::ConstFlat in6,
|
||||
typename TTypes<T>::ConstFlat in7, typename TTypes<T>::ConstFlat in8) {
|
||||
Add8EigenImpl<CPUDevice, T>::Compute(d, out, in1, in2, in3, in4, in5, in6,
|
||||
in7, in8);
|
||||
}
|
||||
};
|
||||
|
||||
template <typename T>
|
||||
struct Add8pFunctor<CPUDevice, T> {
|
||||
void operator()(
|
||||
const CPUDevice& d, typename TTypes<T>::Flat out,
|
||||
typename TTypes<T>::ConstFlat in1, typename TTypes<T>::ConstFlat in2,
|
||||
typename TTypes<T>::ConstFlat in3, typename TTypes<T>::ConstFlat in4,
|
||||
typename TTypes<T>::ConstFlat in5, typename TTypes<T>::ConstFlat in6,
|
||||
typename TTypes<T>::ConstFlat in7, typename TTypes<T>::ConstFlat in8) {
|
||||
Add8pEigenImpl<CPUDevice, T>::Compute(d, out, in1, in2, in3, in4, in5, in6,
|
||||
in7, in8);
|
||||
}
|
||||
};
|
||||
|
||||
template <typename T>
|
||||
struct Add9Functor<CPUDevice, T> {
|
||||
void operator()(
|
||||
const CPUDevice& d, typename TTypes<T>::Flat out,
|
||||
typename TTypes<T>::ConstFlat in1, typename TTypes<T>::ConstFlat in2,
|
||||
typename TTypes<T>::ConstFlat in3, typename TTypes<T>::ConstFlat in4,
|
||||
typename TTypes<T>::ConstFlat in5, typename TTypes<T>::ConstFlat in6,
|
||||
typename TTypes<T>::ConstFlat in7, typename TTypes<T>::ConstFlat in8,
|
||||
typename TTypes<T>::ConstFlat in9) {
|
||||
Add9EigenImpl<CPUDevice, T>::Compute(d, out, in1, in2, in3, in4, in5, in6,
|
||||
in7, in8, in9);
|
||||
}
|
||||
};
|
||||
|
||||
#ifdef TENSORFLOW_USE_SYCL
|
||||
// Partial specializations for a SYCLDevice, that uses the Eigen implementation
|
||||
// from AddNEigenImpl.
|
||||
template <typename T>
|
||||
struct Add2Functor<SYCLDevice, T> {
|
||||
void operator()(const SYCLDevice& d, typename TTypes<T>::Flat out,
|
||||
typename TTypes<T>::ConstFlat in1,
|
||||
typename TTypes<T>::ConstFlat in2) {
|
||||
Add2EigenImpl<SYCLDevice, T>::Compute(d, out, in1, in2);
|
||||
}
|
||||
};
|
||||
template <typename T>
|
||||
struct Add3Functor<SYCLDevice, T> {
|
||||
void operator()(const SYCLDevice& d, typename TTypes<T>::Flat out,
|
||||
typename TTypes<T>::ConstFlat in1,
|
||||
typename TTypes<T>::ConstFlat in2,
|
||||
typename TTypes<T>::ConstFlat in3) {
|
||||
Add3EigenImpl<SYCLDevice, T>::Compute(d, out, in1, in2, in3);
|
||||
}
|
||||
};
|
||||
template <typename T>
|
||||
struct Add4Functor<SYCLDevice, T> {
|
||||
void operator()(const SYCLDevice& d, typename TTypes<T>::Flat out,
|
||||
typename TTypes<T>::ConstFlat in1,
|
||||
typename TTypes<T>::ConstFlat in2,
|
||||
typename TTypes<T>::ConstFlat in3,
|
||||
typename TTypes<T>::ConstFlat in4) {
|
||||
Add4EigenImpl<SYCLDevice, T>::Compute(d, out, in1, in2, in3, in4);
|
||||
}
|
||||
};
|
||||
template <typename T>
|
||||
struct Add5Functor<SYCLDevice, T> {
|
||||
void operator()(const SYCLDevice& d, typename TTypes<T>::Flat out,
|
||||
typename TTypes<T>::ConstFlat in1,
|
||||
typename TTypes<T>::ConstFlat in2,
|
||||
typename TTypes<T>::ConstFlat in3,
|
||||
typename TTypes<T>::ConstFlat in4,
|
||||
typename TTypes<T>::ConstFlat in5) {
|
||||
Add5EigenImpl<SYCLDevice, T>::Compute(d, out, in1, in2, in3, in4, in5);
|
||||
}
|
||||
};
|
||||
template <typename T>
|
||||
struct Add6Functor<SYCLDevice, T> {
|
||||
void operator()(const SYCLDevice& d, typename TTypes<T>::Flat out,
|
||||
typename TTypes<T>::ConstFlat in1,
|
||||
typename TTypes<T>::ConstFlat in2,
|
||||
typename TTypes<T>::ConstFlat in3,
|
||||
typename TTypes<T>::ConstFlat in4,
|
||||
typename TTypes<T>::ConstFlat in5,
|
||||
typename TTypes<T>::ConstFlat in6) {
|
||||
Add6EigenImpl<SYCLDevice, T>::Compute(d, out, in1, in2, in3, in4, in5, in6);
|
||||
}
|
||||
};
|
||||
template <typename T>
|
||||
struct Add7Functor<SYCLDevice, T> {
|
||||
void operator()(const SYCLDevice& d, typename TTypes<T>::Flat out,
|
||||
typename TTypes<T>::ConstFlat in1,
|
||||
typename TTypes<T>::ConstFlat in2,
|
||||
typename TTypes<T>::ConstFlat in3,
|
||||
typename TTypes<T>::ConstFlat in4,
|
||||
typename TTypes<T>::ConstFlat in5,
|
||||
typename TTypes<T>::ConstFlat in6,
|
||||
typename TTypes<T>::ConstFlat in7) {
|
||||
Add7EigenImpl<SYCLDevice, T>::Compute(d, out, in1, in2, in3, in4, in5, in6,
|
||||
in7);
|
||||
}
|
||||
};
|
||||
|
||||
template <typename T>
|
||||
struct Add8Functor<SYCLDevice, T> {
|
||||
void operator()(
|
||||
const SYCLDevice& d, typename TTypes<T>::Flat out,
|
||||
typename TTypes<T>::ConstFlat in1, typename TTypes<T>::ConstFlat in2,
|
||||
typename TTypes<T>::ConstFlat in3, typename TTypes<T>::ConstFlat in4,
|
||||
typename TTypes<T>::ConstFlat in5, typename TTypes<T>::ConstFlat in6,
|
||||
typename TTypes<T>::ConstFlat in7, typename TTypes<T>::ConstFlat in8) {
|
||||
Add8EigenImpl<SYCLDevice, T>::Compute(d, out, in1, in2, in3, in4, in5, in6,
|
||||
in7, in8);
|
||||
}
|
||||
};
|
||||
|
||||
template <typename T>
|
||||
struct Add8pFunctor<SYCLDevice, T> {
|
||||
void operator()(
|
||||
const SYCLDevice& d, typename TTypes<T>::Flat out,
|
||||
typename TTypes<T>::ConstFlat in1, typename TTypes<T>::ConstFlat in2,
|
||||
typename TTypes<T>::ConstFlat in3, typename TTypes<T>::ConstFlat in4,
|
||||
typename TTypes<T>::ConstFlat in5, typename TTypes<T>::ConstFlat in6,
|
||||
typename TTypes<T>::ConstFlat in7, typename TTypes<T>::ConstFlat in8) {
|
||||
Add8pEigenImpl<SYCLDevice, T>::Compute(d, out, in1, in2, in3, in4, in5, in6,
|
||||
in7, in8);
|
||||
}
|
||||
};
|
||||
|
||||
template <typename T>
|
||||
struct Add9Functor<SYCLDevice, T> {
|
||||
void operator()(
|
||||
const SYCLDevice& d, typename TTypes<T>::Flat out,
|
||||
typename TTypes<T>::ConstFlat in1, typename TTypes<T>::ConstFlat in2,
|
||||
typename TTypes<T>::ConstFlat in3, typename TTypes<T>::ConstFlat in4,
|
||||
typename TTypes<T>::ConstFlat in5, typename TTypes<T>::ConstFlat in6,
|
||||
typename TTypes<T>::ConstFlat in7, typename TTypes<T>::ConstFlat in8,
|
||||
typename TTypes<T>::ConstFlat in9) {
|
||||
Add9EigenImpl<SYCLDevice, T>::Compute(d, out, in1, in2, in3, in4, in5, in6,
|
||||
in7, in8, in9);
|
||||
}
|
||||
};
|
||||
#endif // TENSORFLOW_USE_SYCL
|
||||
|
||||
} // namespace functor
|
||||
|
||||
} // namespace tensorflow
|
||||
|
||||
#endif // TENSORFLOW_CORE_KERNELS_AGGREGATE_OPS_CPU_H_
|
||||
@@ -0,0 +1,68 @@
|
||||
/* Copyright 2015 The TensorFlow Authors. All Rights Reserved.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
==============================================================================*/
|
||||
|
||||
#ifndef TENSORFLOW_CORE_KERNELS_ARGMAX_OP_H_
|
||||
#define TENSORFLOW_CORE_KERNELS_ARGMAX_OP_H_
|
||||
// Generator definition for ArgMaxOp, must be compilable by nvcc.
|
||||
|
||||
#include "third_party/eigen3/unsupported/Eigen/CXX11/Tensor"
|
||||
#include "tensorflow/core/framework/tensor_types.h"
|
||||
#include "tensorflow/core/platform/types.h"
|
||||
|
||||
namespace tensorflow {
|
||||
|
||||
namespace functor {
|
||||
|
||||
template <typename Device, typename T, typename Tout>
|
||||
struct ArgMax {
|
||||
#define DECLARE_COMPUTE_SPEC(Dims) \
|
||||
EIGEN_ALWAYS_INLINE static void Reduce##Dims( \
|
||||
const Device& d, typename TTypes<T, Dims>::ConstTensor input, \
|
||||
const int32 dimension, typename TTypes<Tout, Dims - 1>::Tensor output) { \
|
||||
output.device(d) = input.argmax(dimension).template cast<Tout>(); \
|
||||
}
|
||||
|
||||
DECLARE_COMPUTE_SPEC(1);
|
||||
DECLARE_COMPUTE_SPEC(2);
|
||||
DECLARE_COMPUTE_SPEC(3);
|
||||
DECLARE_COMPUTE_SPEC(4);
|
||||
DECLARE_COMPUTE_SPEC(5);
|
||||
|
||||
#undef DECLARE_COMPUTE_SPEC
|
||||
};
|
||||
|
||||
template <typename Device, typename T, typename Tout>
|
||||
struct ArgMin {
|
||||
#define DECLARE_COMPUTE_SPEC(Dims) \
|
||||
EIGEN_ALWAYS_INLINE static void Reduce##Dims( \
|
||||
const Device& d, typename TTypes<T, Dims>::ConstTensor input, \
|
||||
const int32 dimension, typename TTypes<Tout, Dims - 1>::Tensor output) { \
|
||||
output.device(d) = input.argmin(dimension).template cast<Tout>(); \
|
||||
}
|
||||
|
||||
DECLARE_COMPUTE_SPEC(1);
|
||||
DECLARE_COMPUTE_SPEC(2);
|
||||
DECLARE_COMPUTE_SPEC(3);
|
||||
DECLARE_COMPUTE_SPEC(4);
|
||||
DECLARE_COMPUTE_SPEC(5);
|
||||
|
||||
#undef DECLARE_COMPUTE_SPEC
|
||||
};
|
||||
|
||||
} // namespace functor
|
||||
|
||||
} // namespace tensorflow
|
||||
|
||||
#endif // TENSORFLOW_CORE_KERNELS_ARGMAX_OP_H_
|
||||
@@ -0,0 +1,146 @@
|
||||
/* Copyright 2015 The TensorFlow Authors. All Rights Reserved.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
==============================================================================*/
|
||||
|
||||
#ifndef TENSORFLOW_CORE_KERNELS_ASSIGN_OP_H_
|
||||
#define TENSORFLOW_CORE_KERNELS_ASSIGN_OP_H_
|
||||
|
||||
#define EIGEN_USE_THREADS
|
||||
|
||||
#include "third_party/eigen3/unsupported/Eigen/CXX11/Tensor"
|
||||
#include "tensorflow/core/framework/op_kernel.h"
|
||||
#include "tensorflow/core/framework/tensor_types.h"
|
||||
|
||||
namespace tensorflow {
|
||||
|
||||
// TODO(jeff): Get rid of use_exclusive_lock_ option
|
||||
|
||||
// Computes *input[0] = input[1]
|
||||
class AssignOp : public OpKernel {
|
||||
public:
|
||||
explicit AssignOp(OpKernelConstruction* context) : OpKernel(context) {
|
||||
OP_REQUIRES_OK(context,
|
||||
context->GetAttr("use_locking", &use_exclusive_lock_));
|
||||
OP_REQUIRES_OK(context,
|
||||
context->GetAttr("validate_shape", &validate_shape_));
|
||||
OP_REQUIRES(context, IsRefType(context->input_type(0)),
|
||||
errors::InvalidArgument("lhs input needs to be a ref type"));
|
||||
if (!context
|
||||
->GetAttr("_grappler_relax_allocator_constraints",
|
||||
&relax_constraints_)
|
||||
.ok()) {
|
||||
relax_constraints_ = false;
|
||||
}
|
||||
}
|
||||
|
||||
void Compute(OpKernelContext* context) override {
|
||||
const Tensor& rhs = context->input(1);
|
||||
|
||||
// We always return the input ref.
|
||||
context->forward_ref_input_to_ref_output(0, 0);
|
||||
|
||||
// We can't always know how this value will be used downstream, so make
|
||||
// conservative assumptions in specifying constraints on the memory
|
||||
// allocation attributes, unless the Grappler graph analysis determined that
|
||||
// it was safe not to.
|
||||
AllocatorAttributes attr;
|
||||
if (!relax_constraints_) {
|
||||
attr.set_gpu_compatible(true);
|
||||
attr.set_nic_compatible(true);
|
||||
}
|
||||
|
||||
{
|
||||
mutex_lock l(*context->input_ref_mutex(0));
|
||||
const Tensor& old_lhs = context->mutable_input(0, /* lock_held */ true);
|
||||
const bool same_shape = old_lhs.shape().IsSameSize(rhs.shape());
|
||||
if (validate_shape_) {
|
||||
OP_REQUIRES(context, same_shape,
|
||||
errors::InvalidArgument(
|
||||
"Assign requires shapes of both tensors to match. "
|
||||
"lhs shape= ",
|
||||
old_lhs.shape().DebugString(),
|
||||
" rhs shape= ", rhs.shape().DebugString()));
|
||||
}
|
||||
|
||||
// In the code below we try to minimize the amount of memory allocation
|
||||
// and copying by trying the following two shortcuts:
|
||||
// 1. If the lhs is initialized and has the same number of elements as
|
||||
// the rhs we can avoid a memory allocation.
|
||||
// 2. If we can reuse the rhs buffer we avoid both a memory allocation
|
||||
// and copying.
|
||||
|
||||
// 1. Try to copy into an existing buffer.
|
||||
if (old_lhs.IsInitialized() &&
|
||||
old_lhs.shape().num_elements() == rhs.shape().num_elements()) {
|
||||
// The existing lhs tensor has already been initialized and the right
|
||||
// hand side can fit in the underlying buffer.
|
||||
Tensor reshaped_old_lhs;
|
||||
if (same_shape) {
|
||||
reshaped_old_lhs = old_lhs;
|
||||
} else {
|
||||
CHECK(reshaped_old_lhs.CopyFrom(old_lhs, rhs.shape()));
|
||||
context->replace_ref_input(0, reshaped_old_lhs,
|
||||
/* lock_held */ true);
|
||||
}
|
||||
if (use_exclusive_lock_) {
|
||||
Copy(context, &reshaped_old_lhs, rhs);
|
||||
return;
|
||||
}
|
||||
} else {
|
||||
// 2. Try to reuse the rhs.
|
||||
std::unique_ptr<Tensor> input_alias = context->forward_input(
|
||||
1, OpKernelContext::Params::kNoReservation /*output_index*/,
|
||||
rhs.dtype(), rhs.shape(), DEVICE_MEMORY, attr);
|
||||
if (input_alias != nullptr) {
|
||||
// Update the ref to point to the new buffer.
|
||||
context->replace_ref_input(0, *input_alias, /* lock_held */ true);
|
||||
return;
|
||||
}
|
||||
|
||||
// Otherwise, create a new persistent tensor whose shape matches the
|
||||
// right hand side, hand off to lhs and copy the rhs into it.
|
||||
PersistentTensor copy;
|
||||
Tensor* copyTensor = nullptr;
|
||||
OP_REQUIRES_OK(
|
||||
context, context->allocate_persistent(old_lhs.dtype(), rhs.shape(),
|
||||
©, ©Tensor, attr));
|
||||
// We track memory of variables in variable ops instead of in this
|
||||
// assign op.
|
||||
context->clear_recorded_memory();
|
||||
context->replace_ref_input(0, *copyTensor, /* lock_held */ true);
|
||||
if (use_exclusive_lock_) {
|
||||
Copy(context, copyTensor, rhs);
|
||||
return;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// The tensor has already been initialized and the right hand side
|
||||
// matches the left hand side's shape. We have been told to do the
|
||||
// copy outside the lock.
|
||||
Tensor old_unlocked_lhs = context->mutable_input(0, /* lock_held */ false);
|
||||
Copy(context, &old_unlocked_lhs, rhs);
|
||||
}
|
||||
|
||||
virtual void Copy(OpKernelContext* context, Tensor* lhs,
|
||||
const Tensor& rhs) = 0;
|
||||
|
||||
bool use_exclusive_lock_;
|
||||
bool validate_shape_;
|
||||
bool relax_constraints_;
|
||||
};
|
||||
|
||||
} // end namespace tensorflow
|
||||
|
||||
#endif // TENSORFLOW_CORE_KERNELS_ASSIGN_OP_H_
|
||||
@@ -0,0 +1,79 @@
|
||||
/* Copyright 2015 The TensorFlow Authors. All Rights Reserved.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
==============================================================================*/
|
||||
|
||||
#ifndef TENSORFLOW_CORE_KERNELS_AVGPOOLING_OP_H_
|
||||
#define TENSORFLOW_CORE_KERNELS_AVGPOOLING_OP_H_
|
||||
// Functor definition for AvgPoolingOp, must be compilable by nvcc.
|
||||
|
||||
#include "tensorflow/core/framework/tensor_types.h"
|
||||
#include "tensorflow/core/kernels/eigen_pooling.h"
|
||||
#include "tensorflow/core/platform/types.h"
|
||||
|
||||
namespace tensorflow {
|
||||
namespace functor {
|
||||
|
||||
template <typename Device, typename T>
|
||||
struct SpatialAvgPooling {
|
||||
void operator()(const Device& d, typename TTypes<T, 4>::Tensor output,
|
||||
typename TTypes<T, 4>::ConstTensor input, int window_rows,
|
||||
int window_cols, int row_stride, int col_stride,
|
||||
const Eigen::PaddingType& padding) {
|
||||
if (Eigen::internal::is_same<Device, Eigen::GpuDevice>::value) {
|
||||
// Use 32bit indexing to speed up the computations
|
||||
To32Bit(output).swap_layout().device(d) = Eigen::SpatialAvgPooling(
|
||||
To32Bit(input).swap_layout(), window_cols, window_rows, col_stride,
|
||||
row_stride, padding);
|
||||
} else {
|
||||
// Because we swap the layout, we swap the row/cols as well
|
||||
output.swap_layout().device(d) = Eigen::SpatialAvgPooling(
|
||||
input.swap_layout(), window_cols, window_rows, col_stride, row_stride,
|
||||
padding);
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
} // namespace functor
|
||||
|
||||
typedef Eigen::GpuDevice GPUDevice;
|
||||
|
||||
// Launch a custom GPU kernels from Yanqing for the avgpooling backward
|
||||
// operation that works NHWC data formats. Arguments:
|
||||
// top_diff: backprop to the output of the pooling layer
|
||||
// num: number of input batches
|
||||
// height: input height
|
||||
// width: input width
|
||||
// channels: number of input channels
|
||||
// pooled_height: the height of the output to the pooling layer
|
||||
// pooled_width: the width of the output to the pooling layer
|
||||
// kernel_h: the height of the pooling kernel
|
||||
// kernel_w: the width of the pooling kernel
|
||||
// stride_h: the height of the vertical stride
|
||||
// stride_w: the width of the horizontal stride
|
||||
// pad_t: padding size to the top side
|
||||
// pad_l: padding size to the left side
|
||||
// bottom_diff: backprop to the input of the pooling layer.
|
||||
template <typename T>
|
||||
bool RunAvePoolBackwardNHWC(const T* const top_diff, const int num,
|
||||
const int height, const int width,
|
||||
const int channels, const int pooled_height,
|
||||
const int pooled_width, const int kernel_h,
|
||||
const int kernel_w, const int stride_h,
|
||||
const int stride_w, const int pad_t,
|
||||
const int pad_l, T* const bottom_diff,
|
||||
const GPUDevice& d);
|
||||
|
||||
} // namespace tensorflow
|
||||
|
||||
#endif // TENSORFLOW_CORE_KERNELS_AVGPOOLING_OP_H_
|
||||
@@ -0,0 +1,620 @@
|
||||
/* Copyright 2015 The TensorFlow Authors. All Rights Reserved.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
==============================================================================*/
|
||||
|
||||
// See docs in ../ops/math_ops.cc.
|
||||
|
||||
#ifndef TENSORFLOW_CORE_KERNELS_BATCH_MATMUL_OP_IMPL_H_
|
||||
#define TENSORFLOW_CORE_KERNELS_BATCH_MATMUL_OP_IMPL_H_
|
||||
|
||||
#define EIGEN_USE_THREADS
|
||||
|
||||
#include <vector>
|
||||
#include "third_party/eigen3/unsupported/Eigen/CXX11/Tensor"
|
||||
#include "tensorflow/core/framework/op.h"
|
||||
#include "tensorflow/core/framework/op_kernel.h"
|
||||
#include "tensorflow/core/framework/register_types.h"
|
||||
#include "tensorflow/core/framework/tensor.h"
|
||||
#include "tensorflow/core/framework/tensor_shape.h"
|
||||
#include "tensorflow/core/framework/type_traits.h"
|
||||
#include "tensorflow/core/framework/types.h"
|
||||
#include "tensorflow/core/kernels/fill_functor.h"
|
||||
#include "tensorflow/core/platform/logging.h"
|
||||
#include "tensorflow/core/platform/types.h"
|
||||
#include "tensorflow/core/util/work_sharder.h"
|
||||
|
||||
#if GOOGLE_CUDA
|
||||
#include "tensorflow/core/platform/stream_executor.h"
|
||||
#endif // GOOGLE_CUDA
|
||||
|
||||
namespace tensorflow {
|
||||
|
||||
typedef Eigen::ThreadPoolDevice CPUDevice;
|
||||
typedef Eigen::GpuDevice GPUDevice;
|
||||
#ifdef TENSORFLOW_USE_SYCL
|
||||
typedef Eigen::SyclDevice SYCLDevice;
|
||||
#endif // TENSORFLOW_USE_SYCL
|
||||
|
||||
namespace {
|
||||
|
||||
Eigen::IndexPair<Eigen::DenseIndex> ContractionDims(bool adj_x, bool adj_y) {
|
||||
if (!adj_x) {
|
||||
if (!adj_y) {
|
||||
return Eigen::IndexPair<Eigen::DenseIndex>(1, 0);
|
||||
} else {
|
||||
return Eigen::IndexPair<Eigen::DenseIndex>(1, 1);
|
||||
}
|
||||
} else {
|
||||
if (!adj_y) {
|
||||
return Eigen::IndexPair<Eigen::DenseIndex>(0, 0);
|
||||
} else {
|
||||
return Eigen::IndexPair<Eigen::DenseIndex>(0, 1);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Parallel batch matmul kernel based on the multi-threaded tensor contraction
|
||||
// in Eigen.
|
||||
template <typename Scalar, bool IsComplex = true>
|
||||
struct ParallelMatMulKernel {
|
||||
static void Conjugate(const OpKernelContext* context, Tensor* out) {
|
||||
const Eigen::ThreadPoolDevice d = context->eigen_cpu_device();
|
||||
auto z = out->tensor<Scalar, 3>();
|
||||
z.device(d) = z.conjugate();
|
||||
}
|
||||
|
||||
static void Run(const OpKernelContext* context, const Tensor& in_x,
|
||||
const Tensor in_y, bool adj_x, bool adj_y, Tensor* out,
|
||||
int start, int limit) {
|
||||
static_assert(IsComplex, "Complex type expected.");
|
||||
auto Tx = in_x.tensor<Scalar, 3>();
|
||||
auto Ty = in_y.tensor<Scalar, 3>();
|
||||
auto Tz = out->tensor<Scalar, 3>();
|
||||
// We use the identities
|
||||
// conj(a) * conj(b) = conj(a * b)
|
||||
// conj(a) * b = conj(a * conj(b))
|
||||
// to halve the number of cases. The final conjugation of the result is
|
||||
// done at the end of LaunchBatchMatMul<CPUDevice, Scalar>::Launch().
|
||||
Eigen::array<Eigen::IndexPair<Eigen::DenseIndex>, 1> contract_pairs;
|
||||
contract_pairs[0] = ContractionDims(adj_x, adj_y);
|
||||
const Eigen::ThreadPoolDevice d = context->eigen_cpu_device();
|
||||
for (int i = start; i < limit; ++i) {
|
||||
auto x = Tx.template chip<0>(i);
|
||||
auto z = Tz.template chip<0>(i);
|
||||
if (adj_x != adj_y) {
|
||||
auto y = Ty.template chip<0>(i).conjugate();
|
||||
z.device(d) = x.contract(y, contract_pairs);
|
||||
} else {
|
||||
auto y = Ty.template chip<0>(i);
|
||||
z.device(d) = x.contract(y, contract_pairs);
|
||||
}
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
// The Eigen contraction kernel used here is very large and slow to compile,
|
||||
// so we partially specialize ParallelMatMulKernel for real types to avoid all
|
||||
// but one of the instantiations.
|
||||
template <typename Scalar>
|
||||
struct ParallelMatMulKernel<Scalar, false> {
|
||||
static void Conjugate(const OpKernelContext* context, Tensor* out) {}
|
||||
|
||||
static void Run(const OpKernelContext* context, const Tensor& in_x,
|
||||
const Tensor& in_y, bool adj_x, bool adj_y, Tensor* out,
|
||||
int start, int limit) {
|
||||
auto Tx = in_x.tensor<Scalar, 3>();
|
||||
auto Ty = in_y.tensor<Scalar, 3>();
|
||||
auto Tz = out->tensor<Scalar, 3>();
|
||||
Eigen::array<Eigen::IndexPair<Eigen::DenseIndex>, 1> contract_pairs;
|
||||
contract_pairs[0] = ContractionDims(adj_x, adj_y);
|
||||
const Eigen::ThreadPoolDevice d = context->eigen_cpu_device();
|
||||
for (int i = start; i < limit; ++i) {
|
||||
auto x = Tx.template chip<0>(i);
|
||||
auto y = Ty.template chip<0>(i);
|
||||
auto z = Tz.template chip<0>(i);
|
||||
z.device(d) = x.contract(y, contract_pairs);
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
// TODO(rmlarsen): Get rid of this when we have upstreamed improvements
|
||||
// for matrix*vector and vector*matrix to Eigen's general matrix product.
|
||||
template <typename Tx, typename Ty, typename Tz>
|
||||
static void Multiply(bool adj_x, bool adj_y, Tx x, Ty y, Tz z) {
|
||||
if (!adj_x) {
|
||||
if (!adj_y) {
|
||||
z.noalias() = x * y;
|
||||
} else {
|
||||
z.noalias() = x * y.adjoint();
|
||||
}
|
||||
} else {
|
||||
if (!adj_y) {
|
||||
z.noalias() = x.adjoint() * y;
|
||||
} else {
|
||||
z.noalias() = x.adjoint() * y.adjoint();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Sequential batch matmul kernel that calls the regular Eigen matmul.
|
||||
// We prefer this over the tensor contraction because it performs
|
||||
// better on vector-matrix and matrix-vector products.
|
||||
template <typename Scalar>
|
||||
struct SequentialMatMulKernel {
|
||||
using Matrix =
|
||||
Eigen::Matrix<Scalar, Eigen::Dynamic, Eigen::Dynamic, Eigen::RowMajor>;
|
||||
using ConstMatrixMap = Eigen::Map<const Matrix>;
|
||||
using MatrixMap = Eigen::Map<Matrix>;
|
||||
|
||||
static ConstMatrixMap ConstTensorSliceToEigenMatrix(const Tensor& t,
|
||||
int slice) {
|
||||
return ConstMatrixMap(
|
||||
t.flat<Scalar>().data() + slice * t.dim_size(1) * t.dim_size(2),
|
||||
t.dim_size(1), t.dim_size(2));
|
||||
}
|
||||
|
||||
static MatrixMap TensorSliceToEigenMatrix(Tensor* t, int slice) {
|
||||
return MatrixMap(
|
||||
t->flat<Scalar>().data() + slice * t->dim_size(1) * t->dim_size(2),
|
||||
t->dim_size(1), t->dim_size(2));
|
||||
}
|
||||
|
||||
static void Run(const Tensor& in_x, const Tensor& in_y, bool adj_x,
|
||||
bool adj_y, Tensor* out, int start, int limit) {
|
||||
for (int i = start; i < limit; ++i) {
|
||||
auto x = ConstTensorSliceToEigenMatrix(in_x, i);
|
||||
auto y = ConstTensorSliceToEigenMatrix(in_y, i);
|
||||
auto z = TensorSliceToEigenMatrix(out, i);
|
||||
// TODO(rmlarsen): Get rid of the special casing here when we have
|
||||
// upstreamed improvements for matrix*vector and vector*matrix to
|
||||
// Eigen's general matrix product.
|
||||
if (!adj_x && x.rows() == 1) {
|
||||
Multiply(adj_x, adj_y, x.row(0), y, z);
|
||||
} else if (adj_x && x.cols() == 1) {
|
||||
Multiply(adj_x, adj_y, x.col(0), y, z);
|
||||
} else if (!adj_y && y.cols() == 1) {
|
||||
Multiply(adj_x, adj_y, x, y.col(0), z);
|
||||
} else if (adj_y && y.rows() == 1) {
|
||||
Multiply(adj_x, adj_y, x, y.row(0), z);
|
||||
} else {
|
||||
Multiply(adj_x, adj_y, x, y, z);
|
||||
}
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
} // namespace
|
||||
|
||||
template <typename Device, typename Scalar>
|
||||
struct LaunchBatchMatMul;
|
||||
|
||||
template <typename Scalar>
|
||||
struct LaunchBatchMatMul<CPUDevice, Scalar> {
|
||||
static void Launch(OpKernelContext* context, const Tensor& in_x,
|
||||
const Tensor& in_y, bool adj_x, bool adj_y, Tensor* out) {
|
||||
typedef ParallelMatMulKernel<Scalar, Eigen::NumTraits<Scalar>::IsComplex>
|
||||
ParallelMatMulKernel;
|
||||
bool conjugate_result = false;
|
||||
|
||||
// Number of matrix multiplies i.e. size of the batch.
|
||||
const int64 batch_size = in_x.dim_size(0);
|
||||
const int64 cost_per_unit =
|
||||
in_x.dim_size(1) * in_x.dim_size(2) * out->dim_size(2);
|
||||
const int64 small_dim = std::min(
|
||||
std::min(in_x.dim_size(1), in_x.dim_size(2)), out->dim_size(2));
|
||||
const int64 kMaxCostOuterParallelism = 128 * 128 * 256; // heuristic.
|
||||
auto worker_threads = *(context->device()->tensorflow_cpu_worker_threads());
|
||||
if (small_dim > 1 &&
|
||||
(batch_size == 1 || cost_per_unit > kMaxCostOuterParallelism)) {
|
||||
// Parallelize over inner dims.
|
||||
// For large matrix products it is counter-productive to parallelize
|
||||
// over the batch dimension.
|
||||
ParallelMatMulKernel::Run(context, in_x, in_y, adj_x, adj_y, out, 0,
|
||||
batch_size);
|
||||
conjugate_result = adj_x;
|
||||
} else {
|
||||
// Parallelize over outer dims. For small matrices and large batches, it
|
||||
// is counter-productive to parallelize the inner matrix multiplies.
|
||||
Shard(worker_threads.num_threads, worker_threads.workers, batch_size,
|
||||
cost_per_unit,
|
||||
[&in_x, &in_y, adj_x, adj_y, out](int start, int limit) {
|
||||
SequentialMatMulKernel<Scalar>::Run(in_x, in_y, adj_x, adj_y, out,
|
||||
start, limit);
|
||||
});
|
||||
}
|
||||
if (conjugate_result) {
|
||||
// We used one of the identities
|
||||
// conj(a) * conj(b) = conj(a * b)
|
||||
// conj(a) * b = conj(a * conj(b))
|
||||
// above, we need to conjugate the final output. This is a
|
||||
// no-op for non-complex types.
|
||||
ParallelMatMulKernel::Conjugate(context, out);
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
#if GOOGLE_CUDA
|
||||
|
||||
namespace {
|
||||
template <typename T>
|
||||
se::DeviceMemory<T> AsDeviceMemory(const T* cuda_memory) {
|
||||
se::DeviceMemoryBase wrapped(const_cast<T*>(cuda_memory));
|
||||
se::DeviceMemory<T> typed(wrapped);
|
||||
return typed;
|
||||
}
|
||||
|
||||
class CublasScratchAllocator : public se::ScratchAllocator {
|
||||
public:
|
||||
using Stream = se::Stream;
|
||||
using DeviceMemoryBytes = se::DeviceMemory<uint8>;
|
||||
|
||||
CublasScratchAllocator(OpKernelContext* context) : context_(context) {}
|
||||
|
||||
int64 GetMemoryLimitInBytes(Stream* stream) override { return -1; }
|
||||
|
||||
se::port::StatusOr<DeviceMemoryBytes> AllocateBytes(
|
||||
Stream* stream, int64 byte_size) override {
|
||||
Tensor temporary_memory;
|
||||
|
||||
Status allocation_status(context_->allocate_temp(
|
||||
DT_UINT8, TensorShape({byte_size}), &temporary_memory));
|
||||
if (!allocation_status.ok()) {
|
||||
return se::port::StatusOr<DeviceMemoryBytes>(
|
||||
DeviceMemoryBytes::MakeFromByteSize(nullptr, 0));
|
||||
}
|
||||
// Hold the reference of the allocated tensors until the end of the
|
||||
// allocator.
|
||||
allocated_tensors_.push_back(temporary_memory);
|
||||
return se::port::StatusOr<DeviceMemoryBytes>(
|
||||
DeviceMemoryBytes::MakeFromByteSize(
|
||||
temporary_memory.flat<uint8>().data(),
|
||||
temporary_memory.flat<uint8>().size()));
|
||||
}
|
||||
|
||||
private:
|
||||
OpKernelContext* context_;
|
||||
std::vector<Tensor> allocated_tensors_;
|
||||
};
|
||||
} // namespace
|
||||
|
||||
template <typename Scalar>
|
||||
struct LaunchBatchMatMul<GPUDevice, Scalar> {
|
||||
static void Launch(OpKernelContext* context, const Tensor& in_x,
|
||||
const Tensor& in_y, bool adj_x, bool adj_y, Tensor* out) {
|
||||
constexpr se::blas::Transpose kTranspose =
|
||||
is_complex<Scalar>::value ? se::blas::Transpose::kConjugateTranspose
|
||||
: se::blas::Transpose::kTranspose;
|
||||
se::blas::Transpose trans[] = {se::blas::Transpose::kNoTranspose,
|
||||
kTranspose};
|
||||
const uint64 m = in_x.dim_size(adj_x ? 2 : 1);
|
||||
const uint64 k = in_x.dim_size(adj_x ? 1 : 2);
|
||||
const uint64 n = in_y.dim_size(adj_y ? 1 : 2);
|
||||
const uint64 batch_size = in_x.dim_size(0);
|
||||
auto blas_transpose_a = trans[adj_x];
|
||||
auto blas_transpose_b = trans[adj_y];
|
||||
|
||||
auto* stream = context->op_device_context()->stream();
|
||||
OP_REQUIRES(context, stream, errors::Internal("No GPU stream available."));
|
||||
|
||||
typedef se::DeviceMemory<Scalar> DeviceMemoryType;
|
||||
std::vector<DeviceMemoryType> a_device_memory;
|
||||
std::vector<DeviceMemoryType> b_device_memory;
|
||||
std::vector<DeviceMemoryType> c_device_memory;
|
||||
std::vector<DeviceMemoryType*> a_ptrs;
|
||||
std::vector<DeviceMemoryType*> b_ptrs;
|
||||
std::vector<DeviceMemoryType*> c_ptrs;
|
||||
a_device_memory.reserve(batch_size);
|
||||
b_device_memory.reserve(batch_size);
|
||||
c_device_memory.reserve(batch_size);
|
||||
a_ptrs.reserve(batch_size);
|
||||
b_ptrs.reserve(batch_size);
|
||||
c_ptrs.reserve(batch_size);
|
||||
auto* a_base_ptr = in_x.template flat<Scalar>().data();
|
||||
auto* b_base_ptr = in_y.template flat<Scalar>().data();
|
||||
auto* c_base_ptr = out->template flat<Scalar>().data();
|
||||
for (int64 i = 0; i < batch_size; ++i) {
|
||||
a_device_memory.push_back(AsDeviceMemory(a_base_ptr + i * m * k));
|
||||
b_device_memory.push_back(AsDeviceMemory(b_base_ptr + i * k * n));
|
||||
c_device_memory.push_back(AsDeviceMemory(c_base_ptr + i * m * n));
|
||||
a_ptrs.push_back(&a_device_memory.back());
|
||||
b_ptrs.push_back(&b_device_memory.back());
|
||||
c_ptrs.push_back(&c_device_memory.back());
|
||||
}
|
||||
|
||||
typedef Scalar Coefficient;
|
||||
|
||||
// Cublas does
|
||||
// C = A x B
|
||||
// where A, B and C are assumed to be in column major.
|
||||
// We want the output to be in row-major, so we can compute
|
||||
// C' = B' x A', where ' stands for transpose (not adjoint).
|
||||
// TODO(yangzihao): Choose the best of the three strategies using autotune.
|
||||
if (batch_size == 1) {
|
||||
// This is a regular matrix*matrix or matrix*vector multiply. Avoid the
|
||||
// overhead of the scratch allocator and the batch interface.
|
||||
if (n == 1 &&
|
||||
blas_transpose_b != se::blas::Transpose::kConjugateTranspose &&
|
||||
blas_transpose_a != se::blas::Transpose::kConjugateTranspose) {
|
||||
// This is a matrix*vector multiply so use GEMV to compute A * b.
|
||||
// Here we are multiplying in the natural order, so we have to flip
|
||||
// the transposition flag to compensate for the tensor being stored
|
||||
// row-major. Since GEMV doesn't provide a way to just conjugate an
|
||||
// argument, we have to defer those cases to GEMM below.
|
||||
auto gemv_trans_a = blas_transpose_a == se::blas::Transpose::kTranspose
|
||||
? se::blas::Transpose::kNoTranspose
|
||||
: se::blas::Transpose::kTranspose;
|
||||
bool blas_launch_status =
|
||||
stream
|
||||
->ThenBlasGemv(gemv_trans_a, adj_x ? m : k, adj_x ? k : m,
|
||||
static_cast<Coefficient>(1.0), *(a_ptrs[0]),
|
||||
adj_x ? m : k, *(b_ptrs[0]), 1,
|
||||
static_cast<Coefficient>(0.0), c_ptrs[0], 1)
|
||||
.ok();
|
||||
if (!blas_launch_status) {
|
||||
context->SetStatus(errors::Internal(
|
||||
"Blas xGEMV launch failed : a.shape=", in_x.shape().DebugString(),
|
||||
", b.shape=", in_y.shape().DebugString(), ", m=", m, ", n=", n,
|
||||
", k=", k));
|
||||
}
|
||||
} else {
|
||||
bool blas_launch_status =
|
||||
stream
|
||||
->ThenBlasGemm(blas_transpose_b, blas_transpose_a, n, m, k,
|
||||
static_cast<Coefficient>(1.0), *(b_ptrs[0]),
|
||||
adj_y ? k : n, *(a_ptrs[0]), adj_x ? m : k,
|
||||
static_cast<Coefficient>(0.0), c_ptrs[0], n)
|
||||
.ok();
|
||||
if (!blas_launch_status) {
|
||||
context->SetStatus(errors::Internal(
|
||||
"Blas xGEMM launch failed : a.shape=", in_x.shape().DebugString(),
|
||||
", b.shape=", in_y.shape().DebugString(), ", m=", m, ", n=", n,
|
||||
", k=", k));
|
||||
}
|
||||
}
|
||||
} else {
|
||||
CublasScratchAllocator scratch_allocator(context);
|
||||
bool blas_launch_status =
|
||||
stream
|
||||
->ThenBlasGemmBatchedWithScratch(
|
||||
blas_transpose_b, blas_transpose_a, n, m, k,
|
||||
static_cast<Coefficient>(1.0), b_ptrs, adj_y ? k : n, a_ptrs,
|
||||
adj_x ? m : k, static_cast<Coefficient>(0.0), c_ptrs, n,
|
||||
batch_size, &scratch_allocator)
|
||||
.ok();
|
||||
if (!blas_launch_status) {
|
||||
context->SetStatus(errors::Internal(
|
||||
"Blas xGEMMBatched launch failed : a.shape=",
|
||||
in_x.shape().DebugString(),
|
||||
", b.shape=", in_y.shape().DebugString(), ", m=", m, ", n=", n,
|
||||
", k=", k, ", batch_size=", batch_size));
|
||||
}
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
template <>
|
||||
struct LaunchBatchMatMul<GPUDevice, Eigen::half> {
|
||||
static void Launch(OpKernelContext* context, const Tensor& in_x,
|
||||
const Tensor& in_y, bool adj_x, bool adj_y, Tensor* out) {
|
||||
typedef Eigen::half Scalar;
|
||||
constexpr perftools::gputools::blas::Transpose kTranspose =
|
||||
is_complex<Scalar>::value
|
||||
? perftools::gputools::blas::Transpose::kConjugateTranspose
|
||||
: perftools::gputools::blas::Transpose::kTranspose;
|
||||
perftools::gputools::blas::Transpose trans[] = {
|
||||
perftools::gputools::blas::Transpose::kNoTranspose, kTranspose};
|
||||
const uint64 m = in_x.dim_size(adj_x ? 2 : 1);
|
||||
const uint64 k = in_x.dim_size(adj_x ? 1 : 2);
|
||||
const uint64 n = in_y.dim_size(adj_y ? 1 : 2);
|
||||
const uint64 batch_size = in_x.dim_size(0);
|
||||
auto blas_transpose_a = trans[adj_x];
|
||||
auto blas_transpose_b = trans[adj_y];
|
||||
|
||||
auto* stream = context->op_device_context()->stream();
|
||||
OP_REQUIRES(context, stream, errors::Internal("No GPU stream available."));
|
||||
|
||||
typedef perftools::gputools::DeviceMemory<Scalar> DeviceMemoryType;
|
||||
std::vector<DeviceMemoryType> a_device_memory;
|
||||
std::vector<DeviceMemoryType> b_device_memory;
|
||||
std::vector<DeviceMemoryType> c_device_memory;
|
||||
std::vector<DeviceMemoryType*> a_ptrs;
|
||||
std::vector<DeviceMemoryType*> b_ptrs;
|
||||
std::vector<DeviceMemoryType*> c_ptrs;
|
||||
a_device_memory.reserve(batch_size);
|
||||
b_device_memory.reserve(batch_size);
|
||||
c_device_memory.reserve(batch_size);
|
||||
a_ptrs.reserve(batch_size);
|
||||
b_ptrs.reserve(batch_size);
|
||||
c_ptrs.reserve(batch_size);
|
||||
auto* a_base_ptr = in_x.template flat<Scalar>().data();
|
||||
auto* b_base_ptr = in_y.template flat<Scalar>().data();
|
||||
auto* c_base_ptr = out->template flat<Scalar>().data();
|
||||
for (int64 i = 0; i < batch_size; ++i) {
|
||||
a_device_memory.push_back(AsDeviceMemory(a_base_ptr + i * m * k));
|
||||
b_device_memory.push_back(AsDeviceMemory(b_base_ptr + i * k * n));
|
||||
c_device_memory.push_back(AsDeviceMemory(c_base_ptr + i * m * n));
|
||||
a_ptrs.push_back(&a_device_memory.back());
|
||||
b_ptrs.push_back(&b_device_memory.back());
|
||||
c_ptrs.push_back(&c_device_memory.back());
|
||||
}
|
||||
|
||||
typedef float Coefficient;
|
||||
|
||||
// Cublas does
|
||||
// C = A x B
|
||||
// where A, B and C are assumed to be in column major.
|
||||
// We want the output to be in row-major, so we can compute
|
||||
// C' = B' x A', where ' stands for transpose (not adjoint).
|
||||
// TODO(yangzihao): Choose the best of the three strategies using autotune.
|
||||
if (batch_size == 1) {
|
||||
// This is a regular matrix*matrix or matrix*vector multiply. Avoid the
|
||||
// overhead of the scratch allocator and the batch interface.
|
||||
// TODO(benbarsdell): Use fp16 Gemv if it becomes supported by CUBLAS
|
||||
bool blas_launch_status =
|
||||
stream
|
||||
->ThenBlasGemm(blas_transpose_b, blas_transpose_a, n, m, k,
|
||||
static_cast<Coefficient>(1.0), *(b_ptrs[0]),
|
||||
adj_y ? k : n, *(a_ptrs[0]), adj_x ? m : k,
|
||||
static_cast<Coefficient>(0.0), c_ptrs[0], n)
|
||||
.ok();
|
||||
if (!blas_launch_status) {
|
||||
context->SetStatus(errors::Internal(
|
||||
"Blas xGEMM launch failed : a.shape=", in_x.shape().DebugString(),
|
||||
", b.shape=", in_y.shape().DebugString(), ", m=", m, ", n=", n,
|
||||
", k=", k));
|
||||
}
|
||||
} else {
|
||||
CublasScratchAllocator scratch_allocator(context);
|
||||
bool blas_launch_status =
|
||||
stream
|
||||
->ThenBlasGemmBatchedWithScratch(
|
||||
blas_transpose_b, blas_transpose_a, n, m, k,
|
||||
static_cast<Coefficient>(1.0), b_ptrs, adj_y ? k : n, a_ptrs,
|
||||
adj_x ? m : k, static_cast<Coefficient>(0.0), c_ptrs, n,
|
||||
batch_size, &scratch_allocator)
|
||||
.ok();
|
||||
if (!blas_launch_status) {
|
||||
context->SetStatus(
|
||||
errors::Internal("Blas xGEMMBatched launch failed : a.shape=",
|
||||
in_x.shape().DebugString(), ", b.shape=",
|
||||
in_y.shape().DebugString(), ", m=", m, ", n=", n,
|
||||
", k=", k, ", batch_size=", batch_size));
|
||||
}
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
#endif // GOOGLE_CUDA
|
||||
|
||||
#ifdef TENSORFLOW_USE_SYCL
|
||||
template <typename Scalar>
|
||||
struct ParallelMatMulKernelSYCL {
|
||||
static void Run(const OpKernelContext* context, const Tensor& in_x,
|
||||
const Tensor& in_y, bool adj_x, bool adj_y, Tensor* out,
|
||||
int start, int limit) {
|
||||
auto Tx = in_x.tensor<Scalar, 3>();
|
||||
auto Ty = in_y.tensor<Scalar, 3>();
|
||||
auto Tz = out->tensor<Scalar, 3>();
|
||||
Eigen::array<Eigen::IndexPair<Eigen::DenseIndex>, 1> contract_pairs;
|
||||
contract_pairs[0] = ContractionDims(adj_x, adj_y);
|
||||
auto d = context->eigen_sycl_device();
|
||||
for (int i = start; i < limit; ++i) {
|
||||
auto x = Tx.template chip<0>(i);
|
||||
auto y = Ty.template chip<0>(i);
|
||||
auto z = Tz.template chip<0>(i);
|
||||
z.device(d) = x.contract(y, contract_pairs);
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
template <typename Scalar>
|
||||
struct LaunchBatchMatMul<SYCLDevice, Scalar> {
|
||||
static void Launch(OpKernelContext* context, const Tensor& in_x,
|
||||
const Tensor& in_y, bool adj_x, bool adj_y, Tensor* out) {
|
||||
// Number of matrix multiplies i.e. size of the batch.
|
||||
const int64 batch_size = in_x.dim_size(0);
|
||||
ParallelMatMulKernelSYCL<Scalar>::Run(context, in_x, in_y, adj_x, adj_y,
|
||||
out, 0, batch_size);
|
||||
}
|
||||
};
|
||||
#endif // TENSORFLOW_USE_SYCL
|
||||
|
||||
template <typename Device, typename Scalar>
|
||||
class BatchMatMul : public OpKernel {
|
||||
public:
|
||||
explicit BatchMatMul(OpKernelConstruction* context) : OpKernel(context) {
|
||||
OP_REQUIRES_OK(context, context->GetAttr("adj_x", &adj_x_));
|
||||
OP_REQUIRES_OK(context, context->GetAttr("adj_y", &adj_y_));
|
||||
}
|
||||
|
||||
virtual ~BatchMatMul() {}
|
||||
|
||||
void Compute(OpKernelContext* ctx) override {
|
||||
const Tensor& in0 = ctx->input(0);
|
||||
const Tensor& in1 = ctx->input(1);
|
||||
OP_REQUIRES(ctx, in0.dims() == in1.dims(),
|
||||
errors::InvalidArgument("In[0] and In[1] has different ndims: ",
|
||||
in0.shape().DebugString(), " vs. ",
|
||||
in1.shape().DebugString()));
|
||||
const int ndims = in0.dims();
|
||||
OP_REQUIRES(
|
||||
ctx, ndims >= 2,
|
||||
errors::InvalidArgument("In[0] and In[1] ndims must be >= 2: ", ndims));
|
||||
TensorShape out_shape;
|
||||
for (int i = 0; i < ndims - 2; ++i) {
|
||||
OP_REQUIRES(ctx, in0.dim_size(i) == in1.dim_size(i),
|
||||
errors::InvalidArgument(
|
||||
"In[0].dim(", i, ") and In[1].dim(", i,
|
||||
") must be the same: ", in0.shape().DebugString(), " vs ",
|
||||
in1.shape().DebugString()));
|
||||
out_shape.AddDim(in0.dim_size(i));
|
||||
}
|
||||
auto n = (ndims == 2) ? 1 : out_shape.num_elements();
|
||||
auto d0 = in0.dim_size(ndims - 2);
|
||||
auto d1 = in0.dim_size(ndims - 1);
|
||||
Tensor in0_reshaped;
|
||||
CHECK(in0_reshaped.CopyFrom(in0, TensorShape({n, d0, d1})));
|
||||
auto d2 = in1.dim_size(ndims - 2);
|
||||
auto d3 = in1.dim_size(ndims - 1);
|
||||
Tensor in1_reshaped;
|
||||
CHECK(in1_reshaped.CopyFrom(in1, TensorShape({n, d2, d3})));
|
||||
if (adj_x_) std::swap(d0, d1);
|
||||
if (adj_y_) std::swap(d2, d3);
|
||||
OP_REQUIRES(ctx, d1 == d2,
|
||||
errors::InvalidArgument(
|
||||
"In[0] mismatch In[1] shape: ", d1, " vs. ", d2, ": ",
|
||||
in0.shape().DebugString(), " ", in1.shape().DebugString(),
|
||||
" ", adj_x_, " ", adj_y_));
|
||||
out_shape.AddDim(d0);
|
||||
out_shape.AddDim(d3);
|
||||
Tensor* out = nullptr;
|
||||
OP_REQUIRES_OK(ctx, ctx->allocate_output(0, out_shape, &out));
|
||||
if (out->NumElements() == 0) {
|
||||
return;
|
||||
}
|
||||
if (in0.NumElements() == 0 || in1.NumElements() == 0) {
|
||||
functor::SetZeroFunctor<Device, Scalar> f;
|
||||
f(ctx->eigen_device<Device>(), out->flat<Scalar>());
|
||||
return;
|
||||
}
|
||||
Tensor out_reshaped;
|
||||
CHECK(out_reshaped.CopyFrom(*out, TensorShape({n, d0, d3})));
|
||||
LaunchBatchMatMul<Device, Scalar>::Launch(ctx, in0_reshaped, in1_reshaped,
|
||||
adj_x_, adj_y_, &out_reshaped);
|
||||
}
|
||||
|
||||
private:
|
||||
bool adj_x_;
|
||||
bool adj_y_;
|
||||
};
|
||||
|
||||
#define REGISTER_BATCH_MATMUL_CPU(TYPE) \
|
||||
REGISTER_KERNEL_BUILDER( \
|
||||
Name("BatchMatMul").Device(DEVICE_CPU).TypeConstraint<TYPE>("T"), \
|
||||
BatchMatMul<CPUDevice, TYPE>)
|
||||
|
||||
#define REGISTER_BATCH_MATMUL_GPU(TYPE) \
|
||||
REGISTER_KERNEL_BUILDER( \
|
||||
Name("BatchMatMul").Device(DEVICE_GPU).TypeConstraint<TYPE>("T"), \
|
||||
BatchMatMul<GPUDevice, TYPE>)
|
||||
|
||||
#ifdef TENSORFLOW_USE_SYCL
|
||||
#define REGISTER_BATCH_MATMUL_SYCL(TYPE) \
|
||||
REGISTER_KERNEL_BUILDER( \
|
||||
Name("BatchMatMul").Device(DEVICE_SYCL).TypeConstraint<TYPE>("T"), \
|
||||
BatchMatMul<SYCLDevice, TYPE>)
|
||||
#endif // TENSORFLOW_USE_SYCL
|
||||
} // end namespace tensorflow
|
||||
|
||||
#endif // TENSORFLOW_CORE_KERNELS_BATCH_MATMUL_OP_IMPL_H_
|
||||
@@ -0,0 +1,156 @@
|
||||
/* Copyright 2015 The TensorFlow Authors. All Rights Reserved.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
==============================================================================*/
|
||||
|
||||
#ifndef TENSORFLOW_CORE_KERNELS_BATCH_NORM_OP_H_
|
||||
#define TENSORFLOW_CORE_KERNELS_BATCH_NORM_OP_H_
|
||||
// Functor definition for BatchNormOp, must be compilable by nvcc.
|
||||
#include "third_party/eigen3/unsupported/Eigen/CXX11/Tensor"
|
||||
#include "tensorflow/core/framework/tensor_types.h"
|
||||
|
||||
namespace tensorflow {
|
||||
namespace functor {
|
||||
|
||||
// Functor used by BatchNormOp to do the computations.
|
||||
template <typename Device, typename T>
|
||||
struct BatchNorm {
|
||||
void operator()(const Device& d, typename TTypes<T, 4>::ConstTensor input,
|
||||
typename TTypes<T>::ConstVec mean,
|
||||
typename TTypes<T>::ConstVec var,
|
||||
typename TTypes<T>::ConstVec beta,
|
||||
typename TTypes<T>::ConstVec gamma, T variance_epsilon,
|
||||
bool scale_after_normalization,
|
||||
typename TTypes<T, 4>::Tensor output) {
|
||||
const int depth = mean.dimension(0);
|
||||
const int rest_size = input.size() / depth;
|
||||
|
||||
Eigen::DSizes<int, 2> rest_by_depth(rest_size, depth);
|
||||
#if !defined(EIGEN_HAS_INDEX_LIST)
|
||||
Eigen::DSizes<int, 2> rest_by_one(rest_size, 1);
|
||||
Eigen::DSizes<int, 2> one_by_depth(1, depth);
|
||||
Eigen::DSizes<int, 2> depth_by_one(depth, 1);
|
||||
#else
|
||||
Eigen::IndexList<int, Eigen::type2index<1> > rest_by_one;
|
||||
rest_by_one.set(0, rest_size);
|
||||
Eigen::IndexList<Eigen::type2index<1>, int> one_by_depth;
|
||||
one_by_depth.set(1, depth);
|
||||
Eigen::IndexList<int, Eigen::type2index<1> > depth_by_one;
|
||||
depth_by_one.set(0, depth);
|
||||
#endif
|
||||
if (scale_after_normalization) {
|
||||
output.reshape(rest_by_depth).device(d) =
|
||||
(input.reshape(rest_by_depth) -
|
||||
mean.reshape(one_by_depth).broadcast(rest_by_one)) *
|
||||
((var + var.constant(variance_epsilon)).rsqrt() * gamma)
|
||||
.eval()
|
||||
.reshape(one_by_depth)
|
||||
.broadcast(rest_by_one) +
|
||||
beta.reshape(one_by_depth).broadcast(rest_by_one);
|
||||
} else {
|
||||
output.reshape(rest_by_depth).device(d) =
|
||||
(input.reshape(rest_by_depth) -
|
||||
mean.reshape(one_by_depth).broadcast(rest_by_one)) *
|
||||
((var + var.constant(variance_epsilon)).rsqrt())
|
||||
.eval()
|
||||
.reshape(one_by_depth)
|
||||
.broadcast(rest_by_one) +
|
||||
beta.reshape(one_by_depth).broadcast(rest_by_one);
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
template <typename Device, typename T>
|
||||
struct BatchNormGrad {
|
||||
void operator()(const Device& d, typename TTypes<T, 4>::ConstTensor input,
|
||||
typename TTypes<T>::ConstVec mean,
|
||||
typename TTypes<T>::ConstVec var,
|
||||
typename TTypes<T>::ConstVec gamma,
|
||||
typename TTypes<T, 4>::ConstTensor out_backprop,
|
||||
T variance_epsilon, bool scale_after_normalization,
|
||||
typename TTypes<T, 4>::Tensor dx, typename TTypes<T>::Vec dm,
|
||||
typename TTypes<T>::Vec dv, typename TTypes<T>::Vec db,
|
||||
typename TTypes<T>::Vec dg, typename TTypes<T>::Vec scratch1,
|
||||
typename TTypes<T>::Vec scratch2) {
|
||||
const int depth = mean.dimension(0);
|
||||
const int rest_size = input.size() / depth;
|
||||
|
||||
typedef typename TTypes<T>::ConstVec::Index Index;
|
||||
|
||||
Eigen::DSizes<Index, 2> rest_by_depth(rest_size, depth);
|
||||
#if !defined(EIGEN_HAS_INDEX_LIST)
|
||||
Eigen::DSizes<Index, 2> rest_by_one(rest_size, 1);
|
||||
Eigen::DSizes<Index, 2> one_by_depth(1, depth);
|
||||
Eigen::array<Index, 1> reduction_axis;
|
||||
reduction_axis[0] = 0; // Reduces on first dimension.
|
||||
#else
|
||||
Eigen::IndexList<Index, Eigen::type2index<1> > rest_by_one;
|
||||
rest_by_one.set(0, rest_size);
|
||||
Eigen::IndexList<Eigen::type2index<1>, Index> one_by_depth;
|
||||
one_by_depth.set(1, depth);
|
||||
Eigen::IndexList<Eigen::type2index<0> > reduction_axis;
|
||||
#endif
|
||||
|
||||
// db = out_backprop
|
||||
//
|
||||
// dg = out_backprop * ((x - m) * rsqrt(v + epsilon))
|
||||
//
|
||||
// dv = sum_over_rest(out_backprop * gamma * (x - m)) *
|
||||
// (-1/2) * (v + epsilon) ^ (-3/2)
|
||||
//
|
||||
// dm = sum_over_rest(out_backprop * gamma) * (-1 / rsqrt(v + epsilon))
|
||||
//
|
||||
// dx = out_backprop * (gamma * rsqrt(v + epsilon))
|
||||
db.device(d) = out_backprop.reshape(rest_by_depth).sum(reduction_axis);
|
||||
|
||||
// scratch1 = rsqrt(v + epsilon)
|
||||
scratch1.device(d) = (var + var.constant(variance_epsilon)).rsqrt();
|
||||
|
||||
// scratch2 = sum_over_rest(out_backprop * (x - m))
|
||||
scratch2.device(d) = (out_backprop.reshape(rest_by_depth) *
|
||||
(input.reshape(rest_by_depth) -
|
||||
mean.reshape(one_by_depth).broadcast(rest_by_one)))
|
||||
.sum(reduction_axis);
|
||||
|
||||
if (scale_after_normalization) {
|
||||
dx.reshape(rest_by_depth).device(d) =
|
||||
out_backprop.reshape(rest_by_depth) * ((scratch1 * gamma)
|
||||
.eval()
|
||||
.reshape(one_by_depth)
|
||||
.broadcast(rest_by_one));
|
||||
dm.device(d) = -db * (scratch1 * gamma).eval();
|
||||
dg.device(d) = scratch2 * scratch1;
|
||||
} else {
|
||||
dx.reshape(rest_by_depth).device(d) =
|
||||
out_backprop.reshape(rest_by_depth) *
|
||||
scratch1.reshape(one_by_depth).broadcast(rest_by_one);
|
||||
dm.device(d) = -db * scratch1;
|
||||
dg.device(d) = dg.constant(static_cast<T>(0.0)); // Gamma is not learned.
|
||||
}
|
||||
|
||||
// scratch1 = - 1/2 * (var + epsilon) ^ (-3/2)
|
||||
scratch1.device(d) = scratch1 * scratch1.constant(static_cast<T>(-0.5f)) /
|
||||
(var + var.constant(variance_epsilon));
|
||||
|
||||
if (scale_after_normalization) {
|
||||
dv.device(d) = scratch2 * (scratch1 * gamma).eval();
|
||||
} else {
|
||||
dv.device(d) = scratch2 * scratch1;
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
} // namespace functor
|
||||
} // namespace tensorflow
|
||||
|
||||
#endif // TENSORFLOW_CORE_KERNELS_BATCH_NORM_OP_H_
|
||||
@@ -0,0 +1,281 @@
|
||||
/* Copyright 2016 The TensorFlow Authors. All Rights Reserved.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
==============================================================================*/
|
||||
|
||||
// Abstractions for processing small tasks in a batched fashion, to reduce
|
||||
// processing times and costs that can be amortized across multiple tasks.
|
||||
//
|
||||
// The core class is BatchScheduler, which groups tasks into batches.
|
||||
//
|
||||
// BatchScheduler encapsulates logic for aggregating multiple tasks into a
|
||||
// batch, and kicking off processing of a batch on a thread pool it manages.
|
||||
//
|
||||
// This file defines an abstract BatchScheduler class.
|
||||
|
||||
#ifndef TENSORFLOW_CORE_KERNELS_BATCHING_UTIL_BATCH_SCHEDULER_H_
|
||||
#define TENSORFLOW_CORE_KERNELS_BATCHING_UTIL_BATCH_SCHEDULER_H_
|
||||
|
||||
#include <stddef.h>
|
||||
#include <algorithm>
|
||||
#include <functional>
|
||||
#include <memory>
|
||||
#include <utility>
|
||||
#include <vector>
|
||||
|
||||
#include "tensorflow/core/lib/core/notification.h"
|
||||
#include "tensorflow/core/lib/core/status.h"
|
||||
#include "tensorflow/core/platform/logging.h"
|
||||
#include "tensorflow/core/platform/macros.h"
|
||||
#include "tensorflow/core/platform/mutex.h"
|
||||
#include "tensorflow/core/platform/thread_annotations.h"
|
||||
#include "tensorflow/core/platform/types.h"
|
||||
|
||||
namespace tensorflow {
|
||||
namespace serving {
|
||||
|
||||
// The abstract superclass for a unit of work to be done as part of a batch.
|
||||
//
|
||||
// An implementing subclass typically contains (or points to):
|
||||
// (a) input data;
|
||||
// (b) a thread-safe completion signal (e.g. a Notification);
|
||||
// (c) a place to store the outcome (success, or some error), upon completion;
|
||||
// (d) a place to store the output data, upon success.
|
||||
//
|
||||
// Items (b), (c) and (d) are typically non-owned pointers to data homed
|
||||
// elsewhere, because a task's ownership gets transferred to a BatchScheduler
|
||||
// (see below) and it may be deleted as soon as it is done executing.
|
||||
class BatchTask {
|
||||
public:
|
||||
virtual ~BatchTask() = default;
|
||||
|
||||
// Returns the size of the task, in terms of how much it contributes to the
|
||||
// size of a batch. (A batch's size is the sum of its task sizes.)
|
||||
virtual size_t size() const = 0;
|
||||
};
|
||||
|
||||
// A thread-safe collection of BatchTasks, to be executed together in some
|
||||
// fashion.
|
||||
//
|
||||
// At a given time, a batch is either "open" or "closed": an open batch can
|
||||
// accept new tasks; a closed one cannot. A batch is monotonic: initially it is
|
||||
// open and tasks can be added to it; then it is closed and its set of tasks
|
||||
// remains fixed for the remainder of its life. A closed batch cannot be re-
|
||||
// opened. Tasks can never be removed from a batch.
|
||||
//
|
||||
// Type parameter TaskType must be a subclass of BatchTask.
|
||||
template <typename TaskType>
|
||||
class Batch {
|
||||
public:
|
||||
Batch() = default;
|
||||
virtual ~Batch(); // Blocks until the batch is closed.
|
||||
|
||||
// Appends 'task' to the batch. After calling AddTask(), the newly-added task
|
||||
// can be accessed via task(num_tasks()-1) or mutable_task(num_tasks()-1).
|
||||
// Dies if the batch is closed.
|
||||
void AddTask(std::unique_ptr<TaskType> task);
|
||||
|
||||
// Removes the most recently added task. Returns nullptr if the batch is
|
||||
// empty.
|
||||
std::unique_ptr<TaskType> RemoveTask();
|
||||
|
||||
// Returns the number of tasks in the batch.
|
||||
int num_tasks() const;
|
||||
|
||||
// Returns true iff the batch contains 0 tasks.
|
||||
bool empty() const;
|
||||
|
||||
// Returns a reference to the ith task (in terms of insertion order).
|
||||
const TaskType& task(int i) const;
|
||||
|
||||
// Returns a pointer to the ith task (in terms of insertion order).
|
||||
TaskType* mutable_task(int i);
|
||||
|
||||
// Returns the sum of the task sizes.
|
||||
size_t size() const;
|
||||
|
||||
// Returns true iff the batch is currently closed.
|
||||
bool IsClosed() const;
|
||||
|
||||
// Blocks until the batch is closed.
|
||||
void WaitUntilClosed() const;
|
||||
|
||||
// Marks the batch as closed. Dies if called more than once.
|
||||
void Close();
|
||||
|
||||
private:
|
||||
mutable mutex mu_;
|
||||
|
||||
// The tasks in the batch.
|
||||
std::vector<std::unique_ptr<TaskType>> tasks_ GUARDED_BY(mu_);
|
||||
|
||||
// The sum of the sizes of the tasks in 'tasks_'.
|
||||
size_t size_ GUARDED_BY(mu_) = 0;
|
||||
|
||||
// Whether the batch has been closed.
|
||||
Notification closed_;
|
||||
|
||||
TF_DISALLOW_COPY_AND_ASSIGN(Batch);
|
||||
};
|
||||
|
||||
// An abstract batch scheduler class. Collects individual tasks into batches,
|
||||
// and processes each batch on a pool of "batch threads" that it manages. The
|
||||
// actual logic for processing a batch is accomplished via a callback.
|
||||
//
|
||||
// Type parameter TaskType must be a subclass of BatchTask.
|
||||
template <typename TaskType>
|
||||
class BatchScheduler {
|
||||
public:
|
||||
virtual ~BatchScheduler() = default;
|
||||
|
||||
// Submits a task to be processed as part of a batch.
|
||||
//
|
||||
// Ownership of '*task' is transferred to the callee iff the method returns
|
||||
// Status::OK. In that case, '*task' is left as nullptr. Otherwise, '*task' is
|
||||
// left as-is.
|
||||
//
|
||||
// If no batch processing capacity is available to process this task at the
|
||||
// present time, and any task queue maintained by the implementing subclass is
|
||||
// full, this method returns an UNAVAILABLE error code. The client may retry
|
||||
// later.
|
||||
//
|
||||
// Other problems, such as the task size being larger than the maximum batch
|
||||
// size, yield other, permanent error types.
|
||||
//
|
||||
// In all cases, this method returns "quickly" without blocking for any
|
||||
// substantial amount of time. If the method returns Status::OK, the task is
|
||||
// processed asynchronously, and any errors that occur during the processing
|
||||
// of the batch that includes the task can be reported to 'task'.
|
||||
virtual Status Schedule(std::unique_ptr<TaskType>* task) = 0;
|
||||
|
||||
// Returns the number of tasks that have been scheduled (i.e. accepted by
|
||||
// Schedule()), but have yet to be handed to a thread for execution as part of
|
||||
// a batch. Note that this returns the number of tasks, not the aggregate task
|
||||
// size (so if there is one task of size 3 and one task of size 5, this method
|
||||
// returns 2 rather than 8).
|
||||
virtual size_t NumEnqueuedTasks() const = 0;
|
||||
|
||||
// Returns a guaranteed number of size 1 tasks that can be Schedule()d without
|
||||
// getting an UNAVAILABLE error. In a typical implementation, returns the
|
||||
// available space on a queue.
|
||||
//
|
||||
// There are two important caveats:
|
||||
// 1. The guarantee does not extend to varying-size tasks due to possible
|
||||
// internal fragmentation of batches.
|
||||
// 2. The guarantee only holds in a single-thread environment or critical
|
||||
// section, i.e. if an intervening thread cannot call Schedule().
|
||||
//
|
||||
// This method is useful for monitoring, or for guaranteeing a future slot in
|
||||
// the schedule (but being mindful about the caveats listed above).
|
||||
virtual size_t SchedulingCapacity() const = 0;
|
||||
|
||||
// Returns the maximum allowed size of tasks submitted to the scheduler. (This
|
||||
// is typically equal to a configured maximum batch size.)
|
||||
virtual size_t max_task_size() const = 0;
|
||||
};
|
||||
|
||||
//////////
|
||||
// Implementation details follow. API users need not read.
|
||||
|
||||
template <typename TaskType>
|
||||
Batch<TaskType>::~Batch() {
|
||||
WaitUntilClosed();
|
||||
}
|
||||
|
||||
template <typename TaskType>
|
||||
void Batch<TaskType>::AddTask(std::unique_ptr<TaskType> task) {
|
||||
DCHECK(!IsClosed());
|
||||
{
|
||||
mutex_lock l(mu_);
|
||||
size_ += task->size();
|
||||
tasks_.push_back(std::move(task));
|
||||
}
|
||||
}
|
||||
|
||||
template <typename TaskType>
|
||||
std::unique_ptr<TaskType> Batch<TaskType>::RemoveTask() {
|
||||
{
|
||||
mutex_lock l(mu_);
|
||||
if (tasks_.empty()) {
|
||||
return nullptr;
|
||||
}
|
||||
std::unique_ptr<TaskType> task = std::move(tasks_.back());
|
||||
size_ -= task->size();
|
||||
tasks_.pop_back();
|
||||
return task;
|
||||
}
|
||||
}
|
||||
|
||||
template <typename TaskType>
|
||||
int Batch<TaskType>::num_tasks() const {
|
||||
{
|
||||
mutex_lock l(mu_);
|
||||
return tasks_.size();
|
||||
}
|
||||
}
|
||||
|
||||
template <typename TaskType>
|
||||
bool Batch<TaskType>::empty() const {
|
||||
{
|
||||
mutex_lock l(mu_);
|
||||
return tasks_.empty();
|
||||
}
|
||||
}
|
||||
|
||||
template <typename TaskType>
|
||||
const TaskType& Batch<TaskType>::task(int i) const {
|
||||
DCHECK_GE(i, 0);
|
||||
{
|
||||
mutex_lock l(mu_);
|
||||
DCHECK_LT(i, tasks_.size());
|
||||
return *tasks_[i].get();
|
||||
}
|
||||
}
|
||||
|
||||
template <typename TaskType>
|
||||
TaskType* Batch<TaskType>::mutable_task(int i) {
|
||||
DCHECK_GE(i, 0);
|
||||
{
|
||||
mutex_lock l(mu_);
|
||||
DCHECK_LT(i, tasks_.size());
|
||||
return tasks_[i].get();
|
||||
}
|
||||
}
|
||||
|
||||
template <typename TaskType>
|
||||
size_t Batch<TaskType>::size() const {
|
||||
{
|
||||
mutex_lock l(mu_);
|
||||
return size_;
|
||||
}
|
||||
}
|
||||
|
||||
template <typename TaskType>
|
||||
bool Batch<TaskType>::IsClosed() const {
|
||||
return const_cast<Notification*>(&closed_)->HasBeenNotified();
|
||||
}
|
||||
|
||||
template <typename TaskType>
|
||||
void Batch<TaskType>::WaitUntilClosed() const {
|
||||
const_cast<Notification*>(&closed_)->WaitForNotification();
|
||||
}
|
||||
|
||||
template <typename TaskType>
|
||||
void Batch<TaskType>::Close() {
|
||||
closed_.Notify();
|
||||
}
|
||||
|
||||
} // namespace serving
|
||||
} // namespace tensorflow
|
||||
|
||||
#endif // TENSORFLOW_CORE_KERNELS_BATCHING_UTIL_BATCH_SCHEDULER_H_
|
||||
@@ -0,0 +1,134 @@
|
||||
/* Copyright 2016 The TensorFlow Authors. All Rights Reserved.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
==============================================================================*/
|
||||
|
||||
// PeriodicFunction will periodically call the given function with a specified
|
||||
// period in a background thread. After Start() returns, the thread is
|
||||
// guaranteed to have started. The destruction of the class causes the
|
||||
// background thread to be destroyed as well. Start() should not be called more
|
||||
// than once.
|
||||
//
|
||||
// PeriodicFunction runs the function as soon as any previous run both is
|
||||
// complete and was started more than "interval_micros" earlier. Thus, runs are
|
||||
// both serialized, and normally have a period of "interval_micros" if no run
|
||||
// exceeds the time.
|
||||
//
|
||||
// Note that, if the function takes longer than two interval_micross to finish,
|
||||
// then PeriodicFunction will "skip" at least one call to the function. For
|
||||
// instance, if the period is 50ms and the function starts runs at time 0 for
|
||||
// 150ms, then the function will immediately start executing again at time 150,
|
||||
// but there will be no function runs corresponding to times 50 or 100. This is
|
||||
// especially important to remember when using an environment with a simulated
|
||||
// clock: advancing simulated time atomically over N interval_micross will not
|
||||
// cause the function to be called N times.
|
||||
//
|
||||
// This object is thread-safe.
|
||||
//
|
||||
// Example:
|
||||
//
|
||||
// class Foo {
|
||||
// public:
|
||||
// Foo() : periodic_function_([this]() { Bar(); },
|
||||
// 1000 /* 1000us == 1ms*/) {
|
||||
// }
|
||||
//
|
||||
// private:
|
||||
// void Bar() { ... }
|
||||
//
|
||||
// PeriodicFunction periodic_function_;
|
||||
// };
|
||||
|
||||
#ifndef TENSORFLOW_CORE_KERNELS_BATCHING_UTIL_PERIODIC_FUNCTION_H_
|
||||
#define TENSORFLOW_CORE_KERNELS_BATCHING_UTIL_PERIODIC_FUNCTION_H_
|
||||
|
||||
#include "tensorflow/core/kernels/batching_util/periodic_function.h"
|
||||
|
||||
#include <functional>
|
||||
#include <memory>
|
||||
#include <string>
|
||||
|
||||
#include "tensorflow/core/lib/core/notification.h"
|
||||
#include "tensorflow/core/platform/env.h"
|
||||
#include "tensorflow/core/platform/macros.h"
|
||||
#include "tensorflow/core/platform/mutex.h"
|
||||
#include "tensorflow/core/platform/thread_annotations.h"
|
||||
#include "tensorflow/core/platform/types.h"
|
||||
|
||||
namespace tensorflow {
|
||||
namespace serving {
|
||||
|
||||
namespace internal {
|
||||
class PeriodicFunctionTestAccess;
|
||||
}
|
||||
|
||||
class PeriodicFunction {
|
||||
public:
|
||||
// Provides the ability to customize several aspects of the PeriodicFunction.
|
||||
// Passed to constructor of PeriodicFunction.
|
||||
struct Options {
|
||||
Options() {}
|
||||
|
||||
// Any standard thread options, such as stack size, should
|
||||
// be passed via "thread_options".
|
||||
ThreadOptions thread_options;
|
||||
|
||||
// Specifies the thread name prefix (see the description in class
|
||||
// Thread).
|
||||
string thread_name_prefix = "periodic_function";
|
||||
|
||||
// The environment to use. Does not take ownership, but must remain alive
|
||||
// for as long as the PeriodicFunction exists.
|
||||
Env* env = Env::Default();
|
||||
|
||||
// Specifies the length of sleep before the first invocation of the
|
||||
// function.
|
||||
// This can be used for adding a random jitter to avoid synchronous behavior
|
||||
// across multiple periodic functions.
|
||||
int64 startup_delay_micros = 0;
|
||||
};
|
||||
|
||||
// Also starts the background thread which will be calling the function.
|
||||
PeriodicFunction(const std::function<void()>& function, int64 interval_micros,
|
||||
const Options& options = Options());
|
||||
|
||||
~PeriodicFunction();
|
||||
|
||||
private:
|
||||
friend class internal::PeriodicFunctionTestAccess;
|
||||
|
||||
// Notifies the background thread to stop.
|
||||
void NotifyStop();
|
||||
|
||||
// (Blocking.) Loops forever calling "function_" every "interval_micros_".
|
||||
void RunLoop(int64 start) LOCKS_EXCLUDED(mutex_);
|
||||
|
||||
const std::function<void()> function_; // Actual client function
|
||||
const int64 interval_micros_; // Interval between calls.
|
||||
const Options options_;
|
||||
|
||||
// Protects state below.
|
||||
mutable mutex mutex_;
|
||||
// Used to notify the thread to stop.
|
||||
Notification stop_thread_;
|
||||
|
||||
// Thread for running "function_"
|
||||
std::unique_ptr<Thread> thread_ GUARDED_BY(mutex_) = nullptr;
|
||||
|
||||
TF_DISALLOW_COPY_AND_ASSIGN(PeriodicFunction);
|
||||
};
|
||||
|
||||
} // namespace serving
|
||||
} // namespace tensorflow
|
||||
|
||||
#endif // TENSORFLOW_CORE_KERNELS_BATCHING_UTIL_PERIODIC_FUNCTION_H_
|
||||
@@ -0,0 +1,706 @@
|
||||
/* Copyright 2016 The TensorFlow Authors. All Rights Reserved.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
==============================================================================*/
|
||||
|
||||
#ifndef TENSORFLOW_CORE_KERNELS_BATCHING_UTIL_SHARED_BATCH_SCHEDULER_H_
|
||||
#define TENSORFLOW_CORE_KERNELS_BATCHING_UTIL_SHARED_BATCH_SCHEDULER_H_
|
||||
|
||||
#include <stddef.h>
|
||||
#include <deque>
|
||||
#include <functional>
|
||||
#include <list>
|
||||
#include <memory>
|
||||
#include <string>
|
||||
#include <utility>
|
||||
#include <vector>
|
||||
|
||||
#include "tensorflow/core/kernels/batching_util/batch_scheduler.h"
|
||||
#include "tensorflow/core/kernels/batching_util/periodic_function.h"
|
||||
#include "tensorflow/core/lib/core/errors.h"
|
||||
#include "tensorflow/core/lib/core/status.h"
|
||||
#include "tensorflow/core/lib/strings/strcat.h"
|
||||
#include "tensorflow/core/platform/byte_order.h"
|
||||
#include "tensorflow/core/platform/cpu_info.h"
|
||||
#include "tensorflow/core/platform/env.h"
|
||||
#include "tensorflow/core/platform/thread_annotations.h"
|
||||
#include "tensorflow/core/platform/types.h"
|
||||
|
||||
namespace tensorflow {
|
||||
namespace serving {
|
||||
namespace internal {
|
||||
template <typename TaskType>
|
||||
class Queue;
|
||||
} // namespace internal
|
||||
} // namespace serving
|
||||
} // namespace tensorflow
|
||||
|
||||
namespace tensorflow {
|
||||
namespace serving {
|
||||
|
||||
// A batch scheduler for server instances that service multiple request types
|
||||
// (e.g. multiple machine-learned models, or multiple versions of a model served
|
||||
// concurrently), or even multiple distinct tasks for a given request. The
|
||||
// scheduler multiplexes batches of different kinds of tasks onto a fixed-size
|
||||
// thread pool (each batch contains tasks of a single type), in a carefully
|
||||
// controlled manner. A common configuration is to set the number of threads
|
||||
// equal to the number of hardware accelerator units, in which case the
|
||||
// scheduler takes care of multiplexing the task types onto the shared hardware,
|
||||
// in a manner that is both fair and efficient.
|
||||
//
|
||||
// Semantically, SharedBatchScheduler behaves like having N instances of
|
||||
// BasicBatchScheduler (see basic_batch_scheduler.h), one per task type. The
|
||||
// difference is that under the covers there is a single shared thread pool,
|
||||
// instead of N independent ones, with their sharing deliberately coordinated.
|
||||
//
|
||||
// SharedBatchScheduler does not implement the BatchScheduler API; rather, it
|
||||
// presents an abstraction of "queues", where each queue corresponds to one type
|
||||
// of task. Tasks submitted to a given queue are placed in their own batches,
|
||||
// and cannot be mixed with other tasks. Queues can be added and deleted
|
||||
// dynamically, to accommodate e.g. versions of a model being brought up and
|
||||
// down over the lifetime of a server.
|
||||
//
|
||||
// The batch thread pool round-robins through the queues, running one batch
|
||||
// from a queue and then moving to the next queue. Each queue behaves like a
|
||||
// BasicBatchScheduler instance, in the sense that it has maximum batch size and
|
||||
// timeout parameters, which govern when a batch is eligible to be processed.
|
||||
//
|
||||
// Each queue is independently configured with a maximum size (in terms of the
|
||||
// maximum number of batches worth of enqueued tasks). For online serving, it is
|
||||
// recommended that the queue sizes be configured such that the sum of the sizes
|
||||
// of the active queues roughly equal the number of batch threads. (The idea is
|
||||
// that if all threads become available at roughly the same time, there will be
|
||||
// enough enqueued work for them to take on, but no more.)
|
||||
//
|
||||
// If queue sizes are configured in the manner suggested above, the maximum time
|
||||
// a task can spend in a queue before being placed in a batch and assigned to a
|
||||
// thread for processing, is the greater of:
|
||||
// - the maximum time to process one batch of tasks from any active queue
|
||||
// - the configured timeout parameter for the task's queue (which can be 0)
|
||||
//
|
||||
// For bulk processing jobs and throughput-oriented benchmarks, you may want to
|
||||
// set the maximum queue size to a large value.
|
||||
//
|
||||
// TODO(b/26539183): Support queue servicing policies other than round-robin.
|
||||
// E.g. let each queue specify a "share" (an int >= 1), so e.g. with queues A
|
||||
// and B having shares 1 and 2 respectively, the servicing pattern is ABBABB...
|
||||
//
|
||||
//
|
||||
// PERFORMANCE TUNING: See README.md.
|
||||
//
|
||||
template <typename TaskType>
|
||||
class SharedBatchScheduler
|
||||
: public std::enable_shared_from_this<SharedBatchScheduler<TaskType>> {
|
||||
public:
|
||||
// TODO(b/25089730): Tune defaults based on best practices as they develop.
|
||||
struct Options {
|
||||
// The name to use for the pool of batch threads.
|
||||
string thread_pool_name = {"batch_threads"};
|
||||
|
||||
// The number of threads to use to process batches.
|
||||
// Must be >= 1, and should be tuned carefully.
|
||||
int num_batch_threads = port::NumSchedulableCPUs();
|
||||
|
||||
// The environment to use.
|
||||
// (Typically only overridden by test code.)
|
||||
Env* env = Env::Default();
|
||||
};
|
||||
// Ownership is shared between the caller of Create() and any queues created
|
||||
// via AddQueue().
|
||||
static Status Create(
|
||||
const Options& options,
|
||||
std::shared_ptr<SharedBatchScheduler<TaskType>>* scheduler);
|
||||
|
||||
~SharedBatchScheduler();
|
||||
|
||||
// Adds a queue to which tasks may be submitted. The returned queue implements
|
||||
// the BatchScheduler API. Each queue has its own set of scheduling options,
|
||||
// and its own callback to process batches of tasks submitted to the queue.
|
||||
//
|
||||
// The returned queue's destructor blocks until all tasks submitted to it have
|
||||
// been processed.
|
||||
struct QueueOptions {
|
||||
// The maximum size of each batch.
|
||||
//
|
||||
// The scheduler may form batches of any size between 1 and this number
|
||||
// (inclusive). If there is a need to quantize the batch sizes, i.e. only
|
||||
// submit batches whose size is in a small set of allowed sizes, that can be
|
||||
// done by adding padding in the process-batch callback.
|
||||
size_t max_batch_size = 1000;
|
||||
|
||||
// If a task has been enqueued for this amount of time (in microseconds),
|
||||
// and a thread is available, the scheduler will immediately form a batch
|
||||
// from enqueued tasks and assign the batch to the thread for processing,
|
||||
// even if the batch's size is below 'max_batch_size'.
|
||||
//
|
||||
// This parameter offers a way to bound queue latency, so that a task isn't
|
||||
// stuck in the queue indefinitely waiting for enough tasks to arrive to
|
||||
// make a full batch. (The latency bound is given in the class documentation
|
||||
// above.)
|
||||
//
|
||||
// The goal is to smooth out batch sizes under low request rates, and thus
|
||||
// avoid latency spikes.
|
||||
int64 batch_timeout_micros = 0;
|
||||
|
||||
// The maximum allowable number of enqueued (accepted by Schedule() but
|
||||
// not yet being processed on a batch thread) tasks in terms of batches.
|
||||
// If this limit is reached, Schedule() will return an UNAVAILABLE error.
|
||||
// See the class documentation above for guidelines on how to tune this
|
||||
// parameter.
|
||||
size_t max_enqueued_batches = 10;
|
||||
};
|
||||
Status AddQueue(const QueueOptions& options,
|
||||
std::function<void(std::unique_ptr<Batch<TaskType>>)>
|
||||
process_batch_callback,
|
||||
std::unique_ptr<BatchScheduler<TaskType>>* queue);
|
||||
|
||||
private:
|
||||
explicit SharedBatchScheduler(const Options& options);
|
||||
|
||||
// The code executed in 'batch_threads_'. Obtains a batch to process from the
|
||||
// queue pointed to by 'next_queue_to_schedule_', and processes it. If that
|
||||
// queue declines to provide a batch to process, moves onto the next queue. If
|
||||
// no queues provide a batch to process, just sleeps briefly and exits.
|
||||
void ThreadLogic();
|
||||
|
||||
const Options options_;
|
||||
|
||||
mutex mu_;
|
||||
|
||||
// A list of queues. (We use std::list instead of std::vector to ensure that
|
||||
// iterators are not invalidated by adding/removing elements. It also offers
|
||||
// efficient removal of elements from the middle.)
|
||||
using QueueList = std::list<std::unique_ptr<internal::Queue<TaskType>>>;
|
||||
|
||||
// All "active" queues, i.e. ones that either:
|
||||
// - have not been removed, or
|
||||
// - have been removed but are not yet empty.
|
||||
QueueList queues_ GUARDED_BY(mu_);
|
||||
|
||||
// An iterator over 'queues_', pointing to the queue from which the next
|
||||
// available batch thread should grab work.
|
||||
typename QueueList::iterator next_queue_to_schedule_ GUARDED_BY(mu_);
|
||||
|
||||
// Used by idle batch threads to wait for work to enter the system. Notified
|
||||
// whenever a batch becomes schedulable.
|
||||
condition_variable schedulable_batch_cv_;
|
||||
|
||||
// Threads that process batches obtained from the queues.
|
||||
std::vector<std::unique_ptr<PeriodicFunction>> batch_threads_;
|
||||
|
||||
TF_DISALLOW_COPY_AND_ASSIGN(SharedBatchScheduler);
|
||||
};
|
||||
|
||||
//////////
|
||||
// Implementation details follow. API users need not read.
|
||||
|
||||
namespace internal {
|
||||
|
||||
// A task queue for SharedBatchScheduler. Accepts tasks and accumulates them
|
||||
// into batches, and dispenses those batches to be processed via a "pull"
|
||||
// interface. The queue's behavior is governed by maximum batch size, timeout
|
||||
// and maximum queue length parameters; see their documentation in
|
||||
// SharedBatchScheduler.
|
||||
//
|
||||
// The queue is implemented as a deque of batches, with these invariants:
|
||||
// - The number of batches is between 1 and 'options_.max_enqueued_batches'.
|
||||
// - The back-most batch is open; the rest are closed.
|
||||
//
|
||||
// Submitted tasks are added to the open batch. If that batch doesn't have room
|
||||
// but the queue isn't full, then that batch is closed and a new open batch is
|
||||
// started.
|
||||
//
|
||||
// Batch pull requests are handled by dequeuing the front-most batch if it is
|
||||
// closed. If the front-most batch is open (i.e. the queue contains only one
|
||||
// batch) and has reached the timeout, it is immediately closed and returned;
|
||||
// otherwise no batch is returned for the request.
|
||||
template <typename TaskType>
|
||||
class Queue {
|
||||
public:
|
||||
using ProcessBatchCallback =
|
||||
std::function<void(std::unique_ptr<Batch<TaskType>>)>;
|
||||
using SchedulableBatchCallback = std::function<void()>;
|
||||
Queue(const typename SharedBatchScheduler<TaskType>::QueueOptions& options,
|
||||
Env* env, ProcessBatchCallback process_batch_callback,
|
||||
SchedulableBatchCallback schdulable_batch_callback);
|
||||
|
||||
// Illegal to destruct unless the queue is empty.
|
||||
~Queue();
|
||||
|
||||
// Submits a task to the queue, with the same semantics as
|
||||
// BatchScheduler::Schedule().
|
||||
Status Schedule(std::unique_ptr<TaskType>* task);
|
||||
|
||||
// Returns the number of enqueued tasks, with the same semantics as
|
||||
// BatchScheduler::NumEnqueuedTasks().
|
||||
size_t NumEnqueuedTasks() const;
|
||||
|
||||
// Returns the queue capacity, with the same semantics as
|
||||
// BatchScheduler::SchedulingCapacity().
|
||||
size_t SchedulingCapacity() const;
|
||||
|
||||
// Returns the maximum allowed size of tasks submitted to the queue.
|
||||
size_t max_task_size() const { return options_.max_batch_size; }
|
||||
|
||||
// Called by a thread that is ready to process a batch, to request one from
|
||||
// this queue. Either returns a batch that is ready to be processed, or
|
||||
// nullptr if the queue declines to schedule a batch at this time. If it
|
||||
// returns a batch, the batch is guaranteed to be closed.
|
||||
std::unique_ptr<Batch<TaskType>> ScheduleBatch();
|
||||
|
||||
// Processes a batch that has been returned earlier by ScheduleBatch().
|
||||
void ProcessBatch(std::unique_ptr<Batch<TaskType>> batch);
|
||||
|
||||
// Determines whether the queue is empty, i.e. has no tasks waiting or being
|
||||
// processed.
|
||||
bool IsEmpty() const;
|
||||
|
||||
// Marks the queue closed, and waits until it is empty.
|
||||
void CloseAndWaitUntilEmpty();
|
||||
|
||||
bool closed() const {
|
||||
mutex_lock l(mu_);
|
||||
return closed_;
|
||||
}
|
||||
|
||||
private:
|
||||
// Same as IsEmpty(), but assumes the caller already holds a lock on 'mu_'.
|
||||
bool IsEmptyInternal() const EXCLUSIVE_LOCKS_REQUIRED(mu_);
|
||||
|
||||
// Closes the open batch residing at the back of 'batches_', and inserts a
|
||||
// fresh open batch behind it.
|
||||
void StartNewBatch() EXCLUSIVE_LOCKS_REQUIRED(mu_);
|
||||
|
||||
// Determines whether the open batch residing at the back of 'batches_' is
|
||||
// currently schedulable.
|
||||
bool IsOpenBatchSchedulable() const EXCLUSIVE_LOCKS_REQUIRED(mu_);
|
||||
|
||||
const typename SharedBatchScheduler<TaskType>::QueueOptions options_;
|
||||
|
||||
// The environment to use.
|
||||
Env* env_;
|
||||
|
||||
// A callback invoked to processes a batch of work units. Always invoked from
|
||||
// a batch thread.
|
||||
ProcessBatchCallback process_batch_callback_;
|
||||
|
||||
// A callback invoked to notify the scheduler that a new batch has become
|
||||
// schedulable.
|
||||
SchedulableBatchCallback schedulable_batch_callback_;
|
||||
|
||||
mutable mutex mu_;
|
||||
|
||||
// Whether this queue can accept new tasks. This variable is monotonic: it
|
||||
// starts as false, and then at some point gets set to true and remains true
|
||||
// for the duration of this object's life.
|
||||
bool closed_ GUARDED_BY(mu_) = false;
|
||||
|
||||
// The enqueued batches. See the invariants in the class comments above.
|
||||
std::deque<std::unique_ptr<Batch<TaskType>>> batches_ GUARDED_BY(mu_);
|
||||
|
||||
// The time at which the first task was added to the open (back-most) batch
|
||||
// in 'batches_'. Valid iff that batch contains at least one task.
|
||||
uint64 open_batch_start_time_micros_ GUARDED_BY(mu_);
|
||||
|
||||
// Whether this queue contains a batch that is eligible to be scheduled. Used
|
||||
// to keep track of when to call 'schedulable_batch_callback_'.
|
||||
bool schedulable_batch_ GUARDED_BY(mu_) = false;
|
||||
|
||||
// The number of batches currently being processed by batch threads.
|
||||
// Incremented in ScheduleBatch() and decremented in ProcessBatch().
|
||||
int num_batches_being_processed_ GUARDED_BY(mu_) = 0;
|
||||
|
||||
// Used by CloseAndWaitUntilEmpty() to wait until the queue is empty, for the
|
||||
// case in which the queue is not empty when CloseAndWaitUntilEmpty() starts.
|
||||
// When ProcessBatch() dequeues the last batch and makes the queue empty, if
|
||||
// 'empty_notification_' is non-null it calls 'empty_notification_->Notify()'.
|
||||
Notification* empty_notification_ GUARDED_BY(mu_) = nullptr;
|
||||
|
||||
TF_DISALLOW_COPY_AND_ASSIGN(Queue);
|
||||
};
|
||||
|
||||
// A RAII-style object that points to a Queue and implements
|
||||
// the BatchScheduler API. To be handed out to clients who call AddQueue().
|
||||
template <typename TaskType>
|
||||
class QueueHandle : public BatchScheduler<TaskType> {
|
||||
public:
|
||||
QueueHandle(std::shared_ptr<SharedBatchScheduler<TaskType>> scheduler,
|
||||
Queue<TaskType>* queue);
|
||||
~QueueHandle() override;
|
||||
|
||||
Status Schedule(std::unique_ptr<TaskType>* task) override;
|
||||
size_t NumEnqueuedTasks() const override;
|
||||
size_t SchedulingCapacity() const override;
|
||||
|
||||
size_t max_task_size() const override { return queue_->max_task_size(); }
|
||||
|
||||
private:
|
||||
// The scheduler that owns 'queue_'.
|
||||
std::shared_ptr<SharedBatchScheduler<TaskType>> scheduler_;
|
||||
|
||||
// The queue this handle wraps. Owned by 'scheduler_', which keeps it alive at
|
||||
// least until this class's destructor closes it.
|
||||
Queue<TaskType>* queue_;
|
||||
|
||||
TF_DISALLOW_COPY_AND_ASSIGN(QueueHandle);
|
||||
};
|
||||
|
||||
} // namespace internal
|
||||
|
||||
template <typename TaskType>
|
||||
Status SharedBatchScheduler<TaskType>::Create(
|
||||
const Options& options,
|
||||
std::shared_ptr<SharedBatchScheduler<TaskType>>* scheduler) {
|
||||
if (options.num_batch_threads < 1) {
|
||||
return errors::InvalidArgument("num_batch_threads must be positive; was ",
|
||||
options.num_batch_threads);
|
||||
}
|
||||
scheduler->reset(new SharedBatchScheduler<TaskType>(options));
|
||||
return Status::OK();
|
||||
}
|
||||
|
||||
template <typename TaskType>
|
||||
SharedBatchScheduler<TaskType>::~SharedBatchScheduler() {
|
||||
// Wait until the batch threads finish clearing out and deleting the closed
|
||||
// queues.
|
||||
for (;;) {
|
||||
{
|
||||
mutex_lock l(mu_);
|
||||
if (queues_.empty()) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
const int64 kSleepTimeMicros = 100;
|
||||
options_.env->SleepForMicroseconds(kSleepTimeMicros);
|
||||
}
|
||||
// Delete the batch threads before allowing state the threads may access (e.g.
|
||||
// 'mu_') to be deleted.
|
||||
batch_threads_.clear();
|
||||
}
|
||||
|
||||
template <typename TaskType>
|
||||
Status SharedBatchScheduler<TaskType>::AddQueue(
|
||||
const QueueOptions& options,
|
||||
std::function<void(std::unique_ptr<Batch<TaskType>>)>
|
||||
process_batch_callback,
|
||||
std::unique_ptr<BatchScheduler<TaskType>>* queue) {
|
||||
if (options.max_batch_size == 0) {
|
||||
return errors::InvalidArgument("max_batch_size must be positive; was ",
|
||||
options.max_batch_size);
|
||||
}
|
||||
if (options.batch_timeout_micros < 0) {
|
||||
return errors::InvalidArgument(
|
||||
"batch_timeout_micros must be non-negative; was ",
|
||||
options.batch_timeout_micros);
|
||||
}
|
||||
if (options.max_enqueued_batches < 0) {
|
||||
return errors::InvalidArgument(
|
||||
"max_enqueued_batches must be non-negative; was ",
|
||||
options.max_enqueued_batches);
|
||||
}
|
||||
|
||||
auto schedulable_batch_callback = [this] {
|
||||
mutex_lock l(mu_);
|
||||
schedulable_batch_cv_.notify_one();
|
||||
};
|
||||
auto internal_queue =
|
||||
std::unique_ptr<internal::Queue<TaskType>>(new internal::Queue<TaskType>(
|
||||
options, options_.env, process_batch_callback,
|
||||
schedulable_batch_callback));
|
||||
auto handle = std::unique_ptr<BatchScheduler<TaskType>>(
|
||||
new internal::QueueHandle<TaskType>(this->shared_from_this(),
|
||||
internal_queue.get()));
|
||||
{
|
||||
mutex_lock l(mu_);
|
||||
queues_.push_back(std::move(internal_queue));
|
||||
if (next_queue_to_schedule_ == queues_.end()) {
|
||||
next_queue_to_schedule_ = queues_.begin();
|
||||
}
|
||||
}
|
||||
*queue = std::move(handle);
|
||||
return Status::OK();
|
||||
}
|
||||
|
||||
template <typename TaskType>
|
||||
SharedBatchScheduler<TaskType>::SharedBatchScheduler(const Options& options)
|
||||
: options_(options), next_queue_to_schedule_(queues_.end()) {
|
||||
// Kick off the batch threads.
|
||||
PeriodicFunction::Options periodic_fn_options;
|
||||
periodic_fn_options.thread_name_prefix =
|
||||
strings::StrCat(options.thread_pool_name, "_");
|
||||
for (int i = 0; i < options.num_batch_threads; ++i) {
|
||||
std::unique_ptr<PeriodicFunction> thread(new PeriodicFunction(
|
||||
[this] { this->ThreadLogic(); },
|
||||
0 /* function invocation interval time */, periodic_fn_options));
|
||||
batch_threads_.push_back(std::move(thread));
|
||||
}
|
||||
}
|
||||
|
||||
template <typename TaskType>
|
||||
void SharedBatchScheduler<TaskType>::ThreadLogic() {
|
||||
// A batch to process next (or nullptr if no work to do).
|
||||
std::unique_ptr<Batch<TaskType>> batch_to_process;
|
||||
// The queue with which 'batch_to_process' is associated.
|
||||
internal::Queue<TaskType>* queue_for_batch = nullptr;
|
||||
{
|
||||
mutex_lock l(mu_);
|
||||
|
||||
const int num_queues = queues_.size();
|
||||
for (int num_queues_tried = 0;
|
||||
batch_to_process == nullptr && num_queues_tried < num_queues;
|
||||
++num_queues_tried) {
|
||||
DCHECK(next_queue_to_schedule_ != queues_.end());
|
||||
|
||||
// If a closed queue responds to ScheduleBatch() with nullptr, the queue
|
||||
// will never yield any further batches so we can drop it. To avoid a
|
||||
// race, we take a snapshot of the queue's closedness state *before*
|
||||
// calling ScheduleBatch().
|
||||
const bool queue_closed = (*next_queue_to_schedule_)->closed();
|
||||
|
||||
// Ask '*next_queue_to_schedule_' if it wants us to process a batch.
|
||||
batch_to_process = (*next_queue_to_schedule_)->ScheduleBatch();
|
||||
if (batch_to_process != nullptr) {
|
||||
queue_for_batch = next_queue_to_schedule_->get();
|
||||
}
|
||||
|
||||
// Advance 'next_queue_to_schedule_'.
|
||||
if (queue_closed && (*next_queue_to_schedule_)->IsEmpty() &&
|
||||
batch_to_process == nullptr) {
|
||||
// We've encountered a closed queue with no work to do. Drop it.
|
||||
DCHECK_NE(queue_for_batch, next_queue_to_schedule_->get());
|
||||
next_queue_to_schedule_ = queues_.erase(next_queue_to_schedule_);
|
||||
} else {
|
||||
++next_queue_to_schedule_;
|
||||
}
|
||||
if (next_queue_to_schedule_ == queues_.end() && !queues_.empty()) {
|
||||
// We've hit the end. Wrap to the first queue.
|
||||
next_queue_to_schedule_ = queues_.begin();
|
||||
}
|
||||
}
|
||||
|
||||
if (batch_to_process == nullptr) {
|
||||
// We couldn't find any work to do. Wait until a new batch becomes
|
||||
// schedulable, or some time has elapsed, before checking again.
|
||||
const int64 kTimeoutMillis = 1; // The smallest accepted granule of time.
|
||||
WaitForMilliseconds(&l, &schedulable_batch_cv_, kTimeoutMillis);
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
queue_for_batch->ProcessBatch(std::move(batch_to_process));
|
||||
}
|
||||
|
||||
namespace internal {
|
||||
|
||||
template <typename TaskType>
|
||||
Queue<TaskType>::Queue(
|
||||
const typename SharedBatchScheduler<TaskType>::QueueOptions& options,
|
||||
Env* env, ProcessBatchCallback process_batch_callback,
|
||||
SchedulableBatchCallback schedulable_batch_callback)
|
||||
: options_(options),
|
||||
env_(env),
|
||||
process_batch_callback_(process_batch_callback),
|
||||
schedulable_batch_callback_(schedulable_batch_callback) {
|
||||
// Create an initial, open batch.
|
||||
batches_.emplace_back(new Batch<TaskType>);
|
||||
}
|
||||
|
||||
template <typename TaskType>
|
||||
Queue<TaskType>::~Queue() {
|
||||
mutex_lock l(mu_);
|
||||
DCHECK(IsEmptyInternal());
|
||||
|
||||
// Close the (empty) open batch, so its destructor doesn't block.
|
||||
batches_.back()->Close();
|
||||
}
|
||||
|
||||
template <typename TaskType>
|
||||
Status Queue<TaskType>::Schedule(std::unique_ptr<TaskType>* task) {
|
||||
if ((*task)->size() > options_.max_batch_size) {
|
||||
return errors::InvalidArgument("Task size ", (*task)->size(),
|
||||
" is larger than maximum batch size ",
|
||||
options_.max_batch_size);
|
||||
}
|
||||
|
||||
bool notify_of_schedulable_batch = false;
|
||||
{
|
||||
mutex_lock l(mu_);
|
||||
|
||||
DCHECK(!closed_);
|
||||
|
||||
if (batches_.back()->size() + (*task)->size() > options_.max_batch_size) {
|
||||
if (batches_.size() >= options_.max_enqueued_batches) {
|
||||
return errors::Unavailable(
|
||||
"The batch scheduling queue to which this task was submitted is "
|
||||
"full");
|
||||
}
|
||||
StartNewBatch();
|
||||
}
|
||||
if (batches_.back()->empty()) {
|
||||
open_batch_start_time_micros_ = env_->NowMicros();
|
||||
}
|
||||
batches_.back()->AddTask(std::move(*task));
|
||||
|
||||
if (!schedulable_batch_) {
|
||||
if (batches_.size() > 1 || IsOpenBatchSchedulable()) {
|
||||
schedulable_batch_ = true;
|
||||
notify_of_schedulable_batch = true;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (notify_of_schedulable_batch) {
|
||||
schedulable_batch_callback_();
|
||||
}
|
||||
|
||||
return Status::OK();
|
||||
}
|
||||
|
||||
template <typename TaskType>
|
||||
size_t Queue<TaskType>::NumEnqueuedTasks() const {
|
||||
mutex_lock l(mu_);
|
||||
size_t num_enqueued_tasks = 0;
|
||||
for (const auto& batch : batches_) {
|
||||
num_enqueued_tasks += batch->num_tasks();
|
||||
}
|
||||
return num_enqueued_tasks;
|
||||
}
|
||||
|
||||
template <typename TaskType>
|
||||
size_t Queue<TaskType>::SchedulingCapacity() const {
|
||||
mutex_lock l(mu_);
|
||||
const int num_new_batches_schedulable =
|
||||
options_.max_enqueued_batches - batches_.size();
|
||||
const int open_batch_capacity =
|
||||
options_.max_batch_size - batches_.back()->size();
|
||||
return (num_new_batches_schedulable * options_.max_batch_size) +
|
||||
open_batch_capacity;
|
||||
}
|
||||
|
||||
template <typename TaskType>
|
||||
std::unique_ptr<Batch<TaskType>> Queue<TaskType>::ScheduleBatch() {
|
||||
// The batch to schedule, which we may populate below. (If left as nullptr,
|
||||
// that means we are electing not to schedule a batch at this time.)
|
||||
std::unique_ptr<Batch<TaskType>> batch_to_schedule;
|
||||
|
||||
{
|
||||
mutex_lock l(mu_);
|
||||
|
||||
// Consider closing the open batch at this time, to schedule it.
|
||||
if (batches_.size() == 1 && IsOpenBatchSchedulable()) {
|
||||
StartNewBatch();
|
||||
}
|
||||
|
||||
if (batches_.size() >= 2) {
|
||||
// There is at least one closed batch that is ready to be scheduled.
|
||||
++num_batches_being_processed_;
|
||||
batch_to_schedule = std::move(batches_.front());
|
||||
batches_.pop_front();
|
||||
} else {
|
||||
schedulable_batch_ = false;
|
||||
}
|
||||
}
|
||||
|
||||
return batch_to_schedule;
|
||||
}
|
||||
|
||||
template <typename TaskType>
|
||||
void Queue<TaskType>::ProcessBatch(std::unique_ptr<Batch<TaskType>> batch) {
|
||||
process_batch_callback_(std::move(batch));
|
||||
|
||||
{
|
||||
mutex_lock l(mu_);
|
||||
--num_batches_being_processed_;
|
||||
if (empty_notification_ != nullptr && IsEmptyInternal()) {
|
||||
empty_notification_->Notify();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
template <typename TaskType>
|
||||
bool Queue<TaskType>::IsEmpty() const {
|
||||
mutex_lock l(mu_);
|
||||
return IsEmptyInternal();
|
||||
}
|
||||
|
||||
template <typename TaskType>
|
||||
void Queue<TaskType>::CloseAndWaitUntilEmpty() {
|
||||
Notification empty;
|
||||
{
|
||||
mutex_lock l(mu_);
|
||||
closed_ = true;
|
||||
if (IsEmptyInternal()) {
|
||||
empty.Notify();
|
||||
} else {
|
||||
// Arrange for ProcessBatch() to notify when the queue becomes empty.
|
||||
empty_notification_ = ∅
|
||||
}
|
||||
}
|
||||
empty.WaitForNotification();
|
||||
}
|
||||
|
||||
template <typename TaskType>
|
||||
bool Queue<TaskType>::IsEmptyInternal() const {
|
||||
return num_batches_being_processed_ == 0 && batches_.size() == 1 &&
|
||||
batches_.back()->empty();
|
||||
}
|
||||
|
||||
template <typename TaskType>
|
||||
void Queue<TaskType>::StartNewBatch() {
|
||||
batches_.back()->Close();
|
||||
batches_.emplace_back(new Batch<TaskType>);
|
||||
}
|
||||
|
||||
template <typename TaskType>
|
||||
bool Queue<TaskType>::IsOpenBatchSchedulable() const {
|
||||
Batch<TaskType>* open_batch = batches_.back().get();
|
||||
if (open_batch->empty()) {
|
||||
return false;
|
||||
}
|
||||
return closed_ || open_batch->size() >= options_.max_batch_size ||
|
||||
env_->NowMicros() >=
|
||||
open_batch_start_time_micros_ + options_.batch_timeout_micros;
|
||||
}
|
||||
|
||||
template <typename TaskType>
|
||||
QueueHandle<TaskType>::QueueHandle(
|
||||
std::shared_ptr<SharedBatchScheduler<TaskType>> scheduler,
|
||||
Queue<TaskType>* queue)
|
||||
: scheduler_(scheduler), queue_(queue) {}
|
||||
|
||||
template <typename TaskType>
|
||||
QueueHandle<TaskType>::~QueueHandle() {
|
||||
queue_->CloseAndWaitUntilEmpty();
|
||||
}
|
||||
|
||||
template <typename TaskType>
|
||||
Status QueueHandle<TaskType>::Schedule(std::unique_ptr<TaskType>* task) {
|
||||
return queue_->Schedule(task);
|
||||
}
|
||||
|
||||
template <typename TaskType>
|
||||
size_t QueueHandle<TaskType>::NumEnqueuedTasks() const {
|
||||
return queue_->NumEnqueuedTasks();
|
||||
}
|
||||
|
||||
template <typename TaskType>
|
||||
size_t QueueHandle<TaskType>::SchedulingCapacity() const {
|
||||
return queue_->SchedulingCapacity();
|
||||
}
|
||||
|
||||
} // namespace internal
|
||||
|
||||
} // namespace serving
|
||||
} // namespace tensorflow
|
||||
|
||||
#endif // TENSORFLOW_CORE_KERNELS_BATCHING_UTIL_SHARED_BATCH_SCHEDULER_H_
|
||||
@@ -0,0 +1,51 @@
|
||||
/* Copyright 2016 The TensorFlow Authors. All Rights Reserved.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
==============================================================================*/
|
||||
|
||||
#ifndef TENSORFLOW_CORE_KERNELS_BETAINC_OP_H_
|
||||
#define TENSORFLOW_CORE_KERNELS_BETAINC_OP_H_
|
||||
// Functor definition for BetaincOp, must be compilable by nvcc.
|
||||
|
||||
#include "third_party/eigen3/unsupported/Eigen/CXX11/Tensor"
|
||||
#include "tensorflow/core/framework/tensor_types.h"
|
||||
|
||||
namespace tensorflow {
|
||||
namespace functor {
|
||||
|
||||
// Functor used by BetaincOp to do the computations.
|
||||
template <typename Device, typename T, int NDIM>
|
||||
struct Betainc {
|
||||
void operator()(const Device& d, typename TTypes<T, NDIM>::ConstTensor a,
|
||||
typename TTypes<T, NDIM>::ConstTensor b,
|
||||
typename TTypes<T, NDIM>::ConstTensor x,
|
||||
typename TTypes<T, NDIM>::Tensor output) {
|
||||
output.device(d) = Eigen::betainc(a, b, x);
|
||||
}
|
||||
|
||||
void BCast(const Device& d, typename TTypes<T, NDIM>::ConstTensor a,
|
||||
const typename Eigen::array<Eigen::DenseIndex, NDIM>& bcast_a,
|
||||
typename TTypes<T, NDIM>::ConstTensor b,
|
||||
const typename Eigen::array<Eigen::DenseIndex, NDIM>& bcast_b,
|
||||
typename TTypes<T, NDIM>::ConstTensor x,
|
||||
const typename Eigen::array<Eigen::DenseIndex, NDIM>& bcast_x,
|
||||
typename TTypes<T, NDIM>::Tensor output) {
|
||||
output.device(d) = Eigen::betainc(
|
||||
a.broadcast(bcast_a), b.broadcast(bcast_b), x.broadcast(bcast_x));
|
||||
}
|
||||
};
|
||||
|
||||
} // namespace functor
|
||||
} // namespace tensorflow
|
||||
|
||||
#endif // TENSORFLOW_CORE_KERNELS_BETAINC_OP_H_
|
||||
@@ -0,0 +1,55 @@
|
||||
/* Copyright 2015 The TensorFlow Authors. All Rights Reserved.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
==============================================================================*/
|
||||
|
||||
#ifndef TENSORFLOW_CORE_KERNELS_BIAS_OP_H_
|
||||
#define TENSORFLOW_CORE_KERNELS_BIAS_OP_H_
|
||||
// Functor definition for BiasOp, must be compilable by nvcc.
|
||||
|
||||
#include "third_party/eigen3/unsupported/Eigen/CXX11/Tensor"
|
||||
#include "tensorflow/core/framework/tensor_types.h"
|
||||
|
||||
namespace tensorflow {
|
||||
namespace functor {
|
||||
|
||||
// Functor used by BiasOp to do the computations.
|
||||
template <typename Device, typename T, int Dims>
|
||||
struct Bias {
|
||||
// Add "bias" to "input", broadcasting it on all dimensions but the last one.
|
||||
void operator()(const Device& d, typename TTypes<T, Dims>::ConstTensor input,
|
||||
typename TTypes<T>::ConstVec bias,
|
||||
typename TTypes<T, Dims>::Tensor output) {
|
||||
if (input.size() >= INT_MAX) {
|
||||
const int64_t bias_size = bias.dimension(0);
|
||||
const int64_t rest_size = input.size() / bias_size;
|
||||
Eigen::DSizes<int64_t, 1> one_d(input.size());
|
||||
Eigen::DSizes<int64_t, 1> bcast(rest_size);
|
||||
output.reshape(one_d).device(d) =
|
||||
input.reshape(one_d) + bias.broadcast(bcast).reshape(one_d);
|
||||
} else {
|
||||
const int bias_size = bias.dimension(0);
|
||||
const int rest_size = input.size() / bias_size;
|
||||
Eigen::DSizes<int, 1> one_d(input.size());
|
||||
Eigen::DSizes<int, 1> bcast(rest_size);
|
||||
To32Bit(output).reshape(one_d).device(d) =
|
||||
To32Bit(input).reshape(one_d) +
|
||||
To32Bit(bias).broadcast(bcast).reshape(one_d);
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
} // namespace functor
|
||||
} // namespace tensorflow
|
||||
|
||||
#endif // TENSORFLOW_CORE_KERNELS_BIAS_OP_H_
|
||||
@@ -0,0 +1,80 @@
|
||||
/* Copyright 2015 The TensorFlow Authors. All Rights Reserved.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
==============================================================================*/
|
||||
|
||||
#ifndef TENSORFLOW_CORE_KERNELS_BIAS_OP_GPU_H_
|
||||
#define TENSORFLOW_CORE_KERNELS_BIAS_OP_GPU_H_
|
||||
|
||||
#define EIGEN_USE_GPU
|
||||
|
||||
#include "third_party/eigen3/unsupported/Eigen/CXX11/Tensor"
|
||||
#include "tensorflow/core/framework/op_kernel.h"
|
||||
#include "tensorflow/core/framework/tensor_types.h"
|
||||
#include "tensorflow/core/kernels/gpu_utils.h"
|
||||
#include "tensorflow/core/util/tensor_format.h"
|
||||
|
||||
namespace tensorflow {
|
||||
|
||||
typedef Eigen::GpuDevice GPUDevice;
|
||||
|
||||
template <typename T>
|
||||
struct BiasGPU {
|
||||
static void compute(const GPUDevice& d, const T* input, const T* bias,
|
||||
T* output, int32 batch, int32 height, int32 width,
|
||||
int32 channel, TensorFormat data_format);
|
||||
};
|
||||
|
||||
template <typename T>
|
||||
struct BiasGradGPU {
|
||||
static void compute(const GPUDevice& device, const T* output_backprop,
|
||||
T* bias_backprop, int32 batch, int32 height, int32 width,
|
||||
int32 channel, TensorFormat data_format);
|
||||
|
||||
static void DoRowReduction(OpKernelContext* context, T* output,
|
||||
const T* input, int rows, int cols);
|
||||
|
||||
static void DoColReduction(OpKernelContext* context, T* output,
|
||||
const T* input, int rows, int cols);
|
||||
};
|
||||
|
||||
enum class BiasAddGradGPUMode {
|
||||
kInvalid = 0,
|
||||
kNative = 1,
|
||||
kReduction = 2,
|
||||
};
|
||||
|
||||
// Describe the BiasGradGPU result from a perf experiment.
|
||||
//
|
||||
// Arguments:
|
||||
// algorithm: returns the method to use for bias add grad.
|
||||
// elapsed_time; returns the measured elapsed time in microseconds.
|
||||
class BiasGradGPUProfileResult {
|
||||
public:
|
||||
bool is_valid() const {
|
||||
return (algorithm_ != BiasAddGradGPUMode::kInvalid &&
|
||||
elapsed_time_ != std::numeric_limits<float>::max());
|
||||
}
|
||||
BiasAddGradGPUMode algorithm() const { return algorithm_; }
|
||||
void set_algorithm(BiasAddGradGPUMode val) { algorithm_ = val; }
|
||||
uint64 elapsed_time() const { return elapsed_time_; }
|
||||
void set_elapsed_time(uint64 val) { elapsed_time_ = val; }
|
||||
|
||||
private:
|
||||
BiasAddGradGPUMode algorithm_ = BiasAddGradGPUMode::kInvalid;
|
||||
uint64 elapsed_time_ = std::numeric_limits<uint64>::max();
|
||||
};
|
||||
|
||||
} // namespace tensorflow
|
||||
|
||||
#endif // TENSORFLOW_CORE_KERNELS_BIAS_OP_GPU_H_
|
||||
@@ -0,0 +1,41 @@
|
||||
/* Copyright 2015 The TensorFlow Authors. All Rights Reserved.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
==============================================================================*/
|
||||
|
||||
#ifndef TENSORFLOW_CORE_KERNELS_BINCOUNT_OP_H_
|
||||
#define TENSORFLOW_CORE_KERNELS_BINCOUNT_OP_H_
|
||||
|
||||
#include "third_party/eigen3/unsupported/Eigen/CXX11/Tensor"
|
||||
#include "tensorflow/core/framework/op_kernel.h"
|
||||
#include "tensorflow/core/framework/tensor_types.h"
|
||||
#include "tensorflow/core/framework/types.h"
|
||||
#include "tensorflow/core/lib/core/errors.h"
|
||||
|
||||
namespace tensorflow {
|
||||
|
||||
namespace functor {
|
||||
|
||||
template <typename Device, typename T>
|
||||
struct BincountFunctor {
|
||||
static Status Compute(OpKernelContext* context,
|
||||
const typename TTypes<int32, 1>::ConstTensor& arr,
|
||||
const typename TTypes<T, 1>::ConstTensor& weights,
|
||||
typename TTypes<T, 1>::Tensor& output);
|
||||
};
|
||||
|
||||
} // end namespace functor
|
||||
|
||||
} // end namespace tensorflow
|
||||
|
||||
#endif // TENSORFLOW_CORE_KERNELS_BINCOUNT_OP_H_
|
||||
@@ -0,0 +1,30 @@
|
||||
/* Copyright 2015 The TensorFlow Authors. All Rights Reserved.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
==============================================================================*/
|
||||
|
||||
// See docs in ../ops/array_ops.cc.
|
||||
|
||||
#ifndef TENSORFLOW_CORE_KERNELS_BITCAST_OP_H_
|
||||
#define TENSORFLOW_CORE_KERNELS_BITCAST_OP_H_
|
||||
|
||||
#include <string.h> // for memcpy
|
||||
|
||||
#include "tensorflow/core/framework/op_kernel.h"
|
||||
#include "tensorflow/core/framework/register_types.h"
|
||||
#include "tensorflow/core/framework/tensor.h"
|
||||
#include "tensorflow/core/framework/tensor_shape.h"
|
||||
#include "tensorflow/core/framework/types.h"
|
||||
#include "tensorflow/core/lib/core/casts.h"
|
||||
|
||||
#endif // TENSORFLOW_CORE_KERNELS_BITCAST_OP_H_
|
||||
File diff suppressed because it is too large.
Load diff
+96
@@ -0,0 +1,96 @@
|
||||
// Copyright 2017 The TensorFlow Authors. All Rights Reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
// =============================================================================
|
||||
#ifndef TENSORFLOW_CORE_KERNELS_BOOSTED_TREES_QUANTILES_QUANTILE_STREAM_RESOURCE_H_
|
||||
#define TENSORFLOW_CORE_KERNELS_BOOSTED_TREES_QUANTILES_QUANTILE_STREAM_RESOURCE_H_
|
||||
|
||||
#include <vector>
|
||||
#include "tensorflow/core/framework/resource_mgr.h"
|
||||
#include "tensorflow/core/kernels/boosted_trees/quantiles/weighted_quantiles_stream.h"
|
||||
#include "tensorflow/core/platform/macros.h"
|
||||
#include "tensorflow/core/platform/mutex.h"
|
||||
|
||||
namespace tensorflow {
|
||||
|
||||
using QuantileStream =
|
||||
boosted_trees::quantiles::WeightedQuantilesStream<float, float>;
|
||||
|
||||
// Quantile Stream Resource for a list of streams sharing the same number of
|
||||
// quantiles, maximum elements, and epsilon.
|
||||
class BoostedTreesQuantileStreamResource : public ResourceBase {
|
||||
public:
|
||||
BoostedTreesQuantileStreamResource(const float epsilon,
|
||||
const int64 max_elements,
|
||||
const int64 num_streams)
|
||||
: are_buckets_ready_(false),
|
||||
epsilon_(epsilon),
|
||||
num_streams_(num_streams),
|
||||
max_elements_(max_elements) {
|
||||
streams_.reserve(num_streams_);
|
||||
boundaries_.reserve(num_streams_);
|
||||
for (int64 idx = 0; idx < num_streams; ++idx) {
|
||||
streams_.push_back(QuantileStream(epsilon, max_elements));
|
||||
boundaries_.push_back(std::vector<float>());
|
||||
}
|
||||
}
|
||||
|
||||
string DebugString() override { return "QuantileStreamResource"; }
|
||||
|
||||
tensorflow::mutex* mutex() { return &mu_; }
|
||||
|
||||
QuantileStream* stream(const int64 index) { return &streams_[index]; }
|
||||
|
||||
const std::vector<float>& boundaries(const int64 index) {
|
||||
return boundaries_[index];
|
||||
}
|
||||
|
||||
void set_boundaries(const std::vector<float>& boundaries, const int64 index) {
|
||||
boundaries_[index] = boundaries;
|
||||
}
|
||||
|
||||
float epsilon() const { return epsilon_; }
|
||||
int64 num_streams() const { return num_streams_; }
|
||||
|
||||
bool are_buckets_ready() const { return are_buckets_ready_; }
|
||||
void set_buckets_ready(const bool are_buckets_ready) {
|
||||
are_buckets_ready_ = are_buckets_ready;
|
||||
}
|
||||
|
||||
private:
|
||||
~BoostedTreesQuantileStreamResource() override {}
|
||||
|
||||
// Mutex for the whole resource.
|
||||
tensorflow::mutex mu_;
|
||||
|
||||
// Quantile streams.
|
||||
std::vector<QuantileStream> streams_;
|
||||
|
||||
// Stores the boundaries. Same size as streams_.
|
||||
std::vector<std::vector<float>> boundaries_;
|
||||
|
||||
// Whether boundaries are created. Initially boundaries are empty until
|
||||
// set_boundaries are called.
|
||||
bool are_buckets_ready_;
|
||||
|
||||
const float epsilon_;
|
||||
const int64 num_streams_;
|
||||
// An upper-bound for the number of elements.
|
||||
int64 max_elements_;
|
||||
|
||||
TF_DISALLOW_COPY_AND_ASSIGN(BoostedTreesQuantileStreamResource);
|
||||
};
|
||||
|
||||
} // namespace tensorflow
|
||||
|
||||
#endif // TENSORFLOW_CORE_KERNELS_BOOSTED_TREES_QUANTILES_QUANTILE_STREAM_RESOURCE_H_
|
||||
+132
@@ -0,0 +1,132 @@
|
||||
// Copyright 2018 The TensorFlow Authors. All Rights Reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
// =============================================================================
|
||||
#ifndef TENSORFLOW_CORE_KERNELS_BOOSTED_TREES_QUANTILES_WEIGHTED_QUANTILES_BUFFER_H_
|
||||
#define TENSORFLOW_CORE_KERNELS_BOOSTED_TREES_QUANTILES_WEIGHTED_QUANTILES_BUFFER_H_
|
||||
|
||||
#include <algorithm>
|
||||
#include <unordered_map>
|
||||
#include <vector>
|
||||
|
||||
#include "tensorflow/core/platform/logging.h"
|
||||
#include "tensorflow/core/platform/types.h"
|
||||
|
||||
namespace tensorflow {
|
||||
namespace boosted_trees {
|
||||
namespace quantiles {
|
||||
|
||||
// Buffering container ideally suited for scenarios where we need
|
||||
// to sort and dedupe/compact fixed chunks of a stream of weighted elements.
|
||||
template <typename ValueType, typename WeightType,
|
||||
typename CompareFn = std::less<ValueType>>
|
||||
class WeightedQuantilesBuffer {
|
||||
public:
|
||||
struct BufferEntry {
|
||||
BufferEntry(ValueType v, WeightType w)
|
||||
: value(std::move(v)), weight(std::move(w)) {}
|
||||
BufferEntry() : value(), weight(0) {}
|
||||
|
||||
bool operator<(const BufferEntry& other) const {
|
||||
return kCompFn(value, other.value);
|
||||
}
|
||||
bool operator==(const BufferEntry& other) const {
|
||||
return value == other.value && weight == other.weight;
|
||||
}
|
||||
friend std::ostream& operator<<(std::ostream& strm,
|
||||
const BufferEntry& entry) {
|
||||
return strm << "{" << entry.value << ", " << entry.weight << "}";
|
||||
}
|
||||
ValueType value;
|
||||
WeightType weight;
|
||||
};
|
||||
|
||||
explicit WeightedQuantilesBuffer(int64 block_size, int64 max_elements)
|
||||
: max_size_(std::min(block_size << 1, max_elements)) {
|
||||
QCHECK(max_size_ > 0) << "Invalid buffer specification: (" << block_size
|
||||
<< ", " << max_elements << ")";
|
||||
vec_.reserve(max_size_);
|
||||
}
|
||||
|
||||
// Disallow copying as it's semantically non-sensical in the Squawd algorithm
|
||||
// but enable move semantics.
|
||||
WeightedQuantilesBuffer(const WeightedQuantilesBuffer& other) = delete;
|
||||
WeightedQuantilesBuffer& operator=(const WeightedQuantilesBuffer&) = delete;
|
||||
WeightedQuantilesBuffer(WeightedQuantilesBuffer&& other) = default;
|
||||
WeightedQuantilesBuffer& operator=(WeightedQuantilesBuffer&& other) = default;
|
||||
|
||||
// Push entry to buffer and maintain a compact representation within
|
||||
// pre-defined size limit.
|
||||
void PushEntry(ValueType value, WeightType weight) {
|
||||
// Callers are expected to act on a full compacted buffer after the
|
||||
// PushEntry call returns.
|
||||
QCHECK(!IsFull()) << "Buffer already full: " << max_size_;
|
||||
|
||||
// Ignore zero and negative weight entries.
|
||||
if (weight <= 0) {
|
||||
return;
|
||||
}
|
||||
|
||||
// Push back the entry to the buffer.
|
||||
vec_.push_back(BufferEntry(std::move(value), std::move(weight)));
|
||||
}
|
||||
|
||||
// Returns a sorted vector view of the base buffer and clears the buffer.
|
||||
// Callers should minimize how often this is called, ideally only right after
|
||||
// the buffer becomes full.
|
||||
std::vector<BufferEntry> GenerateEntryList() {
|
||||
std::vector<BufferEntry> ret;
|
||||
if (vec_.size() == 0) {
|
||||
return ret;
|
||||
}
|
||||
ret.swap(vec_);
|
||||
vec_.reserve(max_size_);
|
||||
std::sort(ret.begin(), ret.end());
|
||||
size_t num_entries = 0;
|
||||
for (size_t i = 1; i < ret.size(); ++i) {
|
||||
if (ret[i].value != ret[i - 1].value) {
|
||||
BufferEntry tmp = ret[i];
|
||||
++num_entries;
|
||||
ret[num_entries] = tmp;
|
||||
} else {
|
||||
ret[num_entries].weight += ret[i].weight;
|
||||
}
|
||||
}
|
||||
ret.resize(num_entries + 1);
|
||||
return ret;
|
||||
}
|
||||
|
||||
int64 Size() const { return vec_.size(); }
|
||||
bool IsFull() const { return vec_.size() >= max_size_; }
|
||||
void Clear() { vec_.clear(); }
|
||||
|
||||
private:
|
||||
using BufferVector = typename std::vector<BufferEntry>;
|
||||
|
||||
// Comparison function.
|
||||
static constexpr decltype(CompareFn()) kCompFn = CompareFn();
|
||||
|
||||
// Base buffer.
|
||||
size_t max_size_;
|
||||
BufferVector vec_;
|
||||
};
|
||||
|
||||
template <typename ValueType, typename WeightType, typename CompareFn>
|
||||
constexpr decltype(CompareFn())
|
||||
WeightedQuantilesBuffer<ValueType, WeightType, CompareFn>::kCompFn;
|
||||
|
||||
} // namespace quantiles
|
||||
} // namespace boosted_trees
|
||||
} // namespace tensorflow
|
||||
|
||||
#endif // TENSORFLOW_CORE_KERNELS_BOOSTED_TREES_QUANTILES_WEIGHTED_QUANTILES_BUFFER_H_
|
||||
+330
@@ -0,0 +1,330 @@
|
||||
// Copyright 2018 The TensorFlow Authors. All Rights Reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
// =============================================================================
|
||||
#ifndef TENSORFLOW_CORE_KERNELS_BOOSTED_TREES_QUANTILES_WEIGHTED_QUANTILES_STREAM_H_
|
||||
#define TENSORFLOW_CORE_KERNELS_BOOSTED_TREES_QUANTILES_WEIGHTED_QUANTILES_STREAM_H_
|
||||
|
||||
#include <cmath>
|
||||
#include <memory>
|
||||
#include <vector>
|
||||
|
||||
#include "tensorflow/core/kernels/boosted_trees/quantiles/weighted_quantiles_buffer.h"
|
||||
#include "tensorflow/core/kernels/boosted_trees/quantiles/weighted_quantiles_summary.h"
|
||||
#include "tensorflow/core/platform/types.h"
|
||||
|
||||
namespace tensorflow {
|
||||
namespace boosted_trees {
|
||||
namespace quantiles {
|
||||
|
||||
// Class to compute approximate quantiles with error bound guarantees for
|
||||
// weighted data sets.
|
||||
// This implementation is an adaptation of techniques from the following papers:
|
||||
// * (2001) Space-efficient online computation of quantile summaries.
|
||||
// * (2004) Power-conserving computation of order-statistics over
|
||||
// sensor networks.
|
||||
// * (2007) A fast algorithm for approximate quantiles in high speed
|
||||
// data streams.
|
||||
// * (2016) XGBoost: A Scalable Tree Boosting System.
|
||||
//
|
||||
// The key ideas at play are the following:
|
||||
// - Maintain an in-memory multi-level quantile summary in a way to guarantee
|
||||
// a maximum approximation error of eps * W per bucket where W is the total
|
||||
// weight across all points in the input dataset.
|
||||
// - Two base operations are defined: MERGE and COMPRESS. MERGE combines two
|
||||
// summaries guaranteeing a epsNew = max(eps1, eps2). COMPRESS compresses
|
||||
// a summary to b + 1 elements guaranteeing epsNew = epsOld + 1/b.
|
||||
// - b * sizeof(summary entry) must ideally be small enough to fit in an
|
||||
// average CPU L2 cache.
|
||||
// - To distribute this algorithm with maintaining error bounds, we need
|
||||
// the worker-computed summaries to have no more than eps / h error
|
||||
// where h is the height of the distributed computation graph which
|
||||
// is 2 for an MR with no combiner.
|
||||
//
|
||||
// We mainly want to max out IO bw by ensuring we're not compute-bound and
|
||||
// using a reasonable amount of RAM.
|
||||
//
|
||||
// Complexity:
|
||||
// Compute: O(n * log(1/eps * log(eps * n))).
|
||||
// Memory: O(1/eps * log^2(eps * n)) <- for one worker streaming through the
|
||||
// entire dataset.
|
||||
// An epsilon value of zero would make the algorithm extremely inefficent and
|
||||
// therefore, is disallowed.
|
||||
template <typename ValueType, typename WeightType,
|
||||
typename CompareFn = std::less<ValueType>>
|
||||
class WeightedQuantilesStream {
|
||||
public:
|
||||
using Buffer = WeightedQuantilesBuffer<ValueType, WeightType, CompareFn>;
|
||||
using BufferEntry = typename Buffer::BufferEntry;
|
||||
using Summary = WeightedQuantilesSummary<ValueType, WeightType, CompareFn>;
|
||||
using SummaryEntry = typename Summary::SummaryEntry;
|
||||
|
||||
explicit WeightedQuantilesStream(double eps, int64 max_elements)
|
||||
: eps_(eps), buffer_(1LL, 2LL), finalized_(false) {
|
||||
// See the class documentation. An epsilon value of zero could cause
|
||||
// perfoamance issues.
|
||||
QCHECK(eps > 0) << "An epsilon value of zero is not allowed.";
|
||||
std::tie(max_levels_, block_size_) = GetQuantileSpecs(eps, max_elements);
|
||||
buffer_ = Buffer(block_size_, max_elements);
|
||||
summary_levels_.reserve(max_levels_);
|
||||
}
|
||||
|
||||
// Disallow copy and assign but enable move semantics for the stream.
|
||||
WeightedQuantilesStream(const WeightedQuantilesStream& other) = delete;
|
||||
WeightedQuantilesStream& operator=(const WeightedQuantilesStream&) = delete;
|
||||
WeightedQuantilesStream(WeightedQuantilesStream&& other) = default;
|
||||
WeightedQuantilesStream& operator=(WeightedQuantilesStream&& other) = default;
|
||||
|
||||
// Pushes one entry while maintaining approximation error invariants.
|
||||
void PushEntry(const ValueType& value, const WeightType& weight) {
|
||||
// Validate state.
|
||||
QCHECK(!finalized_) << "Finalize() already called.";
|
||||
|
||||
// Push element to base buffer.
|
||||
buffer_.PushEntry(value, weight);
|
||||
|
||||
// When compacted buffer is full we need to compress
|
||||
// and push weighted quantile summary up the level chain.
|
||||
if (buffer_.IsFull()) {
|
||||
PushBuffer(buffer_);
|
||||
}
|
||||
}
|
||||
|
||||
// Pushes full buffer while maintaining approximation error invariants.
|
||||
void PushBuffer(Buffer& buffer) {
|
||||
// Validate state.
|
||||
QCHECK(!finalized_) << "Finalize() already called.";
|
||||
|
||||
// Create local compressed summary and propagate.
|
||||
local_summary_.BuildFromBufferEntries(buffer.GenerateEntryList());
|
||||
local_summary_.Compress(block_size_, eps_);
|
||||
PropagateLocalSummary();
|
||||
}
|
||||
|
||||
// Pushes full summary while maintaining approximation error invariants.
|
||||
void PushSummary(const std::vector<SummaryEntry>& summary) {
|
||||
// Validate state.
|
||||
QCHECK(!finalized_) << "Finalize() already called.";
|
||||
|
||||
// Create local compressed summary and propagate.
|
||||
local_summary_.BuildFromSummaryEntries(summary);
|
||||
local_summary_.Compress(block_size_, eps_);
|
||||
PropagateLocalSummary();
|
||||
}
|
||||
|
||||
// Flushes approximator and finalizes state.
|
||||
void Finalize() {
|
||||
// Validate state.
|
||||
QCHECK(!finalized_) << "Finalize() may only be called once.";
|
||||
|
||||
// Flush any remaining buffer elements.
|
||||
PushBuffer(buffer_);
|
||||
|
||||
// Create final merged summary.
|
||||
local_summary_.Clear();
|
||||
for (auto& summary : summary_levels_) {
|
||||
local_summary_.Merge(summary);
|
||||
summary.Clear();
|
||||
}
|
||||
summary_levels_.clear();
|
||||
summary_levels_.shrink_to_fit();
|
||||
finalized_ = true;
|
||||
}
|
||||
|
||||
// Generates requested number of quantiles after finalizing stream.
|
||||
// The returned quantiles can be queried using std::lower_bound to get
|
||||
// the bucket for a given value.
|
||||
std::vector<ValueType> GenerateQuantiles(int64 num_quantiles) const {
|
||||
// Validate state.
|
||||
QCHECK(finalized_)
|
||||
<< "Finalize() must be called before generating quantiles.";
|
||||
return local_summary_.GenerateQuantiles(num_quantiles);
|
||||
}
|
||||
|
||||
// Generates requested number of boundaries after finalizing stream.
|
||||
// The returned boundaries can be queried using std::lower_bound to get
|
||||
// the bucket for a given value.
|
||||
// The boundaries, while still guaranteeing approximation bounds, don't
|
||||
// necessarily represent the actual quantiles of the distribution.
|
||||
// Boundaries are preferable over quantiles when the caller is less
|
||||
// interested in the actual quantiles distribution and more interested in
|
||||
// getting a representative sample of boundary values.
|
||||
std::vector<ValueType> GenerateBoundaries(int64 num_boundaries) const {
|
||||
// Validate state.
|
||||
QCHECK(finalized_)
|
||||
<< "Finalize() must be called before generating boundaries.";
|
||||
return local_summary_.GenerateBoundaries(num_boundaries);
|
||||
}
|
||||
|
||||
// Calculates approximation error for the specified level.
|
||||
// If the passed level is negative, the approximation error for the entire
|
||||
// summary is returned. Note that after Finalize is called, only the overall
|
||||
// error is available.
|
||||
WeightType ApproximationError(int64 level = -1) const {
|
||||
if (finalized_) {
|
||||
QCHECK(level <= 0) << "Only overall error is available after Finalize()";
|
||||
return local_summary_.ApproximationError();
|
||||
}
|
||||
|
||||
if (summary_levels_.empty()) {
|
||||
// No error even if base buffer isn't empty.
|
||||
return 0;
|
||||
}
|
||||
|
||||
// If level is negative, we get the approximation error
|
||||
// for the top-most level which is the max approximation error
|
||||
// in all summaries by construction.
|
||||
if (level < 0) {
|
||||
level = summary_levels_.size() - 1;
|
||||
}
|
||||
QCHECK(level < summary_levels_.size()) << "Invalid level.";
|
||||
return summary_levels_[level].ApproximationError();
|
||||
}
|
||||
|
||||
size_t MaxDepth() const { return summary_levels_.size(); }
|
||||
|
||||
// Generates requested number of quantiles after finalizing stream.
|
||||
const Summary& GetFinalSummary() const {
|
||||
// Validate state.
|
||||
QCHECK(finalized_)
|
||||
<< "Finalize() must be called before requesting final summary.";
|
||||
return local_summary_;
|
||||
}
|
||||
|
||||
// Helper method which, given the desired approximation error
|
||||
// and an upper bound on the number of elements, computes the optimal
|
||||
// number of levels and block size and returns them in the tuple.
|
||||
static std::tuple<int64, int64> GetQuantileSpecs(double eps,
|
||||
int64 max_elements);
|
||||
|
||||
// Serializes the internal state of the stream.
|
||||
std::vector<Summary> SerializeInternalSummaries() const {
|
||||
// The buffer should be empty for serialize to work.
|
||||
QCHECK_EQ(buffer_.Size(), 0);
|
||||
std::vector<Summary> result;
|
||||
result.reserve(summary_levels_.size() + 1);
|
||||
for (const Summary& summary : summary_levels_) {
|
||||
result.push_back(summary);
|
||||
}
|
||||
result.push_back(local_summary_);
|
||||
return result;
|
||||
}
|
||||
|
||||
// Resets the state of the stream with a serialized state.
|
||||
void DeserializeInternalSummaries(const std::vector<Summary>& summaries) {
|
||||
// Clear the state before deserializing.
|
||||
buffer_.Clear();
|
||||
summary_levels_.clear();
|
||||
local_summary_.Clear();
|
||||
QCHECK_GT(max_levels_, summaries.size() - 1);
|
||||
for (int i = 0; i < summaries.size() - 1; ++i) {
|
||||
summary_levels_.push_back(summaries[i]);
|
||||
}
|
||||
local_summary_ = summaries[summaries.size() - 1];
|
||||
}
|
||||
|
||||
private:
|
||||
// Propagates local summary through summary levels while maintaining
|
||||
// approximation error invariants.
|
||||
void PropagateLocalSummary() {
|
||||
// Validate state.
|
||||
QCHECK(!finalized_) << "Finalize() already called.";
|
||||
|
||||
// No-op if there's nothing to add.
|
||||
if (local_summary_.Size() <= 0) {
|
||||
return;
|
||||
}
|
||||
|
||||
// Propagate summary through levels.
|
||||
size_t level = 0;
|
||||
for (bool settled = false; !settled; ++level) {
|
||||
// Ensure we have enough depth.
|
||||
if (summary_levels_.size() <= level) {
|
||||
summary_levels_.emplace_back();
|
||||
}
|
||||
|
||||
// Merge summaries.
|
||||
Summary& current_summary = summary_levels_[level];
|
||||
local_summary_.Merge(current_summary);
|
||||
|
||||
// Check if we need to compress and propagate summary higher.
|
||||
if (current_summary.Size() == 0 ||
|
||||
local_summary_.Size() <= block_size_ + 1) {
|
||||
current_summary = std::move(local_summary_);
|
||||
settled = true;
|
||||
} else {
|
||||
// Compress, empty current level and propagate.
|
||||
local_summary_.Compress(block_size_, eps_);
|
||||
current_summary.Clear();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Desired approximation precision.
|
||||
double eps_;
|
||||
// Maximum number of levels.
|
||||
int64 max_levels_;
|
||||
// Max block size per level.
|
||||
int64 block_size_;
|
||||
// Base buffer.
|
||||
Buffer buffer_;
|
||||
// Local summary used to minimize memory allocation and cache misses.
|
||||
// After the stream is finalized, this summary holds the final quantile
|
||||
// estimates.
|
||||
Summary local_summary_;
|
||||
// Summary levels;
|
||||
std::vector<Summary> summary_levels_;
|
||||
// Flag indicating whether the stream is finalized.
|
||||
bool finalized_;
|
||||
};
|
||||
|
||||
template <typename ValueType, typename WeightType, typename CompareFn>
|
||||
inline std::tuple<int64, int64>
|
||||
WeightedQuantilesStream<ValueType, WeightType, CompareFn>::GetQuantileSpecs(
|
||||
double eps, int64 max_elements) {
|
||||
int64 max_level = 1LL;
|
||||
int64 block_size = 2LL;
|
||||
QCHECK(eps >= 0 && eps < 1);
|
||||
QCHECK_GT(max_elements, 0);
|
||||
|
||||
if (eps <= std::numeric_limits<double>::epsilon()) {
|
||||
// Exact quantile computation at the expense of RAM.
|
||||
max_level = 1;
|
||||
block_size = std::max(max_elements, int64{2});
|
||||
} else {
|
||||
// The bottom-most level will become full at most
|
||||
// (max_elements / block_size) times, the level above will become full
|
||||
// (max_elements / 2 * block_size) times and generally level l becomes
|
||||
// full (max_elements / 2^l * block_size) times until the last
|
||||
// level max_level becomes full at most once meaning when the inequality
|
||||
// (2^max_level * block_size >= max_elements) is satisfied.
|
||||
// In what follows, we jointly solve for max_level and block_size by
|
||||
// gradually increasing the level until the inequality above is satisfied.
|
||||
// We could alternatively set max_level = ceil(log2(eps * max_elements));
|
||||
// and block_size = ceil(max_level / eps) + 1 but that tends to give more
|
||||
// pessimistic bounds and wastes RAM needlessly.
|
||||
for (max_level = 1, block_size = 2;
|
||||
(1LL << max_level) * block_size < max_elements; ++max_level) {
|
||||
// Update upper bound on block size at current level, we always
|
||||
// increase the estimate by 2 to hold the min/max elements seen so far.
|
||||
block_size = static_cast<size_t>(ceil(max_level / eps)) + 1;
|
||||
}
|
||||
}
|
||||
return std::make_tuple(max_level, std::max(block_size, int64{2}));
|
||||
}
|
||||
|
||||
} // namespace quantiles
|
||||
} // namespace boosted_trees
|
||||
} // namespace tensorflow
|
||||
|
||||
#endif // TENSORFLOW_CORE_KERNELS_BOOSTED_TREES_QUANTILES_WEIGHTED_QUANTILES_STREAM_H_
|
||||
+344
@@ -0,0 +1,344 @@
|
||||
// Copyright 2018 The TensorFlow Authors. All Rights Reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
// =============================================================================
|
||||
#ifndef TENSORFLOW_CORE_KERNELS_BOOSTED_TREES_QUANTILES_WEIGHTED_QUANTILES_SUMMARY_H_
|
||||
#define TENSORFLOW_CORE_KERNELS_BOOSTED_TREES_QUANTILES_WEIGHTED_QUANTILES_SUMMARY_H_
|
||||
|
||||
#include <cstring>
|
||||
#include <vector>
|
||||
|
||||
#include "tensorflow/core/kernels/boosted_trees/quantiles/weighted_quantiles_buffer.h"
|
||||
|
||||
namespace tensorflow {
|
||||
namespace boosted_trees {
|
||||
namespace quantiles {
|
||||
|
||||
// Summary holding a sorted block of entries with upper bound guarantees
|
||||
// over the approximation error.
|
||||
template <typename ValueType, typename WeightType,
|
||||
typename CompareFn = std::less<ValueType>>
|
||||
class WeightedQuantilesSummary {
|
||||
public:
|
||||
using Buffer = WeightedQuantilesBuffer<ValueType, WeightType, CompareFn>;
|
||||
using BufferEntry = typename Buffer::BufferEntry;
|
||||
|
||||
struct SummaryEntry {
|
||||
SummaryEntry(const ValueType& v, const WeightType& w, const WeightType& min,
|
||||
const WeightType& max) {
|
||||
// Explicitly initialize all of memory (including padding from memory
|
||||
// alignment) to allow the struct to be msan-resistant "plain old data".
|
||||
//
|
||||
// POD = http://en.cppreference.com/w/cpp/concept/PODType
|
||||
memset(this, 0, sizeof(*this));
|
||||
|
||||
value = v;
|
||||
weight = w;
|
||||
min_rank = min;
|
||||
max_rank = max;
|
||||
}
|
||||
|
||||
SummaryEntry() {
|
||||
memset(this, 0, sizeof(*this));
|
||||
|
||||
value = ValueType();
|
||||
weight = 0;
|
||||
min_rank = 0;
|
||||
max_rank = 0;
|
||||
}
|
||||
|
||||
bool operator==(const SummaryEntry& other) const {
|
||||
return value == other.value && weight == other.weight &&
|
||||
min_rank == other.min_rank && max_rank == other.max_rank;
|
||||
}
|
||||
friend std::ostream& operator<<(std::ostream& strm,
|
||||
const SummaryEntry& entry) {
|
||||
return strm << "{" << entry.value << ", " << entry.weight << ", "
|
||||
<< entry.min_rank << ", " << entry.max_rank << "}";
|
||||
}
|
||||
|
||||
// Max rank estimate for previous smaller value.
|
||||
WeightType PrevMaxRank() const { return max_rank - weight; }
|
||||
|
||||
// Min rank estimate for next larger value.
|
||||
WeightType NextMinRank() const { return min_rank + weight; }
|
||||
|
||||
ValueType value;
|
||||
WeightType weight;
|
||||
WeightType min_rank;
|
||||
WeightType max_rank;
|
||||
};
|
||||
|
||||
// Re-construct summary from the specified buffer.
|
||||
void BuildFromBufferEntries(const std::vector<BufferEntry>& buffer_entries) {
|
||||
entries_.clear();
|
||||
entries_.reserve(buffer_entries.size());
|
||||
WeightType cumulative_weight = 0;
|
||||
for (const auto& entry : buffer_entries) {
|
||||
WeightType current_weight = entry.weight;
|
||||
entries_.emplace_back(entry.value, entry.weight, cumulative_weight,
|
||||
cumulative_weight + current_weight);
|
||||
cumulative_weight += current_weight;
|
||||
}
|
||||
}
|
||||
|
||||
// Re-construct summary from the specified summary entries.
|
||||
void BuildFromSummaryEntries(
|
||||
const std::vector<SummaryEntry>& summary_entries) {
|
||||
entries_.clear();
|
||||
entries_.reserve(summary_entries.size());
|
||||
entries_.insert(entries_.begin(), summary_entries.begin(),
|
||||
summary_entries.end());
|
||||
}
|
||||
|
||||
// Merges two summaries through an algorithm that's derived from MergeSort
|
||||
// for summary entries while guaranteeing that the max approximation error
|
||||
// of the final merged summary is no greater than the approximation errors
|
||||
// of each individual summary.
|
||||
// For example consider summaries where each entry is of the form
|
||||
// (element, weight, min rank, max rank):
|
||||
// summary entries 1: (1, 3, 0, 3), (4, 2, 3, 5)
|
||||
// summary entries 2: (3, 1, 0, 1), (4, 1, 1, 2)
|
||||
// merged: (1, 3, 0, 3), (3, 1, 3, 4), (4, 3, 4, 7).
|
||||
void Merge(const WeightedQuantilesSummary& other_summary) {
|
||||
// Make sure we have something to merge.
|
||||
const auto& other_entries = other_summary.entries_;
|
||||
if (other_entries.empty()) {
|
||||
return;
|
||||
}
|
||||
if (entries_.empty()) {
|
||||
BuildFromSummaryEntries(other_summary.entries_);
|
||||
return;
|
||||
}
|
||||
|
||||
// Move current entries to make room for a new buffer.
|
||||
std::vector<SummaryEntry> base_entries(std::move(entries_));
|
||||
entries_.clear();
|
||||
entries_.reserve(base_entries.size() + other_entries.size());
|
||||
|
||||
// Merge entries maintaining ranks. The idea is to stack values
|
||||
// in order which we can do in linear time as the two summaries are
|
||||
// already sorted. We keep track of the next lower rank from either
|
||||
// summary and update it as we pop elements from the summaries.
|
||||
// We handle the special case when the next two elements from either
|
||||
// summary are equal, in which case we just merge the two elements
|
||||
// and simultaneously update both ranks.
|
||||
auto it1 = base_entries.cbegin();
|
||||
auto it2 = other_entries.cbegin();
|
||||
WeightType next_min_rank1 = 0;
|
||||
WeightType next_min_rank2 = 0;
|
||||
while (it1 != base_entries.cend() && it2 != other_entries.cend()) {
|
||||
if (kCompFn(it1->value, it2->value)) { // value1 < value2
|
||||
// Take value1 and use the last added value2 to compute
|
||||
// the min rank and the current value2 to compute the max rank.
|
||||
entries_.emplace_back(it1->value, it1->weight,
|
||||
it1->min_rank + next_min_rank2,
|
||||
it1->max_rank + it2->PrevMaxRank());
|
||||
// Update next min rank 1.
|
||||
next_min_rank1 = it1->NextMinRank();
|
||||
++it1;
|
||||
} else if (kCompFn(it2->value, it1->value)) { // value1 > value2
|
||||
// Take value2 and use the last added value1 to compute
|
||||
// the min rank and the current value1 to compute the max rank.
|
||||
entries_.emplace_back(it2->value, it2->weight,
|
||||
it2->min_rank + next_min_rank1,
|
||||
it2->max_rank + it1->PrevMaxRank());
|
||||
// Update next min rank 2.
|
||||
next_min_rank2 = it2->NextMinRank();
|
||||
++it2;
|
||||
} else { // value1 == value2
|
||||
// Straight additive merger of the two entries into one.
|
||||
entries_.emplace_back(it1->value, it1->weight + it2->weight,
|
||||
it1->min_rank + it2->min_rank,
|
||||
it1->max_rank + it2->max_rank);
|
||||
// Update next min ranks for both.
|
||||
next_min_rank1 = it1->NextMinRank();
|
||||
next_min_rank2 = it2->NextMinRank();
|
||||
++it1;
|
||||
++it2;
|
||||
}
|
||||
}
|
||||
|
||||
// Fill in any residual.
|
||||
while (it1 != base_entries.cend()) {
|
||||
entries_.emplace_back(it1->value, it1->weight,
|
||||
it1->min_rank + next_min_rank2,
|
||||
it1->max_rank + other_entries.back().max_rank);
|
||||
++it1;
|
||||
}
|
||||
while (it2 != other_entries.cend()) {
|
||||
entries_.emplace_back(it2->value, it2->weight,
|
||||
it2->min_rank + next_min_rank1,
|
||||
it2->max_rank + base_entries.back().max_rank);
|
||||
++it2;
|
||||
}
|
||||
}
|
||||
|
||||
// Compresses buffer into desired size. The size specification is
|
||||
// considered a hint as we always keep the first and last elements and
|
||||
// maintain strict approximation error bounds.
|
||||
// The approximation error delta is taken as the max of either the requested
|
||||
// min error or 1 / size_hint.
|
||||
// After compression, the approximation error is guaranteed to increase
|
||||
// by no more than that error delta.
|
||||
// This algorithm is linear in the original size of the summary and is
|
||||
// designed to be cache-friendly.
|
||||
void Compress(int64 size_hint, double min_eps = 0) {
|
||||
// No-op if we're already within the size requirement.
|
||||
size_hint = std::max(size_hint, int64{2});
|
||||
if (entries_.size() <= size_hint) {
|
||||
return;
|
||||
}
|
||||
|
||||
// First compute the max error bound delta resulting from this compression.
|
||||
double eps_delta = TotalWeight() * std::max(1.0 / size_hint, min_eps);
|
||||
|
||||
// Compress elements ensuring approximation bounds and elements diversity
|
||||
// are both maintained.
|
||||
int64 add_accumulator = 0, add_step = entries_.size();
|
||||
auto write_it = entries_.begin() + 1, last_it = write_it;
|
||||
for (auto read_it = entries_.begin(); read_it + 1 != entries_.end();) {
|
||||
auto next_it = read_it + 1;
|
||||
while (next_it != entries_.end() && add_accumulator < add_step &&
|
||||
next_it->PrevMaxRank() - read_it->NextMinRank() <= eps_delta) {
|
||||
add_accumulator += size_hint;
|
||||
++next_it;
|
||||
}
|
||||
if (read_it == next_it - 1) {
|
||||
++read_it;
|
||||
} else {
|
||||
read_it = next_it - 1;
|
||||
}
|
||||
(*write_it++) = (*read_it);
|
||||
last_it = read_it;
|
||||
add_accumulator -= add_step;
|
||||
}
|
||||
// Write last element and resize.
|
||||
if (last_it + 1 != entries_.end()) {
|
||||
(*write_it++) = entries_.back();
|
||||
}
|
||||
entries_.resize(write_it - entries_.begin());
|
||||
}
|
||||
|
||||
// To construct the boundaries we first run a soft compress over a copy
|
||||
// of the summary and retrieve the values.
|
||||
// The resulting boundaries are guaranteed to both contain at least
|
||||
// num_boundaries unique elements and maintain approximation bounds.
|
||||
std::vector<ValueType> GenerateBoundaries(int64 num_boundaries) const {
|
||||
std::vector<ValueType> output;
|
||||
if (entries_.empty()) {
|
||||
return output;
|
||||
}
|
||||
|
||||
// Generate soft compressed summary.
|
||||
WeightedQuantilesSummary<ValueType, WeightType, CompareFn>
|
||||
compressed_summary;
|
||||
compressed_summary.BuildFromSummaryEntries(entries_);
|
||||
// Set an epsilon for compression that's at most 1.0 / num_boundaries
|
||||
// more than epsilon of original our summary since the compression operation
|
||||
// adds ~1.0/num_boundaries to final approximation error.
|
||||
float compression_eps = ApproximationError() + (1.0 / num_boundaries);
|
||||
compressed_summary.Compress(num_boundaries, compression_eps);
|
||||
|
||||
// Return boundaries.
|
||||
output.reserve(compressed_summary.entries_.size());
|
||||
for (const auto& entry : compressed_summary.entries_) {
|
||||
output.push_back(entry.value);
|
||||
}
|
||||
return output;
|
||||
}
|
||||
|
||||
// To construct the desired n-quantiles we repetitively query n ranks from the
|
||||
// original summary. The following algorithm is an efficient cache-friendly
|
||||
// O(n) implementation of that idea which avoids the cost of the repetitive
|
||||
// full rank queries O(nlogn).
|
||||
std::vector<ValueType> GenerateQuantiles(int64 num_quantiles) const {
|
||||
std::vector<ValueType> output;
|
||||
if (entries_.empty()) {
|
||||
return output;
|
||||
}
|
||||
num_quantiles = std::max(num_quantiles, int64{2});
|
||||
output.reserve(num_quantiles + 1);
|
||||
|
||||
// Make successive rank queries to get boundaries.
|
||||
// We always keep the first (min) and last (max) entries.
|
||||
for (size_t cur_idx = 0, rank = 0; rank <= num_quantiles; ++rank) {
|
||||
// This step boils down to finding the next element sub-range defined by
|
||||
// r = (rmax[i + 1] + rmin[i + 1]) / 2 where the desired rank d < r.
|
||||
WeightType d_2 = 2 * (rank * entries_.back().max_rank / num_quantiles);
|
||||
size_t next_idx = cur_idx + 1;
|
||||
while (next_idx < entries_.size() &&
|
||||
d_2 >= entries_[next_idx].min_rank + entries_[next_idx].max_rank) {
|
||||
++next_idx;
|
||||
}
|
||||
cur_idx = next_idx - 1;
|
||||
|
||||
// Determine insertion order.
|
||||
if (next_idx == entries_.size() ||
|
||||
d_2 < entries_[cur_idx].NextMinRank() +
|
||||
entries_[next_idx].PrevMaxRank()) {
|
||||
output.push_back(entries_[cur_idx].value);
|
||||
} else {
|
||||
output.push_back(entries_[next_idx].value);
|
||||
}
|
||||
}
|
||||
return output;
|
||||
}
|
||||
|
||||
// Calculates current approximation error which should always be <= eps.
|
||||
double ApproximationError() const {
|
||||
if (entries_.empty()) {
|
||||
return 0;
|
||||
}
|
||||
|
||||
WeightType max_gap = 0;
|
||||
for (auto it = entries_.cbegin() + 1; it < entries_.end(); ++it) {
|
||||
max_gap = std::max(max_gap,
|
||||
std::max(it->max_rank - it->min_rank - it->weight,
|
||||
it->PrevMaxRank() - (it - 1)->NextMinRank()));
|
||||
}
|
||||
return static_cast<double>(max_gap) / TotalWeight();
|
||||
}
|
||||
|
||||
ValueType MinValue() const {
|
||||
return !entries_.empty() ? entries_.front().value
|
||||
: std::numeric_limits<ValueType>::max();
|
||||
}
|
||||
ValueType MaxValue() const {
|
||||
return !entries_.empty() ? entries_.back().value
|
||||
: std::numeric_limits<ValueType>::max();
|
||||
}
|
||||
WeightType TotalWeight() const {
|
||||
return !entries_.empty() ? entries_.back().max_rank : 0;
|
||||
}
|
||||
int64 Size() const { return entries_.size(); }
|
||||
void Clear() { entries_.clear(); }
|
||||
const std::vector<SummaryEntry>& GetEntryList() const { return entries_; }
|
||||
|
||||
private:
|
||||
// Comparison function.
|
||||
static constexpr decltype(CompareFn()) kCompFn = CompareFn();
|
||||
|
||||
// Summary entries.
|
||||
std::vector<SummaryEntry> entries_;
|
||||
};
|
||||
|
||||
template <typename ValueType, typename WeightType, typename CompareFn>
|
||||
constexpr decltype(CompareFn())
|
||||
WeightedQuantilesSummary<ValueType, WeightType, CompareFn>::kCompFn;
|
||||
|
||||
} // namespace quantiles
|
||||
} // namespace boosted_trees
|
||||
} // namespace tensorflow
|
||||
|
||||
#endif // TENSORFLOW_CORE_KERNELS_BOOSTED_TREES_QUANTILES_WEIGHTED_QUANTILES_SUMMARY_H_
|
||||
@@ -0,0 +1,173 @@
|
||||
/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
==============================================================================*/
|
||||
|
||||
#ifndef TENSORFLOW_CORE_KERNELS_BOOSTED_TREES_RESOURCES_H_
|
||||
#define TENSORFLOW_CORE_KERNELS_BOOSTED_TREES_RESOURCES_H_
|
||||
|
||||
#include "tensorflow/core/framework/resource_mgr.h"
|
||||
#include "tensorflow/core/platform/mutex.h"
|
||||
#include "tensorflow/core/platform/protobuf.h"
|
||||
|
||||
namespace tensorflow {
|
||||
|
||||
// Forward declaration for proto class TreeEnsemble
|
||||
namespace boosted_trees {
|
||||
class TreeEnsemble;
|
||||
} // namespace boosted_trees
|
||||
|
||||
// A StampedResource is a resource that has a stamp token associated with it.
|
||||
// Before reading from or applying updates to the resource, the stamp should
|
||||
// be checked to verify that the update is not stale.
|
||||
class StampedResource : public ResourceBase {
|
||||
public:
|
||||
StampedResource() : stamp_(-1) {}
|
||||
|
||||
bool is_stamp_valid(int64 stamp) const { return stamp_ == stamp; }
|
||||
|
||||
int64 stamp() const { return stamp_; }
|
||||
void set_stamp(int64 stamp) { stamp_ = stamp; }
|
||||
|
||||
private:
|
||||
int64 stamp_;
|
||||
};
|
||||
|
||||
// Keep a tree ensemble in memory for efficient evaluation and mutation.
|
||||
class BoostedTreesEnsembleResource : public StampedResource {
|
||||
public:
|
||||
BoostedTreesEnsembleResource();
|
||||
|
||||
string DebugString() override;
|
||||
|
||||
bool InitFromSerialized(const string& serialized, const int64 stamp_token);
|
||||
|
||||
string SerializeAsString() const;
|
||||
|
||||
int32 num_trees() const;
|
||||
|
||||
// Find the next node to which the example (specified by index_in_batch)
|
||||
// traverses down from the current node indicated by tree_id and node_id.
|
||||
// Args:
|
||||
// tree_id: the index of the tree in the ensemble.
|
||||
// node_id: the index of the node within the tree.
|
||||
// index_in_batch: the index of the example within the batch (relevant to
|
||||
// the index of the row to read in each bucketized_features).
|
||||
// bucketized_features: vector of feature Vectors.
|
||||
int32 next_node(
|
||||
const int32 tree_id, const int32 node_id, const int32 index_in_batch,
|
||||
const std::vector<TTypes<int32>::ConstVec>& bucketized_features) const;
|
||||
|
||||
float node_value(const int32 tree_id, const int32 node_id) const;
|
||||
|
||||
void set_node_value(const int32 tree_id, const int32 node_id,
|
||||
const float logits);
|
||||
|
||||
int32 GetNumLayersGrown(const int32 tree_id) const;
|
||||
|
||||
void SetNumLayersGrown(const int32 tree_id, int32 new_num_layers) const;
|
||||
|
||||
void UpdateLastLayerNodesRange(const int32 node_range_start,
|
||||
int32 node_range_end) const;
|
||||
|
||||
void GetLastLayerNodesRange(int32* node_range_start,
|
||||
int32* node_range_end) const;
|
||||
|
||||
int64 GetNumNodes(const int32 tree_id);
|
||||
|
||||
void UpdateGrowingMetadata() const;
|
||||
|
||||
int32 GetNumLayersAttempted();
|
||||
|
||||
bool is_leaf(const int32 tree_id, const int32 node_id) const;
|
||||
|
||||
int32 feature_id(const int32 tree_id, const int32 node_id) const;
|
||||
|
||||
int32 bucket_threshold(const int32 tree_id, const int32 node_id) const;
|
||||
|
||||
int32 left_id(const int32 tree_id, const int32 node_id) const;
|
||||
|
||||
int32 right_id(const int32 tree_id, const int32 node_id) const;
|
||||
|
||||
// Add a tree to the ensemble and returns a new tree_id.
|
||||
int32 AddNewTree(const float weight);
|
||||
|
||||
// Adds new tree with one node to the ensemble and sets node's value to logits
|
||||
int32 AddNewTreeWithLogits(const float weight, const float logits);
|
||||
|
||||
// Grows the tree by adding a split and leaves.
|
||||
void AddBucketizedSplitNode(const int32 tree_id, const int32 node_id,
|
||||
const int32 feature_id, const int32 threshold,
|
||||
const float gain, const float left_contrib,
|
||||
const float right_contrib, int32* left_node_id,
|
||||
int32* right_node_id);
|
||||
|
||||
// Retrieves tree weights and returns as a vector.
|
||||
// It involves a copy, so should be called only sparingly (like once per
|
||||
// iteration, not per example).
|
||||
std::vector<float> GetTreeWeights() const;
|
||||
|
||||
float GetTreeWeight(const int32 tree_id) const;
|
||||
|
||||
float IsTreeFinalized(const int32 tree_id) const;
|
||||
|
||||
float IsTreePostPruned(const int32 tree_id) const;
|
||||
|
||||
void SetIsFinalized(const int32 tree_id, const bool is_finalized);
|
||||
|
||||
// Sets the weight of i'th tree.
|
||||
void SetTreeWeight(const int32 tree_id, const float weight);
|
||||
|
||||
// Resets the resource and frees the protos in arena.
|
||||
// Caller needs to hold the mutex lock while calling this.
|
||||
virtual void Reset();
|
||||
|
||||
void PostPruneTree(const int32 current_tree);
|
||||
|
||||
// For a given node, returns the id in a pruned tree, as well as correction
|
||||
// to the cached prediction that should be applied. If tree was not
|
||||
// post-pruned, current_node_id will be equal to initial_node_id and logit
|
||||
// update will be equal to zero.
|
||||
void GetPostPruneCorrection(const int32 tree_id, const int32 initial_node_id,
|
||||
int32* current_node_id,
|
||||
float* logit_update) const;
|
||||
mutex* get_mutex() { return &mu_; }
|
||||
|
||||
private:
|
||||
// Helper method to check whether a node is a terminal node in that it
|
||||
// only has leaf nodes as children.
|
||||
bool IsTerminalSplitNode(const int32 tree_id, const int32 node_id) const;
|
||||
|
||||
// For each pruned node, finds the leaf where it finally ended up and
|
||||
// calculates the total update from that pruned node prediction.
|
||||
void CalculateParentAndLogitUpdate(
|
||||
const int32 start_node_id,
|
||||
const std::vector<std::pair<int32, float>>& nodes_change,
|
||||
int32* parent_id, float* change) const;
|
||||
|
||||
// Helper method to collect the information to be used to prune some nodes in
|
||||
// the tree.
|
||||
void RecursivelyDoPostPrunePreparation(
|
||||
const int32 tree_id, const int32 node_id,
|
||||
std::vector<int32>* nodes_to_delete,
|
||||
std::vector<std::pair<int32, float>>* nodes_meta);
|
||||
|
||||
protected:
|
||||
protobuf::Arena arena_;
|
||||
mutex mu_;
|
||||
boosted_trees::TreeEnsemble* tree_ensemble_;
|
||||
};
|
||||
|
||||
} // namespace tensorflow
|
||||
|
||||
#endif // TENSORFLOW_CORE_KERNELS_BOOSTED_TREES_RESOURCES_H_
|
||||
@@ -0,0 +1,69 @@
|
||||
/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
==============================================================================*/
|
||||
|
||||
#ifndef TENSORFLOW_CORE_KERNELS_BOOSTED_TREES_TREE_HELPER_H_
|
||||
#define TENSORFLOW_CORE_KERNELS_BOOSTED_TREES_TREE_HELPER_H_
|
||||
#include <cmath>
|
||||
|
||||
namespace tensorflow {
|
||||
|
||||
static bool GainsAreEqual(const float g1, const float g2) {
|
||||
const float kTolerance = 1e-15;
|
||||
return std::abs(g1 - g2) < kTolerance;
|
||||
}
|
||||
|
||||
static bool GainIsLarger(const float g1, const float g2) {
|
||||
const float kTolerance = 1e-15;
|
||||
return g1 - g2 >= kTolerance;
|
||||
}
|
||||
|
||||
static void CalculateWeightsAndGains(const float g, const float h,
|
||||
const float l1, const float l2,
|
||||
float* weight, float* gain) {
|
||||
const float kEps = 1e-15;
|
||||
// The formula for weight is -(g+l1*sgn(w))/(H+l2), for gain it is
|
||||
// (g+l1*sgn(w))^2/(h+l2).
|
||||
// This is because for each leaf we optimize
|
||||
// 1/2(h+l2)*w^2+g*w+l1*abs(w)
|
||||
float g_with_l1 = g;
|
||||
// Apply L1 regularization.
|
||||
// 1) Assume w>0 => w=-(g+l1)/(h+l2)=> g+l1 < 0 => g < -l1
|
||||
// 2) Assume w<0 => w=-(g-l1)/(h+l2)=> g-l1 > 0 => g > l1
|
||||
// For g from (-l1, l1), thus there is no solution => set to 0.
|
||||
if (l1 > 0) {
|
||||
if (g > l1) {
|
||||
g_with_l1 -= l1;
|
||||
} else if (g < -l1) {
|
||||
g_with_l1 += l1;
|
||||
} else {
|
||||
*weight = 0.0;
|
||||
*gain = 0.0;
|
||||
return;
|
||||
}
|
||||
}
|
||||
// Apply L2 regularization.
|
||||
if (h + l2 <= kEps) {
|
||||
// Avoid division by 0 or infinitesimal.
|
||||
*weight = 0;
|
||||
*gain = 0;
|
||||
} else {
|
||||
*weight = -g_with_l1 / (h + l2);
|
||||
*gain = -g_with_l1 * (*weight);
|
||||
}
|
||||
}
|
||||
|
||||
} // namespace tensorflow
|
||||
|
||||
#endif // TENSORFLOW_CORE_KERNELS_BOOSTED_TREES_TREE_HELPER_H_
|
||||
@@ -0,0 +1,54 @@
|
||||
/* Copyright 2015 The TensorFlow Authors. All Rights Reserved.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
==============================================================================*/
|
||||
|
||||
#ifndef TENSORFLOW_CORE_KERNELS_BOUNDS_CHECK_H_
|
||||
#define TENSORFLOW_CORE_KERNELS_BOUNDS_CHECK_H_
|
||||
|
||||
#include <type_traits>
|
||||
|
||||
#include "third_party/eigen3/Eigen/Core"
|
||||
#include "tensorflow/core/platform/macros.h"
|
||||
|
||||
namespace tensorflow {
|
||||
|
||||
// Check that 0 <= index < limit using a single comparison, assuming
|
||||
// that 0 <= limit if Index is signed. Intended for use in performance
|
||||
// critical contexts where 0 <= index < limit is almost always true.
|
||||
template <typename Ta, typename Tb>
|
||||
EIGEN_ALWAYS_INLINE EIGEN_DEVICE_FUNC bool FastBoundsCheck(const Ta index,
|
||||
const Tb limit) {
|
||||
static_assert(std::is_integral<Ta>::value && std::is_integral<Tb>::value,
|
||||
"FastBoundsCheck can only be used on integer types.");
|
||||
typedef typename std::make_unsigned<decltype(index + limit)>::type UIndex;
|
||||
return TF_PREDICT_TRUE(static_cast<UIndex>(index) <
|
||||
static_cast<UIndex>(limit));
|
||||
}
|
||||
|
||||
namespace internal {
|
||||
// Ensure that the compiler cannot elide a copy into a local, for
|
||||
// bounds checking on source tensors that might be updated asynchronously.
|
||||
// This function may only be used on primitive integral types (int32, int64,
|
||||
// etc). It does not guarantee any atomicity or barriers.
|
||||
template <typename T>
|
||||
EIGEN_ALWAYS_INLINE EIGEN_DEVICE_FUNC const T SubtleMustCopy(const T &x) {
|
||||
static_assert(std::is_integral<T>::value,
|
||||
"SubtleMustCopy can only be used on integer types.");
|
||||
auto *to_x = reinterpret_cast<const volatile T *>(&x);
|
||||
return *to_x;
|
||||
}
|
||||
} // namespace internal
|
||||
} // namespace tensorflow
|
||||
|
||||
#endif // TENSORFLOW_CORE_KERNELS_BOUNDS_CHECK_H_
|
||||
@@ -0,0 +1,242 @@
|
||||
/* Copyright 2015 The TensorFlow Authors. All Rights Reserved.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
==============================================================================*/
|
||||
|
||||
#ifndef TENSORFLOW_CORE_KERNELS_BROADCAST_TO_OP_H_
|
||||
#define TENSORFLOW_CORE_KERNELS_BROADCAST_TO_OP_H_
|
||||
|
||||
#include "tensorflow/core/framework/op_kernel.h"
|
||||
#include "tensorflow/core/framework/tensor.h"
|
||||
#include "tensorflow/core/framework/tensor_shape.h"
|
||||
#include "tensorflow/core/framework/tensor_types.h"
|
||||
#include "tensorflow/core/framework/types.h"
|
||||
#include "third_party/eigen3/unsupported/Eigen/CXX11/Tensor"
|
||||
|
||||
namespace tensorflow {
|
||||
|
||||
namespace functor {
|
||||
|
||||
template <typename Device, typename T>
|
||||
struct BroadcastTo {
|
||||
void operator()(const Device &d, OpKernelContext *ctx, Tensor &output_tensor,
|
||||
const TensorShape &output_shape, const Tensor &input_tensor,
|
||||
const TensorShape &input_shape) {
|
||||
#define BROADCAST_SHAPE(broadcast, reshape, NDIMS, input_shape, output_shape) \
|
||||
for (int i = 0; i < NDIMS; i++) { \
|
||||
if (reshape[i] != broadcast[i]) { \
|
||||
OP_REQUIRES(ctx, \
|
||||
((reshape[i] != 0) && (broadcast[i] % reshape[i] == 0)), \
|
||||
errors::InvalidArgument("invalid shape to broadcast from ", \
|
||||
input_shape.DebugString(), " to ", \
|
||||
output_shape.DebugString())); \
|
||||
broadcast[i] = broadcast[i] / reshape[i]; \
|
||||
} else { \
|
||||
broadcast[i] = 1; \
|
||||
} \
|
||||
}
|
||||
|
||||
if (output_shape.num_elements() == 0) {
|
||||
return;
|
||||
}
|
||||
if (output_shape == input_shape) {
|
||||
output_tensor.flat<T>().device(d) = input_tensor.flat<T>();
|
||||
return;
|
||||
}
|
||||
|
||||
switch (output_shape.dims()) {
|
||||
case 0: {
|
||||
if (input_shape.dims() > 0) {
|
||||
ctx->CtxFailure(errors::InvalidArgument(
|
||||
"invalid shape to broadcast from ", input_shape.DebugString(),
|
||||
" to ", output_shape.DebugString()));
|
||||
break;
|
||||
}
|
||||
output_tensor.scalar<T>().device(d) = input_tensor.scalar<T>();
|
||||
break;
|
||||
}
|
||||
case 1: {
|
||||
auto reshape = AsEigenDSizesWithPrefix<1>(input_shape);
|
||||
auto broadcast = output_shape.AsEigenDSizes<1>();
|
||||
|
||||
BROADCAST_SHAPE(broadcast, reshape, 1, input_shape, output_shape);
|
||||
|
||||
auto output = output_tensor.tensor<T, 1>();
|
||||
switch (input_shape.dims()) {
|
||||
case 0: {
|
||||
output.device(d) = output.constant(input_tensor.scalar<T>()());
|
||||
} break;
|
||||
case 1: {
|
||||
auto input = input_tensor.tensor<T, 1>();
|
||||
output.device(d) = input.broadcast(broadcast);
|
||||
} break;
|
||||
default:
|
||||
ctx->CtxFailure(errors::InvalidArgument(
|
||||
"invalid shape to broadcast from ", input_shape.DebugString(),
|
||||
" to ", output_shape.DebugString()));
|
||||
break;
|
||||
}
|
||||
} break;
|
||||
case 2: {
|
||||
auto reshape = AsEigenDSizesWithPrefix<2>(input_shape);
|
||||
auto broadcast = output_shape.AsEigenDSizes<2>();
|
||||
|
||||
BROADCAST_SHAPE(broadcast, reshape, 2, input_shape, output_shape);
|
||||
|
||||
auto output = output_tensor.tensor<T, 2>();
|
||||
switch (input_shape.dims()) {
|
||||
case 0: {
|
||||
output.device(d) = output.constant(input_tensor.scalar<T>()());
|
||||
} break;
|
||||
case 1: {
|
||||
auto input = input_tensor.tensor<T, 1>();
|
||||
output.device(d) = input.reshape(reshape).broadcast(broadcast);
|
||||
} break;
|
||||
case 2: {
|
||||
auto input = input_tensor.tensor<T, 2>();
|
||||
output.device(d) = input.broadcast(broadcast);
|
||||
} break;
|
||||
default:
|
||||
ctx->CtxFailure(errors::InvalidArgument(
|
||||
"invalid shape to broadcast from ", input_shape.DebugString(),
|
||||
" to ", output_shape.DebugString()));
|
||||
break;
|
||||
}
|
||||
} break;
|
||||
case 3: {
|
||||
auto reshape = AsEigenDSizesWithPrefix<3>(input_shape);
|
||||
auto broadcast = output_shape.AsEigenDSizes<3>();
|
||||
|
||||
BROADCAST_SHAPE(broadcast, reshape, 3, input_shape, output_shape);
|
||||
|
||||
auto output = output_tensor.tensor<T, 3>();
|
||||
switch (input_shape.dims()) {
|
||||
case 0: {
|
||||
output.device(d) = output.constant(input_tensor.scalar<T>()());
|
||||
} break;
|
||||
case 1: {
|
||||
auto input = input_tensor.tensor<T, 1>();
|
||||
output.device(d) = input.reshape(reshape).broadcast(broadcast);
|
||||
} break;
|
||||
case 2: {
|
||||
auto input = input_tensor.tensor<T, 2>();
|
||||
output.device(d) = input.reshape(reshape).broadcast(broadcast);
|
||||
} break;
|
||||
case 3: {
|
||||
auto input = input_tensor.tensor<T, 3>();
|
||||
output.device(d) = input.broadcast(broadcast);
|
||||
} break;
|
||||
default:
|
||||
ctx->CtxFailure(errors::InvalidArgument(
|
||||
"invalid shape to broadcast from ", input_shape.DebugString(),
|
||||
" to ", output_shape.DebugString()));
|
||||
break;
|
||||
}
|
||||
} break;
|
||||
case 4: {
|
||||
auto reshape = AsEigenDSizesWithPrefix<4>(input_shape);
|
||||
auto broadcast = output_shape.AsEigenDSizes<4>();
|
||||
|
||||
BROADCAST_SHAPE(broadcast, reshape, 4, input_shape, output_shape);
|
||||
auto output = output_tensor.tensor<T, 4>();
|
||||
switch (input_shape.dims()) {
|
||||
case 0: {
|
||||
output.device(d) = output.constant(input_tensor.scalar<T>()());
|
||||
} break;
|
||||
case 1: {
|
||||
auto input = input_tensor.tensor<T, 1>();
|
||||
output.device(d) = input.reshape(reshape).broadcast(broadcast);
|
||||
} break;
|
||||
case 2: {
|
||||
auto input = input_tensor.tensor<T, 2>();
|
||||
output.device(d) = input.reshape(reshape).broadcast(broadcast);
|
||||
} break;
|
||||
case 3: {
|
||||
auto input = input_tensor.tensor<T, 3>();
|
||||
output.device(d) = input.reshape(reshape).broadcast(broadcast);
|
||||
} break;
|
||||
case 4: {
|
||||
auto input = input_tensor.tensor<T, 4>();
|
||||
output.device(d) = input.broadcast(broadcast);
|
||||
} break;
|
||||
default:
|
||||
ctx->CtxFailure(errors::InvalidArgument(
|
||||
"invalid shape to broadcast from ", input_shape.DebugString(),
|
||||
" to ", output_shape.DebugString()));
|
||||
break;
|
||||
}
|
||||
} break;
|
||||
case 5: {
|
||||
auto reshape = AsEigenDSizesWithPrefix<5>(input_shape);
|
||||
auto broadcast = output_shape.AsEigenDSizes<5>();
|
||||
|
||||
BROADCAST_SHAPE(broadcast, reshape, 5, input_shape, output_shape);
|
||||
auto output = output_tensor.tensor<T, 5>();
|
||||
switch (input_shape.dims()) {
|
||||
case 0: {
|
||||
output.device(d) = output.constant(input_tensor.scalar<T>()());
|
||||
} break;
|
||||
case 1: {
|
||||
auto input = input_tensor.tensor<T, 1>();
|
||||
output.device(d) = input.reshape(reshape).broadcast(broadcast);
|
||||
} break;
|
||||
case 2: {
|
||||
auto input = input_tensor.tensor<T, 2>();
|
||||
output.device(d) = input.reshape(reshape).broadcast(broadcast);
|
||||
} break;
|
||||
case 3: {
|
||||
auto input = input_tensor.tensor<T, 3>();
|
||||
output.device(d) = input.reshape(reshape).broadcast(broadcast);
|
||||
} break;
|
||||
case 4: {
|
||||
auto input = input_tensor.tensor<T, 4>();
|
||||
output.device(d) = input.reshape(reshape).broadcast(broadcast);
|
||||
} break;
|
||||
case 5: {
|
||||
auto input = input_tensor.tensor<T, 5>();
|
||||
output.device(d) = input.broadcast(broadcast);
|
||||
} break;
|
||||
default:
|
||||
ctx->CtxFailure(errors::InvalidArgument(
|
||||
"invalid shape to broadcast from ", input_shape.DebugString(),
|
||||
" to ", output_shape.DebugString()));
|
||||
break;
|
||||
}
|
||||
} break;
|
||||
default:
|
||||
ctx->CtxFailure(errors::InvalidArgument(
|
||||
"invalid shape to broadcast from ", input_shape.DebugString(),
|
||||
" to ", output_shape.DebugString()));
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
private:
|
||||
template <int NDIMS>
|
||||
Eigen::DSizes<Eigen::DenseIndex, NDIMS> AsEigenDSizesWithPrefix(
|
||||
const TensorShape &shape) const {
|
||||
Eigen::DSizes<Eigen::DenseIndex, NDIMS> dsizes;
|
||||
for (int d = 0; d < NDIMS - shape.dims(); d++) {
|
||||
dsizes[d] = 1;
|
||||
}
|
||||
for (int d = NDIMS - shape.dims(); d < NDIMS; d++) {
|
||||
dsizes[d] = shape.dim_size(d - (NDIMS - shape.dims()));
|
||||
}
|
||||
return dsizes;
|
||||
}
|
||||
};
|
||||
|
||||
} // namespace functor
|
||||
} // namespace tensorflow
|
||||
|
||||
#endif // TENSORFLOW_CORE_KERNELS_BROADCAST_TO_OP_H_
|
||||
@@ -0,0 +1,41 @@
|
||||
/* Copyright 2015 The TensorFlow Authors. All Rights Reserved.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
==============================================================================*/
|
||||
|
||||
#ifndef TENSORFLOW_CORE_KERNELS_BUCKETIZE_OP_H_
|
||||
#define TENSORFLOW_CORE_KERNELS_BUCKETIZE_OP_H_
|
||||
|
||||
#include <vector>
|
||||
#include "third_party/eigen3/unsupported/Eigen/CXX11/Tensor"
|
||||
|
||||
#include "tensorflow/core/framework/op_kernel.h"
|
||||
#include "tensorflow/core/framework/tensor_types.h"
|
||||
#include "tensorflow/core/framework/types.h"
|
||||
#include "tensorflow/core/lib/core/errors.h"
|
||||
|
||||
namespace tensorflow {
|
||||
namespace functor {
|
||||
|
||||
template <typename Device, typename T>
|
||||
struct BucketizeFunctor {
|
||||
static Status Compute(OpKernelContext* context,
|
||||
const typename TTypes<T, 1>::ConstTensor& input,
|
||||
const std::vector<float>& boundaries_vector,
|
||||
typename TTypes<int32, 1>::Tensor& output);
|
||||
};
|
||||
|
||||
} // namespace functor
|
||||
} // namespace tensorflow
|
||||
|
||||
#endif // TENSORFLOW_CORE_KERNELS_BUCKETIZE_OP_H_
|
||||
@@ -0,0 +1,326 @@
|
||||
/* Copyright 2015 The TensorFlow Authors. All Rights Reserved.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
==============================================================================*/
|
||||
|
||||
#ifndef TENSORFLOW_CORE_KERNELS_CAST_OP_H_
|
||||
#define TENSORFLOW_CORE_KERNELS_CAST_OP_H_
|
||||
|
||||
#include "third_party/eigen3/unsupported/Eigen/CXX11/Tensor"
|
||||
#include "tensorflow/core/framework/bfloat16.h"
|
||||
#include "tensorflow/core/framework/op_kernel.h"
|
||||
#include "tensorflow/core/framework/tensor_types.h"
|
||||
#include "tensorflow/core/framework/types.h"
|
||||
#include "tensorflow/core/platform/byte_order.h"
|
||||
#include "tensorflow/core/platform/types.h"
|
||||
|
||||
// Note that the GPU cast functor templates need to be instantiated unlike the
|
||||
// CPU ones, and hence their specializations are different than that for CPUs.
|
||||
#ifdef SPECIALIZE_FOR_GPUS
|
||||
#define SPECIALIZE_CAST(DEVICE, OUT_TYPE, IN_OUT) \
|
||||
template <typename Device> \
|
||||
struct CastFunctor<Device, OUT_TYPE, IN_OUT> { \
|
||||
void operator()(const Device& d, \
|
||||
typename TTypes<OUT_TYPE>::Flat out_tensor, \
|
||||
typename TTypes<IN_OUT>::ConstFlat in_tensor, \
|
||||
bool truncate = false) { \
|
||||
if (truncate) { \
|
||||
out_tensor.device(d) = \
|
||||
in_tensor.unaryExpr(LSBZeroSetter<IN_OUT, OUT_TYPE>()) \
|
||||
.template cast<OUT_TYPE>(); \
|
||||
} else { \
|
||||
out_tensor.device(d) = in_tensor.template cast<OUT_TYPE>(); \
|
||||
} \
|
||||
} \
|
||||
}; \
|
||||
template struct CastFunctor<DEVICE, OUT_TYPE, IN_OUT>;
|
||||
#else
|
||||
#define SPECIALIZE_CAST(DEVICE, OUT_TYPE, IN_OUT) \
|
||||
template <> \
|
||||
struct CastFunctor<DEVICE, OUT_TYPE, IN_OUT> { \
|
||||
void operator()(const DEVICE& d, \
|
||||
typename TTypes<OUT_TYPE>::Flat out_tensor, \
|
||||
typename TTypes<IN_OUT>::ConstFlat in_tensor, \
|
||||
bool truncate = false) { \
|
||||
if (truncate) { \
|
||||
out_tensor.device(d) = \
|
||||
in_tensor.unaryExpr(LSBZeroSetter<IN_OUT, OUT_TYPE>()) \
|
||||
.template cast<OUT_TYPE>(); \
|
||||
} else { \
|
||||
out_tensor.device(d) = in_tensor.template cast<OUT_TYPE>(); \
|
||||
} \
|
||||
} \
|
||||
};
|
||||
#endif
|
||||
|
||||
#define CAST_FUNCTORS(devname) \
|
||||
SPECIALIZE_CAST(devname, float, double) \
|
||||
SPECIALIZE_CAST(devname, float, std::complex<double>) \
|
||||
SPECIALIZE_CAST(devname, std::complex<float>, std::complex<double>) \
|
||||
SPECIALIZE_CAST(devname, std::complex<float>, double) \
|
||||
SPECIALIZE_CAST(devname, Eigen::half, double) \
|
||||
SPECIALIZE_CAST(devname, Eigen::half, float) \
|
||||
SPECIALIZE_CAST(devname, Eigen::half, std::complex<double>) \
|
||||
SPECIALIZE_CAST(devname, Eigen::half, std::complex<float>) \
|
||||
SPECIALIZE_CAST(devname, bfloat16, float) \
|
||||
template <typename OUT_TYPE, typename IN_OUT> \
|
||||
struct CastFunctor<devname, OUT_TYPE, IN_OUT> { \
|
||||
void operator()(const devname& d, \
|
||||
typename TTypes<OUT_TYPE>::Flat out_tensor, \
|
||||
typename TTypes<IN_OUT>::ConstFlat in_tensor, \
|
||||
bool truncate = false) { \
|
||||
out_tensor.device(d) = in_tensor.template cast<OUT_TYPE>(); \
|
||||
} \
|
||||
};
|
||||
|
||||
namespace tensorflow {
|
||||
|
||||
typedef std::function<void(OpKernelContext*, const Tensor&, Tensor*,
|
||||
bool trunc)>
|
||||
CastFunctorType;
|
||||
|
||||
// Common base class of Cast kernels
|
||||
class CastOpBase : public OpKernel {
|
||||
public:
|
||||
explicit CastOpBase(OpKernelConstruction* ctx);
|
||||
|
||||
void Compute(OpKernelContext* ctx) override;
|
||||
|
||||
protected:
|
||||
DataType src_dtype_;
|
||||
DataType dst_dtype_;
|
||||
DataType external_src_dtype_;
|
||||
DataType external_dst_dtype_;
|
||||
bool use_truncation_;
|
||||
CastFunctorType work_ = nullptr;
|
||||
Status Unimplemented();
|
||||
|
||||
TF_DISALLOW_COPY_AND_ASSIGN(CastOpBase);
|
||||
};
|
||||
|
||||
// CPU implementation of Cast
|
||||
class CpuCastOp : public CastOpBase {
|
||||
public:
|
||||
explicit CpuCastOp(OpKernelConstruction* ctx);
|
||||
|
||||
private:
|
||||
Status Prepare();
|
||||
};
|
||||
|
||||
namespace functor {
|
||||
|
||||
template <typename I>
|
||||
constexpr int MantissaWidth() {
|
||||
return std::numeric_limits<I>::digits;
|
||||
}
|
||||
|
||||
template <>
|
||||
constexpr int MantissaWidth<Eigen::half>() {
|
||||
// Remember, there's 1 hidden bit
|
||||
return 10 + 1;
|
||||
}
|
||||
|
||||
template <>
|
||||
constexpr int MantissaWidth<bfloat16>() {
|
||||
// Remember, there's 1 hidden bit
|
||||
return 7 + 1;
|
||||
}
|
||||
|
||||
template <typename Device, typename Tout, typename Tin>
|
||||
void Cast(const Device& d, typename TTypes<Tout>::Flat o,
|
||||
typename TTypes<Tin>::ConstFlat i) {
|
||||
o.device(d) = i.template cast<Tout>();
|
||||
}
|
||||
|
||||
template <typename Device, typename Tout, typename Tin>
|
||||
struct CastFunctor {
|
||||
void operator()(const Device& d, typename TTypes<Tout>::Flat o,
|
||||
typename TTypes<Tin>::ConstFlat i, bool truncate = false);
|
||||
};
|
||||
|
||||
// Only enable LSBZeroSetterHelper for 64 and 32 bit input data types.
|
||||
// Specialize for others if needed in future.
|
||||
template <typename I>
|
||||
typename std::enable_if<sizeof(I) == 8, void>::type EIGEN_DEVICE_FUNC
|
||||
EIGEN_STRONG_INLINE static LSBZeroSetterHelper(I& t, int n) {
|
||||
// Only zero the bits for non-NaNs.
|
||||
// For NaNs, let the non-truncation version handle it.
|
||||
if (!std::isnan(t)) {
|
||||
uint64_t* p = reinterpret_cast<uint64_t*>(&t);
|
||||
*p &= (0xFFFFFFFFFFFFFFFF << n);
|
||||
}
|
||||
}
|
||||
|
||||
template <typename I>
|
||||
typename std::enable_if<sizeof(I) == 4, void>::type EIGEN_DEVICE_FUNC
|
||||
EIGEN_STRONG_INLINE static LSBZeroSetterHelper(I& t, int n) {
|
||||
// Only zero the bits for non-NaNs.
|
||||
// For NaNs, let the non-truncation version handle it.
|
||||
if (!std::isnan(t)) {
|
||||
uint32_t* p = reinterpret_cast<uint32_t*>(&t);
|
||||
*p &= (0xFFFFFFFF << n);
|
||||
}
|
||||
}
|
||||
|
||||
// Set n least significant bits to 0
|
||||
template <typename I, typename O>
|
||||
struct LSBZeroSetter {
|
||||
EIGEN_EMPTY_STRUCT_CTOR(LSBZeroSetter)
|
||||
|
||||
EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const I operator()(const I& a) const {
|
||||
constexpr int bits = MantissaWidth<I>() - MantissaWidth<O>();
|
||||
static_assert(
|
||||
bits > 0,
|
||||
"The output type must have fewer mantissa bits than the input type\n");
|
||||
I t = a;
|
||||
LSBZeroSetterHelper(t, bits);
|
||||
return t;
|
||||
}
|
||||
};
|
||||
|
||||
template <typename I, typename O>
|
||||
struct LSBZeroSetter<std::complex<I>, std::complex<O>> {
|
||||
EIGEN_EMPTY_STRUCT_CTOR(LSBZeroSetter)
|
||||
|
||||
EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const std::complex<I> operator()(
|
||||
const std::complex<I>& a) const {
|
||||
constexpr int bits = MantissaWidth<I>() - MantissaWidth<O>();
|
||||
static_assert(
|
||||
bits > 0,
|
||||
"The output type must have fewer mantissa bits than the input type\n");
|
||||
I re = std::real(a);
|
||||
I img = std::imag(a);
|
||||
LSBZeroSetterHelper(re, bits);
|
||||
LSBZeroSetterHelper(img, bits);
|
||||
std::complex<I> toReturn(re, img);
|
||||
return toReturn;
|
||||
}
|
||||
};
|
||||
|
||||
template <typename I, typename O>
|
||||
struct LSBZeroSetter<std::complex<I>, O> {
|
||||
EIGEN_EMPTY_STRUCT_CTOR(LSBZeroSetter)
|
||||
// Sets the 16 LSBits of the float to 0
|
||||
EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const std::complex<I> operator()(
|
||||
const std::complex<I>& a) const {
|
||||
constexpr int bits = MantissaWidth<I>() - MantissaWidth<O>();
|
||||
static_assert(
|
||||
bits > 0,
|
||||
"The output type must have fewer mantissa bits than the input type\n");
|
||||
I re = std::real(a);
|
||||
I img = std::imag(a);
|
||||
LSBZeroSetterHelper(re, bits);
|
||||
LSBZeroSetterHelper(img, bits);
|
||||
std::complex<I> toReturn(re, img);
|
||||
return toReturn;
|
||||
}
|
||||
};
|
||||
|
||||
} // end namespace functor
|
||||
} // end namespace tensorflow
|
||||
|
||||
namespace Eigen {
|
||||
namespace internal {
|
||||
|
||||
// Eigen can't convert to/from complex numbers, because it is limited to cases
|
||||
// that can be static_casted. But numpy is able to cast to/from complex, which
|
||||
// we want to replicate. So we add specializations for complex here.
|
||||
template <typename From, typename To>
|
||||
struct scalar_cast_op<std::complex<From>, To> {
|
||||
EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE To
|
||||
operator()(const std::complex<From>& a) const {
|
||||
// Replicate numpy behavior of returning just the real part
|
||||
return static_cast<To>(a.real());
|
||||
}
|
||||
};
|
||||
|
||||
template <typename From, typename To>
|
||||
struct scalar_cast_op<From, std::complex<To>> {
|
||||
EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE std::complex<To> operator()(
|
||||
const From& a) const {
|
||||
// Replicate numpy behavior of setting the imaginary part to 0
|
||||
return std::complex<To>(static_cast<To>(a), To(0));
|
||||
}
|
||||
};
|
||||
|
||||
template <typename From, typename To>
|
||||
struct scalar_cast_op<std::complex<From>, std::complex<To>> {
|
||||
EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE std::complex<To> operator()(
|
||||
const std::complex<From>& a) const {
|
||||
return std::complex<To>(static_cast<To>(a.real()),
|
||||
static_cast<To>(a.imag()));
|
||||
}
|
||||
};
|
||||
|
||||
template <typename From, typename To>
|
||||
struct functor_traits_complex_impl {
|
||||
enum { Cost = NumTraits<To>::AddCost, PacketAccess = false };
|
||||
};
|
||||
|
||||
template <typename From, typename To>
|
||||
struct functor_traits<scalar_cast_op<std::complex<From>, To>>
|
||||
: functor_traits_complex_impl<std::complex<From>, To> {};
|
||||
template <typename From, typename To>
|
||||
struct functor_traits<scalar_cast_op<From, std::complex<To>>>
|
||||
: functor_traits_complex_impl<From, std::complex<To>> {};
|
||||
// Needed to avoid ambiguous partial specialization
|
||||
template <typename From, typename To>
|
||||
struct functor_traits<scalar_cast_op<std::complex<From>, std::complex<To>>>
|
||||
: functor_traits_complex_impl<std::complex<From>, std::complex<To>> {};
|
||||
|
||||
// Specialized cast op impls for bfloat16.
|
||||
template <>
|
||||
struct scalar_cast_op<::tensorflow::bfloat16, float> {
|
||||
EIGEN_EMPTY_STRUCT_CTOR(scalar_cast_op)
|
||||
typedef float result_type;
|
||||
EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE float operator()(
|
||||
const ::tensorflow::bfloat16& a) const {
|
||||
float ret;
|
||||
uint16_t* p = reinterpret_cast<uint16_t*>(&ret);
|
||||
#if __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__
|
||||
p[0] = a.value;
|
||||
p[1] = 0;
|
||||
#else
|
||||
static_assert(::tensorflow::port::kLittleEndian,
|
||||
"Not a little endian system!");
|
||||
p[0] = 0;
|
||||
p[1] = a.value;
|
||||
#endif
|
||||
return ret;
|
||||
}
|
||||
};
|
||||
|
||||
template <>
|
||||
struct functor_traits<scalar_cast_op<::tensorflow::bfloat16, float>> {
|
||||
enum { Cost = NumTraits<float>::AddCost, PacketAccess = false };
|
||||
};
|
||||
|
||||
template <>
|
||||
struct scalar_cast_op<float, ::tensorflow::bfloat16> {
|
||||
EIGEN_EMPTY_STRUCT_CTOR(scalar_cast_op)
|
||||
typedef ::tensorflow::bfloat16 result_type;
|
||||
EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const ::tensorflow::bfloat16 operator()(
|
||||
const float a) const {
|
||||
return ::tensorflow::bfloat16(a);
|
||||
}
|
||||
};
|
||||
|
||||
template <>
|
||||
struct functor_traits<scalar_cast_op<float, ::tensorflow::bfloat16>> {
|
||||
enum { Cost = NumTraits<float>::AddCost, PacketAccess = false };
|
||||
};
|
||||
|
||||
} // namespace internal
|
||||
} // namespace Eigen
|
||||
|
||||
#endif // TENSORFLOW_CORE_KERNELS_CAST_OP_H_
|
||||
@@ -0,0 +1,160 @@
|
||||
/* Copyright 2016 The TensorFlow Authors. All Rights Reserved.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
==============================================================================*/
|
||||
|
||||
#ifndef TENSORFLOW_CORE_KERNELS_CAST_OP_IMPL_H_
|
||||
#define TENSORFLOW_CORE_KERNELS_CAST_OP_IMPL_H_
|
||||
|
||||
#define EIGEN_USE_THREADS
|
||||
|
||||
#include "tensorflow/core/framework/op_kernel.h"
|
||||
#include "tensorflow/core/kernels/cast_op.h"
|
||||
|
||||
namespace tensorflow {
|
||||
|
||||
namespace functor {
|
||||
|
||||
CAST_FUNCTORS(Eigen::ThreadPoolDevice);
|
||||
|
||||
#ifdef TENSORFLOW_USE_SYCL
|
||||
CAST_FUNCTORS(Eigen::SyclDevice);
|
||||
#endif // TENSORFLOW_USE_SYCL
|
||||
|
||||
} // namespace functor
|
||||
|
||||
#define CURRY_TYPES3_NO_HALF(FN, arg0, arg1) \
|
||||
FN(arg0, arg1, bool); \
|
||||
FN(arg0, arg1, uint8); \
|
||||
FN(arg0, arg1, uint16); \
|
||||
FN(arg0, arg1, uint32); \
|
||||
FN(arg0, arg1, uint64); \
|
||||
FN(arg0, arg1, int8); \
|
||||
FN(arg0, arg1, int16); \
|
||||
FN(arg0, arg1, int32); \
|
||||
FN(arg0, arg1, int64); \
|
||||
FN(arg0, arg1, float); \
|
||||
FN(arg0, arg1, double); \
|
||||
FN(arg0, arg1, std::complex<float>); \
|
||||
FN(arg0, arg1, std::complex<double>)
|
||||
|
||||
#define CURRY_TYPES3_NO_BF16(FN, arg0, arg1) \
|
||||
CURRY_TYPES3_NO_HALF(FN, arg0, arg1) \
|
||||
FN(arg0, arg1, Eigen::half);
|
||||
|
||||
#define CURRY_TYPES3(FN, arg0, arg1) \
|
||||
CURRY_TYPES3_NO_BF16(FN, arg0, arg1) \
|
||||
FN(arg0, arg1, bfloat16);
|
||||
|
||||
#define CAST_CASE(DEVICE, IN, OUT) \
|
||||
if (DataTypeToEnum<OUT>::value == dst_dtype) { \
|
||||
return [](OpKernelContext* ctx, const Tensor& inp, Tensor* out, \
|
||||
bool truncate) { \
|
||||
functor::CastFunctor<DEVICE, OUT, IN> func; \
|
||||
func(ctx->eigen_device<DEVICE>(), out->flat<OUT>(), inp.flat<IN>(), \
|
||||
truncate); \
|
||||
}; \
|
||||
}
|
||||
|
||||
// The functions below are implemented in the cast_op_impl_*.cc files.
|
||||
CastFunctorType GetCpuCastFromBool(DataType dst_dtype);
|
||||
|
||||
CastFunctorType GetCpuCastFromUint8(DataType dst_dtype);
|
||||
|
||||
CastFunctorType GetCpuCastFromUint16(DataType dst_dtype);
|
||||
|
||||
CastFunctorType GetCpuCastFromInt8(DataType dst_dtype);
|
||||
|
||||
CastFunctorType GetCpuCastFromUint32(DataType dst_dtype);
|
||||
|
||||
CastFunctorType GetCpuCastFromUint64(DataType dst_dtype);
|
||||
|
||||
CastFunctorType GetCpuCastFromInt8(DataType dst_dtype);
|
||||
|
||||
CastFunctorType GetCpuCastFromInt16(DataType dst_dtype);
|
||||
|
||||
CastFunctorType GetCpuCastFromInt32(DataType dst_dtype);
|
||||
|
||||
CastFunctorType GetCpuCastFromInt64(DataType dst_dtype);
|
||||
|
||||
CastFunctorType GetCpuCastFromHalf(DataType dst_dtype);
|
||||
|
||||
CastFunctorType GetCpuCastFromFloat(DataType dst_dtype);
|
||||
|
||||
CastFunctorType GetCpuCastFromDouble(DataType dst_dtype);
|
||||
|
||||
CastFunctorType GetCpuCastFromComplex64(DataType dst_dtype);
|
||||
|
||||
CastFunctorType GetCpuCastFromComplex128(DataType dst_dtype);
|
||||
|
||||
CastFunctorType GetCpuCastFromBfloat(DataType dst_dtype);
|
||||
|
||||
#if GOOGLE_CUDA
|
||||
// Same, for GPU.
|
||||
CastFunctorType GetGpuCastFromBool(DataType dst_dtype);
|
||||
|
||||
CastFunctorType GetGpuCastFromUint8(DataType dst_dtype);
|
||||
|
||||
CastFunctorType GetGpuCastFromUint16(DataType dst_dtype);
|
||||
|
||||
CastFunctorType GetGpuCastFromInt8(DataType dst_dtype);
|
||||
|
||||
CastFunctorType GetGpuCastFromUint32(DataType dst_dtype);
|
||||
|
||||
CastFunctorType GetGpuCastFromUint64(DataType dst_dtype);
|
||||
|
||||
CastFunctorType GetGpuCastFromInt16(DataType dst_dtype);
|
||||
|
||||
CastFunctorType GetGpuCastFromInt32(DataType dst_dtype);
|
||||
|
||||
CastFunctorType GetGpuCastFromInt64(DataType dst_dtype);
|
||||
|
||||
CastFunctorType GetGpuCastFromHalf(DataType dst_dtype);
|
||||
|
||||
CastFunctorType GetGpuCastFromFloat(DataType dst_dtype);
|
||||
|
||||
CastFunctorType GetGpuCastFromDouble(DataType dst_dtype);
|
||||
|
||||
CastFunctorType GetGpuCastFromComplex64(DataType dst_dtype);
|
||||
|
||||
CastFunctorType GetGpuCastFromComplex128(DataType dst_dtype);
|
||||
|
||||
CastFunctorType GetGpuCastFromBfloat(DataType dst_dtype);
|
||||
|
||||
#endif // GOOGLE_CUDA
|
||||
|
||||
#ifdef TENSORFLOW_USE_SYCL
|
||||
CastFunctorType GetSyclCastFromBool(DataType dst_dtype);
|
||||
|
||||
CastFunctorType GetSyclCastFromUint8(DataType dst_dtype);
|
||||
|
||||
CastFunctorType GetSyclCastFromUint16(DataType dst_dtype);
|
||||
|
||||
CastFunctorType GetSyclCastFromUint32(DataType dst_dtype);
|
||||
|
||||
CastFunctorType GetSyclCastFromUint64(DataType dst_dtype);
|
||||
|
||||
CastFunctorType GetSyclCastFromInt16(DataType dst_dtype);
|
||||
|
||||
CastFunctorType GetSyclCastFromInt32(DataType dst_dtype);
|
||||
|
||||
CastFunctorType GetSyclCastFromInt64(DataType dst_dtype);
|
||||
|
||||
CastFunctorType GetSyclCastFromFloat(DataType dst_dtype);
|
||||
|
||||
CastFunctorType GetSyclCastFromDouble(DataType dst_dtype);
|
||||
#endif // TENSORFLOW_USE_SYCL
|
||||
|
||||
} // namespace tensorflow
|
||||
|
||||
#endif // TENSORFLOW_CORE_KERNELS_CAST_OP_IMPL_H_
|
||||
@@ -0,0 +1,94 @@
|
||||
/* Copyright 2015 The TensorFlow Authors. All Rights Reserved.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
==============================================================================*/
|
||||
|
||||
#ifndef TENSORFLOW_CORE_KERNELS_COLORSPACE_OP_H_
|
||||
#define TENSORFLOW_CORE_KERNELS_COLORSPACE_OP_H_
|
||||
|
||||
#include "third_party/eigen3/unsupported/Eigen/CXX11/Tensor"
|
||||
#include "tensorflow/core/framework/tensor_shape.h"
|
||||
#include "tensorflow/core/framework/tensor_types.h"
|
||||
|
||||
namespace tensorflow {
|
||||
|
||||
namespace functor {
|
||||
|
||||
template <typename Device, typename T>
|
||||
struct RGBToHSV {
|
||||
void operator()(const Device &d,
|
||||
typename TTypes<T, 2>::ConstTensor input_data,
|
||||
typename TTypes<T, 1>::Tensor range,
|
||||
typename TTypes<T, 2>::Tensor output_data) {
|
||||
auto H = output_data.template chip<1>(0);
|
||||
auto S = output_data.template chip<1>(1);
|
||||
auto V = output_data.template chip<1>(2);
|
||||
|
||||
auto R = input_data.template chip<1>(0);
|
||||
auto G = input_data.template chip<1>(1);
|
||||
auto B = input_data.template chip<1>(2);
|
||||
|
||||
#if !defined(EIGEN_HAS_INDEX_LIST)
|
||||
Eigen::array<int, 1> channel_axis{{1}};
|
||||
#else
|
||||
Eigen::IndexList<Eigen::type2index<1> > channel_axis;
|
||||
#endif
|
||||
|
||||
V.device(d) = input_data.maximum(channel_axis);
|
||||
|
||||
range.device(d) = V - input_data.minimum(channel_axis);
|
||||
|
||||
S.device(d) = (V > T(0)).select(range / V, V.constant(T(0)));
|
||||
|
||||
auto norm = range.inverse() * (T(1) / T(6));
|
||||
// TODO(wicke): all these assignments are only necessary because a combined
|
||||
// expression is larger than kernel parameter space. A custom kernel is
|
||||
// probably in order.
|
||||
H.device(d) = (R == V).select(
|
||||
norm * (G - B), (G == V).select(norm * (B - R) + T(2) / T(6),
|
||||
norm * (R - G) + T(4) / T(6)));
|
||||
H.device(d) = (range > T(0)).select(H, H.constant(T(0)));
|
||||
H.device(d) = (H < T(0)).select(H + T(1), H);
|
||||
}
|
||||
};
|
||||
|
||||
template <typename Device, typename T>
|
||||
struct HSVToRGB {
|
||||
void operator()(const Device &d,
|
||||
typename TTypes<T, 2>::ConstTensor input_data,
|
||||
typename TTypes<T, 2>::Tensor output_data) {
|
||||
auto H = input_data.template chip<1>(0);
|
||||
auto S = input_data.template chip<1>(1);
|
||||
auto V = input_data.template chip<1>(2);
|
||||
|
||||
// TODO(wicke): compute only the fractional part of H for robustness
|
||||
auto dh = H * T(6);
|
||||
auto dr = ((dh - T(3)).abs() - T(1)).cwiseMax(T(0)).cwiseMin(T(1));
|
||||
auto dg = (-(dh - T(2)).abs() + T(2)).cwiseMax(T(0)).cwiseMin(T(1));
|
||||
auto db = (-(dh - T(4)).abs() + T(2)).cwiseMax(T(0)).cwiseMin(T(1));
|
||||
auto one_s = -S + T(1);
|
||||
|
||||
auto R = output_data.template chip<1>(0);
|
||||
auto G = output_data.template chip<1>(1);
|
||||
auto B = output_data.template chip<1>(2);
|
||||
|
||||
R.device(d) = (one_s + S * dr) * V;
|
||||
G.device(d) = (one_s + S * dg) * V;
|
||||
B.device(d) = (one_s + S * db) * V;
|
||||
}
|
||||
};
|
||||
|
||||
} // namespace functor
|
||||
} // namespace tensorflow
|
||||
|
||||
#endif // TENSORFLOW_CORE_KERNELS_COLORSPACE_OP_H_
|
||||
@@ -0,0 +1,42 @@
|
||||
/* Copyright 2017 The TensorFlow Authors. All Rights Reserved.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
==============================================================================*/
|
||||
|
||||
#ifndef TENSORFLOW_CORE_KERNELS_COMPARE_AND_BITPACK_OP_H_
|
||||
#define TENSORFLOW_CORE_KERNELS_COMPARE_AND_BITPACK_OP_H_
|
||||
|
||||
#include "third_party/eigen3/unsupported/Eigen/CXX11/Tensor"
|
||||
#include "tensorflow/core/framework/op_kernel.h"
|
||||
#include "tensorflow/core/framework/tensor_types.h"
|
||||
#include "tensorflow/core/platform/types.h"
|
||||
|
||||
namespace tensorflow {
|
||||
|
||||
namespace functor {
|
||||
|
||||
typedef Eigen::ThreadPoolDevice CPUDevice;
|
||||
typedef Eigen::GpuDevice GPUDevice;
|
||||
|
||||
template <typename Device, typename T>
|
||||
struct CompareAndBitpack {
|
||||
void operator()(OpKernelContext* c, typename TTypes<T>::ConstMatrix input,
|
||||
typename TTypes<T>::ConstScalar threshold,
|
||||
TTypes<uint8>::Matrix output);
|
||||
};
|
||||
|
||||
} // namespace functor
|
||||
|
||||
} // namespace tensorflow
|
||||
|
||||
#endif // TENSORFLOW_CORE_KERNELS_COMPARE_AND_BITPACK_OP_H_
|
||||
@@ -0,0 +1,69 @@
|
||||
/* Copyright 2015 The TensorFlow Authors. All Rights Reserved.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
==============================================================================*/
|
||||
|
||||
#ifndef TENSORFLOW_CORE_KERNELS_CONCAT_LIB_H_
|
||||
#define TENSORFLOW_CORE_KERNELS_CONCAT_LIB_H_
|
||||
|
||||
#include <vector>
|
||||
|
||||
#include "third_party/eigen3/unsupported/Eigen/CXX11/Tensor"
|
||||
#include "tensorflow/core/framework/device_base.h"
|
||||
|
||||
namespace tensorflow {
|
||||
|
||||
// Functors to concatenate tensors. These always take a rank-2 tensor (i.e a
|
||||
// matrix) and concatenate it along the axis 1 ("putting them next to each
|
||||
// other" as opposed to "putting them on top of one another").
|
||||
//
|
||||
// Any concatenation of n-dimensional tensors across any axis can be reduced to
|
||||
// a concatenation of two-dimensional tensors across the axis 1 by first
|
||||
// partitioning the axes of the original tensors into those less than the axis
|
||||
// to be concatenated across and the rest. Then reshape the tensors into a
|
||||
// two-dimensional tensor by collapsing these two sets of axes and concatenate
|
||||
// the resulting matrices across the axis 1, finally reshaping the result to
|
||||
// have the proper shape.
|
||||
//
|
||||
// So, for example, when stacking N tensors, reshape each to have shape
|
||||
// {1, Numelements} and reshape the result matrix to have shape
|
||||
// {1, N * NumElements} before passing it to this functor.
|
||||
|
||||
// Assumes all inputs are nonempty
|
||||
template <typename T>
|
||||
void ConcatCPU(
|
||||
DeviceBase* d,
|
||||
const std::vector<std::unique_ptr<typename TTypes<T, 2>::ConstMatrix>>&
|
||||
inputs,
|
||||
typename TTypes<T, 2>::Matrix* output);
|
||||
#if GOOGLE_CUDA
|
||||
template <typename T>
|
||||
void ConcatGPU(
|
||||
OpKernelContext* c,
|
||||
const std::vector<std::unique_ptr<typename TTypes<T, 2>::ConstMatrix>>&
|
||||
inputs_flat,
|
||||
Tensor* output, typename TTypes<T, 2>::Tensor* output_flat);
|
||||
|
||||
#endif // GOOGLE_CUDA
|
||||
|
||||
#ifdef TENSORFLOW_USE_SYCL
|
||||
template <typename T>
|
||||
void ConcatSYCL(
|
||||
const Eigen::SyclDevice& d,
|
||||
const std::vector<std::unique_ptr<typename TTypes<T, 2>::ConstMatrix>>&
|
||||
inputs,
|
||||
typename TTypes<T, 2>::Matrix* output);
|
||||
#endif // TENSORFLOW_USE_SYCL
|
||||
} // namespace tensorflow
|
||||
|
||||
#endif // TENSORFLOW_CORE_KERNELS_CONCAT_LIB_H_
|
||||
@@ -0,0 +1,169 @@
|
||||
/* Copyright 2015 The TensorFlow Authors. All Rights Reserved.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
==============================================================================*/
|
||||
|
||||
#ifndef TENSORFLOW_CORE_KERNELS_CONCAT_LIB_CPU_H_
|
||||
#define TENSORFLOW_CORE_KERNELS_CONCAT_LIB_CPU_H_
|
||||
|
||||
#define EIGEN_USE_THREADS
|
||||
|
||||
#include <vector>
|
||||
#include "tensorflow/core/framework/register_types.h"
|
||||
#include "tensorflow/core/kernels/concat_lib.h"
|
||||
#include "tensorflow/core/util/work_sharder.h"
|
||||
|
||||
namespace tensorflow {
|
||||
|
||||
// ElementCopier must be a struct with a single Copy function, which is passed
|
||||
// the output pointer, input pointer, input index, and number of elements to
|
||||
// copy from input to output.
|
||||
template <typename T, typename ElementCopier>
|
||||
void ConcatCPUImpl(
|
||||
DeviceBase* d,
|
||||
const std::vector<std::unique_ptr<typename TTypes<T, 2>::ConstMatrix>>&
|
||||
inputs,
|
||||
int64 cost_per_unit, ElementCopier copier,
|
||||
typename TTypes<T, 2>::Matrix* output) {
|
||||
size_t num_inputs = inputs.size();
|
||||
|
||||
std::vector<ptrdiff_t> sizes;
|
||||
sizes.reserve(num_inputs);
|
||||
int64 row_size = 0;
|
||||
for (const auto& input : inputs) {
|
||||
sizes.push_back(input->dimension(1));
|
||||
row_size += sizes.back();
|
||||
}
|
||||
|
||||
auto worker_threads = d->tensorflow_cpu_worker_threads();
|
||||
int num_threads = std::min(4, worker_threads->num_threads);
|
||||
// strings define a different amount of work (generally much more) compared
|
||||
// with standard POD, so we parallelize differently.
|
||||
if (!std::is_same<T, string>::value) {
|
||||
num_threads =
|
||||
static_cast<int>(std::min<int64>(num_threads, output->size() / 4096));
|
||||
}
|
||||
// Single threaded mode.
|
||||
// TODO(dga): Deduplicate this code w.r.t. sharded code below.
|
||||
if (num_threads == 0) {
|
||||
T* out = &(*output)(0, 0);
|
||||
std::vector<const T*> inp;
|
||||
inp.reserve(num_inputs);
|
||||
for (const auto& input : inputs) {
|
||||
inp.push_back(&(*input)(0, 0));
|
||||
}
|
||||
const int64 dim0 = output->dimension(0);
|
||||
for (int64 i = 0; i < dim0; ++i) {
|
||||
for (int64 j = 0; j < num_inputs; ++j) {
|
||||
auto size = sizes[j];
|
||||
copier.Copy(out, inp[j], j, size);
|
||||
out += size;
|
||||
inp[j] += size;
|
||||
}
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
// Sharded mode.
|
||||
auto work = [&row_size, &sizes, &inputs, &output, &copier, &num_inputs](
|
||||
int64 start, int64 end) {
|
||||
int64 skipped_rows = start / row_size;
|
||||
T* out = output->data() + skipped_rows * row_size;
|
||||
T* out_start = output->data() + start;
|
||||
T* out_end = output->data() + end;
|
||||
|
||||
// Handle partial row at start
|
||||
if (out < out_start) {
|
||||
for (size_t j = 0; j < num_inputs; ++j) {
|
||||
ptrdiff_t size = sizes[j];
|
||||
ptrdiff_t offset = out_start - out;
|
||||
if (size <= offset) {
|
||||
out += size;
|
||||
continue;
|
||||
}
|
||||
const T* inp = &(*inputs[j])(skipped_rows, 0);
|
||||
if (offset > 0) {
|
||||
out += offset;
|
||||
inp += offset;
|
||||
size -= offset;
|
||||
}
|
||||
size = std::min(size, out_end - out);
|
||||
if (size <= 0) break;
|
||||
copier.Copy(out, inp, j, size);
|
||||
out += size;
|
||||
}
|
||||
++skipped_rows;
|
||||
}
|
||||
if (out == out_end) return;
|
||||
CHECK(out >= out_start);
|
||||
CHECK(out < out_end);
|
||||
|
||||
// Copy remaining data.
|
||||
std::vector<const T*> inp;
|
||||
inp.reserve(num_inputs);
|
||||
for (const auto& input : inputs) {
|
||||
inp.push_back(&(*input)(skipped_rows, 0));
|
||||
}
|
||||
const int64 dim0 = output->dimension(0);
|
||||
for (int64 i = skipped_rows; i < dim0; ++i) {
|
||||
for (int64 j = 0; j < num_inputs; ++j) {
|
||||
ptrdiff_t size = std::min(sizes[j], out_end - out);
|
||||
copier.Copy(out, inp[j], j, size);
|
||||
out += size;
|
||||
inp[j] += size;
|
||||
if (out == out_end) return;
|
||||
}
|
||||
}
|
||||
};
|
||||
Shard(worker_threads->num_threads, worker_threads->workers, output->size(),
|
||||
cost_per_unit, work);
|
||||
}
|
||||
|
||||
#ifdef TENSORFLOW_USE_SYCL
|
||||
template <typename T, typename ElementCopier>
|
||||
void ConcatSYCLImpl(
|
||||
const Eigen::SyclDevice& d,
|
||||
const std::vector<std::unique_ptr<typename TTypes<T, 2>::ConstMatrix>>&
|
||||
inputs,
|
||||
int64 cost_per_unit, ElementCopier copier,
|
||||
typename TTypes<T, 2>::Matrix* output) {
|
||||
size_t num_inputs = inputs.size();
|
||||
|
||||
std::vector<ptrdiff_t> sizes;
|
||||
sizes.reserve(num_inputs);
|
||||
int64 row_size = 0;
|
||||
for (const auto& input : inputs) {
|
||||
sizes.push_back(input->dimension(1));
|
||||
row_size += sizes.back();
|
||||
}
|
||||
|
||||
T* out = &(*output)(0, 0);
|
||||
std::vector<const T*> inp;
|
||||
inp.reserve(num_inputs);
|
||||
for (const auto& input : inputs) {
|
||||
inp.push_back(&(*input)(0, 0));
|
||||
}
|
||||
const int64 dim0 = output->dimension(0);
|
||||
for (int64 i = 0; i < dim0; ++i) {
|
||||
for (int64 j = 0; j < num_inputs; ++j) {
|
||||
auto size = sizes[j];
|
||||
d.memcpy(out, inp[j], size * sizeof(T));
|
||||
out += size;
|
||||
inp[j] += size;
|
||||
}
|
||||
}
|
||||
}
|
||||
#endif // TENSORFLOW_USE_SYCL
|
||||
} // namespace tensorflow
|
||||
|
||||
#endif // TENSORFLOW_CORE_KERNELS_CONCAT_LIB_CPU_H_
|
||||
@@ -0,0 +1,138 @@
|
||||
/* Copyright 2015 The TensorFlow Authors. All Rights Reserved.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
==============================================================================*/
|
||||
|
||||
#ifndef TENSORFLOW_CORE_KERNELS_CONDITIONAL_ACCUMULATOR_H_
|
||||
#define TENSORFLOW_CORE_KERNELS_CONDITIONAL_ACCUMULATOR_H_
|
||||
|
||||
#include "tensorflow/core/kernels/fill_functor.h"
|
||||
#include "tensorflow/core/kernels/typed_conditional_accumulator_base.h"
|
||||
|
||||
namespace tensorflow {
|
||||
|
||||
/**
|
||||
* An aggregation object for adding dense gradients.
|
||||
*
|
||||
* The two main methods of this class are TryApplyGrad and TryTakeGrad.
|
||||
*
|
||||
* TryApplyGrad tries add a gradient to the accumulator. The attempt is
|
||||
* successful if local_step >= global_step, i.e., if the gradient is not stale,
|
||||
* having been computed using up-to-date information. Otherwise, the gradient is
|
||||
* silently dropped.
|
||||
*
|
||||
* TryTakeGrad logs an attempt to read the average gradient. The attempt is
|
||||
* blocked until the number of gradients accumulated (via TryApplyGrad) is equal
|
||||
* or exceeds the number requested by TryTakeGrad.
|
||||
* Once this condition is satisfied, the following actions are taken:
|
||||
* (1) the value of the average gradient is returned
|
||||
* (2) the count of accumulated gradients is reset to 0
|
||||
* (3) the internal global_step value (current_global_step_) is incremented by 1
|
||||
*
|
||||
* ConditionalAccumulator is the datatype-dependent templated sub-class of
|
||||
* ConditionalAccumulatorBase. It implements the virtual arithmetic methods that
|
||||
* are used by for aggregating, averaging, allocating, returning dense Tensors.
|
||||
*/
|
||||
template <typename Device, typename T>
|
||||
class ConditionalAccumulator
|
||||
: public TypedConditionalAccumulatorBase<const Tensor> {
|
||||
public:
|
||||
// Args:
|
||||
// dtype: The datatype of the gradients to be accumulated.
|
||||
// shape: The shape of the accumulated gradients.
|
||||
// name: A name to use for the ConditionalAccumulator.
|
||||
// reduction_type: The reduction type, i.e., MEAN or SUM
|
||||
ConditionalAccumulator(const DataType& dtype, const PartialTensorShape& shape,
|
||||
const string& name, const string& reduction_type)
|
||||
: TypedConditionalAccumulatorBase<const Tensor>(dtype, shape, name,
|
||||
reduction_type) {}
|
||||
~ConditionalAccumulator() override{};
|
||||
|
||||
protected:
|
||||
// accum_grad is the tensor that holds the aggregate gradient.
|
||||
// It is initialized the first time ApplyGrad is called.
|
||||
Tensor* accum_grad_ = nullptr;
|
||||
PersistentTensor accum_grad_persistent_;
|
||||
|
||||
functor::SetZeroFunctor<Device, T> set_zero_functor_;
|
||||
|
||||
Status ValidateShape(const Tensor* tensor)
|
||||
EXCLUSIVE_LOCKS_REQUIRED(this->mu_) {
|
||||
// Must be compatible with accumulated gradient if available
|
||||
if (counter_ > 0) {
|
||||
if (!accum_grad_->shape().IsSameSize(tensor->shape())) {
|
||||
return errors::InvalidArgument("Shape mismatch: expected ",
|
||||
accum_grad_->shape().DebugString(),
|
||||
", got ", tensor->shape().DebugString());
|
||||
}
|
||||
}
|
||||
// Must also be compatible with given shape
|
||||
if (!shape_.IsCompatibleWith(tensor->shape())) {
|
||||
return errors::InvalidArgument("Shape mismatch: expected ",
|
||||
shape_.DebugString(), ", got ",
|
||||
tensor->shape().DebugString());
|
||||
}
|
||||
return Status::OK();
|
||||
}
|
||||
|
||||
void AllocateAndAssignToAccumGradFunction(OpKernelContext* ctx,
|
||||
const Tensor* grad) override {
|
||||
// TODO(b/32704451): Don't just ignore the ::tensorflow::Status object!
|
||||
ctx->allocate_persistent(dtype_, grad->shape(), &accum_grad_persistent_,
|
||||
&accum_grad_)
|
||||
.IgnoreError();
|
||||
accum_grad_->flat<T>().device(ctx->template eigen_device<Device>()) =
|
||||
grad->flat<T>();
|
||||
}
|
||||
|
||||
void AddToAccumGradFunction(OpKernelContext* ctx,
|
||||
const Tensor* grad) override {
|
||||
accum_grad_->flat<T>().device(ctx->template eigen_device<Device>()) +=
|
||||
grad->flat<T>();
|
||||
}
|
||||
|
||||
void DivideAccumGradByCounter(OpKernelContext* ctx) override
|
||||
EXCLUSIVE_LOCKS_REQUIRED(this->mu_) {
|
||||
Tensor c(DataTypeToEnum<T>::value, {});
|
||||
c.scalar<T>()() = TypeConverter<T, int>::ConvertUToT(this->counter_);
|
||||
this->accum_grad_->template flat<T>().device(
|
||||
ctx->template eigen_device<Device>()) =
|
||||
this->accum_grad_->template flat<T>() / c.scalar<T>()();
|
||||
}
|
||||
|
||||
bool SetOutput(OpKernelContext* ctx) override {
|
||||
ctx->set_output(0, *accum_grad_);
|
||||
return true;
|
||||
}
|
||||
|
||||
bool GetAndValidateTensorInputForApplyGrad(OpKernelContext* ctx,
|
||||
const Tensor** tensor) override
|
||||
EXCLUSIVE_LOCKS_REQUIRED(this->mu_) {
|
||||
// Get input gradient tensor
|
||||
const Tensor* grad_tensor;
|
||||
OP_REQUIRES_OK_BOOLEAN(ctx, ctx->input("gradient", &grad_tensor));
|
||||
*tensor = grad_tensor;
|
||||
OP_REQUIRES_OK_BOOLEAN(ctx, this->ValidateShape(*tensor));
|
||||
return true;
|
||||
}
|
||||
|
||||
void CleanUpGradTensor(const Tensor* tensor) override {
|
||||
// do nothing
|
||||
}
|
||||
|
||||
TF_DISALLOW_COPY_AND_ASSIGN(ConditionalAccumulator);
|
||||
};
|
||||
|
||||
} // namespace tensorflow
|
||||
|
||||
#endif // TENSORFLOW_CORE_KERNELS_CONDITIONAL_ACCUMULATOR_H_
|
||||
@@ -0,0 +1,203 @@
|
||||
/* Copyright 2015 The TensorFlow Authors. All Rights Reserved.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
==============================================================================*/
|
||||
|
||||
#ifndef TENSORFLOW_CORE_KERNELS_CONDITIONAL_ACCUMULATOR_BASE_H_
|
||||
#define TENSORFLOW_CORE_KERNELS_CONDITIONAL_ACCUMULATOR_BASE_H_
|
||||
|
||||
#include <deque>
|
||||
|
||||
#include "third_party/eigen3/unsupported/Eigen/CXX11/Tensor"
|
||||
#include "tensorflow/core/framework/numeric_op.h"
|
||||
|
||||
#include "tensorflow/core/framework/op_kernel.h"
|
||||
#include "tensorflow/core/framework/resource_mgr.h"
|
||||
|
||||
namespace tensorflow {
|
||||
|
||||
/**
|
||||
* ConditionalAccumulator/ConditionalAccumulatorBase implements an aggregation
|
||||
* object for adding gradients.
|
||||
* The two main methods of this class are TryApplyGrad and TryTakeGrad.
|
||||
*
|
||||
* TryApplyGrad tries add a gradient to the accumulator. The attempt is
|
||||
* successful if local_step >= global_step, i.e., if the gradient is not stale,
|
||||
* having been computed using up-to-date information. Otherwise, the gradient is
|
||||
* silently dropped.
|
||||
*
|
||||
* TryTakeGrad logs an attempt to read the average gradient. The attempt is
|
||||
* blocked until the number of gradients accumulated (via TryApplyGrad) is equal
|
||||
* or exceeds the number requested by TryTakeGrad.
|
||||
* Once this condition is satisfied, the following actions are taken:
|
||||
* (1) the value of the average gradient is returned
|
||||
* (2) the count of accumulated gradients is reset to 0
|
||||
* (3) the internal global_step value (current_global_step_) is incremented by 1
|
||||
*/
|
||||
class ConditionalAccumulatorBase : public ResourceBase {
|
||||
public:
|
||||
// Args:
|
||||
// dtype: The datatype of the gradients to be accumulated.
|
||||
// shape: The shape of the accumulated gradients.
|
||||
// name: A name to use for the ConditionalAccumulator.
|
||||
ConditionalAccumulatorBase(const DataType& dtype,
|
||||
const PartialTensorShape& shape,
|
||||
const string& name, const string& reduction_type);
|
||||
|
||||
typedef AsyncOpKernel::DoneCallback DoneCallback;
|
||||
|
||||
virtual void TryApplyGrad(int64 local_step, OpKernelContext* ctx) = 0;
|
||||
void TryTakeGrad(int num_required, OpKernelContext* ctx,
|
||||
DoneCallback callback);
|
||||
|
||||
// Accessor methods
|
||||
uint32 num_accumulated() {
|
||||
mutex_lock lock(mu_);
|
||||
return counter_;
|
||||
}
|
||||
|
||||
const DataType& dtype() const { return dtype_; }
|
||||
|
||||
string DebugString() override { return "A conditional accumulator"; }
|
||||
|
||||
// SetGlobalStep is a modifier method for current_global_step.
|
||||
// It returns an InvalidArgument error if the new_global_step is less than
|
||||
// current_global_step.
|
||||
Status SetGlobalStep(int64 new_global_step);
|
||||
|
||||
Status MatchesNodeDef(const NodeDef& node_def);
|
||||
|
||||
protected:
|
||||
// Virtual methods to be implemented by sub-classes for different datatypes.
|
||||
// Implements arithmetic operations specific to datatype.
|
||||
virtual void DivideAccumGradByCounter(OpKernelContext* ctx)
|
||||
EXCLUSIVE_LOCKS_REQUIRED(mu_) = 0;
|
||||
virtual bool SetOutput(OpKernelContext* ctx) = 0;
|
||||
|
||||
enum RunResult { kNoProgress, kComplete };
|
||||
|
||||
// Helper struct holding information about a TakeGrad attempt
|
||||
struct Attempt;
|
||||
typedef std::function<RunResult(Attempt*)> RunCallback;
|
||||
struct Attempt {
|
||||
int elements_requested;
|
||||
DoneCallback done_callback; // must be run outside mu_
|
||||
OpKernelContext* context;
|
||||
CancellationManager* cancellation_manager; // not owned
|
||||
CancellationToken cancellation_token;
|
||||
RunCallback run_callback; // must be run while holding mu_
|
||||
bool is_cancelled;
|
||||
|
||||
Attempt(int elements_requested, DoneCallback done_callback,
|
||||
OpKernelContext* context, CancellationManager* cancellation_manager,
|
||||
CancellationToken cancellation_token, RunCallback run_callback)
|
||||
: elements_requested(elements_requested),
|
||||
done_callback(std::move(done_callback)),
|
||||
context(context),
|
||||
cancellation_manager(cancellation_manager),
|
||||
cancellation_token(cancellation_token),
|
||||
run_callback(std::move(run_callback)),
|
||||
is_cancelled(false) {}
|
||||
};
|
||||
|
||||
// Helper struct for deregistration of a cancellation token and executing a
|
||||
// DoneCallback after a TakeGrad attempt is complete.
|
||||
struct CleanUp {
|
||||
CleanUp(DoneCallback&& f, CancellationToken ct, CancellationManager* cm)
|
||||
: finished(f), to_deregister(ct), cm(cm) {}
|
||||
DoneCallback finished;
|
||||
CancellationToken to_deregister;
|
||||
CancellationManager* cm;
|
||||
};
|
||||
|
||||
// Fields
|
||||
|
||||
const DataType dtype_;
|
||||
const PartialTensorShape shape_;
|
||||
const string name_;
|
||||
const string reduction_type_;
|
||||
mutex mu_;
|
||||
int counter_ GUARDED_BY(mu_);
|
||||
int64 current_global_step_ GUARDED_BY(mu_);
|
||||
|
||||
std::deque<Attempt> takegrad_attempts_ GUARDED_BY(mu_);
|
||||
|
||||
// Methods
|
||||
|
||||
// Helper function for creating cancellation callback
|
||||
void Cancel(CancellationManager* cancellation_manager,
|
||||
CancellationToken token);
|
||||
|
||||
// Helper functions to process TakeGrad attempts.
|
||||
// FlushUnlocked is called at the end of each TryApplyGrad and TryTakeGrad
|
||||
// calls to try to clear the TakeGrad attempts. This in turn calls
|
||||
// TryAttemptLocked, which then executes the RunCallback of the logged
|
||||
// attempts.
|
||||
// Both functions are modeled after core/kernels/queue_base.
|
||||
// Note: ApplyGrad attempts never block -- unlike in a queue with limited
|
||||
// capacity, we can always add the newest gradient to our accumulator
|
||||
// (if it is not stale) or drop it silently (if it is stale).
|
||||
void FlushUnlocked();
|
||||
bool TryAttemptLocked(std::vector<CleanUp>* clean_up)
|
||||
EXCLUSIVE_LOCKS_REQUIRED(mu_);
|
||||
|
||||
// Helper methods
|
||||
// void DeepCopy(Tensor* dst);
|
||||
bool TakeGradLockedHelper(OpKernelContext* ctx, DoneCallback callback)
|
||||
EXCLUSIVE_LOCKS_REQUIRED(mu_);
|
||||
};
|
||||
|
||||
/*
|
||||
* Modifications to convenience macros defined in core/framework/op_kernel.h.
|
||||
* The below macros return a boolean if the test fails, so that the calling
|
||||
* function can get an indication that a failure has occurred.
|
||||
*/
|
||||
#define OP_REQUIRES_BOOLEAN(CTX, EXP, STATUS) \
|
||||
do { \
|
||||
if (!TF_PREDICT_TRUE(EXP)) { \
|
||||
(CTX)->CtxFailure(__FILE__, __LINE__, (STATUS)); \
|
||||
return false; \
|
||||
} \
|
||||
} while (0)
|
||||
|
||||
#define OP_REQUIRES_OK_BOOLEAN(CTX, STATUS) \
|
||||
do { \
|
||||
::tensorflow::Status _s(STATUS); \
|
||||
if (!TF_PREDICT_TRUE(_s.ok())) { \
|
||||
(CTX)->CtxFailureWithWarning(__FILE__, __LINE__, _s); \
|
||||
return false; \
|
||||
} \
|
||||
} while (0)
|
||||
|
||||
/*
|
||||
* Convenience classes for helping to convert between numeric types.
|
||||
* The specialization for Eigen::half here simplifies specialization of
|
||||
* ConditionalAccumulator classes later.
|
||||
*/
|
||||
template <typename T, typename U>
|
||||
class TypeConverter {
|
||||
public:
|
||||
static T ConvertUToT(U c) { return c; /* implicit conversion */ }
|
||||
};
|
||||
|
||||
template <typename U>
|
||||
class TypeConverter<Eigen::half, U> {
|
||||
public:
|
||||
static Eigen::half ConvertUToT(U c) {
|
||||
return Eigen::half_impl::float_to_half_rtne(c);
|
||||
}
|
||||
};
|
||||
|
||||
} // namespace tensorflow
|
||||
|
||||
#endif // TENSORFLOW_CORE_KERNELS_CONDITIONAL_ACCUMULATOR_BASE_H_
|
||||
@@ -0,0 +1,240 @@
|
||||
/* Copyright 2015 The TensorFlow Authors. All Rights Reserved.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
==============================================================================*/
|
||||
|
||||
#ifndef TENSORFLOW_CORE_KERNELS_CONDITIONAL_ACCUMULATOR_BASE_OP_H_
|
||||
#define TENSORFLOW_CORE_KERNELS_CONDITIONAL_ACCUMULATOR_BASE_OP_H_
|
||||
|
||||
#define EIGEN_USE_THREADS
|
||||
|
||||
#include "tensorflow/core/kernels/conditional_accumulator_base.h"
|
||||
|
||||
#include "tensorflow/core/framework/op_kernel.h"
|
||||
#include "tensorflow/core/framework/register_types.h"
|
||||
#include "tensorflow/core/framework/resource_mgr.h"
|
||||
#include "tensorflow/core/framework/tensor.h"
|
||||
#include "tensorflow/core/framework/tensor_shape.h"
|
||||
#include "tensorflow/core/framework/types.h"
|
||||
#include "tensorflow/core/lib/core/errors.h"
|
||||
#include "tensorflow/core/platform/macros.h"
|
||||
#include "tensorflow/core/platform/mutex.h"
|
||||
#include "tensorflow/core/platform/thread_annotations.h"
|
||||
#include "tensorflow/core/platform/types.h"
|
||||
|
||||
typedef Eigen::ThreadPoolDevice CPUDevice;
|
||||
|
||||
typedef std::function<void()> DoneCallback;
|
||||
|
||||
namespace tensorflow {
|
||||
|
||||
/**
|
||||
* Defines a ConditionalAccumulatorBaseOp, which constructs a
|
||||
* ConditionalAccumulatorBase (via sub-class's Creator) and returns its handle.
|
||||
*/
|
||||
class ConditionalAccumulatorBaseOp : public OpKernel {
|
||||
public:
|
||||
explicit ConditionalAccumulatorBaseOp(OpKernelConstruction* context)
|
||||
: OpKernel(context), accumulator_handle_set_(false) {
|
||||
OP_REQUIRES_OK(context,
|
||||
context->allocate_persistent(DT_STRING, TensorShape({2}),
|
||||
&accumulator_handle_, nullptr));
|
||||
OP_REQUIRES_OK(context, context->GetAttr("shape", &shape_));
|
||||
OP_REQUIRES_OK(context, context->GetAttr("dtype", &dtype_));
|
||||
OP_REQUIRES_OK(context,
|
||||
context->GetAttr("reduction_type", &reduction_type_));
|
||||
}
|
||||
|
||||
void Compute(OpKernelContext* ctx) override {
|
||||
mutex_lock l(mu_);
|
||||
if (!accumulator_handle_set_) {
|
||||
OP_REQUIRES_OK(ctx, SetAccumulatorHandle(ctx));
|
||||
}
|
||||
ctx->set_output_ref(0, &mu_, accumulator_handle_.AccessTensor(ctx));
|
||||
}
|
||||
|
||||
protected:
|
||||
~ConditionalAccumulatorBaseOp() override {
|
||||
// If the accumulator object was not shared, delete it.
|
||||
if (accumulator_handle_set_ && cinfo_.resource_is_private_to_kernel()) {
|
||||
TF_CHECK_OK((cinfo_.resource_manager()
|
||||
->template Delete<ConditionalAccumulatorBase>(
|
||||
cinfo_.container(), cinfo_.name())));
|
||||
}
|
||||
}
|
||||
|
||||
protected:
|
||||
typedef std::function<Status(ConditionalAccumulatorBase**)> Creator;
|
||||
|
||||
// Subclasses must override this
|
||||
virtual Creator GetCreator() const = 0;
|
||||
|
||||
// Variables required to construct ConditionalAccumulator
|
||||
DataType dtype_;
|
||||
PartialTensorShape shape_;
|
||||
ContainerInfo cinfo_;
|
||||
string reduction_type_;
|
||||
|
||||
private:
|
||||
Status SetAccumulatorHandle(OpKernelContext* ctx)
|
||||
EXCLUSIVE_LOCKS_REQUIRED(mu_) {
|
||||
TF_RETURN_IF_ERROR(cinfo_.Init(ctx->resource_manager(), def()));
|
||||
|
||||
// Check input signature
|
||||
DataTypeVector expected_inputs = {};
|
||||
TF_RETURN_IF_ERROR(ctx->MatchSignature(expected_inputs, {DT_STRING_REF}));
|
||||
|
||||
Creator creator = GetCreator();
|
||||
ConditionalAccumulatorBase* accumulator;
|
||||
TF_RETURN_IF_ERROR(
|
||||
(cinfo_.resource_manager()
|
||||
->template LookupOrCreate<ConditionalAccumulatorBase>(
|
||||
cinfo_.container(), cinfo_.name(), &accumulator, creator)));
|
||||
core::ScopedUnref unref_me(accumulator);
|
||||
|
||||
// Verify that the shared accumulator is compatible
|
||||
// with the requested arguments.
|
||||
TF_RETURN_IF_ERROR(accumulator->MatchesNodeDef(def()));
|
||||
auto h = accumulator_handle_.AccessTensor(ctx)->template flat<string>();
|
||||
h(0) = cinfo_.container();
|
||||
h(1) = cinfo_.name();
|
||||
accumulator_handle_set_ = true;
|
||||
return Status::OK();
|
||||
}
|
||||
|
||||
mutex mu_;
|
||||
PersistentTensor accumulator_handle_ GUARDED_BY(mu_);
|
||||
bool accumulator_handle_set_ GUARDED_BY(mu_);
|
||||
};
|
||||
|
||||
/**
|
||||
* General OpKernel for ConditionalAccumulatorBase-related ops.
|
||||
*/
|
||||
class ConditionalAccumulatorBaseAsyncOpKernel : public AsyncOpKernel {
|
||||
public:
|
||||
explicit ConditionalAccumulatorBaseAsyncOpKernel(
|
||||
OpKernelConstruction* context)
|
||||
: AsyncOpKernel(context) {}
|
||||
|
||||
void ComputeAsync(OpKernelContext* ctx, DoneCallback callback) final {
|
||||
ConditionalAccumulatorBase* accumulator;
|
||||
OP_REQUIRES_OK_ASYNC(
|
||||
ctx, GetResourceFromContext(ctx, "handle", &accumulator), callback);
|
||||
ComputeAsync(ctx, accumulator, [callback, accumulator]() {
|
||||
accumulator->Unref();
|
||||
callback();
|
||||
});
|
||||
}
|
||||
|
||||
protected:
|
||||
virtual void ComputeAsync(OpKernelContext* ctx,
|
||||
ConditionalAccumulatorBase* accumulator,
|
||||
DoneCallback callback) = 0;
|
||||
};
|
||||
|
||||
/**
|
||||
* General OpKernel for ConditionalAccumulatorBase-related ops.
|
||||
*/
|
||||
class ConditionalAccumulatorBaseSyncOpKernel : public OpKernel {
|
||||
public:
|
||||
explicit ConditionalAccumulatorBaseSyncOpKernel(OpKernelConstruction* context)
|
||||
: OpKernel(context) {}
|
||||
|
||||
void Compute(OpKernelContext* ctx) final {
|
||||
ConditionalAccumulatorBase* accumulator;
|
||||
OP_REQUIRES_OK(ctx, GetResourceFromContext(ctx, "handle", &accumulator));
|
||||
Compute(ctx, accumulator);
|
||||
accumulator->Unref();
|
||||
}
|
||||
|
||||
protected:
|
||||
virtual void Compute(OpKernelContext* ctx,
|
||||
ConditionalAccumulatorBase* accumulator) = 0;
|
||||
};
|
||||
|
||||
/**
|
||||
* Defines a AccumulateGradientOp, the execution of which adds a gradient to the
|
||||
* given ConditionalAccumulator.
|
||||
*/
|
||||
class ConditionalAccumulatorBaseApplyGradientOp
|
||||
: public ConditionalAccumulatorBaseSyncOpKernel {
|
||||
public:
|
||||
explicit ConditionalAccumulatorBaseApplyGradientOp(
|
||||
OpKernelConstruction* context)
|
||||
: ConditionalAccumulatorBaseSyncOpKernel(context) {}
|
||||
|
||||
protected:
|
||||
virtual void CheckSignature(OpKernelContext* ctx,
|
||||
ConditionalAccumulatorBase* accumulator) = 0;
|
||||
|
||||
void Compute(OpKernelContext* ctx,
|
||||
ConditionalAccumulatorBase* accumulator) override {
|
||||
// Check input signature
|
||||
CheckSignature(ctx, accumulator);
|
||||
|
||||
// Get input local_step
|
||||
const Tensor* local_step_tensor;
|
||||
OP_REQUIRES_OK(ctx, ctx->input("local_step", &local_step_tensor));
|
||||
if (!TensorShapeUtils::IsScalar(local_step_tensor->shape())) {
|
||||
ctx->CtxFailureWithWarning(errors::InvalidArgument(
|
||||
"Argument local_step must be scalar, but had bad shape ",
|
||||
local_step_tensor->shape().DebugString()));
|
||||
}
|
||||
|
||||
// Actually try to apply gradient now
|
||||
accumulator->TryApplyGrad(local_step_tensor->scalar<int64>()(), ctx);
|
||||
}
|
||||
};
|
||||
|
||||
/**
|
||||
* Defines a TakeAccumulatedGradientOp, the execution of which adds a gradient
|
||||
* to the given ConditionalAccumulator.
|
||||
*/
|
||||
class ConditionalAccumulatorBaseTakeGradientOp
|
||||
: public ConditionalAccumulatorBaseAsyncOpKernel {
|
||||
public:
|
||||
explicit ConditionalAccumulatorBaseTakeGradientOp(
|
||||
OpKernelConstruction* context)
|
||||
: ConditionalAccumulatorBaseAsyncOpKernel(context) {}
|
||||
|
||||
protected:
|
||||
virtual void CheckSignature(OpKernelContext* ctx,
|
||||
ConditionalAccumulatorBase* accumulator,
|
||||
DoneCallback callback) = 0;
|
||||
|
||||
void ComputeAsync(OpKernelContext* ctx,
|
||||
ConditionalAccumulatorBase* accumulator,
|
||||
DoneCallback callback) override {
|
||||
// Check signature
|
||||
CheckSignature(ctx, accumulator, callback);
|
||||
|
||||
// Get input num_required
|
||||
const Tensor* num_required_tensor;
|
||||
OP_REQUIRES_OK_ASYNC(ctx, ctx->input("num_required", &num_required_tensor),
|
||||
callback);
|
||||
if (!TensorShapeUtils::IsScalar(num_required_tensor->shape())) {
|
||||
ctx->CtxFailureWithWarning(errors::InvalidArgument(
|
||||
"Argument num_required must be scalar, but had bad shape ",
|
||||
num_required_tensor->shape().DebugString()));
|
||||
callback();
|
||||
}
|
||||
|
||||
// Actually try to take gradient now
|
||||
accumulator->TryTakeGrad(num_required_tensor->scalar<int32>()(), ctx,
|
||||
callback);
|
||||
}
|
||||
};
|
||||
|
||||
} // namespace tensorflow
|
||||
|
||||
#endif // TENSORFLOW_CORE_KERNELS_CONDITIONAL_ACCUMULATOR_BASE_OP_H_
|
||||
@@ -0,0 +1,50 @@
|
||||
/* Copyright 2015 The TensorFlow Authors. All Rights Reserved.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
==============================================================================*/
|
||||
|
||||
#ifndef TENSORFLOW_CORE_KERNELS_CONSTANT_OP_H_
|
||||
#define TENSORFLOW_CORE_KERNELS_CONSTANT_OP_H_
|
||||
|
||||
#include "third_party/eigen3/unsupported/Eigen/CXX11/Tensor"
|
||||
#include "tensorflow/core/framework/op_kernel.h"
|
||||
#include "tensorflow/core/framework/tensor_types.h"
|
||||
#include "tensorflow/core/platform/macros.h"
|
||||
|
||||
namespace tensorflow {
|
||||
|
||||
// ConstantOp returns a tensor specified by ConstantOpDef.
|
||||
class ConstantOp : public OpKernel {
|
||||
public:
|
||||
explicit ConstantOp(OpKernelConstruction* ctx);
|
||||
void Compute(OpKernelContext* ctx) override;
|
||||
bool IsExpensive() override { return false; }
|
||||
~ConstantOp() override;
|
||||
|
||||
private:
|
||||
Tensor tensor_;
|
||||
TF_DISALLOW_COPY_AND_ASSIGN(ConstantOp);
|
||||
};
|
||||
|
||||
class PlaceholderOp : public OpKernel {
|
||||
public:
|
||||
explicit PlaceholderOp(OpKernelConstruction* ctx);
|
||||
void Compute(OpKernelContext* ctx) override;
|
||||
|
||||
private:
|
||||
PartialTensorShape expected_shape_;
|
||||
};
|
||||
|
||||
} // namespace tensorflow
|
||||
|
||||
#endif // TENSORFLOW_CORE_KERNELS_CONSTANT_OP_H_
|
||||
@@ -0,0 +1,118 @@
|
||||
/* Copyright 2015 The TensorFlow Authors. All Rights Reserved.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
==============================================================================*/
|
||||
|
||||
#ifndef TENSORFLOW_CORE_KERNELS_CONTROL_FLOW_OPS_H_
|
||||
#define TENSORFLOW_CORE_KERNELS_CONTROL_FLOW_OPS_H_
|
||||
|
||||
#include "tensorflow/core/framework/op_kernel.h"
|
||||
|
||||
namespace tensorflow {
|
||||
|
||||
// A ControlTriggerOp is similar to a NoOp. However, it always treats the input
|
||||
// control edges as Live edges. Its primary use so far is in the scheduling of
|
||||
// recvs, where we add ControlTrigger nodes and use them to trigger recvs. We
|
||||
// allow ControlTrigger nodes to be enabled by dead nodes.
|
||||
class ControlTriggerOp : public OpKernel {
|
||||
public:
|
||||
explicit ControlTriggerOp(OpKernelConstruction* context)
|
||||
: OpKernel(context) {}
|
||||
void Compute(OpKernelContext* context) override {}
|
||||
bool IsExpensive() override { return false; }
|
||||
};
|
||||
|
||||
// A switch op has two inputs and two outputs. It forwards the value of
|
||||
// Input:0 to the output specified by input:1. Input:1 is a boolean tensor.
|
||||
// Input:0 is forwarded to output:0 if input:1 is false, otherwise to
|
||||
// output:1.
|
||||
class SwitchOp : public OpKernel {
|
||||
public:
|
||||
explicit SwitchOp(OpKernelConstruction* context) : OpKernel(context) {}
|
||||
void Compute(OpKernelContext* context) override;
|
||||
bool IsExpensive() override { return false; }
|
||||
~SwitchOp() override {}
|
||||
|
||||
TF_DISALLOW_COPY_AND_ASSIGN(SwitchOp);
|
||||
};
|
||||
|
||||
// A merge op has n inputs and two outputs. It forwards the value of the
|
||||
// first input that becomes available to its first output, and the
|
||||
// index of the first input to its second output.
|
||||
class MergeOp : public OpKernel {
|
||||
public:
|
||||
explicit MergeOp(OpKernelConstruction* context);
|
||||
void Compute(OpKernelContext* context) override;
|
||||
bool IsExpensive() override { return false; }
|
||||
~MergeOp() override {}
|
||||
|
||||
TF_DISALLOW_COPY_AND_ASSIGN(MergeOp);
|
||||
};
|
||||
|
||||
// An enter op has one input and one output. It creates or finds
|
||||
// the child frame that is uniquely identified by the frame_name,
|
||||
// and makes its input available to the child frame.
|
||||
class EnterOp : public OpKernel {
|
||||
public:
|
||||
explicit EnterOp(OpKernelConstruction* context) : OpKernel(context) {}
|
||||
void Compute(OpKernelContext* context) override;
|
||||
bool IsExpensive() override { return false; }
|
||||
~EnterOp() override {}
|
||||
|
||||
TF_DISALLOW_COPY_AND_ASSIGN(EnterOp);
|
||||
};
|
||||
|
||||
// An exit op has one input and one output. It exits the current
|
||||
// frame to its parent frame, and makes its input available to the
|
||||
// parent frame.
|
||||
class ExitOp : public OpKernel {
|
||||
public:
|
||||
explicit ExitOp(OpKernelConstruction* context) : OpKernel(context) {}
|
||||
void Compute(OpKernelContext* context) override;
|
||||
bool IsExpensive() override { return false; }
|
||||
~ExitOp() override {}
|
||||
|
||||
TF_DISALLOW_COPY_AND_ASSIGN(ExitOp);
|
||||
};
|
||||
|
||||
// A next_iteration op has one input and one output. It makes its input
|
||||
// available to the next iteration.
|
||||
class NextIterationOp : public OpKernel {
|
||||
public:
|
||||
explicit NextIterationOp(OpKernelConstruction* context) : OpKernel(context) {}
|
||||
void Compute(OpKernelContext* context) override;
|
||||
bool IsExpensive() override { return false; }
|
||||
~NextIterationOp() override {}
|
||||
|
||||
TF_DISALLOW_COPY_AND_ASSIGN(NextIterationOp);
|
||||
};
|
||||
|
||||
// A LoopCond op has one input and one output. The input is a boolean
|
||||
// scalar representing the taken branches of the "pivot" Switch that
|
||||
// determines loop termination. As a contract, any high-level front-end
|
||||
// should always use port '0' of the "pivot" switches for loop exit.
|
||||
class LoopCondOp : public OpKernel {
|
||||
public:
|
||||
explicit LoopCondOp(OpKernelConstruction* context);
|
||||
~LoopCondOp() override;
|
||||
|
||||
void Compute(OpKernelContext* context) override;
|
||||
|
||||
bool IsExpensive() override;
|
||||
|
||||
TF_DISALLOW_COPY_AND_ASSIGN(LoopCondOp);
|
||||
};
|
||||
|
||||
} // namespace tensorflow
|
||||
|
||||
#endif // TENSORFLOW_CORE_KERNELS_CONTROL_FLOW_OPS_H_
|
||||
@@ -0,0 +1,316 @@
|
||||
/* Copyright 2015 The TensorFlow Authors. All Rights Reserved.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
==============================================================================*/
|
||||
|
||||
#ifndef TENSORFLOW_CORE_KERNELS_CONV_2D_H_
|
||||
#define TENSORFLOW_CORE_KERNELS_CONV_2D_H_
|
||||
|
||||
#include "third_party/eigen3/unsupported/Eigen/CXX11/Tensor"
|
||||
#include "tensorflow/core/framework/tensor_types.h"
|
||||
#include "tensorflow/core/kernels/eigen_backward_spatial_convolutions.h"
|
||||
#include "tensorflow/core/kernels/eigen_spatial_convolutions.h"
|
||||
#include "tensorflow/core/util/tensor_format.h"
|
||||
|
||||
namespace tensorflow {
|
||||
namespace functor {
|
||||
|
||||
// TODO(yangke): revisit these operations and in particular, see if we can
|
||||
// combine all of them into just one operation without causing nvcc to
|
||||
// timeout.
|
||||
template <typename Device, typename T, int Dims, typename IndexType>
|
||||
struct ShuffleAndReverse {
|
||||
void operator()(const Device& d,
|
||||
typename TTypes<T, Dims, IndexType>::ConstTensor input,
|
||||
const Eigen::DSizes<IndexType, Dims>& order,
|
||||
const Eigen::array<bool, Dims>& reverse_dims,
|
||||
typename TTypes<T, Dims, IndexType>::Tensor output) {
|
||||
output.device(d) = input.shuffle(order).reverse(reverse_dims);
|
||||
}
|
||||
};
|
||||
|
||||
template <typename Device, typename T, int Dims, typename IndexType>
|
||||
struct InflatePadAndShuffle {
|
||||
void operator()(
|
||||
const Device& d, typename TTypes<T, Dims, IndexType>::ConstTensor input,
|
||||
const Eigen::DSizes<IndexType, Dims>& strides,
|
||||
const Eigen::array<Eigen::IndexPair<IndexType>, Dims>& pad_dims,
|
||||
const Eigen::DSizes<IndexType, Dims>& order,
|
||||
typename TTypes<T, Dims, IndexType>::Tensor output) {
|
||||
output.device(d) = input.inflate(strides).pad(pad_dims).shuffle(order);
|
||||
}
|
||||
};
|
||||
|
||||
template <typename Device, typename Input, typename Filter, typename Output>
|
||||
void SpatialConvolutionFunc(const Device& d, Output output, Input input,
|
||||
Filter filter, int row_stride, int col_stride,
|
||||
int row_dilation, int col_dilation,
|
||||
const Eigen::PaddingType& padding) {
|
||||
// Need to swap row/col when calling Eigen.
|
||||
output.device(d) =
|
||||
Eigen::SpatialConvolution(input, filter, col_stride, row_stride, padding,
|
||||
col_dilation, row_dilation);
|
||||
}
|
||||
|
||||
template <typename Device, typename T>
|
||||
struct SpatialConvolution {
|
||||
void operator()(const Device& d, typename TTypes<T, 4>::Tensor output,
|
||||
typename TTypes<T, 4>::ConstTensor input,
|
||||
typename TTypes<T, 4>::ConstTensor filter, int row_stride,
|
||||
int col_stride, int row_dilation, int col_dilation,
|
||||
const Eigen::PaddingType& padding) {
|
||||
SpatialConvolutionFunc(d, output, input, filter, row_stride, col_stride,
|
||||
row_dilation, col_dilation, padding);
|
||||
}
|
||||
};
|
||||
|
||||
template <typename Device>
|
||||
struct SpatialConvolution<Device, Eigen::half> {
|
||||
void operator()(const Device& d,
|
||||
typename TTypes<Eigen::half, 4>::Tensor output,
|
||||
typename TTypes<Eigen::half, 4>::ConstTensor input,
|
||||
typename TTypes<Eigen::half, 4>::ConstTensor filter,
|
||||
int row_stride, int col_stride, int row_dilation,
|
||||
int col_dilation, const Eigen::PaddingType& padding) {
|
||||
output.device(d) =
|
||||
Eigen::SpatialConvolution(input.cast<float>(), filter.cast<float>(),
|
||||
col_stride, row_stride, padding, col_dilation,
|
||||
row_dilation)
|
||||
.cast<Eigen::half>();
|
||||
}
|
||||
};
|
||||
|
||||
template <typename Device, typename T>
|
||||
struct SpatialConvolutionBackwardInput {
|
||||
void operator()(const Device& d, typename TTypes<T, 4>::Tensor input_backward,
|
||||
typename TTypes<T, 4>::ConstTensor kernel,
|
||||
typename TTypes<T, 4>::ConstTensor output_backward,
|
||||
int row_stride, int col_stride, int row_dilation,
|
||||
int col_dilation) {
|
||||
// Need to swap row/col when calling Eigen.
|
||||
input_backward.device(d) = Eigen::SpatialConvolutionBackwardInput(
|
||||
kernel, output_backward, input_backward.dimension(2),
|
||||
input_backward.dimension(1), col_stride, row_stride, col_dilation,
|
||||
row_dilation);
|
||||
}
|
||||
};
|
||||
|
||||
template <typename Device, typename T>
|
||||
struct SpatialConvolutionBackwardFilter {
|
||||
void operator()(const Device& d,
|
||||
typename TTypes<T, 4>::Tensor kernel_backward,
|
||||
typename TTypes<T, 4>::ConstTensor input,
|
||||
typename TTypes<T, 4>::ConstTensor output_backward,
|
||||
int row_stride, int col_stride, int row_dilation,
|
||||
int col_dilation) {
|
||||
// Need to swap row/col when calling Eigen.
|
||||
kernel_backward.device(d) = Eigen::SpatialConvolutionBackwardKernel(
|
||||
input, output_backward, kernel_backward.dimension(1),
|
||||
kernel_backward.dimension(0), col_stride, row_stride, col_dilation,
|
||||
row_dilation);
|
||||
}
|
||||
};
|
||||
|
||||
// TODO(vrv): Figure out how to use the MatMulFunctor in matmul_op.h.
|
||||
// My initial attempt to do this compiled but failed in the pytest
|
||||
// due to a swigdeps error.
|
||||
template <typename Device, typename T>
|
||||
struct MatMulConvFunctor {
|
||||
// Computes on device "d": out = in0 * in1, where * is matrix
|
||||
// multiplication.
|
||||
void operator()(
|
||||
const Device& d, typename TTypes<T, 2>::Tensor out,
|
||||
typename TTypes<T, 2>::ConstTensor in0,
|
||||
typename TTypes<T, 2>::ConstTensor in1,
|
||||
const Eigen::array<Eigen::IndexPair<Eigen::DenseIndex>, 1>& dim_pair) {
|
||||
out.device(d) = in0.contract(in1, dim_pair);
|
||||
}
|
||||
};
|
||||
|
||||
// Shuffles a filter tensor from TensorFlow format HWIO to dst_filter_format.
|
||||
//
|
||||
// Note: Currently OIHW is the only supported destination format. Support for
|
||||
// OHWI format will be added in a follow-up change.
|
||||
template <typename Device, typename T, typename IndexType, int NDIMS>
|
||||
struct TransformFilter {
|
||||
void operator()(const Device& d, FilterTensorFormat dst_filter_format,
|
||||
typename TTypes<T, NDIMS, IndexType>::ConstTensor in,
|
||||
typename TTypes<T, NDIMS, IndexType>::Tensor out) {
|
||||
// Merge the spatial dimensions together to speed up the shuffle operation.
|
||||
Eigen::DSizes<IndexType, 3> merged_dims;
|
||||
merged_dims[0] = in.dimension(0); // spatial dimensions
|
||||
for (int i = 1; i < NDIMS - 2; ++i) {
|
||||
merged_dims[0] *= in.dimension(i);
|
||||
}
|
||||
merged_dims[1] = in.dimension(NDIMS - 2); // input filters
|
||||
merged_dims[2] = in.dimension(NDIMS - 1); // output filters
|
||||
|
||||
CHECK(dst_filter_format == FORMAT_OIHW)
|
||||
<< "Unsupported destination filter format: "
|
||||
<< ToString(dst_filter_format);
|
||||
// Source filter format is FORMAT_HWIO and spatial dimensions HW are merged
|
||||
// in the beginning.
|
||||
Eigen::DSizes<IndexType, 3> shuffling_perm =
|
||||
Eigen::DSizes<IndexType, 3>(2, 1, 0);
|
||||
|
||||
Eigen::DSizes<IndexType, NDIMS> expanded_dims;
|
||||
int out_index = 0;
|
||||
for (int merged_dim = 0; merged_dim < merged_dims.rank(); ++merged_dim) {
|
||||
if (shuffling_perm[merged_dim] == 0) {
|
||||
for (int spatial_dim = 0; spatial_dim < NDIMS - 2; ++spatial_dim) {
|
||||
expanded_dims[out_index++] = in.dimension(spatial_dim);
|
||||
}
|
||||
} else {
|
||||
constexpr int kLastSpatialDim = NDIMS - 3;
|
||||
expanded_dims[out_index++] =
|
||||
in.dimension(kLastSpatialDim + shuffling_perm[merged_dim]);
|
||||
}
|
||||
}
|
||||
|
||||
out.device(d) =
|
||||
in.reshape(merged_dims).shuffle(shuffling_perm).reshape(expanded_dims);
|
||||
}
|
||||
};
|
||||
|
||||
template <typename Device, typename T, typename IndexType>
|
||||
struct TransformDepth {
|
||||
void operator()(const Device& d,
|
||||
typename TTypes<T, 4, IndexType>::ConstTensor in,
|
||||
const Eigen::DSizes<IndexType, 4>& shuffle,
|
||||
typename TTypes<T, 4, IndexType>::Tensor out) {
|
||||
Eigen::DSizes<IndexType, 3> merged_dims;
|
||||
Eigen::DSizes<IndexType, 4> expanded_dims;
|
||||
Eigen::DSizes<IndexType, 3> new_shuffle;
|
||||
|
||||
// Merge dimensions that won't be shuffled together to speed things up.
|
||||
if (shuffle[1] == 2 && shuffle[2] == 3) {
|
||||
merged_dims[0] = in.dimension(0);
|
||||
merged_dims[1] = in.dimension(1);
|
||||
merged_dims[2] = in.dimension(2) * in.dimension(3);
|
||||
new_shuffle[0] = shuffle[0];
|
||||
new_shuffle[1] = 2;
|
||||
new_shuffle[2] = shuffle[3];
|
||||
expanded_dims[0] = in.dimension(shuffle[0]);
|
||||
expanded_dims[1] = in.dimension(2);
|
||||
expanded_dims[2] = in.dimension(3);
|
||||
expanded_dims[3] = in.dimension(shuffle[3]);
|
||||
} else if (shuffle[0] == 2 && shuffle[1] == 3) {
|
||||
merged_dims[0] = in.dimension(0);
|
||||
merged_dims[1] = in.dimension(1);
|
||||
merged_dims[2] = in.dimension(2) * in.dimension(3);
|
||||
new_shuffle[0] = 2;
|
||||
new_shuffle[1] = shuffle[2];
|
||||
new_shuffle[2] = shuffle[3];
|
||||
expanded_dims[0] = in.dimension(2);
|
||||
expanded_dims[1] = in.dimension(3);
|
||||
expanded_dims[2] = in.dimension(shuffle[2]);
|
||||
expanded_dims[3] = in.dimension(shuffle[3]);
|
||||
} else if (shuffle[0] == 0 && shuffle[1] == 3 && shuffle[2] == 1 &&
|
||||
shuffle[3] == 2) {
|
||||
merged_dims[0] = in.dimension(0);
|
||||
merged_dims[1] = in.dimension(1) * in.dimension(2);
|
||||
merged_dims[2] = in.dimension(3);
|
||||
new_shuffle[0] = 0;
|
||||
new_shuffle[1] = 2;
|
||||
new_shuffle[2] = 1;
|
||||
expanded_dims[0] = in.dimension(0);
|
||||
expanded_dims[1] = in.dimension(3);
|
||||
expanded_dims[2] = in.dimension(1);
|
||||
expanded_dims[3] = in.dimension(2);
|
||||
} else {
|
||||
assert(false && "unexpected shuffle");
|
||||
}
|
||||
|
||||
out.device(d) =
|
||||
in.reshape(merged_dims).shuffle(new_shuffle).reshape(expanded_dims);
|
||||
}
|
||||
};
|
||||
|
||||
template <typename Device, typename T, typename IndexType, int NDIMS>
|
||||
struct PadInput {
|
||||
void operator()(const Device& d,
|
||||
typename TTypes<T, NDIMS, IndexType>::ConstTensor in,
|
||||
const std::array<int, NDIMS - 2>& padding_left,
|
||||
const std::array<int, NDIMS - 2>& padding_right,
|
||||
typename TTypes<T, NDIMS, IndexType>::Tensor out,
|
||||
TensorFormat format) {
|
||||
Eigen::array<Eigen::IndexPair<IndexType>, NDIMS> padding;
|
||||
padding[GetTensorDimIndex<NDIMS - 2>(format, 'N')] = {0, 0};
|
||||
for (int i = 0; i < NDIMS - 2; ++i) {
|
||||
padding[GetTensorDimIndex<NDIMS - 2>(format, '0' + i)] = {
|
||||
padding_left[i], padding_right[i]};
|
||||
}
|
||||
padding[GetTensorDimIndex<NDIMS - 2>(format, 'C')] = {0, 0};
|
||||
out.device(d) = in.pad(padding);
|
||||
}
|
||||
};
|
||||
|
||||
// Converts a tensor from:
|
||||
// [batch, <spatial>, filters]
|
||||
// to:
|
||||
// [batch, filters, <spatial>]
|
||||
template <typename Device, typename T, int NDIMS>
|
||||
struct NHWCToNCHW {
|
||||
void operator()(const Device& d, typename TTypes<T, NDIMS>::ConstTensor in,
|
||||
typename TTypes<T, NDIMS>::Tensor out);
|
||||
};
|
||||
|
||||
// Converts a tensor from:
|
||||
// [batch, filters, <spatial>]
|
||||
// to:
|
||||
// [batch, <spatial>, filters]
|
||||
template <typename Device, typename T, int NDIMS>
|
||||
struct NCHWToNHWC {
|
||||
void operator()(const Device& d, typename TTypes<T, NDIMS>::ConstTensor in,
|
||||
typename TTypes<T, NDIMS>::Tensor out);
|
||||
};
|
||||
|
||||
// Converts a tensor from:
|
||||
// [dim0, dim1, dim2]
|
||||
// to:
|
||||
// [dim0, dim2, dim1]
|
||||
template <typename Device, typename T, bool conjugate = false>
|
||||
struct SwapDimension1And2InTensor3 {
|
||||
void operator()(const Device& d, const T* in,
|
||||
const gtl::ArraySlice<int64>& input_dims, T* out);
|
||||
};
|
||||
|
||||
// Converts a tensor from:
|
||||
// [dim0, dim1, dim2]
|
||||
// to:
|
||||
// [dim2, dim1, dim0]
|
||||
template <typename Device, typename T, bool conjugate = false>
|
||||
struct SwapDimension0And2InTensor3 {
|
||||
void operator()(const Device& d, const T* in,
|
||||
const gtl::ArraySlice<int64>& input_dims, T* out);
|
||||
};
|
||||
|
||||
// Transforms back filter from OIHW to HWOI format to reverse effect of
|
||||
// TransformFilter above.
|
||||
// TODO(hinsu): Support reverse transformation from filter format OHWI as well.
|
||||
template <typename Device, typename T, int NDIMS>
|
||||
struct ReverseTransformFilter {
|
||||
void operator()(const Device& d, typename TTypes<T, NDIMS>::ConstTensor in,
|
||||
typename TTypes<T, NDIMS>::Tensor out);
|
||||
};
|
||||
|
||||
} // namespace functor
|
||||
|
||||
template <class T>
|
||||
class ConvAlgorithmMap;
|
||||
|
||||
template <>
|
||||
class ConvAlgorithmMap<Eigen::ThreadPoolDevice> {};
|
||||
} // namespace tensorflow
|
||||
|
||||
#endif // TENSORFLOW_CORE_KERNELS_CONV_2D_H_
|
||||
@@ -0,0 +1,91 @@
|
||||
/* Copyright 2016 The TensorFlow Authors. All Rights Reserved.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
==============================================================================*/
|
||||
|
||||
// Functors for 3d convolution.
|
||||
|
||||
#ifndef TENSORFLOW_CORE_KERNELS_CONV_3D_H_
|
||||
#define TENSORFLOW_CORE_KERNELS_CONV_3D_H_
|
||||
|
||||
#include "tensorflow/core/framework/tensor_types.h"
|
||||
#include "tensorflow/core/kernels/eigen_backward_cuboid_convolutions.h"
|
||||
#include "tensorflow/core/kernels/eigen_cuboid_convolution.h"
|
||||
|
||||
namespace tensorflow {
|
||||
namespace functor {
|
||||
|
||||
// Applies a 3D convolution to a batch of multi-channel volumes.
|
||||
template <typename Device, typename T>
|
||||
struct CuboidConvolution;
|
||||
|
||||
// Backward input pass for the cuboid convolution.
|
||||
template <typename Device, typename T>
|
||||
struct CuboidConvolutionBackwardInput;
|
||||
|
||||
// Backward filter pass for the cuboid convolution.
|
||||
template <typename Device, typename T>
|
||||
struct CuboidConvolutionBackwardFilter;
|
||||
|
||||
typedef Eigen::ThreadPoolDevice CPUDevice;
|
||||
|
||||
template <typename T>
|
||||
struct CuboidConvolution<CPUDevice, T> {
|
||||
void operator()(const CPUDevice& d, typename TTypes<T, 5>::Tensor output,
|
||||
typename TTypes<T, 5>::ConstTensor input,
|
||||
typename TTypes<T, 5>::ConstTensor filter, int stride_planes,
|
||||
int stride_rows, int stride_cols,
|
||||
const Eigen::PaddingType& padding) {
|
||||
output.device(d) = Eigen::CuboidConvolution(
|
||||
input, filter, stride_planes, stride_rows, stride_cols, padding);
|
||||
}
|
||||
};
|
||||
|
||||
template <typename T>
|
||||
struct CuboidConvolutionBackwardInput<CPUDevice, T> {
|
||||
void operator()(const CPUDevice& d,
|
||||
typename TTypes<T, 5>::Tensor input_backward,
|
||||
typename TTypes<T, 5>::ConstTensor filter,
|
||||
typename TTypes<T, 5>::ConstTensor output_backward,
|
||||
int stride_planes, int stride_rows, int stride_cols) {
|
||||
// Need to swap the order of plane/row/col strides when calling Eigen.
|
||||
input_backward.device(d) = Eigen::CuboidConvolutionBackwardInput(
|
||||
filter, output_backward,
|
||||
input_backward.dimension(3), // input_planes
|
||||
input_backward.dimension(2), // input_rows
|
||||
input_backward.dimension(1), // input_cols
|
||||
stride_cols, stride_rows, stride_planes);
|
||||
}
|
||||
};
|
||||
|
||||
template <typename T>
|
||||
struct CuboidConvolutionBackwardFilter<CPUDevice, T> {
|
||||
void operator()(const CPUDevice& d,
|
||||
typename TTypes<T, 5>::Tensor filter_backward,
|
||||
typename TTypes<T, 5>::ConstTensor input,
|
||||
typename TTypes<T, 5>::ConstTensor output_backward,
|
||||
int stride_planes, int stride_rows, int stride_cols) {
|
||||
// Need to swap the order of plane/row/col strides when calling Eigen.
|
||||
filter_backward.device(d) = Eigen::CuboidConvolutionBackwardKernel(
|
||||
input, output_backward,
|
||||
filter_backward.dimension(2), // kernel_planes
|
||||
filter_backward.dimension(1), // kernel_rows
|
||||
filter_backward.dimension(0), // kernel_cols
|
||||
stride_cols, stride_rows, stride_planes);
|
||||
}
|
||||
};
|
||||
|
||||
} // namespace functor
|
||||
} // namespace tensorflow
|
||||
|
||||
#endif // TENSORFLOW_CORE_KERNELS_CONV_3D_H_
|
||||
@@ -0,0 +1,269 @@
|
||||
/* Copyright 2016 The TensorFlow Authors. All Rights Reserved.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
==============================================================================*/
|
||||
|
||||
// This is the common header for the input and filter backprop kernels.
|
||||
//
|
||||
// The operation to compute Conv2D gradients.
|
||||
//
|
||||
// To compute the gradients for Conv2D, we need three input tensors:
|
||||
// input, filter, and backprop for output.
|
||||
// And we need to compute two backprops: one for input and one for filter. We
|
||||
// compute them in two different kernels.
|
||||
//
|
||||
// Both backprops can be computed as straightforward conv2d.
|
||||
//
|
||||
// Consider a case where the input is 3x3 and the filter is 2x1:
|
||||
//
|
||||
// INPUT = [ A B C ]
|
||||
// [ D E F ]
|
||||
// [ G H I ]
|
||||
//
|
||||
// where each "A", "B", etc is batch x in_depth
|
||||
//
|
||||
// FILTER = [ X Y ]
|
||||
//
|
||||
// where both "X" and "Y" are in_depth x out_depth
|
||||
//
|
||||
// With VALID padding, the output is 3x2:
|
||||
//
|
||||
// OUTPUT = [ a b ]
|
||||
// [ c d ]
|
||||
// [ e f ]
|
||||
//
|
||||
// where each "a", "b", etc is batch x out_depth
|
||||
//
|
||||
// So we have:
|
||||
//
|
||||
// a = A * X + B * Y
|
||||
// b = B * X + C * Y
|
||||
// c = D * X + E * Y
|
||||
// d = E * X + F * Y
|
||||
// e = G * X + H * Y
|
||||
// f = H * X + I * Y
|
||||
//
|
||||
// So when we have backprops for the outputs (we denote them by
|
||||
// a', b', ... ):
|
||||
//
|
||||
// The backprops for the input are:
|
||||
//
|
||||
// A' = a' * X^t
|
||||
// B' = a' * Y^t + b' * X^t
|
||||
// C' = b' * Y^t
|
||||
// ...
|
||||
//
|
||||
// This is essentially computing a 2d conv of
|
||||
//
|
||||
// INPUT = [ 0 a' b' 0 ]
|
||||
// [ 0 c' d' 0 ]
|
||||
// [ 0 e' f' 0 ]
|
||||
// and
|
||||
//
|
||||
// FILTER = [ Y^t X^t ]
|
||||
//
|
||||
// The backprops for the filter are:
|
||||
//
|
||||
// X' = A^t * a' + B^t * b' + D^t * c' + E^t * d' + G^t * e' + H^t * f'
|
||||
// Y' = B^t * a' + C^t * b' + E^t + c' + F^t * d' + H^t * e' + I^t * f'
|
||||
//
|
||||
// This is essentially computing a 2d conv of
|
||||
//
|
||||
// INPUT = [ A^t B^t C^t ]
|
||||
// [ D^t E^t F^t ]
|
||||
// [ G^t H^t I^t ]
|
||||
//
|
||||
// and
|
||||
//
|
||||
// FILTER = [ a' b' ]
|
||||
// [ c' d' ]
|
||||
// [ e' f' ]
|
||||
//
|
||||
//
|
||||
//////////////////////////////////////////////////////////
|
||||
//
|
||||
// With stride more than one, it's a bit more complicated (we will need to
|
||||
// create holes to the backprop).
|
||||
//
|
||||
// Consider the case where
|
||||
//
|
||||
// INPUT = [ A B C D E ]
|
||||
// [ F G H I J ]
|
||||
// [ K L M N O ]
|
||||
// and
|
||||
//
|
||||
// FILTER = [ X Y Z ]
|
||||
//
|
||||
// with stride 2.
|
||||
//
|
||||
// The output will be
|
||||
//
|
||||
// OUTPUT = [ a b ]
|
||||
// [ c d ]
|
||||
//
|
||||
// where:
|
||||
//
|
||||
// a = A * X + B * Y + C * Z
|
||||
// b = C * X + D * Y + E * Z
|
||||
// c = K * X + L * Y + M * Z
|
||||
// d = M * X + N * Y + O * Z
|
||||
//
|
||||
//
|
||||
// To compute the backprop for INPUT, we need to convolve
|
||||
//
|
||||
// INPUT = [ 0 0 a' 0 b' 0 0 ]
|
||||
// [ 0 0 0 0 0 0 0 ]
|
||||
// [ 0 0 c' 0 d' 0 0 ]
|
||||
//
|
||||
// (notice the holes in INPUT)
|
||||
//
|
||||
// and
|
||||
//
|
||||
// FILTER = [ Z^t Y^t X^t ]
|
||||
//
|
||||
// with stride 1.
|
||||
//
|
||||
// To compute the backprop for FILTER, we need to convolve
|
||||
|
||||
//
|
||||
// INPUT = [ A^t B^t C^t D^t E^t ]
|
||||
// [ F^t G^t H^t I^t J^t ]
|
||||
// [ K^t L^t M^t N^t O^t ]
|
||||
// and
|
||||
//
|
||||
// FILTER = [ a' 0 b' ]
|
||||
// [ 0 0 0 ]
|
||||
// [ c' 0 d' ]
|
||||
//
|
||||
// (notice the holes in FILTER)
|
||||
//
|
||||
//
|
||||
// with stride 1
|
||||
//
|
||||
//////////////////////////////////////////////////////////
|
||||
//
|
||||
//
|
||||
// The case for SAME padding is in fact very similar to VALID -- we just
|
||||
// need to pad the input tensor a bit when computing the filter_backprop.
|
||||
|
||||
#ifndef TENSORFLOW_CORE_KERNELS_CONV_GRAD_OPS_H_
|
||||
#define TENSORFLOW_CORE_KERNELS_CONV_GRAD_OPS_H_
|
||||
|
||||
#include <vector>
|
||||
|
||||
#include "tensorflow/core/framework/tensor_shape.h"
|
||||
#include "tensorflow/core/lib/core/stringpiece.h"
|
||||
#include "tensorflow/core/util/padding.h"
|
||||
#include "tensorflow/core/util/tensor_format.h"
|
||||
|
||||
namespace tensorflow {
|
||||
|
||||
// Forward declaration.
|
||||
class OpKernelContext;
|
||||
|
||||
template <typename Device, typename T>
|
||||
struct LaunchConv2DBackpropInputOp {
|
||||
void operator()(OpKernelContext* ctx, bool use_cudnn, bool cudnn_use_autotune,
|
||||
const Tensor& out_backprop, const Tensor& filter,
|
||||
int row_dilation, int col_dilation, int row_stride,
|
||||
int col_stride, const Padding& padding, Tensor* in_backprop,
|
||||
TensorFormat data_format);
|
||||
};
|
||||
|
||||
template <typename Device, typename T>
|
||||
struct LaunchConv2DBackpropFilterOp {
|
||||
void operator()(OpKernelContext* ctx, bool use_cudnn, bool cudnn_use_autotune,
|
||||
const Tensor& out_backprop, const Tensor& input,
|
||||
int row_dilation, int col_dilation, int row_stride,
|
||||
int col_stride, const Padding& padding,
|
||||
Tensor* filter_backprop, TensorFormat data_format);
|
||||
};
|
||||
|
||||
#ifdef GOOGLE_CUDA
|
||||
template <typename T>
|
||||
struct LaunchConv2DBackpropInputOp<Eigen::GpuDevice, T> {
|
||||
void operator()(OpKernelContext* ctx, bool use_cudnn, bool cudnn_use_autotune,
|
||||
const Tensor& input, const Tensor& filter, int row_dilation,
|
||||
int col_dilation, int row_stride, int col_stride,
|
||||
const Padding& padding, Tensor* output,
|
||||
TensorFormat data_format);
|
||||
};
|
||||
|
||||
template <typename T>
|
||||
struct LaunchConv2DBackpropFilterOp<Eigen::GpuDevice, T> {
|
||||
void operator()(OpKernelContext* ctx, bool use_cudnn, bool cudnn_use_autotune,
|
||||
const Tensor& out_backprop, const Tensor& input,
|
||||
int row_dilation, int col_dilation, int row_stride,
|
||||
int col_stride, const Padding& padding,
|
||||
Tensor* filter_backprop, TensorFormat data_format);
|
||||
};
|
||||
#endif // GOOGLE_CUDA
|
||||
|
||||
// Information about a single spatial dimension for a convolution
|
||||
// backpropagation.
|
||||
struct ConvBackpropSpatialDimension {
|
||||
int64 input_size;
|
||||
int64 filter_size;
|
||||
int64 output_size;
|
||||
int64 stride;
|
||||
int64 dilation;
|
||||
int64 expanded_output_size;
|
||||
|
||||
// Number of padding elements to be added before/after this dimension of
|
||||
// the input when computing Conv?DBackpropInput.
|
||||
int64 pad_before, pad_after;
|
||||
};
|
||||
|
||||
// Computed dimensions for a backwards convolution.
|
||||
struct ConvBackpropDimensions {
|
||||
// Information about each spatial dimension.
|
||||
gtl::InlinedVector<ConvBackpropSpatialDimension, 3> spatial_dims;
|
||||
|
||||
// Batch size.
|
||||
int64 batch_size;
|
||||
|
||||
// Input and output feature depth.
|
||||
int64 in_depth, out_depth;
|
||||
|
||||
// Convenience access methods for spatial dimensions properties.
|
||||
int64 input_size(int dim) const { return spatial_dims[dim].input_size; }
|
||||
int64 filter_size(int dim) const { return spatial_dims[dim].filter_size; }
|
||||
int64 output_size(int dim) const { return spatial_dims[dim].output_size; }
|
||||
int64 stride(int dim) const { return spatial_dims[dim].stride; }
|
||||
int64 dilation(int dim) const { return spatial_dims[dim].dilation; }
|
||||
|
||||
// Compute padding for the given spatial dimension.
|
||||
int SpatialPadding(const Padding& padding, int dim) const;
|
||||
};
|
||||
|
||||
// Common code between implementations of Conv?DBackpropInput and
|
||||
// Conv?DBackpropFilter. Verifies that the dimensions all match, and computes
|
||||
// sizes/padding for the spatial dimensions.
|
||||
Status ConvBackpropComputeDimensions(StringPiece label, int num_spatial_dims,
|
||||
const TensorShape& input_shape,
|
||||
const TensorShape& filter_shape,
|
||||
const TensorShape& out_backprop_shape,
|
||||
const std::vector<int32>& strides,
|
||||
Padding padding, TensorFormat data_format,
|
||||
ConvBackpropDimensions* dims);
|
||||
|
||||
// The V2 version computes the same outputs with arbitrary dilation rate.
|
||||
// TODO(b/67112639): Merge V2 versions and the original versions eventually.
|
||||
Status ConvBackpropComputeDimensionsV2(
|
||||
StringPiece label, int num_spatial_dims, const TensorShape& input_shape,
|
||||
const TensorShape& filter_shape, const TensorShape& out_backprop_shape,
|
||||
const gtl::ArraySlice<int32>& dilations, const std::vector<int32>& strides,
|
||||
Padding padding, TensorFormat data_format, ConvBackpropDimensions* dims);
|
||||
} // namespace tensorflow
|
||||
|
||||
#endif // TENSORFLOW_CORE_KERNELS_CONV_GRAD_OPS_H_
|
||||
@@ -0,0 +1,115 @@
|
||||
/* Copyright 2016 The TensorFlow Authors. All Rights Reserved.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
==============================================================================*/
|
||||
|
||||
#ifndef TENSORFLOW_CORE_KERNELS_CONV_OPS_H_
|
||||
#define TENSORFLOW_CORE_KERNELS_CONV_OPS_H_
|
||||
|
||||
#include "third_party/eigen3/unsupported/Eigen/CXX11/Tensor"
|
||||
#include "tensorflow/core/framework/resource_mgr.h"
|
||||
#include "tensorflow/core/platform/mem.h"
|
||||
#include "tensorflow/core/util/tensor_format.h"
|
||||
|
||||
#if GOOGLE_CUDA
|
||||
#include "tensorflow/core/kernels/conv_ops_gpu.h"
|
||||
#include "tensorflow/core/platform/stream_executor.h"
|
||||
#endif // GOOGLE_CUDA
|
||||
|
||||
namespace tensorflow {
|
||||
|
||||
// Forward declaration.
|
||||
class OpKernelContext;
|
||||
|
||||
template <typename Device, typename T>
|
||||
struct LaunchConv2DOp {
|
||||
void operator()(OpKernelContext* ctx, bool use_cudnn, bool cudnn_use_autotune,
|
||||
const Tensor& input, const Tensor& filter, int row_dilation,
|
||||
int col_dilation, int row_stride, int col_stride,
|
||||
const Padding& padding, Tensor* output,
|
||||
TensorFormat data_format);
|
||||
};
|
||||
|
||||
#ifdef GOOGLE_CUDA
|
||||
template <typename T>
|
||||
struct LaunchConv2DOp<Eigen::GpuDevice, T> {
|
||||
void operator()(OpKernelContext* ctx, bool use_cudnn, bool cudnn_use_autotune,
|
||||
const Tensor& input, const Tensor& filter, int row_dilation,
|
||||
int col_dilation, int row_stride, int col_stride,
|
||||
const Padding& padding, Tensor* output,
|
||||
TensorFormat data_format);
|
||||
};
|
||||
#endif // GOOGLE_CUDA
|
||||
|
||||
// Used to keep track of persistent memory buffers used within the op.
|
||||
// It uses malloc and free to avoid the time cost of initializing the memory.
|
||||
template <class T, size_t size>
|
||||
struct Im2ColBufferResource : public ResourceBase {
|
||||
Im2ColBufferResource<T, size>() {
|
||||
data = static_cast<T*>(port::Malloc(size * sizeof(T)));
|
||||
}
|
||||
~Im2ColBufferResource<T, size>() { port::Free(data); }
|
||||
// This mutex ensures that only a single operation at a time is able to use
|
||||
// the buffer memory held by this resource.
|
||||
mutex mu;
|
||||
T* data;
|
||||
string DebugString() { return "Im2ColBufferResource"; }
|
||||
};
|
||||
|
||||
// Convolution parameters specified by Op attributes.
|
||||
struct Conv2DParameters {
|
||||
std::vector<int32> dilations;
|
||||
std::vector<int32> strides;
|
||||
Padding padding;
|
||||
TensorFormat data_format;
|
||||
};
|
||||
|
||||
// Convolution dimensions inferred from parameters, input and filter tensors.
|
||||
struct Conv2DDimensions {
|
||||
int batch;
|
||||
int input_rows;
|
||||
int input_cols;
|
||||
int in_depth;
|
||||
|
||||
int filter_rows;
|
||||
int filter_cols;
|
||||
int patch_depth;
|
||||
int out_depth;
|
||||
|
||||
int stride_rows;
|
||||
int stride_cols;
|
||||
|
||||
int dilation_rows;
|
||||
int dilation_cols;
|
||||
|
||||
int64 out_rows;
|
||||
int64 out_cols;
|
||||
int64 pad_rows;
|
||||
int64 pad_cols;
|
||||
};
|
||||
|
||||
// Initializes and validates Conv2D parameters configured by OpKernel
|
||||
// attributes.
|
||||
Status InitConv2DParameters(const OpKernelConstruction* context,
|
||||
Conv2DParameters* params);
|
||||
|
||||
// Computes and validates convolutions dimensions from Conv2D parameters. If
|
||||
// parameters are valid, dimensions will be updated with derived convolution
|
||||
// dimensions, otherwise error will be returned.
|
||||
Status ComputeConv2DDimension(const Conv2DParameters& params,
|
||||
const Tensor& input, const Tensor& filter,
|
||||
Conv2DDimensions* dimensions);
|
||||
|
||||
} // namespace tensorflow
|
||||
|
||||
#endif // TENSORFLOW_CORE_KERNELS_CONV_OPS_H_
|
||||
@@ -0,0 +1,205 @@
|
||||
/* Copyright 2015 The TensorFlow Authors. All Rights Reserved.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
==============================================================================*/
|
||||
|
||||
#ifndef TENSORFLOW_CORE_KERNELS_CONV_OPS_GPU_H_
|
||||
#define TENSORFLOW_CORE_KERNELS_CONV_OPS_GPU_H_
|
||||
|
||||
#if GOOGLE_CUDA
|
||||
|
||||
#include <tuple>
|
||||
#include <unordered_map>
|
||||
#include "tensorflow/core/framework/op_kernel.h"
|
||||
#include "tensorflow/core/kernels/gpu_utils.h"
|
||||
#include "tensorflow/core/lib/gtl/inlined_vector.h"
|
||||
#include "tensorflow/core/lib/hash/hash.h"
|
||||
|
||||
namespace tensorflow {
|
||||
|
||||
// Get the Cudnn workspace limit from the environment variable, which is in MB.
|
||||
// Return the workspace memory limit in bytes. If no value is set, return the
|
||||
// default value.
|
||||
int64 GetCudnnWorkspaceLimit(const string& envvar_in_mb,
|
||||
int64 default_value_in_bytes);
|
||||
|
||||
// A class to provide scratch-space allocator for Stream-Executor Cudnn
|
||||
// callback. TensorFlow is responsible for releasing the temporary buffers after
|
||||
// the kernel finishes.
|
||||
class CudnnScratchAllocator : public se::ScratchAllocator {
|
||||
public:
|
||||
virtual ~CudnnScratchAllocator() {}
|
||||
CudnnScratchAllocator(int64 memory_limit, OpKernelContext* context)
|
||||
: memory_limit_(memory_limit), total_byte_size_(0), context_(context) {}
|
||||
int64 GetMemoryLimitInBytes(se::Stream* stream) override {
|
||||
return memory_limit_;
|
||||
}
|
||||
se::port::StatusOr<se::DeviceMemory<uint8>> AllocateBytes(
|
||||
se::Stream* stream, int64 byte_size) override {
|
||||
Tensor temporary_memory;
|
||||
if (byte_size < 0) {
|
||||
return se::port::Status{se::port::error::INVALID_ARGUMENT,
|
||||
"Requested negative byte size!"};
|
||||
}
|
||||
if (byte_size > memory_limit_) {
|
||||
return se::port::StatusOr<se::DeviceMemory<uint8>>();
|
||||
}
|
||||
AllocationAttributes allocation_attr;
|
||||
allocation_attr.no_retry_on_failure = true;
|
||||
Status allocation_status(context_->allocate_temp(
|
||||
DT_UINT8, TensorShape({byte_size}), &temporary_memory,
|
||||
AllocatorAttributes(), allocation_attr));
|
||||
if (!allocation_status.ok()) {
|
||||
return se::port::StatusOr<se::DeviceMemory<uint8>>();
|
||||
}
|
||||
// Hold the reference of the allocated tensors until the end of the
|
||||
// allocator.
|
||||
allocated_tensors_.push_back(temporary_memory);
|
||||
total_byte_size_ += byte_size;
|
||||
return se::port::StatusOr<se::DeviceMemory<uint8>>(
|
||||
AsDeviceMemory(temporary_memory.flat<uint8>().data(),
|
||||
temporary_memory.flat<uint8>().size()));
|
||||
}
|
||||
int64 TotalByteSize() { return total_byte_size_; }
|
||||
|
||||
private:
|
||||
int64 memory_limit_;
|
||||
int64 total_byte_size_;
|
||||
OpKernelContext* context_;
|
||||
std::vector<Tensor> allocated_tensors_;
|
||||
};
|
||||
|
||||
// Encapsulate all the shape information that is used in both forward and
|
||||
// backward conv operations.
|
||||
class ConvParameters {
|
||||
public:
|
||||
using SpatialArray = gtl::InlinedVector<int64, 3>;
|
||||
ConvParameters(int64 batch, int64 in_depths, const SpatialArray& in,
|
||||
TensorFormat data_format, int64 out_depths,
|
||||
const SpatialArray& filter, const SpatialArray& dilation,
|
||||
const SpatialArray& stride, const SpatialArray& padding,
|
||||
DataType dtype, int device_id)
|
||||
: batch_(batch),
|
||||
in_depths_(in_depths),
|
||||
out_depths_(out_depths),
|
||||
in_(in),
|
||||
data_format_(data_format),
|
||||
filter_(filter),
|
||||
dilation_(dilation),
|
||||
stride_(stride),
|
||||
padding_(padding),
|
||||
dtype_(dtype),
|
||||
device_id_(device_id) {
|
||||
hash_code_ = batch;
|
||||
hash_code_ = Hash64Combine(hash_code_, in_depths);
|
||||
for (int64 val : in) hash_code_ = Hash64Combine(hash_code_, val);
|
||||
hash_code_ = Hash64Combine(hash_code_, data_format);
|
||||
hash_code_ = Hash64Combine(hash_code_, out_depths);
|
||||
for (int64 val : filter) hash_code_ = Hash64Combine(hash_code_, val);
|
||||
for (int64 val : dilation) hash_code_ = Hash64Combine(hash_code_, val);
|
||||
for (int64 val : stride) hash_code_ = Hash64Combine(hash_code_, val);
|
||||
for (int64 val : padding) hash_code_ = Hash64Combine(hash_code_, val);
|
||||
hash_code_ = Hash64Combine(hash_code_, dtype);
|
||||
hash_code_ = Hash64Combine(hash_code_, device_id);
|
||||
}
|
||||
bool operator==(const ConvParameters& other) const {
|
||||
return this->get_data_as_tuple() == other.get_data_as_tuple();
|
||||
}
|
||||
|
||||
bool operator!=(const ConvParameters& other) const {
|
||||
return !(*this == other);
|
||||
}
|
||||
uint64 hash() const { return hash_code_; }
|
||||
|
||||
string ToString() const {
|
||||
// clang-format off
|
||||
return strings::StrCat(
|
||||
batch_, ", ", in_depths_, ", ",
|
||||
"(", str_util::Join(in_, ", "), "), ",
|
||||
::tensorflow::ToString(data_format_), ", ",
|
||||
out_depths_, ", ",
|
||||
"(", str_util::Join(filter_, ", "), "), ",
|
||||
"(", str_util::Join(dilation_, ", "), "), ",
|
||||
"(", str_util::Join(stride_, ", "), "), ",
|
||||
"(", str_util::Join(padding_, ", "), "), ",
|
||||
dtype_, ", ",
|
||||
device_id_);
|
||||
// clang-format on
|
||||
}
|
||||
|
||||
// The purpose of this function is to disable winograd nonfused conv algorithm
|
||||
// for certain input parameters so as to avoid a bug in cuDNNv5 and cuDNNv6.
|
||||
template <typename T>
|
||||
bool ShouldIncludeWinogradNonfusedAlgo(
|
||||
se::StreamExecutor* stream_exec) const {
|
||||
auto* dnn_support = stream_exec->AsDnn();
|
||||
if (!dnn_support) {
|
||||
return false;
|
||||
}
|
||||
// Skip this check for cuDNN 7 and newer.
|
||||
auto version = dnn_support->GetVersion();
|
||||
if (version.ok() && version.ValueOrDie().major_version() >= 7) {
|
||||
return true;
|
||||
}
|
||||
return ShouldIncludeWinogradNonfusedAlgoPreCudnn7<T>();
|
||||
}
|
||||
|
||||
protected:
|
||||
using ParameterDataType =
|
||||
std::tuple<int64, int64, SpatialArray, TensorFormat, int64, SpatialArray,
|
||||
SpatialArray, SpatialArray, SpatialArray, DataType, int>;
|
||||
|
||||
ParameterDataType get_data_as_tuple() const {
|
||||
return std::make_tuple(batch_, in_depths_, in_, data_format_, out_depths_,
|
||||
filter_, dilation_, stride_, padding_, dtype_,
|
||||
device_id_);
|
||||
}
|
||||
|
||||
uint64 hash_code_;
|
||||
|
||||
private:
|
||||
friend struct ConvParametersPeer; // For testing purposes.
|
||||
|
||||
template <typename T>
|
||||
bool ShouldIncludeWinogradNonfusedAlgoPreCudnn7() const {
|
||||
int64 total_size = 16 * std::ceil(batch_ / 16.0) *
|
||||
std::max(in_depths_, out_depths_) * in_[0] * in_[1] *
|
||||
sizeof(T);
|
||||
int64 threshold = 1LL << 31;
|
||||
if (total_size >= threshold) {
|
||||
return false;
|
||||
} else {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
|
||||
int64 batch_;
|
||||
int64 in_depths_;
|
||||
int64 out_depths_;
|
||||
SpatialArray in_;
|
||||
TensorFormat data_format_;
|
||||
SpatialArray filter_;
|
||||
SpatialArray dilation_;
|
||||
SpatialArray stride_;
|
||||
SpatialArray padding_;
|
||||
DataType dtype_;
|
||||
int device_id_;
|
||||
};
|
||||
|
||||
typedef Eigen::GpuDevice GPUDevice;
|
||||
|
||||
} // namespace tensorflow
|
||||
|
||||
#endif // GOOGLE_CUDA
|
||||
|
||||
#endif // TENSORFLOW_CORE_KERNELS_CONV_OPS_GPU_H_
|
||||
@@ -0,0 +1,71 @@
|
||||
/* Copyright 2016 The TensorFlow Authors. All Rights Reserved.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
==============================================================================*/
|
||||
|
||||
#ifndef TENSORFLOW_CORE_KERNELS_CROP_AND_RESIZE_OP_H_
|
||||
#define TENSORFLOW_CORE_KERNELS_CROP_AND_RESIZE_OP_H_
|
||||
|
||||
#include "third_party/eigen3/unsupported/Eigen/CXX11/Tensor"
|
||||
#include "tensorflow/core/framework/numeric_types.h"
|
||||
#include "tensorflow/core/framework/op_kernel.h"
|
||||
#include "tensorflow/core/framework/tensor_types.h"
|
||||
|
||||
namespace tensorflow {
|
||||
namespace functor {
|
||||
|
||||
template <typename Device, typename T>
|
||||
struct CropAndResize {
|
||||
// We assume that the tensor sizes are correct.
|
||||
bool operator()(const OpKernelContext* context,
|
||||
typename TTypes<T, 4>::ConstTensor image,
|
||||
typename TTypes<float, 2>::ConstTensor boxes,
|
||||
typename TTypes<int32, 1>::ConstTensor box_ind,
|
||||
string method_name, float extrapolation_value,
|
||||
typename TTypes<float, 4>::Tensor crops);
|
||||
};
|
||||
|
||||
template <typename Device, typename T>
|
||||
struct CropAndResizeBackpropImage {
|
||||
// We assume that the tensor sizes are correct.
|
||||
bool operator()(const Device& d, typename TTypes<float, 4>::ConstTensor grads,
|
||||
typename TTypes<float, 2>::ConstTensor boxes,
|
||||
typename TTypes<int32, 1>::ConstTensor box_ind,
|
||||
typename TTypes<T, 4>::Tensor grads_image,
|
||||
const string& method_name);
|
||||
};
|
||||
|
||||
template <typename Device, typename T>
|
||||
struct CropAndResizeBackpropBoxes {
|
||||
// We assume that the tensor sizes are correct.
|
||||
bool operator()(const Device& d, typename TTypes<float, 4>::ConstTensor grads,
|
||||
typename TTypes<T, 4>::ConstTensor image,
|
||||
typename TTypes<float, 2>::ConstTensor boxes,
|
||||
typename TTypes<int32, 1>::ConstTensor box_ind,
|
||||
typename TTypes<float, 2>::Tensor grads_boxes);
|
||||
};
|
||||
|
||||
template <typename Device>
|
||||
struct CheckValidBoxIndexHelper {
|
||||
// Checks if all values in box_index are in [0, batch).
|
||||
void operator()(const Device& d,
|
||||
typename TTypes<int32, 1>::ConstTensor box_index, int batch,
|
||||
typename TTypes<bool, 0>::Tensor isvalid) {
|
||||
isvalid.device(d) = ((box_index >= 0) && (box_index < batch)).all();
|
||||
}
|
||||
};
|
||||
|
||||
} // namespace functor
|
||||
} // namespace tensorflow
|
||||
|
||||
#endif // TENSORFLOW_CORE_KERNELS_CROP_AND_RESIZE_OP_H_
|
||||
@@ -0,0 +1,54 @@
|
||||
/* Copyright 2015 The TensorFlow Authors. All Rights Reserved.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
==============================================================================*/
|
||||
|
||||
#ifndef TENSORFLOW_CORE_KERNELS_CROSS_OP_H_
|
||||
#define TENSORFLOW_CORE_KERNELS_CROSS_OP_H_
|
||||
|
||||
#include "third_party/eigen3/unsupported/Eigen/CXX11/Tensor"
|
||||
#include "tensorflow/core/framework/tensor_shape.h"
|
||||
#include "tensorflow/core/framework/tensor_types.h"
|
||||
|
||||
namespace tensorflow {
|
||||
|
||||
namespace functor {
|
||||
|
||||
template <typename Device, typename Type>
|
||||
struct Cross {
|
||||
void operator()(const Device &d,
|
||||
typename TTypes<Type, 2>::ConstTensor in0_data,
|
||||
typename TTypes<Type, 2>::ConstTensor in1_data,
|
||||
typename TTypes<Type, 2>::Tensor output_data) {
|
||||
auto s1 = output_data.template chip<1>(0);
|
||||
auto s2 = output_data.template chip<1>(1);
|
||||
auto s3 = output_data.template chip<1>(2);
|
||||
|
||||
auto u1 = in0_data.template chip<1>(0);
|
||||
auto u2 = in0_data.template chip<1>(1);
|
||||
auto u3 = in0_data.template chip<1>(2);
|
||||
|
||||
auto v1 = in1_data.template chip<1>(0);
|
||||
auto v2 = in1_data.template chip<1>(1);
|
||||
auto v3 = in1_data.template chip<1>(2);
|
||||
|
||||
s1.device(d) = u2 * v3 - u3 * v2;
|
||||
s2.device(d) = u3 * v1 - u1 * v3;
|
||||
s3.device(d) = u1 * v2 - u2 * v1;
|
||||
}
|
||||
};
|
||||
|
||||
} // namespace functor
|
||||
} // namespace tensorflow
|
||||
|
||||
#endif // TENSORFLOW_CORE_KERNELS_CROSS_OP_H_
|
||||
@@ -0,0 +1,120 @@
|
||||
/* Copyright 2016 The TensorFlow Authors. All Rights Reserved.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
==============================================================================*/
|
||||
#ifndef TENSORFLOW_CORE_KERNELS_CUDA_DEVICE_ARRAY_H_
|
||||
#define TENSORFLOW_CORE_KERNELS_CUDA_DEVICE_ARRAY_H_
|
||||
|
||||
#if GOOGLE_CUDA
|
||||
|
||||
#include "tensorflow/core/common_runtime/gpu/gpu_event_mgr.h"
|
||||
#include "tensorflow/core/framework/op_kernel.h"
|
||||
#include "tensorflow/core/kernels/cuda_device_array_gpu.h"
|
||||
|
||||
namespace tensorflow {
|
||||
|
||||
// Create an array of value on the host, to be sent to kernel using
|
||||
// CudaDeviceArrayStruct.
|
||||
//
|
||||
// Usage:
|
||||
// int size = ...;
|
||||
// CudaDeviceArrayOnHost ptrs(context, size);
|
||||
// OP_REQUIRES_OK(ptrs.Init());
|
||||
// for (int i = 0; i < size; ++i) {
|
||||
// ptrs.Set(i, ...);
|
||||
// }
|
||||
// OP_REQUIRES_OK(ptrs.Finalize());
|
||||
// launchKernel(..., ptrs.data, ...);
|
||||
//
|
||||
// ValueType must be memcopyable.
|
||||
template <typename ValueType, int MaxInlineValues = 8>
|
||||
class CudaDeviceArrayOnHost {
|
||||
public:
|
||||
CudaDeviceArrayOnHost(OpKernelContext* context, int32 size)
|
||||
: context_(context),
|
||||
total_bytes_(static_cast<int64>(size) * sizeof(ValueType)) {
|
||||
data_.size = size;
|
||||
}
|
||||
|
||||
Status Init() {
|
||||
if (inlined()) {
|
||||
values_ = data_.inline_values;
|
||||
return Status::OK();
|
||||
}
|
||||
|
||||
// Out-of-line: allocate data that will be memcopied.
|
||||
AllocatorAttributes attr;
|
||||
attr.set_on_host(true);
|
||||
attr.set_gpu_compatible(true);
|
||||
TF_RETURN_IF_ERROR(
|
||||
context_->allocate_temp(DT_INT8, TensorShape{total_bytes_},
|
||||
&out_of_line_values_on_host_, attr));
|
||||
values_ = reinterpret_cast<ValueType*>(
|
||||
out_of_line_values_on_host_.flat<int8>().data());
|
||||
return Status::OK();
|
||||
}
|
||||
|
||||
void Set(int index, ValueType val) {
|
||||
DCHECK(values_); // ensure Init was called.
|
||||
DCHECK_LT(index, data_.size);
|
||||
*(values_ + index) = val;
|
||||
}
|
||||
|
||||
Status Finalize() {
|
||||
if (inlined()) {
|
||||
return Status::OK();
|
||||
}
|
||||
|
||||
// Out-of-line - copy pointers to device.
|
||||
auto stream = context_->op_device_context()->stream();
|
||||
TensorReference tensor_ref(out_of_line_values_on_host_);
|
||||
TF_RETURN_IF_ERROR(context_->allocate_temp(
|
||||
DT_INT8, TensorShape{total_bytes_}, &out_of_line_values_on_gpu_));
|
||||
se::DeviceMemoryBase output_values_base{
|
||||
out_of_line_values_on_gpu_.flat<int8>().data(),
|
||||
static_cast<uint64>(total_bytes_)};
|
||||
stream->ThenMemcpy(&output_values_base,
|
||||
out_of_line_values_on_host_.flat<int8>().data(),
|
||||
total_bytes_);
|
||||
context_->device()->tensorflow_gpu_device_info()->event_mgr->ThenExecute(
|
||||
stream, [tensor_ref]() { tensor_ref.Unref(); });
|
||||
data_.out_of_line_values = reinterpret_cast<ValueType*>(
|
||||
out_of_line_values_on_gpu_.flat<int8>().data());
|
||||
return Status::OK();
|
||||
}
|
||||
|
||||
const CudaDeviceArrayStruct<ValueType, MaxInlineValues>& data() const {
|
||||
// Ensure Finalize is called.
|
||||
DCHECK(inlined() || out_of_line_values_on_gpu_.IsInitialized());
|
||||
return data_;
|
||||
}
|
||||
|
||||
private:
|
||||
bool inlined() const { return data_.size <= MaxInlineValues; }
|
||||
|
||||
OpKernelContext* const context_;
|
||||
const int64 total_bytes_; // total size of all pointers.
|
||||
ValueType* values_ = nullptr;
|
||||
CudaDeviceArrayStruct<ValueType, MaxInlineValues> data_;
|
||||
|
||||
Tensor out_of_line_values_on_host_;
|
||||
Tensor out_of_line_values_on_gpu_;
|
||||
|
||||
TF_DISALLOW_COPY_AND_ASSIGN(CudaDeviceArrayOnHost);
|
||||
};
|
||||
|
||||
} // namespace tensorflow
|
||||
|
||||
#endif // GOOGLE_CUDA
|
||||
|
||||
#endif // TENSORFLOW_CORE_KERNELS_CUDA_DEVICE_ARRAY_H_
|
||||
@@ -0,0 +1,50 @@
|
||||
/* Copyright 2016 The TensorFlow Authors. All Rights Reserved.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
==============================================================================*/
|
||||
|
||||
// Contains structs and functions to be included in device code.
|
||||
|
||||
#ifndef TENSORFLOW_CORE_KERNELS_CUDA_DEVICE_ARRAY_GPU_H_
|
||||
#define TENSORFLOW_CORE_KERNELS_CUDA_DEVICE_ARRAY_GPU_H_
|
||||
|
||||
#if GOOGLE_CUDA
|
||||
|
||||
namespace tensorflow {
|
||||
|
||||
static constexpr int kMaxInlineCudaPointers = 8;
|
||||
// To decode on the device side, use GetCudaDeviceArrayOnDevice.
|
||||
// To encode on the host side, use CudaDeviceArrayOnHost.
|
||||
template <typename ValueType, int MaxInlineValues = 8>
|
||||
struct CudaDeviceArrayStruct {
|
||||
int32 size;
|
||||
// used if size <= MaxInlineValues;
|
||||
ValueType inline_values[MaxInlineValues];
|
||||
ValueType* out_of_line_values = nullptr; // used if size > MaxInlineValues;
|
||||
};
|
||||
|
||||
template <typename ValueType, int MaxInlineValues = 8>
|
||||
EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE ValueType* GetCudaDeviceArrayOnDevice(
|
||||
CudaDeviceArrayStruct<ValueType, MaxInlineValues>* data) {
|
||||
if (data->size <= MaxInlineValues) {
|
||||
return data->inline_values;
|
||||
} else {
|
||||
return data->out_of_line_values;
|
||||
}
|
||||
}
|
||||
|
||||
} // namespace tensorflow
|
||||
|
||||
#endif // GOOGLE_CUDA
|
||||
|
||||
#endif // TENSORFLOW_CORE_KERNELS_CUDA_DEVICE_ARRAY_GPU_H_
|
||||
@@ -0,0 +1,70 @@
|
||||
/* Copyright 2016 The TensorFlow Authors. All Rights Reserved.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
==============================================================================*/
|
||||
|
||||
// Helper functions to run 3d pooling on GPU using CuDNN.
|
||||
|
||||
#ifndef TENSORFLOW_CORE_KERNELS_CUDNN_POOLING_GPU_H_
|
||||
#define TENSORFLOW_CORE_KERNELS_CUDNN_POOLING_GPU_H_
|
||||
|
||||
#include <array>
|
||||
|
||||
#include "tensorflow/core/framework/op_kernel.h"
|
||||
|
||||
#if GOOGLE_CUDA
|
||||
#include "tensorflow/core/platform/stream_executor.h"
|
||||
#endif
|
||||
|
||||
#include "tensorflow/core/util/padding.h"
|
||||
|
||||
namespace tensorflow {
|
||||
|
||||
#if GOOGLE_CUDA
|
||||
|
||||
// Runs (avg/max)pooling on GPU.
|
||||
// Dimension order for all array arguments is: x, y, z.
|
||||
template <typename T>
|
||||
class DnnPooling3dOp {
|
||||
public:
|
||||
static void Compute(OpKernelContext* context,
|
||||
se::dnn::PoolingMode pooling_mode,
|
||||
const std::array<int64, 3>& size,
|
||||
const std::array<int64, 3>& stride,
|
||||
const std::array<int64, 3>& padding,
|
||||
TensorFormat data_format, const Tensor& tensor_in,
|
||||
Tensor* output);
|
||||
};
|
||||
|
||||
// Computes the gradient of (avg/max)pooling on GPU.
|
||||
// Dimension order for all array arguments is: x, y, z.
|
||||
template <typename T>
|
||||
class DnnPooling3dGradOp {
|
||||
public:
|
||||
static void Compute(OpKernelContext* context,
|
||||
se::dnn::PoolingMode pooling_mode,
|
||||
const std::array<int64, 3>& window,
|
||||
const std::array<int64, 3>& stride,
|
||||
const std::array<int64, 3>& padding,
|
||||
const std::array<int64, 3>& output_size,
|
||||
TensorFormat data_format, const Tensor& out_backprop,
|
||||
const TensorShape& tensor_in_shape,
|
||||
const Tensor* tensor_in, const Tensor* tensor_out,
|
||||
Tensor* input_backprop);
|
||||
};
|
||||
|
||||
#endif
|
||||
|
||||
} // namespace tensorflow
|
||||
|
||||
#endif // TENSORFLOW_CORE_KERNELS_CUDNN_POOLING_GPU_H_
|
||||
@@ -0,0 +1,61 @@
|
||||
/* Copyright 2015 The TensorFlow Authors. All Rights Reserved.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
==============================================================================*/
|
||||
|
||||
#ifndef TENSORFLOW_CORE_KERNELS_CWISE_OP_CLIP_H_
|
||||
#define TENSORFLOW_CORE_KERNELS_CWISE_OP_CLIP_H_
|
||||
|
||||
#include "tensorflow/core/kernels/cwise_ops_common.h"
|
||||
|
||||
namespace tensorflow {
|
||||
namespace functor {
|
||||
// Unary functor for clip [Tensor, Scalar, Scalar]
|
||||
template <typename Device, typename T>
|
||||
struct UnaryClipOp {
|
||||
void operator()(const Device &d, typename TTypes<T>::ConstFlat &in0_flat,
|
||||
typename TTypes<T>::ConstFlat &in1_flat,
|
||||
typename TTypes<T>::ConstFlat &in2_flat,
|
||||
typename TTypes<T>::Flat &out_flat) const;
|
||||
};
|
||||
|
||||
// Binary functor for clip [Tensor, Scalar, Tensor]
|
||||
template <typename Device, typename T>
|
||||
struct BinaryRightClipOp {
|
||||
void operator()(const Device &d, typename TTypes<T>::ConstFlat &in0_flat,
|
||||
typename TTypes<T>::ConstFlat &in1_flat,
|
||||
typename TTypes<T>::ConstFlat &in2_flat,
|
||||
typename TTypes<T>::Flat &out_flat) const;
|
||||
};
|
||||
|
||||
// Binary functor for clip [Tensor, Tensor, Scalar]
|
||||
template <typename Device, typename T>
|
||||
struct BinaryLeftClipOp {
|
||||
void operator()(const Device &d, typename TTypes<T>::ConstFlat &in0_flat,
|
||||
typename TTypes<T>::ConstFlat &in1_flat,
|
||||
typename TTypes<T>::ConstFlat &in2_flat,
|
||||
typename TTypes<T>::Flat &out_flat) const;
|
||||
};
|
||||
|
||||
// Ternary functor for clip [Tensor, Tensor, Tensor]
|
||||
template <typename Device, typename T>
|
||||
struct TernaryClipOp {
|
||||
void operator()(const Device &d, typename TTypes<T>::ConstFlat &in0_flat,
|
||||
typename TTypes<T>::ConstFlat &in1_flat,
|
||||
typename TTypes<T>::ConstFlat &in2_flat,
|
||||
typename TTypes<T>::Flat &out_flat) const;
|
||||
};
|
||||
} // namespace functor
|
||||
} // namespace tensorflow
|
||||
|
||||
#endif // TENSORFLOW_CORE_KERNELS_CWISE_OP_CLIP_H_
|
||||
File diff suppressed because it is too large.
Load diff
@@ -0,0 +1,605 @@
|
||||
/* Copyright 2015 The TensorFlow Authors. All Rights Reserved.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
==============================================================================*/
|
||||
|
||||
#ifndef TENSORFLOW_CORE_KERNELS_CWISE_OPS_COMMON_H_
|
||||
#define TENSORFLOW_CORE_KERNELS_CWISE_OPS_COMMON_H_
|
||||
|
||||
// See docs in ../ops/math_ops.cc.
|
||||
|
||||
#define EIGEN_USE_THREADS
|
||||
|
||||
#include "tensorflow/core/lib/bfloat16/bfloat16.h"
|
||||
|
||||
#ifdef TENSORFLOW_USE_SYCL
|
||||
#include "tensorflow/core/kernels/cwise_ops_sycl_common.h"
|
||||
#endif
|
||||
|
||||
#include "tensorflow/core/kernels/cwise_ops.h"
|
||||
#include "tensorflow/core/kernels/cwise_ops_gradients.h"
|
||||
|
||||
#include "tensorflow/core/framework/op.h"
|
||||
#include "tensorflow/core/framework/op_kernel.h"
|
||||
#include "tensorflow/core/framework/tensor_types.h"
|
||||
#include "tensorflow/core/framework/variant_op_registry.h"
|
||||
#include "tensorflow/core/platform/logging.h"
|
||||
#include "tensorflow/core/util/bcast.h"
|
||||
|
||||
namespace tensorflow {
|
||||
|
||||
typedef Eigen::ThreadPoolDevice CPUDevice;
|
||||
typedef Eigen::GpuDevice GPUDevice;
|
||||
#ifdef TENSORFLOW_USE_SYCL
|
||||
typedef Eigen::SyclDevice SYCLDevice;
|
||||
#endif
|
||||
|
||||
class BinaryOpShared : public OpKernel {
|
||||
public:
|
||||
explicit BinaryOpShared(OpKernelConstruction* ctx, DataType out, DataType in);
|
||||
|
||||
protected:
|
||||
struct BinaryOpState {
|
||||
// Sets up bcast with the shape of in0 and in1, ensures that the bcast
|
||||
// is valid, and if so, set out, either by allocating a new buffer using
|
||||
// ctx->output(...) or by creating an alias for an owned input buffer for
|
||||
// in-place computation.
|
||||
// Caller must check ctx->status() upon return for non-ok status.
|
||||
// If ctx->status().ok() is true, then out is guaranteed to be allocated.
|
||||
BinaryOpState(OpKernelContext* ctx);
|
||||
|
||||
const Tensor& in0;
|
||||
const Tensor& in1;
|
||||
|
||||
BCast bcast;
|
||||
Tensor* out = nullptr;
|
||||
int64 out_num_elements;
|
||||
|
||||
int64 in0_num_elements;
|
||||
int64 in1_num_elements;
|
||||
|
||||
int ndims;
|
||||
};
|
||||
|
||||
void SetUnimplementedError(OpKernelContext* ctx);
|
||||
void SetComputeError(OpKernelContext* ctx);
|
||||
};
|
||||
|
||||
// Coefficient-wise binary operations:
|
||||
// Device: E.g., CPUDevice, GPUDevice.
|
||||
// Functor: defined in cwise_ops.h. E.g., functor::add.
|
||||
template <typename Device, typename Functor>
|
||||
class BinaryOp : public BinaryOpShared {
|
||||
public:
|
||||
typedef typename Functor::in_type Tin; // Input scalar data type.
|
||||
typedef typename Functor::out_type Tout; // Output scalar data type.
|
||||
|
||||
explicit BinaryOp(OpKernelConstruction* ctx)
|
||||
: BinaryOpShared(ctx, DataTypeToEnum<Tout>::v(),
|
||||
DataTypeToEnum<Tin>::v()) {}
|
||||
|
||||
void Compute(OpKernelContext* ctx) override {
|
||||
// 'state': Shared helper not dependent on T to reduce code size
|
||||
BinaryOpState state(ctx);
|
||||
if (!ctx->status().ok()) return;
|
||||
Tensor* out = state.out;
|
||||
BCast* bcast = &state.bcast;
|
||||
auto& in0 = state.in0;
|
||||
auto& in1 = state.in1;
|
||||
if (state.out_num_elements == 0) {
|
||||
return;
|
||||
}
|
||||
const int ndims = state.ndims;
|
||||
const Device& eigen_device = ctx->eigen_device<Device>();
|
||||
bool error = false;
|
||||
bool* const error_ptr = Functor::has_errors ? &error : nullptr;
|
||||
if (ndims <= 1) {
|
||||
auto out_flat = out->flat<Tout>();
|
||||
if (state.in1_num_elements == 1) {
|
||||
// tensor op scalar
|
||||
functor::BinaryFunctor<Device, Functor, 1>().Right(
|
||||
eigen_device, out_flat, in0.template flat<Tin>(),
|
||||
in1.template scalar<Tin>(), error_ptr);
|
||||
} else if (state.in0_num_elements == 1) {
|
||||
// scalar op tensor
|
||||
functor::BinaryFunctor<Device, Functor, 1>().Left(
|
||||
eigen_device, out_flat, in0.template scalar<Tin>(),
|
||||
in1.template flat<Tin>(), error_ptr);
|
||||
} else {
|
||||
functor::BinaryFunctor<Device, Functor, 1>()(
|
||||
eigen_device, out_flat, in0.template flat<Tin>(),
|
||||
in1.template flat<Tin>(), error_ptr);
|
||||
}
|
||||
} else if (ndims == 2) {
|
||||
functor::BinaryFunctor<Device, Functor, 2>().BCast(
|
||||
eigen_device, out->shaped<Tout, 2>(bcast->result_shape()),
|
||||
in0.template shaped<Tin, 2>(bcast->x_reshape()),
|
||||
BCast::ToIndexArray<2>(bcast->x_bcast()),
|
||||
in1.template shaped<Tin, 2>(bcast->y_reshape()),
|
||||
BCast::ToIndexArray<2>(bcast->y_bcast()), error_ptr);
|
||||
} else if (ndims == 3) {
|
||||
functor::BinaryFunctor<Device, Functor, 3>().BCast(
|
||||
eigen_device, out->shaped<Tout, 3>(bcast->result_shape()),
|
||||
in0.template shaped<Tin, 3>(bcast->x_reshape()),
|
||||
BCast::ToIndexArray<3>(bcast->x_bcast()),
|
||||
in1.template shaped<Tin, 3>(bcast->y_reshape()),
|
||||
BCast::ToIndexArray<3>(bcast->y_bcast()), error_ptr);
|
||||
} else if (ndims == 4) {
|
||||
functor::BinaryFunctor<Device, Functor, 4>().BCast(
|
||||
eigen_device, out->shaped<Tout, 4>(bcast->result_shape()),
|
||||
in0.template shaped<Tin, 4>(bcast->x_reshape()),
|
||||
BCast::ToIndexArray<4>(bcast->x_bcast()),
|
||||
in1.template shaped<Tin, 4>(bcast->y_reshape()),
|
||||
BCast::ToIndexArray<4>(bcast->y_bcast()), error_ptr);
|
||||
} else if (ndims == 5) {
|
||||
functor::BinaryFunctor<Device, Functor, 5>().BCast(
|
||||
eigen_device, out->shaped<Tout, 5>(bcast->result_shape()),
|
||||
in0.template shaped<Tin, 5>(bcast->x_reshape()),
|
||||
BCast::ToIndexArray<5>(bcast->x_bcast()),
|
||||
in1.template shaped<Tin, 5>(bcast->y_reshape()),
|
||||
BCast::ToIndexArray<5>(bcast->y_bcast()), error_ptr);
|
||||
} else {
|
||||
SetUnimplementedError(ctx);
|
||||
}
|
||||
if (Functor::has_errors && error) {
|
||||
SetComputeError(ctx);
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
template <typename Device, typename T>
|
||||
class ApproximateEqualOp : public OpKernel {
|
||||
public:
|
||||
explicit ApproximateEqualOp(OpKernelConstruction* context)
|
||||
: OpKernel(context) {
|
||||
float tolerance;
|
||||
OP_REQUIRES_OK(context, context->GetAttr("tolerance", &tolerance));
|
||||
tolerance_ = T(tolerance);
|
||||
}
|
||||
void Compute(OpKernelContext* context) override {
|
||||
const Tensor& x_input = context->input(0);
|
||||
const Tensor& y_input = context->input(1);
|
||||
OP_REQUIRES(
|
||||
context, x_input.shape() == y_input.shape(),
|
||||
errors::InvalidArgument("x and y must be of the same shape. ",
|
||||
"x shape: ", x_input.shape().DebugString(),
|
||||
". y shape: ", y_input.shape().DebugString()));
|
||||
Tensor* z_output = nullptr;
|
||||
OP_REQUIRES_OK(context,
|
||||
context->allocate_output(0, x_input.shape(), &z_output));
|
||||
const Device& d = context->eigen_device<Device>();
|
||||
typename TTypes<T>::ConstFlat x(x_input.flat<T>());
|
||||
typename TTypes<T>::ConstFlat y(y_input.flat<T>());
|
||||
typename TTypes<bool>::Flat z(z_output->flat<bool>());
|
||||
functor::ApproximateEqual<Device, T>()(d, x, y, tolerance_, z);
|
||||
}
|
||||
|
||||
private:
|
||||
T tolerance_;
|
||||
};
|
||||
|
||||
// Basic coefficient-wise binary operations that are known to not require
|
||||
// any broadcasting. This is the case for example of the gradients of
|
||||
// unary operations.
|
||||
// Device: E.g., CPUDevice, GPUDevice.
|
||||
// Functor: defined above. E.g., functor::tanh_grad.
|
||||
template <typename Device, typename Functor>
|
||||
class SimpleBinaryOp : public OpKernel {
|
||||
public:
|
||||
typedef typename Functor::in_type Tin; // Input scalar data type.
|
||||
typedef typename Functor::out_type Tout; // Output scalar data type.
|
||||
|
||||
explicit SimpleBinaryOp(OpKernelConstruction* ctx) : OpKernel(ctx) {}
|
||||
|
||||
void Compute(OpKernelContext* ctx) override {
|
||||
const Tensor& in0 = ctx->input(0);
|
||||
const Tensor& in1 = ctx->input(1);
|
||||
auto in0_flat = in0.flat<Tin>();
|
||||
auto in1_flat = in1.flat<Tin>();
|
||||
const Device& eigen_device = ctx->eigen_device<Device>();
|
||||
|
||||
Tensor* out = nullptr;
|
||||
if (std::is_same<Tin, Tout>::value) {
|
||||
OP_REQUIRES_OK(ctx, ctx->forward_input_or_allocate_output(
|
||||
{0, 1}, 0, in0.shape(), &out));
|
||||
} else {
|
||||
OP_REQUIRES_OK(ctx, ctx->allocate_output(0, in0.shape(), &out));
|
||||
}
|
||||
auto out_flat = out->flat<Tout>();
|
||||
functor::SimpleBinaryFunctor<Device, Functor>()(eigen_device, out_flat,
|
||||
in0_flat, in1_flat);
|
||||
}
|
||||
};
|
||||
|
||||
// Coefficient-wise unary operations:
|
||||
// Device: E.g., CPUDevice, GPUDevice.
|
||||
// Functor: defined in cwise_ops.h. E.g., functor::sqrt.
|
||||
template <typename Device, typename Functor>
|
||||
class UnaryOp : public OpKernel {
|
||||
public:
|
||||
typedef typename Functor::in_type Tin; // Input scalar data type.
|
||||
typedef typename Functor::out_type Tout; // Output scalar data type.
|
||||
// Tin may be different from Tout. E.g., abs: complex64 -> float
|
||||
|
||||
explicit UnaryOp(OpKernelConstruction* ctx) : OpKernel(ctx) {
|
||||
auto in = DataTypeToEnum<Tin>::v();
|
||||
auto out = DataTypeToEnum<Tout>::v();
|
||||
OP_REQUIRES_OK(ctx, ctx->MatchSignature({in}, {out}));
|
||||
}
|
||||
|
||||
void Compute(OpKernelContext* ctx) override {
|
||||
const Tensor& inp = ctx->input(0);
|
||||
Tensor* out = nullptr;
|
||||
if (std::is_same<Tin, Tout>::value) {
|
||||
OP_REQUIRES_OK(ctx, ctx->forward_input_or_allocate_output(
|
||||
{0}, 0, inp.shape(), &out));
|
||||
} else {
|
||||
OP_REQUIRES_OK(ctx, ctx->allocate_output(0, inp.shape(), &out));
|
||||
}
|
||||
functor::UnaryFunctor<Device, Functor>()(
|
||||
ctx->eigen_device<Device>(), out->flat<Tout>(), inp.flat<Tin>());
|
||||
}
|
||||
};
|
||||
|
||||
template <typename Device, VariantUnaryOp OpEnum>
|
||||
class UnaryVariantOp : public OpKernel {
|
||||
public:
|
||||
explicit UnaryVariantOp(OpKernelConstruction* ctx) : OpKernel(ctx) {}
|
||||
|
||||
void Compute(OpKernelContext* ctx) override {
|
||||
const Tensor& inp = ctx->input(0);
|
||||
OP_REQUIRES(
|
||||
ctx, TensorShapeUtils::IsScalar(inp.shape()),
|
||||
errors::InvalidArgument("Non-scalar variants are not supported."));
|
||||
const Variant& v = inp.scalar<Variant>()();
|
||||
Variant v_out;
|
||||
OP_REQUIRES_OK(ctx, UnaryOpVariant<Device>(ctx, OpEnum, v, &v_out));
|
||||
Tensor out(cpu_allocator(), DT_VARIANT, TensorShape());
|
||||
out.scalar<Variant>()() = std::move(v_out);
|
||||
ctx->set_output(0, std::move(out));
|
||||
}
|
||||
};
|
||||
|
||||
namespace functor {
|
||||
|
||||
template <typename D, typename Out, typename Rhs>
|
||||
void Assign(const D& d, Out out, Rhs rhs) {
|
||||
out.device(d) = rhs;
|
||||
}
|
||||
|
||||
// Partial specialization of BinaryFunctor<Device=CPUDevice, Functor, NDIMS>
|
||||
// for functors with with no error checking.
|
||||
template <typename Functor, int NDIMS>
|
||||
struct BinaryFunctor<CPUDevice, Functor, NDIMS, false> {
|
||||
void operator()(const CPUDevice& d, typename Functor::tout_type out,
|
||||
typename Functor::tin_type in0,
|
||||
typename Functor::tin_type in1, bool* error) {
|
||||
Assign(d, out, in0.binaryExpr(in1, typename Functor::func()));
|
||||
}
|
||||
|
||||
void Left(const CPUDevice& d, typename Functor::tout_type out,
|
||||
typename Functor::tscalar_type scalar,
|
||||
typename Functor::tin_type in, bool* error) {
|
||||
typedef typename Functor::out_type Tout;
|
||||
typedef typename Functor::in_type Tin;
|
||||
typedef typename Functor::func Binary;
|
||||
typedef typename Eigen::internal::scalar_left<Tout, Tin, Binary> Unary;
|
||||
Assign(d, out, in.unaryExpr(Unary(scalar.data())));
|
||||
}
|
||||
|
||||
void Right(const CPUDevice& d, typename Functor::tout_type out,
|
||||
typename Functor::tin_type in,
|
||||
typename Functor::tscalar_type scalar, bool* error) {
|
||||
typedef typename Functor::out_type Tout;
|
||||
typedef typename Functor::in_type Tin;
|
||||
typedef typename Functor::func Binary;
|
||||
typedef typename Eigen::internal::scalar_right<Tout, Tin, Binary> Unary;
|
||||
Assign(d, out, in.unaryExpr(Unary(scalar.data())));
|
||||
}
|
||||
|
||||
void BCast(const CPUDevice& dev,
|
||||
typename TTypes<typename Functor::out_type, NDIMS>::Tensor out,
|
||||
typename TTypes<typename Functor::in_type, NDIMS>::ConstTensor in0,
|
||||
typename Eigen::array<Eigen::DenseIndex, NDIMS> bcast0,
|
||||
typename TTypes<typename Functor::in_type, NDIMS>::ConstTensor in1,
|
||||
typename Eigen::array<Eigen::DenseIndex, NDIMS> bcast1,
|
||||
bool* error) {
|
||||
typename Functor::func func;
|
||||
if (AllOne<NDIMS>(bcast0) && AllOne<NDIMS>(bcast1)) {
|
||||
Assign(dev, out, in0.binaryExpr(in1, func));
|
||||
} else if (AllOne<NDIMS>(bcast0)) {
|
||||
auto rhs = in1.broadcast(bcast1);
|
||||
Assign(dev, out, in0.binaryExpr(rhs, func));
|
||||
} else if (AllOne<NDIMS>(bcast1)) {
|
||||
auto lhs = in0.broadcast(bcast0);
|
||||
Assign(dev, out, lhs.binaryExpr(in1, func));
|
||||
} else {
|
||||
auto lhs = in0.broadcast(bcast0);
|
||||
auto rhs = in1.broadcast(bcast1);
|
||||
Assign(dev, out, lhs.binaryExpr(rhs, func));
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
// Partial specialization of BinaryFunctor<Device=CPUDevice, Functor, 2>
|
||||
// for functors with with no error checking.
|
||||
template <typename Functor>
|
||||
struct BinaryFunctor<CPUDevice, Functor, 2, false> {
|
||||
enum { NDIMS = 2 };
|
||||
|
||||
void operator()(const CPUDevice& d, typename Functor::tout_type out,
|
||||
typename Functor::tin_type in0,
|
||||
typename Functor::tin_type in1, bool* error) {
|
||||
Assign(d, out, in0.binaryExpr(in1, typename Functor::func()));
|
||||
}
|
||||
|
||||
void Left(const CPUDevice& d, typename Functor::tout_type out,
|
||||
typename Functor::tscalar_type scalar,
|
||||
typename Functor::tin_type in, bool* error) {
|
||||
typedef typename Functor::out_type Tout;
|
||||
typedef typename Functor::in_type Tin;
|
||||
typedef typename Functor::func Binary;
|
||||
typedef typename Eigen::internal::scalar_left<Tout, Tin, Binary> Unary;
|
||||
Assign(d, out, in.unaryExpr(Unary(scalar.data())));
|
||||
}
|
||||
|
||||
void Right(const CPUDevice& d, typename Functor::tout_type out,
|
||||
typename Functor::tin_type in,
|
||||
typename Functor::tscalar_type scalar, bool* error) {
|
||||
typedef typename Functor::out_type Tout;
|
||||
typedef typename Functor::in_type Tin;
|
||||
typedef typename Functor::func Binary;
|
||||
typedef typename Eigen::internal::scalar_right<Tout, Tin, Binary> Unary;
|
||||
Assign(d, out, in.unaryExpr(Unary(scalar.data())));
|
||||
}
|
||||
|
||||
#if !defined(EIGEN_HAS_INDEX_LIST)
|
||||
inline Eigen::DSizes<int, 2> NByOne(int n) {
|
||||
return Eigen::DSizes<int, 2>(n, 1);
|
||||
}
|
||||
inline Eigen::DSizes<int, 2> OneByM(int m) {
|
||||
return Eigen::DSizes<int, 2>(1, m);
|
||||
}
|
||||
#else
|
||||
inline Eigen::IndexList<int, Eigen::type2index<1>> NByOne(int n) {
|
||||
Eigen::IndexList<int, Eigen::type2index<1>> ret;
|
||||
ret.set(0, n);
|
||||
return ret;
|
||||
}
|
||||
inline Eigen::IndexList<Eigen::type2index<1>, int> OneByM(int m) {
|
||||
Eigen::IndexList<Eigen::type2index<1>, int> ret;
|
||||
ret.set(1, m);
|
||||
return ret;
|
||||
}
|
||||
#endif
|
||||
|
||||
void BCast(const CPUDevice& dev,
|
||||
typename TTypes<typename Functor::out_type, NDIMS>::Tensor out,
|
||||
typename TTypes<typename Functor::in_type, NDIMS>::ConstTensor in0,
|
||||
typename Eigen::array<Eigen::DenseIndex, NDIMS> bcast0,
|
||||
typename TTypes<typename Functor::in_type, NDIMS>::ConstTensor in1,
|
||||
typename Eigen::array<Eigen::DenseIndex, NDIMS> bcast1,
|
||||
bool* error) {
|
||||
typedef typename Functor::in_type T;
|
||||
typename Functor::func func;
|
||||
if (Functor::use_bcast_optimization && use_bcast_optimization<T>::value) {
|
||||
// Optimize for speed by using Eigen::type2index and avoid
|
||||
// .broadcast() when we know its a no-op.
|
||||
//
|
||||
// Here, we need to handle 6 cases depending on how many "1"
|
||||
// exist in in0 and in1's shapes (4 numbers in total). It's not
|
||||
// possible that two shapes have more than 2 1s because those
|
||||
// are simplified to NDIMS==1 case.
|
||||
//
|
||||
// Because this optimization increases the binary size for each
|
||||
// Functor (+, -, *, /, <, <=, etc.), type and ndim combination.
|
||||
// we only apply such optimization for selected ops/types/ndims.
|
||||
//
|
||||
// Because NDIMS, Functor::use_broadcast_optimization and
|
||||
// use_broadcast_optimization<T> are compile-time constant, gcc
|
||||
// does a decent job avoiding generating code when conditions
|
||||
// are not met.
|
||||
const int a = in0.dimension(0); // in0 is shape [a, b]
|
||||
const int b = in0.dimension(1);
|
||||
const int c = in1.dimension(0); // in1 is shape [c, d]
|
||||
const int d = in1.dimension(1);
|
||||
if ((a == 1) && (d == 1)) {
|
||||
auto lhs = in0.reshape(OneByM(b)).broadcast(NByOne(c));
|
||||
auto rhs = in1.reshape(NByOne(c)).broadcast(OneByM(b));
|
||||
Assign(dev, out, lhs.binaryExpr(rhs, func));
|
||||
return;
|
||||
}
|
||||
if ((b == 1) && (c == 1)) {
|
||||
auto lhs = in0.reshape(NByOne(a)).broadcast(OneByM(d));
|
||||
auto rhs = in1.reshape(OneByM(d)).broadcast(NByOne(a));
|
||||
Assign(dev, out, lhs.binaryExpr(rhs, func));
|
||||
return;
|
||||
}
|
||||
if (a == 1) {
|
||||
auto lhs = in0.reshape(OneByM(b)).broadcast(NByOne(c));
|
||||
auto rhs = in1;
|
||||
Assign(dev, out, lhs.binaryExpr(rhs, func));
|
||||
return;
|
||||
}
|
||||
if (b == 1) {
|
||||
auto lhs = in0.reshape(NByOne(a)).broadcast(OneByM(d));
|
||||
auto rhs = in1;
|
||||
Assign(dev, out, lhs.binaryExpr(rhs, func));
|
||||
return;
|
||||
}
|
||||
if (c == 1) {
|
||||
auto lhs = in0;
|
||||
auto rhs = in1.reshape(OneByM(d)).broadcast(NByOne(a));
|
||||
Assign(dev, out, lhs.binaryExpr(rhs, func));
|
||||
return;
|
||||
}
|
||||
if (d == 1) {
|
||||
auto lhs = in0;
|
||||
auto rhs = in1.reshape(NByOne(c)).broadcast(OneByM(b));
|
||||
Assign(dev, out, lhs.binaryExpr(rhs, func));
|
||||
return;
|
||||
}
|
||||
|
||||
const bool bcast0_all_one = AllOne<NDIMS>(bcast0);
|
||||
const bool bcast1_all_one = AllOne<NDIMS>(bcast1);
|
||||
if (bcast0_all_one && !bcast1_all_one) {
|
||||
auto lhs = in0; // No need to do broadcast for in0
|
||||
auto rhs = in1.broadcast(bcast1);
|
||||
Assign(dev, out, lhs.binaryExpr(rhs, func));
|
||||
return;
|
||||
}
|
||||
|
||||
if (!bcast0_all_one && bcast1_all_one) {
|
||||
auto lhs = in0.broadcast(bcast0);
|
||||
auto rhs = in1; // No need to do broadcast for in1
|
||||
Assign(dev, out, lhs.binaryExpr(rhs, func));
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
// Fallback path. Always works and probably slower.
|
||||
auto lhs = in0.broadcast(bcast0);
|
||||
auto rhs = in1.broadcast(bcast1);
|
||||
Assign(dev, out, lhs.binaryExpr(rhs, func));
|
||||
}
|
||||
};
|
||||
|
||||
// Version of BinaryFunctor with error handling.
|
||||
template <typename Functor, int NDIMS>
|
||||
struct BinaryFunctor<CPUDevice, Functor, NDIMS, true> {
|
||||
void operator()(const CPUDevice& d, typename Functor::tout_type out,
|
||||
typename Functor::tin_type in0,
|
||||
typename Functor::tin_type in1, bool* error) {
|
||||
Assign(d, out, in0.binaryExpr(in1, typename Functor::func(error)));
|
||||
}
|
||||
|
||||
void Left(const CPUDevice& d, typename Functor::tout_type out,
|
||||
typename Functor::tscalar_type scalar,
|
||||
typename Functor::tin_type in, bool* error) {
|
||||
typedef typename Functor::out_type Tout;
|
||||
typedef typename Functor::in_type Tin;
|
||||
typedef typename Functor::func Binary;
|
||||
typedef typename Eigen::internal::scalar_left<Tout, Tin, Binary> Unary;
|
||||
Assign(d, out, in.unaryExpr(Unary(scalar.data(), error)));
|
||||
}
|
||||
|
||||
void Right(const CPUDevice& d, typename Functor::tout_type out,
|
||||
typename Functor::tin_type in,
|
||||
typename Functor::tscalar_type scalar, bool* error) {
|
||||
typedef typename Functor::out_type Tout;
|
||||
typedef typename Functor::in_type Tin;
|
||||
typedef typename Functor::func Binary;
|
||||
typedef typename Eigen::internal::scalar_right<Tout, Tin, Binary> Unary;
|
||||
Assign(d, out, in.unaryExpr(Unary(scalar.data(), error)));
|
||||
}
|
||||
|
||||
void BCast(const CPUDevice& dev,
|
||||
typename TTypes<typename Functor::out_type, NDIMS>::Tensor out,
|
||||
typename TTypes<typename Functor::in_type, NDIMS>::ConstTensor in0,
|
||||
typename Eigen::array<Eigen::DenseIndex, NDIMS> bcast0,
|
||||
typename TTypes<typename Functor::in_type, NDIMS>::ConstTensor in1,
|
||||
typename Eigen::array<Eigen::DenseIndex, NDIMS> bcast1,
|
||||
bool* error) {
|
||||
typename Functor::func func(error);
|
||||
auto lhs = in0.broadcast(bcast0);
|
||||
auto rhs = in1.broadcast(bcast1);
|
||||
Assign(dev, out, lhs.binaryExpr(rhs, func));
|
||||
}
|
||||
};
|
||||
|
||||
// Partial specialization of UnaryFunctor<Device=CPUDevice, Functor>.
|
||||
template <typename Functor>
|
||||
struct UnaryFunctor<CPUDevice, Functor> {
|
||||
void operator()(const CPUDevice& d, typename Functor::tout_type out,
|
||||
typename Functor::tin_type in) {
|
||||
Assign(d, out, in.unaryExpr(typename Functor::func()));
|
||||
}
|
||||
};
|
||||
|
||||
// Partial specialization of ApproximateEqual<Device=CPUDevice, T>.
|
||||
template <typename T>
|
||||
struct ApproximateEqual<CPUDevice, T> {
|
||||
void operator()(const CPUDevice& d, typename TTypes<T>::ConstFlat x,
|
||||
typename TTypes<T>::ConstFlat y, T tolerance,
|
||||
typename TTypes<bool>::Flat z) {
|
||||
auto diff = x - y;
|
||||
z.device(d) = diff.abs() <= tolerance;
|
||||
}
|
||||
};
|
||||
|
||||
} // end namespace functor
|
||||
|
||||
#define REGISTER(OP, D, N, F, T) \
|
||||
REGISTER_KERNEL_BUILDER(Name(N).Device(DEVICE_##D).TypeConstraint<T>("T"), \
|
||||
OP<D##Device, F<T>>);
|
||||
|
||||
#define REGISTER_VARIANT(OP, D, N, ENUM) \
|
||||
REGISTER_KERNEL_BUILDER( \
|
||||
Name(N).Device(DEVICE_##D).TypeConstraint<Variant>("T"), \
|
||||
OP<D##Device, ENUM>);
|
||||
|
||||
// Macros to register kernels for multiple types (T0, T1, etc.) on
|
||||
// device type "D" (CPU or GPU) for operation "N" (e.g., sqrt) using
|
||||
// the functor "F" (e.g., functor::sqrt).
|
||||
|
||||
#if defined(__ANDROID_TYPES_SLIM__)
|
||||
// Note that __ANDROID_TYPES_SLIM__ is also checked in the cwise_ops*.cc files.
|
||||
// Normally Android TensorFlow is built with a reduced number of types (float).
|
||||
// Override on the command-line using "--copt=-D__ANDROID_TYPES_FULL__"
|
||||
// to generate a library with full type support with a consequent increase in
|
||||
// code size.
|
||||
#define REGISTER2(OP, D, N, F, T0, T1) REGISTER(OP, D, N, F, T0)
|
||||
#define REGISTER3(OP, D, N, F, T0, T1, T2) REGISTER(OP, D, N, F, T0)
|
||||
#define REGISTER4(OP, D, N, F, T0, T1, T2, T3) REGISTER(OP, D, N, F, T0)
|
||||
#define REGISTER5(OP, D, N, F, T0, T1, T2, T3, T4) REGISTER(OP, D, N, F, T0)
|
||||
#define REGISTER6(OP, D, N, F, T0, T1, T2, T3, T4, T5) REGISTER(OP, D, N, F, T0)
|
||||
#define REGISTER7(OP, D, N, F, T0, T1, T2, T3, T4, T5, T6) \
|
||||
REGISTER(OP, D, N, F, T0)
|
||||
#define REGISTER8(OP, D, N, F, T0, T1, T2, T3, T4, T5, T6, T7) \
|
||||
REGISTER(OP, D, N, F, T0)
|
||||
#define REGISTER9(OP, D, N, F, T0, T1, T2, T3, T4, T5, T6, T7, T8) \
|
||||
REGISTER(OP, D, N, F, T0)
|
||||
#else // !defined(__ANDROID_TYPES_SLIM__)
|
||||
#define REGISTER2(OP, D, N, F, T0, T1) \
|
||||
REGISTER(OP, D, N, F, T0) \
|
||||
REGISTER(OP, D, N, F, T1)
|
||||
#define REGISTER3(OP, D, N, F, T0, T1, T2) \
|
||||
REGISTER2(OP, D, N, F, T0, T1) \
|
||||
REGISTER(OP, D, N, F, T2)
|
||||
#define REGISTER4(OP, D, N, F, T0, T1, T2, T3) \
|
||||
REGISTER2(OP, D, N, F, T0, T1) \
|
||||
REGISTER2(OP, D, N, F, T2, T3)
|
||||
#define REGISTER5(OP, D, N, F, T0, T1, T2, T3, T4) \
|
||||
REGISTER3(OP, D, N, F, T0, T1, T2) \
|
||||
REGISTER2(OP, D, N, F, T3, T4)
|
||||
#define REGISTER6(OP, D, N, F, T0, T1, T2, T3, T4, T5) \
|
||||
REGISTER3(OP, D, N, F, T0, T1, T2) \
|
||||
REGISTER3(OP, D, N, F, T3, T4, T5)
|
||||
#define REGISTER7(OP, D, N, F, T0, T1, T2, T3, T4, T5, T6) \
|
||||
REGISTER4(OP, D, N, F, T0, T1, T2, T3) \
|
||||
REGISTER3(OP, D, N, F, T4, T5, T6)
|
||||
#define REGISTER8(OP, D, N, F, T0, T1, T2, T3, T4, T5, T6, T7) \
|
||||
REGISTER4(OP, D, N, F, T0, T1, T2, T3) \
|
||||
REGISTER4(OP, D, N, F, T4, T5, T6, T7)
|
||||
#define REGISTER9(OP, D, N, F, T0, T1, T2, T3, T4, T5, T6, T7, T8) \
|
||||
REGISTER5(OP, D, N, F, T0, T1, T2, T3, T4) \
|
||||
REGISTER4(OP, D, N, F, T5, T6, T7, T8)
|
||||
|
||||
// Instead of adding REGISTER10, etc., shard the .cc files - see
|
||||
// cwise_op_equal_to_*.cc for an example.
|
||||
|
||||
#endif // defined(__ANDROID_TYPES_SLIM__)
|
||||
|
||||
} // end namespace tensorflow
|
||||
|
||||
#endif // TENSORFLOW_CORE_KERNELS_CWISE_OPS_COMMON_H_
|
||||
@@ -0,0 +1,211 @@
|
||||
/* Copyright 2015 The TensorFlow Authors. All Rights Reserved.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
==============================================================================*/
|
||||
|
||||
#ifndef TENSORFLOW_CORE_KERNELS_CWISE_OPS_GRADIENTS_H_
|
||||
#define TENSORFLOW_CORE_KERNELS_CWISE_OPS_GRADIENTS_H_
|
||||
|
||||
#define EIGEN_USE_THREADS
|
||||
#include "third_party/eigen3/unsupported/Eigen/CXX11/Tensor"
|
||||
#include "tensorflow/core/kernels/cwise_ops.h"
|
||||
|
||||
namespace Eigen {
|
||||
namespace internal {
|
||||
|
||||
// Gradient for the tanh function
|
||||
template <typename T>
|
||||
struct scalar_tanh_gradient_op {
|
||||
EIGEN_EMPTY_STRUCT_CTOR(scalar_tanh_gradient_op)
|
||||
EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const T
|
||||
operator()(const T& output, const T& output_gradient) const {
|
||||
return output_gradient * (T(1) - output * output);
|
||||
}
|
||||
template <typename Packet>
|
||||
EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const Packet
|
||||
packetOp(const Packet& output, const Packet& output_gradient) const {
|
||||
return pmul(output_gradient,
|
||||
psub(pset1<Packet>(T(1)), pmul(output, output)));
|
||||
}
|
||||
};
|
||||
template <typename T>
|
||||
struct functor_traits<scalar_tanh_gradient_op<T>> {
|
||||
enum {
|
||||
Cost = NumTraits<T>::AddCost + 2 * NumTraits<T>::MulCost,
|
||||
PacketAccess = packet_traits<T>::HasSub && packet_traits<T>::HasMul,
|
||||
};
|
||||
};
|
||||
|
||||
// Gradient for the sigmoid function
|
||||
template <typename T>
|
||||
struct scalar_sigmoid_gradient_op {
|
||||
EIGEN_EMPTY_STRUCT_CTOR(scalar_sigmoid_gradient_op)
|
||||
EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const T
|
||||
operator()(const T& output, const T& output_gradient) const {
|
||||
return output_gradient * output * (T(1) - output);
|
||||
}
|
||||
template <typename Packet>
|
||||
EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const Packet
|
||||
packetOp(const Packet& output, const Packet& output_gradient) const {
|
||||
return pmul(output_gradient,
|
||||
pmul(output, psub(pset1<Packet>(T(1)), output)));
|
||||
}
|
||||
};
|
||||
template <typename T>
|
||||
struct functor_traits<scalar_sigmoid_gradient_op<T>> {
|
||||
enum {
|
||||
Cost = NumTraits<T>::AddCost + 2 * NumTraits<T>::MulCost,
|
||||
PacketAccess = packet_traits<T>::HasSub && packet_traits<T>::HasMul,
|
||||
};
|
||||
};
|
||||
|
||||
// Gradient for the inverse function
|
||||
template <typename T>
|
||||
struct scalar_inverse_gradient_op {
|
||||
EIGEN_EMPTY_STRUCT_CTOR(scalar_inverse_gradient_op)
|
||||
EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const T
|
||||
operator()(const T& output, const T& output_gradient) const {
|
||||
const T out_conj = numext::conj(output);
|
||||
return -output_gradient * out_conj * out_conj;
|
||||
}
|
||||
template <typename Packet>
|
||||
EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const Packet
|
||||
packetOp(const Packet& output, const Packet& output_gradient) const {
|
||||
const Packet out_conj = pconj(output);
|
||||
return pnegate(pmul(output_gradient, pmul(out_conj, out_conj)));
|
||||
}
|
||||
};
|
||||
template <typename T>
|
||||
struct functor_traits<scalar_inverse_gradient_op<T>> {
|
||||
enum {
|
||||
Cost = NumTraits<T>::AddCost + 2 * NumTraits<T>::MulCost,
|
||||
PacketAccess = packet_traits<T>::HasMul,
|
||||
};
|
||||
};
|
||||
|
||||
// Gradient for the sqrt function
|
||||
template <typename T>
|
||||
struct scalar_sqrt_gradient_op {
|
||||
EIGEN_EMPTY_STRUCT_CTOR(scalar_sqrt_gradient_op)
|
||||
EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const T
|
||||
operator()(const T& output, const T& output_gradient) const {
|
||||
const T out_conj = numext::conj(output);
|
||||
return static_cast<T>(0.5) * output_gradient / out_conj;
|
||||
}
|
||||
template <typename Packet>
|
||||
EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const Packet
|
||||
packetOp(const Packet& output, const Packet& output_gradient) const {
|
||||
const Packet const_half = pset1<Packet>(static_cast<T>(0.5));
|
||||
const Packet out_conj = pconj(output);
|
||||
return pdiv(pmul(const_half, output_gradient), out_conj);
|
||||
}
|
||||
};
|
||||
template <typename T>
|
||||
struct functor_traits<scalar_sqrt_gradient_op<T>> {
|
||||
enum {
|
||||
PacketAccess = packet_traits<T>::HasMul & packet_traits<T>::HasDiv,
|
||||
Cost = NumTraits<T>::MulCost + scalar_div_cost<T, PacketAccess>::value,
|
||||
};
|
||||
};
|
||||
|
||||
// Gradient for the rsqrt function
|
||||
template <typename T>
|
||||
struct scalar_rsqrt_gradient_op {
|
||||
EIGEN_EMPTY_STRUCT_CTOR(scalar_rsqrt_gradient_op)
|
||||
EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const T
|
||||
operator()(const T& output, const T& output_gradient) const {
|
||||
const T out_conj = numext::conj(output);
|
||||
return static_cast<T>(-0.5) * (output_gradient * out_conj) *
|
||||
(out_conj * out_conj);
|
||||
}
|
||||
template <typename Packet>
|
||||
EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const Packet
|
||||
packetOp(const Packet& output, const Packet& output_gradient) const {
|
||||
const Packet const_half = pset1<Packet>(static_cast<T>(-0.5));
|
||||
const Packet out_conj = pconj(output);
|
||||
return pmul(const_half, pmul(pmul(output_gradient, out_conj),
|
||||
pmul(out_conj, out_conj)));
|
||||
}
|
||||
};
|
||||
template <typename T>
|
||||
struct functor_traits<scalar_rsqrt_gradient_op<T>> {
|
||||
enum {
|
||||
Cost = 4 * NumTraits<T>::MulCost,
|
||||
PacketAccess = packet_traits<T>::HasMul,
|
||||
};
|
||||
};
|
||||
|
||||
} // end namespace internal
|
||||
} // end namespace Eigen
|
||||
|
||||
namespace tensorflow {
|
||||
|
||||
namespace functor {
|
||||
|
||||
template <typename Device, typename Functor>
|
||||
struct SimpleBinaryFunctor {
|
||||
void operator()(const Device& d, typename Functor::tout_type out,
|
||||
typename Functor::tin_type in0,
|
||||
typename Functor::tin_type in1);
|
||||
};
|
||||
|
||||
// Partial specialization of BinaryFunctor for CPU devices
|
||||
typedef Eigen::ThreadPoolDevice CPUDevice;
|
||||
|
||||
template <typename Functor>
|
||||
struct SimpleBinaryFunctor<CPUDevice, Functor> {
|
||||
void operator()(const CPUDevice& d, typename Functor::tout_type out,
|
||||
typename Functor::tin_type in0,
|
||||
typename Functor::tin_type in1) {
|
||||
out.device(d) = in0.binaryExpr(in1, typename Functor::func());
|
||||
}
|
||||
};
|
||||
|
||||
#ifdef TENSORFLOW_USE_SYCL
|
||||
// Partial specialization of BinaryFunctor for SYCL devices
|
||||
typedef Eigen::SyclDevice SYCLDevice;
|
||||
template <typename Functor>
|
||||
struct SimpleBinaryFunctor<SYCLDevice, Functor> {
|
||||
void operator()(const SYCLDevice& d, typename Functor::tout_type out,
|
||||
typename Functor::tin_type in0,
|
||||
typename Functor::tin_type in1) {
|
||||
out.device(d) = in0.binaryExpr(in1, typename Functor::func());
|
||||
}
|
||||
};
|
||||
|
||||
#endif // TENSORFLOW_USE_SYCL
|
||||
|
||||
template <typename T>
|
||||
struct tanh_grad : base<T, Eigen::internal::scalar_tanh_gradient_op<T>> {};
|
||||
|
||||
template <typename T>
|
||||
struct sigmoid_grad : base<T, Eigen::internal::scalar_sigmoid_gradient_op<T>> {
|
||||
};
|
||||
|
||||
template <typename T>
|
||||
struct inverse_grad : base<T, Eigen::internal::scalar_inverse_gradient_op<T>> {
|
||||
};
|
||||
|
||||
template <typename T>
|
||||
struct sqrt_grad : base<T, Eigen::internal::scalar_sqrt_gradient_op<T>> {};
|
||||
|
||||
template <typename T>
|
||||
struct rsqrt_grad : base<T, Eigen::internal::scalar_rsqrt_gradient_op<T>> {};
|
||||
|
||||
template <typename T>
|
||||
struct igamma_grad_a : base<T, Eigen::internal::scalar_igamma_der_a_op<T>> {};
|
||||
|
||||
} // end namespace functor
|
||||
|
||||
} // end namespace tensorflow
|
||||
#endif // TENSORFLOW_CORE_KERNELS_CWISE_OPS_GRADIENTS_H_
|
||||
@@ -0,0 +1,163 @@
|
||||
/* Copyright 2016 The TensorFlow Authors. All Rights Reserved.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
==============================================================================*/
|
||||
|
||||
#if !TENSORFLOW_USE_SYCL
|
||||
#error This file must only be included when building TensorFlow with SYCL support
|
||||
#endif
|
||||
|
||||
#ifndef TENSORFLOW_CORE_KERNELS_CWISE_OPS_SYCL_COMMON_H_
|
||||
#define TENSORFLOW_CORE_KERNELS_CWISE_OPS_SYCL_COMMON_H_
|
||||
|
||||
#include "third_party/eigen3/unsupported/Eigen/CXX11/Tensor"
|
||||
|
||||
#include "tensorflow/core/framework/register_types.h"
|
||||
#include "tensorflow/core/kernels/cwise_ops.h"
|
||||
#include "tensorflow/core/platform/types.h"
|
||||
|
||||
namespace tensorflow {
|
||||
namespace functor {
|
||||
|
||||
typedef Eigen::SyclDevice SYCLDevice;
|
||||
|
||||
template <typename OUT, typename RHS>
|
||||
void Assign(const SYCLDevice& d, OUT out, RHS rhs) {
|
||||
out.device(d) = rhs;
|
||||
}
|
||||
|
||||
// Partial specialization of UnaryFunctor<Device=SYCLDevice, Functor>.
|
||||
template <typename Functor>
|
||||
struct UnaryFunctor<SYCLDevice, Functor> {
|
||||
void operator()(const SYCLDevice& d, typename Functor::tout_type out,
|
||||
typename Functor::tin_type in) {
|
||||
To32Bit(out).device(d) = To32Bit(in).unaryExpr(typename Functor::func());
|
||||
}
|
||||
};
|
||||
|
||||
// Partial specialization of BinaryFunctor<Device=SYCLDevice, Functor>.
|
||||
template <typename Functor, int NDIMS, bool has_errors>
|
||||
struct BinaryFunctor<SYCLDevice, Functor, NDIMS, has_errors> {
|
||||
void operator()(const SYCLDevice& d, typename Functor::tout_type out,
|
||||
typename Functor::tin_type in0,
|
||||
typename Functor::tin_type in1, bool* error) {
|
||||
To32Bit(out).device(d) =
|
||||
To32Bit(in0).binaryExpr(To32Bit(in1), typename Functor::func());
|
||||
}
|
||||
|
||||
void Left(const SYCLDevice& d, typename Functor::tout_type out,
|
||||
typename Functor::tscalar_type scalar,
|
||||
typename Functor::tin_type in, bool* error) {
|
||||
typedef typename Functor::func Binary;
|
||||
constexpr int NumDims = Functor::tin_type::NumDimensions;
|
||||
static_assert(NumDims == 1, "Unexpected size");
|
||||
Eigen::Sizes<1> scalar_dim;
|
||||
out.device(d) = scalar.reshape(scalar_dim)
|
||||
.broadcast(in.dimensions())
|
||||
.binaryExpr(in, Binary());
|
||||
}
|
||||
|
||||
void Right(const SYCLDevice& d, typename Functor::tout_type out,
|
||||
typename Functor::tin_type in,
|
||||
typename Functor::tscalar_type scalar, bool* error) {
|
||||
typedef typename Functor::func Binary;
|
||||
constexpr int NumDims = Functor::tin_type::NumDimensions;
|
||||
static_assert(NumDims == 1, "Unexpected size");
|
||||
Eigen::Sizes<1> scalar_dim;
|
||||
out.device(d) = in.binaryExpr(
|
||||
scalar.reshape(scalar_dim).broadcast(in.dimensions()), Binary());
|
||||
}
|
||||
|
||||
void BCast(const SYCLDevice& d,
|
||||
typename TTypes<typename Functor::out_type, NDIMS>::Tensor out,
|
||||
typename TTypes<typename Functor::in_type, NDIMS>::ConstTensor in0,
|
||||
typename Eigen::array<Eigen::DenseIndex, NDIMS> bcast0,
|
||||
typename TTypes<typename Functor::in_type, NDIMS>::ConstTensor in1,
|
||||
typename Eigen::array<Eigen::DenseIndex, NDIMS> bcast1,
|
||||
bool* error) {
|
||||
typedef typename Functor::in_type T;
|
||||
typename Functor::func func;
|
||||
if ((NDIMS == 2) && Functor::use_bcast_optimization &&
|
||||
use_bcast_optimization<T>::value) {
|
||||
const bool bcast0_all_one = AllOne<NDIMS>(bcast0);
|
||||
const bool bcast1_all_one = AllOne<NDIMS>(bcast1);
|
||||
if (bcast0_all_one && !bcast1_all_one) {
|
||||
To32Bit(out).device(d) =
|
||||
To32Bit(in0).binaryExpr(To32Bit(in1).broadcast(bcast1), func);
|
||||
return;
|
||||
}
|
||||
if (!bcast0_all_one && bcast1_all_one) {
|
||||
To32Bit(out).device(d) =
|
||||
To32Bit(in0).broadcast(bcast0).binaryExpr(To32Bit(in1), func);
|
||||
return;
|
||||
}
|
||||
}
|
||||
To32Bit(out).device(d) = To32Bit(in0).broadcast(bcast0).binaryExpr(
|
||||
To32Bit(in1).broadcast(bcast1), func);
|
||||
}
|
||||
};
|
||||
|
||||
// Macros to explicitly instantiate kernels on GPU for multiple types
|
||||
// (T0, T1, etc.) for UnaryFunctor (e.g., functor::sqrt).
|
||||
#define DEFINE_UNARY1(F, T) template struct UnaryFunctor<SYCLDevice, F<T> >
|
||||
#define DEFINE_UNARY2(F, T0, T1) \
|
||||
DEFINE_UNARY1(F, T0); \
|
||||
DEFINE_UNARY1(F, T1)
|
||||
#define DEFINE_UNARY3(F, T0, T1, T2) \
|
||||
DEFINE_UNARY2(F, T0, T1); \
|
||||
DEFINE_UNARY1(F, T2)
|
||||
#define DEFINE_UNARY4(F, T0, T1, T2, T3) \
|
||||
DEFINE_UNARY2(F, T0, T1); \
|
||||
DEFINE_UNARY2(F, T2, T3)
|
||||
#define DEFINE_UNARY5(F, T0, T1, T2, T3, T4) \
|
||||
DEFINE_UNARY2(F, T0, T1); \
|
||||
DEFINE_UNARY3(F, T2, T3, T4)
|
||||
|
||||
// Macros to explicitly instantiate kernels on GPU for multiple types
|
||||
// (T0, T1, etc.) for BinaryFunctor.
|
||||
#define DEFINE_BINARY1(F, T) \
|
||||
template struct BinaryFunctor<SYCLDevice, F<T>, 1>; \
|
||||
template struct BinaryFunctor<SYCLDevice, F<T>, 2>; \
|
||||
template struct BinaryFunctor<SYCLDevice, F<T>, 3>
|
||||
#define DEFINE_BINARY2(F, T0, T1) \
|
||||
DEFINE_BINARY1(F, T0); \
|
||||
DEFINE_BINARY1(F, T1)
|
||||
#define DEFINE_BINARY3(F, T0, T1, T2) \
|
||||
DEFINE_BINARY2(F, T0, T1); \
|
||||
DEFINE_BINARY1(F, T2)
|
||||
#define DEFINE_BINARY4(F, T0, T1, T2, T3) \
|
||||
DEFINE_BINARY2(F, T0, T1); \
|
||||
DEFINE_BINARY2(F, T2, T3)
|
||||
#define DEFINE_BINARY5(F, T0, T1, T2, T3, T4) \
|
||||
DEFINE_BINARY2(F, T0, T1); \
|
||||
DEFINE_BINARY3(F, T2, T3, T4)
|
||||
#define DEFINE_BINARY6(F, T0, T1, T2, T3, T4, T5) \
|
||||
DEFINE_BINARY3(F, T0, T1, T2); \
|
||||
DEFINE_BINARY3(F, T3, T4, T5)
|
||||
#define DEFINE_BINARY7(F, T0, T1, T2, T3, T4, T5, T6) \
|
||||
DEFINE_BINARY3(F, T0, T1, T2); \
|
||||
DEFINE_BINARY4(F, T3, T4, T5, T6)
|
||||
#define DEFINE_BINARY8(F, T0, T1, T2, T3, T4, T5, T6, T7) \
|
||||
DEFINE_BINARY4(F, T0, T1, T2, T3); \
|
||||
DEFINE_BINARY4(F, T4, T5, T6, T7)
|
||||
#define DEFINE_BINARY9(F, T0, T1, T2, T3, T4, T5, T6, T7, T8) \
|
||||
DEFINE_BINARY4(F, T0, T1, T2, T3); \
|
||||
DEFINE_BINARY5(F, T4, T5, T6, T7, T8)
|
||||
#define DEFINE_BINARY10(F, T0, T1, T2, T3, T4, T5, T6, T7, T8, T9) \
|
||||
DEFINE_BINARY5(F, T0, T1, T2, T3, T4); \
|
||||
DEFINE_BINARY5(F, T5, T6, T7, T8, T9)
|
||||
|
||||
} // end namespace functor
|
||||
} // end namespace tensorflow
|
||||
|
||||
#endif // TENSORFLOW_CORE_KERNELS_CWISE_OPS_SYCL_COMMON_H_
|
||||
@@ -0,0 +1,147 @@
|
||||
/* Copyright 2017 The TensorFlow Authors. All Rights Reserved.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
==============================================================================*/
|
||||
#ifndef TENSORFLOW_CORE_KERNELS_DATA_CAPTURED_FUNCTION_H_
|
||||
#define TENSORFLOW_CORE_KERNELS_DATA_CAPTURED_FUNCTION_H_
|
||||
|
||||
#include <memory>
|
||||
#include <vector>
|
||||
|
||||
#include "tensorflow/core/framework/function.h"
|
||||
#include "tensorflow/core/framework/tensor.h"
|
||||
#include "tensorflow/core/kernels/data/dataset.h"
|
||||
#include "tensorflow/core/lib/core/status.h"
|
||||
#include "tensorflow/core/lib/gtl/array_slice.h"
|
||||
#include "tensorflow/core/lib/random/random.h"
|
||||
#include "tensorflow/core/platform/macros.h"
|
||||
|
||||
namespace tensorflow {
|
||||
|
||||
class Device;
|
||||
class OpKernelContext;
|
||||
class ResourceMgr;
|
||||
|
||||
namespace data {
|
||||
|
||||
// A `CapturedFunction` encapsulates a TensorFlow function and all of
|
||||
// the runtime support required to execute it.
|
||||
//
|
||||
// The `Dataset`-related classes use `CapturedFunction` to execute
|
||||
// TensorFlow functions outside a the normal `OpKernel::Compute()`
|
||||
// context.
|
||||
class CapturedFunction {
|
||||
public:
|
||||
// Creates a new instance using a list of named attributes, fetching captured
|
||||
// inputs from a context argument.
|
||||
static Status Create(const NameAttrList& func, OpKernelContext* ctx,
|
||||
const string& argument,
|
||||
std::unique_ptr<CapturedFunction>* out_function);
|
||||
|
||||
// Creates a new instance using a list of named attributes, fetching captured
|
||||
// inputs from a context argument.
|
||||
//
|
||||
// If `use_inter_op_parallelism` is false, the runtime may use an executor
|
||||
// that is optimized for small functions.
|
||||
static Status Create(const NameAttrList& func, OpKernelContext* ctx,
|
||||
const string& argument, bool use_inter_op_parallelism,
|
||||
std::unique_ptr<CapturedFunction>* out_function);
|
||||
|
||||
~CapturedFunction();
|
||||
|
||||
// Runs the "Captured function" using the given FLR and caches the lib and
|
||||
// handle generated during instantiation. If Run is called with a different
|
||||
// lib afterwards, generates an error. This method takes ownership of the
|
||||
// tensors in `args`, in order to be able to deallocate them as early as
|
||||
// possible. Use `RunWithBorrowedArgs()` if the caller needs to retain
|
||||
// ownership of the `args`.
|
||||
Status Run(IteratorContext* ctx, std::vector<Tensor>&& args,
|
||||
std::vector<Tensor>* rets);
|
||||
|
||||
// Synchronously runs the captured function on the given `args`, and stores
|
||||
// the results in `*rets`. Prefer to use `Run()` or `RunAsync()` when
|
||||
// possible.
|
||||
Status RunWithBorrowedArgs(IteratorContext* ctx,
|
||||
const std::vector<Tensor>& args,
|
||||
std::vector<Tensor>* rets);
|
||||
|
||||
// Explicitly instantiate this function for use in the given
|
||||
// context. This method, and the context-less overload
|
||||
// `RunInstantiated()` below can be useful for calling a captured
|
||||
// function in cases where an `IteratorContext*` is not available
|
||||
// (such as a destructor).
|
||||
Status Instantiate(IteratorContext* ctx);
|
||||
|
||||
// Synchronously runs the captured function on the given `args`, and stores
|
||||
// the results in `*rets`. Prefer to use `Run()` or `RunAsync()` when
|
||||
// possible.
|
||||
//
|
||||
// REQUIRES: `this->Instantiate()` must have been called before this method.
|
||||
Status RunInstantiated(const std::vector<Tensor>& args,
|
||||
std::vector<Tensor>* rets);
|
||||
|
||||
// Asynchronously runs the captured function on the given `args`, stores
|
||||
// the results in `*rets`, and calls the given `done` callback when the
|
||||
// function returns. This method takes ownership of the tensors in `args`,
|
||||
// in order to be able to deallocate them as early as possible.
|
||||
void RunAsync(IteratorContext* ctx, std::vector<Tensor>&& args,
|
||||
std::vector<Tensor>* rets,
|
||||
FunctionLibraryRuntime::DoneCallback done,
|
||||
const string& prefix);
|
||||
|
||||
// Returns the named list of function arguments.
|
||||
const NameAttrList& func() { return func_; }
|
||||
|
||||
// Returns that additional captured inputs that will be passed to the function
|
||||
// when `Run*()` is called.
|
||||
const std::vector<Tensor>& captured_inputs() { return captured_inputs_; }
|
||||
|
||||
// Returns a step ID for use when running a `CapturedFunction`.
|
||||
static int64 generate_step_id() {
|
||||
// Choose a step ID that is guaranteed not to clash with any
|
||||
// Session-generated step ID. DirectSession only generates
|
||||
// non-negative step IDs (contiguous, starting from 0), and
|
||||
// MasterSession generates 56-bit random step IDs whose MSB is
|
||||
// always 0, so a negative random step ID should suffice.
|
||||
return -std::abs(static_cast<int64>(random::New64()));
|
||||
}
|
||||
|
||||
private:
|
||||
CapturedFunction(const NameAttrList& func,
|
||||
std::vector<Tensor> captured_inputs,
|
||||
bool use_inter_op_parallelism);
|
||||
|
||||
Status GetHandle(IteratorContext* ctx,
|
||||
FunctionLibraryRuntime::Handle* out_handle);
|
||||
|
||||
mutex mu_;
|
||||
const NameAttrList func_;
|
||||
FunctionLibraryRuntime* lib_ GUARDED_BY(mu_);
|
||||
FunctionLibraryRuntime::Handle f_handle_ GUARDED_BY(mu_);
|
||||
const std::vector<Tensor> captured_inputs_;
|
||||
DataTypeSlice ret_types_;
|
||||
std::function<void(std::function<void()>)> captured_runner_ = nullptr;
|
||||
const bool use_inter_op_parallelism_;
|
||||
|
||||
TF_DISALLOW_COPY_AND_ASSIGN(CapturedFunction);
|
||||
};
|
||||
|
||||
} // namespace data
|
||||
|
||||
// TODO(b/114112161): Remove these aliases when all users have moved over to the
|
||||
// `tensorflow::data` namespace.
|
||||
using data::CapturedFunction;
|
||||
|
||||
} // namespace tensorflow
|
||||
|
||||
#endif // TENSORFLOW_CORE_KERNELS_DATA_CAPTURED_FUNCTION_H_
|
||||
@@ -0,0 +1,20 @@
|
||||
/* Copyright 2017 The TensorFlow Authors. All Rights Reserved.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
==============================================================================*/
|
||||
#ifndef TENSORFLOW_CORE_KERNELS_DATA_DATASET_H_
|
||||
#define TENSORFLOW_CORE_KERNELS_DATA_DATASET_H_
|
||||
|
||||
#include "tensorflow/core/framework/dataset.h"
|
||||
|
||||
#endif // TENSORFLOW_CORE_KERNELS_DATA_DATASET_H_
|
||||
@@ -0,0 +1,43 @@
|
||||
/* Copyright 2017 The TensorFlow Authors. All Rights Reserved.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
==============================================================================*/
|
||||
#ifndef TENSORFLOW_CORE_KERNELS_DATA_DATASET_UTILS_H_
|
||||
#define TENSORFLOW_CORE_KERNELS_DATA_DATASET_UTILS_H_
|
||||
|
||||
#include "tensorflow/core/framework/tensor.h"
|
||||
#include "tensorflow/core/kernels/data/captured_function.h"
|
||||
#include "tensorflow/core/kernels/data/dataset.h"
|
||||
|
||||
namespace tensorflow {
|
||||
namespace data {
|
||||
|
||||
Status MakeIteratorFromInputElement(
|
||||
IteratorContext* ctx, const std::vector<Tensor>& input_element,
|
||||
int64 thread_index, CapturedFunction* captured_func, StringPiece prefix,
|
||||
std::unique_ptr<IteratorBase>* out_iterator);
|
||||
|
||||
// Returns Status::OK() if `expected` and `received` types match,
|
||||
// errors::InvalidArgument otherwise.
|
||||
Status VerifyTypesMatch(const DataTypeVector& expected,
|
||||
const DataTypeVector& received);
|
||||
|
||||
// Returns Status::OK() if `expected` and `received` shapes are compatible,
|
||||
// errors::InvalidArgument otherwise.
|
||||
Status VerifyShapesCompatible(const std::vector<PartialTensorShape>& expected,
|
||||
const std::vector<PartialTensorShape>& received);
|
||||
|
||||
} // namespace data
|
||||
} // namespace tensorflow
|
||||
|
||||
#endif // TENSORFLOW_CORE_KERNELS_DATA_DATASET_UTILS_H_
|
||||
@@ -0,0 +1,119 @@
|
||||
/* Copyright 2017 The TensorFlow Authors. All Rights Reserved.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
==============================================================================*/
|
||||
#ifndef TENSORFLOW_CORE_KERNELS_DATA_EXPERIMENTAL_INDEXED_DATASET_H_
|
||||
#define TENSORFLOW_CORE_KERNELS_DATA_EXPERIMENTAL_INDEXED_DATASET_H_
|
||||
|
||||
#include "tensorflow/core/framework/dataset.h"
|
||||
#include "tensorflow/core/framework/op_kernel.h"
|
||||
|
||||
namespace tensorflow {
|
||||
namespace data {
|
||||
|
||||
// TODO(saeta): Urgh, this is ugly.
|
||||
class MaterializedIndexedDataset {
|
||||
public:
|
||||
virtual ~MaterializedIndexedDataset() = default;
|
||||
|
||||
// Retrieve the element at a given index. The output tensors are stored in
|
||||
// out_tensors.
|
||||
//
|
||||
// If `index` is greater than `Size()`, tensorflow::errors::OutOfRangeError is
|
||||
// returned.
|
||||
//
|
||||
// Get is thread-safe.
|
||||
virtual Status Get(IteratorContext&& ctx, uint64 index,
|
||||
std::vector<Tensor>* out_tensors) const = 0;
|
||||
|
||||
// Size determines the number of elements in this IndexedDataset.
|
||||
//
|
||||
// Size is thread-safe.
|
||||
virtual Status Size(uint64* size) const = 0;
|
||||
|
||||
// Returns a vector of DataType values, representing the respective
|
||||
// element types of each tuple component in the outputs of this dataset.
|
||||
virtual const DataTypeVector& output_dtypes() const = 0;
|
||||
|
||||
// Returns a vector of tensor shapes, representing the respective
|
||||
// (and possibly partially defined) shapes of each tuple component
|
||||
// in the outputs of this dataset.
|
||||
virtual const std::vector<PartialTensorShape>& output_shapes() const = 0;
|
||||
};
|
||||
|
||||
// IndexedDataset represents a dataset that supports random access in addition
|
||||
// to iterator-based sequential access.
|
||||
//
|
||||
// Note: IndexedDatasets are HIGHLY experimental at this time. Expect
|
||||
// significant (backwards incompatible) changes!
|
||||
class IndexedDataset : public DatasetBase {
|
||||
public:
|
||||
IndexedDataset(DatasetContext&& ctx) : DatasetBase(std::move(ctx)) {}
|
||||
|
||||
// Materialize (if necessary) the dataset, and return a pointer.
|
||||
// TODO(saeta): Add in `IteratorContext* ctx` when materializing.
|
||||
virtual Status MaterializeDataset(
|
||||
std::shared_ptr<MaterializedIndexedDataset>* materialized) = 0;
|
||||
};
|
||||
|
||||
// IndexedDatasetOpKernel abstracts away interfacing IndexedDatasets with the
|
||||
// rest of the TensorFlow runtime.
|
||||
//
|
||||
// Most IndexedDataset's will be private members of classes inheriting from this
|
||||
// class.
|
||||
class IndexedDatasetOpKernel : public OpKernel {
|
||||
public:
|
||||
IndexedDatasetOpKernel(OpKernelConstruction* ctx) : OpKernel(ctx) {}
|
||||
void Compute(OpKernelContext* ctx) final;
|
||||
|
||||
protected:
|
||||
// Subclasses should implement this method. It will be called during Compute
|
||||
// execution.
|
||||
virtual void MakeIndexedDataset(OpKernelContext* ctx,
|
||||
IndexedDataset** output) = 0;
|
||||
|
||||
template <typename T>
|
||||
Status ParseScalarArgument(OpKernelContext* ctx,
|
||||
const StringPiece& argument_name, T* output) {
|
||||
const Tensor* argument_t;
|
||||
TF_RETURN_IF_ERROR(ctx->input(argument_name, &argument_t));
|
||||
if (!TensorShapeUtils::IsScalar(argument_t->shape())) {
|
||||
return errors::InvalidArgument(argument_name, " must be a scalar");
|
||||
}
|
||||
*output = argument_t->scalar<T>()();
|
||||
return Status::OK();
|
||||
}
|
||||
};
|
||||
|
||||
// Validates and extracts an `IndexedDataset` object from `tensor`.
|
||||
//
|
||||
// `tensor` must have been written by a call to
|
||||
// `StoreIndexedDatasetInVariantTensor`
|
||||
//
|
||||
// The retrieved pointer isa borrowed reference to the dataset, which is owned
|
||||
// by the tensor. The consumer must either acquire its own reference to the
|
||||
// dataset by calling `(*out_dataset)->Ref()`, or ensure that `tensor` is not
|
||||
// destroyed or mutated while the retrieved pointer is in use.
|
||||
Status GetIndexedDatasetFromVariantTensor(const Tensor& tensor,
|
||||
IndexedDataset** out_dataset);
|
||||
|
||||
// Stores an `IndexedDataset` object in `tensor.`
|
||||
//
|
||||
// The ownership of `dataset` is transferred to `tensor`.
|
||||
Status StoreIndexedDatasetInVariantTensor(IndexedDataset* dataset,
|
||||
Tensor* tensor);
|
||||
|
||||
} // namespace data
|
||||
} // namespace tensorflow
|
||||
|
||||
#endif // TENSORFLOW_CORE_KERNELS_DATA_EXPERIMENTAL_INDEXED_DATASET_H_
|
||||
@@ -0,0 +1,42 @@
|
||||
/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
==============================================================================*/
|
||||
|
||||
#ifndef TENSORFLOW_CORE_KERNELS_DATA_GENERATOR_DATASET_OP_H_
|
||||
#define TENSORFLOW_CORE_KERNELS_DATA_GENERATOR_DATASET_OP_H_
|
||||
|
||||
#include "tensorflow/core/framework/dataset.h"
|
||||
|
||||
namespace tensorflow {
|
||||
namespace data {
|
||||
|
||||
class GeneratorDatasetOp : public DatasetOpKernel {
|
||||
public:
|
||||
explicit GeneratorDatasetOp(OpKernelConstruction* ctx);
|
||||
|
||||
void MakeDataset(OpKernelContext* ctx, DatasetBase** output) override;
|
||||
|
||||
private:
|
||||
class Dataset;
|
||||
|
||||
DataTypeVector output_types_;
|
||||
std::vector<PartialTensorShape> output_shapes_;
|
||||
NameAttrList init_func_;
|
||||
NameAttrList next_func_;
|
||||
NameAttrList finalize_func_;
|
||||
};
|
||||
|
||||
} // namespace data
|
||||
} // namespace tensorflow
|
||||
#endif // TENSORFLOW_CORE_KERNELS_DATA_GENERATOR_DATASET_OP_H_
|
||||
@@ -0,0 +1,149 @@
|
||||
/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
==============================================================================*/
|
||||
|
||||
#ifndef TENSORFLOW_CORE_KERNELS_DATA_ITERATOR_OPS_H_
|
||||
#define TENSORFLOW_CORE_KERNELS_DATA_ITERATOR_OPS_H_
|
||||
|
||||
#include "tensorflow/core/common_runtime/function.h"
|
||||
#include "tensorflow/core/framework/dataset.h"
|
||||
#include "tensorflow/core/framework/op_kernel.h"
|
||||
#include "tensorflow/core/kernels/ops_util.h"
|
||||
|
||||
namespace tensorflow {
|
||||
namespace data {
|
||||
|
||||
class IteratorResource;
|
||||
|
||||
class IteratorHandleOp : public OpKernel {
|
||||
public:
|
||||
explicit IteratorHandleOp(OpKernelConstruction* ctx);
|
||||
|
||||
// The resource is deleted from the resource manager only when it is private
|
||||
// to kernel. Ideally the resource should be deleted when it is no longer held
|
||||
// by anyone, but it would break backward compatibility.
|
||||
~IteratorHandleOp() override;
|
||||
|
||||
void Compute(OpKernelContext* context) override LOCKS_EXCLUDED(mu_);
|
||||
|
||||
private:
|
||||
// During the first Compute(), resource is either created or looked up using
|
||||
// shared_name. In the latter case, the resource found should be verified if
|
||||
// it is compatible with this op's configuration. The verification may fail in
|
||||
// cases such as two graphs asking queues of the same shared name to have
|
||||
// inconsistent capacities.
|
||||
Status VerifyResource(IteratorResource* resource);
|
||||
|
||||
template <typename To, typename From> // use like this: down_cast<T*>(foo);
|
||||
static inline To down_cast(From* f) { // so we only accept pointers
|
||||
static_assert(
|
||||
(std::is_base_of<From, typename std::remove_pointer<To>::type>::value),
|
||||
"target type not derived from source type");
|
||||
|
||||
// We skip the assert and hence the dynamic_cast if RTTI is disabled.
|
||||
#if !defined(__GNUC__) || defined(__GXX_RTTI)
|
||||
// Uses RTTI in dbg and fastbuild. asserts are disabled in opt builds.
|
||||
assert(f == nullptr || dynamic_cast<To>(f) != nullptr);
|
||||
#endif // !defined(__GNUC__) || defined(__GXX_RTTI)
|
||||
return static_cast<To>(f);
|
||||
}
|
||||
|
||||
FunctionLibraryRuntime* CreatePrivateFLR(
|
||||
OpKernelContext* ctx, std::unique_ptr<DeviceMgr>* device_mgr,
|
||||
std::unique_ptr<FunctionLibraryDefinition>* flib_def,
|
||||
std::unique_ptr<ProcessFunctionLibraryRuntime>* pflr);
|
||||
|
||||
mutex mu_;
|
||||
ContainerInfo cinfo_; // Written once under mu_ then constant afterwards.
|
||||
IteratorResource* resource_ GUARDED_BY(mu_) = nullptr;
|
||||
DataTypeVector output_dtypes_;
|
||||
std::vector<PartialTensorShape> output_shapes_;
|
||||
const int graph_def_version_;
|
||||
string name_;
|
||||
};
|
||||
|
||||
// Like IteratorHandleOp, but creates handles which are never shared, and does
|
||||
// not hold a reference to these handles. The latter is important for eager
|
||||
// execution, since OpKernel instances generally live as long as the program
|
||||
// running them.
|
||||
class AnonymousIteratorHandleOp : public OpKernel {
|
||||
public:
|
||||
explicit AnonymousIteratorHandleOp(OpKernelConstruction* context);
|
||||
|
||||
void Compute(OpKernelContext* context) override;
|
||||
|
||||
private:
|
||||
// Coordinates Iterator unique name creation across AnonymousIteratorHandleOp
|
||||
// instances.
|
||||
static mutex static_resource_lookup_mutex_;
|
||||
// current_id_ is just a hint for creating unique names. If it turns out
|
||||
// there's a collision (e.g. because another AnonymousIteratorHandleOp
|
||||
// instance is generating handles) we'll just skip that id.
|
||||
static int64 current_id_ GUARDED_BY(static_resource_lookup_mutex_);
|
||||
DataTypeVector output_dtypes_;
|
||||
std::vector<PartialTensorShape> output_shapes_;
|
||||
const int graph_def_version_;
|
||||
};
|
||||
|
||||
class MakeIteratorOp : public OpKernel {
|
||||
public:
|
||||
explicit MakeIteratorOp(OpKernelConstruction* ctx) : OpKernel(ctx) {}
|
||||
|
||||
void Compute(OpKernelContext* ctx) override;
|
||||
};
|
||||
|
||||
class IteratorGetNextOp : public AsyncOpKernel {
|
||||
public:
|
||||
explicit IteratorGetNextOp(OpKernelConstruction* ctx)
|
||||
: AsyncOpKernel(ctx),
|
||||
background_worker_(ctx->env(),
|
||||
strings::StrCat("iterator_get_next_thread_",
|
||||
SanitizeThreadSuffix(name()))) {}
|
||||
|
||||
void ComputeAsync(OpKernelContext* ctx, DoneCallback done) override;
|
||||
|
||||
private:
|
||||
BackgroundWorker background_worker_;
|
||||
};
|
||||
|
||||
class IteratorGetNextSyncOp : public OpKernel {
|
||||
public:
|
||||
explicit IteratorGetNextSyncOp(OpKernelConstruction* ctx) : OpKernel(ctx) {}
|
||||
|
||||
void Compute(OpKernelContext* ctx) override;
|
||||
};
|
||||
|
||||
class IteratorToStringHandleOp : public OpKernel {
|
||||
public:
|
||||
explicit IteratorToStringHandleOp(OpKernelConstruction* ctx)
|
||||
: OpKernel(ctx) {}
|
||||
|
||||
void Compute(OpKernelContext* ctx) override;
|
||||
};
|
||||
|
||||
class IteratorFromStringHandleOp : public OpKernel {
|
||||
public:
|
||||
explicit IteratorFromStringHandleOp(OpKernelConstruction* ctx);
|
||||
|
||||
void Compute(OpKernelContext* ctx) override;
|
||||
|
||||
private:
|
||||
DataTypeVector output_dtypes_;
|
||||
std::vector<PartialTensorShape> output_shapes_;
|
||||
};
|
||||
|
||||
} // namespace data
|
||||
} // namespace tensorflow
|
||||
|
||||
#endif // TENSORFLOW_CORE_KERNELS_DATA_ITERATOR_OPS_H_
|
||||
@@ -0,0 +1,38 @@
|
||||
/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
==============================================================================*/
|
||||
#ifndef TENSORFLOW_CORE_KERNELS_DATA_OPTIONAL_OPS_H_
|
||||
#define TENSORFLOW_CORE_KERNELS_DATA_OPTIONAL_OPS_H_
|
||||
|
||||
#include <vector>
|
||||
|
||||
#include "tensorflow/core/framework/tensor.h"
|
||||
#include "tensorflow/core/framework/variant_tensor_data.h"
|
||||
|
||||
namespace tensorflow {
|
||||
namespace data {
|
||||
|
||||
// Stores a DT_VARIANT value representing an Optional with the given value
|
||||
// in the `output_index`^th output of the given kernel execution context.
|
||||
Status WriteOptionalWithValueToOutput(OpKernelContext* ctx, int output_index,
|
||||
std::vector<Tensor> value);
|
||||
|
||||
// Stores a DT_VARIANT value representing an Optional with no value
|
||||
// in the `output_index`^th output of the given kernel execution context.
|
||||
Status WriteOptionalNoneToOutput(OpKernelContext* ctx, int output_index);
|
||||
|
||||
} // namespace data
|
||||
} // namespace tensorflow
|
||||
|
||||
#endif // TENSORFLOW_CORE_KERNELS_DATA_OPTIONAL_OPS_H_
|
||||
@@ -0,0 +1,54 @@
|
||||
/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
==============================================================================*/
|
||||
#ifndef TENSORFLOW_CORE_KERNELS_DATA_PARALLEL_MAP_ITERATOR_H_
|
||||
#define TENSORFLOW_CORE_KERNELS_DATA_PARALLEL_MAP_ITERATOR_H_
|
||||
|
||||
#include <memory>
|
||||
|
||||
#include "tensorflow/core/framework/dataset.h"
|
||||
|
||||
namespace tensorflow {
|
||||
namespace data {
|
||||
|
||||
// A function that transforms elements of one dataset into another
|
||||
// asynchronously. The arguments are:
|
||||
// 1. An `IteratorContext*` for the context in which the function should
|
||||
// execute.
|
||||
// 2. A `std::vector<Tensor>` containing the input element.
|
||||
// 3. A `std::vector<Tensor>*` to which the function will write the result.
|
||||
// 4. A `StatusCallback` that should be invoked when the function is complete.
|
||||
using ParallelMapIteratorFunction =
|
||||
std::function<void(IteratorContext*, std::vector<Tensor>,
|
||||
std::vector<Tensor>*, StatusCallback)>;
|
||||
|
||||
// Returns a new iterator that applies `map_func` to the elements of
|
||||
// `input_dataset` using the given degree of parallelism. `init_func` (if
|
||||
// specified) will be executed when the iterator is initialized (see
|
||||
// `IteratorBase::Initialize()`) and enables the user to specify error checking
|
||||
// logic that can fail early.
|
||||
std::unique_ptr<IteratorBase> NewParallelMapIterator(
|
||||
const DatasetBaseIterator::BaseParams& params,
|
||||
const DatasetBase* input_dataset,
|
||||
std::function<Status(IteratorContext*)> init_func,
|
||||
ParallelMapIteratorFunction map_func, int32 num_parallel_calls);
|
||||
std::unique_ptr<IteratorBase> NewParallelMapIterator(
|
||||
const DatasetBaseIterator::BaseParams& params,
|
||||
const DatasetBase* input_dataset, ParallelMapIteratorFunction map_func,
|
||||
int32 num_parallel_calls);
|
||||
|
||||
} // namespace data
|
||||
} // namespace tensorflow
|
||||
|
||||
#endif // TENSORFLOW_CORE_KERNELS_DATA_PARALLEL_MAP_ITERATOR_H_
|
||||
@@ -0,0 +1,73 @@
|
||||
/* Copyright 2017 The TensorFlow Authors. All Rights Reserved.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
==============================================================================*/
|
||||
|
||||
#ifndef TENSORFLOW_CORE_KERNELS_DATA_PREFETCH_AUTOTUNER_H_
|
||||
#define TENSORFLOW_CORE_KERNELS_DATA_PREFETCH_AUTOTUNER_H_
|
||||
|
||||
#include "tensorflow/core/platform/types.h"
|
||||
|
||||
namespace tensorflow {
|
||||
namespace data {
|
||||
|
||||
// PrefetchAutotuner dynamically adjusts the buffer size of a prefetch iterator.
|
||||
//
|
||||
// PrefetchAutotuner attempts to find the minimum buffer size such that there is
|
||||
// always at least 1 element in the prefetch queue every time the downstream
|
||||
// iterator calls GetNext().
|
||||
//
|
||||
// One common failure mode of input pipelines is being throughput bound. No
|
||||
// amount of prefetching can address that performance mode. In order to guard
|
||||
// against this condition, PrefetchAutotuner will only increase the buffer_limit
|
||||
// if the prefetching thread is able to successfully fill the buffer at its
|
||||
// current size.
|
||||
//
|
||||
// Note: in the current implementation, we never decrease the buffer_limit().
|
||||
// This should change in the future!
|
||||
//
|
||||
// PrefetchAutotuner is NOT thread safe.
|
||||
class PrefetchAutotuner {
|
||||
public:
|
||||
static const int64 kAutoTune = -1;
|
||||
|
||||
explicit PrefetchAutotuner(int64 initial_buffer_size);
|
||||
|
||||
int64 buffer_limit() const { return buffer_limit_; }
|
||||
|
||||
void RecordConsumption(size_t current_buffer_size);
|
||||
void RecordEmpty() { RecordConsumption(0); }
|
||||
|
||||
private:
|
||||
// PrefetchAutotuner operates as a state machine.
|
||||
enum class Mode {
|
||||
// Disables the autotuning.
|
||||
kDisabled,
|
||||
|
||||
// We have increased the size of the buffer, and will transition to
|
||||
// kDownswing if we successfully fill the buffer.
|
||||
kUpswing,
|
||||
|
||||
// We have successfully filled a buffer of this size. If we ever block the
|
||||
// downstream iterator, we should increase the buffer size.
|
||||
kDownswing,
|
||||
};
|
||||
|
||||
int64 buffer_limit_;
|
||||
Mode mode_ = Mode::kDisabled;
|
||||
};
|
||||
|
||||
} // namespace data
|
||||
} // namespace tensorflow
|
||||
|
||||
#endif // TENSORFLOW_CORE_KERNELS_DATA_PREFETCH_AUTOTUNER_H_
|
||||
@@ -0,0 +1,41 @@
|
||||
/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
==============================================================================*/
|
||||
|
||||
#ifndef TENSORFLOW_CORE_KERNELS_DATA_PREFETCH_DATASET_OP_H_
|
||||
#define TENSORFLOW_CORE_KERNELS_DATA_PREFETCH_DATASET_OP_H_
|
||||
|
||||
#include "tensorflow/core/kernels/data/dataset.h"
|
||||
#include "tensorflow/core/kernels/data/prefetch_autotuner.h"
|
||||
|
||||
namespace tensorflow {
|
||||
namespace data {
|
||||
|
||||
class PrefetchDatasetOp : public UnaryDatasetOpKernel {
|
||||
public:
|
||||
explicit PrefetchDatasetOp(OpKernelConstruction* ctx)
|
||||
: UnaryDatasetOpKernel(ctx) {}
|
||||
|
||||
protected:
|
||||
void MakeDataset(OpKernelContext* ctx, DatasetBase* input,
|
||||
DatasetBase** output) override;
|
||||
|
||||
private:
|
||||
class Dataset;
|
||||
};
|
||||
|
||||
} // namespace data
|
||||
} // namespace tensorflow
|
||||
|
||||
#endif // TENSORFLOW_CORE_KERNELS_DATA_PREFETCH_DATASET_OP_H_
|
||||
@@ -0,0 +1,62 @@
|
||||
/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
==============================================================================*/
|
||||
|
||||
#ifndef TENSORFLOW_CORE_KERNELS_DATA_SINGLE_THREADED_EXECUTOR_H_
|
||||
#define TENSORFLOW_CORE_KERNELS_DATA_SINGLE_THREADED_EXECUTOR_H_
|
||||
|
||||
#include "tensorflow/core/common_runtime/executor.h"
|
||||
|
||||
namespace tensorflow {
|
||||
namespace data {
|
||||
|
||||
// Creates a new `Executor` for executing `graph` synchronously on the caller
|
||||
// thread.
|
||||
//
|
||||
// NOTE(mrry): The returned executor is optimized to impose low overhead on
|
||||
// graphs that perform a small amount of work (e.g. <15us of work per graph on
|
||||
// present architectures). It eschews concurrency, because issuing work to
|
||||
// multiple threads can dominate the cost of executing small ops synchronously,
|
||||
// and because contention in the executor data structures can reduce throughput
|
||||
// (in terms of ops executed per unit time).
|
||||
//
|
||||
// However, the current implementation has the following limitations:
|
||||
//
|
||||
// 1. Reference-typed tensors are not supported and will not be supported in
|
||||
// future.
|
||||
// 2. Graphs with control flow (containing "Switch" and "Merge" nodes) are not
|
||||
// currently supported. The current plan is to extend support to "functional"
|
||||
// control flow after the TensorFlow APIs transition to building graphs in
|
||||
// that form (e.g. `tf.cond_v2()`).
|
||||
// 3. Partitioned graphs (containing "_Recv" nodes) are not currently supported.
|
||||
// The present implementation executes kernels one at a time in topological
|
||||
// order, and cannot currently distinguish between disconnected subgraphs
|
||||
// that are logically connected by subgraphs on a different device.
|
||||
// 4. Memory logging is not currently supported.
|
||||
// 5. Allocation forwarding is not currently supported.
|
||||
// 6. Non-default device contexts are not currently supported. In effect, this
|
||||
// limits the executor to CPU devices.
|
||||
// 7. Ops that rely on `OpKernelContext::slice_reader_cache()` being non-null
|
||||
// are not currently supported.
|
||||
//
|
||||
// The single-threaded executor is primarily suitable for executing simple
|
||||
// TensorFlow functions, such as one might find in a `tf.data` pipeline.
|
||||
Status NewSingleThreadedExecutor(const LocalExecutorParams& params,
|
||||
std::unique_ptr<const Graph> graph,
|
||||
Executor** executor);
|
||||
|
||||
} // namespace data
|
||||
} // namespace tensorflow
|
||||
|
||||
#endif // TENSORFLOW_CORE_KERNELS_DATA_SINGLE_THREADED_EXECUTOR_H_
|
||||
@@ -0,0 +1,41 @@
|
||||
/* Copyright 2017 The TensorFlow Authors. All Rights Reserved.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
==============================================================================*/
|
||||
#ifndef TENSORFLOW_CORE_KERNELS_DATA_SQL_DRIVER_MANAGER_H_
|
||||
#define TENSORFLOW_CORE_KERNELS_DATA_SQL_DRIVER_MANAGER_H_
|
||||
|
||||
#include "tensorflow/core/kernels/data/sql/query_connection.h"
|
||||
|
||||
namespace tensorflow {
|
||||
namespace data {
|
||||
namespace sql {
|
||||
|
||||
// A factory class for creating `QueryConnection` instances.
|
||||
class DriverManager {
|
||||
public:
|
||||
// A factory method for creating `QueryConnection` instances.
|
||||
//
|
||||
// `driver_name` is the database type (e.g. 'sqlite'). `driver_name`
|
||||
// corresponds to a `QueryConnection` subclass. For example, if `driver_name`
|
||||
// == `sqlite`, then `CreateQueryConnection` will create a
|
||||
// `SqliteQueryConnection` instance.
|
||||
static std::unique_ptr<QueryConnection> CreateQueryConnection(
|
||||
const string& driver_name);
|
||||
};
|
||||
|
||||
} // namespace sql
|
||||
} // namespace data
|
||||
} // namespace tensorflow
|
||||
|
||||
#endif // TENSORFLOW_CORE_KERNELS_DATA_SQL_DRIVER_MANAGER_H_
|
||||
@@ -0,0 +1,70 @@
|
||||
/* Copyright 2017 The TensorFlow Authors. All Rights Reserved.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
==============================================================================*/
|
||||
#ifndef TENSORFLOW_CORE_KERNELS_DATA_SQL_QUERY_CONNECTION_H_
|
||||
#define TENSORFLOW_CORE_KERNELS_DATA_SQL_QUERY_CONNECTION_H_
|
||||
|
||||
#include "tensorflow/core/framework/tensor.h"
|
||||
|
||||
namespace tensorflow {
|
||||
namespace data {
|
||||
|
||||
class IteratorContext;
|
||||
|
||||
namespace sql {
|
||||
// This interface allows a user to connect to a database, execute a query, and
|
||||
// iterate over the result set, putting the results into an output tensor.
|
||||
// A subclass implementation is required for each type of database
|
||||
// (e.g. sqlite3, mysql, etc.)
|
||||
//
|
||||
// Presently, a `QueryConnection` instance can only handle one query at a time.
|
||||
// In a future extension, this class may be refactored so that it creates
|
||||
// instances of a new class (named, say, `Statement`) which could have a
|
||||
// one-to-one correspondence with queries. This would make `QueryConnection`
|
||||
// more consistent with `Connection` classes of other database APIs.
|
||||
// `QueryConnection` would then be renamed simply `Connection`.
|
||||
//
|
||||
// This class is not thread safe. Access to it is guarded by a mutex in
|
||||
// `SqlDatasetOp::Dataset::Iterator`.
|
||||
class QueryConnection {
|
||||
public:
|
||||
virtual ~QueryConnection() {}
|
||||
// Opens a connection to the database named by `data_source_name`. Prepares to
|
||||
// execute `query` against the database.
|
||||
//
|
||||
// The client must call `Close()` to release the connection resources, even
|
||||
// if `Open()` fails. `Close()` must be called before making another call
|
||||
// to `Open()`.
|
||||
virtual Status Open(const string& data_source_name, const string& query,
|
||||
const DataTypeVector& output_types) = 0;
|
||||
// Closes an opened connection.
|
||||
virtual Status Close() = 0;
|
||||
// Retrieves the next row of the result set of the query from the most recent
|
||||
// call to `Open()`.
|
||||
//
|
||||
// If such a row exists, then the row will be stored in `*out_tensors`, and
|
||||
// `false` will be stored in `*end_of_sequence`.
|
||||
//
|
||||
// If there are no more rows in the result set, then instead `true` will be
|
||||
// stored in `*end_of_sequence`, and the content of `*out_tensors` will be
|
||||
// undefined.
|
||||
virtual Status GetNext(IteratorContext* ctx, std::vector<Tensor>* out_tensors,
|
||||
bool* end_of_sequence) = 0;
|
||||
};
|
||||
|
||||
} // namespace sql
|
||||
} // namespace data
|
||||
} // namespace tensorflow
|
||||
|
||||
#endif // TENSORFLOW_CORE_KERNELS_DATA_SQL_QUERY_CONNECTION_H_
|
||||
@@ -0,0 +1,56 @@
|
||||
/* Copyright 2017 The TensorFlow Authors. All Rights Reserved.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
==============================================================================*/
|
||||
#ifndef TENSORFLOW_CORE_KERNELS_DATA_SQL_SQLITE_QUERY_CONNECTION_H_
|
||||
#define TENSORFLOW_CORE_KERNELS_DATA_SQL_SQLITE_QUERY_CONNECTION_H_
|
||||
|
||||
#include <memory>
|
||||
|
||||
#include "tensorflow/core/kernels/data/sql/query_connection.h"
|
||||
#include "tensorflow/core/lib/db/sqlite.h"
|
||||
#include "tensorflow/core/platform/types.h"
|
||||
|
||||
namespace tensorflow {
|
||||
namespace data {
|
||||
namespace sql {
|
||||
|
||||
class SqliteQueryConnection : public QueryConnection {
|
||||
public:
|
||||
SqliteQueryConnection();
|
||||
~SqliteQueryConnection() override;
|
||||
Status Open(const string& data_source_name, const string& query,
|
||||
const DataTypeVector& output_types) override;
|
||||
Status Close() override;
|
||||
Status GetNext(IteratorContext* ctx, std::vector<Tensor>* out_tensors,
|
||||
bool* end_of_sequence) override;
|
||||
|
||||
private:
|
||||
// Prepares the query string `query_`.
|
||||
Status PrepareQuery();
|
||||
// Fills `tensor` with the column_index_th element of the current row of
|
||||
// `stmt_`.
|
||||
void FillTensorWithResultSetEntry(const DataType& data_type, int column_index,
|
||||
Tensor* tensor);
|
||||
Sqlite* db_ = nullptr;
|
||||
SqliteStatement stmt_;
|
||||
int column_count_ = 0;
|
||||
string query_;
|
||||
DataTypeVector output_types_;
|
||||
};
|
||||
|
||||
} // namespace sql
|
||||
} // namespace data
|
||||
} // namespace tensorflow
|
||||
|
||||
#endif // TENSORFLOW_CORE_KERNELS_DATA_SQL_SQLITE_QUERY_CONNECTION_H_
|
||||
@@ -0,0 +1,50 @@
|
||||
/* Copyright 2017 The TensorFlow Authors. All Rights Reserved.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
==============================================================================*/
|
||||
#ifndef TENSORFLOW_CORE_KERNELS_DATA_WINDOW_DATASET_H_
|
||||
#define TENSORFLOW_CORE_KERNELS_DATA_WINDOW_DATASET_H_
|
||||
|
||||
#include <vector>
|
||||
|
||||
#include "tensorflow/core/framework/partial_tensor_shape.h"
|
||||
#include "tensorflow/core/framework/tensor.h"
|
||||
#include "tensorflow/core/framework/types.h"
|
||||
#include "tensorflow/core/kernels/data/dataset.h"
|
||||
|
||||
namespace tensorflow {
|
||||
namespace data {
|
||||
|
||||
// Creates a dataset representing an eagerly-collected window of elements.
|
||||
//
|
||||
// The `elements` argument defines the elements of the resulting
|
||||
// dataset, which is stored in `out_dataset`.
|
||||
//
|
||||
// This dataset is constructed internally for use in datasets that
|
||||
// build nested dataset expressions (e.g. the reducer function for
|
||||
// GroupByWindowDataset). It efficiently supports multiple iterators on
|
||||
// the same window without recomputation.
|
||||
//
|
||||
// REQUIRES: `output_types` must match the types of the respective
|
||||
// element components in `elements`.
|
||||
// REQUIRES: `output_shapes` must be compatible with the shapes of the
|
||||
// respective element components in `elements`.a
|
||||
Status NewWindowDataset(std::vector<std::vector<Tensor>> elements,
|
||||
DataTypeVector output_types,
|
||||
std::vector<PartialTensorShape> output_shapes,
|
||||
DatasetBase** out_dataset);
|
||||
|
||||
} // namespace data
|
||||
} // namespace tensorflow
|
||||
|
||||
#endif // TENSORFLOW_CORE_KERNELS_DATA_WINDOW_DATASET_H_
|
||||
@@ -0,0 +1,86 @@
|
||||
/* Copyright 2015 The TensorFlow Authors. All Rights Reserved.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
==============================================================================*/
|
||||
|
||||
#ifndef TENSORFLOW_CORE_KERNELS_DATA_FORMAT_OPS_H_
|
||||
#define TENSORFLOW_CORE_KERNELS_DATA_FORMAT_OPS_H_
|
||||
// Functor definition for data format dim mapping ops, must be compilable
|
||||
// by nvcc.
|
||||
#include "third_party/eigen3/unsupported/Eigen/CXX11/Tensor"
|
||||
#include "tensorflow/core/framework/tensor_types.h"
|
||||
|
||||
namespace tensorflow {
|
||||
namespace functor {
|
||||
|
||||
// Functor used by DataFormatDimMapOP to do the computations.
|
||||
template <typename Device, typename T>
|
||||
struct DataFormatDimMap {
|
||||
void operator()(const Device& d, typename TTypes<T>::ConstFlat x,
|
||||
typename TTypes<T>::Flat y, const TTypes<int>::Vec dst) {
|
||||
auto zero = x.constant(0);
|
||||
auto one = x.constant(1);
|
||||
auto two = x.constant(2);
|
||||
|
||||
auto f_zero = x.constant(dst(0));
|
||||
auto f_one = x.constant(dst(1));
|
||||
auto f_two = x.constant(dst(2));
|
||||
auto f_three = x.constant(dst(3));
|
||||
|
||||
auto four = x.constant(4);
|
||||
auto x_mod = (x + four) % 4;
|
||||
|
||||
auto is_zero = (x_mod == zero);
|
||||
auto is_one = (x_mod == one);
|
||||
auto is_two = (x_mod == two);
|
||||
|
||||
y.device(d) = is_zero.select(
|
||||
f_zero, is_one.select(f_one, is_two.select(f_two, f_three)));
|
||||
}
|
||||
};
|
||||
|
||||
template <typename T>
|
||||
struct VecPermute {
|
||||
VecPermute(const Eigen::DSizes<Eigen::DenseIndex, 8>& dst) : dst_(dst) {}
|
||||
Eigen::DSizes<Eigen::DenseIndex, 1> dimensions(
|
||||
typename TTypes<T>::ConstFlat input) const {
|
||||
Eigen::DSizes<Eigen::DenseIndex, 1> result;
|
||||
result[0] = input.dimension(0);
|
||||
return result;
|
||||
}
|
||||
template <typename Output, typename Device>
|
||||
void eval(typename TTypes<T>::ConstFlat input, Output& output,
|
||||
const Device& d) const {
|
||||
for (int i = 0; i < input.size(); ++i) {
|
||||
output.template chip<0>(dst_[i]).device(d) = input.template chip<0>(i);
|
||||
}
|
||||
}
|
||||
|
||||
private:
|
||||
Eigen::DSizes<Eigen::DenseIndex, 8> dst_;
|
||||
};
|
||||
|
||||
// Functor used by DataFormatVecPermuteOp to do the computations.
|
||||
template <typename Device, typename T>
|
||||
struct DataFormatVecPermute {
|
||||
void operator()(const Device& d, typename TTypes<T>::ConstFlat x,
|
||||
typename TTypes<T>::Flat y,
|
||||
const Eigen::DSizes<Eigen::DenseIndex, 8>& dst) {
|
||||
y.device(d) = x.customOp(VecPermute<T>(dst));
|
||||
}
|
||||
};
|
||||
|
||||
} // namespace functor
|
||||
} // namespace tensorflow
|
||||
|
||||
#endif // TENSORFLOW_CORE_KERNELS_DATA_FORMAT_OPS_H_
|
||||
@@ -0,0 +1,117 @@
|
||||
/* Copyright 2016 The TensorFlow Authors. All Rights Reserved.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
==============================================================================*/
|
||||
|
||||
#ifndef TENSORFLOW_CORE_KERNELS_DEEP_CONV2D_H_
|
||||
#define TENSORFLOW_CORE_KERNELS_DEEP_CONV2D_H_
|
||||
|
||||
#include "tensorflow/core/framework/types.h"
|
||||
|
||||
namespace tensorflow {
|
||||
|
||||
class OpKernelContext;
|
||||
|
||||
// DeepConv2D is a Conv2D implementation specialized for deep (i.e. large
|
||||
// in_depth * out_depth product) convolutions (see deep_conv2d.cc for details).
|
||||
|
||||
// DeepConv2DTransform is an interface for implementing transforms for
|
||||
// DeepConv2D. Implementations must specify transform matrices and
|
||||
// input/output/filter shapes. DeepConv2d computes:
|
||||
//
|
||||
// y = C[Ad * Bg]
|
||||
//
|
||||
// C: output transform matrix
|
||||
// A: input data transform matrix
|
||||
// B: filter transform matrix
|
||||
// d: vectorized 2D data tile
|
||||
// g: vectorized 2D filter tile
|
||||
// y: vectorized 2D output tile
|
||||
|
||||
template <typename T>
|
||||
class DeepConv2DTransform {
|
||||
public:
|
||||
virtual ~DeepConv2DTransform() {}
|
||||
|
||||
virtual void GetFilterTransformMatrix(const int64 rows, const int64 cols,
|
||||
T* transform_matrix) const = 0;
|
||||
|
||||
virtual void GetInputTransformMatrix(const int64 rows, const int64 cols,
|
||||
T* transform_matrix) const = 0;
|
||||
|
||||
virtual void GetOutputTransformMatrix(const int64 rows, const int64 cols,
|
||||
T* transform_matrix) const = 0;
|
||||
|
||||
struct Shape {
|
||||
Shape(int64 r, int64 c) : rows(r), cols(c) {}
|
||||
int64 rows;
|
||||
int64 cols;
|
||||
};
|
||||
|
||||
virtual const Shape& filter_shape() const = 0;
|
||||
virtual const Shape& input_shape() const = 0;
|
||||
virtual const Shape& output_shape() const = 0;
|
||||
};
|
||||
|
||||
// Conv2D arguments used by DeepConv2D implementation.
|
||||
struct Conv2DArgs {
|
||||
// Input layer dimensions
|
||||
int batch;
|
||||
int in_rows;
|
||||
int in_cols;
|
||||
int in_depth;
|
||||
int filter_rows;
|
||||
int filter_cols;
|
||||
int pad_rows;
|
||||
int pad_cols;
|
||||
|
||||
// Output layer dimensions
|
||||
int out_rows;
|
||||
int out_cols;
|
||||
int out_depth;
|
||||
|
||||
Conv2DArgs()
|
||||
: batch(0),
|
||||
in_rows(0),
|
||||
in_cols(0),
|
||||
in_depth(0),
|
||||
filter_rows(0),
|
||||
filter_cols(0),
|
||||
pad_rows(0),
|
||||
pad_cols(0),
|
||||
out_rows(0),
|
||||
out_cols(0),
|
||||
out_depth(0) {}
|
||||
};
|
||||
|
||||
// Returns true if convolution operation specified by function arguments
|
||||
// can use DeepConv2D implementation, and false otherwise.
|
||||
// May return false based on parameters, cost, or whether feature is disabled.
|
||||
bool CanUseDeepConv2D(int stride_rows, int stride_cols, int filter_rows,
|
||||
int filter_cols, int in_depth, int out_depth,
|
||||
int out_rows, int out_cols);
|
||||
|
||||
namespace functor {
|
||||
|
||||
// Calls DeepConv2D implementation (see deep_conv2d.cc for details).
|
||||
template <typename Device, typename T>
|
||||
struct DeepConv2D {
|
||||
void operator()(OpKernelContext* ctx, const Conv2DArgs& args, const T* input,
|
||||
const T* filter, T* output);
|
||||
};
|
||||
|
||||
} // namespace functor
|
||||
|
||||
} // namespace tensorflow
|
||||
|
||||
#endif // TENSORFLOW_CORE_KERNELS_DEEP_CONV2D_H_
|
||||
@@ -0,0 +1,108 @@
|
||||
/* Copyright 2015 The TensorFlow Authors. All Rights Reserved.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
==============================================================================*/
|
||||
|
||||
#ifndef TENSORFLOW_CORE_KERNELS_DENSE_UPDATE_FUNCTOR_H_
|
||||
#define TENSORFLOW_CORE_KERNELS_DENSE_UPDATE_FUNCTOR_H_
|
||||
|
||||
#define EIGEN_USE_THREADS
|
||||
|
||||
#include "third_party/eigen3/unsupported/Eigen/CXX11/Tensor"
|
||||
#include "tensorflow/core/framework/op_kernel.h"
|
||||
#include "tensorflow/core/framework/tensor_types.h"
|
||||
|
||||
namespace tensorflow {
|
||||
|
||||
typedef Eigen::ThreadPoolDevice CPUDevice;
|
||||
typedef Eigen::GpuDevice GPUDevice;
|
||||
|
||||
#ifdef TENSORFLOW_USE_SYCL
|
||||
typedef Eigen::SyclDevice SYCLDevice;
|
||||
#endif // TENSORFLOW_USE_SYCL
|
||||
|
||||
enum DenseUpdateType { ADD, SUB, ASSIGN };
|
||||
|
||||
namespace functor {
|
||||
|
||||
template <typename Device, typename T, DenseUpdateType OP>
|
||||
struct DenseUpdate {
|
||||
void operator()(const Device& d, typename TTypes<T>::Flat params,
|
||||
typename TTypes<T>::ConstFlat update);
|
||||
};
|
||||
|
||||
template <typename T>
|
||||
struct DenseUpdate<CPUDevice, T, ADD> {
|
||||
void operator()(const CPUDevice& d, typename TTypes<T>::Flat params,
|
||||
typename TTypes<T>::ConstFlat update) {
|
||||
params.device(d) += update;
|
||||
}
|
||||
};
|
||||
|
||||
template <typename T>
|
||||
struct DenseUpdate<CPUDevice, T, SUB> {
|
||||
void operator()(const CPUDevice& d, typename TTypes<T>::Flat params,
|
||||
typename TTypes<T>::ConstFlat update) {
|
||||
params.device(d) -= update;
|
||||
}
|
||||
};
|
||||
|
||||
template <typename T>
|
||||
struct DenseUpdate<CPUDevice, T, ASSIGN> {
|
||||
void operator()(const CPUDevice& d, typename TTypes<T>::Flat params,
|
||||
typename TTypes<T>::ConstFlat update) {
|
||||
params.device(d) = update;
|
||||
}
|
||||
};
|
||||
|
||||
#ifdef TENSORFLOW_USE_SYCL
|
||||
template <typename T>
|
||||
struct DenseUpdate<SYCLDevice, T, ADD> {
|
||||
void operator()(const SYCLDevice& d, typename TTypes<T>::Flat params,
|
||||
typename TTypes<T>::ConstFlat update) {
|
||||
params.device(d) += update;
|
||||
}
|
||||
};
|
||||
|
||||
template <typename T>
|
||||
struct DenseUpdate<SYCLDevice, T, SUB> {
|
||||
void operator()(const SYCLDevice& d, typename TTypes<T>::Flat params,
|
||||
typename TTypes<T>::ConstFlat update) {
|
||||
params.device(d) -= update;
|
||||
}
|
||||
};
|
||||
|
||||
template <typename T>
|
||||
struct DenseUpdate<SYCLDevice, T, ASSIGN> {
|
||||
void operator()(const SYCLDevice& d, typename TTypes<T>::Flat params,
|
||||
typename TTypes<T>::ConstFlat update) {
|
||||
params.device(d) = update;
|
||||
}
|
||||
};
|
||||
#endif // TENSORFLOW_USE_SYCL
|
||||
|
||||
} // end namespace functor
|
||||
|
||||
template <typename Device>
|
||||
Status VariantCopyFn(OpKernelContext* context, const Tensor& from, Tensor* to);
|
||||
|
||||
template <>
|
||||
Status VariantCopyFn<CPUDevice>(OpKernelContext* context, const Tensor& from,
|
||||
Tensor* to);
|
||||
template <>
|
||||
Status VariantCopyFn<GPUDevice>(OpKernelContext* context, const Tensor& from,
|
||||
Tensor* to);
|
||||
|
||||
} // end namespace tensorflow
|
||||
|
||||
#endif // TENSORFLOW_CORE_KERNELS_DENSE_UPDATE_FUNCTOR_H_
|
||||
@@ -0,0 +1,56 @@
|
||||
/* Copyright 2015 The TensorFlow Authors. All Rights Reserved.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
==============================================================================*/
|
||||
|
||||
#ifndef TENSORFLOW_CORE_KERNELS_DEPTHTOSPACE_OP_H_
|
||||
#define TENSORFLOW_CORE_KERNELS_DEPTHTOSPACE_OP_H_
|
||||
|
||||
#include "third_party/eigen3/unsupported/Eigen/CXX11/Tensor"
|
||||
#include "tensorflow/core/framework/tensor_types.h"
|
||||
#include "tensorflow/core/util/tensor_format.h"
|
||||
|
||||
namespace tensorflow {
|
||||
namespace functor {
|
||||
|
||||
// Functor used by DepthToSpaceOp to do the computations.
|
||||
// Implements a family of Depth to Space transforms for a 4D 'input' tensor
|
||||
// to a 4D 'output' tensor, both tensors use type 'T' and layout 'data_format'.
|
||||
// These transforms multiply the vertical and horizontal image sizes by
|
||||
// 'block_size', and divide the depth dimension by (block_size * block_size)
|
||||
// which must divide evenly.
|
||||
// Each pixel in the input image is converted to a square block of pixels in
|
||||
// the output image. The Y, X coordinates within each block comes from the
|
||||
// high component of the input depth (channel) index.
|
||||
// e.g. for data_format = NHWC:
|
||||
// Each element in the input tensor can be specified via 6 coordinates,
|
||||
// ordered by decreasing memory layout significance as:
|
||||
// n,iY,iX,bY,bX,oC (where n=batch index, iX, iY means X or Y coordinates
|
||||
// within the input image, bX, bY means coordinates
|
||||
// within the output block, oC means output channel).
|
||||
// The output would be a transpose to the following layout:
|
||||
// n,iY,bY,iX,bX,oC
|
||||
template <typename Device, typename T, TensorFormat data_format>
|
||||
struct DepthToSpaceOpFunctor {
|
||||
void operator()(const Device& d, typename TTypes<T, 4>::ConstTensor input,
|
||||
int block_size, typename TTypes<T, 4>::Tensor output);
|
||||
|
||||
// This 5-D version is to support NCHW_VECT_C.
|
||||
void operator()(const Device& d, typename TTypes<T, 5>::ConstTensor input,
|
||||
int block_size, typename TTypes<T, 5>::Tensor output);
|
||||
};
|
||||
|
||||
} // namespace functor
|
||||
} // namespace tensorflow
|
||||
|
||||
#endif // TENSORFLOW_CORE_KERNELS_DEPTHTOSPACE_OP_H_
|
||||
@@ -0,0 +1,287 @@
|
||||
/* Copyright 2015 The TensorFlow Authors. All Rights Reserved.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
==============================================================================*/
|
||||
|
||||
#ifndef TENSORFLOW_CORE_KERNELS_DEPTHWISE_CONV_OP_H_
|
||||
#define TENSORFLOW_CORE_KERNELS_DEPTHWISE_CONV_OP_H_
|
||||
|
||||
#include "third_party/eigen3/unsupported/Eigen/CXX11/Tensor"
|
||||
#include "tensorflow/core/framework/types.h"
|
||||
#include "tensorflow/core/util/tensor_format.h"
|
||||
|
||||
namespace tensorflow {
|
||||
|
||||
struct DepthwiseArgs {
|
||||
// Input layer dimensions
|
||||
int batch;
|
||||
int in_rows;
|
||||
int in_cols;
|
||||
int in_depth;
|
||||
int filter_rows;
|
||||
int filter_cols;
|
||||
int depth_multiplier;
|
||||
int stride;
|
||||
int pad_rows;
|
||||
int pad_cols;
|
||||
|
||||
// Output layer dimensions
|
||||
int out_rows;
|
||||
int out_cols;
|
||||
int out_depth;
|
||||
|
||||
DepthwiseArgs()
|
||||
: batch(0),
|
||||
in_rows(0),
|
||||
in_cols(0),
|
||||
in_depth(0),
|
||||
filter_rows(0),
|
||||
filter_cols(0),
|
||||
depth_multiplier(0),
|
||||
stride(0),
|
||||
pad_rows(0),
|
||||
pad_cols(0),
|
||||
out_rows(0),
|
||||
out_cols(0),
|
||||
out_depth(0) {}
|
||||
};
|
||||
|
||||
// Forward declaration.
|
||||
class OpKernelContext;
|
||||
|
||||
template <typename Device, typename T>
|
||||
struct LaunchDepthwiseConvOp {
|
||||
void operator()(OpKernelContext* ctx, const DepthwiseArgs& args,
|
||||
const T* input, const T* filter, T* output,
|
||||
TensorFormat data_format);
|
||||
};
|
||||
|
||||
template <typename Device, typename T>
|
||||
struct LaunchDepthwiseConvBackpropInputOp {
|
||||
void operator()(OpKernelContext* ctx, const DepthwiseArgs& args,
|
||||
const T* out_backprop, const T* filter, T* in_backprop,
|
||||
TensorFormat data_format);
|
||||
};
|
||||
|
||||
template <typename Device, typename T>
|
||||
struct LaunchDepthwiseConvBackpropFilterOp {
|
||||
void operator()(OpKernelContext* ctx, const DepthwiseArgs& args,
|
||||
const T* out_backprop, const T* input, T* filter_backprop,
|
||||
TensorFormat data_format);
|
||||
};
|
||||
|
||||
#if GOOGLE_CUDA
|
||||
template <typename T>
|
||||
struct LaunchDepthwiseConvOp<Eigen::GpuDevice, T> {
|
||||
void operator()(OpKernelContext* ctx, const DepthwiseArgs& args,
|
||||
const T* input, const T* filter, T* output,
|
||||
TensorFormat data_format);
|
||||
};
|
||||
|
||||
template <typename T>
|
||||
struct LaunchDepthwiseConvBackpropInputOp<Eigen::GpuDevice, T> {
|
||||
void operator()(class OpKernelContext* ctx, const DepthwiseArgs& args,
|
||||
const T* out_backprop, const T* filter, T* in_backprop,
|
||||
TensorFormat data_format);
|
||||
};
|
||||
|
||||
template <typename T>
|
||||
struct LaunchDepthwiseConvBackpropFilterOp<Eigen::GpuDevice, T> {
|
||||
void operator()(class OpKernelContext* ctx, const DepthwiseArgs& args,
|
||||
const T* out_backprop, const T* input, T* filter_backprop,
|
||||
TensorFormat data_format);
|
||||
};
|
||||
#endif
|
||||
|
||||
} // namespace tensorflow
|
||||
|
||||
namespace tensorflow {
|
||||
namespace functor {
|
||||
|
||||
// Pads 'filter' to vector-register boundary along its inner dimension:
|
||||
// filter_inner_dim_size = in_depth * depth_multiplier
|
||||
// Requires 'filter' to have the following storage order:
|
||||
// [filter_rows, filter_cols, in_depth, depth_multiplier]
|
||||
// Returns zero-padded filter in 'padded_filter'.
|
||||
//
|
||||
// EX:
|
||||
// in_depth = 3, depth_multiplier = 2, filter [2, 2], register_width = 4
|
||||
// So we have a total of 3 * 2 = 6 filters, each of spatial size 2 x 2.
|
||||
//
|
||||
// filter [rows, cols, in_depth, depth_multiplier]
|
||||
// [u0, v0, w0, x0] [y0, z0, u1, v1] [w1, x1, y1, z1]
|
||||
// [u2, v2, w2, x2] [y2, z2, u3, v3] [w3, x3, y3, z3]
|
||||
//
|
||||
// padded_filter [rows, cols, in_depth, depth_multiplier]
|
||||
// [u0, v0, w0, x0] [y0, z0, 0, 0] [u1, v1, w1, x1] [y1, z1, 0, 0]
|
||||
// [u2, v2, w2, x2] [y2, z2, 0, 0] [u3, v3, w3, x3] [y3, z3, 0, 0]
|
||||
|
||||
template <typename T>
|
||||
struct DepthwiseFilterPadOp {
|
||||
void operator()(const DepthwiseArgs& args, const T* filter,
|
||||
T* padded_filter) {
|
||||
typedef typename Eigen::internal::packet_traits<T>::type Packet;
|
||||
static const int64 kPacketSize = (sizeof(Packet) / sizeof(T));
|
||||
|
||||
// Calculate vectorized and scalar lengths of filter's inner dimension.
|
||||
const int64 filter_inner_dim_size = args.out_depth;
|
||||
const int64 vectorized_size =
|
||||
(filter_inner_dim_size / kPacketSize) * kPacketSize;
|
||||
const int64 scalar_size = filter_inner_dim_size - vectorized_size;
|
||||
// Calculate required padding and padded output buffer stride.
|
||||
const int64 pad_size = scalar_size > 0 ? kPacketSize - scalar_size : 0;
|
||||
const int64 padded_filter_stride = vectorized_size + kPacketSize;
|
||||
|
||||
const int64 filter_spatial_size = args.filter_rows * args.filter_cols;
|
||||
for (int64 i = 0; i < filter_spatial_size; ++i) {
|
||||
const int64 input_base = i * filter_inner_dim_size;
|
||||
const int64 output_base = i * padded_filter_stride;
|
||||
// Write vectorized length of filter's inner dimension to output.
|
||||
for (int64 j = 0; j < vectorized_size; j += kPacketSize) {
|
||||
const auto v = Eigen::internal::ploadu<Packet>(filter + input_base + j);
|
||||
Eigen::internal::pstoreu<T>(padded_filter + output_base + j, v);
|
||||
}
|
||||
// Write scalar length of filter's inner dimension to output.
|
||||
for (int64 j = 0; j < scalar_size; ++j) {
|
||||
padded_filter[output_base + vectorized_size + j] =
|
||||
filter[input_base + vectorized_size + j];
|
||||
}
|
||||
// Pad the remainder of output to vector-register boundary.
|
||||
for (int64 j = 0; j < pad_size; ++j) {
|
||||
padded_filter[output_base + vectorized_size + scalar_size + j] =
|
||||
static_cast<T>(0);
|
||||
}
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
// Copies data from local region in 'input' specified by 'out_r' and 'out_'c'
|
||||
// to 'input_buffer'. The copied data is replicated by factor
|
||||
// 'args.depth_mulitplier', and padded to vector register-width boundaries so
|
||||
// that it is aligned for efficient traversal and vector multiply-add by the
|
||||
// depthwise kernel.
|
||||
//
|
||||
// EX:
|
||||
// in_depth = 3, depth_multiplier = 2, filter [2, 2], register_width = 4
|
||||
//
|
||||
// input: [batch, in_rows, in_cols, in_depth]
|
||||
//
|
||||
// [a0, a1, a2, b0, b1, b2, ..., e0, e1, e2, f0, f1, f2, ...]
|
||||
//
|
||||
// input_buffer (register boundaries shown):
|
||||
// [a0, a0, a1, a1] [a2, a2, 0, 0] in_row = 0, in_col = 0
|
||||
// [b0, b0, b1, b1] [b2, b2, 0, 0] in_row = 0, in_col = 1
|
||||
// [e0, e0, e1, e1] [e2, e2, 0, 0] in_row = 1, in_col = 0
|
||||
// [f0, f0, f1, f1] [f2, f2, 0, 0] in_row = 1, in_col = 1
|
||||
//
|
||||
// Returns replicated and padded data from specified input region in
|
||||
// 'input_buffer'.
|
||||
|
||||
template <typename T>
|
||||
struct DepthwiseInputCopyOp {
|
||||
void operator()(const DepthwiseArgs& args,
|
||||
const int64 padded_filter_inner_dim_size, const int64 out_r,
|
||||
const int64 out_c, const T* input, T* input_buffer) {
|
||||
typedef typename Eigen::internal::packet_traits<T>::type Packet;
|
||||
static const int64 kPacketSize = (sizeof(Packet) / sizeof(T));
|
||||
|
||||
// Calculate vectorized and scalar (residual) lengths for 'in_depth'.
|
||||
const int64 input_vectorized_size =
|
||||
(args.in_depth / kPacketSize) * kPacketSize;
|
||||
const int64 input_scalar_size = args.in_depth % kPacketSize;
|
||||
|
||||
// Calculate vectorized and scalar (residual) lengths for
|
||||
// 'depth_multiplier'. This is used to efficiently replicate data for
|
||||
// when 'depth_multiplier' > kPacketSize.
|
||||
const int64 dm_vectorized_size =
|
||||
(args.depth_multiplier / kPacketSize) * kPacketSize;
|
||||
const int64 dm_scalar_size = args.depth_multiplier % kPacketSize;
|
||||
|
||||
// Calculate output padding length.
|
||||
const int64 output_scalar_size = args.out_depth % kPacketSize;
|
||||
const int64 output_pad_size =
|
||||
output_scalar_size > 0 ? kPacketSize - output_scalar_size : 0;
|
||||
|
||||
const int64 replicated_packet_size = kPacketSize * args.depth_multiplier;
|
||||
|
||||
// Iterate through all rows x cols reading 'in_depth' from 'input' and
|
||||
// replicating by 'depth_multiplier' into 'input_buffer' (otherwise
|
||||
// zero-padding input buffer as needed).
|
||||
auto* in_buf = input_buffer;
|
||||
const int64 in_r_start = out_r * args.stride - args.pad_rows;
|
||||
const int64 in_c_start = out_c * args.stride - args.pad_cols;
|
||||
|
||||
for (int64 f_r = 0; f_r < args.filter_rows; ++f_r) {
|
||||
const int64 in_r = in_r_start + f_r;
|
||||
|
||||
for (int64 f_c = 0; f_c < args.filter_cols; ++f_c) {
|
||||
const int64 in_c = in_c_start + f_c;
|
||||
|
||||
if (in_r >= 0 && in_r < args.in_rows && in_c >= 0 &&
|
||||
in_c < args.in_cols) {
|
||||
auto* in = input + (in_r * args.in_cols + in_c) * args.in_depth;
|
||||
// Copy vectorized portion of inner dimension.
|
||||
for (int64 d = 0; d < input_vectorized_size; d += kPacketSize) {
|
||||
auto v = Eigen::internal::ploadu<Packet>(in + d);
|
||||
for (int dm = 0; dm < args.depth_multiplier; ++dm) {
|
||||
Eigen::internal::pscatter<T, Packet>(in_buf + dm, v,
|
||||
args.depth_multiplier);
|
||||
}
|
||||
in_buf += replicated_packet_size;
|
||||
}
|
||||
|
||||
// Copy scalar portion of inner dimension.
|
||||
for (int64 d = 0; d < input_scalar_size; ++d) {
|
||||
T v = in[input_vectorized_size + d];
|
||||
const int64 base = d * args.depth_multiplier;
|
||||
if (dm_vectorized_size > 0) {
|
||||
// Copy vectorized portion of replicated output.
|
||||
// This branch is only taken if 'args.depth_multiplier' is
|
||||
// vectorizable (i.e. args.depth_multiplier >= register width).
|
||||
auto p = Eigen::internal::pset1<Packet>(v);
|
||||
for (int64 dm = 0; dm < dm_vectorized_size; dm += kPacketSize) {
|
||||
Eigen::internal::pstoreu<T>(in_buf + base + dm, p);
|
||||
}
|
||||
// Copy scalar portion of replicated output.
|
||||
for (int64 dm = 0; dm < dm_scalar_size; ++dm) {
|
||||
in_buf[base + dm_vectorized_size + dm] = v;
|
||||
}
|
||||
} else {
|
||||
// Depth multiplier is less than one packet: scalar copy.
|
||||
for (int dm = 0; dm < args.depth_multiplier; ++dm) {
|
||||
in_buf[base + dm] = v;
|
||||
}
|
||||
}
|
||||
}
|
||||
in_buf += input_scalar_size * args.depth_multiplier;
|
||||
|
||||
// Pad the remainder of the output to vector register boundary.
|
||||
for (int64 d = 0; d < output_pad_size; ++d) {
|
||||
in_buf[d] = static_cast<T>(0);
|
||||
}
|
||||
in_buf += output_pad_size;
|
||||
|
||||
} else {
|
||||
// Zero pad.
|
||||
memset(in_buf, 0, sizeof(T) * padded_filter_inner_dim_size);
|
||||
in_buf += padded_filter_inner_dim_size;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
} // namespace functor
|
||||
} // namespace tensorflow
|
||||
|
||||
#endif // TENSORFLOW_CORE_KERNELS_DEPTHWISE_CONV_OP_H_
|
||||
@@ -0,0 +1,47 @@
|
||||
/* Copyright 2017 The TensorFlow Authors. All Rights Reserved.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
==============================================================================*/
|
||||
|
||||
#ifndef TENSORFLOW_CORE_KERNELS_DETERMINANT_OP_H_
|
||||
#define TENSORFLOW_CORE_KERNELS_DETERMINANT_OP_H_
|
||||
|
||||
#include "tensorflow/core/framework/tensor_types.h"
|
||||
|
||||
namespace tensorflow {
|
||||
namespace functor {
|
||||
|
||||
// Helper functor to compute Determinant from a partially pivoted LU
|
||||
// factorization.
|
||||
template <typename Device, typename Scalar>
|
||||
struct DeterminantFromPivotedLUFunctor {
|
||||
void operator()(const Device& device,
|
||||
typename TTypes<Scalar, 3>::ConstTensor lu_factor,
|
||||
const int* pivots, typename TTypes<Scalar, 1>::Tensor output,
|
||||
int* info);
|
||||
};
|
||||
|
||||
// Helper functor to compute sign and log of the absolute value of the
|
||||
// determinant from a partially pivoted LU factorization.
|
||||
template <typename Device, typename Scalar>
|
||||
struct LogDeterminantFromPivotedLUFunctor {
|
||||
void operator()(const Device& device,
|
||||
typename TTypes<Scalar, 3>::ConstTensor lu_factor,
|
||||
const int* pivots, typename TTypes<Scalar, 1>::Tensor sign,
|
||||
typename TTypes<Scalar, 1>::Tensor log_abs_det);
|
||||
};
|
||||
|
||||
} // namespace functor
|
||||
} // namespace tensorflow
|
||||
|
||||
#endif // TENSORFLOW_CORE_KERNELS_DETERMINANT_OP_H_
|
||||
@@ -0,0 +1,43 @@
|
||||
/* Copyright 2017 The TensorFlow Authors. All Rights Reserved.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
==============================================================================*/
|
||||
|
||||
#ifndef TENSORFLOW_CORE_KERNELS_DIAG_OP_H_
|
||||
#define TENSORFLOW_CORE_KERNELS_DIAG_OP_H_
|
||||
|
||||
#include "tensorflow/core/framework/op_kernel.h"
|
||||
#include "tensorflow/core/framework/tensor_types.h"
|
||||
#include "tensorflow/core/platform/types.h"
|
||||
|
||||
namespace tensorflow {
|
||||
|
||||
namespace functor {
|
||||
|
||||
template <typename Device, typename T>
|
||||
struct DiagFunctor {
|
||||
Status operator()(OpKernelContext* context, const int64 size, const T* in,
|
||||
T* out);
|
||||
};
|
||||
|
||||
template <typename Device, typename T>
|
||||
struct DiagPartFunctor {
|
||||
Status operator()(OpKernelContext* context, const int64 size, const T* in,
|
||||
T* out);
|
||||
};
|
||||
|
||||
} // namespace functor
|
||||
|
||||
} // namespace tensorflow
|
||||
|
||||
#endif // TENSORFLOW_CORE_KERNELS_DIAG_OP_H_
|
||||
@@ -0,0 +1,66 @@
|
||||
/* Copyright 2016 The TensorFlow Authors. All Rights Reserved.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
==============================================================================*/
|
||||
|
||||
#ifndef TENSORFLOW_CORE_KERNELS_DILATION_OPS_H_
|
||||
#define TENSORFLOW_CORE_KERNELS_DILATION_OPS_H_
|
||||
|
||||
#include "third_party/eigen3/unsupported/Eigen/CXX11/Tensor"
|
||||
#include "tensorflow/core/framework/tensor_types.h"
|
||||
#include "tensorflow/core/platform/types.h"
|
||||
|
||||
namespace tensorflow {
|
||||
namespace functor {
|
||||
|
||||
template <typename Device, typename T>
|
||||
struct Dilation {
|
||||
// We assume that the tensor sizes are correct.
|
||||
void operator()(const Device& d, typename TTypes<T, 4>::ConstTensor input,
|
||||
typename TTypes<T, 3>::ConstTensor filter, int stride_rows,
|
||||
int stride_cols, int rate_rows, int rate_cols, int pad_top,
|
||||
int pad_left, typename TTypes<T, 4>::Tensor output);
|
||||
};
|
||||
|
||||
template <typename Device, typename T>
|
||||
struct DilationBackpropInput {
|
||||
// We assume that the tensor sizes are correct.
|
||||
// To avoid storing the argmax values during forward computation, we recompute
|
||||
// the argmax during backward computation, which is the reason why we provide
|
||||
// filter as argument to the backward computation routine.
|
||||
void operator()(const Device& d, typename TTypes<T, 4>::ConstTensor input,
|
||||
typename TTypes<T, 3>::ConstTensor filter,
|
||||
typename TTypes<T, 4>::ConstTensor out_backprop,
|
||||
int stride_rows, int stride_cols, int rate_rows,
|
||||
int rate_cols, int pad_top, int pad_left,
|
||||
typename TTypes<T, 4>::Tensor in_backprop);
|
||||
};
|
||||
|
||||
template <typename Device, typename T>
|
||||
struct DilationBackpropFilter {
|
||||
// We assume that the tensor sizes are correct.
|
||||
// To avoid storing the argmax values during forward computation, we recompute
|
||||
// the argmax during backward computation, which is the reason why we provide
|
||||
// filter as argument to the backward computation routine.
|
||||
void operator()(const Device& d, typename TTypes<T, 4>::ConstTensor input,
|
||||
typename TTypes<T, 3>::ConstTensor filter,
|
||||
typename TTypes<T, 4>::ConstTensor out_backprop,
|
||||
int stride_rows, int stride_cols, int rate_rows,
|
||||
int rate_cols, int pad_top, int pad_left,
|
||||
typename TTypes<T, 3>::Tensor filter_backprop);
|
||||
};
|
||||
|
||||
} // namespace functor
|
||||
} // namespace tensorflow
|
||||
|
||||
#endif // TENSORFLOW_CORE_KERNELS_DILATION_OPS_H_
|
||||
@@ -0,0 +1,125 @@
|
||||
/* Copyright 2015 The TensorFlow Authors. All Rights Reserved.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
==============================================================================*/
|
||||
|
||||
#ifndef TENSORFLOW_CORE_KERNELS_EIGEN_ACTIVATIONS_H_
|
||||
#define TENSORFLOW_CORE_KERNELS_EIGEN_ACTIVATIONS_H_
|
||||
|
||||
#include "third_party/eigen3/unsupported/Eigen/CXX11/Tensor"
|
||||
|
||||
namespace Eigen {
|
||||
|
||||
/** scalar_sigmoid_fast_derivative_op
|
||||
* \ingroup CXX11_NeuralNetworks_Module
|
||||
* \brief Template functor to compute the fast derivative of a sigmoid
|
||||
*
|
||||
* Input should be the backpropagated gradient.
|
||||
*
|
||||
* \sa class CwiseUnaryOp, Cwise::sigmoid_fast_derivative()
|
||||
*/
|
||||
template <typename T>
|
||||
struct scalar_sigmoid_fast_derivative_op {
|
||||
EIGEN_EMPTY_STRUCT_CTOR(scalar_sigmoid_fast_derivative_op)
|
||||
EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE T operator()(const T& y) const {
|
||||
const T one = T(1);
|
||||
return (one - y) * y;
|
||||
}
|
||||
|
||||
template <typename Packet>
|
||||
inline Packet packetOp(const Packet& y) const {
|
||||
const Packet one = internal::pset1<Packet>(1);
|
||||
return internal::pmul(internal::psub(one, y), y);
|
||||
}
|
||||
};
|
||||
|
||||
namespace internal {
|
||||
template <typename T>
|
||||
struct functor_traits<scalar_sigmoid_fast_derivative_op<T> > {
|
||||
enum {
|
||||
Cost = NumTraits<T>::AddCost * 2 + NumTraits<T>::MulCost,
|
||||
PacketAccess = packet_traits<T>::HasAdd && packet_traits<T>::HasMul &&
|
||||
packet_traits<T>::HasNegate
|
||||
};
|
||||
};
|
||||
} // namespace internal
|
||||
|
||||
/** scalar_tanh_fast_derivative_op
|
||||
* \ingroup CXX11_NeuralNetworks_Module
|
||||
* \brief Template functor to compute the fast derivative of a tanh
|
||||
*
|
||||
* Input should be the backpropagated gradient.
|
||||
*
|
||||
* \sa class CwiseUnaryOp, Cwise::tanh_fast_derivative()
|
||||
*/
|
||||
template <typename T>
|
||||
struct scalar_tanh_fast_derivative_op {
|
||||
EIGEN_EMPTY_STRUCT_CTOR(scalar_tanh_fast_derivative_op)
|
||||
EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE T operator()(const T& y) const {
|
||||
const T one = T(1);
|
||||
return one - (y * y);
|
||||
}
|
||||
|
||||
template <typename Packet>
|
||||
inline Packet packetOp(const Packet& y) const {
|
||||
const Packet one = internal::pset1<Packet>(1);
|
||||
return internal::psub(one, internal::pmul(y, y));
|
||||
}
|
||||
};
|
||||
|
||||
namespace internal {
|
||||
template <typename T>
|
||||
struct functor_traits<scalar_tanh_fast_derivative_op<T> > {
|
||||
enum {
|
||||
Cost = NumTraits<T>::AddCost * 2 + NumTraits<T>::MulCost * 1,
|
||||
PacketAccess = packet_traits<T>::HasAdd && packet_traits<T>::HasMul &&
|
||||
packet_traits<T>::HasNegate
|
||||
};
|
||||
};
|
||||
} // namespace internal
|
||||
|
||||
/**
|
||||
* \ingroup CXX11_NeuralNetworks_Module
|
||||
* \brief Template functor to clip the magnitude of the first scalar.
|
||||
*
|
||||
* \sa class CwiseBinaryOp, MatrixBase::Clip
|
||||
*/
|
||||
template <typename Scalar>
|
||||
struct scalar_clip_op {
|
||||
EIGEN_EMPTY_STRUCT_CTOR(scalar_clip_op)
|
||||
EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const Scalar
|
||||
operator()(const Scalar& a, const Scalar& b) const {
|
||||
return numext::mini(numext::maxi(a, -b), b);
|
||||
}
|
||||
template <typename Packet>
|
||||
EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const Packet
|
||||
packetOp(const Packet& a, const Packet& b) const {
|
||||
return internal::pmin(internal::pmax(a, internal::pnegate(b)), b);
|
||||
}
|
||||
};
|
||||
|
||||
namespace internal {
|
||||
template <typename Scalar>
|
||||
struct functor_traits<scalar_clip_op<Scalar> > {
|
||||
enum {
|
||||
Cost = NumTraits<Scalar>::AddCost * 3,
|
||||
PacketAccess = packet_traits<Scalar>::HasMax &&
|
||||
packet_traits<Scalar>::HasMin &&
|
||||
packet_traits<Scalar>::HasNegate
|
||||
};
|
||||
};
|
||||
} // namespace internal
|
||||
|
||||
} // end namespace Eigen
|
||||
|
||||
#endif // TENSORFLOW_CORE_KERNELS_EIGEN_ACTIVATIONS_H_
|
||||
@@ -0,0 +1,255 @@
|
||||
/* Copyright 2015 The TensorFlow Authors. All Rights Reserved.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
==============================================================================*/
|
||||
|
||||
#ifndef TENSORFLOW_CORE_KERNELS_EIGEN_ATTENTION_H_
|
||||
#define TENSORFLOW_CORE_KERNELS_EIGEN_ATTENTION_H_
|
||||
|
||||
#include "third_party/eigen3/unsupported/Eigen/CXX11/Tensor"
|
||||
|
||||
namespace Eigen {
|
||||
|
||||
/** ExtractGlimpses
|
||||
* \ingroup CXX11_NeuralNetworks_Module
|
||||
*
|
||||
* \brief Extract glimpses from an input tensor.
|
||||
*
|
||||
* The input parameter is expected to be a col-major tensor with a rank of 4
|
||||
* (depth, x, y, and batch). The width and height parameters specify the
|
||||
* extension of the returned glimpses. The offsets parameter specifies the x, y
|
||||
* locations of the center of the glimpses relative to the center of the input
|
||||
* image. The vector is expected to contain one IndexPair for each image in the
|
||||
* batch dimension. The normalized boolean indicates if incoming coordinates are
|
||||
* normalized so that 0.0 and 1.0 correspond to the minimum and maximum of each
|
||||
* height and width dimension. The centered boolean indicates if incoming
|
||||
* coordinates are centered relative to the image, in which case -1.0 and 1.0
|
||||
* correspond to minimum and maximum of each dimension while 0.0 corresponds to
|
||||
* the center.
|
||||
*
|
||||
* The result can be assigned to a tensor of rank equal to that of the input.
|
||||
* The result will be laid out in col-major order (depth, x, y, batch). The
|
||||
* dimensions of the result will be equal to the dimensions of the input except
|
||||
* for width and height which will be equal to the requested glimpse size.
|
||||
*/
|
||||
namespace {
|
||||
template <typename Index>
|
||||
struct GlimpseExtractionOp {
|
||||
GlimpseExtractionOp(const Index width, const Index height,
|
||||
const std::vector<IndexPair<float> >& offsets,
|
||||
const bool normalized, const bool centered,
|
||||
const bool uniform_noise)
|
||||
: width_(width),
|
||||
height_(height),
|
||||
offsets_(offsets),
|
||||
normalized_(normalized),
|
||||
centered_(centered),
|
||||
uniform_noise_(uniform_noise) {}
|
||||
|
||||
template <typename Input>
|
||||
DSizes<Index, 4> dimensions(const Input& input) const {
|
||||
typedef typename internal::traits<Input>::Index IndexType;
|
||||
typedef TensorRef<Tensor<typename internal::traits<Input>::Scalar, 4,
|
||||
internal::traits<Input>::Layout, IndexType> >
|
||||
Ref;
|
||||
Ref in(input);
|
||||
|
||||
DSizes<Index, 4> dims = in.dimensions();
|
||||
|
||||
dims[0] = in.dimension(0);
|
||||
dims[1] = width_;
|
||||
dims[2] = height_;
|
||||
dims[3] = in.dimension(3);
|
||||
return dims;
|
||||
}
|
||||
|
||||
template <typename Input, typename Output, typename Device>
|
||||
EIGEN_DEVICE_FUNC void eval(const Input& input, Output& output,
|
||||
const Device& device) const {
|
||||
typedef typename internal::traits<Input>::Index IndexType;
|
||||
typedef TensorRef<Tensor<typename internal::traits<Input>::Scalar, 4,
|
||||
internal::traits<Input>::Layout, IndexType> >
|
||||
Ref;
|
||||
Ref in(input);
|
||||
const Index num_channels = in.dimension(0);
|
||||
const Index input_width = in.dimension(1);
|
||||
const Index input_height = in.dimension(2);
|
||||
const Index batch_size = in.dimension(3);
|
||||
eigen_assert(input_width > 0);
|
||||
eigen_assert(input_height > 0);
|
||||
internal::NormalRandomGenerator<float> gen;
|
||||
internal::UniformRandomGenerator<float> unigen;
|
||||
|
||||
for (Index i = 0; i < batch_size; ++i) {
|
||||
float x = offsets_[i].first, y = offsets_[i].second;
|
||||
|
||||
// Un-normalize coordinates back to pixel space if normalized.
|
||||
if (normalized_) {
|
||||
x *= input_width;
|
||||
y *= input_height;
|
||||
}
|
||||
// Un-center if coordinates are centered on the image center.
|
||||
if (centered_) {
|
||||
x /= 2.0f;
|
||||
y /= 2.0f;
|
||||
x += input_width / 2.0f;
|
||||
y += input_height / 2.0f;
|
||||
}
|
||||
// Remove half of the glimpse window.
|
||||
x -= width_ / 2.0f;
|
||||
y -= height_ / 2.0f;
|
||||
|
||||
const Index offset_x = (Index)x;
|
||||
const Index offset_y = (Index)y;
|
||||
Index glimpse_width = width_;
|
||||
Index glimpse_height = height_;
|
||||
bool partial_overlap = false;
|
||||
DSizes<Index, 3> slice_offset(0, offset_x, offset_y);
|
||||
DSizes<Index, 3> slice_extent(num_channels, width_, height_);
|
||||
DSizes<Index, 3> base_offset(0, 0, 0);
|
||||
|
||||
if (offset_x < 0) {
|
||||
slice_offset[1] = 0;
|
||||
glimpse_width = (std::max<Index>)(0, width_ + offset_x);
|
||||
slice_extent[1] = glimpse_width;
|
||||
base_offset[1] = width_ - glimpse_width;
|
||||
partial_overlap = true;
|
||||
} else if (offset_x + width_ >= input_width) {
|
||||
glimpse_width = (std::max<Index>)(0, input_width - offset_x);
|
||||
slice_extent[1] = glimpse_width;
|
||||
partial_overlap = true;
|
||||
}
|
||||
if (offset_y < 0) {
|
||||
slice_offset[2] = 0;
|
||||
glimpse_height = (std::max<Index>)(0, height_ + offset_y);
|
||||
slice_extent[2] = glimpse_height;
|
||||
base_offset[2] = height_ - glimpse_height;
|
||||
partial_overlap = true;
|
||||
} else if (offset_y + height_ >= input_height) {
|
||||
glimpse_height = (std::max<Index>)(0, input_height - offset_y);
|
||||
slice_extent[2] = glimpse_height;
|
||||
partial_overlap = true;
|
||||
}
|
||||
slice_extent[1] = std::min<Index>(input_width, slice_extent[1]);
|
||||
slice_extent[2] = std::min<Index>(input_height, slice_extent[2]);
|
||||
|
||||
if (partial_overlap) {
|
||||
if (uniform_noise_) {
|
||||
// Initialize the glimpse with uniform noise.
|
||||
typedef typename internal::remove_const<
|
||||
typename internal::traits<Input>::Scalar>::type Scalar;
|
||||
TensorFixedSize<Scalar, Sizes<> > mini;
|
||||
mini.device(device) = input.template chip<3>(i).minimum();
|
||||
TensorFixedSize<float, Sizes<> > range;
|
||||
range.device(device) = (input.template chip<3>(i).maximum() - mini)
|
||||
.template cast<float>();
|
||||
|
||||
DSizes<Index, 3> glimpse_size(num_channels, width_, height_);
|
||||
TensorMap<Tensor<float, 3> > tmp(NULL, glimpse_size);
|
||||
output.template chip<3>(i).device(device) =
|
||||
mini.reshape(Sizes<1, 1, 1>()).broadcast(glimpse_size) +
|
||||
(tmp.random(unigen) *
|
||||
range.reshape(Sizes<1, 1, 1>()).broadcast(glimpse_size))
|
||||
.template cast<Scalar>();
|
||||
} else {
|
||||
// Initialize the glimpse with white noise: compute the mean and sigma
|
||||
// of each channel, and use them to shape the gaussian.
|
||||
DSizes<Index, 2> glimpse_size(width_, height_);
|
||||
DSizes<Index, 2> input_size(input_width, input_height);
|
||||
typedef typename internal::remove_const<
|
||||
typename internal::traits<Input>::Scalar>::type Scalar;
|
||||
|
||||
for (int j = 0; j < num_channels; ++j) {
|
||||
TensorFixedSize<Scalar, Sizes<> > mean;
|
||||
mean.device(device) = input.template chip<3>(i)
|
||||
.template chip<0>(j)
|
||||
.template cast<float>()
|
||||
.mean();
|
||||
TensorFixedSize<float, Sizes<> > sigma;
|
||||
sigma.device(device) =
|
||||
(input.template chip<3>(i)
|
||||
.template chip<0>(j)
|
||||
.template cast<float>() -
|
||||
mean.reshape(Sizes<1, 1>()).broadcast(input_size))
|
||||
.square()
|
||||
.mean()
|
||||
.sqrt();
|
||||
TensorFixedSize<Scalar, Sizes<> > mini;
|
||||
mini.device(device) =
|
||||
input.template chip<3>(i).template chip<0>(j).minimum();
|
||||
TensorFixedSize<float, Sizes<> > maxi;
|
||||
maxi.device(device) =
|
||||
input.template chip<3>(i).template chip<0>(j).maximum();
|
||||
|
||||
TensorMap<Tensor<float, 2> > tmp(NULL, glimpse_size);
|
||||
output.template chip<3>(i).template chip<0>(j).device(device) =
|
||||
(mean.reshape(Sizes<1, 1>()).broadcast(glimpse_size) +
|
||||
(tmp.random(gen) *
|
||||
sigma.reshape(Sizes<1, 1>()).broadcast(glimpse_size))
|
||||
.template cast<Scalar>())
|
||||
.cwiseMin(
|
||||
maxi.reshape(Sizes<1, 1>()).broadcast(glimpse_size))
|
||||
.cwiseMax(
|
||||
mini.reshape(Sizes<1, 1>()).broadcast(glimpse_size));
|
||||
}
|
||||
}
|
||||
|
||||
// Copy the part of the glimpse that cover the input image if any.
|
||||
if (glimpse_width == 0 || glimpse_height == 0) {
|
||||
continue;
|
||||
}
|
||||
output.template chip<3>(i)
|
||||
.slice(base_offset, slice_extent)
|
||||
.device(device) =
|
||||
input.template chip<3>(i).slice(slice_offset, slice_extent);
|
||||
} else {
|
||||
output.template chip<3>(i).device(device) =
|
||||
input.template chip<3>(i).slice(slice_offset, slice_extent);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private:
|
||||
const Index width_;
|
||||
const Index height_;
|
||||
const std::vector<IndexPair<float> > offsets_;
|
||||
const bool normalized_;
|
||||
const bool centered_;
|
||||
const bool uniform_noise_;
|
||||
};
|
||||
} // namespace
|
||||
|
||||
template <typename Input>
|
||||
EIGEN_ALWAYS_INLINE static const TensorCustomUnaryOp<
|
||||
const GlimpseExtractionOp<typename internal::traits<Input>::Index>,
|
||||
const Input>
|
||||
ExtractGlimpses(const Input& input,
|
||||
const typename internal::traits<Input>::Index width,
|
||||
const typename internal::traits<Input>::Index height,
|
||||
const std::vector<IndexPair<float> >& offsets,
|
||||
const bool normalized = true, const bool centered = true,
|
||||
const bool uniform_noise = true) {
|
||||
EIGEN_STATIC_ASSERT(internal::traits<Input>::Layout == ColMajor,
|
||||
YOU_MADE_A_PROGRAMMING_MISTAKE);
|
||||
EIGEN_STATIC_ASSERT(internal::traits<Input>::NumDimensions == 4,
|
||||
YOU_MADE_A_PROGRAMMING_MISTAKE);
|
||||
|
||||
typedef typename internal::traits<Input>::Index Index;
|
||||
const GlimpseExtractionOp<Index> op(width, height, offsets, normalized,
|
||||
centered, uniform_noise);
|
||||
return input.customOp(op);
|
||||
}
|
||||
|
||||
} // end namespace Eigen
|
||||
|
||||
#endif // TENSORFLOW_CORE_KERNELS_EIGEN_ATTENTION_H_
|
||||
@@ -0,0 +1,610 @@
|
||||
/* Copyright 2015 The TensorFlow Authors. All Rights Reserved.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
==============================================================================*/
|
||||
|
||||
#ifndef TENSORFLOW_CORE_KERNELS_EIGEN_BACKWARD_CUBOID_CONVOLUTIONS_H_
|
||||
#define TENSORFLOW_CORE_KERNELS_EIGEN_BACKWARD_CUBOID_CONVOLUTIONS_H_
|
||||
|
||||
#include "third_party/eigen3/unsupported/Eigen/CXX11/Tensor"
|
||||
#include "tensorflow/core/kernels/eigen_volume_patch.h"
|
||||
|
||||
namespace Eigen {
|
||||
|
||||
/** CuboidConvolutionBackwardInput
|
||||
* \ingroup CXX11_NeuralNetworks_Module
|
||||
*
|
||||
* \brief Computes the backprop for the input of a 3D convolution.
|
||||
*
|
||||
* The output_backward parameter is expected to be a tensor with a rank of 4 or
|
||||
* more (channels, depth, height, width, and optionally others)
|
||||
* The kernel parameter is expected to be a 5D tensor (filters, channels,
|
||||
* kernel_depth, kernel_height, kernel_width)
|
||||
* output_backward and kernel have to be in the same layout.
|
||||
*
|
||||
* The dimensions of the result will be filters, depth, height, width (and
|
||||
* others if applicable).
|
||||
*
|
||||
* It is possible to swap the order of the depth, width and height dimensions
|
||||
* provided that the same order is used in the input, the kernel, and the
|
||||
* output.
|
||||
*
|
||||
* All dimension orders above are given for col-major, and should be reversed
|
||||
* for row-major.
|
||||
*/
|
||||
|
||||
template <typename OutputBackward, typename Kernel>
|
||||
EIGEN_ALWAYS_INLINE static const typename internal::conditional<
|
||||
internal::traits<OutputBackward>::Layout == ColMajor,
|
||||
TensorReshapingOp<
|
||||
const DSizes<typename internal::traits<OutputBackward>::Index,
|
||||
internal::traits<OutputBackward>::NumDimensions>,
|
||||
const TensorContractionOp<
|
||||
const array<
|
||||
IndexPair<typename internal::traits<OutputBackward>::Index>, 1>,
|
||||
const Eigen::TensorForcedEvalOp<const TensorReshapingOp<
|
||||
const DSizes<typename internal::traits<OutputBackward>::Index,
|
||||
2>,
|
||||
const TensorShufflingOp<
|
||||
const array<
|
||||
typename internal::traits<OutputBackward>::Index, 5>,
|
||||
const TensorReverseOp<const Eigen::array<bool, 5>,
|
||||
const Kernel>>>>,
|
||||
const TensorReshapingOp<
|
||||
const DSizes<typename internal::traits<OutputBackward>::Index,
|
||||
2>,
|
||||
const TensorVolumePatchOp<Dynamic, Dynamic, Dynamic,
|
||||
const OutputBackward>>>>,
|
||||
TensorReshapingOp<
|
||||
const DSizes<typename internal::traits<OutputBackward>::Index,
|
||||
internal::traits<OutputBackward>::NumDimensions>,
|
||||
const TensorContractionOp<
|
||||
const array<
|
||||
IndexPair<typename internal::traits<OutputBackward>::Index>, 1>,
|
||||
const TensorReshapingOp<
|
||||
const DSizes<typename internal::traits<OutputBackward>::Index,
|
||||
2>,
|
||||
const TensorVolumePatchOp<Dynamic, Dynamic, Dynamic,
|
||||
const OutputBackward>>,
|
||||
const Eigen::TensorForcedEvalOp<const TensorReshapingOp<
|
||||
const DSizes<typename internal::traits<OutputBackward>::Index,
|
||||
2>,
|
||||
const TensorShufflingOp<
|
||||
const array<
|
||||
typename internal::traits<OutputBackward>::Index, 5>,
|
||||
const TensorReverseOp<const Eigen::array<bool, 5>,
|
||||
const Kernel>>>>>>>::type
|
||||
CuboidConvolutionBackwardInput(
|
||||
const Kernel& kernel, const OutputBackward& output_backward,
|
||||
typename internal::traits<OutputBackward>::Index inputPlanes,
|
||||
typename internal::traits<OutputBackward>::Index inputRows,
|
||||
typename internal::traits<OutputBackward>::Index inputCols,
|
||||
const DenseIndex plane_stride = 1, const DenseIndex row_stride = 1,
|
||||
const DenseIndex col_stride = 1) {
|
||||
typedef typename internal::traits<OutputBackward>::Index TensorIndex;
|
||||
const TensorRef<const Tensor<typename internal::traits<Kernel>::Scalar,
|
||||
internal::traits<Kernel>::NumDimensions,
|
||||
internal::traits<Kernel>::Layout, TensorIndex>>
|
||||
kern(kernel);
|
||||
const TensorRef<
|
||||
const Tensor<typename internal::traits<OutputBackward>::Scalar,
|
||||
internal::traits<OutputBackward>::NumDimensions,
|
||||
internal::traits<OutputBackward>::Layout, TensorIndex>>
|
||||
out(output_backward);
|
||||
|
||||
EIGEN_STATIC_ASSERT(internal::traits<Kernel>::Layout ==
|
||||
internal::traits<OutputBackward>::Layout,
|
||||
YOU_MADE_A_PROGRAMMING_MISTAKE);
|
||||
|
||||
static const bool isColMajor =
|
||||
(internal::traits<OutputBackward>::Layout == ColMajor);
|
||||
|
||||
static const int NumDims = internal::traits<OutputBackward>::NumDimensions;
|
||||
|
||||
// Number of filters to apply. This is the same as the output depth of the
|
||||
// result
|
||||
const TensorIndex kernelFilters =
|
||||
isColMajor ? kern.dimensions()[0] : kern.dimensions()[4];
|
||||
// Number of channels. This is the same as the input depth.
|
||||
const TensorIndex kernelChannels =
|
||||
isColMajor ? kern.dimensions()[1] : kern.dimensions()[3];
|
||||
const TensorIndex kernelPlanes =
|
||||
isColMajor ? kern.dimensions()[2] : kern.dimensions()[2];
|
||||
const TensorIndex kernelRows =
|
||||
isColMajor ? kern.dimensions()[3] : kern.dimensions()[1];
|
||||
const TensorIndex kernelCols =
|
||||
isColMajor ? kern.dimensions()[4] : kern.dimensions()[0];
|
||||
|
||||
const TensorIndex outputPlanes =
|
||||
isColMajor ? out.dimensions()[1] : out.dimensions()[NumDims - 2];
|
||||
const TensorIndex outputRows =
|
||||
isColMajor ? out.dimensions()[2] : out.dimensions()[NumDims - 3];
|
||||
const TensorIndex outputCols =
|
||||
isColMajor ? out.dimensions()[3] : out.dimensions()[NumDims - 4];
|
||||
|
||||
// TODO(ezhulenev): Add support for inflated strides. Without inflated strides
|
||||
// effective kernel planes/rows/cols are always the same as the kernel itself
|
||||
// (see eigen_spatial_convolutions for details).
|
||||
const TensorIndex kernelPlanesEff = kernelPlanes;
|
||||
const TensorIndex kernelRowsEff = kernelRows;
|
||||
const TensorIndex kernelColsEff = kernelCols;
|
||||
|
||||
// Computing the forward padding.
|
||||
const TensorIndex forward_pad_top_z = numext::maxi<Index>(
|
||||
0,
|
||||
((outputPlanes - 1) * plane_stride + kernelPlanesEff - inputPlanes) / 2);
|
||||
const TensorIndex forward_pad_top = numext::maxi<Index>(
|
||||
0, ((outputRows - 1) * row_stride + kernelRowsEff - inputRows) / 2);
|
||||
const TensorIndex forward_pad_left = numext::maxi<Index>(
|
||||
0, ((outputCols - 1) * col_stride + kernelColsEff - inputCols) / 2);
|
||||
|
||||
const TensorIndex padding_top_z = kernelPlanesEff - 1 - forward_pad_top_z;
|
||||
const TensorIndex padding_top = kernelRowsEff - 1 - forward_pad_top;
|
||||
const TensorIndex padding_left = kernelColsEff - 1 - forward_pad_left;
|
||||
|
||||
const TensorIndex padding_bottom_z = inputPlanes -
|
||||
(outputPlanes - 1) * plane_stride - 2 -
|
||||
padding_top_z + kernelPlanesEff;
|
||||
const TensorIndex padding_bottom = inputRows - (outputRows - 1) * row_stride -
|
||||
2 - padding_top + kernelRowsEff;
|
||||
const TensorIndex padding_right = inputCols - (outputCols - 1) * col_stride -
|
||||
2 - padding_left + kernelColsEff;
|
||||
|
||||
eigen_assert(padding_top_z >= 0);
|
||||
eigen_assert(padding_top >= 0);
|
||||
eigen_assert(padding_left >= 0);
|
||||
eigen_assert(padding_bottom_z >= 0);
|
||||
eigen_assert(padding_bottom >= 0);
|
||||
eigen_assert(padding_right >= 0);
|
||||
|
||||
// The kernel has dimensions :
|
||||
// filters x channels x patch_planes x patch_rows x patch_cols.
|
||||
// We need to reverse the kernel along the spatial dimensions.
|
||||
Eigen::array<bool, 5> kernel_reverse;
|
||||
if (isColMajor) {
|
||||
kernel_reverse[0] = false;
|
||||
kernel_reverse[1] = false;
|
||||
kernel_reverse[2] = true;
|
||||
kernel_reverse[3] = true;
|
||||
kernel_reverse[4] = true;
|
||||
} else {
|
||||
kernel_reverse[0] = true;
|
||||
kernel_reverse[1] = true;
|
||||
kernel_reverse[2] = true;
|
||||
kernel_reverse[3] = false;
|
||||
kernel_reverse[4] = false;
|
||||
}
|
||||
|
||||
// Reorder the dimensions to:
|
||||
// filters x patch_planes x patch_rows x patch_cols x channels
|
||||
array<TensorIndex, 5> kernel_shuffle;
|
||||
if (isColMajor) {
|
||||
// From: filters x channels x planes x rows x cols
|
||||
// To: filters x planes x rows x cols x channels
|
||||
kernel_shuffle[0] = 0;
|
||||
kernel_shuffle[1] = 2;
|
||||
kernel_shuffle[2] = 3;
|
||||
kernel_shuffle[3] = 4;
|
||||
kernel_shuffle[4] = 1;
|
||||
} else {
|
||||
// From: cols x rows x planes x channels x filters
|
||||
// To: channels x cols x rows x planes x filters
|
||||
kernel_shuffle[0] = 3;
|
||||
kernel_shuffle[1] = 0;
|
||||
kernel_shuffle[2] = 1;
|
||||
kernel_shuffle[3] = 2;
|
||||
kernel_shuffle[4] = 4;
|
||||
}
|
||||
|
||||
// Collapse the dims
|
||||
DSizes<TensorIndex, 2> kernel_dims;
|
||||
if (isColMajor) {
|
||||
kernel_dims[0] = kernelFilters * kernelPlanes * kernelRows * kernelCols;
|
||||
kernel_dims[1] = kernelChannels;
|
||||
} else {
|
||||
kernel_dims[1] = kernelFilters * kernelPlanes * kernelRows * kernelCols;
|
||||
kernel_dims[0] = kernelChannels;
|
||||
}
|
||||
|
||||
// The output_backward has dimensions out_depth X out_planes X out_rows X
|
||||
// out_cols X OTHERS
|
||||
// When we extract the image patches from output_backward, it will have
|
||||
// dimensions:
|
||||
// out_depth X (patch_planes * patch_rows * patch_cols) X (input_planes *
|
||||
// input_rows * input_cols * OTHERS)
|
||||
DSizes<TensorIndex, 2> pre_contract_dims;
|
||||
if (isColMajor) {
|
||||
pre_contract_dims[0] =
|
||||
kernelFilters * kernelPlanes * kernelRows * kernelCols;
|
||||
pre_contract_dims[1] = inputPlanes * inputRows * inputCols;
|
||||
for (int i = 4; i < NumDims; ++i) {
|
||||
pre_contract_dims[1] *= out.dimension(i);
|
||||
}
|
||||
} else {
|
||||
pre_contract_dims[1] =
|
||||
kernelFilters * kernelPlanes * kernelRows * kernelCols;
|
||||
pre_contract_dims[0] = inputPlanes * inputRows * inputCols;
|
||||
for (int i = 0; i < NumDims - 4; ++i) {
|
||||
pre_contract_dims[0] *= out.dimension(i);
|
||||
}
|
||||
}
|
||||
|
||||
// We will contract along the collapsed dimension that contains the
|
||||
// kernelFilters, kernelPlanes, kernelRows and kernelCols.
|
||||
array<IndexPair<TensorIndex>, 1> contract_dims;
|
||||
if (isColMajor) {
|
||||
// col-major: kernel.contract(output.patches)
|
||||
contract_dims[0] = IndexPair<TensorIndex>(0, 0);
|
||||
} else {
|
||||
// row-major: output.patches.contract(kernel)
|
||||
contract_dims[0] = IndexPair<TensorIndex>(1, 1);
|
||||
}
|
||||
|
||||
// Post contraction, the dimensions of the input_backprop is
|
||||
// channels X input_planes X input_rows X input_cols X OTHERS
|
||||
DSizes<TensorIndex, NumDims> post_contract_dims;
|
||||
if (isColMajor) {
|
||||
post_contract_dims[0] = kernelChannels;
|
||||
post_contract_dims[1] = inputPlanes;
|
||||
post_contract_dims[2] = inputRows;
|
||||
post_contract_dims[3] = inputCols;
|
||||
for (int i = 4; i < NumDims; ++i) {
|
||||
post_contract_dims[i] = out.dimension(i);
|
||||
}
|
||||
} else {
|
||||
post_contract_dims[NumDims - 1] = kernelChannels;
|
||||
post_contract_dims[NumDims - 2] = inputPlanes;
|
||||
post_contract_dims[NumDims - 3] = inputRows;
|
||||
post_contract_dims[NumDims - 4] = inputCols;
|
||||
for (int i = 0; i < NumDims - 4; ++i) {
|
||||
post_contract_dims[i] = out.dimension(i);
|
||||
}
|
||||
}
|
||||
|
||||
return choose(
|
||||
Cond<internal::traits<OutputBackward>::Layout == ColMajor>(),
|
||||
kernel.reverse(kernel_reverse)
|
||||
.shuffle(kernel_shuffle)
|
||||
.reshape(kernel_dims)
|
||||
.eval()
|
||||
.contract(output_backward
|
||||
.extract_volume_patches(
|
||||
kernelPlanes, kernelRows, kernelCols, 1, 1, 1,
|
||||
plane_stride, row_stride, col_stride, padding_top_z,
|
||||
padding_bottom_z, padding_top, padding_bottom,
|
||||
padding_left, padding_right)
|
||||
.reshape(pre_contract_dims),
|
||||
contract_dims)
|
||||
.reshape(post_contract_dims),
|
||||
output_backward
|
||||
.extract_volume_patches(kernelPlanes, kernelRows, kernelCols, 1, 1, 1,
|
||||
plane_stride, row_stride, col_stride,
|
||||
padding_top_z, padding_bottom_z, padding_top,
|
||||
padding_bottom, padding_left, padding_right)
|
||||
.reshape(pre_contract_dims)
|
||||
.contract(kernel.reverse(kernel_reverse)
|
||||
.shuffle(kernel_shuffle)
|
||||
.reshape(kernel_dims)
|
||||
.eval(),
|
||||
contract_dims)
|
||||
.reshape(post_contract_dims));
|
||||
}
|
||||
|
||||
/** CuboidConvolutionBackwardKernel
|
||||
* \ingroup CXX11_NeuralNetworks_Module
|
||||
*
|
||||
* \brief Computes the backprop for the filter of a 3D convolution.
|
||||
*
|
||||
* The output_backward parameter is expected to be a tensor with a rank of 4 or
|
||||
* more (channels, depth, height, width, and optionally others)
|
||||
* The kernel parameter is expected to be a 4D tensor (filters, channels,
|
||||
* kernel_depth, kernel_height, kernel_width)
|
||||
* output_backward and kernel have to be in the same layout.
|
||||
*
|
||||
* The dimensions of the result will be filters, depth, height, width (and
|
||||
* others if applicable).
|
||||
*
|
||||
* It is possible to swap the order of the depth, width and height dimensions
|
||||
* provided that the same order is used in the input, the kernel, and the
|
||||
* output.
|
||||
*
|
||||
* All dimension orders above are given for col-major, and should be reversed
|
||||
* for row-major.
|
||||
*/
|
||||
template <typename OutputBackward, typename Input>
|
||||
EIGEN_ALWAYS_INLINE static const typename internal::conditional<
|
||||
internal::traits<Input>::Layout == ColMajor,
|
||||
const TensorReverseOp<
|
||||
const Eigen::array<typename internal::traits<Input>::Index,
|
||||
internal::traits<Input>::NumDimensions>,
|
||||
const Eigen::TensorShufflingOp<
|
||||
const Eigen::array<typename internal::traits<Input>::Index,
|
||||
internal::traits<Input>::NumDimensions>,
|
||||
const Eigen::TensorReshapingOp<
|
||||
const Eigen::DSizes<typename internal::traits<Input>::Index,
|
||||
internal::traits<Input>::NumDimensions>,
|
||||
const TensorContractionOp<
|
||||
const array<
|
||||
IndexPair<typename internal::traits<Input>::Index>, 1>,
|
||||
const Eigen::TensorForcedEvalOp<const TensorReshapingOp<
|
||||
const DSizes<typename internal::traits<Input>::Index,
|
||||
2>,
|
||||
const Eigen::TensorShufflingOp<
|
||||
const Eigen::array<
|
||||
typename internal::traits<Input>::Index,
|
||||
internal::traits<Input>::NumDimensions>,
|
||||
const OutputBackward>>>,
|
||||
const TensorReshapingOp<
|
||||
const DSizes<typename internal::traits<Input>::Index,
|
||||
2>,
|
||||
const TensorVolumePatchOp<
|
||||
Dynamic, Dynamic, Dynamic,
|
||||
const Eigen::TensorForcedEvalOp<
|
||||
const Eigen::TensorShufflingOp<
|
||||
const Eigen::array<
|
||||
typename internal::traits<Input>::Index,
|
||||
internal::traits<Input>::NumDimensions>,
|
||||
const Input>>>>>>>>,
|
||||
const TensorReverseOp<
|
||||
const Eigen::array<typename internal::traits<Input>::Index,
|
||||
internal::traits<Input>::NumDimensions>,
|
||||
const Eigen::TensorShufflingOp<
|
||||
const Eigen::array<typename internal::traits<Input>::Index,
|
||||
internal::traits<Input>::NumDimensions>,
|
||||
const Eigen::TensorReshapingOp<
|
||||
const Eigen::DSizes<typename internal::traits<Input>::Index,
|
||||
internal::traits<Input>::NumDimensions>,
|
||||
const TensorContractionOp<
|
||||
const array<
|
||||
IndexPair<typename internal::traits<Input>::Index>, 1>,
|
||||
const TensorReshapingOp<
|
||||
const DSizes<typename internal::traits<Input>::Index,
|
||||
2>,
|
||||
const TensorVolumePatchOp<
|
||||
Dynamic, Dynamic, Dynamic,
|
||||
const Eigen::TensorForcedEvalOp<
|
||||
const Eigen::TensorShufflingOp<
|
||||
const Eigen::array<
|
||||
typename internal::traits<Input>::Index,
|
||||
internal::traits<Input>::NumDimensions>,
|
||||
const Input>>>>,
|
||||
const Eigen::TensorForcedEvalOp<const TensorReshapingOp<
|
||||
const DSizes<typename internal::traits<Input>::Index,
|
||||
2>,
|
||||
const Eigen::TensorShufflingOp<
|
||||
const Eigen::array<
|
||||
typename internal::traits<Input>::Index,
|
||||
internal::traits<Input>::NumDimensions>,
|
||||
const OutputBackward>>>>>>>>::type
|
||||
CuboidConvolutionBackwardKernel(
|
||||
const Input& input, const OutputBackward& output_backward,
|
||||
typename internal::traits<Input>::Index kernelPlanes,
|
||||
typename internal::traits<Input>::Index kernelRows,
|
||||
typename internal::traits<Input>::Index kernelCols,
|
||||
const DenseIndex stridePlanes = 1, const DenseIndex strideRows = 1,
|
||||
const DenseIndex strideCols = 1) {
|
||||
typedef typename internal::traits<Input>::Index TensorIndex;
|
||||
TensorRef<Tensor<typename internal::traits<Input>::Scalar,
|
||||
internal::traits<Input>::NumDimensions,
|
||||
internal::traits<Input>::Layout, TensorIndex>>
|
||||
in(input);
|
||||
TensorRef<Tensor<typename internal::traits<OutputBackward>::Scalar,
|
||||
internal::traits<OutputBackward>::NumDimensions,
|
||||
internal::traits<OutputBackward>::Layout, TensorIndex>>
|
||||
out(output_backward);
|
||||
|
||||
EIGEN_STATIC_ASSERT(internal::traits<Input>::Layout ==
|
||||
internal::traits<OutputBackward>::Layout,
|
||||
YOU_MADE_A_PROGRAMMING_MISTAKE);
|
||||
|
||||
static const bool isColMajor = (internal::traits<Input>::Layout == ColMajor);
|
||||
|
||||
static const int NumDims = internal::traits<Input>::NumDimensions;
|
||||
EIGEN_STATIC_ASSERT(internal::traits<Input>::NumDimensions ==
|
||||
internal::traits<OutputBackward>::NumDimensions,
|
||||
YOU_MADE_A_PROGRAMMING_MISTAKE);
|
||||
|
||||
// We do not support higher dimensional backward convolutions, or convolutions
|
||||
// without batch dimension.
|
||||
// TODO(ezhulenev): Relax this constraint, and turn on tests without batch
|
||||
// dimension in eigen_backward_cuboid_convolutions_test.cc.
|
||||
EIGEN_STATIC_ASSERT(internal::traits<Input>::NumDimensions == 5,
|
||||
YOU_MADE_A_PROGRAMMING_MISTAKE);
|
||||
|
||||
const TensorIndex inputPlanes =
|
||||
isColMajor ? in.dimension(1) : in.dimension(NumDims - 2);
|
||||
const TensorIndex inputRows =
|
||||
isColMajor ? in.dimension(2) : in.dimension(NumDims - 3);
|
||||
const TensorIndex inputCols =
|
||||
isColMajor ? in.dimension(3) : in.dimension(NumDims - 4);
|
||||
|
||||
const TensorIndex outputPlanes =
|
||||
isColMajor ? out.dimension(1) : out.dimension(NumDims - 2);
|
||||
const TensorIndex outputRows =
|
||||
isColMajor ? out.dimension(2) : out.dimension(NumDims - 3);
|
||||
const TensorIndex outputCols =
|
||||
isColMajor ? out.dimension(3) : out.dimension(NumDims - 4);
|
||||
|
||||
// Number of filters. This is the same as the output depth.
|
||||
const TensorIndex kernelFilters =
|
||||
isColMajor ? out.dimension(0) : out.dimension(NumDims - 1);
|
||||
// Number of channels. This is the same as the input depth.
|
||||
const TensorIndex kernelChannels =
|
||||
isColMajor ? in.dimension(0) : in.dimension(NumDims - 1);
|
||||
|
||||
// Number of batches in the input tensor.
|
||||
const TensorIndex batch =
|
||||
isColMajor ? in.dimension(4) : in.dimension(NumDims - 5);
|
||||
|
||||
// TODO(ezhulenev): Add support for inflated strides. Without inflated strides
|
||||
// effective kernel planes/rows/cols are always the same as the kernel itself
|
||||
// (see eigen_spatial_convolutions for details).
|
||||
const TensorIndex kernelPlanesEff = kernelPlanes;
|
||||
const TensorIndex kernelRowsEff = kernelRows;
|
||||
const TensorIndex kernelColsEff = kernelCols;
|
||||
|
||||
// Compute forward padding from input and output_backward dimensions.
|
||||
const TensorIndex padPlanes = numext::maxi<Index>(
|
||||
0, (outputPlanes - 1) * stridePlanes + kernelPlanesEff - inputPlanes);
|
||||
const TensorIndex padRows = numext::maxi<Index>(
|
||||
0, (outputRows - 1) * strideRows + kernelRowsEff - inputRows);
|
||||
const TensorIndex padCols = numext::maxi<Index>(
|
||||
0, (outputCols - 1) * strideCols + kernelColsEff - inputCols);
|
||||
|
||||
const TensorIndex padding_top_z = padPlanes / 2;
|
||||
const TensorIndex padding_top = padRows / 2;
|
||||
const TensorIndex padding_left = padCols / 2;
|
||||
|
||||
// Compute paddings for output_backward before extracting patches.
|
||||
const auto expanded_out_planes = (outputPlanes - 1) * stridePlanes + 1;
|
||||
const auto expanded_out_rows = (outputRows - 1) * strideRows + 1;
|
||||
const auto expanded_out_cols = (outputCols - 1) * strideCols + 1;
|
||||
const auto padded_out_planes = inputPlanes + kernelPlanes - 1;
|
||||
const auto padded_out_rows = inputRows + kernelRows - 1;
|
||||
const auto padded_out_cols = inputCols + kernelCols - 1;
|
||||
const auto top_pad_planes = kernelPlanes - 1 - padding_top_z;
|
||||
const auto top_pad_rows = kernelRows - 1 - padding_top;
|
||||
const auto left_pad_cols = kernelCols - 1 - padding_left;
|
||||
const auto bottom_pad_planes =
|
||||
padded_out_planes - expanded_out_planes - top_pad_planes;
|
||||
const auto bottom_pad_rows =
|
||||
padded_out_rows - expanded_out_rows - top_pad_rows;
|
||||
const auto right_pad_cols =
|
||||
padded_out_cols - expanded_out_cols - left_pad_cols;
|
||||
|
||||
// Reorder output_backward dimensions.
|
||||
array<TensorIndex, 5> output_backward_shuffle;
|
||||
if (isColMajor) {
|
||||
// From: [out_depth, out_planes, out_rows, out_cols, batch]
|
||||
// To: [batch, out_planes, out_rows, out_cols, out_depth]
|
||||
output_backward_shuffle = {4, 1, 2, 3, 0};
|
||||
} else {
|
||||
// From: [batch, out_cols, out_rows, out_planes, out_depth]
|
||||
// To: [out_depth, out_cols, out_rows, out_planes, batch]
|
||||
output_backward_shuffle = {4, 1, 2, 3, 0};
|
||||
}
|
||||
|
||||
// Reorder input dimensions.
|
||||
array<TensorIndex, 5> input_shuffle;
|
||||
if (isColMajor) {
|
||||
// From: [in_depth, in_planes, in_rows, in_cols, batch]
|
||||
// To: [in_depth, batch, in_planes, in_rows, in_cols]
|
||||
input_shuffle = {0, 4, 1, 2, 3};
|
||||
} else {
|
||||
// From: [batch, in_cols, in_rows, in_planes, in_depth]
|
||||
// To: [in_cols, in_rows, in_planes, batch, in_depth]
|
||||
input_shuffle = {1, 2, 3, 0, 4};
|
||||
}
|
||||
|
||||
// Input is playing the role of a "kernel" in this convolution.
|
||||
DSizes<TensorIndex, 2> input_dims;
|
||||
if (isColMajor) {
|
||||
input_dims[0] = kernelChannels;
|
||||
input_dims[1] = batch * inputPlanes * inputRows * inputCols;
|
||||
} else {
|
||||
input_dims[1] = kernelChannels;
|
||||
input_dims[0] = inputCols * inputRows * inputPlanes * batch;
|
||||
}
|
||||
|
||||
// Molds the output of the patch extraction result into a 2D tensor:
|
||||
// - the first dimension (dims[0]): the patch values to be multiplied with the
|
||||
// kernels
|
||||
// - the second dimension (dims[1]): everything else
|
||||
DSizes<TensorIndex, 2> pre_contract_dims;
|
||||
if (isColMajor) {
|
||||
pre_contract_dims[0] = batch * inputPlanes * inputRows * inputCols;
|
||||
pre_contract_dims[1] =
|
||||
kernelPlanes * kernelRows * kernelCols * kernelFilters;
|
||||
} else {
|
||||
pre_contract_dims[1] = inputCols * inputRows * inputPlanes * batch;
|
||||
pre_contract_dims[0] =
|
||||
kernelFilters * kernelCols * kernelRows * kernelPlanes;
|
||||
}
|
||||
|
||||
// We will contract along the collapsed dimension that contains the
|
||||
// batch, inputPlanes, inputRows and inputCols.
|
||||
array<IndexPair<TensorIndex>, 1> contract_dims;
|
||||
contract_dims[0] = IndexPair<TensorIndex>(1, 0);
|
||||
|
||||
// Dimensions after contraction.
|
||||
DSizes<TensorIndex, NumDims> post_contract_dims;
|
||||
if (isColMajor) {
|
||||
post_contract_dims[0] = kernelChannels;
|
||||
post_contract_dims[1] = kernelPlanes;
|
||||
post_contract_dims[2] = kernelRows;
|
||||
post_contract_dims[3] = kernelCols;
|
||||
post_contract_dims[4] = kernelFilters;
|
||||
} else {
|
||||
post_contract_dims[0] = kernelFilters;
|
||||
post_contract_dims[1] = kernelCols;
|
||||
post_contract_dims[2] = kernelRows;
|
||||
post_contract_dims[3] = kernelPlanes;
|
||||
post_contract_dims[4] = kernelChannels;
|
||||
}
|
||||
|
||||
// Reorder output of contraction to valid filter shape.
|
||||
array<TensorIndex, 5> kernel_shuffle;
|
||||
if (isColMajor) {
|
||||
// From: [in_depth, kernel_planes, kernel_rows, kernel_cols, out_depth]
|
||||
// To: [out_depth, in_depth, kernel_planes, kernel_rows, kernel_cols]
|
||||
kernel_shuffle = {4, 0, 1, 2, 3};
|
||||
} else {
|
||||
// From: [out_depth, kernel_cols, kernel_rows, kernel_planes, in_depth]
|
||||
// To: [kernel_cols, kernel_rows, kernel_planes, in_depth, out_depth]
|
||||
kernel_shuffle = {1, 2, 3, 4, 0};
|
||||
}
|
||||
|
||||
// Reverse kernel backprop dimensions.
|
||||
array<TensorIndex, 5> kernel_reverse;
|
||||
if (isColMajor) {
|
||||
kernel_reverse = {false, false, true, true, true};
|
||||
} else {
|
||||
kernel_reverse = {true, true, true, false, false};
|
||||
}
|
||||
|
||||
// Create convolution input (aka source of patches) from output backward
|
||||
// tensor by shuffling dimensions.
|
||||
const auto the_input =
|
||||
output_backward.shuffle(output_backward_shuffle).eval();
|
||||
|
||||
// Create convolution kernel (aka filter) from input by shuffling and
|
||||
// reshaping.
|
||||
const auto the_kernel =
|
||||
input.shuffle(input_shuffle).reshape(input_dims).eval();
|
||||
|
||||
return choose(Cond<internal::traits<Input>::Layout == ColMajor>(),
|
||||
the_kernel.contract(
|
||||
the_input
|
||||
.extract_volume_patches(
|
||||
inputPlanes, inputRows, inputCols, 1, 1, 1,
|
||||
stridePlanes, strideRows, strideCols,
|
||||
top_pad_planes, bottom_pad_planes, top_pad_rows,
|
||||
bottom_pad_rows, left_pad_cols, right_pad_cols)
|
||||
.reshape(pre_contract_dims),
|
||||
contract_dims),
|
||||
the_input
|
||||
.extract_volume_patches(
|
||||
inputPlanes, inputRows, inputCols, 1, 1, 1,
|
||||
stridePlanes, strideRows, strideCols, top_pad_planes,
|
||||
bottom_pad_planes, top_pad_rows, bottom_pad_rows,
|
||||
left_pad_cols, right_pad_cols)
|
||||
.reshape(pre_contract_dims)
|
||||
.contract(the_kernel, contract_dims))
|
||||
.reshape(post_contract_dims)
|
||||
.shuffle(kernel_shuffle)
|
||||
.reverse(kernel_reverse);
|
||||
}
|
||||
|
||||
} // end namespace Eigen
|
||||
|
||||
#endif // TENSORFLOW_CORE_KERNELS_EIGEN_BACKWARD_CUBOID_CONVOLUTIONS_H_
|
||||
@@ -0,0 +1,502 @@
|
||||
/* Copyright 2015 The TensorFlow Authors. All Rights Reserved.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
==============================================================================*/
|
||||
|
||||
#ifndef TENSORFLOW_CORE_KERNELS_EIGEN_BACKWARD_SPATIAL_CONVOLUTIONS_H_
|
||||
#define TENSORFLOW_CORE_KERNELS_EIGEN_BACKWARD_SPATIAL_CONVOLUTIONS_H_
|
||||
|
||||
#include "third_party/eigen3/unsupported/Eigen/CXX11/Tensor"
|
||||
|
||||
namespace Eigen {
|
||||
|
||||
/** SpatialConvolutionBackwardInput
|
||||
* \ingroup CXX11_NeuralNetworks_Module
|
||||
*
|
||||
* \brief Computes the backprop for the input of a 2D convolution.
|
||||
*
|
||||
* The output_backward parameter is expected to be a tensor with a rank of 3 or
|
||||
* more (channels, height, width, and optionally others)
|
||||
* The kernel parameter is expected to be a 4D tensor (filters, channels,
|
||||
* kernel_height, kernel_width)
|
||||
* The output_backward and the kernel must both be in col-major layout. The
|
||||
* result will also be in col-major layout.
|
||||
*
|
||||
* If row_in_stride, col_in_stride > 1, then applies convolution with holes
|
||||
* (aka atrous convolution), sampling every row_in_stride, col_in_stride input
|
||||
* pixels.
|
||||
*
|
||||
* The result can be assigned to a tensor of rank equal to the rank of the
|
||||
* output_backward. The dimensions of the result will be filters, height, width
|
||||
* (and others if applicable).
|
||||
*
|
||||
* It is possible to swap the order of the width and height dimensions provided
|
||||
* that the same order is used in the input, the kernel, and the output.
|
||||
*
|
||||
*/
|
||||
#ifdef EIGEN_HAS_INDEX_LIST
|
||||
typedef IndexList<type2index<0>, type2index<0>, type2index<1>, type2index<1> >
|
||||
ReverseColMajor;
|
||||
typedef IndexList<type2index<1>, type2index<1>, type2index<0>, type2index<0> >
|
||||
ReverseRowMajor;
|
||||
#else
|
||||
typedef array<bool, 4> ReverseColMajor;
|
||||
typedef array<bool, 4> ReverseRowMajor;
|
||||
#endif
|
||||
|
||||
template <typename OutputBackward, typename Kernel>
|
||||
EIGEN_ALWAYS_INLINE static const typename internal::conditional<
|
||||
internal::traits<OutputBackward>::Layout == ColMajor,
|
||||
TensorReshapingOp<
|
||||
const DSizes<typename internal::traits<OutputBackward>::Index,
|
||||
internal::traits<OutputBackward>::NumDimensions>,
|
||||
const TensorContractionOp<
|
||||
const array<
|
||||
IndexPair<typename internal::traits<OutputBackward>::Index>, 1>,
|
||||
const Eigen::TensorForcedEvalOp<const TensorReshapingOp<
|
||||
const DSizes<typename internal::traits<OutputBackward>::Index,
|
||||
2>,
|
||||
const TensorShufflingOp<
|
||||
const array<
|
||||
typename internal::traits<OutputBackward>::Index, 4>,
|
||||
const TensorReverseOp<const ReverseColMajor,
|
||||
const Kernel> > > >,
|
||||
const TensorReshapingOp<
|
||||
const DSizes<typename internal::traits<OutputBackward>::Index,
|
||||
2>,
|
||||
const TensorImagePatchOp<Dynamic, Dynamic,
|
||||
const OutputBackward> > > >,
|
||||
TensorReshapingOp<
|
||||
const DSizes<typename internal::traits<OutputBackward>::Index,
|
||||
internal::traits<OutputBackward>::NumDimensions>,
|
||||
const TensorContractionOp<
|
||||
const array<
|
||||
IndexPair<typename internal::traits<OutputBackward>::Index>, 1>,
|
||||
const TensorReshapingOp<
|
||||
const DSizes<typename internal::traits<OutputBackward>::Index,
|
||||
2>,
|
||||
const TensorImagePatchOp<Dynamic, Dynamic,
|
||||
const OutputBackward> >,
|
||||
const Eigen::TensorForcedEvalOp<const TensorReshapingOp<
|
||||
const DSizes<typename internal::traits<OutputBackward>::Index,
|
||||
2>,
|
||||
const TensorShufflingOp<
|
||||
const array<
|
||||
typename internal::traits<OutputBackward>::Index, 4>,
|
||||
const TensorReverseOp<const ReverseRowMajor,
|
||||
const Kernel> > > > > > >::type
|
||||
SpatialConvolutionBackwardInput(
|
||||
const Kernel& kernel, const OutputBackward& output_backward,
|
||||
typename internal::traits<OutputBackward>::Index inputRows,
|
||||
typename internal::traits<OutputBackward>::Index inputCols,
|
||||
const DenseIndex row_stride = 1, const DenseIndex col_stride = 1,
|
||||
const DenseIndex row_in_stride = 1, const DenseIndex col_in_stride = 1) {
|
||||
typedef typename internal::traits<OutputBackward>::Index TensorIndex;
|
||||
typedef typename internal::traits<OutputBackward>::Scalar OutScalar;
|
||||
TensorRef<Tensor<typename internal::traits<Kernel>::Scalar,
|
||||
internal::traits<Kernel>::NumDimensions,
|
||||
internal::traits<Kernel>::Layout, TensorIndex> >
|
||||
kern(kernel);
|
||||
TensorRef<Tensor<OutScalar, internal::traits<OutputBackward>::NumDimensions,
|
||||
internal::traits<OutputBackward>::Layout, TensorIndex> >
|
||||
out(output_backward);
|
||||
|
||||
EIGEN_STATIC_ASSERT(internal::traits<Kernel>::Layout ==
|
||||
internal::traits<OutputBackward>::Layout,
|
||||
YOU_MADE_A_PROGRAMMING_MISTAKE);
|
||||
|
||||
static const bool isColMajor =
|
||||
(internal::traits<OutputBackward>::Layout == ColMajor);
|
||||
|
||||
static const int NumDims = internal::traits<OutputBackward>::NumDimensions;
|
||||
|
||||
// Number of filters to apply. This is the same as the output depth of the
|
||||
// result
|
||||
const TensorIndex kernelFilters =
|
||||
isColMajor ? kern.dimensions()[0] : kern.dimensions()[3];
|
||||
// Number of channels. This is the same as the input depth.
|
||||
const TensorIndex kernelChannels =
|
||||
isColMajor ? kern.dimensions()[1] : kern.dimensions()[2];
|
||||
const TensorIndex kernelRows =
|
||||
isColMajor ? kern.dimensions()[2] : kern.dimensions()[1];
|
||||
const TensorIndex kernelCols =
|
||||
isColMajor ? kern.dimensions()[3] : kern.dimensions()[0];
|
||||
|
||||
// This is the effective kernel size, taking into account the (*_in_stride -
|
||||
// 1) zero-values
|
||||
// inserted between consecutive kernel elements in atrous convolution
|
||||
const TensorIndex kernelRowsEff =
|
||||
kernelRows + (kernelRows - 1) * (row_in_stride - 1);
|
||||
const TensorIndex kernelColsEff =
|
||||
kernelCols + (kernelCols - 1) * (col_in_stride - 1);
|
||||
|
||||
const TensorIndex outputRows = isColMajor
|
||||
? output_backward.dimension(1)
|
||||
: output_backward.dimension(NumDims - 2);
|
||||
const TensorIndex outputCols = isColMajor
|
||||
? output_backward.dimension(2)
|
||||
: output_backward.dimension(NumDims - 3);
|
||||
|
||||
// Computing the forward padding
|
||||
const TensorIndex forward_pad_top = numext::maxi<Index>(
|
||||
0, ((outputRows - 1) * row_stride + kernelRowsEff - inputRows) / 2);
|
||||
const TensorIndex forward_pad_left = numext::maxi<Index>(
|
||||
0, ((outputCols - 1) * col_stride + kernelColsEff - inputCols) / 2);
|
||||
const TensorIndex padding_top = kernelRowsEff - 1 - forward_pad_top;
|
||||
const TensorIndex padding_left = kernelColsEff - 1 - forward_pad_left;
|
||||
|
||||
const TensorIndex padding_bottom = inputRows - (outputRows - 1) * row_stride -
|
||||
2 - padding_top + kernelRowsEff;
|
||||
const TensorIndex padding_right = inputCols - (outputCols - 1) * col_stride -
|
||||
2 - padding_left + kernelColsEff;
|
||||
|
||||
eigen_assert(padding_top >= 0);
|
||||
eigen_assert(padding_left >= 0);
|
||||
eigen_assert(padding_bottom >= 0);
|
||||
eigen_assert(padding_right >= 0);
|
||||
|
||||
// The kernel has dimensions filters X channels X patch_rows X patch_cols
|
||||
// We need to reverse the kernel along dimensions corresponding to rows and
|
||||
// cols.
|
||||
// TODO(yangke): we can make things slightly faster by collapsing the
|
||||
// dimensions
|
||||
// where we don't reverse. Try that once we have a faster compiler.
|
||||
typedef typename internal::conditional<isColMajor, ReverseColMajor,
|
||||
ReverseRowMajor>::type Reverse;
|
||||
Reverse kernel_reverse;
|
||||
|
||||
#ifndef EIGEN_HAS_INDEX_LIST
|
||||
if (isColMajor) {
|
||||
kernel_reverse[0] = false;
|
||||
kernel_reverse[1] = false;
|
||||
kernel_reverse[2] = true;
|
||||
kernel_reverse[3] = true;
|
||||
} else {
|
||||
kernel_reverse[0] = true;
|
||||
kernel_reverse[1] = true;
|
||||
kernel_reverse[2] = false;
|
||||
kernel_reverse[3] = false;
|
||||
}
|
||||
#endif
|
||||
|
||||
// Reorder the dimensions to:
|
||||
// filters x patch_rows x patch_cols x channels
|
||||
array<TensorIndex, 4> kernel_shuffle;
|
||||
if (isColMajor) {
|
||||
// From: filters x channels x rows x cols
|
||||
// To: filters x rows x cols x channels
|
||||
kernel_shuffle[0] = 0;
|
||||
kernel_shuffle[1] = 2;
|
||||
kernel_shuffle[2] = 3;
|
||||
kernel_shuffle[3] = 1;
|
||||
} else {
|
||||
// From: cols x rows x channels x filters
|
||||
// To: channels x cols x rows x filters
|
||||
kernel_shuffle[0] = 2;
|
||||
kernel_shuffle[1] = 0;
|
||||
kernel_shuffle[2] = 1;
|
||||
kernel_shuffle[3] = 3;
|
||||
}
|
||||
|
||||
// Collapse the dims
|
||||
DSizes<TensorIndex, 2> kernel_dims;
|
||||
if (isColMajor) {
|
||||
kernel_dims[0] = kernelFilters * kernelRows * kernelCols;
|
||||
kernel_dims[1] = kernelChannels;
|
||||
} else {
|
||||
kernel_dims[1] = kernelFilters * kernelRows * kernelCols;
|
||||
kernel_dims[0] = kernelChannels;
|
||||
}
|
||||
|
||||
// The output_backward has dimensions out_depth X out_rows X out_cols X OTHERS
|
||||
// When we extract the image patches from output_backward, it will have
|
||||
// dimensions
|
||||
// out_depth X (patch_rows * patch_cols) X (input_rows * input_cols *
|
||||
// OTHERS)
|
||||
DSizes<TensorIndex, 2> pre_contract_dims;
|
||||
if (isColMajor) {
|
||||
pre_contract_dims[0] = kernelFilters * kernelRows * kernelCols;
|
||||
pre_contract_dims[1] = inputRows * inputCols;
|
||||
for (int i = 3; i < NumDims; ++i) {
|
||||
pre_contract_dims[1] *= out.dimension(i);
|
||||
}
|
||||
} else {
|
||||
pre_contract_dims[1] = kernelFilters * kernelRows * kernelCols;
|
||||
pre_contract_dims[0] = inputRows * inputCols;
|
||||
for (int i = 0; i < NumDims - 3; ++i) {
|
||||
pre_contract_dims[0] *= out.dimension(i);
|
||||
}
|
||||
}
|
||||
|
||||
// We will contract along the collapsed dimension that contains the
|
||||
// kernelFilters, the kernelRows and the kernelCols.
|
||||
array<IndexPair<TensorIndex>, 1> contract_dims;
|
||||
if (isColMajor) {
|
||||
// col-major: kernel.contract(output.patches)
|
||||
contract_dims[0] = IndexPair<TensorIndex>(0, 0);
|
||||
} else {
|
||||
// row-major: output.patches.contract(kernel)
|
||||
contract_dims[0] = IndexPair<TensorIndex>(1, 1);
|
||||
}
|
||||
|
||||
// Post contraction, the dimensions of the input_backprop is
|
||||
// channels X input_rows X input_cols X OTHERS
|
||||
DSizes<TensorIndex, NumDims> post_contract_dims;
|
||||
if (isColMajor) {
|
||||
post_contract_dims[0] = kernelChannels;
|
||||
post_contract_dims[1] = inputRows;
|
||||
post_contract_dims[2] = inputCols;
|
||||
for (int i = 3; i < NumDims; ++i) {
|
||||
post_contract_dims[i] = out.dimension(i);
|
||||
}
|
||||
} else {
|
||||
post_contract_dims[NumDims - 1] = kernelChannels;
|
||||
post_contract_dims[NumDims - 2] = inputRows;
|
||||
post_contract_dims[NumDims - 3] = inputCols;
|
||||
for (int i = 0; i < NumDims - 3; ++i) {
|
||||
post_contract_dims[i] = out.dimension(i);
|
||||
}
|
||||
}
|
||||
|
||||
return choose(
|
||||
Cond<internal::traits<OutputBackward>::Layout == ColMajor>(),
|
||||
kernel.reverse(kernel_reverse)
|
||||
.shuffle(kernel_shuffle)
|
||||
.reshape(kernel_dims)
|
||||
.eval()
|
||||
.contract(
|
||||
output_backward
|
||||
.extract_image_patches(
|
||||
kernelRows, kernelCols, 1, 1, row_in_stride,
|
||||
col_in_stride, row_stride, col_stride, padding_top,
|
||||
padding_bottom, padding_left, padding_right, OutScalar(0))
|
||||
.reshape(pre_contract_dims),
|
||||
contract_dims)
|
||||
.reshape(post_contract_dims),
|
||||
output_backward
|
||||
.extract_image_patches(kernelRows, kernelCols, 1, 1, row_in_stride,
|
||||
col_in_stride, row_stride, col_stride,
|
||||
padding_top, padding_bottom, padding_left,
|
||||
padding_right, OutScalar(0))
|
||||
.reshape(pre_contract_dims)
|
||||
.contract(kernel.reverse(kernel_reverse)
|
||||
.shuffle(kernel_shuffle)
|
||||
.reshape(kernel_dims)
|
||||
.eval(),
|
||||
contract_dims)
|
||||
.reshape(post_contract_dims));
|
||||
}
|
||||
|
||||
/** SpatialConvolutionBackwardKernel
|
||||
* \ingroup CXX11_NeuralNetworks_Module
|
||||
*
|
||||
* \brief Computes the backprop for the filter of a 2D convolution.
|
||||
*
|
||||
* The output_backward parameter is expected to be a tensor with a rank of 3 or
|
||||
* more (channels, height, width, and optionally others)
|
||||
* The kernel parameter is expected to be a 4D tensor (filters, channels,
|
||||
* kernel_height, kernel_width)
|
||||
* The output_backward and the kernel must both be in col-major layout. The
|
||||
* result will also be in col-major layout.
|
||||
*
|
||||
* If row_in_stride, col_stride > 1, then applies convolution with holes (aka
|
||||
* atrous convolution), sampling every row_in_stride, col_in_stride input
|
||||
* pixels.
|
||||
*
|
||||
* The result can be assigned to a tensor of rank equal to the rank of the
|
||||
* output_backward. The dimensions of the result will be filters, height, width
|
||||
* (and others if applicable).
|
||||
*
|
||||
* It is possible to swap the order of the width and height dimensions provided
|
||||
* that the same order is used in the input, the kernel, and the output.
|
||||
*
|
||||
*/
|
||||
|
||||
template <typename OutputBackward, typename Input>
|
||||
EIGEN_ALWAYS_INLINE static const typename internal::conditional<
|
||||
internal::traits<OutputBackward>::Layout == ColMajor,
|
||||
TensorReshapingOp<
|
||||
const DSizes<typename internal::traits<Input>::Index, 4>,
|
||||
const TensorContractionOp<
|
||||
const array<IndexPair<typename internal::traits<Input>::Index>, 1>,
|
||||
const TensorReshapingOp<
|
||||
const DSizes<typename internal::traits<Input>::Index, 2>,
|
||||
const OutputBackward>,
|
||||
const TensorReshapingOp<
|
||||
const DSizes<typename internal::traits<Input>::Index, 2>,
|
||||
const TensorImagePatchOp<Dynamic, Dynamic, const Input> > > >,
|
||||
TensorReshapingOp<
|
||||
const DSizes<typename internal::traits<Input>::Index, 4>,
|
||||
const TensorContractionOp<
|
||||
const array<IndexPair<typename internal::traits<Input>::Index>, 1>,
|
||||
const TensorReshapingOp<
|
||||
const DSizes<typename internal::traits<Input>::Index, 2>,
|
||||
const TensorImagePatchOp<Dynamic, Dynamic, const Input> >,
|
||||
const TensorReshapingOp<
|
||||
const DSizes<typename internal::traits<Input>::Index, 2>,
|
||||
const OutputBackward> > > >::type
|
||||
SpatialConvolutionBackwardKernel(
|
||||
const Input& input, const OutputBackward& output_backward,
|
||||
typename internal::traits<Input>::Index kernelRows,
|
||||
typename internal::traits<Input>::Index kernelCols,
|
||||
const DenseIndex row_stride = 1, const DenseIndex col_stride = 1,
|
||||
const DenseIndex row_in_stride = 1, const DenseIndex col_in_stride = 1) {
|
||||
typedef typename internal::traits<Input>::Index TensorIndex;
|
||||
typedef typename internal::traits<OutputBackward>::Scalar OutScalar;
|
||||
TensorRef<Tensor<typename internal::traits<Input>::Scalar,
|
||||
internal::traits<Input>::NumDimensions,
|
||||
internal::traits<Input>::Layout, TensorIndex> >
|
||||
in(input);
|
||||
TensorRef<Tensor<OutScalar, internal::traits<OutputBackward>::NumDimensions,
|
||||
internal::traits<OutputBackward>::Layout, TensorIndex> >
|
||||
out(output_backward);
|
||||
|
||||
EIGEN_STATIC_ASSERT(internal::traits<Input>::Layout ==
|
||||
internal::traits<OutputBackward>::Layout,
|
||||
YOU_MADE_A_PROGRAMMING_MISTAKE);
|
||||
|
||||
// stride and in_stride cannot both be larger than 1
|
||||
eigen_assert(!(row_stride > 1 && row_in_stride > 1) &&
|
||||
!(col_stride > 1 && col_in_stride > 1));
|
||||
|
||||
static const bool isColMajor = (internal::traits<Input>::Layout == ColMajor);
|
||||
|
||||
static const int NumDims = internal::traits<Input>::NumDimensions;
|
||||
EIGEN_STATIC_ASSERT(internal::traits<Input>::NumDimensions ==
|
||||
internal::traits<OutputBackward>::NumDimensions,
|
||||
YOU_MADE_A_PROGRAMMING_MISTAKE);
|
||||
|
||||
const TensorIndex inputRows =
|
||||
isColMajor ? in.dimension(1) : in.dimension(NumDims - 2);
|
||||
const TensorIndex inputCols =
|
||||
isColMajor ? in.dimension(2) : in.dimension(NumDims - 3);
|
||||
|
||||
const TensorIndex outputRows = isColMajor
|
||||
? output_backward.dimension(1)
|
||||
: output_backward.dimension(NumDims - 2);
|
||||
const TensorIndex outputCols = isColMajor
|
||||
? output_backward.dimension(2)
|
||||
: output_backward.dimension(NumDims - 3);
|
||||
|
||||
// Number of filters to apply. This is the same as the output depth of the
|
||||
// result
|
||||
const TensorIndex kernelFilters =
|
||||
isColMajor ? out.dimensions()[0] : out.dimensions()[NumDims - 1];
|
||||
|
||||
// Number of channels. This is the same as the input depth.
|
||||
const TensorIndex kernelChannels =
|
||||
isColMajor ? in.dimensions()[0] : in.dimensions()[NumDims - 1];
|
||||
|
||||
// This is the effective kernel size, taking into account the (*_in_stride -
|
||||
// 1) zero-values
|
||||
// inserted between consecutive kernel elements in atrous convolution
|
||||
const TensorIndex kernelRowsEff =
|
||||
kernelRows + (kernelRows - 1) * (row_in_stride - 1);
|
||||
const TensorIndex kernelColsEff =
|
||||
kernelCols + (kernelCols - 1) * (col_in_stride - 1);
|
||||
|
||||
// Computing the forward padding
|
||||
const TensorIndex padRows = numext::maxi<Index>(
|
||||
0, (outputRows - 1) * row_stride + kernelRowsEff - inputRows);
|
||||
const TensorIndex padCols = numext::maxi<Index>(
|
||||
0, (outputCols - 1) * col_stride + kernelColsEff - inputCols);
|
||||
const TensorIndex padding_top = padRows / 2;
|
||||
const TensorIndex padding_bottom = padRows - padding_top;
|
||||
const TensorIndex padding_left = padCols / 2;
|
||||
const TensorIndex padding_right = padCols - padding_left;
|
||||
|
||||
// Reshaped out
|
||||
DSizes<TensorIndex, 2> output_dims;
|
||||
if (isColMajor) {
|
||||
output_dims[0] = kernelFilters;
|
||||
output_dims[1] = outputRows * outputCols;
|
||||
for (int i = 3; i < NumDims; ++i) {
|
||||
output_dims[1] *= out.dimension(i);
|
||||
}
|
||||
} else {
|
||||
output_dims[1] = kernelFilters;
|
||||
output_dims[0] = outputCols * outputRows;
|
||||
for (int i = 0; i < NumDims - 3; ++i) {
|
||||
output_dims[0] *= out.dimension(i);
|
||||
}
|
||||
}
|
||||
|
||||
// Reshaped extract_image_patches(in)
|
||||
DSizes<TensorIndex, 2> pre_contract_dims;
|
||||
if (isColMajor) {
|
||||
pre_contract_dims[0] = kernelChannels * kernelRows * kernelCols;
|
||||
pre_contract_dims[1] = outputRows * outputCols;
|
||||
for (int i = 3; i < NumDims; ++i) {
|
||||
pre_contract_dims[1] *= in.dimension(i);
|
||||
}
|
||||
eigen_assert(output_dims[1] == pre_contract_dims[1]);
|
||||
} else {
|
||||
pre_contract_dims[1] = kernelCols * kernelRows * kernelChannels;
|
||||
pre_contract_dims[0] = outputRows * outputCols;
|
||||
for (int i = 0; i < NumDims - 3; ++i) {
|
||||
pre_contract_dims[0] *= in.dimension(i);
|
||||
}
|
||||
eigen_assert(output_dims[0] == pre_contract_dims[0]);
|
||||
}
|
||||
|
||||
// We will contract along the collapsed dimension that contains the
|
||||
// outputCols, outputRows and OTHERS.
|
||||
array<IndexPair<TensorIndex>, 1> contract_dims;
|
||||
if (isColMajor) {
|
||||
// col-major: output_backward.contract(input.patches)
|
||||
contract_dims[0] = IndexPair<TensorIndex>(1, 1);
|
||||
} else {
|
||||
// row-major: input.patches.contract(output_backward)
|
||||
contract_dims[0] = IndexPair<TensorIndex>(0, 0);
|
||||
}
|
||||
|
||||
// After the contraction, the kernel will have the desired shape
|
||||
// out_depth X in_shape X kernel_rows X kernel_cols
|
||||
DSizes<TensorIndex, 4> kernel_dims;
|
||||
if (isColMajor) {
|
||||
kernel_dims[0] = kernelFilters;
|
||||
kernel_dims[1] = kernelChannels;
|
||||
kernel_dims[2] = kernelRows;
|
||||
kernel_dims[3] = kernelCols;
|
||||
} else {
|
||||
kernel_dims[3] = kernelFilters;
|
||||
kernel_dims[2] = kernelChannels;
|
||||
kernel_dims[1] = kernelRows;
|
||||
kernel_dims[0] = kernelCols;
|
||||
}
|
||||
|
||||
return choose(
|
||||
Cond<internal::traits<Input>::Layout == ColMajor>(),
|
||||
output_backward.reshape(output_dims)
|
||||
.contract(
|
||||
input
|
||||
.extract_image_patches(
|
||||
kernelRows, kernelCols, row_stride, col_stride,
|
||||
row_in_stride, col_in_stride, 1, 1, padding_top,
|
||||
padding_bottom, padding_left, padding_right, OutScalar(0))
|
||||
.reshape(pre_contract_dims),
|
||||
contract_dims)
|
||||
.reshape(kernel_dims),
|
||||
input
|
||||
.extract_image_patches(kernelRows, kernelCols, row_stride, col_stride,
|
||||
row_in_stride, col_in_stride, 1, 1,
|
||||
padding_top, padding_bottom, padding_left,
|
||||
padding_right, OutScalar(0))
|
||||
.reshape(pre_contract_dims)
|
||||
.contract(output_backward.reshape(output_dims), contract_dims)
|
||||
.reshape(kernel_dims));
|
||||
}
|
||||
|
||||
} // end namespace Eigen
|
||||
|
||||
#endif // TENSORFLOW_CORE_KERNELS_EIGEN_BACKWARD_SPATIAL_CONVOLUTIONS_H_
|
||||
File diff suppressed because it is too large.
Load diff
@@ -0,0 +1,619 @@
|
||||
/* Copyright 2015 The TensorFlow Authors. All Rights Reserved.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
==============================================================================*/
|
||||
|
||||
#ifndef TENSORFLOW_CORE_KERNELS_EIGEN_POOLING_H_
|
||||
#define TENSORFLOW_CORE_KERNELS_EIGEN_POOLING_H_
|
||||
|
||||
#include "third_party/eigen3/unsupported/Eigen/CXX11/Tensor"
|
||||
#include "tensorflow/core/kernels/eigen_volume_patch.h"
|
||||
|
||||
namespace Eigen {
|
||||
|
||||
/** SpatialMaxPooling
|
||||
* \ingroup CXX11_NeuralNetworks_Module
|
||||
*
|
||||
* \brief Applies a max-pooling over a multichannel input image.
|
||||
*
|
||||
* The input parameter is expected to be a with a rank of 4 (channels, height,
|
||||
* width, others in col-major, and the reverse of that in row-major).
|
||||
*
|
||||
* The result can be assigned to a tensor of rank equal to the rank of the
|
||||
* input. The dimensions of the result will be channels, height, width, and
|
||||
* others (in col-major, and the reverse of that if the input was row-major).
|
||||
*
|
||||
* The order of the width and height dimensions can be swapped if needed.
|
||||
*
|
||||
*/
|
||||
#if !defined(EIGEN_HAS_INDEX_LIST)
|
||||
template <typename Input>
|
||||
EIGEN_ALWAYS_INLINE static const TensorReshapingOp<
|
||||
const Eigen::DSizes<typename internal::traits<Input>::Index,
|
||||
internal::traits<Input>::NumDimensions>,
|
||||
const TensorReductionOp<
|
||||
internal::MaxReducer<typename internal::remove_const<
|
||||
typename internal::traits<Input>::Scalar>::type>,
|
||||
const Eigen::array<int, 2>,
|
||||
const TensorImagePatchOp<Dynamic, Dynamic, const Input> > >
|
||||
#else
|
||||
template <typename Input>
|
||||
EIGEN_ALWAYS_INLINE static const TensorReshapingOp<
|
||||
const Eigen::DSizes<typename internal::traits<Input>::Index,
|
||||
internal::traits<Input>::NumDimensions>,
|
||||
const TensorReductionOp<
|
||||
internal::MaxReducer<typename internal::remove_const<
|
||||
typename internal::traits<Input>::Scalar>::type>,
|
||||
typename internal::conditional<
|
||||
internal::traits<Input>::Layout == ColMajor,
|
||||
const Eigen::IndexList<Eigen::type2index<1>, Eigen::type2index<2> >,
|
||||
const Eigen::IndexList<Eigen::type2index<2>,
|
||||
Eigen::type2index<3> > >::type,
|
||||
const TensorImagePatchOp<Dynamic, Dynamic, const Input> > >
|
||||
#endif
|
||||
SpatialMaxPooling(const Input& input, DenseIndex patchRows,
|
||||
DenseIndex patchCols, DenseIndex strideRows,
|
||||
DenseIndex strideCols, const PaddingType padding_type,
|
||||
DenseIndex in_strideRows = 1, DenseIndex in_strideCols = 1) {
|
||||
EIGEN_STATIC_ASSERT(internal::traits<Input>::NumDimensions == 4,
|
||||
YOU_MADE_A_PROGRAMMING_MISTAKE);
|
||||
|
||||
typedef typename internal::traits<Input>::Index TensorIndex;
|
||||
TensorRef<Tensor<typename internal::traits<Input>::Scalar,
|
||||
internal::traits<Input>::NumDimensions,
|
||||
internal::traits<Input>::Layout, TensorIndex> >
|
||||
in(input);
|
||||
|
||||
const DenseIndex patchRowsEff =
|
||||
patchRows + (patchRows - 1) * (in_strideRows - 1);
|
||||
const DenseIndex patchColsEff =
|
||||
patchCols + (patchCols - 1) * (in_strideCols - 1);
|
||||
|
||||
static const bool isColMajor = (internal::traits<Input>::Layout == ColMajor);
|
||||
static const int idxRows = isColMajor ? 1 : 2;
|
||||
static const int idxCols = isColMajor ? 2 : 1;
|
||||
|
||||
// Molds the output of the reduction into the shape expected by the user.
|
||||
// (assuming col-major):
|
||||
// - 1st dim: channels
|
||||
// - 2nd dim: output height
|
||||
// - 3rd dim: output width
|
||||
// - 4th dim and beyond: everything else including batch size
|
||||
Eigen::DSizes<TensorIndex, internal::traits<Input>::NumDimensions>
|
||||
post_reduce_dims;
|
||||
post_reduce_dims[0] = in.dimension(0);
|
||||
if (padding_type == PADDING_VALID) {
|
||||
post_reduce_dims[idxRows] = Eigen::divup(
|
||||
static_cast<DenseIndex>(in.dimension(idxRows)) - patchRowsEff + 1,
|
||||
strideRows);
|
||||
post_reduce_dims[idxCols] = Eigen::divup(
|
||||
static_cast<DenseIndex>(in.dimension(idxCols)) - patchColsEff + 1,
|
||||
strideCols);
|
||||
} else {
|
||||
post_reduce_dims[idxRows] = Eigen::divup(
|
||||
static_cast<DenseIndex>(in.dimension(idxRows)), strideRows);
|
||||
post_reduce_dims[idxCols] = Eigen::divup(
|
||||
static_cast<DenseIndex>(in.dimension(idxCols)), strideCols);
|
||||
}
|
||||
post_reduce_dims[3] = in.dimension(3);
|
||||
|
||||
#if !defined(EIGEN_HAS_INDEX_LIST)
|
||||
// nvcc doesn't support cxx11
|
||||
Eigen::array<int, 2> reduction_dims;
|
||||
if (isColMajor) {
|
||||
reduction_dims[0] = 1;
|
||||
reduction_dims[1] = 2;
|
||||
} else {
|
||||
reduction_dims[0] = 2;
|
||||
reduction_dims[1] = 3;
|
||||
}
|
||||
#else
|
||||
// Take advantage of cxx11 to give the compiler information it can use to
|
||||
// optimize the code.
|
||||
typename internal::conditional<
|
||||
internal::traits<Input>::Layout == ColMajor,
|
||||
const Eigen::IndexList<Eigen::type2index<1>, Eigen::type2index<2> >,
|
||||
const Eigen::IndexList<Eigen::type2index<2>,
|
||||
Eigen::type2index<3> > >::type reduction_dims;
|
||||
#endif
|
||||
|
||||
return input
|
||||
.extract_image_patches(
|
||||
patchRows, patchCols, strideRows, strideCols, in_strideRows,
|
||||
in_strideCols, padding_type,
|
||||
-Eigen::NumTraits<typename internal::remove_const<
|
||||
typename internal::traits<Input>::Scalar>::type>::highest())
|
||||
.maximum(reduction_dims)
|
||||
.reshape(post_reduce_dims);
|
||||
}
|
||||
|
||||
/** CuboidMaxPooling
|
||||
* \ingroup CXX11_NeuralNetworks_Module
|
||||
*
|
||||
* \brief Applies a max-pooling over a multichannel input volume.
|
||||
*
|
||||
* The input parameter is expected to be a tensor with a rank of 5 (channels,
|
||||
* depth, height, width, others in col-major, and the reverse of that in
|
||||
* row-major).
|
||||
*
|
||||
* The result can be assigned to a tensor of rank equal to the rank of the
|
||||
* input. The dimensions of the result will be channels, depth, height, width,
|
||||
* and others (in col-major, and the reverse of that if the input was
|
||||
* row-major).
|
||||
*
|
||||
* The order of the depth, width and height dimensions can be swapped if
|
||||
* needed.
|
||||
*
|
||||
*/
|
||||
#if !defined(EIGEN_HAS_INDEX_LIST)
|
||||
template <typename Input>
|
||||
EIGEN_ALWAYS_INLINE static const TensorReshapingOp<
|
||||
const Eigen::DSizes<DenseIndex, internal::traits<Input>::NumDimensions>,
|
||||
const TensorReductionOp<
|
||||
internal::MaxReducer<float>, const Eigen::array<int, 1>,
|
||||
const TensorReshapingOp<
|
||||
const Eigen::DSizes<DenseIndex, 3>,
|
||||
const TensorVolumePatchOp<Dynamic, Dynamic, Dynamic,
|
||||
const Input> > > >
|
||||
#else
|
||||
template <typename Input>
|
||||
EIGEN_ALWAYS_INLINE static const TensorReshapingOp<
|
||||
const Eigen::DSizes<DenseIndex, internal::traits<Input>::NumDimensions>,
|
||||
const TensorReductionOp<
|
||||
internal::MaxReducer<float>,
|
||||
const Eigen::IndexList<Eigen::type2index<1> >,
|
||||
const TensorReshapingOp<
|
||||
const Eigen::DSizes<DenseIndex, 3>,
|
||||
const TensorVolumePatchOp<Dynamic, Dynamic, Dynamic,
|
||||
const Input> > > >
|
||||
#endif
|
||||
CuboidMaxPooling(const Input& input, DenseIndex patchPlanes,
|
||||
DenseIndex patchRows, DenseIndex patchCols,
|
||||
DenseIndex stridePlanes, DenseIndex strideRows,
|
||||
DenseIndex strideCols, const PaddingType padding_type) {
|
||||
EIGEN_STATIC_ASSERT(internal::traits<Input>::NumDimensions == 5,
|
||||
YOU_MADE_A_PROGRAMMING_MISTAKE);
|
||||
static const bool isColMajor = (internal::traits<Input>::Layout == ColMajor);
|
||||
|
||||
typedef typename internal::traits<Input>::Index TensorIndex;
|
||||
TensorRef<Tensor<typename internal::traits<Input>::Scalar,
|
||||
internal::traits<Input>::NumDimensions,
|
||||
internal::traits<Input>::Layout, TensorIndex> >
|
||||
in(input);
|
||||
|
||||
static const int idxPlanes = isColMajor ? 1 : 3;
|
||||
static const int idxRows = 2;
|
||||
static const int idxCols = isColMajor ? 3 : 1;
|
||||
|
||||
// Molds the output of the reduction into the shape expected by the used
|
||||
// (assuming col-major):
|
||||
// - 1st dim: channels
|
||||
// - 2nd dim: output depth
|
||||
// - 3rd dim: output height
|
||||
// - 4th dim: output width
|
||||
// - 5th dim and beyond: everything else including batch size
|
||||
Eigen::DSizes<DenseIndex, internal::traits<Input>::NumDimensions>
|
||||
post_reduce_dims;
|
||||
post_reduce_dims[0] = in.dimension(0);
|
||||
if (padding_type == PADDING_VALID) {
|
||||
post_reduce_dims[idxPlanes] = Eigen::divup(
|
||||
static_cast<DenseIndex>(in.dimension(idxPlanes)) - patchPlanes + 1,
|
||||
stridePlanes);
|
||||
post_reduce_dims[idxRows] = Eigen::divup(
|
||||
static_cast<DenseIndex>(in.dimension(idxRows)) - patchRows + 1,
|
||||
strideRows);
|
||||
post_reduce_dims[idxCols] = Eigen::divup(
|
||||
static_cast<DenseIndex>(in.dimension(idxCols)) - patchCols + 1,
|
||||
strideCols);
|
||||
} else {
|
||||
post_reduce_dims[idxPlanes] = Eigen::divup(
|
||||
static_cast<DenseIndex>(in.dimension(idxPlanes)), stridePlanes);
|
||||
post_reduce_dims[idxRows] = Eigen::divup(
|
||||
static_cast<DenseIndex>(in.dimension(idxRows)), strideRows);
|
||||
post_reduce_dims[idxCols] = Eigen::divup(
|
||||
static_cast<DenseIndex>(in.dimension(idxCols)), strideCols);
|
||||
}
|
||||
post_reduce_dims[4] = in.dimension(4);
|
||||
|
||||
Eigen::DSizes<DenseIndex, 3> pre_reduce_dims;
|
||||
pre_reduce_dims[1] = patchRows * patchCols * patchPlanes;
|
||||
if (isColMajor) {
|
||||
pre_reduce_dims[0] = post_reduce_dims[0];
|
||||
pre_reduce_dims[2] = post_reduce_dims[1] * post_reduce_dims[2] *
|
||||
post_reduce_dims[3] * post_reduce_dims[4];
|
||||
} else {
|
||||
pre_reduce_dims[0] = post_reduce_dims[0] * post_reduce_dims[1] *
|
||||
post_reduce_dims[2] * post_reduce_dims[3];
|
||||
pre_reduce_dims[2] = post_reduce_dims[4];
|
||||
}
|
||||
|
||||
#if !defined(EIGEN_HAS_INDEX_LIST)
|
||||
// nvcc doesn't support cxx11
|
||||
Eigen::array<int, 1> reduction_dims;
|
||||
reduction_dims[0] = 1;
|
||||
#else
|
||||
// Take advantage of cxx11 to give the compiler information it can use to
|
||||
// optimize the code.
|
||||
Eigen::IndexList<Eigen::type2index<1> > reduction_dims;
|
||||
#endif
|
||||
return input
|
||||
.extract_volume_patches(patchPlanes, patchRows, patchCols, stridePlanes,
|
||||
strideRows, strideCols, padding_type,
|
||||
-Eigen::NumTraits<float>::highest())
|
||||
.reshape(pre_reduce_dims)
|
||||
.maximum(reduction_dims)
|
||||
.reshape(post_reduce_dims);
|
||||
}
|
||||
|
||||
/** SpatialAvgPooling
|
||||
* \ingroup CXX11_NeuralNetworks_Module
|
||||
*
|
||||
* \brief Applies an average pooling over a multichannel input image.
|
||||
*
|
||||
* The input parameter is expected to be a tensor with a rank of 4 (channels,
|
||||
* height, width, others in col-major, and the reverse of that in row-major).
|
||||
*
|
||||
* The result can be assigned to a tensor of rank equal to the rank of the
|
||||
* input. The dimensions of the result will be channels, height, width, and
|
||||
* others (in col-major, and the reverse of that if the input was row-major).
|
||||
*
|
||||
* The order of the width and height dimensions can be swapped if needed.
|
||||
*
|
||||
*/
|
||||
namespace internal {
|
||||
|
||||
template <typename T>
|
||||
struct AvgPoolMeanReducer {
|
||||
#if (EIGEN_ARCH_i386 || EIGEN_ARCH_x86_64) && !defined(__CUDACC__)
|
||||
// We only support packet access for floats.
|
||||
static const bool PacketAccess = internal::is_same<T, float>::value;
|
||||
#else
|
||||
static const bool PacketAccess = false;
|
||||
#endif
|
||||
static const bool IsStateful = true;
|
||||
|
||||
EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE AvgPoolMeanReducer() : scalarCount_(0) {
|
||||
typedef typename packet_traits<T>::type Packet;
|
||||
packetCount_ = pset1<Packet>(T(0.0));
|
||||
}
|
||||
|
||||
EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void reduce(const T t, T* accum) {
|
||||
if (t != -Eigen::NumTraits<T>::highest()) {
|
||||
(*accum) = (*accum) + t;
|
||||
scalarCount_++;
|
||||
}
|
||||
}
|
||||
|
||||
EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE T initialize() const {
|
||||
return static_cast<T>(0);
|
||||
}
|
||||
|
||||
EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE T finalize(const T accum) const {
|
||||
eigen_assert(scalarCount_ > 0);
|
||||
return accum / T(scalarCount_);
|
||||
}
|
||||
|
||||
#if (EIGEN_ARCH_i386 || EIGEN_ARCH_x86_64) && !defined(__CUDACC__)
|
||||
#ifdef EIGEN_VECTORIZE_AVX512
|
||||
#define pequal(a, b) \
|
||||
_mm512_castsi512_ps( \
|
||||
_mm512_maskz_set1_epi32(_mm512_cmp_ps_mask(a, b, _CMP_EQ_UQ), -1))
|
||||
|
||||
// The ternarylogic function immediate determines the values in the result
|
||||
// In the case below, 0xd8 implies (false_mask) ? (b) : (a)
|
||||
// For details, refer to the vpternlogd instruction table at
|
||||
// http://www.intel.com/content/dam/www/public/us/en/documents/manuals/64-ia-32-architectures-software-developer-vol-2c-manual.pdf
|
||||
|
||||
#define psel(a, b, false_mask) \
|
||||
_mm512_castsi512_ps(_mm512_ternarylogic_epi32( \
|
||||
_mm512_castps_si512(a), _mm512_castps_si512(b), \
|
||||
_mm512_castps_si512(false_mask), 0xd8))
|
||||
#elif defined EIGEN_VECTORIZE_AVX
|
||||
#define pequal(a, b) _mm256_cmp_ps(a, b, _CMP_EQ_UQ)
|
||||
#define psel(a, b, false_mask) _mm256_blendv_ps(a, b, false_mask)
|
||||
#else
|
||||
#define pequal(a, b) _mm_cmpeq_ps(a, b)
|
||||
#define psel(a, b, false_mask) \
|
||||
_mm_or_ps(_mm_andnot_ps(false_mask, a), _mm_and_ps(false_mask, b))
|
||||
#endif
|
||||
|
||||
template <typename Packet>
|
||||
EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void reducePacket(const Packet& p,
|
||||
Packet* accum) {
|
||||
reducePacketWithType(static_cast<T>(0), p, accum);
|
||||
}
|
||||
|
||||
template <typename Packet>
|
||||
EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void reducePacketWithType(
|
||||
T, const Packet& p, Packet* accum) {
|
||||
Packet skip_mask =
|
||||
pequal(p, pset1<Packet>(-Eigen::NumTraits<T>::highest()));
|
||||
(*accum) = padd<Packet>(*accum, psel(p, pset1<Packet>(0), skip_mask));
|
||||
packetCount_ = padd<Packet>(
|
||||
packetCount_, psel(pset1<Packet>(1), pset1<Packet>(0), skip_mask));
|
||||
}
|
||||
|
||||
template <typename Packet>
|
||||
EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet initializePacket() const {
|
||||
return pset1<Packet>(0);
|
||||
}
|
||||
|
||||
template <typename Packet>
|
||||
EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet
|
||||
finalizePacket(const Packet& vaccum) const {
|
||||
return pdiv(vaccum, packetCount_);
|
||||
}
|
||||
template <typename Packet>
|
||||
EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE T
|
||||
finalizeBoth(const T saccum, const Packet& vaccum) const {
|
||||
return (saccum + predux(vaccum)) / (scalarCount_ + predux(packetCount_));
|
||||
}
|
||||
#endif
|
||||
|
||||
protected:
|
||||
typedef typename packet_traits<T>::type Packet;
|
||||
int scalarCount_;
|
||||
Packet packetCount_;
|
||||
};
|
||||
|
||||
template <typename Device>
|
||||
struct reducer_traits<AvgPoolMeanReducer<float>, Device> {
|
||||
enum {
|
||||
Cost = 1,
|
||||
#if (EIGEN_ARCH_i386 || EIGEN_ARCH_x86_64) && !defined(__CUDACC__)
|
||||
// We only support packet access for floats.
|
||||
PacketAccess = true,
|
||||
#else
|
||||
PacketAccess = false,
|
||||
#endif
|
||||
IsStateful = true,
|
||||
IsExactlyAssociative = false
|
||||
};
|
||||
};
|
||||
|
||||
template <>
|
||||
struct reducer_traits<AvgPoolMeanReducer<float>, GpuDevice> {
|
||||
enum {
|
||||
Cost = 1,
|
||||
PacketAccess = false,
|
||||
IsStateful = true,
|
||||
IsExactlyAssociative = false
|
||||
};
|
||||
};
|
||||
|
||||
} // namespace internal
|
||||
|
||||
#if !defined(EIGEN_HAS_INDEX_LIST)
|
||||
template <typename Input>
|
||||
EIGEN_ALWAYS_INLINE static const TensorReshapingOp<
|
||||
const Eigen::DSizes<typename internal::traits<Input>::Index,
|
||||
internal::traits<Input>::NumDimensions>,
|
||||
const TensorReductionOp<
|
||||
internal::AvgPoolMeanReducer<typename internal::remove_const<
|
||||
typename internal::traits<Input>::Scalar>::type>,
|
||||
const Eigen::array<int, 2>,
|
||||
const TensorImagePatchOp<Dynamic, Dynamic, const Input> > >
|
||||
#else
|
||||
template <typename Input>
|
||||
EIGEN_ALWAYS_INLINE static const TensorReshapingOp<
|
||||
const Eigen::DSizes<typename internal::traits<Input>::Index,
|
||||
internal::traits<Input>::NumDimensions>,
|
||||
const TensorReductionOp<
|
||||
internal::AvgPoolMeanReducer<typename internal::remove_const<
|
||||
typename internal::traits<Input>::Scalar>::type>,
|
||||
typename internal::conditional<
|
||||
internal::traits<Input>::Layout == ColMajor,
|
||||
const Eigen::IndexList<Eigen::type2index<1>, Eigen::type2index<2> >,
|
||||
const Eigen::IndexList<Eigen::type2index<2>,
|
||||
Eigen::type2index<3> > >::type,
|
||||
const TensorImagePatchOp<Dynamic, Dynamic, const Input> > >
|
||||
#endif
|
||||
SpatialAvgPooling(const Input& input, DenseIndex patchRows,
|
||||
DenseIndex patchCols, DenseIndex strideRows,
|
||||
DenseIndex strideCols, const PaddingType padding_type,
|
||||
DenseIndex in_strideRows = 1, DenseIndex in_strideCols = 1) {
|
||||
EIGEN_STATIC_ASSERT(internal::traits<Input>::NumDimensions == 4,
|
||||
YOU_MADE_A_PROGRAMMING_MISTAKE);
|
||||
|
||||
typedef typename internal::traits<Input>::Index TensorIndex;
|
||||
TensorRef<Tensor<typename internal::traits<Input>::Scalar,
|
||||
internal::traits<Input>::NumDimensions,
|
||||
internal::traits<Input>::Layout, TensorIndex> >
|
||||
in(input);
|
||||
|
||||
const DenseIndex patchRowsEff =
|
||||
patchRows + (patchRows - 1) * (in_strideRows - 1);
|
||||
const DenseIndex patchColsEff =
|
||||
patchCols + (patchCols - 1) * (in_strideCols - 1);
|
||||
|
||||
static const bool isColMajor = (internal::traits<Input>::Layout == ColMajor);
|
||||
static const int idxRows = isColMajor ? 1 : 2;
|
||||
static const int idxCols = isColMajor ? 2 : 1;
|
||||
|
||||
// Molds the output of the reduction into the shape expected by the user.
|
||||
// (assuming col-major):
|
||||
// - 1st dim: channels
|
||||
// - 2nd dim: output height
|
||||
// - 3rd dim: output width
|
||||
// - 4th dim and beyond: everything else including batch size
|
||||
Eigen::DSizes<TensorIndex, internal::traits<Input>::NumDimensions>
|
||||
post_reduce_dims;
|
||||
post_reduce_dims[0] = in.dimension(0);
|
||||
if (padding_type == PADDING_VALID) {
|
||||
post_reduce_dims[idxRows] = Eigen::divup(
|
||||
static_cast<DenseIndex>(in.dimension(idxRows)) - patchRowsEff + 1,
|
||||
strideRows);
|
||||
post_reduce_dims[idxCols] = Eigen::divup(
|
||||
static_cast<DenseIndex>(in.dimension(idxCols)) - patchColsEff + 1,
|
||||
strideCols);
|
||||
} else {
|
||||
post_reduce_dims[idxRows] = Eigen::divup(
|
||||
static_cast<DenseIndex>(in.dimension(idxRows)), strideRows);
|
||||
post_reduce_dims[idxCols] = Eigen::divup(
|
||||
static_cast<DenseIndex>(in.dimension(idxCols)), strideCols);
|
||||
}
|
||||
post_reduce_dims[3] = in.dimension(3);
|
||||
|
||||
typedef typename internal::remove_const<
|
||||
typename internal::traits<Input>::Scalar>::type CoeffReturnType;
|
||||
internal::AvgPoolMeanReducer<CoeffReturnType> mean_with_nan;
|
||||
|
||||
#if !defined(EIGEN_HAS_INDEX_LIST)
|
||||
// nvcc doesn't support cxx11
|
||||
Eigen::array<int, 2> reduction_dims;
|
||||
if (isColMajor) {
|
||||
reduction_dims[0] = 1;
|
||||
reduction_dims[1] = 2;
|
||||
} else {
|
||||
reduction_dims[0] = 2;
|
||||
reduction_dims[1] = 3;
|
||||
}
|
||||
#else
|
||||
// Take advantage of cxx11 to give the compiler information it can use to
|
||||
// optimize the code.
|
||||
typename internal::conditional<
|
||||
internal::traits<Input>::Layout == ColMajor,
|
||||
const Eigen::IndexList<Eigen::type2index<1>, Eigen::type2index<2> >,
|
||||
const Eigen::IndexList<Eigen::type2index<2>,
|
||||
Eigen::type2index<3> > >::type reduction_dims;
|
||||
#endif
|
||||
return input
|
||||
.extract_image_patches(patchRows, patchCols, strideRows, strideCols,
|
||||
in_strideRows, in_strideCols, padding_type,
|
||||
-Eigen::NumTraits<CoeffReturnType>::highest())
|
||||
.reduce(reduction_dims, mean_with_nan)
|
||||
.reshape(post_reduce_dims);
|
||||
}
|
||||
|
||||
/** CuboidAvgPooling
|
||||
* \ingroup CXX11_NeuralNetworks_Module
|
||||
*
|
||||
* \brief Applies an average pooling over a multichannel input volume.
|
||||
*
|
||||
* The input parameter is expected to be a tensor with a rank of 5 (channels,
|
||||
* depth, height, width, others, and the reverse of that in row-major).
|
||||
*
|
||||
* The result can be assigned to a tensor of rank equal to the rank of the
|
||||
* input. The dimensions of the result will be channels, depth, width, and
|
||||
* others (in col-major, and the reverse of that if the input was row-major).
|
||||
*
|
||||
* The order of the depth, width and height dimensions can be swapped if
|
||||
* needed.
|
||||
*
|
||||
*/
|
||||
#if !defined(EIGEN_HAS_INDEX_LIST)
|
||||
template <typename Input>
|
||||
EIGEN_ALWAYS_INLINE static const TensorReshapingOp<
|
||||
const Eigen::DSizes<DenseIndex, internal::traits<Input>::NumDimensions>,
|
||||
const TensorReductionOp<
|
||||
internal::AvgPoolMeanReducer<float>, const Eigen::array<int, 1>,
|
||||
const TensorReshapingOp<
|
||||
const Eigen::DSizes<DenseIndex, 3>,
|
||||
const TensorVolumePatchOp<Dynamic, Dynamic, Dynamic,
|
||||
const Input> > > >
|
||||
#else
|
||||
template <typename Input>
|
||||
EIGEN_ALWAYS_INLINE static const TensorReshapingOp<
|
||||
const Eigen::DSizes<DenseIndex, internal::traits<Input>::NumDimensions>,
|
||||
const TensorReductionOp<
|
||||
internal::AvgPoolMeanReducer<float>,
|
||||
const Eigen::IndexList<Eigen::type2index<1> >,
|
||||
const TensorReshapingOp<
|
||||
const Eigen::DSizes<DenseIndex, 3>,
|
||||
const TensorVolumePatchOp<Dynamic, Dynamic, Dynamic,
|
||||
const Input> > > >
|
||||
#endif
|
||||
CuboidAvgPooling(const Input& input, DenseIndex patchPlanes,
|
||||
DenseIndex patchRows, DenseIndex patchCols,
|
||||
DenseIndex stridePlanes, DenseIndex strideRows,
|
||||
DenseIndex strideCols, const PaddingType padding_type) {
|
||||
EIGEN_STATIC_ASSERT(internal::traits<Input>::NumDimensions == 5,
|
||||
YOU_MADE_A_PROGRAMMING_MISTAKE);
|
||||
static const bool isColMajor = (internal::traits<Input>::Layout == ColMajor);
|
||||
|
||||
typedef typename internal::traits<Input>::Index TensorIndex;
|
||||
TensorRef<Tensor<typename internal::traits<Input>::Scalar,
|
||||
internal::traits<Input>::NumDimensions,
|
||||
internal::traits<Input>::Layout, TensorIndex> >
|
||||
in(input);
|
||||
|
||||
static const int idxPlanes = isColMajor ? 1 : 3;
|
||||
static const int idxRows = 2;
|
||||
static const int idxCols = isColMajor ? 3 : 1;
|
||||
// Molds the output of the reduction into the shape expected by the used
|
||||
// (assuming col-major):
|
||||
// - 1st dim: channels
|
||||
// - 2nd dim: outupt depth
|
||||
// - 3rd dim: output height
|
||||
// - 4th dim: output width
|
||||
// - 5th dim and beyond: everything else including batch size
|
||||
Eigen::DSizes<DenseIndex, internal::traits<Input>::NumDimensions>
|
||||
post_reduce_dims;
|
||||
post_reduce_dims[0] = in.dimension(0);
|
||||
if (padding_type == PADDING_VALID) {
|
||||
post_reduce_dims[idxPlanes] = Eigen::divup(
|
||||
static_cast<DenseIndex>(in.dimension(idxPlanes)) - patchPlanes + 1,
|
||||
stridePlanes);
|
||||
post_reduce_dims[idxRows] = Eigen::divup(
|
||||
static_cast<DenseIndex>(in.dimension(idxRows)) - patchRows + 1,
|
||||
strideRows);
|
||||
post_reduce_dims[idxCols] = Eigen::divup(
|
||||
static_cast<DenseIndex>(in.dimension(idxCols)) - patchCols + 1,
|
||||
strideCols);
|
||||
} else {
|
||||
post_reduce_dims[idxPlanes] = Eigen::divup(
|
||||
static_cast<DenseIndex>(in.dimension(idxPlanes)), stridePlanes);
|
||||
post_reduce_dims[idxRows] = Eigen::divup(
|
||||
static_cast<DenseIndex>(in.dimension(idxRows)), strideRows);
|
||||
post_reduce_dims[idxCols] = Eigen::divup(
|
||||
static_cast<DenseIndex>(in.dimension(idxCols)), strideCols);
|
||||
}
|
||||
post_reduce_dims[4] = in.dimension(4);
|
||||
|
||||
Eigen::DSizes<DenseIndex, 3> pre_reduce_dims;
|
||||
pre_reduce_dims[1] = patchRows * patchCols * patchPlanes;
|
||||
if (isColMajor) {
|
||||
pre_reduce_dims[0] = post_reduce_dims[0];
|
||||
pre_reduce_dims[2] = post_reduce_dims[1] * post_reduce_dims[2] *
|
||||
post_reduce_dims[3] * post_reduce_dims[4];
|
||||
} else {
|
||||
pre_reduce_dims[0] = post_reduce_dims[0] * post_reduce_dims[1] *
|
||||
post_reduce_dims[2] * post_reduce_dims[3];
|
||||
pre_reduce_dims[2] = post_reduce_dims[4];
|
||||
}
|
||||
|
||||
typedef typename internal::remove_const<
|
||||
typename internal::traits<Input>::Scalar>::type CoeffReturnType;
|
||||
internal::AvgPoolMeanReducer<CoeffReturnType> mean_with_nan;
|
||||
|
||||
#if !defined(EIGEN_HAS_INDEX_LIST)
|
||||
// nvcc doesn't support cxx11
|
||||
Eigen::array<int, 1> reduction_dims;
|
||||
reduction_dims[0] = 1;
|
||||
#else
|
||||
// Take advantage of cxx11 to give the compiler information it can use to
|
||||
// optimize the code.
|
||||
Eigen::IndexList<Eigen::type2index<1> > reduction_dims;
|
||||
#endif
|
||||
return input
|
||||
.extract_volume_patches(patchPlanes, patchRows, patchCols, stridePlanes,
|
||||
strideRows, strideCols, padding_type,
|
||||
-Eigen::NumTraits<float>::highest())
|
||||
.reshape(pre_reduce_dims)
|
||||
.reduce(reduction_dims, mean_with_nan)
|
||||
.reshape(post_reduce_dims);
|
||||
}
|
||||
|
||||
} // end namespace Eigen
|
||||
|
||||
#endif // TENSORFLOW_CORE_KERNELS_EIGEN_POOLING_H_
|
||||
@@ -0,0 +1,99 @@
|
||||
/* Copyright 2015 The TensorFlow Authors. All Rights Reserved.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
==============================================================================*/
|
||||
|
||||
#ifndef TENSORFLOW_CORE_KERNELS_EIGEN_SOFTMAX_H_
|
||||
#define TENSORFLOW_CORE_KERNELS_EIGEN_SOFTMAX_H_
|
||||
|
||||
#include "third_party/eigen3/unsupported/Eigen/CXX11/Tensor"
|
||||
|
||||
namespace Eigen {
|
||||
|
||||
/** SoftMax
|
||||
* \ingroup CXX11_NeuralNetworks_Module
|
||||
*
|
||||
* \brief Applies a softmax
|
||||
*
|
||||
* The input parameter is expected to be a col-major tensor with a rank of 2
|
||||
* (depth and other).
|
||||
*
|
||||
* The result can be assigned to a tensor of rank and dimensions equal to that
|
||||
* of the input. The result will be laid out in col-major order.
|
||||
*
|
||||
*/
|
||||
|
||||
namespace {
|
||||
struct SoftmaxOp {
|
||||
SoftmaxOp(const float beta) : beta_(beta) {}
|
||||
|
||||
template <typename Input>
|
||||
typename Input::Dimensions dimensions(const Input& input) const {
|
||||
return input.dimensions();
|
||||
}
|
||||
|
||||
template <typename Input, typename Output, typename Device>
|
||||
void eval(const Input& input, Output& output, const Device& device) const {
|
||||
#if !defined(EIGEN_HAS_INDEX_LIST)
|
||||
// nvcc doesn't support cxx11
|
||||
Eigen::array<typename internal::traits<Input>::Index, 1> depth_dim;
|
||||
depth_dim[0] = 0;
|
||||
Eigen::array<typename internal::traits<Input>::Index, 2> bcast;
|
||||
bcast[0] = dimensions(input)[0];
|
||||
bcast[1] = 1;
|
||||
DSizes<typename internal::traits<Input>::Index, 2> dims2d;
|
||||
dims2d[0] = 1;
|
||||
dims2d[1] = dimensions(input)[1];
|
||||
#else
|
||||
// Take advantage of cxx11 to give the compiler information it can use to
|
||||
// optimize the code.
|
||||
Eigen::IndexList<Eigen::type2index<0> > depth_dim;
|
||||
Eigen::IndexList<int, Eigen::type2index<1> > bcast;
|
||||
bcast.set(0, dimensions(input)[0]);
|
||||
Eigen::IndexList<Eigen::type2index<1>,
|
||||
typename internal::traits<Input>::Index>
|
||||
dims2d;
|
||||
dims2d.set(1, dimensions(input)[1]);
|
||||
#endif
|
||||
|
||||
output.device(device) =
|
||||
((input -
|
||||
input.maximum(depth_dim).eval().reshape(dims2d).broadcast(bcast)) *
|
||||
beta_)
|
||||
.exp();
|
||||
output.device(device) =
|
||||
output /
|
||||
(output.sum(depth_dim).eval().reshape(dims2d).broadcast(bcast));
|
||||
}
|
||||
|
||||
private:
|
||||
const float beta_;
|
||||
};
|
||||
} // namespace
|
||||
|
||||
template <typename Input>
|
||||
EIGEN_ALWAYS_INLINE static const TensorCustomUnaryOp<const SoftmaxOp,
|
||||
const Input>
|
||||
SoftMax(const Input& input, const float beta) {
|
||||
EIGEN_STATIC_ASSERT(internal::traits<Input>::Layout == ColMajor,
|
||||
YOU_MADE_A_PROGRAMMING_MISTAKE);
|
||||
EIGEN_STATIC_ASSERT(internal::traits<Input>::NumDimensions == 2,
|
||||
YOU_MADE_A_PROGRAMMING_MISTAKE);
|
||||
|
||||
const SoftmaxOp op(beta);
|
||||
return input.customOp(op);
|
||||
}
|
||||
|
||||
} // end namespace Eigen
|
||||
|
||||
#endif // TENSORFLOW_CORE_KERNELS_EIGEN_SOFTMAX_H_
|
||||
File diff suppressed because it is too large.
Load diff
@@ -0,0 +1,657 @@
|
||||
/* Copyright 2017 The TensorFlow Authors. All Rights Reserved.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
==============================================================================*/
|
||||
|
||||
#ifndef TENSORFLOW_CORE_KERNELS_EIGEN_VOLUME_PATCH_H_
|
||||
#define TENSORFLOW_CORE_KERNELS_EIGEN_VOLUME_PATCH_H_
|
||||
|
||||
#include "third_party/eigen3/unsupported/Eigen/CXX11/Tensor"
|
||||
|
||||
namespace Eigen {
|
||||
|
||||
// Changes the interpretation of padding in TensorVolumePatchOp to be compatible
|
||||
// with the rest of TensorFlow (odd padding is split so that more padding is put
|
||||
// on the right end of the tensor).
|
||||
template <DenseIndex Planes, DenseIndex Rows, DenseIndex Cols, typename ArgType,
|
||||
typename Device>
|
||||
struct CustomTensorEvaluator {
|
||||
typedef TensorVolumePatchOp<Planes, Rows, Cols, ArgType> XprType;
|
||||
typedef typename XprType::Index Index;
|
||||
static const int NumInputDims = internal::array_size<
|
||||
typename TensorEvaluator<ArgType, Device>::Dimensions>::value;
|
||||
static const int NumDims = NumInputDims + 1;
|
||||
typedef DSizes<Index, NumDims> Dimensions;
|
||||
typedef
|
||||
typename internal::remove_const<typename XprType::Scalar>::type Scalar;
|
||||
typedef typename XprType::CoeffReturnType CoeffReturnType;
|
||||
typedef typename PacketType<CoeffReturnType, Device>::type PacketReturnType;
|
||||
static const Index PacketSize =
|
||||
internal::unpacket_traits<PacketReturnType>::size;
|
||||
|
||||
enum {
|
||||
IsAligned = false,
|
||||
PacketAccess = TensorEvaluator<ArgType, Device>::PacketAccess,
|
||||
BlockAccess = false,
|
||||
PreferBlockAccess = false,
|
||||
Layout = TensorEvaluator<ArgType, Device>::Layout,
|
||||
CoordAccess = NumDims == 6,
|
||||
RawAccess = false
|
||||
};
|
||||
|
||||
EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE
|
||||
CustomTensorEvaluator(const XprType& op, const Device& device)
|
||||
: m_impl(op.expression(), device) {
|
||||
EIGEN_STATIC_ASSERT(NumDims >= 5, YOU_MADE_A_PROGRAMMING_MISTAKE);
|
||||
|
||||
m_paddingValue = op.padding_value();
|
||||
|
||||
const typename TensorEvaluator<ArgType, Device>::Dimensions& input_dims =
|
||||
m_impl.dimensions();
|
||||
|
||||
// Cache a few variables.
|
||||
if (static_cast<int>(Layout) == static_cast<int>(ColMajor)) {
|
||||
m_inputDepth = input_dims[0];
|
||||
m_inputPlanes = input_dims[1];
|
||||
m_inputRows = input_dims[2];
|
||||
m_inputCols = input_dims[3];
|
||||
} else {
|
||||
m_inputDepth = input_dims[NumInputDims - 1];
|
||||
m_inputPlanes = input_dims[NumInputDims - 2];
|
||||
m_inputRows = input_dims[NumInputDims - 3];
|
||||
m_inputCols = input_dims[NumInputDims - 4];
|
||||
}
|
||||
|
||||
m_plane_strides = op.plane_strides();
|
||||
m_row_strides = op.row_strides();
|
||||
m_col_strides = op.col_strides();
|
||||
|
||||
// Input strides and effective input/patch size
|
||||
m_in_plane_strides = op.in_plane_strides();
|
||||
m_in_row_strides = op.in_row_strides();
|
||||
m_in_col_strides = op.in_col_strides();
|
||||
m_plane_inflate_strides = op.plane_inflate_strides();
|
||||
m_row_inflate_strides = op.row_inflate_strides();
|
||||
m_col_inflate_strides = op.col_inflate_strides();
|
||||
|
||||
// The "effective" spatial size after inflating data with zeros.
|
||||
m_input_planes_eff = (m_inputPlanes - 1) * m_plane_inflate_strides + 1;
|
||||
m_input_rows_eff = (m_inputRows - 1) * m_row_inflate_strides + 1;
|
||||
m_input_cols_eff = (m_inputCols - 1) * m_col_inflate_strides + 1;
|
||||
m_patch_planes_eff =
|
||||
op.patch_planes() + (op.patch_planes() - 1) * (m_in_plane_strides - 1);
|
||||
m_patch_rows_eff =
|
||||
op.patch_rows() + (op.patch_rows() - 1) * (m_in_row_strides - 1);
|
||||
m_patch_cols_eff =
|
||||
op.patch_cols() + (op.patch_cols() - 1) * (m_in_col_strides - 1);
|
||||
|
||||
if (op.padding_explicit()) {
|
||||
m_outputPlanes = Eigen::divup(
|
||||
m_input_planes_eff +
|
||||
static_cast<Index>(op.padding_top_z() + op.padding_bottom_z()) -
|
||||
m_patch_planes_eff + 1,
|
||||
m_plane_strides);
|
||||
m_outputRows = Eigen::divup(
|
||||
m_input_rows_eff +
|
||||
static_cast<Index>(op.padding_top() + op.padding_bottom()) -
|
||||
m_patch_rows_eff + 1,
|
||||
m_row_strides);
|
||||
m_outputCols = Eigen::divup(
|
||||
m_input_cols_eff +
|
||||
static_cast<Index>(op.padding_left() + op.padding_right()) -
|
||||
m_patch_cols_eff + 1,
|
||||
m_col_strides);
|
||||
m_planePaddingTop = op.padding_top_z();
|
||||
m_rowPaddingTop = op.padding_top();
|
||||
m_colPaddingLeft = op.padding_left();
|
||||
} else {
|
||||
// Computing padding from the type
|
||||
switch (op.padding_type()) {
|
||||
case PADDING_VALID:
|
||||
m_outputPlanes = Eigen::divup(
|
||||
m_input_planes_eff - m_patch_planes_eff + 1, m_plane_strides);
|
||||
m_outputRows = Eigen::divup(m_input_rows_eff - m_patch_rows_eff + 1,
|
||||
m_row_strides);
|
||||
m_outputCols = Eigen::divup(m_input_cols_eff - m_patch_cols_eff + 1,
|
||||
m_col_strides);
|
||||
m_planePaddingTop = 0;
|
||||
m_rowPaddingTop = 0;
|
||||
m_colPaddingLeft = 0;
|
||||
break;
|
||||
case PADDING_SAME: {
|
||||
m_outputPlanes = Eigen::divup(m_input_planes_eff, m_plane_strides);
|
||||
m_outputRows = Eigen::divup(m_input_rows_eff, m_row_strides);
|
||||
m_outputCols = Eigen::divup(m_input_cols_eff, m_col_strides);
|
||||
const Index dz = numext::maxi<DenseIndex>(
|
||||
0, (m_outputPlanes - 1) * m_plane_strides + m_patch_planes_eff -
|
||||
m_input_planes_eff);
|
||||
const Index dy = numext::maxi<DenseIndex>(
|
||||
0, (m_outputRows - 1) * m_row_strides + m_patch_rows_eff -
|
||||
m_input_rows_eff);
|
||||
const Index dx = numext::maxi<DenseIndex>(
|
||||
0, (m_outputCols - 1) * m_col_strides + m_patch_cols_eff -
|
||||
m_input_cols_eff);
|
||||
m_planePaddingTop = dz / 2;
|
||||
m_rowPaddingTop = dy / 2;
|
||||
m_colPaddingLeft = dx / 2;
|
||||
break;
|
||||
}
|
||||
default:
|
||||
eigen_assert(false && "unexpected padding");
|
||||
}
|
||||
}
|
||||
eigen_assert(m_outputRows > 0);
|
||||
eigen_assert(m_outputCols > 0);
|
||||
eigen_assert(m_outputPlanes > 0);
|
||||
|
||||
// Dimensions for result of extraction.
|
||||
if (static_cast<int>(Layout) == static_cast<int>(ColMajor)) {
|
||||
// ColMajor
|
||||
// 0: depth
|
||||
// 1: patch_planes
|
||||
// 2: patch_rows
|
||||
// 3: patch_cols
|
||||
// 4: number of patches
|
||||
// 5 and beyond: anything else (such as batch).
|
||||
m_dimensions[0] = input_dims[0];
|
||||
m_dimensions[1] = op.patch_planes();
|
||||
m_dimensions[2] = op.patch_rows();
|
||||
m_dimensions[3] = op.patch_cols();
|
||||
m_dimensions[4] = m_outputPlanes * m_outputRows * m_outputCols;
|
||||
for (int i = 5; i < NumDims; ++i) {
|
||||
m_dimensions[i] = input_dims[i - 1];
|
||||
}
|
||||
} else {
|
||||
// RowMajor
|
||||
// NumDims-1: depth
|
||||
// NumDims-2: patch_planes
|
||||
// NumDims-3: patch_rows
|
||||
// NumDims-4: patch_cols
|
||||
// NumDims-5: number of patches
|
||||
// NumDims-6 and beyond: anything else (such as batch).
|
||||
m_dimensions[NumDims - 1] = input_dims[NumInputDims - 1];
|
||||
m_dimensions[NumDims - 2] = op.patch_planes();
|
||||
m_dimensions[NumDims - 3] = op.patch_rows();
|
||||
m_dimensions[NumDims - 4] = op.patch_cols();
|
||||
m_dimensions[NumDims - 5] = m_outputPlanes * m_outputRows * m_outputCols;
|
||||
for (int i = NumDims - 6; i >= 0; --i) {
|
||||
m_dimensions[i] = input_dims[i];
|
||||
}
|
||||
}
|
||||
|
||||
// Strides for the output tensor.
|
||||
if (static_cast<int>(Layout) == static_cast<int>(ColMajor)) {
|
||||
m_rowStride = m_dimensions[1];
|
||||
m_colStride = m_dimensions[2] * m_rowStride;
|
||||
m_patchStride = m_colStride * m_dimensions[3] * m_dimensions[0];
|
||||
m_otherStride = m_patchStride * m_dimensions[4];
|
||||
} else {
|
||||
m_rowStride = m_dimensions[NumDims - 2];
|
||||
m_colStride = m_dimensions[NumDims - 3] * m_rowStride;
|
||||
m_patchStride =
|
||||
m_colStride * m_dimensions[NumDims - 4] * m_dimensions[NumDims - 1];
|
||||
m_otherStride = m_patchStride * m_dimensions[NumDims - 5];
|
||||
}
|
||||
|
||||
// Strides for navigating through the input tensor.
|
||||
m_planeInputStride = m_inputDepth;
|
||||
m_rowInputStride = m_inputDepth * m_inputPlanes;
|
||||
m_colInputStride = m_inputDepth * m_inputRows * m_inputPlanes;
|
||||
m_otherInputStride =
|
||||
m_inputDepth * m_inputRows * m_inputCols * m_inputPlanes;
|
||||
|
||||
m_outputPlanesRows = m_outputPlanes * m_outputRows;
|
||||
|
||||
// Fast representations of different variables.
|
||||
m_fastOtherStride = internal::TensorIntDivisor<Index>(m_otherStride);
|
||||
m_fastPatchStride = internal::TensorIntDivisor<Index>(m_patchStride);
|
||||
m_fastColStride = internal::TensorIntDivisor<Index>(m_colStride);
|
||||
m_fastRowStride = internal::TensorIntDivisor<Index>(m_rowStride);
|
||||
m_fastInputRowStride =
|
||||
internal::TensorIntDivisor<Index>(m_row_inflate_strides);
|
||||
m_fastInputColStride =
|
||||
internal::TensorIntDivisor<Index>(m_col_inflate_strides);
|
||||
m_fastInputPlaneStride =
|
||||
internal::TensorIntDivisor<Index>(m_plane_inflate_strides);
|
||||
m_fastInputColsEff = internal::TensorIntDivisor<Index>(m_input_cols_eff);
|
||||
m_fastOutputPlanes = internal::TensorIntDivisor<Index>(m_outputPlanes);
|
||||
m_fastOutputPlanesRows =
|
||||
internal::TensorIntDivisor<Index>(m_outputPlanesRows);
|
||||
|
||||
if (static_cast<int>(Layout) == static_cast<int>(ColMajor)) {
|
||||
m_fastOutputDepth = internal::TensorIntDivisor<Index>(m_dimensions[0]);
|
||||
} else {
|
||||
m_fastOutputDepth =
|
||||
internal::TensorIntDivisor<Index>(m_dimensions[NumDims - 1]);
|
||||
}
|
||||
}
|
||||
|
||||
EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const Dimensions& dimensions() const {
|
||||
return m_dimensions;
|
||||
}
|
||||
|
||||
EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE bool evalSubExprsIfNeeded(
|
||||
Scalar* /*data*/) {
|
||||
m_impl.evalSubExprsIfNeeded(NULL);
|
||||
return true;
|
||||
}
|
||||
|
||||
EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void cleanup() { m_impl.cleanup(); }
|
||||
|
||||
EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE CoeffReturnType
|
||||
coeff(Index index) const {
|
||||
// Patch index corresponding to the passed in index.
|
||||
const Index patchIndex = index / m_fastPatchStride;
|
||||
|
||||
// Spatial offset within the patch. This has to be translated into 3D
|
||||
// coordinates within the patch.
|
||||
const Index patchOffset =
|
||||
(index - patchIndex * m_patchStride) / m_fastOutputDepth;
|
||||
|
||||
// Batch, etc.
|
||||
const Index otherIndex = (NumDims == 5) ? 0 : index / m_fastOtherStride;
|
||||
const Index patch3DIndex =
|
||||
(NumDims == 5)
|
||||
? patchIndex
|
||||
: (index - otherIndex * m_otherStride) / m_fastPatchStride;
|
||||
|
||||
// Calculate column index in the input original tensor.
|
||||
const Index colIndex = patch3DIndex / m_fastOutputPlanesRows;
|
||||
const Index colOffset = patchOffset / m_fastColStride;
|
||||
const Index inputCol = colIndex * m_col_strides +
|
||||
colOffset * m_in_col_strides - m_colPaddingLeft;
|
||||
const Index origInputCol =
|
||||
(m_col_inflate_strides == 1)
|
||||
? inputCol
|
||||
: ((inputCol >= 0) ? (inputCol / m_fastInputColStride) : 0);
|
||||
if (inputCol < 0 || inputCol >= m_input_cols_eff ||
|
||||
((m_col_inflate_strides != 1) &&
|
||||
(inputCol != origInputCol * m_col_inflate_strides))) {
|
||||
return Scalar(m_paddingValue);
|
||||
}
|
||||
|
||||
// Calculate row index in the original input tensor.
|
||||
const Index rowIndex =
|
||||
(patch3DIndex - colIndex * m_outputPlanesRows) / m_fastOutputPlanes;
|
||||
const Index rowOffset =
|
||||
(patchOffset - colOffset * m_colStride) / m_fastRowStride;
|
||||
const Index inputRow = rowIndex * m_row_strides +
|
||||
rowOffset * m_in_row_strides - m_rowPaddingTop;
|
||||
const Index origInputRow =
|
||||
(m_row_inflate_strides == 1)
|
||||
? inputRow
|
||||
: ((inputRow >= 0) ? (inputRow / m_fastInputRowStride) : 0);
|
||||
if (inputRow < 0 || inputRow >= m_input_rows_eff ||
|
||||
((m_row_inflate_strides != 1) &&
|
||||
(inputRow != origInputRow * m_row_inflate_strides))) {
|
||||
return Scalar(m_paddingValue);
|
||||
}
|
||||
|
||||
// Calculate plane index in the original input tensor.
|
||||
const Index planeIndex =
|
||||
(patch3DIndex - m_outputPlanes * (colIndex * m_outputRows + rowIndex));
|
||||
const Index planeOffset =
|
||||
patchOffset - colOffset * m_colStride - rowOffset * m_rowStride;
|
||||
const Index inputPlane = planeIndex * m_plane_strides +
|
||||
planeOffset * m_in_plane_strides -
|
||||
m_planePaddingTop;
|
||||
const Index origInputPlane =
|
||||
(m_plane_inflate_strides == 1)
|
||||
? inputPlane
|
||||
: ((inputPlane >= 0) ? (inputPlane / m_fastInputPlaneStride) : 0);
|
||||
if (inputPlane < 0 || inputPlane >= m_input_planes_eff ||
|
||||
((m_plane_inflate_strides != 1) &&
|
||||
(inputPlane != origInputPlane * m_plane_inflate_strides))) {
|
||||
return Scalar(m_paddingValue);
|
||||
}
|
||||
|
||||
const int depth_index =
|
||||
static_cast<int>(Layout) == static_cast<int>(ColMajor) ? 0
|
||||
: NumDims - 1;
|
||||
const Index depth =
|
||||
index - (index / m_fastOutputDepth) * m_dimensions[depth_index];
|
||||
|
||||
const Index inputIndex = depth + origInputRow * m_rowInputStride +
|
||||
origInputCol * m_colInputStride +
|
||||
origInputPlane * m_planeInputStride +
|
||||
otherIndex * m_otherInputStride;
|
||||
|
||||
return m_impl.coeff(inputIndex);
|
||||
}
|
||||
|
||||
template <int LoadMode>
|
||||
EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE PacketReturnType
|
||||
packet(Index index) const {
|
||||
EIGEN_STATIC_ASSERT(PacketSize > 1, YOU_MADE_A_PROGRAMMING_MISTAKE)
|
||||
eigen_assert(index + PacketSize - 1 < dimensions().TotalSize());
|
||||
|
||||
if (m_in_row_strides != 1 || m_in_col_strides != 1 ||
|
||||
m_row_inflate_strides != 1 || m_col_inflate_strides != 1 ||
|
||||
m_in_plane_strides != 1 || m_plane_inflate_strides != 1) {
|
||||
return packetWithPossibleZero(index);
|
||||
}
|
||||
|
||||
const Index indices[2] = {index, index + PacketSize - 1};
|
||||
const Index patchIndex = indices[0] / m_fastPatchStride;
|
||||
if (patchIndex != indices[1] / m_fastPatchStride) {
|
||||
return packetWithPossibleZero(index);
|
||||
}
|
||||
const Index otherIndex =
|
||||
(NumDims == 5) ? 0 : indices[0] / m_fastOtherStride;
|
||||
eigen_assert(otherIndex == indices[1] / m_fastOtherStride);
|
||||
|
||||
// Find the offset of the element wrt the location of the first element.
|
||||
const Index patchOffsets[2] = {
|
||||
(indices[0] - patchIndex * m_patchStride) / m_fastOutputDepth,
|
||||
(indices[1] - patchIndex * m_patchStride) / m_fastOutputDepth};
|
||||
|
||||
const Index patch3DIndex =
|
||||
(NumDims == 5)
|
||||
? patchIndex
|
||||
: (indices[0] - otherIndex * m_otherStride) / m_fastPatchStride;
|
||||
eigen_assert(patch3DIndex ==
|
||||
(indices[1] - otherIndex * m_otherStride) / m_fastPatchStride);
|
||||
|
||||
const Index colIndex = patch3DIndex / m_fastOutputPlanesRows;
|
||||
const Index colOffsets[2] = {patchOffsets[0] / m_fastColStride,
|
||||
patchOffsets[1] / m_fastColStride};
|
||||
|
||||
// Calculate col indices in the original input tensor.
|
||||
const Index inputCols[2] = {
|
||||
colIndex * m_col_strides + colOffsets[0] - m_colPaddingLeft,
|
||||
colIndex * m_col_strides + colOffsets[1] - m_colPaddingLeft};
|
||||
if (inputCols[1] < 0 || inputCols[0] >= m_inputCols) {
|
||||
return internal::pset1<PacketReturnType>(Scalar(m_paddingValue));
|
||||
}
|
||||
|
||||
if (inputCols[0] != inputCols[1]) {
|
||||
return packetWithPossibleZero(index);
|
||||
}
|
||||
|
||||
const Index rowIndex =
|
||||
(patch3DIndex - colIndex * m_outputPlanesRows) / m_fastOutputPlanes;
|
||||
const Index rowOffsets[2] = {
|
||||
(patchOffsets[0] - colOffsets[0] * m_colStride) / m_fastRowStride,
|
||||
(patchOffsets[1] - colOffsets[1] * m_colStride) / m_fastRowStride};
|
||||
eigen_assert(rowOffsets[0] <= rowOffsets[1]);
|
||||
// Calculate col indices in the original input tensor.
|
||||
const Index inputRows[2] = {
|
||||
rowIndex * m_row_strides + rowOffsets[0] - m_rowPaddingTop,
|
||||
rowIndex * m_row_strides + rowOffsets[1] - m_rowPaddingTop};
|
||||
|
||||
if (inputRows[1] < 0 || inputRows[0] >= m_inputRows) {
|
||||
return internal::pset1<PacketReturnType>(Scalar(m_paddingValue));
|
||||
}
|
||||
|
||||
if (inputRows[0] != inputRows[1]) {
|
||||
return packetWithPossibleZero(index);
|
||||
}
|
||||
|
||||
const Index planeIndex =
|
||||
(patch3DIndex - m_outputPlanes * (colIndex * m_outputRows + rowIndex));
|
||||
const Index planeOffsets[2] = {
|
||||
patchOffsets[0] - colOffsets[0] * m_colStride -
|
||||
rowOffsets[0] * m_rowStride,
|
||||
patchOffsets[1] - colOffsets[1] * m_colStride -
|
||||
rowOffsets[1] * m_rowStride};
|
||||
eigen_assert(planeOffsets[0] <= planeOffsets[1]);
|
||||
const Index inputPlanes[2] = {
|
||||
planeIndex * m_plane_strides + planeOffsets[0] - m_planePaddingTop,
|
||||
planeIndex * m_plane_strides + planeOffsets[1] - m_planePaddingTop};
|
||||
|
||||
if (inputPlanes[1] < 0 || inputPlanes[0] >= m_inputPlanes) {
|
||||
return internal::pset1<PacketReturnType>(Scalar(m_paddingValue));
|
||||
}
|
||||
|
||||
if (inputPlanes[0] >= 0 && inputPlanes[1] < m_inputPlanes) {
|
||||
// no padding
|
||||
const int depth_index =
|
||||
static_cast<int>(Layout) == static_cast<int>(ColMajor) ? 0
|
||||
: NumDims - 1;
|
||||
const Index depth =
|
||||
index - (index / m_fastOutputDepth) * m_dimensions[depth_index];
|
||||
const Index inputIndex = depth + inputRows[0] * m_rowInputStride +
|
||||
inputCols[0] * m_colInputStride +
|
||||
m_planeInputStride * inputPlanes[0] +
|
||||
otherIndex * m_otherInputStride;
|
||||
return m_impl.template packet<Unaligned>(inputIndex);
|
||||
}
|
||||
|
||||
return packetWithPossibleZero(index);
|
||||
}
|
||||
|
||||
EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE TensorOpCost
|
||||
costPerCoeff(bool vectorized) const {
|
||||
const double compute_cost = 10 * TensorOpCost::DivCost<Index>() +
|
||||
21 * TensorOpCost::MulCost<Index>() +
|
||||
8 * TensorOpCost::AddCost<Index>();
|
||||
return TensorOpCost(0, 0, compute_cost, vectorized, PacketSize);
|
||||
}
|
||||
|
||||
EIGEN_DEVICE_FUNC Scalar* data() const { return NULL; }
|
||||
|
||||
const TensorEvaluator<ArgType, Device>& impl() const { return m_impl; }
|
||||
|
||||
Index planePaddingTop() const { return m_planePaddingTop; }
|
||||
Index rowPaddingTop() const { return m_rowPaddingTop; }
|
||||
Index colPaddingLeft() const { return m_colPaddingLeft; }
|
||||
Index outputPlanes() const { return m_outputPlanes; }
|
||||
Index outputRows() const { return m_outputRows; }
|
||||
Index outputCols() const { return m_outputCols; }
|
||||
Index userPlaneStride() const { return m_plane_strides; }
|
||||
Index userRowStride() const { return m_row_strides; }
|
||||
Index userColStride() const { return m_col_strides; }
|
||||
Index userInPlaneStride() const { return m_in_plane_strides; }
|
||||
Index userInRowStride() const { return m_in_row_strides; }
|
||||
Index userInColStride() const { return m_in_col_strides; }
|
||||
Index planeInflateStride() const { return m_plane_inflate_strides; }
|
||||
Index rowInflateStride() const { return m_row_inflate_strides; }
|
||||
Index colInflateStride() const { return m_col_inflate_strides; }
|
||||
|
||||
EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE CoeffReturnType
|
||||
coeff(const array<Index, NumDims>& coords) const {
|
||||
// ColMajor
|
||||
// 0: depth, 1: patch_planes, 2: patch_rows, 3: patch_cols, 4: number of
|
||||
// patches, 5: batches
|
||||
// RowMajor
|
||||
// 0: batches, 1: number of patches, 2: patch_cols , 3: patch_rows, 4:
|
||||
// patch_planes, 5: depth
|
||||
const Index patch3DIndex =
|
||||
coords[static_cast<int>(Layout) == static_cast<int>(ColMajor) ? 4 : 1];
|
||||
const Index colOffset =
|
||||
coords[static_cast<int>(Layout) == static_cast<int>(ColMajor) ? 3 : 2];
|
||||
const Index rowOffset =
|
||||
coords[static_cast<int>(Layout) == static_cast<int>(ColMajor) ? 2 : 3];
|
||||
const Index planeOffset =
|
||||
coords[static_cast<int>(Layout) == static_cast<int>(ColMajor) ? 1 : 4];
|
||||
|
||||
array<Index, NumDims - 1> inputCoords;
|
||||
|
||||
const Index colIndex = patch3DIndex / m_fastOutputPlanesRows;
|
||||
const Index inputCol = colIndex * m_col_strides +
|
||||
colOffset * m_in_col_strides - m_colPaddingLeft;
|
||||
const Index origInputCol =
|
||||
(m_col_inflate_strides == 1)
|
||||
? inputCol
|
||||
: ((inputCol >= 0) ? (inputCol / m_fastInputColStride) : 0);
|
||||
if (inputCol < 0 || inputCol >= m_input_cols_eff ||
|
||||
((m_col_inflate_strides != 1) &&
|
||||
(inputCol != origInputCol * m_col_inflate_strides))) {
|
||||
return Scalar(m_paddingValue);
|
||||
}
|
||||
|
||||
const Index rowIndex =
|
||||
(patch3DIndex - colIndex * m_outputPlanesRows) / m_fastOutputPlanes;
|
||||
const Index inputRow = rowIndex * m_row_strides +
|
||||
rowOffset * m_in_row_strides - m_rowPaddingTop;
|
||||
const Index origInputRow =
|
||||
(m_row_inflate_strides == 1)
|
||||
? inputRow
|
||||
: ((inputRow >= 0) ? (inputRow / m_fastInputRowStride) : 0);
|
||||
if (inputRow < 0 || inputRow >= m_input_rows_eff ||
|
||||
((m_row_inflate_strides != 1) &&
|
||||
(inputRow != origInputRow * m_row_inflate_strides))) {
|
||||
return Scalar(m_paddingValue);
|
||||
}
|
||||
|
||||
const Index planeIndex =
|
||||
patch3DIndex - colIndex * m_outputPlanesRows - rowIndex * m_outputRows;
|
||||
const Index inputPlane = planeIndex * m_plane_strides +
|
||||
planeOffset * m_in_plane_strides -
|
||||
m_planePaddingTop;
|
||||
const Index origInputPlane =
|
||||
(m_plane_inflate_strides == 1)
|
||||
? inputPlane
|
||||
: ((inputPlane >= 0) ? (inputPlane / m_fastInputPlaneStride) : 0);
|
||||
if (inputPlane < 0 || inputPlane >= m_input_planes_eff ||
|
||||
((m_plane_inflate_strides != 1) &&
|
||||
(inputPlane != origInputPlane * m_plane_inflate_strides))) {
|
||||
return Scalar(m_paddingValue);
|
||||
}
|
||||
|
||||
if (static_cast<int>(Layout) == static_cast<int>(ColMajor)) {
|
||||
inputCoords[0] = coords[0]; // depth
|
||||
inputCoords[1] = origInputPlane;
|
||||
inputCoords[2] = origInputRow;
|
||||
inputCoords[3] = origInputCol;
|
||||
inputCoords[4] = coords[5]; // batch
|
||||
} else {
|
||||
inputCoords[4] = coords[5]; // depth
|
||||
inputCoords[3] = origInputPlane;
|
||||
inputCoords[2] = origInputRow;
|
||||
inputCoords[1] = origInputCol;
|
||||
inputCoords[0] = coords[0]; // batch
|
||||
}
|
||||
if (TensorEvaluator<ArgType, Device>::CoordAccess) {
|
||||
return m_impl.coeff(inputCoords);
|
||||
} else {
|
||||
Index inputIndex;
|
||||
if (static_cast<int>(Layout) == static_cast<int>(ColMajor)) {
|
||||
inputIndex = inputCoords[4] * m_otherInputStride +
|
||||
inputCoords[3] * m_colInputStride +
|
||||
inputCoords[2] * m_rowInputStride +
|
||||
inputCoords[1] * m_planeInputStride + inputCoords[0];
|
||||
} else {
|
||||
inputIndex = inputCoords[0] * m_otherInputStride +
|
||||
inputCoords[1] * m_colInputStride +
|
||||
inputCoords[2] * m_rowInputStride +
|
||||
inputCoords[3] * m_planeInputStride + inputCoords[4];
|
||||
}
|
||||
return m_impl.coeff(inputIndex);
|
||||
}
|
||||
}
|
||||
|
||||
protected:
|
||||
EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE PacketReturnType
|
||||
packetWithPossibleZero(Index index) const {
|
||||
EIGEN_ALIGN_MAX typename internal::remove_const<CoeffReturnType>::type
|
||||
values[PacketSize];
|
||||
for (int i = 0; i < PacketSize; ++i) {
|
||||
values[i] = coeff(index + i);
|
||||
}
|
||||
PacketReturnType rslt = internal::pload<PacketReturnType>(values);
|
||||
return rslt;
|
||||
}
|
||||
|
||||
Dimensions m_dimensions;
|
||||
|
||||
// Parameters passed to the costructor.
|
||||
Index m_plane_strides;
|
||||
Index m_row_strides;
|
||||
Index m_col_strides;
|
||||
|
||||
Index m_outputPlanes;
|
||||
Index m_outputRows;
|
||||
Index m_outputCols;
|
||||
|
||||
Index m_planePaddingTop;
|
||||
Index m_rowPaddingTop;
|
||||
Index m_colPaddingLeft;
|
||||
|
||||
Index m_in_plane_strides;
|
||||
Index m_in_row_strides;
|
||||
Index m_in_col_strides;
|
||||
|
||||
Index m_plane_inflate_strides;
|
||||
Index m_row_inflate_strides;
|
||||
Index m_col_inflate_strides;
|
||||
|
||||
// Cached input size.
|
||||
Index m_inputDepth;
|
||||
Index m_inputPlanes;
|
||||
Index m_inputRows;
|
||||
Index m_inputCols;
|
||||
|
||||
// Other cached variables.
|
||||
Index m_outputPlanesRows;
|
||||
|
||||
// Effective input/patch post-inflation size.
|
||||
Index m_input_planes_eff;
|
||||
Index m_input_rows_eff;
|
||||
Index m_input_cols_eff;
|
||||
Index m_patch_planes_eff;
|
||||
Index m_patch_rows_eff;
|
||||
Index m_patch_cols_eff;
|
||||
|
||||
// Strides for the output tensor.
|
||||
Index m_otherStride;
|
||||
Index m_patchStride;
|
||||
Index m_rowStride;
|
||||
Index m_colStride;
|
||||
|
||||
// Strides for the input tensor.
|
||||
Index m_planeInputStride;
|
||||
Index m_rowInputStride;
|
||||
Index m_colInputStride;
|
||||
Index m_otherInputStride;
|
||||
|
||||
internal::TensorIntDivisor<Index> m_fastOtherStride;
|
||||
internal::TensorIntDivisor<Index> m_fastPatchStride;
|
||||
internal::TensorIntDivisor<Index> m_fastColStride;
|
||||
internal::TensorIntDivisor<Index> m_fastRowStride;
|
||||
internal::TensorIntDivisor<Index> m_fastInputPlaneStride;
|
||||
internal::TensorIntDivisor<Index> m_fastInputRowStride;
|
||||
internal::TensorIntDivisor<Index> m_fastInputColStride;
|
||||
internal::TensorIntDivisor<Index> m_fastInputColsEff;
|
||||
internal::TensorIntDivisor<Index> m_fastOutputPlanesRows;
|
||||
internal::TensorIntDivisor<Index> m_fastOutputPlanes;
|
||||
internal::TensorIntDivisor<Index> m_fastOutputDepth;
|
||||
|
||||
Scalar m_paddingValue;
|
||||
|
||||
TensorEvaluator<ArgType, Device> m_impl;
|
||||
};
|
||||
|
||||
// Override the default TensorEvaluator for TensorVolumePatchOp for CPU.
|
||||
#define OVERRIDE_EVALUATOR(Device) \
|
||||
template <DenseIndex Planes, DenseIndex Rows, DenseIndex Cols, \
|
||||
typename ArgType> \
|
||||
struct TensorEvaluator< \
|
||||
const TensorVolumePatchOp<Planes, Rows, Cols, ArgType>, Device> \
|
||||
: public CustomTensorEvaluator<Planes, Rows, Cols, ArgType, Device> { \
|
||||
EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE TensorEvaluator( \
|
||||
const typename CustomTensorEvaluator<Planes, Rows, Cols, ArgType, \
|
||||
Device>::XprType& op, \
|
||||
const Device& device) \
|
||||
: CustomTensorEvaluator<Planes, Rows, Cols, ArgType, Device>( \
|
||||
op, device) {} \
|
||||
};
|
||||
|
||||
OVERRIDE_EVALUATOR(Eigen::ThreadPoolDevice);
|
||||
OVERRIDE_EVALUATOR(Eigen::DefaultDevice);
|
||||
|
||||
#undef OVERRIDE_EVALUATOR
|
||||
|
||||
}; // namespace Eigen
|
||||
|
||||
#endif // TENSORFLOW_CORE_KERNELS_EIGEN_VOLUME_PATCH_H_
|
||||
@@ -0,0 +1,56 @@
|
||||
/* Copyright 2016 The TensorFlow Authors. All Rights Reserved.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
==============================================================================*/
|
||||
|
||||
#ifndef TENSORFLOW_CORE_KERNELS_EXTRACT_IMAGE_PATCHES_OP_H_
|
||||
#define TENSORFLOW_CORE_KERNELS_EXTRACT_IMAGE_PATCHES_OP_H_
|
||||
|
||||
#include "third_party/eigen3/unsupported/Eigen/CXX11/Tensor"
|
||||
#include "tensorflow/core/framework/tensor_shape.h"
|
||||
#include "tensorflow/core/framework/tensor_types.h"
|
||||
|
||||
namespace tensorflow {
|
||||
namespace functor {
|
||||
|
||||
template <typename Device, typename T>
|
||||
struct ExtractImagePatchesForward {
|
||||
void operator()(const Device& d, typename TTypes<T, 4>::ConstTensor input,
|
||||
int patch_rows, int patch_cols, int stride_rows,
|
||||
int stride_cols, int rate_rows, int rate_cols,
|
||||
const Eigen::PaddingType& padding,
|
||||
typename TTypes<T, 4>::Tensor output) {
|
||||
// Need to swap row/col when calling Eigen, because our data is in
|
||||
// NHWC format while Eigen assumes NWHC format.
|
||||
const int64 N = std::max(input.size(), output.size());
|
||||
if (N <= std::numeric_limits<Index32>::max()) {
|
||||
auto output_32bit = To32Bit(output);
|
||||
output_32bit.device(d) =
|
||||
To32Bit(input)
|
||||
.extract_image_patches(patch_cols, patch_rows, stride_cols,
|
||||
stride_rows, rate_cols, rate_rows, padding)
|
||||
.reshape(output_32bit.dimensions());
|
||||
} else {
|
||||
output.device(d) =
|
||||
input
|
||||
.extract_image_patches(patch_cols, patch_rows, stride_cols,
|
||||
stride_rows, rate_cols, rate_rows, padding)
|
||||
.reshape(output.dimensions());
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
} // namespace functor
|
||||
} // namespace tensorflow
|
||||
|
||||
#endif // TENSORFLOW_CORE_KERNELS_EXTRACT_IMAGE_PATCHES_OP_H_
|
||||
@@ -0,0 +1,58 @@
|
||||
/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
==============================================================================*/
|
||||
|
||||
#ifndef TENSORFLOW_KERNELS_EXTRACT_VOLUME_PATCHES_OP_H_
|
||||
#define TENSORFLOW_KERNELS_EXTRACT_VOLUME_PATCHES_OP_H_
|
||||
|
||||
#include "tensorflow/core/framework/tensor_shape.h"
|
||||
#include "tensorflow/core/framework/tensor_types.h"
|
||||
#include "tensorflow/core/kernels/eigen_volume_patch.h"
|
||||
#include "third_party/eigen3/unsupported/Eigen/CXX11/Tensor"
|
||||
|
||||
namespace tensorflow {
|
||||
namespace functor {
|
||||
|
||||
template <typename Device, typename T>
|
||||
struct ExtractVolumePatchesForward {
|
||||
void operator()(const Device& d, typename TTypes<T, 5>::ConstTensor input,
|
||||
int patch_planes, int patch_rows, int patch_cols,
|
||||
int stride_planes, int stride_rows, int stride_cols,
|
||||
/* int rate_planes, int rate_rows, int rate_cols, */
|
||||
const Eigen::PaddingType& padding,
|
||||
typename TTypes<T, 5>::Tensor output) {
|
||||
const int64 N = std::max(input.size(), output.size());
|
||||
if (N <= std::numeric_limits<Index32>::max()) {
|
||||
auto output_32bit = To32Bit(output);
|
||||
output_32bit.device(d) =
|
||||
To32Bit(input)
|
||||
.extract_volume_patches(patch_cols, patch_rows, patch_planes,
|
||||
stride_cols, stride_rows, stride_planes,
|
||||
padding)
|
||||
.reshape(output_32bit.dimensions());
|
||||
} else {
|
||||
output.device(d) =
|
||||
input
|
||||
.extract_volume_patches(patch_cols, patch_rows, patch_planes,
|
||||
stride_cols, stride_rows, stride_planes,
|
||||
padding)
|
||||
.reshape(output.dimensions());
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
} // namespace functor
|
||||
} // namespace tensorflow
|
||||
|
||||
#endif // TENSORFLOW_KERNELS_EXTRACT_VOLUME_PATCHES_OP_H_
|
||||
@@ -0,0 +1,280 @@
|
||||
/* Copyright 2016 The TensorFlow Authors. All Rights Reserved.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
==============================================================================*/
|
||||
|
||||
#ifndef TENSORFLOW_CORE_KERNELS_FAKE_QUANT_OPS_FUNCTOR_H_
|
||||
#define TENSORFLOW_CORE_KERNELS_FAKE_QUANT_OPS_FUNCTOR_H_
|
||||
|
||||
#include <tuple>
|
||||
|
||||
#define EIGEN_STACK_ALLOCATION_LIMIT 0
|
||||
#define EIGEN_USE_THREADS
|
||||
#include "third_party/eigen3/unsupported/Eigen/CXX11/Tensor"
|
||||
#include "tensorflow/core/framework/tensor_types.h"
|
||||
#include "tensorflow/core/platform/types.h"
|
||||
|
||||
EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE float StdRound(float input) {
|
||||
// On Android, std::round() isn't present, just round().
|
||||
#if defined(__ANDROID__)
|
||||
return round(input);
|
||||
#else
|
||||
return std::round(input);
|
||||
#endif
|
||||
}
|
||||
|
||||
namespace tensorflow {
|
||||
|
||||
// Gymnastics with nudged zero point is to ensure that real zero maps to
|
||||
// an integer, which is required for e.g. zero-padding in convolutional layers.
|
||||
// Outputs nudged_min, nudged_max, nudged_scale.
|
||||
EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE void Nudge(
|
||||
const float min, const float max, const int quant_min, const int quant_max,
|
||||
float* nudged_min, float* nudged_max, float* scale) {
|
||||
const float quant_min_float = static_cast<float>(quant_min);
|
||||
const float quant_max_float = static_cast<float>(quant_max);
|
||||
*scale = (max - min) / (quant_max_float - quant_min_float);
|
||||
const float zero_point_from_min = quant_min_float - min / *scale;
|
||||
const uint16 nudged_zero_point = [zero_point_from_min, quant_min,
|
||||
quant_min_float, quant_max,
|
||||
quant_max_float] {
|
||||
if (zero_point_from_min < quant_min_float) {
|
||||
return static_cast<uint16>(quant_min);
|
||||
}
|
||||
if (zero_point_from_min > quant_max_float) {
|
||||
return static_cast<uint16>(quant_max);
|
||||
}
|
||||
return static_cast<uint16>(StdRound(zero_point_from_min));
|
||||
}();
|
||||
*nudged_min = (quant_min_float - nudged_zero_point) * (*scale);
|
||||
*nudged_max = (quant_max_float - nudged_zero_point) * (*scale);
|
||||
}
|
||||
|
||||
template <typename T>
|
||||
using ConstScalar = typename tensorflow::TTypes<T>::ConstScalar;
|
||||
template <typename T>
|
||||
using Scalar = typename tensorflow::TTypes<T>::Scalar;
|
||||
template <typename T>
|
||||
using ConstVec = typename tensorflow::TTypes<T>::ConstVec;
|
||||
template <typename T>
|
||||
using Vec = typename tensorflow::TTypes<T>::Vec;
|
||||
template <typename T>
|
||||
using ConstFlat = typename tensorflow::TTypes<T>::ConstFlat;
|
||||
template <typename T>
|
||||
using Flat = typename tensorflow::TTypes<T>::Flat;
|
||||
|
||||
// Functor called by FakeQuantWithMinMaxArgsOp to do the work. Compiles both
|
||||
// for CPU and GPU.
|
||||
template <typename Device>
|
||||
struct FakeQuantWithMinMaxArgsFunctor {
|
||||
void operator()(const Device& d, ConstFlat<float> inputs, const float min,
|
||||
const float max, const int quant_min, const int quant_max,
|
||||
Flat<float> outputs) {
|
||||
eigen_assert(min <= 0.0f && "min should be <= 0.0");
|
||||
eigen_assert(max >= 0.0f && "max should be >= 0.0");
|
||||
eigen_assert(min < max && "min should be < max");
|
||||
|
||||
float nudged_min, nudged_max, nudged_scale;
|
||||
Nudge(min, max, quant_min, quant_max, &nudged_min, &nudged_max,
|
||||
&nudged_scale);
|
||||
const float inv_nudged_scale = 1.0f / nudged_scale;
|
||||
|
||||
auto clamped = inputs.cwiseMin(nudged_max).cwiseMax(nudged_min);
|
||||
auto clamped_shifted = clamped - nudged_min;
|
||||
outputs.device(d) =
|
||||
(clamped_shifted * inv_nudged_scale + 0.5f).floor() * nudged_scale +
|
||||
nudged_min;
|
||||
}
|
||||
};
|
||||
|
||||
// Functor called by FakeQuantWithMinMaxArgsGradientOp to do the work. Compiles
|
||||
// both for CPU and GPU.
|
||||
template <typename Device>
|
||||
struct FakeQuantWithMinMaxArgsGradientFunctor {
|
||||
void operator()(const Device& d, ConstFlat<float> gradients,
|
||||
ConstFlat<float> inputs, const float min, const float max,
|
||||
const int quant_min, const int quant_max,
|
||||
Flat<float> backprops) {
|
||||
eigen_assert(min <= 0.0f && "min should be <= 0.0");
|
||||
eigen_assert(max >= 0.0f && "max should be >= 0.0");
|
||||
eigen_assert(min < max && "min should be < max");
|
||||
|
||||
float nudged_min, nudged_max, nudged_scale;
|
||||
Nudge(min, max, quant_min, quant_max, &nudged_min, &nudged_max,
|
||||
&nudged_scale);
|
||||
|
||||
auto between_nudged_min_max =
|
||||
(inputs >= nudged_min && inputs <= nudged_max)
|
||||
.select(inputs.constant(1.0f), inputs.constant(0.0f));
|
||||
backprops.device(d) = gradients * between_nudged_min_max;
|
||||
}
|
||||
};
|
||||
|
||||
// Functor called by FakeQuantWithMinMaxVarsOp to do the work. Compiles both
|
||||
// for CPU and GPU.
|
||||
template <typename Device>
|
||||
struct FakeQuantWithMinMaxVarsFunctor {
|
||||
void operator()(const Device& d, ConstFlat<float> inputs,
|
||||
ConstScalar<float> min, ConstScalar<float> max,
|
||||
const int quant_min, const int quant_max,
|
||||
Flat<float> outputs) {
|
||||
const float min_val = min();
|
||||
const float max_val = max();
|
||||
// If min and max are both zero, we should just return zero.
|
||||
if (min_val == 0.0f && max_val == 0.0f) {
|
||||
outputs.device(d) = outputs.constant(0.0f);
|
||||
return;
|
||||
}
|
||||
float nudged_min, nudged_max, nudged_scale;
|
||||
Nudge(min_val, max_val, quant_min, quant_max, &nudged_min, &nudged_max,
|
||||
&nudged_scale);
|
||||
const auto nudged_scale_repl = inputs.constant(nudged_scale);
|
||||
|
||||
const auto clamped = inputs.cwiseMin(nudged_max).cwiseMax(nudged_min);
|
||||
const auto clamped_shifted = clamped - nudged_min;
|
||||
outputs.device(d) = (clamped_shifted / nudged_scale_repl + 0.5f).floor() *
|
||||
nudged_scale_repl +
|
||||
nudged_min;
|
||||
}
|
||||
};
|
||||
|
||||
// Functor called by FakeQuantWithMinMaxVarsGradientOp to do the work. Compiles
|
||||
// both for CPU and GPU.
|
||||
template <typename Device>
|
||||
struct FakeQuantWithMinMaxVarsGradientFunctor {
|
||||
void operator()(const Device& d, ConstFlat<float> gradients,
|
||||
ConstFlat<float> inputs, ConstScalar<float> min,
|
||||
ConstScalar<float> max, const int quant_min,
|
||||
const int quant_max, Flat<float> backprops_wrt_input,
|
||||
Scalar<float> backprop_wrt_min,
|
||||
Scalar<float> backprop_wrt_max) {
|
||||
const float min_val = min();
|
||||
const float max_val = max();
|
||||
// If min and max are both zero, we propagate everything to inputs.
|
||||
if (min_val == 0.0f && max_val == 0.0f) {
|
||||
backprops_wrt_input.device(d) = gradients;
|
||||
backprop_wrt_min.device(d) = backprop_wrt_min.constant(0.0f);
|
||||
backprop_wrt_max.device(d) = backprop_wrt_max.constant(0.0f);
|
||||
return;
|
||||
}
|
||||
float nudged_min, nudged_max, nudged_scale;
|
||||
Nudge(min_val, max_val, quant_min, quant_max, &nudged_min, &nudged_max,
|
||||
&nudged_scale);
|
||||
|
||||
const auto between_min_max =
|
||||
(inputs >= nudged_min && inputs <= nudged_max)
|
||||
.select(inputs.constant(1.0f), inputs.constant(0.0f));
|
||||
backprops_wrt_input.device(d) = gradients * between_min_max;
|
||||
|
||||
const auto below_min =
|
||||
(inputs < nudged_min)
|
||||
.select(inputs.constant(1.0f), inputs.constant(0.0f));
|
||||
backprop_wrt_min.device(d) = (gradients * below_min).sum();
|
||||
|
||||
const auto above_max =
|
||||
(inputs > nudged_max)
|
||||
.select(inputs.constant(1.0f), inputs.constant(0.0f));
|
||||
backprop_wrt_max.device(d) = (gradients * above_max).sum();
|
||||
}
|
||||
};
|
||||
|
||||
using Index = typename tensorflow::TTypes<float>::ConstTensor::Index;
|
||||
|
||||
// Functor called by FakeQuantWithMinMaxVarsPerChannelOp to do the work.
|
||||
// Compiles both for CPU and GPU.
|
||||
//
|
||||
// Already verified: inputs, outputs are of shape [b, d], min, max are of shape
|
||||
// [d].
|
||||
template <typename Device>
|
||||
struct FakeQuantWithMinMaxVarsPerChannelFunctor {
|
||||
void operator()(const Device& d, TTypes<float>::ConstMatrix inputs,
|
||||
ConstVec<float> min, ConstVec<float> max, const int quant_min,
|
||||
const int quant_max, TTypes<float>::Matrix outputs) {
|
||||
for (Index i = 0; i < min.size(); ++i) {
|
||||
const float min_val = min(i);
|
||||
const float max_val = max(i);
|
||||
// If min and max are both zero, we should just return zero.
|
||||
if (min_val == 0.0f && max_val == 0.0f) {
|
||||
auto chip = outputs.chip<1>(i);
|
||||
chip.device(d) = chip.constant(0.0f);
|
||||
continue;
|
||||
}
|
||||
float nudged_min, nudged_max, nudged_scale;
|
||||
Nudge(min_val, max_val, quant_min, quant_max, &nudged_min, &nudged_max,
|
||||
&nudged_scale);
|
||||
const auto clamped =
|
||||
inputs.chip<1>(i).cwiseMin(nudged_max).cwiseMax(nudged_min);
|
||||
const auto clamped_shifted = clamped - nudged_min;
|
||||
|
||||
outputs.chip<1>(i).device(d) =
|
||||
(clamped_shifted / nudged_scale + 0.5f).floor() * nudged_scale +
|
||||
nudged_min;
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
// Functor called by FakeQuantWithMinMaxVarsPerChannelGradientOp to do the work.
|
||||
// Compiles both for CPU and GPU.
|
||||
//
|
||||
// Already verified: gradients, inputs, backprops_wrt_input are of shape [b, d],
|
||||
// min, max, backprop_wrt_min, backprop_wrt_max are of shape [d].
|
||||
template <typename Device>
|
||||
struct FakeQuantWithMinMaxVarsPerChannelGradientFunctor {
|
||||
void operator()(const Device& d, TTypes<float>::ConstMatrix gradients,
|
||||
TTypes<float>::ConstMatrix inputs, ConstVec<float> min,
|
||||
ConstVec<float> max, const int quant_min, const int quant_max,
|
||||
TTypes<float>::Matrix backprops_wrt_input,
|
||||
Vec<float> backprop_wrt_min, Vec<float> backprop_wrt_max) {
|
||||
for (Index i = 0; i < min.size(); ++i) {
|
||||
const float min_val = min(i);
|
||||
const float max_val = max(i);
|
||||
const auto gradients_chip = gradients.chip<1>(i);
|
||||
const auto inputs_chip = inputs.chip<1>(i);
|
||||
// If min and max are both zero, we propagate everything to inputs.
|
||||
if (min_val == 0.0f && max_val == 0.0f) {
|
||||
backprops_wrt_input.chip<1>(i).device(d) = gradients_chip;
|
||||
auto min_chip = backprop_wrt_min.chip<0>(i);
|
||||
auto max_chip = backprop_wrt_max.chip<0>(i);
|
||||
min_chip.device(d) = min_chip.constant(0.0f);
|
||||
max_chip.device(d) = max_chip.constant(0.0f);
|
||||
continue;
|
||||
}
|
||||
float nudged_min, nudged_max, nudged_scale;
|
||||
Nudge(min_val, max_val, quant_min, quant_max, &nudged_min, &nudged_max,
|
||||
&nudged_scale);
|
||||
|
||||
const auto between_min_max =
|
||||
(inputs_chip >= nudged_min && inputs_chip <= nudged_max)
|
||||
.select(inputs_chip.constant(1.0f), inputs_chip.constant(0.0f));
|
||||
backprops_wrt_input.chip<1>(i).device(d) =
|
||||
gradients_chip * between_min_max;
|
||||
|
||||
const auto below_min =
|
||||
(inputs_chip < nudged_min)
|
||||
.select(inputs_chip.constant(1.0f), inputs_chip.constant(0.0f));
|
||||
Eigen::DSizes<Index, 1> reduce(0);
|
||||
backprop_wrt_min.chip<0>(i).device(d) =
|
||||
(gradients_chip * below_min).sum(reduce);
|
||||
|
||||
const auto above_max =
|
||||
(inputs_chip > nudged_max)
|
||||
.select(inputs_chip.constant(1.0f), inputs_chip.constant(0.0f));
|
||||
backprop_wrt_max.chip<0>(i).device(d) =
|
||||
(gradients_chip * above_max).sum(reduce);
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
} // namespace tensorflow
|
||||
|
||||
#endif // TENSORFLOW_CORE_KERNELS_FAKE_QUANT_OPS_FUNCTOR_H_
|
||||
@@ -0,0 +1,91 @@
|
||||
/* Copyright 2015 The TensorFlow Authors. All Rights Reserved.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
==============================================================================*/
|
||||
|
||||
#ifndef TENSORFLOW_CORE_KERNELS_FIFO_QUEUE_H_
|
||||
#define TENSORFLOW_CORE_KERNELS_FIFO_QUEUE_H_
|
||||
|
||||
#include <deque>
|
||||
#include <vector>
|
||||
|
||||
#include "tensorflow/core/framework/op_kernel.h"
|
||||
#include "tensorflow/core/framework/tensor.h"
|
||||
#include "tensorflow/core/framework/tensor_shape.h"
|
||||
#include "tensorflow/core/framework/types.h"
|
||||
#include "tensorflow/core/kernels/queue_op.h"
|
||||
#include "tensorflow/core/kernels/typed_queue.h"
|
||||
#include "tensorflow/core/platform/macros.h"
|
||||
#include "tensorflow/core/platform/mutex.h"
|
||||
#include "tensorflow/core/platform/types.h"
|
||||
|
||||
namespace tensorflow {
|
||||
|
||||
class FIFOQueue : public TypedQueue<std::deque<PersistentTensor> > {
|
||||
public:
|
||||
FIFOQueue(int32 capacity, const DataTypeVector& component_dtypes,
|
||||
const std::vector<TensorShape>& component_shapes,
|
||||
const string& name);
|
||||
|
||||
// Implementations of QueueInterface methods --------------------------------
|
||||
|
||||
void TryEnqueue(const Tuple& tuple, OpKernelContext* ctx,
|
||||
DoneCallback callback) override;
|
||||
void TryEnqueueMany(const Tuple& tuple, OpKernelContext* ctx,
|
||||
DoneCallback callback) override;
|
||||
void TryDequeue(OpKernelContext* ctx, CallbackWithTuple callback) override;
|
||||
void TryDequeueMany(int num_elements, OpKernelContext* ctx,
|
||||
bool allow_small_batch,
|
||||
CallbackWithTuple callback) override;
|
||||
Status MatchesNodeDef(const NodeDef& node_def) override;
|
||||
|
||||
int32 size() override {
|
||||
mutex_lock lock(mu_);
|
||||
return queues_[0].size();
|
||||
}
|
||||
|
||||
protected:
|
||||
~FIFOQueue() override {}
|
||||
|
||||
// Helper for dequeuing a single element from queues_.
|
||||
void DequeueLocked(OpKernelContext* ctx, Tuple* tuple)
|
||||
EXCLUSIVE_LOCKS_REQUIRED(mu_);
|
||||
|
||||
static Status GetElementComponentFromBatch(const Tuple& tuple, int64 index,
|
||||
int component,
|
||||
OpKernelContext* ctx,
|
||||
PersistentTensor* out_element);
|
||||
|
||||
private:
|
||||
TF_DISALLOW_COPY_AND_ASSIGN(FIFOQueue);
|
||||
};
|
||||
|
||||
// Defines a FIFOQueueOp, which produces a Queue (specifically, one
|
||||
// backed by FIFOQueue) that persists across different graph
|
||||
// executions, and sessions. Running this op produces a single-element
|
||||
// tensor of handles to Queues in the corresponding device.
|
||||
class FIFOQueueOp : public TypedQueueOp {
|
||||
public:
|
||||
explicit FIFOQueueOp(OpKernelConstruction* context);
|
||||
|
||||
private:
|
||||
Status CreateResource(QueueInterface** ret) override
|
||||
EXCLUSIVE_LOCKS_REQUIRED(mu_);
|
||||
|
||||
std::vector<TensorShape> component_shapes_;
|
||||
TF_DISALLOW_COPY_AND_ASSIGN(FIFOQueueOp);
|
||||
};
|
||||
|
||||
} // namespace tensorflow
|
||||
|
||||
#endif // TENSORFLOW_CORE_KERNELS_FIFO_QUEUE_H_
|
||||
@@ -0,0 +1,92 @@
|
||||
/* Copyright 2015 The TensorFlow Authors. All Rights Reserved.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
==============================================================================*/
|
||||
|
||||
#ifndef TENSORFLOW_CORE_KERNELS_FILL_FUNCTOR_H_
|
||||
#define TENSORFLOW_CORE_KERNELS_FILL_FUNCTOR_H_
|
||||
|
||||
#define EIGEN_USE_THREADS
|
||||
|
||||
#include "third_party/eigen3/unsupported/Eigen/CXX11/Tensor"
|
||||
#include "tensorflow/core/framework/tensor_types.h"
|
||||
#include "tensorflow/core/framework/types.h"
|
||||
|
||||
namespace tensorflow {
|
||||
namespace functor {
|
||||
|
||||
template <typename Device, typename T>
|
||||
struct FillFunctor {
|
||||
// Computes on device "d": out = out.constant(in(0)),
|
||||
void operator()(const Device& d, typename TTypes<T>::Flat out,
|
||||
typename TTypes<T>::ConstScalar in);
|
||||
};
|
||||
|
||||
template <typename Device, typename T>
|
||||
struct SetZeroFunctor {
|
||||
// Computes on device "d": out = out.setZero(),
|
||||
void operator()(const Device& d, typename TTypes<T>::Flat out);
|
||||
};
|
||||
|
||||
// Partial specialization of SetZeroFunctor<Device=Eigen::ThreadPoolDevice, T>.
|
||||
template <typename T>
|
||||
struct SetZeroFunctor<Eigen::ThreadPoolDevice, T> {
|
||||
void operator()(const Eigen::ThreadPoolDevice& d,
|
||||
typename TTypes<T>::Flat out);
|
||||
};
|
||||
|
||||
#ifdef TENSORFLOW_USE_SYCL
|
||||
// Partial specialization of SetZeroFunctor<Device=Eigen::SyclDevice, T>.
|
||||
template <typename T>
|
||||
struct SetZeroFunctor<Eigen::SyclDevice, T> {
|
||||
void operator()(const Eigen::SyclDevice& d, typename TTypes<T>::Flat out);
|
||||
};
|
||||
#endif // TENSORFLOW_USE_SYCL
|
||||
|
||||
template <>
|
||||
struct SetZeroFunctor<Eigen::ThreadPoolDevice, string> {
|
||||
void operator()(const Eigen::ThreadPoolDevice& d,
|
||||
typename TTypes<string>::Flat out);
|
||||
};
|
||||
|
||||
template <typename Device, typename T>
|
||||
struct SetOneFunctor {
|
||||
// Computes on device "d": out = out.setOne(),
|
||||
void operator()(const Device& d, typename TTypes<T>::Flat out);
|
||||
};
|
||||
|
||||
// Partial specialization of SetOneFunctor<Device=Eigen::ThreadPoolDevice, T>.
|
||||
template <typename T>
|
||||
struct SetOneFunctor<Eigen::ThreadPoolDevice, T> {
|
||||
void operator()(const Eigen::ThreadPoolDevice& d,
|
||||
typename TTypes<T>::Flat out);
|
||||
};
|
||||
|
||||
#ifdef TENSORFLOW_USE_SYCL
|
||||
// Partial specialization of SetOneFunctor<Device=Eigen::SyclDevice, T>.
|
||||
template <typename T>
|
||||
struct SetOneFunctor<Eigen::SyclDevice, T> {
|
||||
void operator()(const Eigen::SyclDevice& d, typename TTypes<T>::Flat out);
|
||||
};
|
||||
#endif // TENSORFLOW_USE_SYCL
|
||||
|
||||
template <>
|
||||
struct SetOneFunctor<Eigen::ThreadPoolDevice, string> {
|
||||
void operator()(const Eigen::ThreadPoolDevice& d,
|
||||
typename TTypes<string>::Flat out);
|
||||
};
|
||||
|
||||
} // namespace functor
|
||||
} // namespace tensorflow
|
||||
|
||||
#endif // TENSORFLOW_CORE_KERNELS_FILL_FUNCTOR_H_
|
||||
@@ -0,0 +1,78 @@
|
||||
/* Copyright 2016 The TensorFlow Authors. All Rights Reserved.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
==============================================================================*/
|
||||
#ifndef TENSORFLOW_CORE_KERNELS_FRACTIONAL_POOL_COMMON_H_
|
||||
#define TENSORFLOW_CORE_KERNELS_FRACTIONAL_POOL_COMMON_H_
|
||||
|
||||
#include <algorithm>
|
||||
#include <vector>
|
||||
|
||||
#include "tensorflow/core/util/guarded_philox_random.h"
|
||||
|
||||
namespace tensorflow {
|
||||
|
||||
// Shuffle a container randomly, copied from random_shuffle_op.cc
|
||||
template <class Iter, class Random>
|
||||
static inline void RandomShuffle(Iter first, Iter last, const Random& uniform) {
|
||||
if (first == last) {
|
||||
return;
|
||||
}
|
||||
const auto stop = last - 1;
|
||||
for (auto i = first; i != stop; ++i) {
|
||||
using std::iter_swap;
|
||||
iter_swap(i, i + uniform(last - i));
|
||||
}
|
||||
}
|
||||
|
||||
// Generate pooling sequence for fractional pooling along one dimension.
|
||||
//
|
||||
// Regular max/avg pooling can be viewed as a special case, in which given the
|
||||
// * input_length: e.g. 10
|
||||
// * output_length: e.g. 5
|
||||
// it will generate pooling sequence as
|
||||
// diff sequence: [2, 2, 2, 2, 2]
|
||||
// or as
|
||||
// cumulative sequence: [0, 2, 4, 6, 8, 10]
|
||||
//
|
||||
// In the case of fractional pooling, input_length is not an integer multiple of
|
||||
// output_length, randomness plays a role when generating pooling sequence.
|
||||
// There are two type of randomness (random vs pseudo-random) defined in paper:
|
||||
// http://arxiv.org/abs/1412.6071
|
||||
// You can check the paper for the difference between these two types.
|
||||
//
|
||||
// In summary, the generated diff sequence satisfy the following properties for
|
||||
// both types of randomness:
|
||||
// * length(generated_diff_pooling_sequence) = output_length
|
||||
// * sum(generated_diff_pooling_sequence) = input_length
|
||||
// * Let's define floor(input_length / output_length) = K, then
|
||||
// K <= generated_diff_pooling_sequence[i] <= K+1
|
||||
// For example, when input_length = 10, output_length = 6, the following are
|
||||
// valid pooling sequence:
|
||||
// * [1, 2, 2, 1, 2, 2]
|
||||
// * [1, 1, 2, 2, 2, 2]
|
||||
// [1, 3, 2, 2, 2, 2] is not valid.
|
||||
//
|
||||
// Args:
|
||||
// input_length: See above explanation
|
||||
// output_length: See above explanation
|
||||
// generator: Parallel version of random number generator
|
||||
// pseudo_random: Whether or not use pseudo-random
|
||||
// Returns:
|
||||
// pooling_sequence: This is the cumulative pooling sequence.
|
||||
std::vector<int64> GeneratePoolingSequence(int input_length, int output_length,
|
||||
GuardedPhiloxRandom* generator,
|
||||
bool pseudo_random);
|
||||
} // namespace tensorflow
|
||||
|
||||
#endif // TENSORFLOW_CORE_KERNELS_FRACTIONAL_POOL_COMMON_H_
|
||||
@@ -0,0 +1,79 @@
|
||||
/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
==============================================================================*/
|
||||
|
||||
#ifndef TENSORFLOW_CORE_KERNELS_FUNCTION_OPS_H_
|
||||
#define TENSORFLOW_CORE_KERNELS_FUNCTION_OPS_H_
|
||||
|
||||
#include "tensorflow/core/framework/function.h"
|
||||
#include "tensorflow/core/framework/op_kernel.h"
|
||||
|
||||
namespace tensorflow {
|
||||
|
||||
static const char* const kArgOp = FunctionLibraryDefinition::kArgOp;
|
||||
static const char* const kRetOp = FunctionLibraryDefinition::kRetOp;
|
||||
|
||||
class ArgOp : public OpKernel {
|
||||
public:
|
||||
explicit ArgOp(OpKernelConstruction* ctx);
|
||||
|
||||
void Compute(OpKernelContext* ctx) override;
|
||||
|
||||
bool IsExpensive() override { return false; }
|
||||
|
||||
private:
|
||||
int index_;
|
||||
DataType dtype_;
|
||||
|
||||
TF_DISALLOW_COPY_AND_ASSIGN(ArgOp);
|
||||
};
|
||||
|
||||
class RetvalOp : public OpKernel {
|
||||
public:
|
||||
explicit RetvalOp(OpKernelConstruction* ctx);
|
||||
|
||||
void Compute(OpKernelContext* ctx) override;
|
||||
|
||||
bool IsExpensive() override { return false; }
|
||||
|
||||
private:
|
||||
int index_;
|
||||
DataType dtype_;
|
||||
|
||||
TF_DISALLOW_COPY_AND_ASSIGN(RetvalOp);
|
||||
};
|
||||
|
||||
class RemoteCallOp : public AsyncOpKernel {
|
||||
public:
|
||||
explicit RemoteCallOp(OpKernelConstruction* ctx);
|
||||
|
||||
~RemoteCallOp() override {}
|
||||
|
||||
void ComputeAsync(OpKernelContext* ctx, DoneCallback done) override;
|
||||
|
||||
private:
|
||||
NameAttrList func_;
|
||||
DataTypeVector input_dtypes_;
|
||||
DataTypeVector output_dtypes_;
|
||||
|
||||
mutex mu_;
|
||||
typedef std::pair<string, FunctionLibraryRuntime*> FunctionTarget;
|
||||
std::map<FunctionTarget, FunctionLibraryRuntime::Handle> handle_cache_
|
||||
GUARDED_BY(mu_);
|
||||
|
||||
TF_DISALLOW_COPY_AND_ASSIGN(RemoteCallOp);
|
||||
};
|
||||
|
||||
} // namespace tensorflow
|
||||
#endif // TENSORFLOW_CORE_KERNELS_FUNCTION_OPS_H_
|
||||
@@ -0,0 +1,131 @@
|
||||
/* Copyright 2016 The TensorFlow Authors. All Rights Reserved.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
==============================================================================*/
|
||||
|
||||
#ifndef TENSORFLOW_CORE_KERNELS_FUSED_BATCH_NORM_OP_H_
|
||||
#define TENSORFLOW_CORE_KERNELS_FUSED_BATCH_NORM_OP_H_
|
||||
|
||||
#include "third_party/eigen3/unsupported/Eigen/CXX11/Tensor"
|
||||
#include "tensorflow/core/framework/tensor.h"
|
||||
#include "tensorflow/core/framework/tensor_types.h"
|
||||
|
||||
namespace tensorflow {
|
||||
namespace functor {
|
||||
|
||||
#if GOOGLE_CUDA
|
||||
|
||||
// There is a behavior difference between cuDNN v4 and v5 with regard to the
|
||||
// scaling factor for function cudnnBatchNormalizationForwardInference.
|
||||
// This function corrects the scaling factor if cuDNN v4 is used, so that
|
||||
// this behavior inconsistency is hidden from TensorFlow users.
|
||||
// Details: in cuDNN v4, y = bnScale * (x - mean) * variance + bnBias;
|
||||
// in v5, y = bnScale * (x - mean) / sqrt(variance + epsilon) + bnBias
|
||||
// The template is instantiated with T as float in batch_norm_ops.cu.cc; for
|
||||
// other types, the instantiation needs to be added accordingly.
|
||||
template <class T>
|
||||
struct VarianceToInvVariance {
|
||||
void operator()(const Eigen::GpuDevice& d, const T* variance, double epsilon,
|
||||
int channels, T* inv_variance);
|
||||
};
|
||||
|
||||
// This function converts the inverted variance of the cuDNN forward training
|
||||
// output to variance for TensorFlow to calculate the running variance.
|
||||
// The template is instantiated with T as float in batch_norm_ops.cu.cc; for
|
||||
// other types, the instantiation needs to be added accordingly.
|
||||
template <class T>
|
||||
struct InvVarianceToVariance {
|
||||
void operator()(const Eigen::GpuDevice& d, double epsilon, int sample_size,
|
||||
int channels, T* variance);
|
||||
};
|
||||
|
||||
// This function sets a GPU tensor to NaNs.
|
||||
template <class T>
|
||||
struct SetNanFunctor {
|
||||
void operator()(const Eigen::GpuDevice& d, typename TTypes<T>::Flat out);
|
||||
};
|
||||
|
||||
#endif // GOOGLE_CUDA
|
||||
|
||||
// Functor used by FusedBatchNormGradOp to do the computations when
|
||||
// is_training=False. Both CPU and GPU will use this functor.
|
||||
template <typename Device, typename T, typename U>
|
||||
struct FusedBatchNormFreezeGrad {
|
||||
void operator()(const Device& d, const Tensor& y_backprop_input,
|
||||
const Tensor& x_input, const Tensor& scale_input,
|
||||
const Tensor& pop_mean_input,
|
||||
const Tensor& pop_variance_input, U epsilon,
|
||||
Tensor* x_backprop_output, Tensor* scale_backprop_output,
|
||||
Tensor* offset_backprop_output,
|
||||
typename TTypes<U>::Vec scratch1,
|
||||
typename TTypes<U>::Vec scratch2) {
|
||||
typename TTypes<T, 4>::ConstTensor y_backprop(
|
||||
y_backprop_input.tensor<T, 4>());
|
||||
typename TTypes<T, 4>::ConstTensor input(x_input.tensor<T, 4>());
|
||||
typename TTypes<U>::ConstVec scale(scale_input.vec<U>());
|
||||
typename TTypes<U>::ConstVec pop_mean(pop_mean_input.vec<U>());
|
||||
typename TTypes<U>::ConstVec pop_var(pop_variance_input.vec<U>());
|
||||
typename TTypes<T, 4>::Tensor x_backprop(x_backprop_output->tensor<T, 4>());
|
||||
typename TTypes<U>::Vec scale_backprop(scale_backprop_output->vec<U>());
|
||||
typename TTypes<U>::Vec offset_backprop(offset_backprop_output->vec<U>());
|
||||
|
||||
const int depth = pop_mean.dimension(0);
|
||||
const int rest_size = input.size() / depth;
|
||||
|
||||
Eigen::DSizes<Eigen::Index, 2> rest_by_depth(rest_size, depth);
|
||||
#if !defined(EIGEN_HAS_INDEX_LIST)
|
||||
Eigen::DSizes<Eigen::Index, 2> one_by_depth(1, depth);
|
||||
Eigen::array<int, 1> reduction_axis{0};
|
||||
Eigen::array<int, 2> rest_by_one({rest_size, 1});
|
||||
#else
|
||||
Eigen::IndexList<Eigen::type2index<1>, Eigen::Index> one_by_depth;
|
||||
one_by_depth.set(1, depth);
|
||||
Eigen::IndexList<Eigen::type2index<0> > reduction_axis;
|
||||
Eigen::IndexList<Eigen::Index, Eigen::type2index<1> > rest_by_one;
|
||||
rest_by_one.set(0, rest_size);
|
||||
#endif
|
||||
|
||||
// offset_backprop = sum(y_backprop)
|
||||
// scale_backprop = y_backprop * ((x - pop_mean) * rsqrt(pop_var + epsilon))
|
||||
// x_backprop = y_backprop * (scale * rsqrt(pop_var + epsilon))
|
||||
|
||||
auto y_backprop_rest_by_depth =
|
||||
y_backprop.reshape(rest_by_depth).template cast<U>();
|
||||
auto input_rest_by_depth = input.reshape(rest_by_depth).template cast<U>();
|
||||
|
||||
offset_backprop.device(d) = y_backprop_rest_by_depth.sum(reduction_axis);
|
||||
|
||||
// scratch1 = rsqrt(pop_var + epsilon)
|
||||
scratch1.device(d) = (pop_var + pop_var.constant(epsilon)).rsqrt();
|
||||
|
||||
// scratch2 = sum(y_backprop * (x - mean))
|
||||
scratch2.device(d) =
|
||||
(y_backprop_rest_by_depth *
|
||||
(input_rest_by_depth -
|
||||
pop_mean.reshape(one_by_depth).broadcast(rest_by_one)))
|
||||
.sum(reduction_axis);
|
||||
|
||||
x_backprop.reshape(rest_by_depth).device(d) =
|
||||
(y_backprop_rest_by_depth * ((scratch1 * scale)
|
||||
.eval()
|
||||
.reshape(one_by_depth)
|
||||
.broadcast(rest_by_one)))
|
||||
.template cast<T>();
|
||||
scale_backprop.device(d) = scratch2 * scratch1;
|
||||
}
|
||||
};
|
||||
|
||||
} // namespace functor
|
||||
} // namespace tensorflow
|
||||
|
||||
#endif // TENSORFLOW_CORE_KERNELS_FUSED_BATCH_NORM_OP_H_
|
||||
@@ -0,0 +1,180 @@
|
||||
/* Copyright 2016 The TensorFlow Authors. All Rights Reserved.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
==============================================================================*/
|
||||
|
||||
#ifndef TENSORFLOW_CORE_KERNELS_GATHER_FUNCTOR_H_
|
||||
#define TENSORFLOW_CORE_KERNELS_GATHER_FUNCTOR_H_
|
||||
|
||||
#include "third_party/eigen3/unsupported/Eigen/CXX11/Tensor"
|
||||
|
||||
#include "tensorflow/core/framework/op_kernel.h"
|
||||
#include "tensorflow/core/framework/tensor_types.h"
|
||||
#include "tensorflow/core/framework/type_traits.h"
|
||||
#include "tensorflow/core/framework/variant.h"
|
||||
#include "tensorflow/core/kernels/bounds_check.h"
|
||||
#include "tensorflow/core/platform/prefetch.h"
|
||||
#include "tensorflow/core/platform/types.h"
|
||||
#include "tensorflow/core/util/work_sharder.h"
|
||||
|
||||
namespace tensorflow {
|
||||
typedef Eigen::ThreadPoolDevice CPUDevice;
|
||||
typedef Eigen::GpuDevice GPUDevice;
|
||||
|
||||
namespace functor {
|
||||
|
||||
// Helper method to copy using memcpy.
|
||||
template <typename T, typename Index, typename SliceIndex,
|
||||
SliceIndex static_slice_elems>
|
||||
SliceIndex HandleCopies(OpKernelContext* ctx,
|
||||
typename TTypes<T, 3>::ConstTensor params,
|
||||
typename TTypes<Index>::ConstFlat indices,
|
||||
SliceIndex slice_elems,
|
||||
typename TTypes<T, 3>::Tensor out) {
|
||||
const SliceIndex indices_size = static_cast<SliceIndex>(indices.dimension(0));
|
||||
const SliceIndex batch_size = static_cast<SliceIndex>(params.dimension(0));
|
||||
const Index limit = static_cast<Index>(params.dimension(1));
|
||||
T* out_base = &out(0, 0, 0);
|
||||
const T* params_base = ¶ms(0, 0, 0);
|
||||
if (static_slice_elems >= 0) {
|
||||
// Give compiler static knowledge of the number of elements/bytes
|
||||
slice_elems = static_slice_elems;
|
||||
}
|
||||
// Compute slice_bytes here so that static knowledge is available
|
||||
const size_t slice_bytes = slice_elems * sizeof(T);
|
||||
auto* worker_threads = ctx->device()->tensorflow_cpu_worker_threads();
|
||||
mutex mu;
|
||||
// Store the value of invalidate index for printing error information, it's a
|
||||
// shared variable.
|
||||
SliceIndex result = -1;
|
||||
auto work = [&](int64 start, int64 end) {
|
||||
SliceIndex batch_idx = static_cast<SliceIndex>(start / indices_size);
|
||||
SliceIndex indices_idx = static_cast<SliceIndex>(start % indices_size);
|
||||
SliceIndex batch_idx_end = static_cast<SliceIndex>(end / indices_size);
|
||||
SliceIndex indices_idx_end = static_cast<SliceIndex>(end % indices_size);
|
||||
|
||||
while ((batch_idx < batch_idx_end) ||
|
||||
(batch_idx == batch_idx_end && indices_idx < indices_idx_end)) {
|
||||
SliceIndex i_next = indices_idx + 1;
|
||||
SliceIndex b_next = batch_idx + 1;
|
||||
if ((batch_idx == batch_idx_end && i_next < indices_idx_end) ||
|
||||
(i_next < indices_size)) {
|
||||
port::prefetch<port::PREFETCH_HINT_T0>(
|
||||
¶ms(batch_idx, indices(i_next), 0));
|
||||
port::prefetch<port::PREFETCH_HINT_T0>(&out(batch_idx, i_next, 0));
|
||||
b_next = batch_idx;
|
||||
} else if (b_next <= batch_idx_end) {
|
||||
port::prefetch<port::PREFETCH_HINT_T0>(¶ms(b_next, indices(0), 0));
|
||||
port::prefetch<port::PREFETCH_HINT_T0>(&out(b_next, 0, 0));
|
||||
i_next = 0;
|
||||
}
|
||||
const Index index = internal::SubtleMustCopy(indices(indices_idx));
|
||||
if (!FastBoundsCheck(index, limit)) {
|
||||
mutex_lock l(mu);
|
||||
result = indices_idx;
|
||||
return;
|
||||
}
|
||||
// Copy using memcpy if possible, otherwise an Eigen loop
|
||||
// TODO(cwhipkey): avoid linking to framework to get Allocator (to improve
|
||||
// ahead-of-time compilation binary size).
|
||||
if (is_simple_type<T>::value) {
|
||||
// Avoid auto-promotion to Index from SliceIndex by casting.
|
||||
memcpy(
|
||||
out_base + (batch_idx * indices_size + indices_idx) * slice_elems,
|
||||
params_base + (batch_idx * static_cast<SliceIndex>(limit) +
|
||||
static_cast<SliceIndex>(index)) *
|
||||
slice_elems,
|
||||
slice_bytes);
|
||||
} else {
|
||||
// For non-"simple" types (e.g. strings).
|
||||
out.template chip<1>(indices_idx) = params.template chip<1>(index);
|
||||
}
|
||||
indices_idx = i_next;
|
||||
batch_idx = b_next;
|
||||
}
|
||||
};
|
||||
|
||||
Shard(worker_threads->num_threads, worker_threads->workers,
|
||||
batch_size * indices_size, slice_elems * sizeof(T), work);
|
||||
return result;
|
||||
}
|
||||
|
||||
template <typename T, typename Index>
|
||||
struct GatherFunctorCPU {
|
||||
int64 operator()(OpKernelContext* ctx,
|
||||
typename TTypes<T, 3>::ConstTensor params,
|
||||
typename TTypes<Index>::ConstFlat indices,
|
||||
typename TTypes<T, 3>::Tensor out) {
|
||||
const int64 N = indices.size();
|
||||
const int64 slice_size = out.dimension(2);
|
||||
int64 bad_i;
|
||||
|
||||
bool use_large = (slice_size > std::numeric_limits<int32>::max() ||
|
||||
params.size() > std::numeric_limits<int32>::max() ||
|
||||
N > std::numeric_limits<int32>::max());
|
||||
#define CALL(elems) \
|
||||
do { \
|
||||
if (use_large) { \
|
||||
bad_i = HandleCopies<T, Index, int64, elems>(ctx, params, indices, \
|
||||
slice_size, out); \
|
||||
} else { \
|
||||
const int32 small_slice = static_cast<int32>(slice_size); \
|
||||
bad_i = HandleCopies<T, Index, int32, elems>(ctx, params, indices, \
|
||||
small_slice, out); \
|
||||
} \
|
||||
} while (0)
|
||||
|
||||
if (slice_size == 10)
|
||||
CALL(10);
|
||||
else if (slice_size == 20)
|
||||
CALL(20);
|
||||
else
|
||||
CALL(-1);
|
||||
#undef CALL
|
||||
|
||||
return bad_i;
|
||||
}
|
||||
};
|
||||
|
||||
template <typename Device, typename T, typename Index>
|
||||
struct GatherFunctor {
|
||||
int64 operator()(OpKernelContext* ctx,
|
||||
typename TTypes<T, 3>::ConstTensor params,
|
||||
typename TTypes<Index>::ConstFlat indices,
|
||||
typename TTypes<T, 3>::Tensor out);
|
||||
};
|
||||
|
||||
template <typename T, typename Index>
|
||||
struct GatherFunctor<CPUDevice, T, Index> {
|
||||
int64 operator()(OpKernelContext* ctx,
|
||||
typename TTypes<T, 3>::ConstTensor params,
|
||||
typename TTypes<Index>::ConstFlat indices,
|
||||
typename TTypes<T, 3>::Tensor out) {
|
||||
return GatherFunctorCPU<T, Index>()(ctx, params, indices, out);
|
||||
}
|
||||
};
|
||||
|
||||
template <typename Index>
|
||||
struct GatherFunctor<GPUDevice, Variant, Index> {
|
||||
int64 operator()(OpKernelContext* ctx,
|
||||
typename TTypes<Variant, 3>::ConstTensor params,
|
||||
typename TTypes<Index>::ConstFlat indices,
|
||||
typename TTypes<Variant, 3>::Tensor out) {
|
||||
return GatherFunctorCPU<Variant, Index>()(ctx, params, indices, out);
|
||||
}
|
||||
};
|
||||
|
||||
} // namespace functor
|
||||
} // namespace tensorflow
|
||||
|
||||
#endif // TENSORFLOW_CORE_KERNELS_GATHER_FUNCTOR_H_
|
||||
@@ -0,0 +1,50 @@
|
||||
/* Copyright 2016 The TensorFlow Authors. All Rights Reserved.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
==============================================================================*/
|
||||
|
||||
#ifndef TENSORFLOW_CORE_KERNELS_GATHER_ND_OP_H_
|
||||
#define TENSORFLOW_CORE_KERNELS_GATHER_ND_OP_H_
|
||||
// Functor definition for GatherOp, must be compilable by nvcc.
|
||||
|
||||
#include "third_party/eigen3/unsupported/Eigen/CXX11/Tensor"
|
||||
#include "tensorflow/core/framework/tensor_types.h"
|
||||
#include "tensorflow/core/kernels/bounds_check.h"
|
||||
#include "tensorflow/core/platform/types.h"
|
||||
|
||||
namespace tensorflow {
|
||||
|
||||
class OpKernelContext;
|
||||
class Status;
|
||||
class Tensor;
|
||||
|
||||
namespace functor {
|
||||
template <typename Device, typename T, typename Index, int IXDIM>
|
||||
struct GatherNdSlice {
|
||||
// Performs a slice gather op on (Tparams, Tindices), writing to Tout.
|
||||
// Returns an index to Tindices if the value at that index is out of range.
|
||||
// Returns -1 if all values of Tindices are in range.
|
||||
Index operator()(const Device& d, const Index slice_size,
|
||||
typename TTypes<int32>::Scalar Tscratch,
|
||||
typename TTypes<T, IXDIM + 1>::ConstTensor Tparams,
|
||||
typename TTypes<Index>::ConstMatrix Tindices,
|
||||
typename TTypes<T>::Matrix Tout);
|
||||
};
|
||||
|
||||
template <typename Device, typename T, typename Index>
|
||||
Status DoGatherNd(OpKernelContext* c, const Tensor& params,
|
||||
const Tensor& indices, Tensor* out);
|
||||
} // namespace functor
|
||||
} // namespace tensorflow
|
||||
|
||||
#endif // TENSORFLOW_CORE_KERNELS_GATHER_ND_OP_H_
|
||||
@@ -0,0 +1,160 @@
|
||||
/* Copyright 2016 The TensorFlow Authors. All Rights Reserved.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
==============================================================================*/
|
||||
|
||||
#ifndef TENSORFLOW_CORE_KERNELS_GATHER_ND_OP_CPU_IMPL_H_
|
||||
#define TENSORFLOW_CORE_KERNELS_GATHER_ND_OP_CPU_IMPL_H_
|
||||
|
||||
// Specialization of GatherNdSlice to CPU
|
||||
|
||||
#define EIGEN_USE_THREADS
|
||||
|
||||
#include <atomic>
|
||||
|
||||
#include "tensorflow/core/framework/op_kernel.h"
|
||||
#include "tensorflow/core/framework/register_types.h"
|
||||
#include "tensorflow/core/framework/tensor.h"
|
||||
#include "tensorflow/core/kernels/bounds_check.h"
|
||||
#include "tensorflow/core/kernels/gather_nd_op.h"
|
||||
#include "tensorflow/core/platform/logging.h"
|
||||
#include "tensorflow/core/platform/mem.h"
|
||||
#include "tensorflow/core/platform/types.h"
|
||||
#include "tensorflow/core/util/util.h"
|
||||
|
||||
namespace tensorflow {
|
||||
|
||||
typedef Eigen::ThreadPoolDevice CPUDevice;
|
||||
|
||||
namespace generator {
|
||||
|
||||
template <typename T, typename Index, int IXDIM>
|
||||
class GatherNdSliceGenerator {
|
||||
public:
|
||||
EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE GatherNdSliceGenerator(
|
||||
const Index slice_size, typename TTypes<Index>::ConstMatrix Tindices,
|
||||
typename TTypes<T, IXDIM + 1>::ConstTensor Tparams,
|
||||
typename TTypes<T>::Matrix Tout, std::atomic<Index>* error_loc)
|
||||
: slice_size_(slice_size),
|
||||
Tindices_(Tindices),
|
||||
Tparams_(Tparams),
|
||||
Tout_(Tout),
|
||||
error_loc_(error_loc) {}
|
||||
|
||||
EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE bool GenerateIndices(
|
||||
const Index loc, Eigen::array<Eigen::DenseIndex, IXDIM + 1>* ix) const {
|
||||
(*ix)[IXDIM] = 0;
|
||||
bool out_of_bounds = false;
|
||||
for (int i = 0; i < IXDIM; ++i) {
|
||||
const Index ix_i = internal::SubtleMustCopy(Tindices_(loc, i));
|
||||
(*ix)[i] = ix_i;
|
||||
out_of_bounds |= !FastBoundsCheck(ix_i, Tparams_.dimension(i));
|
||||
}
|
||||
return out_of_bounds;
|
||||
}
|
||||
|
||||
EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE int32
|
||||
operator()(const Eigen::array<Eigen::DenseIndex, 1>& loc_array) const {
|
||||
const Index loc = loc_array[0];
|
||||
Eigen::array<Eigen::DenseIndex, IXDIM + 1> ix;
|
||||
Eigen::array<Eigen::DenseIndex, 2> ix_out;
|
||||
ix_out[0] = loc;
|
||||
ix_out[1] = 0;
|
||||
const bool out_of_bounds = GenerateIndices(loc, &ix);
|
||||
if (TF_PREDICT_FALSE(out_of_bounds)) {
|
||||
error_loc_->store(loc);
|
||||
std::fill_n(&Tout_(ix_out), slice_size_, T());
|
||||
} else {
|
||||
std::copy_n(&Tparams_(ix), slice_size_, &Tout_(ix_out));
|
||||
}
|
||||
|
||||
return static_cast<int32>(0); // Return something...
|
||||
}
|
||||
|
||||
private:
|
||||
const Index slice_size_;
|
||||
const typename TTypes<Index>::ConstMatrix Tindices_;
|
||||
const typename TTypes<T, IXDIM + 1>::ConstTensor Tparams_;
|
||||
mutable typename TTypes<T>::Matrix Tout_;
|
||||
std::atomic<Index>* error_loc_;
|
||||
};
|
||||
|
||||
} // namespace generator
|
||||
|
||||
namespace functor {
|
||||
|
||||
template <typename T, typename Index, int IXDIM>
|
||||
struct GatherNdSlice<CPUDevice, T, Index, IXDIM> {
|
||||
Index operator()(const CPUDevice& d, const Index slice_size,
|
||||
typename TTypes<int32>::Scalar Tscratch,
|
||||
typename TTypes<T, IXDIM + 1>::ConstTensor Tparams,
|
||||
typename TTypes<Index>::ConstMatrix Tindices,
|
||||
typename TTypes<T>::Matrix Tout) {
|
||||
std::atomic<Index> error_loc(-1);
|
||||
|
||||
const Eigen::DenseIndex batch_size = Tindices.dimension(0);
|
||||
#if !defined(EIGEN_HAS_INDEX_LIST)
|
||||
Eigen::Tensor<Eigen::DenseIndex, 1>::Dimensions reshape_dims{{ 1 }};
|
||||
Eigen::array<Eigen::DenseIndex, 1> broadcast_dims{{ batch_size }};
|
||||
#else
|
||||
Eigen::IndexList<Eigen::type2index<1> > reshape_dims;
|
||||
Eigen::IndexList<Eigen::DenseIndex> broadcast_dims;
|
||||
broadcast_dims.set(0, batch_size);
|
||||
#endif
|
||||
generator::GatherNdSliceGenerator<T, Index, IXDIM> gather_nd_generator(
|
||||
slice_size, Tindices, Tparams, Tout, &error_loc);
|
||||
|
||||
#if defined(INTEL_MKL) && defined(ENABLE_MKL)
|
||||
// Eigen implementation below is not highly performant. gather_nd_generator
|
||||
// does not seem to be called in parallel, leading to very poor performance.
|
||||
// Additionally, since it uses scalar (Tscratch) to invoke 'generate', it
|
||||
// needs to go through redundant operations like 'reshape', 'broadcast' and
|
||||
// 'sum'. OpenMP loop below essentially does same thing as Eigen code, but
|
||||
// is considerably more efficient.
|
||||
#pragma omp parallel for
|
||||
for (Eigen::DenseIndex i = 0; i < batch_size; i++) {
|
||||
const Eigen::array<Eigen::DenseIndex, 1> loc{i};
|
||||
gather_nd_generator(loc);
|
||||
}
|
||||
#else // INTEL_MKL && ENABLE_MKL
|
||||
Tscratch.device(d) = Tscratch.reshape(reshape_dims)
|
||||
.broadcast(broadcast_dims)
|
||||
.generate(gather_nd_generator)
|
||||
.sum();
|
||||
#endif // INTEL_MKL && ENABLE_MKL
|
||||
|
||||
// error_loc() returns -1 if there's no out-of-bounds index,
|
||||
// otherwise it returns the location of an OOB index in Tindices.
|
||||
return error_loc.load();
|
||||
}
|
||||
};
|
||||
|
||||
#define REGISTER_GATHER_ND_FULL(T, Index) \
|
||||
template Index GatherNdSlice<CPUDevice, T, Index, CPU_PROVIDED_IXDIM>:: \
|
||||
operator()(const CPUDevice& d, const Index slice_size, \
|
||||
typename TTypes<int32>::Scalar Tscratch, \
|
||||
typename TTypes<T, CPU_PROVIDED_IXDIM + 1>::ConstTensor Tparams, \
|
||||
typename TTypes<Index>::ConstMatrix Tindices, \
|
||||
typename TTypes<T>::Matrix Tout);
|
||||
|
||||
#define REGISTER_GATHER_ND_CPU(type) \
|
||||
REGISTER_GATHER_ND_FULL(type, int32); \
|
||||
REGISTER_GATHER_ND_FULL(type, int64)
|
||||
|
||||
TF_CALL_ALL_TYPES(REGISTER_GATHER_ND_CPU);
|
||||
|
||||
} // namespace functor
|
||||
|
||||
} // namespace tensorflow
|
||||
|
||||
#endif // TENSORFLOW_CORE_KERNELS_GATHER_ND_OP_CPU_IMPL_H_
|
||||
@@ -0,0 +1,123 @@
|
||||
/* Copyright 2016 The TensorFlow Authors. All Rights Reserved.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
==============================================================================*/
|
||||
|
||||
// This is a set of different implementations for the basic matrix by matrix
|
||||
// multiply function, commonly known as GEMM after the BLAS library's naming.
|
||||
// Having a standard interface enables us to swap out implementations on
|
||||
// different platforms, to make sure we're using the optimal version. They are
|
||||
// implemented as C++ template functors, so they're easy to swap into all of the
|
||||
// different kernels that use them.
|
||||
|
||||
#if !defined(EIGEN_USE_THREADS)
|
||||
#error "EIGEN_USE_THREADS must be enabled by all .cc files including this."
|
||||
#endif // EIGEN_USE_THREADS
|
||||
|
||||
#ifndef TENSORFLOW_CORE_KERNELS_GEMM_FUNCTORS_H_
|
||||
#define TENSORFLOW_CORE_KERNELS_GEMM_FUNCTORS_H_
|
||||
|
||||
#include <string.h>
|
||||
#include <map>
|
||||
#include <vector>
|
||||
|
||||
#include "tensorflow/core/common_runtime/threadpool_device.h"
|
||||
#include "tensorflow/core/framework/op_kernel.h"
|
||||
#include "tensorflow/core/framework/tensor.h"
|
||||
#include "tensorflow/core/framework/tensor_types.h"
|
||||
|
||||
// Apple provides an optimized BLAS library that is better than Eigen for their
|
||||
// devices, so use that if possible.
|
||||
#if defined(__APPLE__) && defined(USE_GEMM_FOR_CONV)
|
||||
#include <Accelerate/Accelerate.h>
|
||||
#define USE_CBLAS_GEMM
|
||||
#endif // __APPLE__
|
||||
|
||||
// Older Raspberry Pi systems don't have NEON SIMD acceleration, so Eigen falls
|
||||
// back to scalar code, but OpenBLAS has much faster support so prefer that.
|
||||
#if defined(RASPBERRY_PI) && defined(USE_GEMM_FOR_CONV) && defined(USE_OPENBLAS)
|
||||
#include <cblas.h>
|
||||
#define USE_CBLAS_GEMM
|
||||
#endif
|
||||
|
||||
// A readable but slow implementation of matrix multiplication, useful for
|
||||
// debugging and understanding the algorithm. Use instead of FastGemmFunctor in
|
||||
// the Im2ColConvFunctor template definition inside the op registration to
|
||||
// enable. Assumes row-major ordering of the values in memory.
|
||||
template <class T1, class T2, class T3>
|
||||
class ReferenceGemmFunctor {
|
||||
public:
|
||||
void operator()(tensorflow::OpKernelContext* ctx, size_t m, size_t n,
|
||||
size_t k, const T1* a, size_t lda, const T2* b, size_t ldb,
|
||||
T3* c, size_t ldc) {
|
||||
const size_t a_i_stride = lda;
|
||||
const size_t a_l_stride = 1;
|
||||
const size_t b_j_stride = 1;
|
||||
const size_t b_l_stride = ldb;
|
||||
const size_t c_i_stride = ldc;
|
||||
const size_t c_j_stride = 1;
|
||||
size_t i, j, l;
|
||||
for (j = 0; j < n; j++) {
|
||||
for (i = 0; i < m; i++) {
|
||||
T3 total(0);
|
||||
for (l = 0; l < k; l++) {
|
||||
const size_t a_index = ((i * a_i_stride) + (l * a_l_stride));
|
||||
const T1 a_value = a[a_index];
|
||||
const size_t b_index = ((j * b_j_stride) + (l * b_l_stride));
|
||||
const T2 b_value = b[b_index];
|
||||
total += (a_value * b_value);
|
||||
}
|
||||
const size_t c_index = ((i * c_i_stride) + (j * c_j_stride));
|
||||
c[c_index] = total;
|
||||
}
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
// Uses the optimized EigenTensor library to implement the matrix multiplication
|
||||
// required by the Im2ColConvFunctor class. We supply the two input and one
|
||||
// output types so that the accumulator can potentially be higher-precision than
|
||||
// the inputs, even though we don't currently take advantage of this.
|
||||
template <class T1, class T2, class T3>
|
||||
class FastGemmFunctor {
|
||||
public:
|
||||
void operator()(tensorflow::OpKernelContext* ctx, size_t m, size_t n,
|
||||
size_t k, const T1* a, size_t lda, const T2* b, size_t ldb,
|
||||
T3* c, size_t ldc) {
|
||||
typename tensorflow::TTypes<const T1>::Matrix a_matrix(a, m, k);
|
||||
typename tensorflow::TTypes<const T2>::Matrix b_matrix(b, k, n);
|
||||
typename tensorflow::TTypes<T3>::Matrix c_matrix(c, m, n);
|
||||
|
||||
Eigen::array<Eigen::IndexPair<Eigen::DenseIndex>, 1> dim_pair;
|
||||
dim_pair[0].first = 1;
|
||||
dim_pair[0].second = 0;
|
||||
c_matrix.device(ctx->eigen_device<Eigen::ThreadPoolDevice>()) =
|
||||
a_matrix.contract(b_matrix, dim_pair);
|
||||
}
|
||||
};
|
||||
|
||||
// If we have a fast CBLAS library, use its implementation through a wrapper.
|
||||
#if defined(USE_CBLAS_GEMM)
|
||||
template <>
|
||||
class FastGemmFunctor<float, float, float> {
|
||||
public:
|
||||
void operator()(tensorflow::OpKernelContext* ctx, size_t m, size_t n,
|
||||
size_t k, const float* a, size_t lda, const float* b,
|
||||
size_t ldb, float* c, size_t ldc) {
|
||||
cblas_sgemm(CblasRowMajor, CblasNoTrans, CblasNoTrans, m, n, k, 1.0f, a,
|
||||
lda, b, ldb, 0.0f, c, ldc);
|
||||
}
|
||||
};
|
||||
#endif // USE_CBLAS_GEMM
|
||||
|
||||
#endif // TENSORFLOW_CORE_KERNELS_GEMM_FUNCTORS_H_
|
||||
@@ -0,0 +1,163 @@
|
||||
/* Copyright 2015 The TensorFlow Authors. All Rights Reserved.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
==============================================================================*/
|
||||
|
||||
#ifndef TENSORFLOW_CORE_KERNELS_GPU_UTILS_H_
|
||||
#define TENSORFLOW_CORE_KERNELS_GPU_UTILS_H_
|
||||
|
||||
#if GOOGLE_CUDA
|
||||
|
||||
#include <unordered_map>
|
||||
|
||||
#include "tensorflow/core/lib/strings/str_util.h"
|
||||
#include "tensorflow/core/lib/strings/strcat.h"
|
||||
#include "tensorflow/core/lib/strings/stringprintf.h"
|
||||
#include "tensorflow/core/platform/logging.h"
|
||||
#include "tensorflow/core/platform/stream_executor.h"
|
||||
|
||||
namespace tensorflow {
|
||||
|
||||
template <typename T>
|
||||
inline se::DeviceMemory<T> AsDeviceMemory(const T* cuda_memory, uint64 size) {
|
||||
se::DeviceMemoryBase wrapped(const_cast<T*>(cuda_memory), size * sizeof(T));
|
||||
se::DeviceMemory<T> typed(wrapped);
|
||||
return typed;
|
||||
}
|
||||
|
||||
// A helper class that looks up the best autotuned config from parameters.
|
||||
// Due to the noisy nature of autotune, especially with multiple devices, it
|
||||
// only accepts a config if its margin exceeds a threshold.
|
||||
// For the same shape configs, if a new best config matches the previous best,
|
||||
// they get promoted; otherwise, the winner gets demoted. This process stops
|
||||
// when the winner's score exceeds the threshold.
|
||||
// In a bad case when two configs are very close to each other and flips
|
||||
// back and forth randomly, the expected number of experiments before autotune
|
||||
// settles is O(threshold ^ 2). So we recommend that number of warmup runs
|
||||
// for any benchmarks.
|
||||
template <typename Parameters, typename Config>
|
||||
class AutoTuneMap {
|
||||
public:
|
||||
bool Find(const Parameters& params, Config* config) const {
|
||||
mutex_lock lock(mu_);
|
||||
auto iter = params_config_map_.find(params);
|
||||
if (iter == params_config_map_.end() ||
|
||||
(iter->second.score < min_score_threshold_ &&
|
||||
iter->second.count <= max_autotune_count_)) {
|
||||
return false;
|
||||
}
|
||||
*config = iter->second.config;
|
||||
return true;
|
||||
}
|
||||
void Insert(const Parameters& params, const Config& config) {
|
||||
mutex_lock lock(mu_);
|
||||
auto iter = params_config_map_.find(params);
|
||||
int new_score = 0;
|
||||
if (iter == params_config_map_.end()) {
|
||||
// Create a new entry if params is new.
|
||||
VLOG(1) << GetActionSummary("creates", params, config);
|
||||
params_config_map_.insert(
|
||||
std::make_pair(params, ValueType{config, 1, 1}));
|
||||
new_score = 1;
|
||||
} else if (iter->second.score < min_score_threshold_ &&
|
||||
iter->second.count <= max_autotune_count_) {
|
||||
DCHECK_GT(iter->second.score, 0);
|
||||
if (iter->second.config != config) {
|
||||
// If it is different from the current winner, demotes the winner.
|
||||
VLOG(1) << GetActionSummary("demotes", params, config);
|
||||
new_score = --iter->second.score;
|
||||
++iter->second.count;
|
||||
if (new_score <= 0) {
|
||||
VLOG(1) << GetActionSummary("erases", params, config);
|
||||
params_config_map_.erase(iter);
|
||||
}
|
||||
} else {
|
||||
// If it is the same as the current winner, promotes the winner.
|
||||
VLOG(1) << GetActionSummary("promotes", params, config);
|
||||
new_score = ++iter->second.score;
|
||||
++iter->second.count;
|
||||
}
|
||||
}
|
||||
if (new_score >= min_score_threshold_) {
|
||||
VLOG(1) << GetActionSummary("accepts", params, config);
|
||||
}
|
||||
}
|
||||
|
||||
private:
|
||||
AutoTuneMap(const string& name) : name_(name) {
|
||||
min_score_threshold_ = 1;
|
||||
int min_warmup_iterations = 10;
|
||||
const char* threshold_str = getenv("TF_AUTOTUNE_THRESHOLD");
|
||||
if (threshold_str != nullptr) {
|
||||
strings::safe_strto32(threshold_str, &min_score_threshold_);
|
||||
}
|
||||
const char* min_warmup_iteration_str =
|
||||
getenv("TF_AUTOTUNE_MIN_WARMUP_ITERATIONS");
|
||||
if (min_warmup_iteration_str != nullptr) {
|
||||
strings::safe_strto32(min_warmup_iteration_str, &min_warmup_iterations);
|
||||
}
|
||||
min_score_threshold_ = std::max(min_score_threshold_, 1);
|
||||
max_autotune_count_ = std::max(
|
||||
5 * min_score_threshold_ * min_score_threshold_, min_warmup_iterations);
|
||||
}
|
||||
|
||||
template <class Group, class Params, class Cfg>
|
||||
friend class AutoTuneSingleton;
|
||||
|
||||
struct Hasher {
|
||||
std::size_t operator()(const Parameters& parameter) const {
|
||||
return parameter.hash();
|
||||
}
|
||||
};
|
||||
|
||||
string GetActionSummary(StringPiece action, const Parameters& params,
|
||||
const Config& config) {
|
||||
return strings::Printf("autotune_map %s %s: %s -> (%s)", name_.c_str(),
|
||||
string(action).c_str(), params.ToString().c_str(),
|
||||
config.ToString().c_str());
|
||||
}
|
||||
|
||||
mutable mutex mu_;
|
||||
struct ValueType {
|
||||
Config config;
|
||||
int32 score;
|
||||
int32 count;
|
||||
};
|
||||
std::unordered_map<Parameters, ValueType, Hasher> params_config_map_
|
||||
GUARDED_BY(mu_);
|
||||
string name_;
|
||||
int32 min_score_threshold_;
|
||||
int32 max_autotune_count_;
|
||||
|
||||
TF_DISALLOW_COPY_AND_ASSIGN(AutoTuneMap);
|
||||
};
|
||||
|
||||
// A Singleton helper that manages the global autotune results by groups.
|
||||
// The caller specified arbitrary Group type that can distinguish between
|
||||
// different autotune results, even if their Parameters and Configs are the
|
||||
// same.
|
||||
template <class Group, typename Parameters, typename Config>
|
||||
class AutoTuneSingleton {
|
||||
public:
|
||||
typedef AutoTuneMap<Parameters, Config> AutoTuneType;
|
||||
static AutoTuneType* GetInstance() {
|
||||
static AutoTuneType* instance = new AutoTuneType(Group::name());
|
||||
return instance;
|
||||
}
|
||||
};
|
||||
|
||||
} // namespace tensorflow
|
||||
|
||||
#endif // GOOGLE_CUDA
|
||||
|
||||
#endif // TENSORFLOW_CORE_KERNELS_GPU_UTILS_H_
|
||||
Loaded 100 of 1283 files, more files were not shown because too many files have changed in this diff.
Show more
Reference in new issue
Block a user