This commit is contained in:
Kevin Darby committed 2019-01-02 17:04:10 -05:00
1 parent eb1b42a5e8
commit 458eb2aef1
1283 files changed
+323373

No files matched your search

@@ -0,0 +1,60 @@
/* Copyright 2015 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
#ifndef TENSORFLOW_CORE_COMMON_RUNTIME_ALLOCATOR_RETRY_H_
#define TENSORFLOW_CORE_COMMON_RUNTIME_ALLOCATOR_RETRY_H_
#include "tensorflow/core/platform/env.h"
#include "tensorflow/core/platform/mutex.h"
#include "tensorflow/core/platform/types.h"
namespace tensorflow {
// A retrying wrapper for a memory allocator.
class AllocatorRetry {
public:
AllocatorRetry();
// Call 'alloc_func' to obtain memory. On first call,
// 'verbose_failure' will be false. If return value is nullptr,
// then wait up to 'max_millis_to_wait' milliseconds, retrying each
// time a call to DeallocateRaw() is detected, until either a good
// pointer is returned or the deadline is exhausted. If the
// deadline is exhausted, try one more time with 'verbose_failure'
// set to true. The value returned is either the first good pointer
// obtained from 'alloc_func' or nullptr.
void* AllocateRaw(std::function<void*(size_t alignment, size_t num_bytes,
bool verbose_failure)>
alloc_func,
int max_millis_to_wait, size_t alignment, size_t bytes);
// Called to notify clients that some memory was returned.
void NotifyDealloc();
private:
Env* env_;
mutex mu_;
condition_variable memory_returned_;
};
// Implementation details below
inline void AllocatorRetry::NotifyDealloc() {
mutex_lock l(mu_);
memory_returned_.notify_all();
}
} // namespace tensorflow
#endif // TENSORFLOW_CORE_COMMON_RUNTIME_ALLOCATOR_RETRY_H_
@@ -0,0 +1,143 @@
/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
#ifndef TENSORFLOW_CORE_COMMON_RUNTIME_BASE_COLLECTIVE_EXECUTOR_H_
#define TENSORFLOW_CORE_COMMON_RUNTIME_BASE_COLLECTIVE_EXECUTOR_H_
#include <memory>
#include <string>
#include "tensorflow/core/common_runtime/buf_rendezvous.h"
#include "tensorflow/core/framework/collective.h"
#include "tensorflow/core/framework/device_attributes.pb.h"
namespace tensorflow {
class CollectiveImplementation;
class DeviceMgr;
class Device;
// Helper interface that aliases regular subfields of a Tensor as separate
// Tensors for in-place update.
class CollectiveAdapter {
public:
virtual ~CollectiveAdapter() {}
// Move the backing tensor to 'output' with its original storage and
// shape. After this call this CollectiveAdapter object should be
// deleted immediately without calling any of its other methods.
virtual void ConsumeFinalValue(Tensor* output) = 0;
// const access to entire intermediate value for debugging
virtual const Tensor& Value() const = 0;
// Returns tensor for chunk i which aliases the backing buffer.
virtual Tensor ChunkAlias(int i) = 0;
// Returns tensor allocated on the same device but with its own
// separate backing buffer. Will have same type and size as
// chunk i.
virtual Tensor TempChunk(int i) const = 0;
// Bytes in chunk i
virtual int64 ChunkBytes(int i) const = 0;
// Generate a CPU RAM scalar tensor of the same DataType as the
// backing tensor with the given integer value.
virtual Tensor Scalar(int v) const = 0;
// Generate a scalar tensor of same DataType and on the same device
// as the backing tensor.
virtual Tensor Scalar(Allocator* a) const = 0;
// Debugging string describing buffer location
virtual string TBounds(const Tensor& t) const = 0;
virtual string DebugString() const = 0;
// Computes the number of elements per alias chunk tensor.
//
// A CHECK in tensor.cc expects that the memory buffer backing a
// Tensor will be aligned according to EIGEN_MAX_ALIGN_BYTES. To
// ensure that all chunk aliasing Tensors maintain this alignment we
// need to pick a chunk size that preserves it. Note than in extreme
// cases (impractical, but possible with very small tensors) one or
// more tail chunks can end up emptby.
static int64 AlignedChunkElts(int64 elt_bytes, int64 total_elts,
int64 num_chunks);
};
// Create a CollectiveAdaptor wrapping 'output', specialized to its
// data-type and shape.
CollectiveAdapter* MakeCollectiveAdapter(Tensor* output, int num_chunks,
Allocator* allocator);
// Default implementation of CollectiveExecutor. Delegates the actual
// work of moving data to a class specialized for the operation type,
// arguments and device+interconnect topology.
class BaseCollectiveExecutor : public CollectiveExecutor {
public:
BaseCollectiveExecutor(CollectiveExecutorMgrInterface* cem,
PerStepCollectiveRemoteAccess* remote_access,
int64 step_id, const DeviceMgr* dev_mgr)
: CollectiveExecutor(cem),
step_id_(step_id),
dev_mgr_(dev_mgr),
remote_access_(remote_access) {}
~BaseCollectiveExecutor() override;
void StartAbort(const Status& s) override;
void ExecuteAsync(OpKernelContext* ctx, const CollectiveParams& col_params,
const string& exec_key, StatusCallback done) override;
PerStepCollectiveRemoteAccess* remote_access() override {
return remote_access_.get();
}
void RecvFromPeer(const string& peer_device, const string& peer_task,
bool peer_is_local, const string& key, Device* to_device,
DeviceContext* to_device_ctx,
const AllocatorAttributes& to_alloc_attr, Tensor* to_tensor,
const DeviceLocality& client_locality, int stream_index,
const StatusCallback& done) override {
remote_access_->RecvFromPeer(
peer_device, peer_task, peer_is_local, key, to_device, to_device_ctx,
to_alloc_attr, to_tensor, client_locality, stream_index, done);
}
void PostToPeer(const string& peer_device, const string& peer_task,
const string& key, Device* from_device,
DeviceContext* from_device_ctx,
const AllocatorAttributes& from_alloc_attr,
const Tensor* from_tensor,
const DeviceLocality& client_locality,
const StatusCallback& done) override {
remote_access_->PostToPeer(peer_device, peer_task, key, from_device,
from_device_ctx, from_alloc_attr, from_tensor,
client_locality, done);
}
protected:
const int64 step_id_;
const DeviceMgr* dev_mgr_; // Not owned.
std::unique_ptr<PerStepCollectiveRemoteAccess> remote_access_;
private:
Status CreateCollective(const CollectiveParams& col_params,
CollectiveImplementationInterface** col_impl);
};
} // namespace tensorflow
#endif // TENSORFLOW_CORE_COMMON_RUNTIME_BASE_COLLECTIVE_EXECUTOR_H_
@@ -0,0 +1,446 @@
/* Copyright 2015 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
#ifndef TENSORFLOW_CORE_COMMON_RUNTIME_BFC_ALLOCATOR_H_
#define TENSORFLOW_CORE_COMMON_RUNTIME_BFC_ALLOCATOR_H_
#include <array>
#include <memory>
#include <string>
#include <unordered_map>
#include <vector>
#include "tensorflow/core/common_runtime/allocator_retry.h"
#include "tensorflow/core/framework/allocator.h"
#include "tensorflow/core/lib/gtl/stl_util.h"
#include "tensorflow/core/lib/strings/strcat.h"
#include "tensorflow/core/platform/macros.h"
#include "tensorflow/core/platform/mutex.h"
#include "tensorflow/core/platform/thread_annotations.h"
#include "tensorflow/core/platform/types.h"
#include "tensorflow/core/protobuf/config.pb.h"
namespace tensorflow {
// A memory allocator that implements a 'best-fit with coalescing'
// algorithm. This is essentially a very simple version of Doug Lea's
// malloc (dlmalloc).
//
// The goal of this allocator is to support defragmentation via
// coalescing. One assumption we make is that the process using this
// allocator owns pretty much all of the memory, and that nearly
// all requests to allocate memory go through this interface.
class BFCAllocator : public Allocator {
public:
// Takes ownership of sub_allocator.
BFCAllocator(SubAllocator* sub_allocator, size_t total_memory,
bool allow_growth, const string& name);
~BFCAllocator() override;
string Name() override { return name_; }
void* AllocateRaw(size_t alignment, size_t num_bytes) override;
void* AllocateRaw(size_t alignment, size_t num_bytes,
const AllocationAttributes& allocation_attr) override;
void DeallocateRaw(void* ptr) override;
bool TracksAllocationSizes() override;
size_t RequestedSize(const void* ptr) override;
size_t AllocatedSize(const void* ptr) override;
int64 AllocationId(const void* ptr) override;
void GetStats(AllocatorStats* stats) override;
void ClearStats() override;
private:
struct Bin;
void* AllocateRawInternal(size_t alignment, size_t num_bytes,
bool dump_log_on_failure);
void DeallocateRawInternal(void* ptr);
// A ChunkHandle is an index into the chunks_ vector in BFCAllocator
// kInvalidChunkHandle means an invalid chunk
typedef size_t ChunkHandle;
static const int kInvalidChunkHandle = -1;
typedef int BinNum;
static const int kInvalidBinNum = -1;
static const int kNumBins = 21;
// A Chunk points to a piece of memory that's either entirely free or entirely
// in use by one user memory allocation.
//
// An AllocationRegion's memory is split up into one or more disjoint Chunks,
// which together cover the whole region without gaps. Chunks participate in
// a doubly-linked list, and the prev/next pointers point to the physically
// adjacent chunks.
//
// Since a chunk cannot be partially in use, we may need to split a free chunk
// in order to service a user allocation. We always merge adjacent free
// chunks.
//
// Chunks contain information about whether they are in use or whether they
// are free, and contain a pointer to the bin they are in.
struct Chunk {
size_t size = 0; // Full size of buffer.
// We sometimes give chunks that are larger than needed to reduce
// fragmentation. requested_size keeps track of what the client
// actually wanted so we can understand whether our splitting
// strategy is efficient.
size_t requested_size = 0;
// allocation_id is set to -1 when the chunk is not in use. It is assigned a
// value greater than zero before the chunk is returned from
// AllocateRaw, and this value is unique among values assigned by
// the parent allocator.
int64 allocation_id = -1;
void* ptr = nullptr; // pointer to granted subbuffer.
// If not kInvalidChunkHandle, the memory referred to by 'prev' is directly
// preceding the memory used by this chunk. E.g., It should start
// at 'ptr - prev->size'
ChunkHandle prev = kInvalidChunkHandle;
// If not kInvalidChunkHandle, the memory referred to by 'next' is directly
// following the memory used by this chunk. E.g., It should be at
// 'ptr + size'
ChunkHandle next = kInvalidChunkHandle;
// What bin are we in?
BinNum bin_num = kInvalidBinNum;
bool in_use() const { return allocation_id != -1; }
string DebugString(BFCAllocator* a,
bool recurse) NO_THREAD_SAFETY_ANALYSIS {
string dbg;
strings::StrAppend(
&dbg, " Size: ", strings::HumanReadableNumBytes(size),
" | Requested Size: ", strings::HumanReadableNumBytes(requested_size),
" | in_use: ", in_use());
if (recurse && prev != BFCAllocator::kInvalidChunkHandle) {
Chunk* p = a->ChunkFromHandle(prev);
strings::StrAppend(&dbg, ", prev: ", p->DebugString(a, false));
}
if (recurse && next != BFCAllocator::kInvalidChunkHandle) {
Chunk* n = a->ChunkFromHandle(next);
strings::StrAppend(&dbg, ", next: ", n->DebugString(a, false));
}
return dbg;
}
};
// A Bin is a collection of similar-sized free chunks.
struct Bin {
// All chunks in this bin have >= bin_size memory.
size_t bin_size = 0;
struct ChunkComparator {
explicit ChunkComparator(BFCAllocator* allocator)
: allocator_(allocator) {}
// Sort first by size and then use pointer address as a tie breaker.
bool operator()(const ChunkHandle ha,
const ChunkHandle hb) const NO_THREAD_SAFETY_ANALYSIS {
const Chunk* a = allocator_->ChunkFromHandle(ha);
const Chunk* b = allocator_->ChunkFromHandle(hb);
if (a->size != b->size) {
return a->size < b->size;
}
return a->ptr < b->ptr;
}
private:
BFCAllocator* allocator_; // The parent allocator
};
typedef std::set<ChunkHandle, ChunkComparator> FreeChunkSet;
// List of free chunks within the bin, sorted by chunk size.
// Chunk * not owned.
FreeChunkSet free_chunks;
Bin(BFCAllocator* allocator, size_t bs)
: bin_size(bs), free_chunks(ChunkComparator(allocator)) {}
};
static const size_t kMinAllocationBits = 8;
static const size_t kMinAllocationSize = 1 << kMinAllocationBits;
// BFCAllocator allocates memory into a collection of disjoint
// AllocationRegions. Each AllocationRegion corresponds to one call to
// SubAllocator::Alloc().
//
// An AllocationRegion contains one or more Chunks, covering all of its
// memory. Its primary job is to map a pointers to ChunkHandles.
//
// This class is thread-compatible.
class AllocationRegion {
public:
AllocationRegion(void* ptr, size_t memory_size)
: ptr_(ptr),
memory_size_(memory_size),
end_ptr_(
static_cast<void*>(static_cast<char*>(ptr_) + memory_size_)) {
DCHECK_EQ(0, memory_size % kMinAllocationSize);
const size_t n_handles =
(memory_size + kMinAllocationSize - 1) / kMinAllocationSize;
handles_.reset(new ChunkHandle[n_handles]);
for (size_t i = 0; i < n_handles; i++) {
handles_[i] = kInvalidChunkHandle;
}
}
AllocationRegion() = default;
AllocationRegion(AllocationRegion&& other) { Swap(other); }
AllocationRegion& operator=(AllocationRegion&& other) {
Swap(other);
return *this;
}
void* ptr() const { return ptr_; }
void* end_ptr() const { return end_ptr_; }
size_t memory_size() const { return memory_size_; }
ChunkHandle get_handle(const void* p) const {
return handles_[IndexFor(p)];
}
void set_handle(const void* p, ChunkHandle h) { handles_[IndexFor(p)] = h; }
void erase(const void* p) { set_handle(p, kInvalidChunkHandle); }
private:
void Swap(AllocationRegion& other) {
std::swap(ptr_, other.ptr_);
std::swap(memory_size_, other.memory_size_);
std::swap(end_ptr_, other.end_ptr_);
std::swap(handles_, other.handles_);
}
int IndexFor(const void* p) const {
std::uintptr_t p_int = reinterpret_cast<std::uintptr_t>(p);
std::uintptr_t base_int = reinterpret_cast<std::uintptr_t>(ptr_);
DCHECK_GE(p_int, base_int);
DCHECK_LT(p_int, base_int + memory_size_);
return static_cast<int>(((p_int - base_int) >> kMinAllocationBits));
}
// Metadata about the allocation region.
void* ptr_ = nullptr;
size_t memory_size_ = 0;
void* end_ptr_ = nullptr;
// Array of size "memory_size / kMinAllocationSize". It is
// indexed by (p-base) / kMinAllocationSize, contains ChunkHandle
// for the memory allocation represented by "p"
std::unique_ptr<ChunkHandle[]> handles_;
TF_DISALLOW_COPY_AND_ASSIGN(AllocationRegion);
};
// RegionManager aggregates one or more "AllocationRegions" and provides
// a layer of indirection from pointers to the underlying ChunkHandle,
// allowing allocation across multiple discontiguous memory regions.
//
// This class is thread-compatible.
class RegionManager {
public:
RegionManager() {}
~RegionManager() {}
void AddAllocationRegion(void* ptr, size_t memory_size) {
// Insert sorted by end_ptr
auto entry =
std::upper_bound(regions_.begin(), regions_.end(), ptr, &Comparator);
regions_.insert(entry, AllocationRegion(ptr, memory_size));
}
ChunkHandle get_handle(const void* p) const {
return RegionFor(p)->get_handle(p);
}
void set_handle(const void* p, ChunkHandle h) {
return MutableRegionFor(p)->set_handle(p, h);
}
void erase(const void* p) { return MutableRegionFor(p)->erase(p); }
const std::vector<AllocationRegion>& regions() const { return regions_; }
private:
static bool Comparator(const void* ptr, const AllocationRegion& other) {
return ptr < other.end_ptr();
}
AllocationRegion* MutableRegionFor(const void* p) {
return const_cast<AllocationRegion*>(RegionFor(p));
}
const AllocationRegion* RegionFor(const void* p) const {
auto entry =
std::upper_bound(regions_.begin(), regions_.end(), p, &Comparator);
if (entry != regions_.end()) {
return &(*entry);
}
LOG(FATAL) << "Could not find Region for " << p;
return nullptr;
}
private:
std::vector<AllocationRegion> regions_;
};
// Returns 'bytes' rounded up to the next highest kMinAllocationSize.
static size_t RoundedBytes(size_t bytes);
// Try to add a new memory region that can satisfy an allocation of
// 'rounded_bytes' bytes. Returns true on success and false on
// failure.
bool Extend(size_t alignment, size_t rounded_bytes)
EXCLUSIVE_LOCKS_REQUIRED(lock_);
// Returns a pointer to an underlying allocated chunk of size
// 'rounded_bytes'.
void* FindChunkPtr(BinNum bin_num, size_t rounded_bytes, size_t num_bytes)
EXCLUSIVE_LOCKS_REQUIRED(lock_);
// Splits the chunk specified by 'h' into two chunks, one at least
// of size 'num_bytes'.
void SplitChunk(ChunkHandle h, size_t num_bytes)
EXCLUSIVE_LOCKS_REQUIRED(lock_);
// Merges the two chunk handles. Requires that the chunks are
// contiguous in their allocation.
void Merge(ChunkHandle h, ChunkHandle h2) EXCLUSIVE_LOCKS_REQUIRED(lock_);
// Frees the memory represented by 'h', coalescing the chunk if
// possible.
void FreeAndMaybeCoalesce(ChunkHandle h) EXCLUSIVE_LOCKS_REQUIRED(lock_);
// Adds the chunk 'h' to the proper free bin.
void InsertFreeChunkIntoBin(ChunkHandle h) EXCLUSIVE_LOCKS_REQUIRED(lock_);
// Removes the free chunk pointed to by 'c' from the set free_chunks.
void RemoveFreeChunkIterFromBin(Bin::FreeChunkSet* free_chunks,
const Bin::FreeChunkSet::iterator& c)
EXCLUSIVE_LOCKS_REQUIRED(lock_);
// Removes a free chunk from the bin.
void RemoveFreeChunkFromBin(ChunkHandle h) EXCLUSIVE_LOCKS_REQUIRED(lock_);
// Removes the chunk metadata represented by 'h'.
void DeleteChunk(ChunkHandle h) EXCLUSIVE_LOCKS_REQUIRED(lock_);
string RenderOccupancy() EXCLUSIVE_LOCKS_REQUIRED(lock_);
void DumpMemoryLog(size_t num_bytes) EXCLUSIVE_LOCKS_REQUIRED(lock_);
ChunkHandle AllocateChunk() EXCLUSIVE_LOCKS_REQUIRED(lock_);
void DeallocateChunk(ChunkHandle h) EXCLUSIVE_LOCKS_REQUIRED(lock_);
Chunk* ChunkFromHandle(ChunkHandle h) EXCLUSIVE_LOCKS_REQUIRED(lock_);
// Information about a Bin that is useful for debugging.
struct BinDebugInfo {
size_t total_bytes_in_use = 0;
size_t total_bytes_in_bin = 0;
size_t total_requested_bytes_in_use = 0;
size_t total_chunks_in_use = 0;
size_t total_chunks_in_bin = 0;
};
// Computes and returns a BinDebugInfo for each Bin.
std::array<BinDebugInfo, kNumBins> get_bin_debug_info()
EXCLUSIVE_LOCKS_REQUIRED(lock_);
AllocatorRetry retry_helper_;
// Structures immutable after construction
size_t memory_limit_ = 0;
inline int Log2FloorNonZeroSlow(uint64 n) {
int r = 0;
while (n > 0) {
r++;
n >>= 1;
}
return r - 1;
}
// Returns floor(log2(n)).
inline int Log2FloorNonZero(uint64 n) {
#if defined(__GNUC__)
return 63 ^ __builtin_clzll(n);
#elif defined(PLATFORM_WINDOWS) && (_WIN64)
unsigned long index;
_BitScanReverse64(&index, n);
return index;
#else
return Log2FloorNonZeroSlow(n);
#endif
}
// Map from bin size to Bin
Bin* BinFromIndex(BinNum index) {
return reinterpret_cast<Bin*>(&(bins_space_[index * sizeof(Bin)]));
}
size_t BinNumToSize(BinNum index) {
return static_cast<size_t>(256) << index;
}
BinNum BinNumForSize(size_t bytes) {
uint64 v = std::max<size_t>(bytes, 256) >> kMinAllocationBits;
int b = std::min(kNumBins - 1, Log2FloorNonZero(v));
return b;
}
Bin* BinForSize(size_t bytes) { return BinFromIndex(BinNumForSize(bytes)); }
char bins_space_[sizeof(Bin) * kNumBins];
// The size of the current region allocation.
size_t curr_region_allocation_bytes_;
// The total number of allocated bytes by the allocator.
size_t total_region_allocated_bytes_ = 0;
// An indicator that expansion of a region has hit the limits
// of the available memory.
bool started_backpedal_ = false;
std::unique_ptr<SubAllocator> sub_allocator_;
string name_;
// Structures mutable after construction
mutable mutex lock_;
RegionManager region_manager_ GUARDED_BY(lock_);
std::vector<Chunk> chunks_ GUARDED_BY(lock_);
// Pointer to head of linked list of free Chunks
ChunkHandle free_chunks_list_ GUARDED_BY(lock_);
// Counter containing the next unique identifier to assign to a
// newly-created chunk.
int64 next_allocation_id_ GUARDED_BY(lock_);
// Stats.
AllocatorStats stats_ GUARDED_BY(lock_);
friend class GPUBFCAllocatorPrivateMethodsTest;
TF_DISALLOW_COPY_AND_ASSIGN(BFCAllocator);
};
} // namespace tensorflow
#endif // TENSORFLOW_CORE_COMMON_RUNTIME_BFC_ALLOCATOR_H_
@@ -0,0 +1,103 @@
/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
#ifndef TENSORFLOW_CORE_COMMON_RUNTIME_BUF_RENDEZVOUS_H_
#define TENSORFLOW_CORE_COMMON_RUNTIME_BUF_RENDEZVOUS_H_
#include <functional>
#include <string>
#include "tensorflow/core/framework/allocator.h"
#include "tensorflow/core/lib/core/status.h"
#include "tensorflow/core/lib/gtl/flatmap.h"
#include "tensorflow/core/platform/mutex.h"
namespace tensorflow {
class Device;
class DeviceContext;
class Tensor;
// EXPERIMENTAL: RDMA oriented producer/consumer rendezvous on a local
// Tensor value for which DMAHelper::CanUseDMA() is true, i.e. dense
// numeric types. Similar to Rendezvous but never owns a Ref on the
// tensor, instead it uses an explicit callback to the producer when
// the consumer side is finished with the value. This allows the
// producer to perform in-place updates on the source buffer or to take
// other actions that depend on knowing the consumer has passed a certain
// execution point.
class BufRendezvous {
public:
explicit BufRendezvous(uint64 step_id) : step_id_(step_id) {}
~BufRendezvous();
// Inform all all waiting parties that this BufRendezvous is defunct
// because of an error Status interrupting the Step.
void StartAbort(const Status& s);
struct Hook;
// Provided by the consumer to be called when access to the buffer
// is available. If the Status arg is not OK, then hook will not
// be populated. Ownership of Hook passes to consumer with the
// callback.
typedef std::function<void(const Status&, Hook*)> ConsumerCallback;
// Provided by the producer to be called when the consumer has finished
// reading the buffer and will no longer access it.
typedef std::function<void(const Status&)> ProducerCallback;
struct Hook {
Device* prod_dev;
DeviceContext* prod_ctx;
const Tensor* prod_value;
AllocatorAttributes prod_attr;
ProducerCallback prod_cb;
ConsumerCallback cons_cb;
Hook()
: prod_dev(nullptr),
prod_ctx(nullptr),
prod_value(nullptr),
prod_cb(nullptr),
cons_cb(nullptr) {}
string DebugString() const;
};
// Called to advertise availability of a Tensor value corresponding
// to key. That value must stay valid until done is called.
void ProvideBuf(const string& key, Device* dev, DeviceContext* dev_ctx,
const Tensor* v, const AllocatorAttributes& attr,
const ProducerCallback& done);
// Called to request access to a Tensor value corresponding to key.
// Consumer is provide with a Hook as soon as available.
void ConsumeBuf(const string& key, const ConsumerCallback& done);
// Consumer must call this function when it's done reading the Hook provided
// by the ConsumerCallback. This function will invoke the producer callback
// and then delete h.
static void DoneWithHook(Hook* h);
// Write the current contents of the table to the INFO log.
void LogContents();
protected:
const uint64 step_id_;
mutex mu_;
Status status_ GUARDED_BY(mu_);
typedef gtl::FlatMap<string, Hook*> HookTable;
HookTable hook_table_ GUARDED_BY(mu_);
void PurgeTable(const Status& s, HookTable* table);
};
} // namespace tensorflow
#endif // TENSORFLOW_CORE_COMMON_RUNTIME_BUF_RENDEZVOUS_H_
@@ -0,0 +1,42 @@
/* Copyright 2016 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
#ifndef TENSORFLOW_CORE_COMMON_RUNTIME_BUILD_GRAPH_OPTIONS_H_
#define TENSORFLOW_CORE_COMMON_RUNTIME_BUILD_GRAPH_OPTIONS_H_
#include <vector>
#include "tensorflow/core/platform/types.h"
#include "tensorflow/core/protobuf/config.pb.h"
namespace tensorflow {
struct BuildGraphOptions {
CallableOptions callable_options;
// If `true`, uses Arg/Retval to implement feeds/fetches; otherwise
// uses Recv/Send to implement feeds/fetches.
// TODO(mrry): Remove this when the distributed runtime supports Arg/Retval.
bool use_function_convention = false;
static const int64 kNoCollectiveGraphKey = 0;
int64 collective_graph_key = kNoCollectiveGraphKey;
string DebugString() const;
};
} // namespace tensorflow
#endif // TENSORFLOW_CORE_COMMON_RUNTIME_BUILD_GRAPH_OPTIONS_H_
@@ -0,0 +1,75 @@
/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
#ifndef TENSORFLOW_CORE_COMMON_RUNTIME_COLLECTIVE_EXECUTOR_MGR_H_
#define TENSORFLOW_CORE_COMMON_RUNTIME_COLLECTIVE_EXECUTOR_MGR_H_
#include "tensorflow/core/framework/collective.h"
#include "tensorflow/core/lib/gtl/flatmap.h"
namespace tensorflow {
class ConfigProto;
class DeviceMgr;
class CollectiveExecutorMgr : public CollectiveExecutorMgrInterface {
public:
CollectiveExecutorMgr(const ConfigProto& config, const DeviceMgr* dev_mgr,
std::unique_ptr<DeviceResolverInterface> dev_resolver,
std::unique_ptr<ParamResolverInterface> param_resolver);
virtual ~CollectiveExecutorMgr();
CollectiveExecutor* FindOrCreate(int64 step_id) override;
void Cleanup(int64 step_id) override;
ParamResolverInterface* GetParamResolver() const override {
return param_resolver_.get();
}
DeviceResolverInterface* GetDeviceResolver() const override {
return dev_resolver_.get();
}
void GetStepSequenceAsync(const GetStepSequenceRequest* request,
GetStepSequenceResponse* response,
const StatusCallback& done) override;
void RefreshStepIdSequenceAsync(int64 graph_key,
const StatusCallback& done) override;
int64 NextStepId(int64 graph_key) override {
return CollectiveExecutor::kInvalidId;
}
void RetireStepId(int64 graph_key, int64 step_id) override {}
protected:
// Called by FindOrCreate when table entry does not yet exist.
virtual CollectiveExecutor* Create(int64 step_id);
const DeviceMgr* dev_mgr_;
std::unique_ptr<DeviceResolverInterface> dev_resolver_;
std::unique_ptr<ParamResolverInterface> param_resolver_;
CollectiveRemoteAccess* remote_access_;
string task_name_;
private:
mutex exec_mu_;
// Map from step_id to CollectiveExecutor
gtl::FlatMap<int64, CollectiveExecutor*> executor_table_ GUARDED_BY(exec_mu_);
};
} // namespace tensorflow
#endif // TENSORFLOW_CORE_COMMON_RUNTIME_COLLECTIVE_EXECUTOR_MGR_H_
@@ -0,0 +1,233 @@
/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
#ifndef TENSORFLOW_CORE_COMMON_RUNTIME_COLLECTIVE_PARAM_RESOLVER_LOCAL_H_
#define TENSORFLOW_CORE_COMMON_RUNTIME_COLLECTIVE_PARAM_RESOLVER_LOCAL_H_
#include <functional>
#include <memory>
#include <set>
#include <string>
#include <vector>
#include "tensorflow/core/framework/collective.h"
#include "tensorflow/core/lib/gtl/flatmap.h"
namespace tensorflow {
class CompleteGroupRequest;
class CompleteGroupResponse;
class CompleteInstanceRequest;
class CompleteInstanceResponse;
class DeviceMgr;
// Implements ParamResolverInterface for a single-task context.
// It also implements the functionality necessary to serve as the
// group leader for param resolution in a multi-task context.
class CollectiveParamResolverLocal : public ParamResolverInterface {
public:
CollectiveParamResolverLocal(const DeviceMgr* dev_mgr,
DeviceResolverInterface* dev_resolver,
const string& task_name);
~CollectiveParamResolverLocal() override {}
void CompleteParamsAsync(const string& device, CollectiveParams* cp,
CancellationManager* cancel_mgr,
const StatusCallback& done) override;
void CompleteGroupAsync(const CompleteGroupRequest* request,
CompleteGroupResponse* response,
CancellationManager* cancel_mgr,
const StatusCallback& done) override;
void CompleteInstanceAsync(const CompleteInstanceRequest* request,
CompleteInstanceResponse* response,
CancellationManager* cancel_mgr,
const StatusCallback& done) override;
protected:
// Used to complete/verify CollGroup.
struct GroupRec {
CollGroupParams group;
mutable mutex mu;
Status status GUARDED_BY(mu);
std::set<string> device_set GUARDED_BY(mu);
std::vector<string> device_list GUARDED_BY(mu);
std::set<string> task_set GUARDED_BY(mu);
std::vector<string> task_list GUARDED_BY(mu);
std::vector<StatusCallback> waiting GUARDED_BY(mu);
};
// Finds the GroupRec that corresponds to cp->group_key.
// Also populates cp->group from that group_rec.
// Will wait until GroupRec is fully populated or an error arises before
// calling done. Callback GroupRec* arg is only valid if status is ok.
// Ownership of GroupRec stays with this object and does not pass to the
// callback.
typedef std::function<void(const Status& s, const GroupRec* gr)>
GroupRecCallback;
void CompleteGroupLocal(const string& device, CollectiveParams* cp,
const GroupRecCallback& done)
LOCKS_EXCLUDED(group_mu_);
// Used to complete/verify CollInstance.
struct InstanceRec;
typedef std::function<void(InstanceRec*)> IRConsumer;
struct InstanceRec {
// This structure has two mutexes so that a possibly long
// initialization can be done without holding the instance_mu_
// table lock the whole time (which can cause an excessive number
// of threads to block on it), and because the compiler may not
// permit mutex locks to be taken in more than one order.
//
// out_mu guards access to most of the fields.
// in_mu guards access to a queue of consumer callbacks wanting to
// read the fields guarded by out_mu.
//
// The in_mu should be locked only while holding instance_mu_; the
// out_mu should be locked only while not holding
// instance_mu_.
//
// When is_init is false (the initial value) any potential user
// other than the creator should queue a callback on init_waiters.
// As soon as the shared member of this structure is fully
// initialized is_init will be set true and those callbacks will
// be invoked.
//
// Once inserted in the table this structure will never be replaced
// so users can capture the pointer while holding instance_mu_,
// drop that lock, then take a lock on out_mu before
// reading/modifying its values.
mutex in_mu;
bool is_init GUARDED_BY(in_mu);
std::vector<IRConsumer> init_waiters GUARDED_BY(in_mu);
// A thread that wishes to acquire out_mu must ensure that it is available
// by invoking WaitForOutMu().
mutex out_mu;
condition_variable out_cv;
bool out_mu_available GUARDED_BY(out_mu);
// Values to be shared by all instances, constant after initialization.
CollectiveParams shared GUARDED_BY(out_mu);
// If an error occurs during initialization this structure stays in
// the table with a non-OK status. Purging the table and restarting
// needs to be done at a higher level.
Status status GUARDED_BY(out_mu);
// These fields are used to count the instances that have called
// in and become known while resolving broadcast source identity.
int source_rank GUARDED_BY(out_mu);
int known_count GUARDED_BY(out_mu);
std::vector<bool> known GUARDED_BY(out_mu);
std::vector<IRConsumer> known_waiters GUARDED_BY(out_mu);
InstanceRec()
: is_init(false),
out_mu_available(true),
source_rank(-1),
known_count(0) {}
// If out_mu is unavailable during distributed device locality
// initialization, wait on out_cv until it is available again.
void WaitForOutMu(mutex_lock& lock) EXCLUSIVE_LOCKS_REQUIRED(out_mu);
};
// Find the InstanceRec with the same instance_key as cp. If it doesn't
// already exist, create and initialize from gr and cp.
//
// Precondition: *gr must be a complete GroupRec, i.e. the value set
// by CompleteGroupLocal. *cp must be populated with all the fields
// required by InitInstanceSharedParams. Ownership of InstanceRec stays
// with this object and does not pass to the callback.
typedef std::function<void(const Status& s, InstanceRec* ir)>
InstanceRecCallback;
void FindInstanceRec(const GroupRec* gr, CollectiveParams* cp,
const InstanceRecCallback& done)
LOCKS_EXCLUDED(instance_mu_, gr->mu, group_mu_);
// Populate *ir with device membership from gr, then initialize to be specific
// to cp->instance_key, i.e. order the devices and tasks.
//
// Preconditions:
// cp is populated with all DeviceLocalities
void InitInstanceSharedParams(const GroupRec* gr, const CollectiveParams* cp,
InstanceRec* ir, const StatusCallback& done)
UNLOCK_FUNCTION(ir->out_mu) LOCKS_EXCLUDED(gr->mu);
void CallInitInstanceSharedParams(const GroupRec* gr,
const CollectiveParams* cp, InstanceRec* ir,
const InstanceRecCallback& done)
LOCKS_EXCLUDED(ir->out_mu, gr->mu);
// Establishes the final order of ir->shared.instance.device_names and
// ir->shared.instance.task_names by considering localities of all devices.
void CompleteDefaultRanking(const GroupRec* gr, const CollectiveParams* cp,
InstanceRec* ir,
const std::vector<DeviceLocality>& localities)
EXCLUSIVE_LOCKS_REQUIRED(ir->out_mu);
// Finish populating *cp.
// Precondition: *gr has been fully populated by CompleteGroupLocal.
void CompleteInstanceLocal(const string& device, const GroupRec* gr,
CollectiveParams* cp, bool is_source,
const StatusCallback& done)
LOCKS_EXCLUDED(instance_mu_, gr->mu, group_mu_);
// Finish populating *cp from fully initialized *ir.
// Precondition: *gr and *ir are fully populated.
void CompleteInstanceFromInitializedIRec(const string& device,
const GroupRec* gr,
CollectiveParams* cp,
InstanceRec* ir, bool is_source,
const StatusCallback& done)
LOCKS_EXCLUDED(ir->out_mu);
// Complete source data for a broadcast instance.
// Precondition: *cp has complete group data and default_rank.
void CompleteInstanceSource(InstanceRec* ir, CollectiveParams* cp,
bool is_source, const IRConsumer& f)
LOCKS_EXCLUDED(ir->out_mu);
// If cp.device_names contains only devices local to this process
// populates *localities, else returns an error.
Status GetLocalDeviceLocalities(const CollectiveParams& cp,
std::vector<DeviceLocality>* localities);
// Sets CollTaskParams.is_local and CollectiveParams.default_rank.
// Precondition: cp->device_names is fully populated and in final order.
void CompleteTaskIsLocal(const string& task_name, CollectiveParams* cp);
// Sets cp->instance_default_rank according to location of device in
// current ordering of cp->instance.device_names.
void SetDefaultRank(const string& device, CollectiveParams* cp);
// Helper to grab status under lock, invoke callback out of lock.
void CallbackWithStatus(const InstanceRecCallback& done, InstanceRec* irec)
LOCKS_EXCLUDED(irec->out_mu);
const DeviceMgr* dev_mgr_;
DeviceResolverInterface* dev_resolver_; // Not owned.
string task_name_;
mutex group_mu_;
gtl::FlatMap<int32, std::unique_ptr<GroupRec>> group_table_
GUARDED_BY(group_mu_);
mutex instance_mu_;
gtl::FlatMap<int32, std::unique_ptr<InstanceRec>> instance_table_
GUARDED_BY(instance_mu_);
};
} // namespace tensorflow
#endif // TENSORFLOW_CORE_COMMON_RUNTIME_COLLECTIVE_PARAM_RESOLVER_LOCAL_H_
@@ -0,0 +1,92 @@
/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
#ifndef TENSORFLOW_CORE_COMMON_RUNTIME_COLLECTIVE_RMA_LOCAL_H_
#define TENSORFLOW_CORE_COMMON_RUNTIME_COLLECTIVE_RMA_LOCAL_H_
#include "tensorflow/core/common_runtime/buf_rendezvous.h"
#include "tensorflow/core/common_runtime/device_mgr.h"
#include "tensorflow/core/framework/collective.h"
#include "tensorflow/core/framework/rendezvous.h"
namespace tensorflow {
// Basic implementation of PerStepCollectiveRemoteAccess.
class CollectiveRemoteAccessLocal : public PerStepCollectiveRemoteAccess {
public:
CollectiveRemoteAccessLocal(const DeviceMgr* dev_mgr,
DeviceResolverInterface* dev_resolver,
int64 step_id)
: dev_mgr_(dev_mgr),
dev_resolver_(dev_resolver),
buf_rendezvous_(step_id),
step_id_(step_id) {}
virtual ~CollectiveRemoteAccessLocal() {}
void StartAbort(const Status& s) override;
void RecvFromPeer(const string& peer_device, const string& peer_task,
bool peer_is_local, const string& key, Device* to_device,
DeviceContext* to_device_ctx,
const AllocatorAttributes& to_alloc_attr, Tensor* to_tensor,
const DeviceLocality& client_locality,
int dev_to_dev_stream_index,
const StatusCallback& done) override;
void PostToPeer(const string& peer_device, const string& peer_task,
const string& key, Device* from_device,
DeviceContext* from_device_ctx,
const AllocatorAttributes& from_alloc_attr,
const Tensor* from_tensor,
const DeviceLocality& client_locality,
const StatusCallback& done) override;
void GetDeviceLocalitiesAsync(const CollInstanceParams& ci_params,
std::vector<DeviceLocality>* localities,
const StatusCallback& done) override {
dev_resolver_->GetDeviceLocalitiesAsync(ci_params, localities, done);
}
void GetLocalityAsync(const string& device, const string& task,
DeviceLocality* locality,
const StatusCallback& done) override {
dev_resolver_->GetLocalityAsync(device, task, locality, done);
}
void ClearTask(const string& task) override {
dev_resolver_->ClearTask(task);
}
BufRendezvous* buf_rendezvous() override { return &buf_rendezvous_; }
// Copy utility that always copies bytes from src to dst even if
// they are on the same device, unlike CopyTensor::ViaDMA which will
// just change the dst buffer pointer in that case.
static void MemCpyAsync(DeviceContext* src_dev_ctx,
DeviceContext* dst_dev_ctx, Device* src_dev,
Device* dst_dev, const AllocatorAttributes& src_attr,
const AllocatorAttributes& dst_attr,
const Tensor* src, Tensor* dst,
int dev_to_dev_stream_index,
const StatusCallback& done);
protected:
const DeviceMgr* dev_mgr_; // not owned
DeviceResolverInterface* dev_resolver_; // not owned
BufRendezvous buf_rendezvous_;
int64 step_id_;
};
} // namespace tensorflow
#endif // TENSORFLOW_CORE_COMMON_RUNTIME_COLLECTIVE_RMA_LOCAL_H_
@@ -0,0 +1,38 @@
/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
#ifndef TENSORFLOW_CORE_COMMON_RUNTIME_COLLECTIVE_UTIL_H_
#define TENSORFLOW_CORE_COMMON_RUNTIME_COLLECTIVE_UTIL_H_
#include <string>
#include "tensorflow/core/common_runtime/device.h"
#include "tensorflow/core/common_runtime/device_mgr.h"
#include "tensorflow/core/framework/collective.h"
#include "tensorflow/core/framework/device_attributes.pb.h"
#include "tensorflow/core/framework/tensor.h"
#include "tensorflow/core/lib/core/status.h"
namespace tensorflow {
namespace collective_util {
Status InitializeDeviceAndLocality(const DeviceMgr* dev_mgr,
const string& device_name, Device** device,
DeviceLocality* device_locality);
string SubdivPermDebugString(const CollectiveParams& col_params);
} // namespace collective_util
} // namespace tensorflow
#endif // TENSORFLOW_CORE_COMMON_RUNTIME_COLLECTIVE_UTIL_H_
@@ -0,0 +1,69 @@
/* Copyright 2017 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
#ifndef TENSORFLOW_CORE_COMMON_RUNTIME_CONSTANT_FOLDING_H_
#define TENSORFLOW_CORE_COMMON_RUNTIME_CONSTANT_FOLDING_H_
#include "tensorflow/core/common_runtime/device.h"
#include "tensorflow/core/framework/function.h"
#include "tensorflow/core/framework/tensor.h"
#include "tensorflow/core/graph/graph.h"
#include "tensorflow/core/platform/env.h"
// TODO(skyewm): can this be combined with EvaluateConstantTensor?
namespace tensorflow {
// This generator type is used to generate a name for the newly folded node
// based on the node's old name.
using ConstantFoldNameGenerator =
std::function<string(Graph* graph, string old_name)>;
// Options specific to constant folding optimizations.
struct ConstantFoldingOptions {
// If "consider" is not a nullptr, then only constant fold a node "n" if
// consider(n) returns true.
std::function<bool(const Node*)> consider = nullptr;
// If shape_map is not a nullptr, it is a map from node n to a
// vector of the (potentially partially-known) shapes of its
// outputs.
const std::unordered_map<string, std::vector<PartialTensorShape>>* shape_map =
nullptr; // not owned
// The maximum size of each constant created during constant folding
// optimization.
int64 max_constant_size_in_bytes = 10 * 1024 * 1024;
// A generator for the name suffix of constant folded nodes. A
// default id generator that monotonically increases is used if nullptr is
// passed.
ConstantFoldNameGenerator generate_new_name = nullptr;
};
// Perform constant folding optimization on "graph".
// Looks for nodes in "graph" that can be completely evaluated statically, i.e.,
// that are only dependent on constants. Evaluates those nodes on a CPU device
// and replaces those nodes with the result of the evaluation.
// "partition_device", if non-null, is the device where all the graph nodes are
// assumed to execute.
// Sets `was_mutated` to true if and only if "graph" has been mutated.
// The status is only set to a non-OK state if an unexpected error is hit
// running the graph.
Status ConstantFold(const ConstantFoldingOptions& opts,
FunctionLibraryRuntime* function_library, Env* env,
Device* partition_device, Graph* graph, bool* was_mutated);
} // namespace tensorflow
#endif // TENSORFLOW_CORE_COMMON_RUNTIME_CONSTANT_FOLDING_H_
@@ -0,0 +1,73 @@
/* Copyright 2015 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
#ifndef TENSORFLOW_CORE_COMMON_RUNTIME_COPY_TENSOR_H_
#define TENSORFLOW_CORE_COMMON_RUNTIME_COPY_TENSOR_H_
#include "tensorflow/core/common_runtime/device.h"
#include "tensorflow/core/framework/allocator.h"
#include "tensorflow/core/framework/device_base.h"
#include "tensorflow/core/framework/tensor.h"
#include "tensorflow/core/framework/types.h"
#include "tensorflow/core/lib/core/status.h"
#include "tensorflow/core/platform/types.h"
namespace tensorflow {
class CopyTensor {
public:
typedef void (*CopyFunction)(
DeviceContext* send_dev_context, DeviceContext* recv_dev_context,
Device* src, Device* dst, const AllocatorAttributes src_alloc_attr,
const AllocatorAttributes dst_alloc_attr, const Tensor* input,
Tensor* output, int dev_to_dev_stream_index, StatusCallback done);
// Copies "input" to "output" between devices accessible to the
// local process via some DMA-like method. "edge_name" is the name
// of the tensor being copied, for debugging purposes. Depending on
// the type of devices and memory in use, the copy may be performed
// synchronously or asynchronously. 'done' will be invoked only
// after the copy is actually complete.
static void ViaDMA(StringPiece edge_name, DeviceContext* send_dev_context,
DeviceContext* recv_dev_context, Device* src, Device* dst,
const AllocatorAttributes src_alloc_attr,
const AllocatorAttributes dst_alloc_attr,
const Tensor* input, Tensor* output,
int dev_to_dev_stream_index, StatusCallback done);
// Object used to call Register() at static-initialization time.
// Note: This should only ever be used as a global-static object; no stack
// or heap instances.
class Registration {
public:
Registration(DeviceType sender_device_type, DeviceType receiver_device_type,
CopyFunction copy_function) {
TF_QCHECK_OK(
Register(sender_device_type, receiver_device_type, copy_function));
}
};
private:
// Register a function for copying between two specific DeviceTypes.
// Note: This should only be called via the constructor of
// CopyTensor::Registration.
static Status Register(DeviceType sender_device_type,
DeviceType receiver_device_type,
CopyFunction copy_function);
};
} // namespace tensorflow
#endif // TENSORFLOW_CORE_COMMON_RUNTIME_COPY_TENSOR_H_
@@ -0,0 +1,55 @@
/* Copyright 2015 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
#ifndef TENSORFLOW_CORE_COMMON_RUNTIME_COSTMODEL_MANAGER_H_
#define TENSORFLOW_CORE_COMMON_RUNTIME_COSTMODEL_MANAGER_H_
#include <unordered_map>
#include "tensorflow/core/framework/cost_graph.pb.h"
#include "tensorflow/core/graph/costmodel.h"
#include "tensorflow/core/graph/graph.h"
#include "tensorflow/core/lib/core/errors.h"
#include "tensorflow/core/lib/gtl/iterator_range.h"
namespace tensorflow {
// Used to manage all the cost models for a session.
class CostModelManager {
public:
~CostModelManager();
typedef std::unordered_map<const Graph*, CostModel*> CostModelMap;
typedef CostModelMap::iterator CostModelMapIter;
void ExportCostModels(CostModelMap* cost_models) {
mutex_lock l(mu_);
*cost_models = cost_models_;
}
CostModel* FindOrCreateCostModel(const Graph* graph);
bool RemoveCostModelForGraph(const Graph* graph);
Status AddToCostGraphDef(const Graph* graph, CostGraphDef* cost_graph);
private:
mutex mu_;
CostModelMap cost_models_ GUARDED_BY(mu_);
};
} // namespace tensorflow
#endif // TENSORFLOW_CORE_COMMON_RUNTIME_COSTMODEL_MANAGER_H_
@@ -0,0 +1,120 @@
/* Copyright 2016 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
#ifndef TENSORFLOW_CORE_COMMON_RUNTIME_DEBUGGER_STATE_INTERFACE_H_
#define TENSORFLOW_CORE_COMMON_RUNTIME_DEBUGGER_STATE_INTERFACE_H_
#include <memory>
#include "tensorflow/core/common_runtime/device.h"
#include "tensorflow/core/graph/graph.h"
#include "tensorflow/core/lib/core/status.h"
#include "tensorflow/core/platform/macros.h"
#include "tensorflow/core/platform/protobuf.h"
#include "tensorflow/core/protobuf/debug.pb.h"
namespace tensorflow {
// Returns a summary string for the list of debug tensor watches.
const string SummarizeDebugTensorWatches(
const protobuf::RepeatedPtrField<DebugTensorWatch>& watches);
// An abstract interface for storing and retrieving debugging information.
class DebuggerStateInterface {
public:
virtual ~DebuggerStateInterface() {}
// Publish metadata about the debugged Session::Run() call.
//
// Args:
// global_step: A global step count supplied by the caller of
// Session::Run().
// session_run_index: A chronologically sorted index for calls to the Run()
// method of the Session object.
// executor_step_index: A chronologically sorted index of invocations of the
// executor charged to serve this Session::Run() call.
// input_names: Name of the input Tensors (feed keys).
// output_names: Names of the fetched Tensors.
// target_names: Names of the target nodes.
virtual Status PublishDebugMetadata(
const int64 global_step, const int64 session_run_index,
const int64 executor_step_index, const std::vector<string>& input_names,
const std::vector<string>& output_names,
const std::vector<string>& target_nodes) = 0;
};
class DebugGraphDecoratorInterface {
public:
virtual ~DebugGraphDecoratorInterface() {}
// Insert special-purpose debug nodes to graph and dump the graph for
// record. See the documentation of DebugNodeInserter::InsertNodes() for
// details.
virtual Status DecorateGraph(Graph* graph, Device* device) = 0;
// Publish Graph to debug URLs.
virtual Status PublishGraph(const Graph& graph,
const string& device_name) = 0;
};
typedef std::function<std::unique_ptr<DebuggerStateInterface>(
const DebugOptions& options)>
DebuggerStateFactory;
// Contains only static methods for registering DebuggerStateFactory.
// We don't expect to create any instances of this class.
// Call DebuggerStateRegistry::RegisterFactory() at initialization time to
// define a global factory that creates instances of DebuggerState, then call
// DebuggerStateRegistry::CreateState() to create a single instance.
class DebuggerStateRegistry {
public:
// Registers a function that creates a concrete DebuggerStateInterface
// implementation based on DebugOptions.
static void RegisterFactory(const DebuggerStateFactory& factory);
// If RegisterFactory() has been called, creates and supplies a concrete
// DebuggerStateInterface implementation using the registered factory,
// owned by the caller and return an OK Status. Otherwise returns an error
// Status.
static Status CreateState(const DebugOptions& debug_options,
std::unique_ptr<DebuggerStateInterface>* state);
private:
static DebuggerStateFactory* factory_;
TF_DISALLOW_COPY_AND_ASSIGN(DebuggerStateRegistry);
};
typedef std::function<std::unique_ptr<DebugGraphDecoratorInterface>(
const DebugOptions& options)>
DebugGraphDecoratorFactory;
class DebugGraphDecoratorRegistry {
public:
static void RegisterFactory(const DebugGraphDecoratorFactory& factory);
static Status CreateDecorator(
const DebugOptions& options,
std::unique_ptr<DebugGraphDecoratorInterface>* decorator);
private:
static DebugGraphDecoratorFactory* factory_;
TF_DISALLOW_COPY_AND_ASSIGN(DebugGraphDecoratorRegistry);
};
} // end namespace tensorflow
#endif // TENSORFLOW_CORE_COMMON_RUNTIME_DEBUGGER_STATE_INTERFACE_H_
@@ -0,0 +1,196 @@
/* Copyright 2015 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
// A Device is a something that can perform computations as part of a
// model. Devices can be local (runs computation on this machine), or
// remote (contacts a device local to another machine using an RPC to
// do the work). Devices are registered in a DeviceSet, which is also
// responsible for the Device <-> id mapping.
//
// Device names
// * Every Device should have a unique name with the format:
// /job:___/replica:___/task:___/(gpu|cpu):___
// An example name would be "/job:train/replica:0/task:3/device:GPU:2".
// * Task numbers are within the specified replica, so there are as
// many "task zeros" as replicas.
#ifndef TENSORFLOW_CORE_COMMON_RUNTIME_DEVICE_H_
#define TENSORFLOW_CORE_COMMON_RUNTIME_DEVICE_H_
#include <memory>
#include <string>
#include "tensorflow/core/framework/allocator.h"
#include "tensorflow/core/framework/control_flow.h"
#include "tensorflow/core/framework/device_attributes.pb_text.h"
#include "tensorflow/core/framework/device_attributes.pb.h"
#include "tensorflow/core/framework/device_base.h"
#include "tensorflow/core/framework/graph.pb.h"
#include "tensorflow/core/framework/op_kernel.h"
#include "tensorflow/core/framework/op_segment.h"
#include "tensorflow/core/framework/resource_mgr.h"
#include "tensorflow/core/framework/types.h"
#include "tensorflow/core/graph/graph.h"
#include "tensorflow/core/graph/types.h"
#include "tensorflow/core/lib/core/status.h"
#include "tensorflow/core/platform/macros.h"
#include "tensorflow/core/platform/types.h"
#include "tensorflow/core/util/device_name_utils.h"
namespace tensorflow {
class DeviceMgr;
class Device : public DeviceBase {
public:
Device(Env* env, const DeviceAttributes& device_attributes);
~Device() override;
// Full name of this device (see top comment).
const string& name() const override { return device_attributes_.name(); }
// Parsed name of this device
const DeviceNameUtils::ParsedName& parsed_name() const {
return parsed_name_;
}
// Describes what kind of device this is. This is intended to be
// human-readable and not computer-parsed, except that two devices
// with the same device_type() are expected to perform similarly
// (both from a computation and communication perspective).
const string& device_type() const { return device_attributes_.device_type(); }
// Returns an aggregation of device attributes.
const DeviceAttributes& attributes() const override {
return device_attributes_;
}
// Performs the actual compute function.
//
// Subclasses may override this function if they wish to perform
// some initialization before each compute.
virtual void Compute(OpKernel* op_kernel, OpKernelContext* context) {
op_kernel->Compute(context);
}
// Asynchronous kernel's compute.
virtual void ComputeAsync(AsyncOpKernel* op_kernel, OpKernelContext* context,
AsyncOpKernel::DoneCallback done) {
op_kernel->ComputeAsync(context, std::move(done));
}
// Takes ownership of the references in tensors. If necessary, a
// device may override this method to keep a reference to the
// accessed tensors until the async computation has completed.
virtual void ConsumeListOfAccessedTensors(
DeviceContext* context, const TensorReferenceVector& tensors) {
for (const auto& ref : tensors) {
ref.Unref();
}
}
// If true, and tracing is enabled, the `tracing::ScopedAnnotation()` tracing
// mechanism will be used instead of `tracing::ScopedActivity()`. Some devices
// may override this method to use annotations, which enable child activities
// (such as GPU kernel launches) to be related to the OpKernel invocation.
virtual bool TraceUsingAnnotations() const { return false; }
// Blocks until all operations queued on the device at the time of
// the call have completed. Returns any error pending on the device
// at completion.
virtual Status Sync() = 0;
// Override this to return true for devices that require a Sync() call before
// session completion.
virtual bool RequiresSyncOnCompletion() const { return false; }
// Optionally modify the device's GraphDef before execution.
//
// This method should be considered experimental and is supplied to enable
// prototyping of TensorFlow device implementations that need to modify
// the GraphDef before execution.
//
// 'graph' supplies the partition of the graph assigned to this
// device.
virtual Status MaybeRewriteGraph(std::unique_ptr<Graph>* /*graph*/) {
return Status::OK();
}
// Fill in the context map for the graph. Default behavior is to do
// nothing.
//
// The caller takes ownership over the DeviceContext objects given
// by the device.
virtual Status FillContextMap(const Graph* graph,
DeviceContextMap* device_context_map) {
return Status::OK();
}
// Returns the op segment of this device. The caller can reuse op
// kernels registered for the same session running on this device.
OpSegment* op_segment() { return &op_seg_; }
// Returns the resource manager associated w/ this device.
virtual ResourceMgr* resource_manager() { return rmgr_; }
// Returns the device manager that owns this device, or nullptr if this Device
// is not owned by a device manager.
DeviceMgr* device_mgr() const { return device_mgr_; }
// Summarizes the status of this Device, for debugging.
string DebugString() const { return ProtoDebugString(device_attributes_); }
// Assembles the parameter components into a complete DeviceAttributes value.
static DeviceAttributes BuildDeviceAttributes(
const string& name, DeviceType device, Bytes memory_limit,
const DeviceLocality& locality, const string& physical_device_desc);
static DeviceAttributes BuildDeviceAttributes(
const string& name, DeviceType device, Bytes memory_limit,
const DeviceLocality& locality) {
// Pass in an empty string as physical device name.
return BuildDeviceAttributes(name, device, memory_limit, locality, "");
}
// Clears the resource manager associated with this device.
void ClearResourceMgr() { rmgr_->Clear(); }
protected:
void DeleteResourceMgr() {
delete rmgr_;
rmgr_ = nullptr;
}
private:
friend class DeviceMgr;
// Pointer to the device manager that owns this device. Not owned.
DeviceMgr* device_mgr_ = nullptr;
const DeviceAttributes device_attributes_;
DeviceNameUtils::ParsedName parsed_name_;
// op_seg_ maps session handle and op name to OpKernel objects.
OpSegment op_seg_;
// Resources associated w/ this device. E.g., shared variables, etc.
ResourceMgr* rmgr_ = nullptr;
TF_DISALLOW_COPY_AND_ASSIGN(Device);
};
} // namespace tensorflow
#endif // TENSORFLOW_CORE_COMMON_RUNTIME_DEVICE_H_
@@ -0,0 +1,129 @@
/* Copyright 2015 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
#ifndef TENSORFLOW_CORE_COMMON_RUNTIME_DEVICE_FACTORY_H_
#define TENSORFLOW_CORE_COMMON_RUNTIME_DEVICE_FACTORY_H_
#include <string>
#include <vector>
#include "tensorflow/core/lib/core/status.h"
#include "tensorflow/core/platform/types.h"
namespace tensorflow {
class Device;
struct SessionOptions;
class DeviceFactory {
public:
virtual ~DeviceFactory() {}
static void Register(const string& device_type, DeviceFactory* factory,
int priority);
static DeviceFactory* GetFactory(const string& device_type);
// Append to "*devices" all suitable devices, respecting
// any device type specific properties/counts listed in "options".
//
// CPU devices are added first.
static Status AddDevices(const SessionOptions& options,
const string& name_prefix,
std::vector<Device*>* devices);
// Helper for tests. Create a single device of type "type". The
// returned device is always numbered zero, so if creating multiple
// devices of the same type, supply distinct name_prefix arguments.
static Device* NewDevice(const string& type, const SessionOptions& options,
const string& name_prefix);
// Most clients should call AddDevices() instead.
virtual Status CreateDevices(const SessionOptions& options,
const string& name_prefix,
std::vector<Device*>* devices) = 0;
// Return the device priority number for a "device_type" string.
//
// Higher number implies higher priority.
//
// In standard TensorFlow distributions, GPU device types are
// preferred over CPU, and by default, custom devices that don't set
// a custom priority during registration will be prioritized lower
// than CPU. Custom devices that want a higher priority can set the
// 'priority' field when registering their device to something
// higher than the packaged devices. See calls to
// REGISTER_LOCAL_DEVICE_FACTORY to see the existing priorities used
// for built-in devices.
static int32 DevicePriority(const string& device_type);
};
namespace dfactory {
template <class Factory>
class Registrar {
public:
// Multiple registrations for the same device type with different priorities
// are allowed. Priorities are used in two different ways:
//
// 1) When choosing which factory (that is, which device
// implementation) to use for a specific 'device_type', the
// factory registered with the highest priority will be chosen.
// For example, if there are two registrations:
//
// Registrar<CPUFactory1>("CPU", 125);
// Registrar<CPUFactory2>("CPU", 150);
//
// then CPUFactory2 will be chosen when
// DeviceFactory::GetFactory("CPU") is called.
//
// 2) When choosing which 'device_type' is preferred over other
// DeviceTypes in a DeviceSet, the ordering is determined
// by the 'priority' set during registration. For example, if there
// are two registrations:
//
// Registrar<CPUFactory>("CPU", 100);
// Registrar<GPUFactory>("GPU", 200);
//
// then DeviceType("GPU") will be prioritized higher than
// DeviceType("CPU").
//
// The default priority values for built-in devices is:
// GPU: 210
// SYCL: 200
// GPUCompatibleCPU: 70
// ThreadPoolDevice: 60
// Default: 50
explicit Registrar(const string& device_type, int priority = 50) {
DeviceFactory::Register(device_type, new Factory(), priority);
}
};
} // namespace dfactory
#define REGISTER_LOCAL_DEVICE_FACTORY(device_type, device_factory, ...) \
INTERNAL_REGISTER_LOCAL_DEVICE_FACTORY(device_type, device_factory, \
__COUNTER__, ##__VA_ARGS__)
#define INTERNAL_REGISTER_LOCAL_DEVICE_FACTORY(device_type, device_factory, \
ctr, ...) \
static ::tensorflow::dfactory::Registrar<device_factory> \
INTERNAL_REGISTER_LOCAL_DEVICE_FACTORY_NAME(ctr)(device_type, \
##__VA_ARGS__)
// __COUNTER__ must go through another macro to be properly expanded
#define INTERNAL_REGISTER_LOCAL_DEVICE_FACTORY_NAME(ctr) ___##ctr##__object_
} // namespace tensorflow
#endif // TENSORFLOW_CORE_COMMON_RUNTIME_DEVICE_FACTORY_H_
@@ -0,0 +1,80 @@
/* Copyright 2015 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
#ifndef TENSORFLOW_CORE_COMMON_RUNTIME_DEVICE_MGR_H_
#define TENSORFLOW_CORE_COMMON_RUNTIME_DEVICE_MGR_H_
#include <string>
#include <unordered_map>
#include <unordered_set>
#include <vector>
#include "tensorflow/core/common_runtime/device.h"
#include "tensorflow/core/lib/core/arena.h"
#include "tensorflow/core/lib/core/status.h"
#include "tensorflow/core/lib/core/stringpiece.h"
#include "tensorflow/core/lib/gtl/inlined_vector.h"
#include "tensorflow/core/platform/macros.h"
namespace tensorflow {
class DeviceAttributes;
class DeviceMgr {
public:
// Takes ownership of each device in 'devices'.
// TODO(zhifengc): Other initialization information.
// TODO(b/37437134): Use std::unique_ptr's to track ownership.
explicit DeviceMgr(const std::vector<Device*>& devices);
~DeviceMgr();
// Returns attributes of all devices.
void ListDeviceAttributes(std::vector<DeviceAttributes>* devices) const;
std::vector<Device*> ListDevices() const;
// Returns a string listing all devices.
string DebugString() const;
// Returns a string of all the device mapping.
string DeviceMappingString() const;
// Assigns *device with pointer to Device of the given name.
// Accepts either a full device name, or just the replica-local suffix.
Status LookupDevice(StringPiece name, Device** device) const;
// Clears given containers of all devices if 'container' is
// non-empty. Otherwise, clears default containers of all devices.
void ClearContainers(gtl::ArraySlice<string> containers) const;
int NumDeviceType(const string& type) const;
private:
// TODO(b/37437134): Use std::unique_ptr's to track ownership.
typedef gtl::InlinedVector<Device*, 8> DeviceVec;
DeviceVec devices_;
StringPiece CopyToBackingStore(StringPiece s);
std::unordered_map<StringPiece, Device*, StringPieceHasher> device_map_;
core::Arena name_backing_store_; // Storage for keys in device_map_
std::unordered_map<string, int> device_type_counts_;
TF_DISALLOW_COPY_AND_ASSIGN(DeviceMgr);
};
} // namespace tensorflow
#endif // TENSORFLOW_CORE_COMMON_RUNTIME_DEVICE_MGR_H_
@@ -0,0 +1,48 @@
/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
#ifndef TENSORFLOW_CORE_COMMON_RUNTIME_DEVICE_RESOLVER_LOCAL_H_
#define TENSORFLOW_CORE_COMMON_RUNTIME_DEVICE_RESOLVER_LOCAL_H_
#include <string>
#include "tensorflow/core/framework/collective.h"
#include "tensorflow/core/framework/device_attributes.pb.h"
namespace tensorflow {
class DeviceMgr;
// Implements DeviceResolverInterface in a single-task context.
class DeviceResolverLocal : public DeviceResolverInterface {
public:
DeviceResolverLocal(const DeviceMgr* dev_mgr) : dev_mgr_(dev_mgr) {}
virtual ~DeviceResolverLocal() {}
void GetDeviceLocalitiesAsync(const CollInstanceParams& ci_params,
std::vector<DeviceLocality>* localities,
const StatusCallback& done) override;
void GetLocalityAsync(const string& device, const string& task,
DeviceLocality* locality,
const StatusCallback& done) override;
void ClearTask(const string& task) override {}
protected:
const DeviceMgr* dev_mgr_;
};
} // namespace tensorflow
#endif // TENSORFLOW_CORE_COMMON_RUNTIME_DEVICE_RESOLVER_LOCAL_H_
@@ -0,0 +1,89 @@
/* Copyright 2015 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
#ifndef TENSORFLOW_CORE_COMMON_RUNTIME_DEVICE_SET_H_
#define TENSORFLOW_CORE_COMMON_RUNTIME_DEVICE_SET_H_
#include <memory>
#include <unordered_map>
#include <vector>
#include "tensorflow/core/common_runtime/device.h"
#include "tensorflow/core/platform/macros.h"
#include "tensorflow/core/platform/types.h"
#include "tensorflow/core/util/device_name_utils.h"
namespace tensorflow {
// DeviceSet is a container class for managing the various types of
// devices used by a model.
class DeviceSet {
public:
DeviceSet();
~DeviceSet();
// Does not take ownership of 'device'.
void AddDevice(Device* device);
// Set the device designated as the "client". This device
// must also be registered via AddDevice().
void set_client_device(Device* device) {
DCHECK(client_device_ == nullptr);
client_device_ = device;
}
// Returns a pointer to the device designated as the "client".
Device* client_device() const { return client_device_; }
// Return the list of devices in this set.
const std::vector<Device*>& devices() const { return devices_; }
// Given a DeviceNameUtils::ParsedName (which may have some
// wildcards for different components), fills "*devices" with all
// devices in "*this" that match "spec".
void FindMatchingDevices(const DeviceNameUtils::ParsedName& spec,
std::vector<Device*>* devices) const;
// Finds the device with the given "fullname". Returns nullptr if
// not found.
Device* FindDeviceByName(const string& fullname) const;
// Return the list of unique device types in this set, ordered
// with more preferable devices earlier.
std::vector<DeviceType> PrioritizedDeviceTypeList() const;
// An order to sort by device types according to system-determined
// priority.
//
// Higher result implies higher priority.
static int DeviceTypeOrder(const DeviceType& d);
private:
// Not owned.
std::vector<Device*> devices_;
// Fullname -> device* for device in devices_.
std::unordered_map<string, Device*> device_by_name_;
// client_device_ points to an element of devices_ that we consider
// to be the client device (in this local process).
Device* client_device_ = nullptr;
TF_DISALLOW_COPY_AND_ASSIGN(DeviceSet);
};
} // namespace tensorflow
#endif // TENSORFLOW_CORE_COMMON_RUNTIME_DEVICE_SET_H_
@@ -0,0 +1,411 @@
/* Copyright 2015 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
#ifndef TENSORFLOW_CORE_COMMON_RUNTIME_DIRECT_SESSION_H_
#define TENSORFLOW_CORE_COMMON_RUNTIME_DIRECT_SESSION_H_
#include <atomic>
#include <memory>
#include <string>
#include <unordered_map>
#include <unordered_set>
#include <vector>
#include "tensorflow/core/common_runtime/costmodel_manager.h"
#include "tensorflow/core/common_runtime/debugger_state_interface.h"
#include "tensorflow/core/common_runtime/device_mgr.h"
#include "tensorflow/core/common_runtime/device_set.h"
#include "tensorflow/core/common_runtime/executor.h"
#include "tensorflow/core/common_runtime/graph_execution_state.h"
#include "tensorflow/core/common_runtime/process_function_library_runtime.h"
#include "tensorflow/core/common_runtime/rendezvous_mgr.h"
#include "tensorflow/core/common_runtime/session_factory.h"
#include "tensorflow/core/framework/cancellation.h"
#include "tensorflow/core/framework/collective.h"
#include "tensorflow/core/framework/graph.pb.h"
#include "tensorflow/core/framework/session_state.h"
#include "tensorflow/core/framework/tensor.h"
#include "tensorflow/core/lib/core/errors.h"
#include "tensorflow/core/lib/core/status.h"
#include "tensorflow/core/platform/macros.h"
#include "tensorflow/core/platform/mutex.h"
#include "tensorflow/core/platform/thread_annotations.h"
#include "tensorflow/core/platform/types.h"
#include "tensorflow/core/public/session.h"
namespace tensorflow {
class CostModel;
class DebugGateway;
class Device;
class DirectSessionFactory;
class DirectSession : public Session {
public:
typedef std::function<void(Session*)> CloseCallback;
// Takes ownership of 'device_mgr'.
// 'factory' is used to unregister the DirectSession with 'factory' when its
// closed. This ensures that Reset requests from the 'factory' don't get sent
// to sessions that are already closed.
DirectSession(const SessionOptions& options, const DeviceMgr* device_mgr,
DirectSessionFactory* factory);
~DirectSession() override;
typedef std::vector<std::pair<string, Tensor>> NamedTensorList;
typedef std::unordered_map<StringPiece, Node*, StringPieceHasher> NameNodeMap;
::tensorflow::Status Create(const GraphDef& graph) override;
::tensorflow::Status Extend(const GraphDef& graph) override;
::tensorflow::Status Run(const NamedTensorList& inputs,
const std::vector<string>& output_names,
const std::vector<string>& target_nodes,
std::vector<Tensor>* outputs) override;
// NOTE: Experimental and subject to change.
::tensorflow::Status Run(const ::tensorflow::RunOptions& run_options,
const NamedTensorList& inputs,
const std::vector<string>& output_names,
const std::vector<string>& target_nodes,
std::vector<Tensor>* outputs,
RunMetadata* run_metadata) override;
// NOTE: PRunSetup and PRun are added to support partial execution. This
// feature is experimental and subject to change.
::tensorflow::Status PRunSetup(const std::vector<string>& input_names,
const std::vector<string>& output_names,
const std::vector<string>& target_nodes,
string* handle) override;
::tensorflow::Status PRun(const string& handle, const NamedTensorList& inputs,
const std::vector<string>& output_names,
std::vector<Tensor>* outputs) override;
// Reset clears 'containers' from the device_mgr of the DirectSession.
// If 'containers' is empty, then Reset clears the default container.
::tensorflow::Status Reset(const std::vector<string>& containers);
::tensorflow::Status ListDevices(
std::vector<DeviceAttributes>* response) override;
::tensorflow::Status Close() override;
::tensorflow::Status LocalDeviceManager(const DeviceMgr** output) override {
*output = device_mgr_.get();
return ::tensorflow::Status::OK();
}
void ExportCostModels(CostModelManager::CostModelMap* cost_models) {
cost_model_manager_.ExportCostModels(cost_models);
}
::tensorflow::Status MakeCallable(const CallableOptions& callable_options,
CallableHandle* out_handle) override;
::tensorflow::Status RunCallable(CallableHandle handle,
const std::vector<Tensor>& feed_tensors,
std::vector<Tensor>* fetch_tensors,
RunMetadata* run_metadata) override;
::tensorflow::Status ReleaseCallable(CallableHandle handle) override;
private:
// For access to collective_graph_key_.
friend class DirectSessionCollectiveTest;
// We create one executor and its dependent library runtime for
// every partition.
struct PerPartitionExecutorsAndLib {
Graph* graph = nullptr; // not owned.
Device* device = nullptr; // not owned.
FunctionLibraryRuntime* flib = nullptr; // not owned.
std::unique_ptr<Executor> executor;
};
// An ExecutorsAndKeys is created for a given set of feeds/fetches.
// 'step_count' is the number of times this graph is executed.
// 'graph' is the entire graph being executed. 'name_to_node'
// maps node name to node. We keep 'graph' and 'name_to_node' only in
// the case of partial runs. Each item in 'items' is the executor for
// a partition of the graph bundled with its dependent library runtime.
// 'input_keys' are the rendezvous keys for the feeds and 'output_keys'
// are rendezvous keys for the fetches.
struct ExecutorsAndKeys {
ExecutorsAndKeys() : step_count(0) {}
std::atomic_int_fast64_t step_count;
std::unique_ptr<Graph> graph;
NameNodeMap name_to_node;
std::vector<PerPartitionExecutorsAndLib> items;
std::unordered_map<string, size_t> input_name_to_index;
std::unordered_map<string, string> input_name_to_rendezvous_key;
std::unordered_map<string, size_t> output_name_to_index;
std::unordered_map<string, string> output_name_to_rendezvous_key;
DataTypeVector input_types;
DataTypeVector output_types;
CallableOptions callable_options;
int64 collective_graph_key = BuildGraphOptions::kNoCollectiveGraphKey;
};
// A FunctionInfo object is created for every unique set of feeds/fetches.
// This info could be folded into the ExecutorsAndKeys object but we would
// like to maintain a deletion order in which the OpKernels (owned by the
// executor) should be destroyed first, followed by the resources in the
// device and then followed by the function stuff.
// TODO(rohanj): Consolidate function library definitions so that we can
// instantiate only one ProcFLR and lib_def and make this just a member
// variable and not a vector.
// 'flib_def' is the function library used.
// 'proc_flr' is the collection of FunctionLibraryRuntime objects, one per
// device.
struct FunctionInfo {
std::unique_ptr<FunctionLibraryDefinition> flib_def;
std::unique_ptr<ProcessFunctionLibraryRuntime> proc_flr;
};
// For each live partial execution, the session maintains a RunState.
// 'status' is the current status of this partial execution. 'executor_done'
// is "notified" when all executors are done. 'pending_inputs' are the set
// of pending feeds and 'pending_outputs' are the set of pending fetches.
struct RunState {
mutex mu_;
Status status GUARDED_BY(mu_);
IntraProcessRendezvous* rendez = nullptr;
std::unique_ptr<CollectiveExecutor::Handle> collective_executor;
std::unique_ptr<StepStatsCollector> collector;
Notification executors_done;
std::unordered_map<string, bool> pending_inputs; // true if fed
std::unordered_map<string, bool> pending_outputs; // true if fetched
TensorStore tensor_store;
ScopedStepContainer step_container;
RunState(int64 step_id, const std::vector<Device*>* devices);
RunState(const std::vector<string>& pending_input_names,
const std::vector<string>& pending_output_names, int64 step_id,
const std::vector<Device*>* devices);
// Returns true if all pending inputs and outputs have been completed.
bool PendingDone() const;
~RunState();
};
struct RunStateArgs {
RunStateArgs(const DebugOptions& options) : debug_options(options) {}
bool is_partial_run = false;
string handle;
std::unique_ptr<Graph> graph;
const DebugOptions& debug_options;
int64 collective_graph_key = BuildGraphOptions::kNoCollectiveGraphKey;
};
// Initializes the base execution state given the 'graph',
// if not already initialized.
Status MaybeInitializeExecutionState(const GraphDef& graph,
bool* out_already_initialized)
EXCLUSIVE_LOCKS_REQUIRED(graph_state_lock_);
// Retrieves an already existing set of executors to run 'inputs' and
// 'outputs', or creates and caches them for future use.
::tensorflow::Status GetOrCreateExecutors(
gtl::ArraySlice<string> inputs, gtl::ArraySlice<string> outputs,
gtl::ArraySlice<string> target_nodes,
ExecutorsAndKeys** executors_and_keys, RunStateArgs* run_state_args);
// Creates a set of executors to run the subgraph defined by
// `callable_options`.
::tensorflow::Status CreateExecutors(
const CallableOptions& callable_options,
std::unique_ptr<ExecutorsAndKeys>* out_executors_and_keys,
std::unique_ptr<FunctionInfo>* out_func_info,
RunStateArgs* run_state_args);
// Creates several graphs given the existing graph_def_ and the
// input feeds and fetches, given 'devices'. The graphs share a common
// function library 'flib_def'.
::tensorflow::Status CreateGraphs(
const BuildGraphOptions& options,
std::unordered_map<string, std::unique_ptr<Graph>>* outputs,
std::unique_ptr<FunctionLibraryDefinition>* flib_def,
RunStateArgs* run_state_args, DataTypeVector* input_types,
DataTypeVector* output_types, int64* collective_graph_key);
::tensorflow::Status RunInternal(int64 step_id, const RunOptions& run_options,
CallFrameInterface* call_frame,
ExecutorsAndKeys* executors_and_keys,
RunMetadata* run_metadata);
// Returns whether inter-op execution uses a global pool.
bool ShouldUseRunHandlerPool() const;
::tensorflow::Status ExtendLocked(const GraphDef& graph)
EXCLUSIVE_LOCKS_REQUIRED(graph_state_lock_);
::tensorflow::Status ResourceHandleToInputTensor(
const Tensor& resource_tensor, Tensor* retrieved_tensor);
// Feeds more inputs to the executors, triggering further execution.
::tensorflow::Status SendPRunInputs(
const std::vector<std::pair<string, Tensor>>& inputs,
const ExecutorsAndKeys* executors_and_keys,
IntraProcessRendezvous* rendez);
// Fetches more outputs from the executors. It waits until the output
// tensors are computed.
::tensorflow::Status RecvPRunOutputs(
const std::vector<string>& output_names,
const ExecutorsAndKeys* executors_and_keys, RunState* run_state,
std::vector<Tensor>* outputs);
// Check if the specified fetches can be computed from the feeds
// that we have already provided.
::tensorflow::Status CheckFetch(
const std::vector<std::pair<string, Tensor>>& feeds,
const std::vector<string>& fetches,
const ExecutorsAndKeys* executors_and_keys, const RunState* run_state);
// Use the appropriate WaitForNotification function based on whether
// operation_timeout_in_ms is greater than 0.
//
// If the timeout expires, the `cm->StartCancel()` will be called.
::tensorflow::Status WaitForNotification(Notification* n,
int64 timeout_in_ms);
void WaitForNotification(RunState* run_state, CancellationManager* cm,
int64 timeout_in_ms);
::tensorflow::Status CheckNotClosed() {
mutex_lock l(closed_lock_);
if (closed_) return errors::Cancelled("Session has been closed.");
return ::tensorflow::Status::OK();
}
::tensorflow::Status CheckGraphCreated(const char* method) {
mutex_lock l(graph_state_lock_);
if (!graph_created_) {
return errors::InvalidArgument(
"Session was not created with a graph before ", method, "!");
}
return ::tensorflow::Status::OK();
}
::tensorflow::Status CreateDebuggerState(
const CallableOptions& options, int64 global_step,
int64 session_run_index, int64 executor_step_index,
std::unique_ptr<DebuggerStateInterface>* debugger_state);
::tensorflow::Status DecorateAndPublishGraphForDebug(
const DebugOptions& debug_options, Graph* graph, Device* device);
const SessionOptions options_;
// Device structures.
const std::unique_ptr<const DeviceMgr> device_mgr_;
std::vector<Device*> devices_; // not owned
DeviceSet device_set_;
string session_handle_;
mutex graph_state_lock_;
bool graph_created_ GUARDED_BY(graph_state_lock_) = false;
// The thread-pools to use for running ops, with a bool indicating if the pool
// is owned.
std::vector<std::pair<thread::ThreadPool*, bool>> thread_pools_;
Status init_error_; // Set to an error if construction failed.
// If true, blocks until device has finished all queued operations in a step.
bool sync_on_finish_ = true;
// Schedules 'c' for execution on pool.
void SchedClosure(thread::ThreadPool* pool, std::function<void()> c);
std::vector<std::unique_ptr<FunctionInfo>> functions_
GUARDED_BY(executor_lock_);
mutex executor_lock_; // protects executors_
// Holds mappings from signature to the executors that process
// it. The reason for a level of indirection around mapped_type is
// to guarantee address stability.
// The map value is a shared_ptr since multiple map keys can point to the
// same ExecutorsAndKey object.
std::unordered_map<string, std::shared_ptr<ExecutorsAndKeys>> executors_
GUARDED_BY(executor_lock_);
class RunCallableCallFrame;
struct Callable {
std::shared_ptr<ExecutorsAndKeys> executors_and_keys;
std::shared_ptr<FunctionInfo> function_info;
~Callable();
};
mutex callables_lock_;
int64 next_callable_handle_ GUARDED_BY(callables_lock_) = 0;
std::unordered_map<int64, Callable> callables_ GUARDED_BY(callables_lock_);
// Holds mappings from handle to partial run state.
std::unordered_map<string, std::unique_ptr<RunState>> partial_runs_
GUARDED_BY(executor_lock_);
// This holds all the tensors that are currently alive in the session.
SessionState session_state_;
DirectSessionFactory* const factory_; // not owned
CancellationManager* cancellation_manager_;
std::unique_ptr<CollectiveExecutorMgrInterface> collective_executor_mgr_;
// Map of placed stateful nodes, i.e. nodes for which is_stateful()
// is true, such as "params" and "queue" nodes. Once placed these
// nodes can not be moved to a different device. Maps node names to
// device names.
std::unordered_map<string, string> stateful_placements_
GUARDED_BY(graph_state_lock_);
// Execution_state; used when placing the entire graph.
std::unique_ptr<GraphExecutionState> execution_state_
GUARDED_BY(graph_state_lock_);
// The function library, before any rewrites or optimizations have been
// performed. In particular, CreateGraphs() may need to modify the function
// library; it copies and modifies the function library.
std::unique_ptr<FunctionLibraryDefinition> flib_def_;
// true if the Session has been Closed.
mutex closed_lock_;
bool closed_ GUARDED_BY(closed_lock_) = false;
// For generating unique names for this session instance.
std::atomic<int64> edge_name_counter_ = {0};
std::atomic<int64> handle_name_counter_ = {0};
// For generating step ids that are unique across this sessions.
static std::atomic_int_fast64_t step_id_counter_;
// Global timeout for all blocking operations in this session.
const int64 operation_timeout_in_ms_ = 0;
// Manages all the cost models for the graphs executed in this session.
CostModelManager cost_model_manager_;
// For testing collective graph key generation.
mutex collective_graph_key_lock_;
int64 collective_graph_key_ GUARDED_BY(collective_graph_key_lock_) = -1;
TF_DISALLOW_COPY_AND_ASSIGN(DirectSession);
// EXPERIMENTAL: debugger (tfdbg) related
friend class DebugGateway;
};
} // end namespace tensorflow
#endif // TENSORFLOW_CORE_COMMON_RUNTIME_DIRECT_SESSION_H_
@@ -0,0 +1,38 @@
/* Copyright 2015 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
#ifndef TENSORFLOW_CORE_COMMON_RUNTIME_DMA_HELPER_H_
#define TENSORFLOW_CORE_COMMON_RUNTIME_DMA_HELPER_H_
#include "tensorflow/core/framework/tensor.h"
namespace tensorflow {
// For TensorFlow internal use only.
class DMAHelper {
public:
static bool CanUseDMA(const Tensor* t) { return t->CanUseDMA(); }
static const void* base(const Tensor* t) { return t->base<const void>(); }
static void* base(Tensor* t) { return t->base<void>(); }
static TensorBuffer* buffer(Tensor* t) { return t->buf_; }
static const TensorBuffer* buffer(const Tensor* t) { return t->buf_; }
static void UnsafeSetShape(Tensor* t, const TensorShape& s) {
t->set_shape(s);
}
};
} // namespace tensorflow
#endif // TENSORFLOW_CORE_COMMON_RUNTIME_DMA_HELPER_H_
@@ -0,0 +1,41 @@
/* Copyright 2015 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
#ifndef TENSORFLOW_COMMON_RUNTIME_EIGEN_THREAD_POOL_H_
#define TENSORFLOW_COMMON_RUNTIME_EIGEN_THREAD_POOL_H_
#include "third_party/eigen3/unsupported/Eigen/CXX11/Tensor"
#include "tensorflow/core/lib/core/threadpool.h"
namespace tensorflow {
class EigenThreadPoolWrapper : public Eigen::ThreadPoolInterface {
public:
explicit EigenThreadPoolWrapper(thread::ThreadPool* pool) : pool_(pool) {}
~EigenThreadPoolWrapper() override {}
void Schedule(std::function<void()> fn) override {
pool_->Schedule(std::move(fn));
}
int NumThreads() const override { return pool_->NumThreads(); }
int CurrentThreadId() const override { return pool_->CurrentThreadId(); }
private:
thread::ThreadPool* pool_ = nullptr;
};
} // namespace tensorflow
#endif // TENSORFLOW_COMMON_RUNTIME_EIGEN_THREAD_POOL_H_
@@ -0,0 +1,66 @@
/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
#ifndef TENSORFLOW_CORE_COMMON_RUNTIME_EVAL_CONST_TENSOR_H_
#define TENSORFLOW_CORE_COMMON_RUNTIME_EVAL_CONST_TENSOR_H_
#include "tensorflow/core/graph/graph.h"
#include "tensorflow/core/lib/core/status.h"
// TODO(skyewm): can this be combined with ConstantFold?
namespace tensorflow {
class GraphRunner;
class OpRegistryInterface;
class ShapeRefiner;
class Tensor;
// Attempts to evaluate `tensor`. This will only be possible if `tensor` doesn't
// depend on any graph inputs (this function is safe to call if this isn't the
// case though).
//
// If the evaluation is successful, `evaluated` will be set to true and
// `tensor`s value returned in `result`. Otherwise `evaluated` will be set to
// false. An error status is returned if something is wrong with the graph or
// input. Note that `evaluated` may set to false if Status::OK() is returned.
//
// Params:
// tensor - the tensor to be evaluated.
// refiner - used to fetch the InferenceContexts for nodes in the graph.
// ops - the OpRegistryInterface for the graph.
// graph_def_version - the producer version of the graph.
// evaluated - output param indicating whether evaluation was successful.
// result - output param containing the result if evaluated is true.
// graph_runner - optional. If not set, a GraphRunner will be created for
// evaluating tensor. This can be set to avoid creating a new GraphRunner
// for every call.
// cached_values - optional. This can be used to cache evaluated results
// across calls, to avoid evaluating the same parts of the graph multiple
// times.
// max_cached_value_size - optional. If `cached_values` is set, the maximum
// result size to cache.
// disable_constant_propagation - if true, only Const node values will be
// returned.
Status EvaluateConstantTensor(
OutputTensor tensor, const ShapeRefiner& refiner,
const OpRegistryInterface& ops, int32 graph_def_version, bool* evaluated,
Tensor* result, GraphRunner* graph_runner = nullptr,
std::unordered_map<string, Tensor>* cached_values = nullptr,
int64 max_cached_value_size = 1024,
bool disable_constant_propagation = false);
} // namespace tensorflow
#endif // TENSORFLOW_CORE_COMMON_RUNTIME_EVAL_CONST_TENSOR_H_
@@ -0,0 +1,232 @@
/* Copyright 2015 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
#ifndef TENSORFLOW_CORE_COMMON_RUNTIME_EXECUTOR_H_
#define TENSORFLOW_CORE_COMMON_RUNTIME_EXECUTOR_H_
#include "tensorflow/core/common_runtime/device.h"
#include "tensorflow/core/framework/rendezvous.h"
#include "tensorflow/core/framework/session_state.h"
#include "tensorflow/core/framework/tensor.h"
#include "tensorflow/core/graph/graph.h"
#include "tensorflow/core/lib/core/notification.h"
#include "tensorflow/core/lib/core/status.h"
#include "tensorflow/core/platform/logging.h"
#include "tensorflow/core/platform/macros.h"
namespace tensorflow {
class StepStatsCollector;
// Executor runs a graph computation.
// Example:
// Graph* graph = ...;
// ... construct graph ...
// Executor* executor;
// TF_CHECK_OK(NewSimpleExecutor(my_device, graph, &executor));
// Rendezvous* rendezvous = NewNaiveRendezvous();
// TF_CHECK_OK(rendezvous->Send("input", some_input_tensor));
// TF_CHECK_OK(executor->Run({ExecutorOpts, rendezvous, nullptr}));
// TF_CHECK_OK(rendezvous->Recv("output", &output_tensor));
// ... ...
//
// Multiple threads can call Executor::Run concurrently.
class Executor {
public:
virtual ~Executor() {}
// RunAsync() executes the graph computation. "done" is run when the
// graph computation completes. If any error happens during the
// computation, "done" is run and the error is passed to "done".
//
// RunAsync() is given a few arguments in Args. The caller must
// ensure objects passed in Args (rendezvous, stats_collector, etc.)
// are alive at least until done is invoked. All pointers to the
// argument objects can be nullptr.
//
// "step_id" is a process-wide unique identifier for the step being
// run. Executors on different devices may receive the same step_id
// in the case that a step runs Ops on more than one device. The
// step_id is used for tracking resource usage of a given step.
//
// RunAsync() uses the given "rendezvous", if not null, as the
// mechanism to communicate inputs and outputs of the underlying
// graph computation.
//
// RunAsync() calls "stats_collector", if not null, to keep track of
// stats. This allows us to collect statistics and traces on demand.
//
// RunAsync() is provided a "call_frame", if the executor is used
// for executing a function, is used to pass arguments and return
// values between the caller and the callee.
//
// RunAsync() uses "cancellation_manager", if not nullptr, to
// register callbacks that should be called if the graph computation
// is canceled. Note that the callbacks merely unblock any
// long-running computation, and a canceled step will terminate by
// returning/calling the DoneCallback as usual.
//
// RunAsync() dispatches closures to "runner". Typically, "runner"
// is backed up by a bounded threadpool.
struct Args {
int64 step_id = 0;
Rendezvous* rendezvous = nullptr;
StepStatsCollectorInterface* stats_collector = nullptr;
CallFrameInterface* call_frame = nullptr;
CancellationManager* cancellation_manager = nullptr;
SessionState* session_state = nullptr;
TensorStore* tensor_store = nullptr;
ScopedStepContainer* step_container = nullptr;
CollectiveExecutor* collective_executor = nullptr;
// If true, calls Sync() on the device.
bool sync_on_finish = false;
typedef std::function<void()> Closure;
typedef std::function<void(Closure)> Runner;
Runner runner = nullptr;
};
typedef std::function<void(const Status&)> DoneCallback;
virtual void RunAsync(const Args& args, DoneCallback done) = 0;
// Synchronous wrapper for RunAsync().
Status Run(const Args& args) {
Status ret;
Notification n;
RunAsync(args, [&ret, &n](const Status& s) {
ret = s;
n.Notify();
});
n.WaitForNotification();
return ret;
}
};
// Creates an Executor that computes the given "graph".
//
// If successful, returns the constructed executor in "*executor". Otherwise,
// returns an error status.
//
// "params" provides a set of context for the executor. We expect that
// different context would provide different implementations.
struct LocalExecutorParams {
Device* device;
// The library runtime support.
FunctionLibraryRuntime* function_library = nullptr;
// create_kernel returns an instance of op kernel based on NodeDef.
// delete_kernel is called for every kernel used by the executor
// when the executor is deleted.
std::function<Status(const NodeDef&, OpKernel**)> create_kernel;
std::function<void(OpKernel*)> delete_kernel;
};
::tensorflow::Status NewLocalExecutor(const LocalExecutorParams& params,
std::unique_ptr<const Graph> graph,
Executor** executor);
// A class to help run multiple executors in parallel and wait until
// all of them are complete.
//
// ExecutorBarrier deletes itself after the function returned by Get()
// is called.
class ExecutorBarrier {
public:
typedef std::function<void(const Status&)> StatusCallback;
// Create an ExecutorBarrier for 'num' different executors.
//
// 'r' is the shared Rendezvous object that is used to communicate
// state. If any of the executors experiences an error, the
// rendezvous object will be aborted exactly once.
//
// 'done' is called after the last executor completes, and
// ExecutorBarrier is deleted.
ExecutorBarrier(size_t num, Rendezvous* r, StatusCallback done)
: rendez_(r), done_cb_(done), pending_(num) {}
~ExecutorBarrier() {}
// Returns a closure that Executors must call when they are done
// computing, passing the status of their execution as an argument.
StatusCallback Get() {
return std::bind(&ExecutorBarrier::WhenDone, this, std::placeholders::_1);
}
private:
Rendezvous* rendez_ = nullptr;
StatusCallback done_cb_ = nullptr;
mutable mutex mu_;
int pending_ GUARDED_BY(mu_) = 0;
Status status_ GUARDED_BY(mu_);
void WhenDone(const Status& s) {
bool error = false;
Rendezvous* error_rendez = nullptr;
StatusCallback done = nullptr;
Status status;
{
mutex_lock l(mu_);
// If we are the first error encountered, mark the status
// appropriately and later trigger an abort of the Rendezvous
// object by this thread only.
if (status_.ok() && !s.ok()) {
error = true;
error_rendez = rendez_;
error_rendez->Ref();
status_ = s;
}
// If this is the last call to WhenDone, call the final callback
// below.
if (--pending_ == 0) {
CHECK(done_cb_ != nullptr);
std::swap(done, done_cb_);
}
if (!status_.ok()) {
status = status_;
}
}
if (error) {
error_rendez->StartAbort(status);
error_rendez->Unref();
}
if (done != nullptr) {
delete this;
done(status);
}
}
TF_DISALLOW_COPY_AND_ASSIGN(ExecutorBarrier);
};
// A few helpers to facilitate create/delete kernels.
// Creates a kernel based on "ndef" on device "device". The kernel can
// access the functions in the "flib". The caller takes ownership of
// returned "*kernel".
Status CreateNonCachedKernel(Device* device, FunctionLibraryRuntime* flib,
const NodeDef& ndef, int graph_def_version,
OpKernel** kernel);
// Deletes "kernel" returned by CreateKernel.
void DeleteNonCachedKernel(OpKernel* kernel);
} // end namespace tensorflow
#endif // TENSORFLOW_CORE_COMMON_RUNTIME_EXECUTOR_H_
@@ -0,0 +1,51 @@
/* Copyright 2015 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
#ifndef TENSORFLOW_CORE_COMMON_RUNTIME_EXECUTOR_FACTORY_H_
#define TENSORFLOW_CORE_COMMON_RUNTIME_EXECUTOR_FACTORY_H_
#include <string>
#include "tensorflow/core/lib/core/errors.h"
#include "tensorflow/core/lib/core/status.h"
#include "tensorflow/core/lib/gtl/array_slice.h"
#include "tensorflow/core/platform/types.h"
namespace tensorflow {
class Executor;
class Graph;
struct LocalExecutorParams;
class ExecutorFactory {
public:
virtual Status NewExecutor(const LocalExecutorParams& params,
std::unique_ptr<const Graph> graph,
std::unique_ptr<Executor>* out_executor) = 0;
virtual ~ExecutorFactory() {}
static void Register(const string& executor_type, ExecutorFactory* factory);
static Status GetFactory(const string& executor_type,
ExecutorFactory** out_factory);
};
Status NewExecutor(const string& executor_type,
const LocalExecutorParams& params,
std::unique_ptr<const Graph> graph,
std::unique_ptr<Executor>* out_executor);
} // namespace tensorflow
#endif // TENSORFLOW_CORE_COMMON_RUNTIME_EXECUTOR_FACTORY_H_
@@ -0,0 +1,173 @@
/* Copyright 2015 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
#ifndef TENSORFLOW_CORE_COMMON_RUNTIME_FUNCTION_H_
#define TENSORFLOW_CORE_COMMON_RUNTIME_FUNCTION_H_
#include <functional>
#include <memory>
#include "tensorflow/core/common_runtime/device.h"
#include "tensorflow/core/common_runtime/device_mgr.h"
#include "tensorflow/core/common_runtime/process_function_library_runtime.h"
#include "tensorflow/core/framework/function.h"
#include "tensorflow/core/graph/graph.h"
#include "tensorflow/core/protobuf/config.pb.h"
namespace tensorflow {
static constexpr const char* const kNoInlineAttr = "_noinline";
// Registers a default customizable kernel creator for a function call.
//
// If 'cb()' returns a non-OK, we still fall back to an executor-based
// interpreter op kernel to execute a function. If 'cb()' returns OK,
// takes ownership of the returned OpKernel.
//
// TODO(zhifengc/phawkins): b/32379046
void RegisterDefaultCustomKernelCreator(CustomKernelCreator cb);
// Creates a FunctionLibraryRuntime, which instantiates functions
// defined in "lib_def" and executes functions on the "device".
// "device_mgr" must contain the "device". If not nullptr,
// "custom_kernel_creator" is consulted by the returned runtime to
// create kernels.
//
// The returned object does not take ownerships of "device" or
// "lib_def". The caller must ensure "device" and "lib_def" outlives
// the returned object.
//
// The "parent" is a pointer to the ProcessFunctionLibraryRuntime object that
// typically owns the created FunctionLibraryRuntime object. The parent pointer
// is not owned by the FunctionLibraryRuntime object.
std::unique_ptr<FunctionLibraryRuntime> NewFunctionLibraryRuntime(
const DeviceMgr* device_mgr, Env* env, Device* device,
int graph_def_version, const FunctionLibraryDefinition* lib_def,
thread::ThreadPool* thread_pool, const OptimizerOptions& optimizer_options,
CustomKernelCreator custom_kernel_creator,
ProcessFunctionLibraryRuntime* parent);
// Same as above except that the returned runtime consults with the
// global default custom kernel creator registered by
// RegisterDefaultCustomKernelCreator.
std::unique_ptr<FunctionLibraryRuntime> NewFunctionLibraryRuntime(
const DeviceMgr* device_mgr, Env* env, Device* device,
int graph_def_version, const FunctionLibraryDefinition* lib_def,
thread::ThreadPool* thread_pool, const OptimizerOptions& optimizer_options,
ProcessFunctionLibraryRuntime* parent);
// FunctionLibraryRuntime::GetFunctionBody returns a description of an
// instantiated function that is represented as a Graph with arg/ret
// nodes annotated.
struct FunctionBody {
FunctionDef fdef;
Graph* graph = nullptr; // owned.
DataTypeVector arg_types;
DataTypeVector ret_types;
gtl::InlinedVector<Node*, 4> arg_nodes;
gtl::InlinedVector<Node*, 4> ret_nodes;
FunctionBody() {}
FunctionBody(const FunctionDef& f, DataTypeSlice arg_types,
DataTypeSlice ret_types, Graph* g);
~FunctionBody();
};
// Debugging facility. Returns a debug string for a graph
// representing an instantiated function.
string DebugString(const Graph* instantiated_func_graph);
// A few hand-crafted optimization on the instantiated function body
// (a Graph*).
// Removes nodes that are
// 1. not stateful; and
// 2. not _Arg; and
// 3. not reachable from _Retval.
// Returns true iff any node is removed from "g".
bool RemoveDeadNodes(Graph* g);
// Find a pattern:
// src -(in)-> node -(out)-> dst, where
// 1) node is an identity node;
// 2) in is the only incoming data edge;
// 3) out is the only outgoing data edge;
//
// Rewrites the above pattern with src->dst and relevant data
// dependencies updated. Repeat the process until no such pattern
// left.
bool RemoveIdentityNodes(Graph* g);
// Rewrites _ListToArray and _ArrayToList to a set of Identity nodes.
bool RemoveListArrayConverter(Graph* g);
// For each node in "graph", if "lib" indicates that the node is a
// function call, inline the function body. Returns true if at least
// one node is inlined.
//
// This routine goes through "graph" nodes once and applies the
// inlining. The caller may decide to apply the inlining on "graph"
// multiple times by calling ExpandInlineFunctions a few times.
bool ExpandInlineFunctions(FunctionLibraryRuntime* lib, Graph* graph);
// Dump the contents of the "graph" to log files if the logging level is
// sufficiently high.
void DumpGraph(StringPiece label, const Graph* g);
// Applies graph rewrite optimization such as inlining, dead code
// removal, etc.
//
// **g is a graph constructed based on the runtime library 'lib'.
// OptimizeGraph mutates **g extensively and replaces '*g' with a
// complete copy. Therefore, the caller should not keep any references
// to nodes *g.
void OptimizeGraph(FunctionLibraryRuntime* lib, std::unique_ptr<Graph>* g);
// Convert the Graph of a function to a GraphDef.
//
// Handles renaming of nodes to avoid duplicate names which may
// be present after various rewriting operations.
void ToGraphDef(const Graph* g, GraphDef* gdef, bool pretty = false);
// Given a numerical function "f", returns another numerical function
// "g", such that if "f" takes N inputs and produces M outputs, "g"
// takes N + M inputs and produces N outputs. I.e., if
// (y1, y2, ..., y_M) = f(x1, x2, ..., x_N),
// g is a function which is
// (dL/dx1, dL/dx2, ..., dL/dx_N) = g(x1, x2, ..., x_N,
// dL/dy1, dL/dy2, ..., dL/dy_M),
// where L is a scalar-value function of (...x_i...).
//
// TODO(zhifengc): Asks math expert to say the comment again.
FunctionBody* SymbolicGradient(const FunctionBody& f);
// Given a "caller" in graph "g", which is a function call of a function
// to "fbody". Replaces the "caller" with fbody->graph and connects
// edges properly. "override_device" specifies whether inlining should replace
// explicitly specified devices inside fbody with the callee's device.
void InlineFunctionBody(const FunctionLibraryDefinition& flib_def, Graph* g,
Node* caller, const FunctionBody* fbody,
bool override_device = true);
// Instantiates FunctionDef into a graph. Set *fbody to point to the
// FunctionBody that holds the instantiated FunctionDef.
Status FunctionDefToBodyHelper(
const FunctionDef& fdef, const AttrSlice& attrs,
const FunctionLibraryDefinition* const lib_def,
const std::function<Status(const string&, const OpDef**)>& get_func_sig,
FunctionBody** fbody);
} // end namespace tensorflow
#endif // TENSORFLOW_CORE_COMMON_RUNTIME_FUNCTION_H_
@@ -0,0 +1,68 @@
/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
#ifndef TENSORFLOW_CORE_COMMON_RUNTIME_GPU_CUDA_HOST_ALLOCATOR_H_
#define TENSORFLOW_CORE_COMMON_RUNTIME_GPU_CUDA_HOST_ALLOCATOR_H_
#include "tensorflow/core/framework/allocator.h"
#include "tensorflow/core/platform/macros.h"
#include "tensorflow/core/platform/stream_executor.h"
namespace tensorflow {
// Allocator for pinned CPU RAM that is made known to CUDA for the
// purpose of efficient DMA with a GPU.
class CUDAHostAllocator : public SubAllocator {
public:
// Note: stream_exec cannot be null.
explicit CUDAHostAllocator(se::StreamExecutor* stream_exec, int numa_node,
const std::vector<Visitor>& alloc_visitors,
const std::vector<Visitor>& free_visitors)
: SubAllocator(alloc_visitors, free_visitors),
stream_exec_(stream_exec),
numa_node_(numa_node) {
CHECK(stream_exec_ != nullptr);
}
~CUDAHostAllocator() override {}
void* Alloc(size_t alignment, size_t num_bytes) override {
void* ptr = nullptr;
if (num_bytes > 0) {
ptr = stream_exec_->HostMemoryAllocate(num_bytes);
if (ptr == nullptr) {
LOG(WARNING) << "could not allocate pinned host memory of size: "
<< num_bytes;
return ptr;
}
VisitAlloc(ptr, numa_node_, num_bytes);
}
return ptr;
}
void Free(void* ptr, size_t num_bytes) override {
if (ptr != nullptr) {
VisitFree(ptr, numa_node_, num_bytes);
stream_exec_->HostMemoryDeallocate(ptr);
}
}
private:
se::StreamExecutor* stream_exec_; // not owned, non-null
const int numa_node_;
TF_DISALLOW_COPY_AND_ASSIGN(CUDAHostAllocator);
};
} // namespace tensorflow
#endif // TENSORFLOW_CORE_COMMON_RUNTIME_GPU_CUDA_HOST_ALLOCATOR_H_
@@ -0,0 +1,103 @@
/* Copyright 2015 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
#ifndef TENSORFLOW_CORE_COMMON_RUNTIME_GPU_GPU_BFC_ALLOCATOR_H_
#define TENSORFLOW_CORE_COMMON_RUNTIME_GPU_GPU_BFC_ALLOCATOR_H_
#include <memory>
#include <string>
#include <unordered_map>
#include <vector>
#include "tensorflow/core/common_runtime/allocator_retry.h"
#include "tensorflow/core/common_runtime/bfc_allocator.h"
#include "tensorflow/core/common_runtime/gpu/gpu_id.h"
#include "tensorflow/core/platform/stream_executor.h"
#include "tensorflow/core/platform/thread_annotations.h"
#include "tensorflow/core/platform/types.h"
#include "tensorflow/core/protobuf/config.pb.h"
namespace tensorflow {
// Suballocator for GPU memory.
class GPUMemAllocator : public SubAllocator {
public:
// 'platform_gpu_id' refers to the ID of the GPU device within
// the process and must reference a valid ID in the process.
// Note: stream_exec cannot be null.
explicit GPUMemAllocator(se::StreamExecutor* stream_exec,
PlatformGpuId gpu_id, bool use_unified_memory,
const std::vector<Visitor>& alloc_visitors,
const std::vector<Visitor>& free_visitors)
: SubAllocator(alloc_visitors, free_visitors),
stream_exec_(stream_exec),
gpu_id_(gpu_id),
use_unified_memory_(use_unified_memory) {
CHECK(stream_exec_ != nullptr);
}
~GPUMemAllocator() override {}
void* Alloc(size_t alignment, size_t num_bytes) override {
void* ptr = nullptr;
if (num_bytes > 0) {
if (use_unified_memory_) {
ptr = stream_exec_->UnifiedMemoryAllocate(num_bytes);
} else {
ptr = stream_exec_->AllocateArray<char>(num_bytes).opaque();
}
VisitAlloc(ptr, gpu_id_.value(), num_bytes);
}
return ptr;
}
void Free(void* ptr, size_t num_bytes) override {
if (ptr != nullptr) {
VisitFree(ptr, gpu_id_.value(), num_bytes);
if (use_unified_memory_) {
stream_exec_->UnifiedMemoryDeallocate(ptr);
} else {
se::DeviceMemoryBase gpu_ptr(ptr);
stream_exec_->Deallocate(&gpu_ptr);
}
}
}
private:
se::StreamExecutor* stream_exec_; // not owned, non-null
const PlatformGpuId gpu_id_;
const bool use_unified_memory_ = false;
TF_DISALLOW_COPY_AND_ASSIGN(GPUMemAllocator);
};
// A GPU memory allocator that implements a 'best-fit with coalescing'
// algorithm.
class GPUBFCAllocator : public BFCAllocator {
public:
GPUBFCAllocator(GPUMemAllocator* sub_allocator, size_t total_memory,
const string& name);
GPUBFCAllocator(GPUMemAllocator* sub_allocator, size_t total_memory,
const GPUOptions& gpu_options, const string& name);
~GPUBFCAllocator() override {}
TF_DISALLOW_COPY_AND_ASSIGN(GPUBFCAllocator);
private:
static bool GetAllowGrowthValue(const GPUOptions& gpu_options);
};
} // namespace tensorflow
#endif // TENSORFLOW_CORE_COMMON_RUNTIME_GPU_GPU_BFC_ALLOCATOR_H_
@@ -0,0 +1,52 @@
/* Copyright 2017 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
#ifndef TENSORFLOW_CORE_COMMON_RUNTIME_GPU_GPU_CUDAMALLOC_ALLOCATOR_H_
#define TENSORFLOW_CORE_COMMON_RUNTIME_GPU_GPU_CUDAMALLOC_ALLOCATOR_H_
#include <memory>
#include "tensorflow/core/common_runtime/gpu/gpu_id.h"
#include "tensorflow/core/framework/allocator.h"
#include "tensorflow/core/platform/macros.h"
#include "tensorflow/core/platform/stream_executor.h"
#include "tensorflow/core/platform/types.h"
namespace tensorflow {
// An allocator that wraps a GPU allocator and adds debugging
// functionality that verifies that users do not write outside their
// allocated memory.
class GPUcudaMallocAllocator : public Allocator {
public:
explicit GPUcudaMallocAllocator(Allocator* allocator,
PlatformGpuId platform_gpu_id);
~GPUcudaMallocAllocator() override;
string Name() override { return "gpu_debug"; }
void* AllocateRaw(size_t alignment, size_t num_bytes) override;
void DeallocateRaw(void* ptr) override;
bool TracksAllocationSizes() override;
private:
Allocator* base_allocator_ = nullptr; // owned
se::StreamExecutor* stream_exec_; // Not owned.
TF_DISALLOW_COPY_AND_ASSIGN(GPUcudaMallocAllocator);
};
} // namespace tensorflow
#endif // TENSORFLOW_CORE_COMMON_RUNTIME_GPU_GPU_CUDAMALLOC_ALLOCATOR_H_
@@ -0,0 +1,87 @@
/* Copyright 2015 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
#ifndef TENSORFLOW_CORE_COMMON_RUNTIME_GPU_GPU_DEBUG_ALLOCATOR_H_
#define TENSORFLOW_CORE_COMMON_RUNTIME_GPU_GPU_DEBUG_ALLOCATOR_H_
#include <memory>
#include <string>
#include <unordered_map>
#include "tensorflow/core/common_runtime/gpu/gpu_id.h"
#include "tensorflow/core/framework/allocator.h"
#include "tensorflow/core/platform/macros.h"
#include "tensorflow/core/platform/stream_executor.h"
#include "tensorflow/core/platform/types.h"
namespace tensorflow {
// An allocator that wraps a GPU allocator and adds debugging
// functionality that verifies that users do not write outside their
// allocated memory.
class GPUDebugAllocator : public Allocator {
public:
explicit GPUDebugAllocator(Allocator* allocator,
PlatformGpuId platform_gpu_id);
~GPUDebugAllocator() override;
string Name() override { return "gpu_debug"; }
void* AllocateRaw(size_t alignment, size_t num_bytes) override;
void DeallocateRaw(void* ptr) override;
bool TracksAllocationSizes() override;
size_t RequestedSize(const void* ptr) override;
size_t AllocatedSize(const void* ptr) override;
int64 AllocationId(const void* ptr) override;
void GetStats(AllocatorStats* stats) override;
void ClearStats() override;
// For testing.
bool CheckHeader(void* ptr);
bool CheckFooter(void* ptr);
private:
Allocator* base_allocator_ = nullptr; // owned
se::StreamExecutor* stream_exec_; // Not owned.
TF_DISALLOW_COPY_AND_ASSIGN(GPUDebugAllocator);
};
// An allocator that wraps a GPU allocator and resets the memory on
// allocation and free to 'NaN', helping to identify cases where the
// user forgets to initialize the memory.
class GPUNanResetAllocator : public Allocator {
public:
explicit GPUNanResetAllocator(Allocator* allocator,
PlatformGpuId platform_gpu_id);
~GPUNanResetAllocator() override;
string Name() override { return "gpu_nan_reset"; }
void* AllocateRaw(size_t alignment, size_t num_bytes) override;
void DeallocateRaw(void* ptr) override;
size_t RequestedSize(const void* ptr) override;
size_t AllocatedSize(const void* ptr) override;
void GetStats(AllocatorStats* stats) override;
void ClearStats() override;
private:
Allocator* base_allocator_ = nullptr; // owned
se::StreamExecutor* stream_exec_; // Not owned.
TF_DISALLOW_COPY_AND_ASSIGN(GPUNanResetAllocator);
};
} // namespace tensorflow
#endif // TENSORFLOW_CORE_COMMON_RUNTIME_GPU_GPU_DEBUG_ALLOCATOR_H_
@@ -0,0 +1,235 @@
/* Copyright 2015 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
#if !GOOGLE_CUDA
#error This file must only be included when building with Cuda support
#endif
#ifndef TENSORFLOW_CORE_COMMON_RUNTIME_GPU_GPU_DEVICE_H_
#define TENSORFLOW_CORE_COMMON_RUNTIME_GPU_GPU_DEVICE_H_
#include <memory>
#include <string>
#include <unordered_map>
#include <vector>
#include "third_party/eigen3/unsupported/Eigen/CXX11/Tensor"
#include "tensorflow/core/common_runtime/device_factory.h"
#include "tensorflow/core/common_runtime/gpu/gpu_event_mgr.h"
#include "tensorflow/core/common_runtime/gpu/gpu_id.h"
#include "tensorflow/core/common_runtime/gpu/gpu_id_manager.h"
#include "tensorflow/core/common_runtime/gpu/gpu_id_utils.h"
#include "tensorflow/core/common_runtime/gpu_device_context.h"
#include "tensorflow/core/common_runtime/local_device.h"
#include "tensorflow/core/common_runtime/scoped_allocator_mgr.h"
#include "tensorflow/core/framework/allocator.h"
#include "tensorflow/core/framework/device_base.h"
#include "tensorflow/core/framework/op_kernel.h"
#include "tensorflow/core/framework/tensor.h"
#include "tensorflow/core/lib/core/status.h"
#include "tensorflow/core/lib/gtl/inlined_vector.h"
#include "tensorflow/core/platform/mutex.h"
#include "tensorflow/core/platform/stream_executor.h"
#include "tensorflow/core/platform/types.h"
#include "tensorflow/core/public/session_options.h"
namespace tensorflow {
class BaseGPUDevice : public LocalDevice {
public:
BaseGPUDevice(const SessionOptions& options, const string& name,
Bytes memory_limit, const DeviceLocality& locality,
TfGpuId tf_gpu_id, const string& physical_device_desc,
Allocator* gpu_allocator, Allocator* cpu_allocator,
bool sync_every_op, int32 max_streams);
~BaseGPUDevice() override;
// Initialize the device and return the status of initialization.
Status Init(const SessionOptions& options);
// GPU devices require the Op Compute method to save a reference to
// any temporary tensors that are allocated until the Op execution
// completes.
bool RequiresRecordingAccessedTensors() const override;
// GPU kernel execution requires us to use `tracing::ScopedAnnotation()`
// rather than `tracing::ScopedActivity()`, in order to relate asynchronously
// launched GPU kernels to the OpKernel.
bool TraceUsingAnnotations() const { return true; }
void ConsumeListOfAccessedTensors(
DeviceContext* device_context,
const TensorReferenceVector& tensor_refs) override;
Status FillContextMap(const Graph* graph,
DeviceContextMap* device_context_map) override;
void Compute(OpKernel* op_kernel, OpKernelContext* context) override;
Status Sync() override;
void ComputeAsync(AsyncOpKernel* op_kernel, OpKernelContext* context,
AsyncOpKernel::DoneCallback done) override;
Status MakeTensorFromProto(const TensorProto& tensor_proto,
const AllocatorAttributes alloc_attrs,
Tensor* tensor) override;
// The caller owns the returned device.
PerOpGpuDevice* MakeGpuDevice() override;
Status ReinitializeGpuDevice(OpKernelContext* context, PerOpGpuDevice* device,
DeviceContext* dc,
Allocator* allocator) override;
// Returns the platform GPU id of this device within the native driver system;
// e.g., for CUDA this is the ordinal of the GPU within the system.
int gpu_id() const {
PlatformGpuId platform_gpu_id;
TF_CHECK_OK(GpuIdManager::TfToPlatformGpuId(tf_gpu_id_, &platform_gpu_id));
return platform_gpu_id.value();
}
// The executor that provides control for the device; e.g., for CUDA this
// corresponds to the cuda context.
se::StreamExecutor* executor() const { return executor_; }
Allocator* GetScopedAllocator(AllocatorAttributes attr,
int64 step_id) override;
ScopedAllocatorMgr* GetScopedAllocatorMgr() const override {
return scoped_allocator_mgr_.get();
}
protected:
Allocator* gpu_allocator_; // not owned
Allocator* cpu_allocator_; // not owned
se::StreamExecutor* executor_; // not owned
std::unique_ptr<ScopedAllocatorMgr> scoped_allocator_mgr_;
private:
struct StreamGroup {
se::Stream* compute = nullptr;
se::Stream* host_to_device = nullptr;
se::Stream* device_to_host = nullptr;
gtl::InlinedVector<se::Stream*, 4> device_to_device;
};
class StreamGroupFactory;
gtl::InlinedVector<StreamGroup*, 4> streams_;
mutex scratch_init_mutex_;
gtl::InlinedVector<char*, 4> scratch_;
std::vector<GPUDeviceContext*> device_contexts_;
GpuDeviceInfo* gpu_device_info_ = nullptr;
mutex trace_mu_;
TfGpuId tf_gpu_id_;
const bool sync_every_op_ = false;
const int32 max_streams_;
std::unique_ptr<EventMgr> em_;
std::unique_ptr<thread::ThreadPool> thread_pool_;
// Initialize scractch buffers used by Eigen.
Status InitScratchBuffers();
void ReinitializeDevice(OpKernelContext* context, PerOpGpuDevice* device,
int stream_id, Allocator* allocator);
void ComputeHelper(OpKernel* op_kernel, OpKernelContext* context);
string ComputeOpKernelDebugString(const OpKernel& op_kernel,
const int& stream_id);
// This method returns an initialization status, in addition to
// calling the "done" StatusCallback, if there is a failure to
// allocate memory or if the tensor "from" is not DMA-copyable.
// If there is no error prior to enqueueing the copy, an OK status
// is returned.
Status MaybeCopyTensorToGPU(const AllocatorAttributes& alloc_attrs,
const Tensor& from, Tensor* to,
StatusCallback done);
};
class BaseGPUDeviceFactory : public DeviceFactory {
public:
Status CreateDevices(const SessionOptions& options, const string& name_prefix,
std::vector<Device*>* devices) override;
struct InterconnectMap {
// Name of interconnect technology, if known.
string name;
// If possible, strength should approximate Gb/sec bandwidth rate.
// Where architecture-specific subclassing is not done that won't
// always be possible. The minimum expectation is that
// faster links should have a higher value than slower links.
int32 strength;
static const int kSameDeviceStrength;
static const int kStreamExecutorStrength;
std::set<std::pair<PlatformGpuId, PlatformGpuId>> directed_links;
};
protected:
// Populates *maps with interconnect maps for all local direct access
// pathways between GPUs.
virtual Status GetInterconnectMaps(
const std::vector<PlatformGpuId>& visible_gpu_order,
se::Platform* gpu_manager, std::vector<InterconnectMap>* maps);
struct TfGpuIdHash {
std::size_t operator()(const TfGpuId& id) const noexcept {
return std::hash<int>{}(id.value());
}
};
typedef std::unordered_map<TfGpuId, DeviceLocality, TfGpuIdHash> LocalityMap;
// Populates *localities with the DeviceLocality descriptor for
// every TfGpuId.
virtual Status GetDeviceLocalities(
int num_tf_gpus, const std::vector<InterconnectMap>& interconnects,
LocalityMap* localities);
private:
// Creates a BaseGPUDevice associated with 'tf_gpu_id', allocates (strictly)
// 'memory_limit' bytes of GPU memory to it, and adds it to the 'devices'
// vector.
Status CreateGPUDevice(const SessionOptions& options,
const string& name_prefix, TfGpuId tf_gpu_id,
int64 memory_limit, const DeviceLocality& dev_locality,
std::vector<Device*>* devices);
virtual BaseGPUDevice* CreateGPUDevice(const SessionOptions& options,
const string& name, Bytes memory_limit,
const DeviceLocality& dev_locality,
TfGpuId tf_gpu_id,
const string& physical_device_desc,
Allocator* gpu_allocator,
Allocator* cpu_allocator) = 0;
// Returns into 'ids' the list of valid platform GPU ids, in the order that
// they should map to TF GPU ids "/device:GPU:0", "/device:GPU:1", etc,
// based upon 'visible_gpu_order' which was generated by parsing
// GPUOptions::visible_device_list which is a comma-separated list of CUDA GPU
// ids.
Status GetValidDeviceIds(const std::vector<PlatformGpuId>& visible_gpu_order,
std::vector<PlatformGpuId>* ids);
// visible_gpu_initialized_[platform_gpu_id] is true if visible GPU
// platform_gpu_id has been initialized by the process.
std::unordered_map<int, bool> visible_gpu_initialized_;
};
} // namespace tensorflow
#endif // TENSORFLOW_CORE_COMMON_RUNTIME_GPU_GPU_DEVICE_H_
@@ -0,0 +1,206 @@
/* Copyright 2015 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
#ifndef TENSORFLOW_CORE_COMMON_RUNTIME_GPU_GPU_EVENT_MGR_H_
#define TENSORFLOW_CORE_COMMON_RUNTIME_GPU_GPU_EVENT_MGR_H_
#include <deque>
#include <vector>
#include "tensorflow/core/framework/log_memory.h"
#include "tensorflow/core/framework/tensor.h"
#include "tensorflow/core/framework/tensor_reference.h"
#include "tensorflow/core/lib/core/notification.h"
#include "tensorflow/core/lib/core/threadpool.h"
#include "tensorflow/core/lib/gtl/inlined_vector.h"
#include "tensorflow/core/platform/mutex.h"
#include "tensorflow/core/platform/stream_executor.h"
#include "tensorflow/core/platform/thread_annotations.h"
#include "tensorflow/core/platform/types.h"
namespace stream_executor {
class Event;
class Stream;
class StreamExecutor;
} // namespace stream_executor
namespace tensorflow {
class GPUOptions;
// The callback provided to EventMgr::ThenExecute must not block or take a long
// time. If it does, performance may be impacted and GPU memory may be
// exhausted. This macro is for checking that an EventMgr thread is not
// accidentally entering blocking parts of the code, e.g. the RPC subsystem.
//
// Intended use is something like
//
// void RespondToAnRPC(Params* params) {
// WARN_IF_IN_EVENT_MGR_THREAD;
// if (params->status.ok()) { ...
//
namespace gpu_event_mgr {
// Logs a stack trace if current execution thread belongs to this EventMgr
// object. If f is not nullptr, executes instead of logging the stack trace.
// trace.
void WarnIfInCallback(std::function<void()> f);
} // namespace gpu_event_mgr
#define WARN_IF_IN_EVENT_MGR_THREAD gpu_event_mgr::WarnIfInCallback(nullptr)
// An object to keep track of pending Events in the StreamExecutor streams
// and associated Tensors that cannot safely be deleted until the associated
// Events are recorded.
class EventMgr {
public:
EventMgr(se::StreamExecutor* se, const GPUOptions& gpu_options);
~EventMgr();
// Releases the references on the elements of "tensors" as soon as
// all events currently enqueued on "stream" have completed.
void ThenDeleteTensors(se::Stream* stream,
const TensorReferenceVector& tensors);
struct BufRec {
Allocator* alloc;
void* buf;
// operation and step_id are only populated when
// LogMemory::IsEnabled() is true.
string operation;
int64 step_id;
};
// Takes ownership of *bufrec.buf and calls bufrec.alloc->DeallocateRaw()
// on it as soon as all events currently enqueued on *stream have completed.
inline void ThenDeleteBuffer(se::Stream* stream, BufRec bufrec) {
ToFreeVector to_free;
{
mutex_lock l(mu_);
QueueBuffer(stream, bufrec);
PollEvents(false, &to_free);
}
FreeMemory(to_free);
}
// Execute func when all pending stream actions have completed.
// func must be brief and non-blocking since it executes in the one
// thread used for all such callbacks and also buffer deletions.
inline void ThenExecute(se::Stream* stream, std::function<void()> func) {
ToFreeVector to_free;
{
mutex_lock l(mu_);
QueueFunc(stream, std::move(func));
PollEvents(false, &to_free);
}
FreeMemory(to_free);
}
private:
friend class TEST_EventMgrHelper;
se::StreamExecutor* const exec_;
const int64 deferred_bytes_threshold_;
const int32 polling_active_delay_usecs_;
mutex mu_;
condition_variable events_pending_ GUARDED_BY(mu_);
void FlushAccumulatedTensors() EXCLUSIVE_LOCKS_REQUIRED(mu_);
struct InUse {
se::Event* event;
TensorReferenceVector* mem;
BufRec bufrec;
std::function<void()> func;
};
typedef gtl::InlinedVector<InUse, 4> ToFreeVector;
void FreeMemory(const ToFreeVector& to_free) {
for (const auto& iu : to_free) {
if (iu.mem != nullptr) {
for (auto& t : *(iu.mem)) {
t.Unref();
}
delete iu.mem;
}
if (iu.bufrec.buf) {
if (LogMemory::IsEnabled()) {
LogMemory::RecordRawDeallocation(iu.bufrec.operation,
iu.bufrec.step_id, iu.bufrec.buf,
iu.bufrec.alloc, false);
}
iu.bufrec.alloc->DeallocateRaw(iu.bufrec.buf);
}
// The function must be called in another thread.
if (iu.func != nullptr) threadpool_.Schedule(iu.func);
}
}
// Stream-enqueue an unused Event and save with it a collection of
// Tensors and/or a BufRec to be deleted only after the Event
// records.
void QueueInUse(se::Stream* stream, InUse in_use)
EXCLUSIVE_LOCKS_REQUIRED(mu_);
void QueueTensors(se::Stream* stream, TensorReferenceVector* tensors)
EXCLUSIVE_LOCKS_REQUIRED(mu_) {
QueueInUse(stream, {nullptr, tensors, BufRec(), nullptr});
}
void QueueBuffer(se::Stream* stream, BufRec bufrec)
EXCLUSIVE_LOCKS_REQUIRED(mu_) {
QueueInUse(stream, {nullptr, nullptr, bufrec, nullptr});
}
void QueueFunc(se::Stream* stream, std::function<void()> func)
EXCLUSIVE_LOCKS_REQUIRED(mu_) {
QueueInUse(stream, {nullptr, nullptr, BufRec(), std::move(func)});
}
// This function should be called at roughly the same tempo as
// QueueTensors() to check whether pending events have recorded,
// and then retire them. It appends InUse elements that need cleanup
// to "*to_free". The caller should call FreeMemory(to_free)
// when this returns.
void PollEvents(bool is_dedicated_poller, ToFreeVector* to_free)
EXCLUSIVE_LOCKS_REQUIRED(mu_);
// An internal polling loop that runs at a low frequency to clear
// straggler Events.
void PollLoop();
// Setup/Teardown functions for the polling loop.
void StartPollingLoop();
void StopPollingLoop();
// A stack of unused events
std::vector<se::Event*> free_events_ GUARDED_BY(mu_);
// Buffered list of tensors waiting to have an event queued for deletion
se::Stream* accumulated_stream_ GUARDED_BY(mu_);
TensorReferenceVector* accumulated_tensors_ GUARDED_BY(mu_);
// Sum of the TotalBytes() of the tensors in "accumulated_tensors_"
int64 accumulated_tensor_bytes_ GUARDED_BY(mu_);
// A FIFO queue of InUse events and associated tensors.
std::deque<InUse> used_events_ GUARDED_BY(mu_);
bool stop_polling_ GUARDED_BY(mu_);
std::unique_ptr<Notification> polling_stopped_;
// The main PollLoop for the event manager runs in this threadpool.
thread::ThreadPool threadpool_;
};
} // namespace tensorflow
#endif // TENSORFLOW_CORE_COMMON_RUNTIME_GPU_GPU_EVENT_MGR_H_
@@ -0,0 +1,88 @@
/* Copyright 2015 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
#ifndef TENSORFLOW_CORE_COMMON_RUNTIME_GPU_GPU_ID_H_
#define TENSORFLOW_CORE_COMMON_RUNTIME_GPU_GPU_ID_H_
#include "tensorflow/core/lib/gtl/int_type.h"
namespace tensorflow {
// There are three types of GPU ids:
// - *physical* GPU id: this is the integer index of a GPU hardware in the
// physical machine, it can be filtered by CUDA environment variable
// CUDA_VISIBLE_DEVICES. Note that this id is not visible to Tensorflow, but
// result after filtering by CUDA_VISIBLE_DEVICES is visible to TF and is
// called platform GPU id as below. See
// http://docs.nvidia.com/cuda/cuda-c-programming-guide/index.html#env-vars
// for more details.
// - *platform* GPU id (also called *visible* GPU id in
// third_party/tensorflow/core/protobuf/config.proto): this is the id that is
// visible to Tensorflow after filtering by CUDA_VISIBLE_DEVICES, and is
// generated by the CUDA GPU driver. It starts from 0 and is used for CUDA API
// calls like cuDeviceGet().
// - TF GPU id (also called *virtual* GPU id in
// third_party/tensorflow/core/protobuf/config.proto): this is the id that
// Tensorflow generates and exposes to its users. It is the id in the <id>
// field of the device name "/device:GPU:<id>", and is also the identifier of
// a BaseGPUDevice. Note that the configuration allows us to create multiple
// BaseGPUDevice per GPU hardware in order to use multi CUDA streams on the
// hardware, so the mapping between TF GPU id and platform GPU id is not a 1:1
// mapping, see the example below.
//
// For example, assuming that in the machine we have GPU device with index 0, 1,
// 2 and 3 (physical GPU id). Setting "CUDA_VISIBLE_DEVICES=1,2,3" will create
// the following mapping between platform GPU id and physical GPU id:
//
// platform GPU id -> physical GPU id
// 0 -> 1
// 1 -> 2
// 2 -> 3
//
// Note that physical GPU id 0 is invisible to TF so there is no mapping entry
// for it.
//
// Assuming we configure the Session to create one BaseGPUDevice per GPU
// hardware, then setting GPUOptions::visible_device_list to "2,0" will create
// the following mappting between TF GPU id and platform GPU id:
//
// TF GPU id -> platform GPU ID
// 0 (i.e. /device:GPU:0) -> 2
// 1 (i.e. /device:GPU:1) -> 0
//
// Note that platform GPU id 1 is filtered out by
// GPUOptions::visible_device_list, so it won't be used by the TF process.
//
// On the other hand, if we configure it to create 2 BaseGPUDevice per GPU
// hardware, then setting GPUOptions::visible_device_list to "2,0" will create
// the following mappting between TF GPU id and platform GPU id:
//
// TF GPU id -> platform GPU ID
// 0 (i.e. /device:GPU:0) -> 2
// 1 (i.e. /device:GPU:1) -> 2
// 2 (i.e. /device:GPU:2) -> 0
// 3 (i.e. /device:GPU:3) -> 0
//
// We create strong-typed integer classes for both TF GPU id and platform GPU id
// to minimize programming errors and improve code readability. Except for the
// StreamExecutor interface (as we don't change its API), whenever we need a
// TF GPU id (or platform GPU id) we should use TfGpuId (or PlatformGpuId)
// instead of a raw integer.
TF_LIB_GTL_DEFINE_INT_TYPE(TfGpuId, int32);
TF_LIB_GTL_DEFINE_INT_TYPE(PlatformGpuId, int32);
} // namespace tensorflow
#endif // TENSORFLOW_CORE_COMMON_RUNTIME_GPU_GPU_ID_H_
@@ -0,0 +1,42 @@
/* Copyright 2015 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
#ifndef TENSORFLOW_CORE_COMMON_RUNTIME_GPU_GPU_ID_MANAGER_H_
#define TENSORFLOW_CORE_COMMON_RUNTIME_GPU_GPU_ID_MANAGER_H_
#include "tensorflow/core/common_runtime/gpu/gpu_id.h"
#include "tensorflow/core/lib/core/status.h"
namespace tensorflow {
// Class that maintains a map from TfGpuId to PlatformGpuId, and manages the
// translation between them.
class GpuIdManager {
public:
// Adds a mapping from tf_gpu_id to platform_gpu_id.
static Status InsertTfPlatformGpuIdPair(TfGpuId tf_gpu_id,
PlatformGpuId platform_gpu_id);
// Gets the platform_gpu_id associated with tf_gpu_id. Returns OK if found.
static Status TfToPlatformGpuId(TfGpuId tf_gpu_id,
PlatformGpuId* platform_gpu_id);
// Clears the map. Used in unit tests only.
static void TestOnlyReset();
};
} // namespace tensorflow
#endif // TENSORFLOW_CORE_COMMON_RUNTIME_GPU_GPU_ID_MANAGER_H_
@@ -0,0 +1,64 @@
/* Copyright 2015 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
#ifndef TENSORFLOW_CORE_COMMON_RUNTIME_GPU_GPU_ID_UTILS_H_
#define TENSORFLOW_CORE_COMMON_RUNTIME_GPU_GPU_ID_UTILS_H_
#include "tensorflow/core/common_runtime/gpu/gpu_id.h"
#include "tensorflow/core/common_runtime/gpu/gpu_id_manager.h"
#include "tensorflow/core/common_runtime/gpu/gpu_init.h"
#include "tensorflow/core/lib/gtl/int_type.h"
#include "tensorflow/core/platform/stream_executor.h"
namespace tensorflow {
// Utility methods for translation between Tensorflow GPU ids and platform GPU
// ids.
class GpuIdUtil {
public:
// Convenient methods for getting the associated executor given a TfGpuId or
// PlatformGpuId.
static se::port::StatusOr<se::StreamExecutor*> ExecutorForPlatformGpuId(
se::Platform* gpu_manager, PlatformGpuId platform_gpu_id) {
return gpu_manager->ExecutorForDevice(platform_gpu_id.value());
}
static se::port::StatusOr<se::StreamExecutor*> ExecutorForPlatformGpuId(
PlatformGpuId platform_gpu_id) {
return ExecutorForPlatformGpuId(GPUMachineManager(), platform_gpu_id);
}
static se::port::StatusOr<se::StreamExecutor*> ExecutorForTfGpuId(
TfGpuId tf_gpu_id) {
PlatformGpuId platform_gpu_id;
TF_RETURN_IF_ERROR(
GpuIdManager::TfToPlatformGpuId(tf_gpu_id, &platform_gpu_id));
return ExecutorForPlatformGpuId(platform_gpu_id);
}
// Verify that the platform_gpu_id associated with a TfGpuId is legitimate.
static void CheckValidTfGpuId(TfGpuId tf_gpu_id) {
PlatformGpuId platform_gpu_id;
TF_CHECK_OK(GpuIdManager::TfToPlatformGpuId(tf_gpu_id, &platform_gpu_id));
const int visible_device_count = GPUMachineManager()->VisibleDeviceCount();
CHECK_LT(platform_gpu_id.value(), visible_device_count)
<< "platform_gpu_id is outside discovered device range."
<< " TF GPU id: " << tf_gpu_id
<< " platform GPU id: " << platform_gpu_id
<< " visible device count: " << visible_device_count;
}
};
} // namespace tensorflow
#endif // TENSORFLOW_CORE_COMMON_RUNTIME_GPU_GPU_ID_UTILS_H_
@@ -0,0 +1,39 @@
/* Copyright 2015 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
#ifndef TENSORFLOW_CORE_COMMON_RUNTIME_GPU_GPU_INIT_H_
#define TENSORFLOW_CORE_COMMON_RUNTIME_GPU_GPU_INIT_H_
#include "tensorflow/core/lib/core/status.h"
namespace stream_executor {
class Platform;
} // namespace stream_executor
namespace tensorflow {
// Initializes the CUDA platform and returns OK if the CUDA
// platform could be initialized.
Status ValidateGPUMachineManager();
// Returns the GPU machine manager singleton, creating it and
// initializing the GPUs on the machine if needed the first time it is
// called. Must only be called when there is a valid GPU environment
// in the process (e.g., ValidateGPUMachineManager() returns OK).
stream_executor::Platform* GPUMachineManager();
} // namespace tensorflow
#endif // TENSORFLOW_CORE_COMMON_RUNTIME_GPU_GPU_INIT_H_
@@ -0,0 +1,36 @@
/* Copyright 2017 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
#ifndef TENSORFLOW_CORE_COMMON_RUNTIME_GPU_GPU_MANAGED_ALLOCATOR_H_
#define TENSORFLOW_CORE_COMMON_RUNTIME_GPU_GPU_MANAGED_ALLOCATOR_H_
#include "tensorflow/core/framework/allocator.h"
namespace tensorflow {
// An allocator for CUDA unified memory. Memory allocated with this allocator
// can be accessed from both host and device. CUDA transparently migrates dirty
// pages, which can be slow. Therefore, this allocator is intended for
// convenience in functional tests only.
class GpuManagedAllocator : public Allocator {
public:
string Name() override { return "GpuManagedAllocator"; }
void* AllocateRaw(size_t alignment, size_t num_bytes) override;
void DeallocateRaw(void* ptr) override;
};
} // namespace tensorflow
#endif // TENSORFLOW_CORE_COMMON_RUNTIME_GPU_GPU_MANAGED_ALLOCATOR_H_
@@ -0,0 +1,137 @@
/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
#ifndef TENSORFLOW_CORE_COMMON_RUNTIME_GPU_GPU_PROCESS_STATE_H_
#define TENSORFLOW_CORE_COMMON_RUNTIME_GPU_GPU_PROCESS_STATE_H_
#include <functional>
#include <map>
#include <unordered_map>
#include <vector>
#include "tensorflow/core/common_runtime/gpu/gpu_id.h"
#include "tensorflow/core/common_runtime/process_state.h"
#include "tensorflow/core/framework/allocator.h"
#include "tensorflow/core/platform/mutex.h"
#include "tensorflow/core/platform/thread_annotations.h"
#include "tensorflow/core/platform/types.h"
#include "tensorflow/core/protobuf/config.pb.h"
namespace tensorflow {
class Allocator;
class PoolAllocator;
// Singleton that manages per-process state when GPUs are present.
class GPUProcessState {
public:
static GPUProcessState* singleton();
// Query whether any GPU device has been created so far.
// Disable thread safety analysis since a race is benign here.
bool HasGPUDevice() const NO_THREAD_SAFETY_ANALYSIS {
return gpu_device_enabled_;
}
// Set the flag to indicate a GPU device has been created.
// Disable thread safety analysis since a race is benign here.
void EnableGPUDevice() NO_THREAD_SAFETY_ANALYSIS {
gpu_device_enabled_ = true;
}
// Returns the one GPU allocator used for the indexed GPU.
// Note that this is a system GPU index, not (necessarily) a brain
// device index.
//
// 'total_bytes' is the total number of bytes that should be made
// available to the allocator. The first call to this function for
// a given tf_gpu_id creates the allocator, so only the total_bytes
// used on that first call is used.
//
// "Allocator type" describes the type of algorithm to use for the
// underlying allocator. REQUIRES: Must be a valid type (see
// config.proto for the list of supported strings.).
//
// REQUIRES: tf_gpu_id must be a valid id for a BaseGPUDevice available in the
// current system environment. Otherwise returns nullptr.
virtual Allocator* GetGPUAllocator(const GPUOptions& options,
TfGpuId tf_gpu_id, size_t total_bytes);
virtual Allocator* GetCUDAHostAllocator(int numa_node);
// Registers a Visitor to be invoked on new chunks of memory allocated by the
// SubAllocator of every GPU proximate to the specified bus. The AllocVisitor
// is provided with a memory pointer, a GPU id, and the size of the area it
// identifies. The pointer is not guaranteed to be valid after the call
// terminates. The intention is for this interface to be used for network
// device memory registration. "bus_id" is platform-specific. On many
// platforms it should be 0. On machines with multiple PCIe buses, it should
// be the index of one of the PCIe buses (maybe the NUMA node at which the
// PCIe is rooted). If the bus_id is invalid, results are undefined.
virtual void AddGPUAllocVisitor(int bus_id,
const SubAllocator::Visitor& visitor);
// Registers a Visitor to be invoked on new chunks of memory allocated by
// the SubAllocator of the CUDAHostAllocator for the given numa_node.
virtual void AddCUDAHostAllocVisitor(int numa_node,
const SubAllocator::Visitor& visitor);
// Registers a Visitor to be invoked on each chunk handed back for freeing to
// the SubAllocator of the CUDAHostAllocator for the given numa_node.
virtual void AddCUDAHostFreeVisitor(int numa_node,
const SubAllocator::Visitor& visitor);
// Returns bus_id for the given GPU id.
virtual int BusIdForGPU(TfGpuId tf_gpu_id);
protected:
GPUProcessState();
// Helper method for unit tests to reset the ProcessState singleton by
// cleaning up everything. Never use in production.
virtual void TestOnlyReset();
ProcessState::MDMap* mem_desc_map() {
if (process_state_) return &process_state_->mem_desc_map_;
return nullptr;
}
static GPUProcessState* instance_;
ProcessState* process_state_; // Not owned.
bool gpu_device_enabled_;
mutex mu_;
struct AllocatorParts {
std::unique_ptr<Allocator> allocator;
SubAllocator* sub_allocator; // owned by allocator
std::unique_ptr<Allocator> recording_allocator;
};
std::vector<AllocatorParts> gpu_allocators_ GUARDED_BY(mu_);
std::vector<std::vector<SubAllocator::Visitor>> gpu_visitors_ GUARDED_BY(mu_);
std::vector<AllocatorParts> cuda_host_allocators_ GUARDED_BY(mu_);
std::vector<std::vector<SubAllocator::Visitor>> cuda_host_alloc_visitors_
GUARDED_BY(mu_);
std::vector<std::vector<SubAllocator::Visitor>> cuda_host_free_visitors_
GUARDED_BY(mu_);
virtual ~GPUProcessState();
friend class GPUDeviceTest;
};
} // namespace tensorflow
#endif // TENSORFLOW_CORE_COMMON_RUNTIME_GPU_GPU_PROCESS_STATE_H_
@@ -0,0 +1,45 @@
/* Copyright 2015 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
#ifndef TENSORFLOW_CORE_COMMON_RUNTIME_GPU_GPU_STREAM_UTIL_H_
#define TENSORFLOW_CORE_COMMON_RUNTIME_GPU_GPU_STREAM_UTIL_H_
#include <unordered_map>
#include "tensorflow/core/graph/graph.h"
#include "tensorflow/core/lib/core/status.h"
namespace tensorflow {
namespace gpu_stream_util {
struct AssignStreamsOpts {
int32 max_streams = 1;
// The following options specify a stream to use for specific op
// types. The value -1 allows ops to be assigned to any stream.
int32 send_stream = -1;
int32 recv_stream = -1;
int32 const_stream = -1;
int32 compute_stream = -1;
};
// Given the input graph, assigns every node in the graph with a
// stream_id that should be used.
Status AssignStreams(const Graph* graph, const AssignStreamsOpts& opts,
std::unordered_map<int, int>* node_to_stream_id);
} // namespace gpu_stream_util
} // namespace tensorflow
#endif // TENSORFLOW_CORE_COMMON_RUNTIME_GPU_GPU_STREAM_UTIL_H_
@@ -0,0 +1,111 @@
/* Copyright 2015 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
#ifndef TENSORFLOW_CORE_COMMON_RUNTIME_GPU_GPU_UTIL_H_
#define TENSORFLOW_CORE_COMMON_RUNTIME_GPU_GPU_UTIL_H_
#include "tensorflow/core/common_runtime/device.h"
#include "tensorflow/core/common_runtime/dma_helper.h"
#include "tensorflow/core/framework/tensor.h"
#include "tensorflow/core/lib/core/status.h"
#include "tensorflow/core/platform/stream_executor.h"
namespace tensorflow {
class RecvTensorResponse;
class TensorProto;
class GPUUtil {
public:
// "tensor" is GPU-local. "dev" is the hosting GPU.
// "device_context" should be the context of the GPU "_Send" op
// which provides the Tensor.
// Sets all necessary fields of "proto" by transferring value
// bytes from GPU to CPU RAM. "is_dead" indicates that the
// tensor is dead with an uninit value.
static void SetProtoFromGPU(const Tensor& tensor, Device* dev,
const DeviceContext* device_context,
TensorProto* proto, bool is_dead,
StatusCallback done);
// Copies the data in 'gpu_tensor' into 'cpu_tensor'.
// 'gpu_tensor''s backing memory must be on 'gpu_device' and
// 'cpu_tensor' must be allocated to be of the same size as
// 'gpu_tensor'. Synchronous: may block.
static void CopyGPUTensorToCPU(Device* gpu_device,
const DeviceContext* device_context,
const Tensor* gpu_tensor, Tensor* cpu_tensor,
StatusCallback done);
// Blocks until all operations queued on the stream associated with
// "gpu_device" at the time of the call have completed. Returns any
// error pending on the stream at completion.
static Status Sync(Device* gpu_device);
// Blocks until all operations queued on all streams associated with the
// corresponding GPU device at the time of call have completed.
// Returns any error pending on the stream at completion.
static Status SyncAll(Device* gpu_device);
// For debugging purpose, given a "device" and a "tensor" allocated
// on the device, return a string printing each byte in the tensor
// (up to a limit). "device" can be either a CPU or a GPU device.
static string MemoryDebugString(const Device* device, Tensor* tensor);
// Map a Tensor as a DeviceMemory object wrapping the given typed
// buffer.
//
// NOTE: will be removed soon, see StreamExecutorUtil::AsDeviceMemory
// instead.
template <typename T>
static se::DeviceMemory<T> AsDeviceMemory(const Tensor& t) {
T* ptr = reinterpret_cast<T*>(const_cast<void*>(DMAHelper::base(&t)));
return se::DeviceMemory<T>(se::DeviceMemoryBase(ptr, t.TotalBytes()));
}
// Computes a checksum over the contents of "tensor", which is allocated
// on "gpu_device".
static uint64 Checksum(Device* gpu_device,
const DeviceContext* device_context,
const Tensor& tensor);
// Computes a checksum over the contents of "tensor", which is allocated
// in local CPU RAM.
static uint64 Checksum(const Tensor& tensor);
static void CopyCPUTensorToGPU(const Tensor* cpu_tensor,
const DeviceContext* device_context,
Device* gpu_device, Tensor* gpu_tensor,
StatusCallback done);
static void DeviceToDeviceCopy(
DeviceContext* send_dev_context, DeviceContext* recv_dev_context,
Device* src, Device* dst, AllocatorAttributes src_alloc_attr,
AllocatorAttributes dst_alloc_attr, const Tensor* input, Tensor* output,
int dev_to_dev_stream_index, StatusCallback done);
// Deep-copying of GPU tensor on the same device.
// 'src_gpu_tensor''s and 'dst_gpu_tensor''s backing memory must be on
// 'gpu_device' and 'dst_cpu_tensor' must be allocated to be of the same
// size as 'src_gpu_tensor'.
static void CopyGPUTensorToSameGPU(Device* gpu_device,
const DeviceContext* device_context,
const Tensor* src_gpu_tensor,
Tensor* dst_gpu_tensor,
StatusCallback done);
};
} // namespace tensorflow
#endif // TENSORFLOW_CORE_COMMON_RUNTIME_GPU_GPU_UTIL_H_
@@ -0,0 +1,81 @@
/* Copyright 2015 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
#ifndef TENSORFLOW_CORE_COMMON_RUNTIME_GPU_DEVICE_CONTEXT_H_
#define TENSORFLOW_CORE_COMMON_RUNTIME_GPU_DEVICE_CONTEXT_H_
#include "tensorflow/core/common_runtime/device.h"
#include "tensorflow/core/framework/device_base.h"
#include "tensorflow/core/lib/gtl/inlined_vector.h"
namespace stream_executor {
class Stream;
} // namespace stream_executor
namespace tensorflow {
class GPUDeviceContext : public DeviceContext {
public:
// Does not take ownership of streams.
GPUDeviceContext(int stream_id, se::Stream* stream,
se::Stream* host_to_device_stream,
se::Stream* device_to_host_stream,
gtl::InlinedVector<se::Stream*, 4> device_to_device_stream)
: stream_id_(stream_id),
stream_(stream),
host_to_device_stream_(host_to_device_stream),
device_to_host_stream_(device_to_host_stream),
device_to_device_stream_(device_to_device_stream) {}
~GPUDeviceContext() override {}
se::Stream* stream() const override { return stream_; }
se::Stream* host_to_device_stream() const { return host_to_device_stream_; }
se::Stream* device_to_host_stream() const { return device_to_host_stream_; }
se::Stream* device_to_device_stream(int index) const {
return device_to_device_stream_[index % device_to_device_stream_.size()];
}
int stream_id() const { return stream_id_; }
void CopyCPUTensorToDevice(const Tensor* cpu_tensor, Device* device,
Tensor* device_tensor,
StatusCallback done) const override;
void CopyDeviceTensorToCPU(const Tensor* device_tensor, StringPiece edge_name,
Device* device, Tensor* cpu_tensor,
StatusCallback done) override;
void MaintainLifetimeOnStream(const Tensor* t,
se::Stream* stream) const override {}
Status ThenExecute(Device* device, se::Stream* stream,
std::function<void()> func) override;
private:
int stream_id_;
// The default primary stream to use for this context.
// All the memory belongs to this stream.
se::Stream* stream_;
// The stream to use for copying data from host into GPU.
se::Stream* host_to_device_stream_;
// The stream to use for copying data from GPU to host.
se::Stream* device_to_host_stream_;
// Streams to use for copying data between GPUs.
gtl::InlinedVector<se::Stream*, 4> device_to_device_stream_;
};
} // namespace tensorflow
#endif // TENSORFLOW_CORE_COMMON_RUNTIME_GPU_DEVICE_CONTEXT_H_
@@ -0,0 +1,215 @@
/* Copyright 2016 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
#ifndef TENSORFLOW_CORE_COMMON_RUNTIME_GRAPH_EXECUTION_STATE_H_
#define TENSORFLOW_CORE_COMMON_RUNTIME_GRAPH_EXECUTION_STATE_H_
#include <functional>
#include <memory>
#include <string>
#include <vector>
#include "tensorflow/core/common_runtime/build_graph_options.h"
#include "tensorflow/core/common_runtime/device.h"
#include "tensorflow/core/common_runtime/device_set.h"
#include "tensorflow/core/framework/graph.pb.h"
#include "tensorflow/core/graph/costmodel.h"
#include "tensorflow/core/graph/graph.h"
#include "tensorflow/core/lib/core/status.h"
#include "tensorflow/core/platform/macros.h"
#include "tensorflow/core/platform/types.h"
namespace tensorflow {
struct SessionOptions;
namespace subgraph {
struct RewriteGraphMetadata;
}
struct GraphExecutionStateOptions {
const DeviceSet* device_set = nullptr;
const SessionOptions* session_options = nullptr;
// A map from node name to device name, representing the unchangeable
// placement of stateful nodes.
std::unordered_map<string, string> stateful_placements;
};
// A ClientGraph is simply a sub-graph of the full graph as induced by
// BuildGraphOptions.
struct ClientGraph {
explicit ClientGraph(std::unique_ptr<FunctionLibraryDefinition> flib,
DataTypeVector feed_types, DataTypeVector fetch_types,
int64 collective_graph_key)
: flib_def(std::move(flib)),
graph(flib_def.get()),
feed_types(std::move(feed_types)),
fetch_types(std::move(fetch_types)),
collective_graph_key(collective_graph_key) {}
// Each client-graph gets its own function library since optimization passes
// post rewrite for execution might want to introduce new functions.
std::unique_ptr<FunctionLibraryDefinition> flib_def;
Graph graph;
DataTypeVector feed_types;
DataTypeVector fetch_types;
int64 collective_graph_key;
};
// GraphExecutionState is responsible for generating an
// executable ClientGraph from the original GraphDef that specifies
// the complete graph and from BuildGraphOptions which specifies
// input/output nodes.
//
// An executable Graph differs from a GraphDef by being Placed,
// meaning that each Node is assigned to a single Device in the
// available set.
//
// When GraphExecutionState is first constructed it instantiates
// a full Graph from the provided GraphDef, and places it, using only
// the static device assignments from the GraphDef. Nodes without are
// currently placed in a very naive way. Since stateful Nodes cannot
// be moved after initial placement, it is important that stateful
// Nodes get sensible initial device assignments in the graph
// definition.
//
// Subsequently, GraphExecutionState generates a SimpleClientGraph on
// demand, which is a sub-graph of the latest placement of the full
// Graph. MasterSession uses such a ClientGraph to execute one or
// more similar client requests.
//
// GraphExecutionState is thread-safe.
class GraphExecutionState {
public:
virtual ~GraphExecutionState();
// Creates a new `GraphExecutionState` for the given
// `graph_def`, which represents the entire graph for a session.
//
// N.B. This method uses `GraphDef::Swap()` and leaves `graph_def`
// in an undefined state. If it is necessary to use `*graph_def`
// after this call, make an explicit copy of the graph before
// calling this method.
static Status MakeForBaseGraph(
GraphDef* graph_def, const GraphExecutionStateOptions& options,
std::unique_ptr<GraphExecutionState>* out_state);
// Creates a new `GraphExecutionState` and `SimpleClientGraph`
// for the subgraph of `original_graph_def` defined by
// `subgraph_options`.
static Status MakeForPrunedGraph(
const FunctionDefLibrary& func_def_lib,
const GraphExecutionStateOptions& options,
const GraphDef& original_graph_def,
const BuildGraphOptions& subgraph_options,
std::unique_ptr<GraphExecutionState>* out_state,
std::unique_ptr<ClientGraph>* out_client_graph);
// Creates a new GraphExecutionState representing the
// concatenation of this graph, and the graph defined by
// "extension_def". The same name may not be used to define a node
// in both this graph and "extension_def".
//
// If successful, returns OK and the caller takes ownership of "*out".
// Otherwise returns an error and does not modify "*out".
//
// After calling `old_state->Extend()`, `old_state` may no longer be
// used.
//
// NOTE(mrry): This method respects the placement of stateful nodes in
// in *this, but currently does not transfer any other placement
// or cost model information to the new graph.
Status Extend(const GraphDef& extension_def,
std::unique_ptr<GraphExecutionState>* out) const;
// Builds a ClientGraph (a sub-graph of the full graph as induced by
// the Node set specified in "options"). If successful, returns OK
// and the caller takes the ownership of "*out". Otherwise, returns
// an error.
Status BuildGraph(const BuildGraphOptions& options,
std::unique_ptr<ClientGraph>* out);
// The graph returned by BuildGraph may contain only the pruned
// graph, whereas some clients may want access to the full graph.
const Graph* full_graph() { return graph_; }
// Returns the node with the given name, or null if it does not exist.
const Node* get_node_by_name(const string& name) const {
NodeNameToCostIdMap::const_iterator iter =
node_name_to_cost_id_map_.find(name);
if (iter != node_name_to_cost_id_map_.end()) {
return graph_->FindNodeId(iter->second);
} else {
return nullptr;
}
}
// Returns a reference to the current graph_def. Use must
// not extend beyond lifetime of GrahExecutionState object.
const GraphDef& original_graph_def() { return original_graph_def_; }
// Returns the map of stateful placements as a map of
// node name to placement string.
std::unordered_map<string, string> GetStatefulPlacements() const {
return stateful_placements_;
}
private:
GraphExecutionState(GraphDef* graph_def,
const GraphExecutionStateOptions& options);
Status InitBaseGraph(const BuildGraphOptions& options);
// Map of placed stateful nodes, i.e. nodes for which is_stateful()
// is true, such as "params" and "queue" nodes. Once placed these
// nodes can not be moved to a different device. Maps node names to
// device names.
std::unordered_map<string, string> stateful_placements_; // Immutable after
// ctor.
void SaveStatefulNodes(Graph* graph);
void RestoreStatefulNodes(Graph* graph);
// Extract the subset of the graph that needs to be run, adding feed/fetch
// ops as needed.
Status PruneGraph(const BuildGraphOptions& options, Graph* graph,
subgraph::RewriteGraphMetadata* out_rewrite_metadata);
Status OptimizeGraph(
const BuildGraphOptions& options, std::unique_ptr<Graph>* optimized_graph,
std::unique_ptr<FunctionLibraryDefinition>* optimized_flib);
GraphDef original_graph_def_; // Immutable after ctor.
const DeviceSet* device_set_; // Not owned
const SessionOptions* session_options_; // Not owned
// Map from name to Node for the full graph in placed_.
NodeNameToCostIdMap node_name_to_cost_id_map_;
// 'flib_def_' is initialized from the initial graph def's library,
// and may be updated by a graph optimization pass.
std::unique_ptr<FunctionLibraryDefinition> flib_def_;
// `rewrite_metadata_` is only set for GraphExecutionState
// objects created by `MakeForPrunedGraph()`.
std::unique_ptr<subgraph::RewriteGraphMetadata> rewrite_metadata_;
// The dataflow graph owned by this object.
Graph* graph_;
TF_DISALLOW_COPY_AND_ASSIGN(GraphExecutionState);
};
} // namespace tensorflow
#endif // TENSORFLOW_CORE_COMMON_RUNTIME_GRAPH_EXECUTION_STATE_H_
@@ -0,0 +1,68 @@
/* Copyright 2015 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
#ifndef TENSORFLOW_CORE_COMMON_RUNTIME_GRAPH_OPTIMIZER_H_
#define TENSORFLOW_CORE_COMMON_RUNTIME_GRAPH_OPTIMIZER_H_
#include "tensorflow/core/framework/function.h"
#include "tensorflow/core/graph/graph.h"
#include "tensorflow/core/lib/core/status.h"
#include "tensorflow/core/platform/env.h"
#include "tensorflow/core/protobuf/config.pb.h"
namespace tensorflow {
class GraphOptimizer {
public:
GraphOptimizer(const OptimizerOptions& opts);
~GraphOptimizer();
// Applies optimization passes specified in 'opts' to 'graph'.
// Maybe replace *graph with a new graph object. 'device' is device
// on which the 'graph' will execute. It's passed to the optimizers
// so that they can respect constraints if any, that should be
// respected.
//
// If shape_map is not null it maps from nodes in graph to partially-known
// shapes of their outputs, and may be used, e.g., in the constant folding
// pass. The use of shape_map implies that the mapping from node name to the
// vector of partial shapes of its outputs is stable, i.e., no optimization
// pass may replace a node with a different node of the same name that has a
// different number of outputs, or outputs with different known shapes.
// TODO(b/65453533) introduce a unique way to name nodes in a graph.
//
// If cse_consider_fn is not null then only nodes for which cse_consider_fn
// returns true will be considered for CSE.
// If cf_consider_fn is not null then only nodes for which cf_consider_fn
// returns true will be considered for CF.
void Optimize(
FunctionLibraryRuntime* runtime, Env* env, Device* device,
std::unique_ptr<Graph>* graph,
const std::unordered_map<string, std::vector<PartialTensorShape>>*
shape_map,
const std::function<bool(const Node*)>& cse_consider_fn = nullptr,
const std::function<bool(const Node*)>& cf_consider_fn = nullptr);
const OptimizerOptions& options() { return opts_; }
private:
OptimizerOptions opts_;
TF_DISALLOW_COPY_AND_ASSIGN(GraphOptimizer);
};
} // end namespace tensorflow
#endif // TENSORFLOW_CORE_COMMON_RUNTIME_GRAPH_OPTIMIZER_H_
@@ -0,0 +1,70 @@
/* Copyright 2016 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
#ifndef TENSORFLOW_CORE_COMMON_RUNTIME_GRAPH_RUNNER_H_
#define TENSORFLOW_CORE_COMMON_RUNTIME_GRAPH_RUNNER_H_
#include <memory>
#include <string>
#include <vector>
#include "tensorflow/core/common_runtime/device.h"
#include "tensorflow/core/framework/function.h"
#include "tensorflow/core/framework/tensor.h"
#include "tensorflow/core/graph/graph.h"
#include "tensorflow/core/lib/core/status.h"
#include "tensorflow/core/platform/env.h"
namespace tensorflow {
// GraphRunner takes a Graph, some inputs to feed, and some outputs
// to fetch and executes the graph required to feed and fetch the
// inputs and outputs.
//
// This class is only meant for internal use where one needs to
// partially evaluate inexpensive nodes in a graph, such as for shape
// inference or for constant folding. Because of its limited, simple
// use-cases, it executes all computation on the given device (CPU by default)
// and is not meant to be particularly lightweight, fast, or efficient.
class GraphRunner {
public:
// REQUIRES: `env` is not nullptr.
GraphRunner(Env* env);
// REQUIRES: 'device' is not nullptr. Not owned.
GraphRunner(Device* device);
~GraphRunner();
// Function semantics for `inputs`, `output_names` and `outputs`
// matches those from Session::Run().
//
// NOTE: The output tensors share lifetime with the GraphRunner, and could
// be destroyed once the GraphRunner is destroyed.
//
// REQUIRES: `graph`, `env`, and `outputs` are not nullptr.
// `function_library` may be nullptr.
typedef std::vector<std::pair<string, Tensor>> NamedTensorList;
Status Run(Graph* graph, FunctionLibraryRuntime* function_library,
const NamedTensorList& inputs,
const std::vector<string>& output_names,
std::vector<Tensor>* outputs);
private:
std::unique_ptr<Device> device_deleter_;
Device* const device_;
};
} // namespace tensorflow
#endif // TENSORFLOW_CORE_COMMON_RUNTIME_GRAPH_RUNNER_H_
@@ -0,0 +1,85 @@
/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
#ifndef TENSORFLOW_CORE_COMMON_RUNTIME_HIERARCHICAL_TREE_BROADCASTER_H_
#define TENSORFLOW_CORE_COMMON_RUNTIME_HIERARCHICAL_TREE_BROADCASTER_H_
#include <vector>
#include "tensorflow/core/common_runtime/base_collective_executor.h"
#include "tensorflow/core/framework/collective.h"
namespace tensorflow {
// Hierarchical tree-algorithm implementation of collective broadcast.
class HierarchicalTreeBroadcaster : public CollectiveImplementationInterface {
public:
HierarchicalTreeBroadcaster();
~HierarchicalTreeBroadcaster() override = default;
// Establishes the subdiv permutations needed for a hierarchical broadcast.
// If all devices are local, establishes a single subdiv comprising all
// devices. If any devices are on a different task, establishes n+1 subdivs
// for n tasks.
// The first subdiv comprises one device per task which gets the tensor on
// each task. Subdiv i+1 corresponds to a task-local tree-broadcast for task
// i.
Status InitializeCollectiveParams(CollectiveParams* col_params) override;
// Initializes members of CollectiveContext not yet initialized, i.e. device
// and device_locality. Also saves the CollectiveContext in this object.
Status InitializeCollectiveContext(CollectiveContext* col_ctx) override;
// Begins async execution of the hierarchical tree broadcast.
// Must be called in a blockable thread.
// TODO(b/80529858): remove the previous warning when we have a dedicated
// collective threadpool.
void Run(StatusCallback done) override;
// Returns the rank of the device from which this device should receive
// its value, -1 if no value should be received.
static int TreeRecvFrom(const CollectiveParams& cp, int subdiv);
// Populates targets with the ranks of the devices to which this device
// should forward the value.
static void TreeSendTo(const CollectiveParams& cp, int subdiv,
std::vector<int>* targets);
private:
// Get the task to which the device at `device_rank` belongs.
int GetDeviceTask(int device_rank, const std::vector<int>& dev_per_task);
// Sends `src_tensor` asynchronously from this device to device at `dst_rank`
// in `subdiv`. Calls `done` upon completion.
void DispatchSend(int subdiv, int dst_rank, int src_rank,
const Tensor* src_tensor, const StatusCallback& done);
// Receives a tensor into the memory buffer owned by `dst_tensor` at this
// device from device at `src_rank` in `subdiv`. Calls `done` upon
// completion.
void DispatchRecv(int subdiv, int src_rank, int dst_rank, Tensor* dst_tensor,
const StatusCallback& done);
// Executes the hierarchical broadcast defined by this op.
void RunTree();
CollectiveContext* col_ctx_; // Not owned
const CollectiveParams* col_params_; // Not owned
StatusCallback done_;
Status status_;
bool is_source_;
};
} // namespace tensorflow
#endif // TENSORFLOW_CORE_COMMON_RUNTIME_HIERARCHICAL_TREE_BROADCASTER_H_
@@ -0,0 +1,57 @@
/* Copyright 2015 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
#ifndef TENSORFLOW_CORE_COMMON_RUNTIME_LOCAL_DEVICE_H_
#define TENSORFLOW_CORE_COMMON_RUNTIME_LOCAL_DEVICE_H_
#include "tensorflow/core/common_runtime/device.h"
#include "tensorflow/core/framework/device_attributes.pb.h"
#include "tensorflow/core/platform/macros.h"
namespace tensorflow {
namespace test {
class Benchmark;
}
struct SessionOptions;
// This class is shared by ThreadPoolDevice and GPUDevice and
// initializes a shared Eigen compute device used by both. This
// should eventually be removed once we refactor ThreadPoolDevice and
// GPUDevice into more 'process-wide' abstractions.
class LocalDevice : public Device {
public:
LocalDevice(const SessionOptions& options,
const DeviceAttributes& attributes);
~LocalDevice() override;
private:
static bool use_global_threadpool_;
static void set_use_global_threadpool(bool use_global_threadpool) {
use_global_threadpool_ = use_global_threadpool;
}
struct EigenThreadPoolInfo;
std::unique_ptr<EigenThreadPoolInfo> owned_tp_info_;
friend class test::Benchmark;
TF_DISALLOW_COPY_AND_ASSIGN(LocalDevice);
};
} // namespace tensorflow
#endif // TENSORFLOW_CORE_COMMON_RUNTIME_LOCAL_DEVICE_H_
@@ -0,0 +1,39 @@
/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
#ifndef TENSORFLOW_CORE_COMMON_RUNTIME_LOWER_IF_OP_H_
#define TENSORFLOW_CORE_COMMON_RUNTIME_LOWER_IF_OP_H_
#include "tensorflow/core/common_runtime/optimization_registry.h"
#include "tensorflow/core/lib/core/status.h"
namespace tensorflow {
// Rewrite If ops to use switch and merge nodes instead.
class LowerIfOpPass : public GraphOptimizationPass {
public:
static const char* const kLowerUsingSwitchMergeAttr;
Status Run(const GraphOptimizationPassOptions& options) override;
private:
// Rewrite the given If node `n` in graph `g` to use the switch-merge
// form. `flib` should contain the branch functions referenced by `n`.
Status RewriteNode(Node* n, const FunctionLibraryDefinition& flib, Graph* g);
};
} // namespace tensorflow
#endif // TENSORFLOW_CORE_COMMON_RUNTIME_LOWER_IF_OP_H_
@@ -0,0 +1,37 @@
/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
#ifndef TENSORFLOW_CORE_COMMON_RUNTIME_LOWER_WHILE_OP_H_
#define TENSORFLOW_CORE_COMMON_RUNTIME_LOWER_WHILE_OP_H_
#include "tensorflow/core/common_runtime/optimization_registry.h"
#include "tensorflow/core/lib/core/status.h"
namespace tensorflow {
// Rewrite While ops to use lower level control flow primitives instead.
class LowerWhileOpPass : public GraphOptimizationPass {
public:
Status Run(const GraphOptimizationPassOptions& options) override;
private:
// Rewrite the given While node `n` in graph `g` to use the lower level
// primitives Enter, Exit, Switch, Merge and NextIteration.
Status RewriteNode(Node* n, Graph* g);
};
} // namespace tensorflow
#endif // TENSORFLOW_CORE_COMMON_RUNTIME_LOWER_WHILE_OP_H_
@@ -0,0 +1,48 @@
/* Copyright 2016 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
#ifndef TENSORFLOW_CORE_COMMON_RUNTIME_MEMORY_TYPES_H_
#define TENSORFLOW_CORE_COMMON_RUNTIME_MEMORY_TYPES_H_
#include "tensorflow/core/framework/memory_types.h"
#include "tensorflow/core/graph/graph.h"
#include "tensorflow/core/lib/core/status.h"
namespace tensorflow {
// Returns an error iff *g running on a single device of 'device_type'
// has memory type mismatch for any edge's source and destination.
Status ValidateMemoryTypes(const DeviceType& device_type, const Graph* g);
// Updates '*g' so that every edge's source and destination has
// compatible memory types by inserting proper HostSend/Recv and
// Send/HostRecv nodes. 'device_type' specifies the type of device on
// which '*g' is going to run on and that device has the name
// 'device_name'.
//
// Returns OK if '*g' is updated properly (ValidateMemoryTypes(g) must
// be OK). Otherwise, returns an error and '*g' may be in an
// invalidate state and the caller should discard it.
Status EnsureMemoryTypes(const DeviceType& device_type,
const string& device_name, Graph* g);
// Get the memory type for 'index'th output of node 'n' in graph 'g', when
// running on 'device_type'.
Status MemoryTypeForOutput(const DeviceType& device_type, const Graph* g,
const Node* n, int index, MemoryType* memory_type);
} // end namespace tensorflow
#endif // TENSORFLOW_CORE_COMMON_RUNTIME_MEMORY_TYPES_H_
@@ -0,0 +1,315 @@
/* Copyright 2017 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
// A simple CPU allocator that intercepts malloc/free calls from MKL library
// and redirects them to Tensorflow allocator
#ifndef TENSORFLOW_CORE_COMMON_RUNTIME_MKL_CPU_ALLOCATOR_H_
#define TENSORFLOW_CORE_COMMON_RUNTIME_MKL_CPU_ALLOCATOR_H_
#ifdef INTEL_MKL
#include <cstdlib>
#include "tensorflow/core/common_runtime/bfc_allocator.h"
#include "tensorflow/core/common_runtime/pool_allocator.h"
#include "tensorflow/core/lib/strings/numbers.h"
#include "tensorflow/core/lib/strings/str_util.h"
#include "tensorflow/core/platform/mem.h"
#include "tensorflow/core/platform/numa.h"
#ifndef INTEL_MKL_DNN_ONLY
#include "i_malloc.h"
#endif
#ifdef _WIN32
typedef unsigned int uint;
#endif
namespace tensorflow {
class MklSubAllocator : public BasicCPUAllocator {
public:
MklSubAllocator() : BasicCPUAllocator(port::kNUMANoAffinity, {}, {}) {}
~MklSubAllocator() override {}
};
// CPU allocator that handles small-size allocations by calling
// suballocator directly. Mostly, it is just a wrapper around a suballocator
// (that calls malloc and free directly) with support for bookkeeping.
class MklSmallSizeAllocator : public Allocator {
public:
MklSmallSizeAllocator(SubAllocator* sub_allocator, size_t total_memory,
const string& name)
: sub_allocator_(sub_allocator), name_(name) {
stats_.bytes_limit = total_memory;
}
~MklSmallSizeAllocator() override {}
TF_DISALLOW_COPY_AND_ASSIGN(MklSmallSizeAllocator);
inline string Name() override { return name_; }
void* AllocateRaw(size_t alignment, size_t num_bytes) override {
void* ptr = sub_allocator_->Alloc(alignment, num_bytes);
if (ptr != nullptr) {
std::pair<void*, size_t> map_val(ptr, num_bytes);
mutex_lock l(mutex_);
// Check that insertion in the hash map was successful.
CHECK(map_.insert(map_val).second);
// Increment statistics for small-size allocations.
IncrementStats(num_bytes);
}
return ptr;
}
void DeallocateRaw(void* ptr) override {
if (ptr == nullptr) {
LOG(ERROR) << "tried to deallocate nullptr";
return;
}
mutex_lock l(mutex_);
auto map_iter = map_.find(ptr);
if (map_iter != map_.end()) {
// Call free visitors.
size_t dealloc_bytes = map_iter->second;
sub_allocator_->Free(ptr, dealloc_bytes);
DecrementStats(dealloc_bytes);
map_.erase(map_iter);
} else {
LOG(ERROR) << "tried to deallocate invalid pointer";
return;
}
}
inline bool IsSmallSizeAllocation(const void* ptr) const {
mutex_lock l(mutex_);
return map_.find(ptr) != map_.end();
}
void GetStats(AllocatorStats* stats) override {
mutex_lock l(mutex_);
*stats = stats_;
}
void ClearStats() override {
mutex_lock l(mutex_);
stats_.Clear();
}
private:
// Increment statistics for the allocator handling small allocations.
inline void IncrementStats(size_t alloc_size)
EXCLUSIVE_LOCKS_REQUIRED(mutex_) {
++stats_.num_allocs;
stats_.bytes_in_use += alloc_size;
stats_.max_bytes_in_use =
std::max(stats_.max_bytes_in_use, stats_.bytes_in_use);
stats_.max_alloc_size =
std::max(alloc_size, static_cast<size_t>(stats_.max_alloc_size));
}
// Decrement statistics for the allocator handling small allocations.
inline void DecrementStats(size_t dealloc_size)
EXCLUSIVE_LOCKS_REQUIRED(mutex_) {
stats_.bytes_in_use -= dealloc_size;
}
SubAllocator* sub_allocator_; // Not owned by this class.
// Mutex for protecting updates to map of allocations.
mutable mutex mutex_;
// Allocator name
string name_;
// Hash map to keep track of "small" allocations
// We do not use BFC allocator for small allocations.
std::unordered_map<const void*, size_t> map_ GUARDED_BY(mutex_);
// Allocator stats for small allocs
AllocatorStats stats_ GUARDED_BY(mutex_);
};
/// CPU allocator for MKL that wraps BFC allocator and intercepts
/// and redirects memory allocation calls from MKL.
class MklCPUAllocator : public Allocator {
public:
// Constructor and other standard functions
/// Environment variable that user can set to upper bound on memory allocation
static constexpr const char* kMaxLimitStr = "TF_MKL_ALLOC_MAX_BYTES";
/// Default upper limit on allocator size - 64GB
static constexpr size_t kDefaultMaxLimit = 64LL << 30;
MklCPUAllocator() { TF_CHECK_OK(Initialize()); }
~MklCPUAllocator() override {
delete small_size_allocator_;
delete large_size_allocator_;
}
Status Initialize() {
VLOG(2) << "MklCPUAllocator: In MklCPUAllocator";
// Set upper bound on memory allocation to physical RAM available on the
// CPU unless explicitly specified by user
uint64 max_mem_bytes = kDefaultMaxLimit;
#if defined(_SC_PHYS_PAGES) && defined(_SC_PAGESIZE)
max_mem_bytes =
(uint64)sysconf(_SC_PHYS_PAGES) * (uint64)sysconf(_SC_PAGESIZE);
#endif
char* user_mem_bytes = getenv(kMaxLimitStr);
if (user_mem_bytes != NULL) {
uint64 user_val = 0;
if (!strings::safe_strtou64(user_mem_bytes, &user_val)) {
return errors::InvalidArgument("Invalid memory limit (", user_mem_bytes,
") specified for MKL allocator through ",
kMaxLimitStr);
}
#if defined(_SC_PHYS_PAGES) && defined(_SC_PAGESIZE)
if (user_val > max_mem_bytes) {
LOG(WARNING) << "The user specified a memory limit " << kMaxLimitStr
<< "=" << user_val
<< " greater than available physical memory: "
<< max_mem_bytes
<< ". This could significantly reduce performance!";
}
#endif
max_mem_bytes = user_val;
}
VLOG(1) << "MklCPUAllocator: Setting max_mem_bytes: " << max_mem_bytes;
sub_allocator_ = new MklSubAllocator();
// SubAllocator is owned by BFCAllocator, so we do not need to deallocate
// it in MklSmallSizeAllocator.
small_size_allocator_ =
new MklSmallSizeAllocator(sub_allocator_, max_mem_bytes, kName);
large_size_allocator_ =
new BFCAllocator(sub_allocator_, max_mem_bytes, kAllowGrowth, kName);
#ifndef INTEL_MKL_DNN_ONLY
// For redirecting all allocations from MKL to this allocator
// From: http://software.intel.com/en-us/node/528565
i_malloc = MallocHook;
i_calloc = CallocHook;
i_realloc = ReallocHook;
i_free = FreeHook;
#endif
return Status::OK();
}
inline string Name() override { return kName; }
inline void* AllocateRaw(size_t alignment, size_t num_bytes) override {
// If the allocation size is less than threshold, call small allocator,
// otherwise call large-size allocator (BFC). We found that BFC allocator
// does not deliver good performance for small allocations when
// inter_op_parallelism_threads is high.
return (num_bytes < kSmallAllocationsThreshold)
? small_size_allocator_->AllocateRaw(alignment, num_bytes)
: large_size_allocator_->AllocateRaw(alignment, num_bytes);
}
inline void DeallocateRaw(void* ptr) override {
// Check if ptr is for "small" allocation. If it is, then call Free
// directly. Otherwise, call BFC to handle free.
if (small_size_allocator_->IsSmallSizeAllocation(ptr)) {
small_size_allocator_->DeallocateRaw(ptr);
} else {
large_size_allocator_->DeallocateRaw(ptr);
}
}
void GetStats(AllocatorStats* stats) override {
AllocatorStats l_stats, s_stats;
small_size_allocator_->GetStats(&s_stats);
large_size_allocator_->GetStats(&l_stats);
// Combine statistics from small-size and large-size allocator.
stats->num_allocs = l_stats.num_allocs + s_stats.num_allocs;
stats->bytes_in_use = l_stats.bytes_in_use + s_stats.bytes_in_use;
stats->max_bytes_in_use =
l_stats.max_bytes_in_use + s_stats.max_bytes_in_use;
// Since small-size allocations go to MklSmallSizeAllocator,
// max_alloc_size from large_size_allocator would be the maximum
// size allocated by MklCPUAllocator.
stats->max_alloc_size = l_stats.max_alloc_size;
stats->bytes_limit = std::max(s_stats.bytes_limit, l_stats.bytes_limit);
}
void ClearStats() override {
small_size_allocator_->ClearStats();
large_size_allocator_->ClearStats();
}
private:
// Hooks provided by this allocator for memory allocation routines from MKL
static inline void* MallocHook(size_t size) {
VLOG(3) << "MklCPUAllocator: In MallocHook";
return cpu_allocator()->AllocateRaw(kAlignment, size);
}
static inline void FreeHook(void* ptr) {
VLOG(3) << "MklCPUAllocator: In FreeHook";
cpu_allocator()->DeallocateRaw(ptr);
}
static inline void* CallocHook(size_t num, size_t size) {
Status s = Status(error::Code::UNIMPLEMENTED,
"Unimplemented case for hooking MKL function.");
TF_CHECK_OK(s); // way to assert with an error message
return nullptr; // return a value and make static code analyzers happy
}
static inline void* ReallocHook(void* ptr, size_t size) {
Status s = Status(error::Code::UNIMPLEMENTED,
"Unimplemented case for hooking MKL function.");
TF_CHECK_OK(s); // way to assert with an error message
return nullptr; // return a value and make static code analyzers happy
}
// Do we allow growth in BFC Allocator
static const bool kAllowGrowth = true;
// Name
static constexpr const char* kName = "mklcpu";
// The alignment that we need for the allocations
static constexpr const size_t kAlignment = 64;
Allocator* large_size_allocator_; // owned by this class
MklSmallSizeAllocator* small_size_allocator_; // owned by this class.
SubAllocator* sub_allocator_; // not owned by this class
// Size in bytes that defines the upper-bound for "small" allocations.
// Any allocation below this threshold is "small" allocation.
static constexpr const size_t kSmallAllocationsThreshold = 4096;
// Prevent copying and assignment
TF_DISALLOW_COPY_AND_ASSIGN(MklCPUAllocator);
};
} // namespace tensorflow
#endif // INTEL_MKL
#endif // TENSORFLOW_CORE_COMMON_RUNTIME_MKL_CPU_ALLOCATOR_H_
@@ -0,0 +1,145 @@
/* Copyright 2016 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
// Classes to maintain a static registry of whole-graph optimization
// passes to be applied by the Session when it initializes a graph.
#ifndef TENSORFLOW_CORE_COMMON_RUNTIME_OPTIMIZATION_REGISTRY_H_
#define TENSORFLOW_CORE_COMMON_RUNTIME_OPTIMIZATION_REGISTRY_H_
#include <functional>
#include <map>
#include <vector>
#include "tensorflow/core/common_runtime/device_set.h"
#include "tensorflow/core/framework/function.h"
#include "tensorflow/core/graph/costmodel.h"
#include "tensorflow/core/graph/graph.h"
namespace tensorflow {
struct SessionOptions;
// All the parameters used by an optimization pass are packaged in
// this struct. They should be enough for the optimization pass to use
// as a key into a state dictionary if it wants to keep state across
// calls.
struct GraphOptimizationPassOptions {
string session_handle;
const SessionOptions* session_options = nullptr;
const CostModel* cost_model = nullptr;
FunctionLibraryDefinition* flib_def = nullptr; // Not owned.
// The DeviceSet contains all the devices known to the system and is
// filled in for optimizations run by the session master, i.e.,
// PRE_PLACEMENT, POST_PLACEMENT, and POST_REWRITE_FOR_EXEC. It is
// nullptr for POST_PARTITIONING optimizations which are run at the
// workers.
const DeviceSet* device_set = nullptr; // Not owned.
// The graph to optimize, for optimization passes that run before
// partitioning. Null for post-partitioning passes.
// An optimization pass may replace *graph with a new graph object.
std::unique_ptr<Graph>* graph = nullptr;
// Graphs for each partition, if running post-partitioning. Optimization
// passes may alter the graphs, but must not add or remove partitions.
// Null for pre-partitioning passes.
std::unordered_map<string, std::unique_ptr<Graph>>* partition_graphs =
nullptr;
};
// Optimization passes are implemented by inheriting from
// GraphOptimizationPass.
class GraphOptimizationPass {
public:
virtual ~GraphOptimizationPass() {}
virtual Status Run(const GraphOptimizationPassOptions& options) = 0;
void set_name(const string& name) { name_ = name; }
string name() const { return name_; }
private:
// The name of the opitimization pass, which is the same as the inherited
// class name.
string name_;
};
// The key is a 'phase' number. Phases are executed in increasing
// order. Within each phase the order of passes is undefined.
typedef std::map<int, std::vector<std::unique_ptr<GraphOptimizationPass>>>
GraphOptimizationPasses;
// A global OptimizationPassRegistry is used to hold all passes.
class OptimizationPassRegistry {
public:
// Groups of passes are run at different points in initialization.
enum Grouping {
PRE_PLACEMENT, // after cost model assignment, before placement.
POST_PLACEMENT, // after placement.
POST_REWRITE_FOR_EXEC, // after re-write using feed/fetch endpoints.
POST_PARTITIONING, // after partitioning
};
// Add an optimization pass to the registry.
void Register(Grouping grouping, int phase,
std::unique_ptr<GraphOptimizationPass> pass);
// Run all passes in grouping, ordered by phase, with the same
// options.
Status RunGrouping(Grouping grouping,
const GraphOptimizationPassOptions& options);
// Returns the global registry of optimization passes.
static OptimizationPassRegistry* Global();
// Prints registered optimization passes for debugging.
void LogGrouping(Grouping grouping, int vlog_level);
void LogAllGroupings(int vlog_level);
private:
std::map<Grouping, GraphOptimizationPasses> groups_;
};
namespace optimization_registration {
class OptimizationPassRegistration {
public:
OptimizationPassRegistration(OptimizationPassRegistry::Grouping grouping,
int phase,
std::unique_ptr<GraphOptimizationPass> pass,
string optimization_pass_name) {
pass->set_name(optimization_pass_name);
OptimizationPassRegistry::Global()->Register(grouping, phase,
std::move(pass));
}
};
} // namespace optimization_registration
#define REGISTER_OPTIMIZATION(grouping, phase, optimization) \
REGISTER_OPTIMIZATION_UNIQ_HELPER(__COUNTER__, grouping, phase, optimization)
#define REGISTER_OPTIMIZATION_UNIQ_HELPER(ctr, grouping, phase, optimization) \
REGISTER_OPTIMIZATION_UNIQ(ctr, grouping, phase, optimization)
#define REGISTER_OPTIMIZATION_UNIQ(ctr, grouping, phase, optimization) \
static ::tensorflow::optimization_registration::OptimizationPassRegistration \
register_optimization_##ctr( \
grouping, phase, \
::std::unique_ptr<::tensorflow::GraphOptimizationPass>( \
new optimization()), \
#optimization)
} // namespace tensorflow
#endif // TENSORFLOW_CORE_COMMON_RUNTIME_OPTIMIZATION_REGISTRY_H_
@@ -0,0 +1,331 @@
#ifndef TENSORFLOW_CORE_COMMON_RUNTIME_PENDING_COUNTS_H_
#define TENSORFLOW_CORE_COMMON_RUNTIME_PENDING_COUNTS_H_
/* Copyright 2015 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
#include "tensorflow/core/lib/gtl/flatmap.h"
#include "tensorflow/core/lib/hash/hash.h"
#include "tensorflow/core/platform/logging.h"
#include "tensorflow/core/platform/macros.h"
#include "tensorflow/core/util/port.h"
namespace tensorflow {
// PendingCounts is an internal helper class to keep track of pending and
// dead counts for nodes, for use in the ExecutorState module. It
// holds a map from Handles to various counts for that handle. This
// information is needed per frame iteration. The amount of memory
// needed for an iteration is the same across all executions of the
// iteration. The memory amount and handles are precomputed at startup
// using a Layout object.
//
// PendingCounts::Layout layout;
// std::vector<PendingCounts::Handle> h(C);
// for (int id = 0; id < C; id++) {
// h[id] = r.AddHandle(max_pending[id], max_dead[id]);
// }
//
// When we actually want to start an iteration we first create a
// PendingCounts object and then index into it using the precomputed
// handles:
// PendingCounts counts(layout);
// ...
// counts.decrement_pending(h[id], 1);
class PendingCounts {
public:
// The state machine for a node's execution.
enum NodeState {
// The pending count for the node > 0.
PENDING_NOTREADY,
// The pending count for the node == 0, but the node has not
// started executing.
PENDING_READY,
// The node has started executing.
STARTED,
// The node has finished executing.
COMPLETED
};
// An opaque handle indicating where in the PendingCounts data structure
// the appropriate count information can be found.
class Handle;
// Given a node that needs to represent counts no larger than the
// specified "max_pending_count" and "max_dead_count", create a
// handle that can be passed to various PendingCounts routines
// to retrieve the count data for this node.
class Layout {
public:
Handle CreateHandle(size_t max_pending_count, size_t max_dead_count);
private:
friend class PendingCounts;
int next_offset_ = 0; // Next byte offset to allocate
};
// Create a new PendingCounts object that can hold the state of
// all the Handles allocated from "final_allocator".
explicit PendingCounts(Layout layout)
: num_bytes_(layout.next_offset_), bytes_(new char[num_bytes_]) {}
// Create a new PendingCounts object with the same layout and counts
// as "other".
explicit PendingCounts(const PendingCounts& other)
: num_bytes_(other.num_bytes_), bytes_(new char[num_bytes_]) {
CHECK_EQ(uintptr_t(bytes_) % alignof(LargeCounts), 0);
memcpy(bytes_, other.bytes_, other.num_bytes_);
}
~PendingCounts() { delete[] bytes_; }
void set_initial_count(Handle h, size_t pending_count) {
if (h.is_large_) {
LargeCounts* c = Large(h);
c->pending = pending_count;
c->dead_count = 0;
c->has_started = 0;
} else {
PackedCounts* c = Packed(h);
DCHECK_LE(pending_count, kMaxCountForPackedCounts);
c->pending = pending_count;
c->dead_count = 0;
c->has_started = 0;
}
}
NodeState node_state(Handle h) {
if (h.is_large_) {
return NodeStateForStruct(Large(h));
} else {
return NodeStateForStruct(Packed(h));
}
}
void mark_started(Handle h) {
DCHECK_EQ(pending(h), 0);
if (h.is_large_) {
LargeCounts* c = Large(h);
DCHECK_EQ(c->has_started, 0);
c->has_started = 1;
} else {
PackedCounts* c = Packed(h);
DCHECK_EQ(c->has_started, 0);
c->has_started = 1;
}
}
void mark_completed(Handle h) {
if (h.is_large_) {
LargeCounts* c = Large(h);
DCHECK_EQ(c->has_started, 1);
c->pending = 1;
} else {
PackedCounts* c = Packed(h);
DCHECK_EQ(c->has_started, 1);
c->pending = 1;
}
}
int pending(Handle h) {
if (h.is_large_) {
LargeCounts* c = Large(h);
if (PENDING_NOTREADY == NodeStateForStruct(c)) {
return c->pending;
} else {
// The pending count encodes the state once the node has
// started, so just return 0.
return 0;
}
} else {
PackedCounts* c = Packed(h);
if (PENDING_NOTREADY == NodeStateForStruct(c)) {
return c->pending;
} else {
// The pending count encodes the state once the node has
// started, so just return 0.
return 0;
}
}
}
int decrement_pending(Handle h, int v) {
DCHECK_GE(pending(h), v);
if (h.is_large_) {
LargeCounts* c = Large(h);
c->pending -= v;
return c->pending;
} else {
PackedCounts* c = Packed(h);
c->pending -= v;
return c->pending;
}
}
// Mark a merge node as live
// REQUIRES: Node corresponding to "h" is a merge node
void mark_live(Handle h) {
if (h.is_large_) {
LargeCounts* c = Large(h);
// Only do anything if the node hasn't already started executing.
if (PENDING_NOTREADY == NodeStateForStruct(c)) {
c->pending &= ~static_cast<int>(0x1);
}
} else {
PackedCounts* c = Packed(h);
// Only do anything if the node hasn't already started executing.
if (PENDING_NOTREADY == NodeStateForStruct(c)) {
static_assert(7 == kMaxCountForPackedCounts,
"Live flag incorrect for max packed count");
c->pending &= 0x6;
}
}
}
int dead_count(Handle h) {
int r = h.is_large_ ? Large(h)->dead_count : Packed(h)->dead_count;
return r;
}
void increment_dead_count(Handle h) {
if (h.is_large_) {
LargeCounts* c = Large(h);
if (PENDING_NOTREADY == NodeStateForStruct(c)) {
c->dead_count++;
}
} else {
PackedCounts* c = Packed(h);
if (PENDING_NOTREADY == NodeStateForStruct(c)) {
DCHECK_LT(c->dead_count, kMaxCountForPackedCounts);
c->dead_count++;
}
}
}
// A streamlined routine that does several pieces of bookkeeping at
// once. Equivalent to:
// if (increment_dead) increment_dead_count(h);
// decrement_pending(h, 1);
// *pending_result = pending(h);
// *dead_result = dead_count(h);
void adjust_for_activation(Handle h, bool increment_dead, int* pending_result,
int* dead_result) {
DCHECK_GE(pending(h), 1);
if (h.is_large_) {
adjust_for_activation_shared(Large(h), increment_dead, pending_result,
dead_result);
} else {
adjust_for_activation_shared(Packed(h), increment_dead, pending_result,
dead_result);
}
}
class Handle {
public:
Handle() : byte_offset_(0), is_large_(0) {}
private:
friend class PendingCounts;
int byte_offset_ : 31; // Byte offset of the rep in PendingCounts object
bool is_large_ : 1; // If true, rep is LargeCounts; otherwise PackedCounts
};
private:
template <typename T>
inline void adjust_for_activation_shared(T* c, bool increment_dead,
int* pending_result,
int* dead_result) {
if (increment_dead) {
if (PENDING_NOTREADY == NodeStateForStruct(c)) {
c->dead_count++;
}
}
c->pending -= 1;
*dead_result = c->dead_count;
*pending_result = c->pending;
}
// We keep track of the pending count and dead input count for each
// graph node. The representation used here is designed to be cache
// efficient for graphs with large numbers of nodes, where most
// nodes have relatively small maximum pending counts (e.g. for one
// LSTM model, 99% of 5000+ nodes had in-degrees of 3 or less). We
// use one byte to hold both the pending and dead count for a node
// where these together can fit in one byte, and we use a hash table
// to handle the rare node ids that need larger counts than this.
// Each frame in this subgraph has its own PendingCounts.
// We use 3 bits each for dead_count and pending.
static const int kMaxCountForPackedCounts = 7;
// Most counts are small, so we pack a pending count and a dead
// count into 3 bits each, use 1 bit to indicate that the node has
// started computing.
struct PackedCounts {
uint8 pending : 3;
uint8 dead_count : 3;
uint8 has_started : 1;
};
struct LargeCounts {
uint32 pending;
uint32 dead_count : 31;
uint8 has_started : 1;
};
template <typename T>
NodeState NodeStateForStruct(T* c) const {
if (c->has_started) {
return (c->pending == 0) ? STARTED : COMPLETED;
} else {
return (c->pending == 0) ? PENDING_READY : PENDING_NOTREADY;
}
}
inline LargeCounts* Large(Handle h) {
DCHECK(h.is_large_);
DCHECK_LE(h.byte_offset_ + sizeof(LargeCounts), num_bytes_);
DCHECK_EQ(h.byte_offset_ % alignof(LargeCounts), 0);
return reinterpret_cast<LargeCounts*>(bytes_ + h.byte_offset_);
}
inline PackedCounts* Packed(Handle h) {
DCHECK(!h.is_large_);
DCHECK_LE(h.byte_offset_ + sizeof(PackedCounts), num_bytes_);
return reinterpret_cast<PackedCounts*>(bytes_ + h.byte_offset_);
}
const int num_bytes_; // Just for bounds checking in debug mode
char* bytes_; // Array of num_bytes_ bytes
void operator=(const PendingCounts&) = delete;
};
inline PendingCounts::Handle PendingCounts::Layout::CreateHandle(
size_t max_pending_count, size_t max_dead_count) {
Handle result;
if ((max_pending_count > kMaxCountForPackedCounts) ||
(max_dead_count > kMaxCountForPackedCounts)) {
int B = sizeof(LargeCounts);
// Round byte offset to proper alignment
DCHECK_GE(sizeof(LargeCounts), alignof(LargeCounts));
int64 offset = ((static_cast<int64>(next_offset_) + B - 1) / B) * B;
result.byte_offset_ = offset;
result.is_large_ = true;
next_offset_ = result.byte_offset_ + B;
} else {
result.byte_offset_ = next_offset_;
result.is_large_ = false;
DCHECK_EQ(sizeof(PackedCounts), 1);
next_offset_ += sizeof(PackedCounts);
}
return result;
}
} // end namespace tensorflow
#endif // TENSORFLOW_CORE_COMMON_RUNTIME_PENDING_COUNTS_H_
@@ -0,0 +1,101 @@
/* Copyright 2015 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
#ifndef TENSORFLOW_CORE_COMMON_RUNTIME_PLACER_H_
#define TENSORFLOW_CORE_COMMON_RUNTIME_PLACER_H_
#include <string>
#include <unordered_map>
#include "tensorflow/core/common_runtime/device_set.h"
#include "tensorflow/core/graph/graph.h"
#include "tensorflow/core/lib/core/status.h"
#include "tensorflow/core/platform/macros.h"
#include "tensorflow/core/platform/types.h"
#include "tensorflow/core/public/session_options.h"
#include "tensorflow/core/util/device_name_utils.h"
namespace tensorflow {
// A placement algorithm that assigns the nodes of the given Graph to
// devices the given DeviceSet, respecting the following constraints:
//
// 1. Existing device assignments remain unchanged.
// 2. Requested (partial or complete) device specifications given by device name
// for each node are granted.
// 3. Nodes connected by edges of a reference type are colocated on
// the same device.
// 4. Given nodes "A" and "B", if node "B" has a colocation group
// "@loc:A", nodes "A" and "B" will be colocated on the same device.
//
// The implementation builds a constraint graph with the same set of
// nodes, and edges that represent colocation constraints between
// nodes. Each connected component in the resulting constraint graph
// is then assigned to a set of valid devices.
//
// Run() will finally assign the device to each node given the list of
// possible devices.
//
// TODO(mrry): "Soft" constraints, such as "place node 'x' as close as
// possible to node 'y' while respecting the other constraints"?
// TODO(mrry): Create a common interface for this and the other
// placement algorithms so that they may be injected into the graph
// builder.
class Placer {
public:
// A map from graph node names to numerical IDs (in a Graph object).
typedef std::unordered_map<string, int> NodeNameToIdMap;
// Creates an instance of the Placer algorithm for the given
// Graph "graph" (nodes in which may or may not be assigned) on the
// given DeviceSet "devices".
//
// The "graph", and "devices" pointer arguments
// are borrowed by this Placer, and must outlive it.
Placer(Graph* graph, const DeviceSet* devices, const SessionOptions* options);
Placer(Graph* graph, const DeviceSet* devices);
~Placer();
// Assigns each node in this Placer's graph to a device in its
// set of devices.
//
// This method is not thread-safe.
// Run() may be invoked at most once.
Status Run();
private:
// Returns true if the device type of 'candidate_device_name' is
// found in 'devices'.
bool CanAssignToDevice(const string& candidate_device_name,
const std::vector<Device*>& devices) const;
// Assigns 'node's devices to 'assigned_device', and logs the
// placement if the SessionOptions entry in 'options_' requests it.
void AssignAndLog(int assigned_device, Node* node) const;
void LogDeviceAssignment(const Node* node) const;
Graph* const graph_; // Not owned.
const DeviceSet* const devices_; // Not owned.
const SessionOptions* options_; // Not owned.
const bool log_device_placement_;
TF_DISALLOW_COPY_AND_ASSIGN(Placer);
};
} // namespace tensorflow
#endif // TENSORFLOW_CORE_COMMON_RUNTIME_PLACER_H_
@@ -0,0 +1,171 @@
/* Copyright 2015 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
#ifndef TENSORFLOW_CORE_COMMON_RUNTIME_POOL_ALLOCATOR_H_
#define TENSORFLOW_CORE_COMMON_RUNTIME_POOL_ALLOCATOR_H_
// Simple LRU pool allocators for various flavors of CPU RAM.
#include <atomic>
#include <map>
#include <memory>
#include <vector>
#include "tensorflow/core/framework/allocator.h"
#include "tensorflow/core/lib/core/bits.h"
#include "tensorflow/core/platform/logging.h"
#include "tensorflow/core/platform/macros.h"
#include "tensorflow/core/platform/mutex.h"
#include "tensorflow/core/platform/types.h"
namespace tensorflow {
// Interface of an object that rounds up integers.
class RoundUpInterface {
public:
virtual ~RoundUpInterface() {}
virtual size_t RoundUp(size_t num_bytes) = 0;
};
// Size-limited pool of memory buffers obtained from a SubAllocator
// instance. Pool eviction policy is LRU.
class PoolAllocator : public Allocator {
public:
// "pool_size_limit" is the maximum number of returned, re-usable
// memory buffers to keep in the pool. If pool_size_limit == 0, the
// pool is effectively a thin wrapper around the allocator.
// If "auto_resize" is true, then the pool_size_limit will gradually
// be raised so that deallocations happen very rarely, if at all.
// Transitory start-up objects may deallocate, but the long-term
// working-set should not. Auto-resizing can raise pool_size_limit
// but will never lower it.
// "allocator" is the object that performs the underlying memory
// malloc/free operations. This object takes ownership of allocator.
PoolAllocator(size_t pool_size_limit, bool auto_resize,
SubAllocator* allocator, RoundUpInterface* size_rounder,
string name);
~PoolAllocator() override;
string Name() override { return name_; }
void* AllocateRaw(size_t alignment, size_t num_bytes) override;
void DeallocateRaw(void* ptr) override;
// Allocate an unused memory region of size "num_bytes". Fetch from
// the pool if available, otherwise call allocator_.
void* Get(size_t num_bytes);
// Return a no-longer needed memory region to the pool. It is an error
// to deference "ptr" after this call. If the pool is full, the least
// recently used region will be deallocated.
void Put(void* ptr, size_t num_bytes);
// Reset the pool to empty.
void Clear();
// The following accessors permit monitoring the effectiveness of
// the pool at avoiding repeated malloc/frees on the underlying
// allocator. Read locks are not taken on the theory that value
// consistency with other threads is not important.
// Number of Get() requests satisfied from pool.
int64 get_from_pool_count() const NO_THREAD_SAFETY_ANALYSIS {
return get_from_pool_count_;
}
// Number of Put() requests.
int64 put_count() const NO_THREAD_SAFETY_ANALYSIS { return put_count_; }
// Number of Get() requests requiring a fresh allocation.
int64 allocated_count() const NO_THREAD_SAFETY_ANALYSIS {
return allocated_count_;
}
// Number of pool evictions.
int64 evicted_count() const NO_THREAD_SAFETY_ANALYSIS {
return evicted_count_;
}
// Current size limit.
size_t size_limit() const NO_THREAD_SAFETY_ANALYSIS {
return pool_size_limit_;
}
void GetStats(AllocatorStats* stats) override { stats->Clear(); }
private:
struct PtrRecord {
void* ptr;
size_t num_bytes;
PtrRecord* prev;
PtrRecord* next;
};
// Remove "pr" from the double-linked LRU list.
void RemoveFromList(PtrRecord* pr) EXCLUSIVE_LOCKS_REQUIRED(mutex_);
// Add "pr" to the head of the double-linked LRU list.
void AddToList(PtrRecord* pr) EXCLUSIVE_LOCKS_REQUIRED(mutex_);
// Delete the least recently used record.
void EvictOne() EXCLUSIVE_LOCKS_REQUIRED(mutex_);
const string name_;
const bool has_size_limit_;
const bool auto_resize_;
size_t pool_size_limit_;
std::unique_ptr<SubAllocator> allocator_;
std::unique_ptr<RoundUpInterface> size_rounder_;
mutex mutex_;
std::multimap<const size_t, PtrRecord*> pool_ GUARDED_BY(mutex_);
PtrRecord* lru_head_ GUARDED_BY(mutex_) = nullptr;
PtrRecord* lru_tail_ GUARDED_BY(mutex_) = nullptr;
int64 get_from_pool_count_ GUARDED_BY(mutex_) = 0;
int64 put_count_ GUARDED_BY(mutex_) = 0;
int64 allocated_count_ GUARDED_BY(mutex_) = 0;
int64 evicted_count_ GUARDED_BY(mutex_) = 0;
};
// Do-nothing rounder. Passes through sizes unchanged.
class NoopRounder : public RoundUpInterface {
public:
size_t RoundUp(size_t num_bytes) override { return num_bytes; }
};
// Power of 2 rounder: rounds up to nearest power of 2 size.
class Pow2Rounder : public RoundUpInterface {
public:
size_t RoundUp(size_t num_bytes) override {
return 1uLL << Log2Ceiling64(num_bytes);
}
};
class BasicCPUAllocator : public SubAllocator {
public:
// Argument numa_node is currently ignored.
BasicCPUAllocator(int numa_node, const std::vector<Visitor>& alloc_visitors,
const std::vector<Visitor>& free_visitors)
: SubAllocator(alloc_visitors, free_visitors), numa_node_(numa_node) {}
~BasicCPUAllocator() override {}
void* Alloc(size_t alignment, size_t num_bytes) override;
void Free(void* ptr, size_t num_bytes) override;
private:
int numa_node_;
TF_DISALLOW_COPY_AND_ASSIGN(BasicCPUAllocator);
};
} // namespace tensorflow
#endif // TENSORFLOW_CORE_COMMON_RUNTIME_POOL_ALLOCATOR_H_
@@ -0,0 +1,200 @@
/* Copyright 2017 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
#ifndef TENSORFLOW_CORE_COMMON_RUNTIME_PROCESS_FUNCTION_LIBRARY_RUNTIME_H_
#define TENSORFLOW_CORE_COMMON_RUNTIME_PROCESS_FUNCTION_LIBRARY_RUNTIME_H_
#include <unordered_map>
#include "tensorflow/core/common_runtime/device_mgr.h"
#include "tensorflow/core/framework/function.h"
#include "tensorflow/core/lib/core/status.h"
#include "tensorflow/core/protobuf/config.pb.h"
namespace tensorflow {
// A class that stores all the FunctionLibraryRuntime objects, one per device.
class ProcessFunctionLibraryRuntime {
public:
// Creates FunctionLibraryRuntime objects for each device in the provided
// DeviceMgr. Caller needs to make sure that device_mgr, lib_def and parent
// (if provided) outlive this object.
ProcessFunctionLibraryRuntime(
const DeviceMgr* device_mgr, Env* env, int graph_def_version,
const FunctionLibraryDefinition* lib_def,
const OptimizerOptions& optimizer_options,
thread::ThreadPool* thread_pool = nullptr,
DistributedFunctionLibraryRuntime* parent = nullptr);
// With `custom_kernel_creator`.
ProcessFunctionLibraryRuntime(const DeviceMgr* device_mgr, Env* env,
int graph_def_version,
const FunctionLibraryDefinition* lib_def,
const OptimizerOptions& optimizer_options,
CustomKernelCreator custom_kernel_creator,
thread::ThreadPool* thread_pool,
DistributedFunctionLibraryRuntime* parent);
// Sends `tensors_to_send` from `source_device` to `target_device` using
// `rendezvous`. `key_prefix` is used as a prefix for the keys sent to the
// Rendezvous. `device_context` should be the DeviceContext of the device
// doing the sending. `alloc_attrs` should either be empty or be the size of
// `tensors_to_send` and indicates how the input tensors are allocated. Method
// takes references on each of the `tensors_to_send`. Method doesn't block.
static Status SendTensors(const string& source_device,
const string& target_device,
const string& key_prefix, int64 src_incarnation,
gtl::ArraySlice<Tensor> tensors_to_send,
DeviceContext* device_context,
const std::vector<AllocatorAttributes>& alloc_attrs,
Rendezvous* rendezvous);
// Receives `received_tensors` from `target_device` (originally sent from
// `source_device`) using `rendezvous`. Uses `key_prefix` to construct the
// keys to be retrieved. `device_context` should be for the device receiving
// the tensors. `alloc_attrs` indicates how to allocate the received
// tensors and should either be empty or `num_tensors` in size. Method doesn't
// block and calls `done` when `num_tensors` are fetched.
static void ReceiveTensorsAsync(
const string& source_device, const string& target_device,
const string& key_prefix, int64 src_incarnation, int64 num_tensors,
DeviceContext* device_context,
const std::vector<AllocatorAttributes>& alloc_attrs,
Rendezvous* rendezvous, std::vector<Tensor>* received_tensors,
StatusCallback done);
static const char kDefaultFLRDevice[];
// Returns the FunctionLibraryRuntime for the corresponding device_name.
FunctionLibraryRuntime* GetFLR(const string& device_name) const;
// Returns the device incarnation for the given device_name.
Status GetDeviceIncarnation(const string& device_name, int64* incarnation);
// For a given canonicalized key signature of the function instantiated
// on device `device_name` and a `local_handle`, creates a handle and returns
// that value. Uses core/common_runtime/framework/function.h::Canonicalize
// to canonicalize the function signature.
FunctionLibraryRuntime::Handle AddHandle(
const string& function_key, const string& device_name,
FunctionLibraryRuntime::LocalHandle local_handle);
// Returns a handle if found for the given key, else returns kInvalidHandle.
FunctionLibraryRuntime::Handle GetHandle(const string& function_key) const;
// For the given handle instantiated on device `device_name` returns the local
// index of instantiation of that function. If the function was not
// instantiated on `device_name` returns kInvalidLocalHandle.
FunctionLibraryRuntime::LocalHandle GetHandleOnDevice(
const string& device_name, FunctionLibraryRuntime::Handle handle);
// Returns true if function with handle `handle` was instantiated on device
// `device_name`.
bool IsInstantiatedOnDevice(const string& device_name,
FunctionLibraryRuntime::Handle handle);
// Instantiates the function. See framework/function.h for more details.
// Allows for function_name to be instantiated on different devices
// as specified in attrs.
Status Instantiate(const string& function_name, AttrSlice attrs,
const FunctionLibraryRuntime::InstantiateOptions& options,
FunctionLibraryRuntime::Handle* handle);
// Delegates to the local FLR that owns state corresponding to `handle` and
// tells it to release it. If the `handle` isnt' needed at all, the local FLR
// might call RemoveHandle on this to get rid of the state owned by the Proc
// FLR.
Status ReleaseHandle(FunctionLibraryRuntime::Handle handle);
// Runs the function with given `handle`. Function could have been
// instantiated on any device. More details in framework/function.h
void Run(const FunctionLibraryRuntime::Options& opts,
FunctionLibraryRuntime::Handle handle, gtl::ArraySlice<Tensor> args,
std::vector<Tensor>* rets,
FunctionLibraryRuntime::DoneCallback done);
private:
// For a given device_name, returns a DeviceContext for copying
// tensors to/from the device.
Status GetDeviceContext(const string& device_name,
DeviceContext** device_context);
// Looks up the information for the given `handle` and returns the name
// of the device where the function is registered.
string GetDeviceName(FunctionLibraryRuntime::Handle handle);
// Removes handle from the state owned by this object.
Status RemoveHandle(FunctionLibraryRuntime::Handle handle);
Status Clone(Env* env, int graph_def_version,
const OptimizerOptions& optimizer_options,
CustomKernelCreator custom_kernel_creator,
std::unique_ptr<FunctionLibraryDefinition>* out_lib_def,
std::unique_ptr<ProcessFunctionLibraryRuntime>* out_pflr);
friend class FunctionLibraryRuntimeImpl;
mutable mutex mu_;
class FunctionData {
public:
FunctionData(const string& target_device,
FunctionLibraryRuntime::LocalHandle local_handle,
const string& function_key)
: target_device_(target_device),
local_handle_(local_handle),
function_key_(function_key) {}
string target_device() { return target_device_; }
const string& function_key() { return function_key_; }
FunctionLibraryRuntime::LocalHandle local_handle() {
mutex_lock l(mu_);
return local_handle_;
}
// Initializes the FunctionData object by potentially making an Initialize
// call to the DistributedFunctionLibraryRuntime.
Status DistributedInit(
DistributedFunctionLibraryRuntime* parent, const string& function_name,
const FunctionLibraryDefinition& lib_def, AttrSlice attrs,
const FunctionLibraryRuntime::InstantiateOptions& options);
private:
mutex mu_;
const string target_device_;
FunctionLibraryRuntime::LocalHandle local_handle_ GUARDED_BY(mu_);
const string function_key_;
bool init_started_ GUARDED_BY(mu_) = false;
Status init_result_ GUARDED_BY(mu_);
Notification init_done_;
};
const DeviceMgr* const device_mgr_;
const FunctionLibraryDefinition* lib_def_;
thread::ThreadPool* default_thread_pool_;
// Holds all the function invocations here.
std::unordered_map<string, FunctionLibraryRuntime::Handle> table_
GUARDED_BY(mu_);
std::unordered_map<FunctionLibraryRuntime::Handle,
std::unique_ptr<FunctionData>>
function_data_ GUARDED_BY(mu_);
std::unordered_map<Device*, std::unique_ptr<FunctionLibraryRuntime>> flr_map_;
int next_handle_ GUARDED_BY(mu_);
DistributedFunctionLibraryRuntime* const parent_;
};
} // namespace tensorflow
#endif // TENSORFLOW_CORE_COMMON_RUNTIME_PROCESS_FUNCTION_LIBRARY_RUNTIME_H_
@@ -0,0 +1,141 @@
/* Copyright 2015 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
#ifndef TENSORFLOW_CORE_COMMON_RUNTIME_PROCESS_STATE_H_
#define TENSORFLOW_CORE_COMMON_RUNTIME_PROCESS_STATE_H_
#include <functional>
#include <map>
#include <unordered_map>
#include <vector>
#include "tensorflow/core/framework/allocator.h"
#include "tensorflow/core/platform/mutex.h"
#include "tensorflow/core/platform/thread_annotations.h"
#include "tensorflow/core/platform/types.h"
#include "tensorflow/core/protobuf/config.pb.h"
namespace tensorflow {
class Allocator;
class PoolAllocator;
// Singleton that manages per-process state, e.g. allocation of
// shared resources.
class ProcessState {
public:
static ProcessState* singleton();
// Descriptor for memory allocation attributes, used by optional
// runtime correctness analysis logic.
struct MemDesc {
enum MemLoc { CPU, GPU };
MemLoc loc;
int dev_index;
bool gpu_registered;
bool nic_registered;
MemDesc()
: loc(CPU),
dev_index(0),
gpu_registered(false),
nic_registered(false) {}
string DebugString();
};
// If NUMA Allocators are desired, call this before calling any
// Allocator accessor.
void EnableNUMA() { numa_enabled_ = true; }
// Returns what we know about the memory at ptr.
// If we know nothing, it's called CPU 0 with no other attributes.
MemDesc PtrType(const void* ptr);
// Returns the one CPUAllocator used for the given numa_node.
// TEMPORARY: ignores numa_node.
Allocator* GetCPUAllocator(int numa_node);
// Registers alloc visitor for the CPU allocator(s).
// REQUIRES: must be called before GetCPUAllocator.
void AddCPUAllocVisitor(SubAllocator::Visitor v);
// Registers free visitor for the CPU allocator(s).
// REQUIRES: must be called before GetCPUAllocator.
void AddCPUFreeVisitor(SubAllocator::Visitor v);
typedef std::unordered_map<const void*, MemDesc> MDMap;
protected:
ProcessState();
friend class GPUProcessState;
// If these flags need to be runtime configurable consider adding
// them to ConfigProto.
static const bool FLAGS_brain_mem_reg_cuda_dma = true;
static const bool FLAGS_brain_gpu_record_mem_types = false;
// Helper method for unit tests to reset the ProcessState singleton by
// cleaning up everything. Never use in production.
virtual void TestOnlyReset();
static ProcessState* instance_;
bool numa_enabled_;
mutex mu_;
std::vector<Allocator*> cpu_allocators_ GUARDED_BY(mu_);
std::vector<SubAllocator::Visitor> cpu_alloc_visitors_ GUARDED_BY(mu_);
std::vector<SubAllocator::Visitor> cpu_free_visitors_ GUARDED_BY(mu_);
virtual ~ProcessState();
// Optional RecordingAllocators that wrap the corresponding
// Allocators for runtime attribute use analysis.
MDMap mem_desc_map_;
std::vector<Allocator*> cpu_al_ GUARDED_BY(mu_);
};
namespace internal {
class RecordingAllocator : public Allocator {
public:
RecordingAllocator(ProcessState::MDMap* mm, Allocator* a,
ProcessState::MemDesc md, mutex* mu)
: mm_(mm), a_(a), md_(md), mu_(mu) {}
string Name() override { return a_->Name(); }
void* AllocateRaw(size_t alignment, size_t num_bytes) override {
void* p = a_->AllocateRaw(alignment, num_bytes);
mutex_lock l(*mu_);
(*mm_)[p] = md_;
return p;
}
void DeallocateRaw(void* p) override {
mutex_lock l(*mu_);
auto iter = mm_->find(p);
mm_->erase(iter);
a_->DeallocateRaw(p);
}
bool TracksAllocationSizes() override { return a_->TracksAllocationSizes(); }
size_t RequestedSize(const void* p) override { return a_->RequestedSize(p); }
size_t AllocatedSize(const void* p) override { return a_->AllocatedSize(p); }
void GetStats(AllocatorStats* stats) override { a_->GetStats(stats); }
void ClearStats() override { a_->ClearStats(); }
ProcessState::MDMap* mm_; // not owned
Allocator* a_; // not owned
ProcessState::MemDesc md_;
mutex* mu_;
};
} // namespace internal
} // namespace tensorflow
#endif // TENSORFLOW_CORE_COMMON_RUNTIME_PROCESS_STATE_H_
@@ -0,0 +1,49 @@
/* Copyright 2016 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
#ifndef TENSORFLOW_CORE_COMMON_RUNTIME_PROCESS_UTIL_H_
#define TENSORFLOW_CORE_COMMON_RUNTIME_PROCESS_UTIL_H_
#include <functional>
#include "tensorflow/core/lib/core/threadpool.h"
#include "tensorflow/core/public/session_options.h"
// TODO(vrv, mrry): Remove this library: its interface circumvents the
// callers' Env and calls Env::Default() directly.
namespace tensorflow {
// Returns a process-wide ThreadPool for scheduling compute operations
// using 'options'. Caller does not take ownership over threadpool.
thread::ThreadPool* ComputePool(const SessionOptions& options);
// Returns number of inter op threads.
int32 NumInterOpThreadsFromSessionOptions(const SessionOptions& options);
// Creates a thread pool with number of inter op threads.
thread::ThreadPool* NewThreadPoolFromSessionOptions(
const SessionOptions& options);
// Schedule "closure" in the default thread queue.
void SchedClosure(std::function<void()> closure);
// Schedule "closure" after the given number of microseconds in the
// fixed-size ThreadPool used for non-blocking compute tasks.
void SchedNonBlockingClosureAfter(int64 micros, std::function<void()> closure);
} // namespace tensorflow
#endif // TENSORFLOW_CORE_COMMON_RUNTIME_PROCESS_UTIL_H_
@@ -0,0 +1,67 @@
/* Copyright 2016 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
#ifndef TENSORFLOW_CORE_COMMON_RUNTIME_PROFILE_HANDLER_H_
#define TENSORFLOW_CORE_COMMON_RUNTIME_PROFILE_HANDLER_H_
#include "tensorflow/core/framework/step_stats.pb.h"
#include "tensorflow/core/graph/types.h"
#include "tensorflow/core/lib/core/status.h"
#include "tensorflow/core/lib/core/stringpiece.h"
namespace tensorflow {
// A profile handler collects event stats from a running step.
class ProfileHandler {
public:
ProfileHandler() {}
virtual ~ProfileHandler() {}
// Records that a single Op was executed in the current step.
//
// Implementations of this method must be thread-safe.
//
// Args:
// - device: Device on which the Op was executed.
// - stats: Statistics of node execution timing.
// - is_copy: True if the op was a copy, send or recv.
// - label: Extra content for timeline click text.
// - op_type: String name of the Op.
// - details: Main content for timeline click text.
virtual void RecordOneOp(const string& device, const NodeExecStats& stats,
bool is_copy, StringPiece label, StringPiece op_type,
StringPiece details) = 0;
// Records that the current step finished.
//
// Implementations of this method need not be thread-safe.
//
// Args:
// - start_time: The time at which the step started.
// - finish_time: The time at which the step finished.
// - cleanup_time: The time at which cleanup for the step finished.
// - total_runops: The number of ops that ran during this step.
// - final_status: The status that this step finished with.
virtual void StepDone(Microseconds start_time, Microseconds finish_time,
Microseconds cleanup_time, int total_runops,
Status final_status) = 0;
// Returns true if the caller should collect rpc activity.
virtual bool should_collect_rpcs() = 0;
};
} // namespace tensorflow
#endif // TENSORFLOW_CORE_COMMON_RUNTIME_PROFILE_HANDLER_H_
@@ -0,0 +1,142 @@
/* Copyright 2016 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
#ifndef TENSORFLOW_CORE_COMMON_RUNTIME_RENAMED_DEVICE_H_
#define TENSORFLOW_CORE_COMMON_RUNTIME_RENAMED_DEVICE_H_
#include "tensorflow/core/common_runtime/device.h"
#include "tensorflow/core/util/device_name_utils.h"
namespace tensorflow {
// Wraps a device with a new name, delegating work to the wrapped device.
//
// This class is used to wrap local devices when using clusterspec propagation
// where the name of a particular device may change in the context of a given
// session.
class RenamedDevice : public Device {
public:
static Device* NewRenamedDevice(const string& new_base, Device* underlying,
bool owns_underlying,
bool isolate_session_state);
~RenamedDevice() override;
// Below are virtual methods defined on DeviceBase
bool RequiresRecordingAccessedTensors() const override {
return underlying_->RequiresRecordingAccessedTensors();
}
const DeviceBase* UnderlyingDevice() const override {
return underlying_->UnderlyingDevice();
}
DeviceBase* UnderlyingDevice() override {
return underlying_->UnderlyingDevice();
}
const CpuWorkerThreads* tensorflow_cpu_worker_threads() const override {
return underlying_->tensorflow_cpu_worker_threads();
}
const GpuDeviceInfo* tensorflow_gpu_device_info() const override {
return underlying_->tensorflow_gpu_device_info();
}
Allocator* GetAllocator(AllocatorAttributes attr) override {
return underlying_->GetAllocator(attr);
}
Allocator* GetScopedAllocator(AllocatorAttributes attr,
int64 step_id) override {
return underlying_->GetScopedAllocator(attr, step_id);
}
ScopedAllocatorMgr* GetScopedAllocatorMgr() const override {
return underlying_->GetScopedAllocatorMgr();
}
const Eigen::ThreadPoolDevice* eigen_cpu_device() override {
return underlying_->eigen_cpu_device();
}
#ifdef TENSORFLOW_USE_SYCL
const Eigen::SyclDevice* eigen_sycl_device() const override {
return underlying_->eigen_sycl_device();
}
#endif
PerOpGpuDevice* MakeGpuDevice() override {
return underlying_->MakeGpuDevice();
}
Status ReinitializeGpuDevice(OpKernelContext* context, PerOpGpuDevice* device,
DeviceContext* dc,
Allocator* allocator) override {
return underlying_->ReinitializeGpuDevice(context, device, dc, allocator);
}
Status MakeTensorFromProto(const TensorProto& tensor_proto,
const AllocatorAttributes alloc_attrs,
Tensor* tensor) override {
return underlying_->MakeTensorFromProto(tensor_proto, alloc_attrs, tensor);
}
// Below are virtual methods defined on Device
void Compute(OpKernel* op_kernel, OpKernelContext* context) override {
underlying_->Compute(op_kernel, context);
}
void ComputeAsync(AsyncOpKernel* op_kernel, OpKernelContext* context,
AsyncOpKernel::DoneCallback done) override {
underlying_->ComputeAsync(op_kernel, context, std::move(done));
}
void ConsumeListOfAccessedTensors(
DeviceContext* context, const TensorReferenceVector& tensors) override {
underlying_->ConsumeListOfAccessedTensors(context, tensors);
}
Status Sync() override { return underlying_->Sync(); }
Status MaybeRewriteGraph(std::unique_ptr<Graph>* graph) override {
return underlying_->MaybeRewriteGraph(graph);
}
Status FillContextMap(const Graph* graph,
DeviceContextMap* device_context_map) override {
return underlying_->FillContextMap(graph, device_context_map);
}
// Returns the resource manager associated w/ this device.
ResourceMgr* resource_manager() override {
if (isolate_session_state_) {
return Device::resource_manager();
} else {
return underlying_->resource_manager();
}
}
private:
RenamedDevice(Device* underlying, const DeviceAttributes& attributes,
bool owns_underlying, bool isolate_session_state);
Device* const underlying_;
const bool owns_underlying_;
const bool isolate_session_state_;
};
} // namespace tensorflow
#endif // TENSORFLOW_CORE_COMMON_RUNTIME_RENAMED_DEVICE_H_
@@ -0,0 +1,90 @@
/* Copyright 2015 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
#ifndef TENSORFLOW_CORE_COMMON_RUNTIME_RENDEZVOUS_MGR_H_
#define TENSORFLOW_CORE_COMMON_RUNTIME_RENDEZVOUS_MGR_H_
#include <string>
#include <unordered_map>
#include "tensorflow/core/common_runtime/device_mgr.h"
#include "tensorflow/core/framework/rendezvous.h"
#include "tensorflow/core/framework/tensor.h"
#include "tensorflow/core/lib/core/status.h"
#include "tensorflow/core/platform/macros.h"
#include "tensorflow/core/platform/mutex.h"
#include "tensorflow/core/platform/types.h"
namespace tensorflow {
// IntraProcessRendezvous is a Rendezvous which expects all producers
// and consumers to be devices immediately accessible within the
// process. That is, it will never be necessary to perform an RPC to
// communicate with either.
//
// Buffering of Tensor values is delegated to a "local" Rendezvous
// obtained from NewLocalRendezvous(). This class just adds
// functionality to coordinate multiple process-local devices.
class IntraProcessRendezvous : public Rendezvous {
public:
explicit IntraProcessRendezvous(const DeviceMgr* device_mgr);
// Forwards to local_, where the Tensor "val" will be buffered and
// any waiting callback stored.
Status Send(const ParsedKey& key, const Rendezvous::Args& args,
const Tensor& val, const bool is_dead) override;
// This method is called only by the RecvOp. It tests to see
// whether the value will be produced by a local or remote device
// and handles accordingly. In the local case it forwards to
// local_, in the remote case it initiates an RPC request.
void RecvAsync(const ParsedKey& key, const Rendezvous::Args& args,
DoneCallback done) override;
void StartAbort(const Status& status) override;
private:
const DeviceMgr* device_mgr_;
Rendezvous* local_; // Owns a Ref on this object.
mutable mutex mu_;
// Status given by StartAbort() if any.
Status status_ GUARDED_BY(mu_);
~IntraProcessRendezvous() override;
// Parses "key" into "parsed". If "is_src" is true, checks that the
// rendezvous key's source is in this process. If "is_src" is false,
// checks that the rendezvous key's destination is in this process.
Status ParseKey(const string& key, bool is_src,
Rendezvous::ParsedKey* parsed);
// Callback handling the case when a rendezvous has been
// accomplished in local_ and the consumer is local to this process.
// Tensor "in" will be copied into "out". The key "parsed" encodes
// the src and dst devices.
typedef std::function<void(const Status&)> StatusCallback;
void SameWorkerRecvDone(const Rendezvous::ParsedKey& parsed,
const Rendezvous::Args& send_args,
const Rendezvous::Args& recv_args, const Tensor& in,
Tensor* out, StatusCallback done);
TF_DISALLOW_COPY_AND_ASSIGN(IntraProcessRendezvous);
};
} // end namespace tensorflow
#endif // TENSORFLOW_CORE_COMMON_RUNTIME_RENDEZVOUS_MGR_H_
@@ -0,0 +1,53 @@
/* Copyright 2017 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
#ifndef TENSORFLOW_CORE_COMMON_RUNTIME_RENDEZVOUS_UTIL_H_
#define TENSORFLOW_CORE_COMMON_RUNTIME_RENDEZVOUS_UTIL_H_
#include <map>
#include "tensorflow/core/framework/rendezvous.h"
#include "tensorflow/core/lib/core/status.h"
namespace tensorflow {
typedef std::map<string, Tensor> NamedTensors;
typedef std::function<void(const Status&)> StatusCallback;
// Uses `rendezvous` to send tensors in `tensors_to_send`. `device_context`
// should be the DeviceContext associated with the source of the tensors.
// `alloc_attrs` contains information about how the `tensors_to_send` are
// allocated. `alloc_attrs` should either be {} or should match the length of
// `keys`.
Status SendTensorsToRendezvous(
Rendezvous* rendezvous, DeviceContext* device_context,
const std::vector<AllocatorAttributes>& alloc_attrs,
const std::vector<string>& keys, gtl::ArraySlice<Tensor> tensors_to_send);
// Uses `rendezvous` to obtain tensors. `device_context` should be the
// DeviceContext associated with the receiving device. `alloc_attrs` contains
// information as how to store the received tensors. Should be {} or match the
// length of `keys`.
void RecvOutputsFromRendezvousAsync(
Rendezvous* rendezvous, DeviceContext* device_context,
const std::vector<AllocatorAttributes>& alloc_attrs,
const std::vector<string>& keys, std::vector<Tensor>* received_tensors,
StatusCallback done);
Status RecvOutputsFromRendezvous(Rendezvous* rendezvous, NamedTensors* out,
const Rendezvous::Args& args);
} // namespace tensorflow
#endif // TENSORFLOW_CORE_COMMON_RUNTIME_RENDEZVOUS_UTIL_H_
@@ -0,0 +1,145 @@
/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
#ifndef TENSORFLOW_CORE_COMMON_RUNTIME_RING_REDUCER_H_
#define TENSORFLOW_CORE_COMMON_RUNTIME_RING_REDUCER_H_
#include <deque>
#include <memory>
#include <string>
#include <vector>
#include "tensorflow/core/common_runtime/base_collective_executor.h"
#include "tensorflow/core/framework/collective.h"
namespace tensorflow {
class Device;
// Ring-algorithm implementation of collective all-reduce.
class RingReducer : public CollectiveImplementationInterface {
public:
RingReducer();
~RingReducer() override;
// Establishes the requested number of subdivision permutations based on the
// ring order implicit in the device order.
Status InitializeCollectiveParams(CollectiveParams* col_params) override;
// Initializes members of CollectiveContext not yet initialized, i.e. device
// and device_locality. Also saves the CollectiveContext in this object.
Status InitializeCollectiveContext(CollectiveContext* col_ctx) override;
// Begins async execution of the ring reduce algorithm.
// Must be called in a blockable thread.
// TODO(b/80529858): remove the previous warning when we have a dedicated
// collective threadpool.
void Run(StatusCallback done) override;
private:
// Called when a bad status is received that implies we should terminate
// execution and return a bad status.
void StartAbort(const Status& s);
void ContinueAfterInputCopy();
void Finish(bool ok);
Status ComputeBinOp(Device* device, OpKernel* op, Tensor* output,
Tensor* input);
bool RunAsyncParts();
// Used for executing a sub-operation, e.g. a merge_op instance, with
// an OpKernelContext based on the one passed into this Op.
class SubContext {
public:
OpKernelContext::Params sub_params_;
gtl::InlinedVector<TensorValue, 4> sub_inputs_;
gtl::InlinedVector<AllocatorAttributes, 4> sub_input_attr_;
gtl::InlinedVector<DeviceContext*, 4> sub_input_dc_;
// Used only for Binary and Unary Ops for which we require
// the calculation to be in-place on the first input.
int forward_from_ = 0;
OpKernelContext* sub_ctx_;
SubContext(OpKernelContext* ctx, OpKernelContext::Params* params,
OpKernel* op, Tensor* output, Tensor* input);
~SubContext() { delete sub_ctx_; }
};
// Current status of a RingField
enum RingFieldAction {
RF_INIT = 0, // Just initialized for a pass
RF_RECV, // Recv pending
RF_REDUCE, // Reduce pending
RF_FINALIZE, // FinalOp pending
RF_SEND_READY, // Ready to send
RF_SEND, // Send pending
RF_DONE, // No more work
};
// Tracks progress of actions on a single subfield of the entire tensor.
struct RingField {
int16 chunk_idx; // major division index
int16 subdiv_idx; // minor division index
int16 sc_idx; // subchunk index
int16 rank; // rank within subdiv permutation
int16 recv_dev_idx; // dev from which value should be recv'd
RingFieldAction action;
bool second_pass;
bool recv_is_remote = false;
bool send_is_remote = false;
bool do_send = false; // is the value sent in this pass?
bool do_recv = false; // is the value recv'd in this pass?
bool is_final = false; // is the last field in the pass for this rank
Tensor chunk; // alias to field values
Tensor tmp_chunk;
Status status;
string DebugString() const;
};
void AdvanceToSecondPass(RingField* rf);
void InitRingField(RingField* rf, int chunk_idx, int subdiv_idx,
int field_idx);
void DispatchSend(RingField* rf, const StatusCallback& done);
void DispatchRecv(RingField* rf, const StatusCallback& done);
// For constructing log messages for debugging.
string FieldState();
string TensorDebugString(const Tensor& tensor);
// Producer/Consumer Queue of RingField structs.
class PCQueue {
public:
void Enqueue(RingField* rf);
RingField* Dequeue();
private:
mutex pcq_mu_;
condition_variable cv_;
int waiter_count_ GUARDED_BY(pcq_mu_) = 0;
std::deque<RingField*> deque_ GUARDED_BY(pcq_mu_);
};
CollectiveContext* col_ctx_; // Not owned
const CollectiveParams* col_params_; // Not owned
StatusCallback done_;
int group_size_;
int num_subdivs_;
Tensor group_size_tensor_;
Notification group_size_tensor_ready_;
std::unique_ptr<CollectiveAdapter> ca_;
mutex status_mu_;
Status status_ GUARDED_BY(status_mu_);
std::vector<RingField> rfv_;
friend class RingReducerTest;
};
} // namespace tensorflow
#endif // TENSORFLOW_CORE_COMMON_RUNTIME_RING_REDUCER_H_
@@ -0,0 +1,124 @@
/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
#ifndef TENSORFLOW_CORE_COMMON_RUNTIME_SCOPED_ALLOCATOR_H_
#define TENSORFLOW_CORE_COMMON_RUNTIME_SCOPED_ALLOCATOR_H_
#include "tensorflow/core/framework/allocator.h"
#include "tensorflow/core/framework/tensor.h"
#include "tensorflow/core/lib/core/refcount.h"
#include "tensorflow/core/platform/mutex.h"
namespace tensorflow {
class ScopedAllocatorContainer;
class ScopedAllocatorInstance;
// Manages a single backing tensor and a collection of aliases.
class ScopedAllocator {
public:
static const int32 kInvalidId = 0;
static const size_t kMaxAlignment = 64;
// A subrange of the TensorBuffer associated with this object that
// will be the backing memory for one aliased tensor.
struct Field {
int32 scope_id;
size_t offset;
size_t bytes;
};
// Field index that refers to backing tensor, not any aliased field.
static const int32 kBackingIndex = -1;
// backing_tensor is expected to be newly allocated by a ScopedAllocatorOp
// instance. It must be large enough to back all of the specified
// (offset, byte) ranges of the fields.
ScopedAllocator(const Tensor& backing_tensor, int32 scope_id,
const string& name, const gtl::ArraySlice<Field>& fields,
int32 expected_call_count,
ScopedAllocatorContainer* container);
// Automatically deletes when last use expires, or when
// ScopedAllocatorContainer decides to delete.
~ScopedAllocator() LOCKS_EXCLUDED(mu_);
// For debugging: returns true iff p is a pointer that could have
// been returned by AllocateRaw.
bool VerifyPointer(const void* p);
bool VerifyTensor(const Tensor* t);
const Tensor& tensor() const { return backing_tensor_; }
const string& name() const { return name_; }
private:
friend class ScopedAllocatorInstance;
// Only ScopedAllocatorInstances can call AllocateRaw and DeallocateRaw on a
// ScopedAllocator
void* AllocateRaw(int32 field_index, size_t num_bytes) LOCKS_EXCLUDED(mu_);
void DeallocateRaw(void* p) LOCKS_EXCLUDED(mu_);
Tensor backing_tensor_;
TensorBuffer* tbuf_;
int32 id_;
string name_;
ScopedAllocatorContainer* container_;
std::vector<Field> fields_;
mutex mu_;
int32 expected_call_count_ GUARDED_BY(mu_);
int32 live_alloc_count_ GUARDED_BY(mu_);
};
// An Allocator that will return a pointer into the backing buffer of
// a previously allocated tensor, allowing creation of an alias
// tensor. There is a one-to-one mapping between the fields of a
// ScopedAllocator and ScopedAllocatorInstances. There is also a one-to-one
// mapping between scope_ids and ScopedAllocatorInstances. It should be
// discarded immediately after a single use.
class ScopedAllocatorInstance : public Allocator {
public:
explicit ScopedAllocatorInstance(ScopedAllocator* sa, int32 field_index);
private:
~ScopedAllocatorInstance() { VLOG(1) << "~ScopedAllocatorInstance " << this; }
public:
// When a ScopedAllocatorContainer "Drops" a scope_id, it calls DropFromTable
// on the underlying ScopedAllocatorInstance. If this instance has already
// deallocated the tensor slice, we can safely delete this.
void DropFromTable() LOCKS_EXCLUDED(mu_);
void* AllocateRaw(size_t alignment, size_t num_bytes)
LOCKS_EXCLUDED(mu_) override;
void* AllocateRaw(size_t alignment, size_t num_bytes,
const AllocationAttributes& allocator_attr) override {
return AllocateRaw(alignment, num_bytes);
}
void DeallocateRaw(void* p) LOCKS_EXCLUDED(mu_) override;
bool TracksAllocationSizes() override { return false; }
bool ShouldAllocateEmptyTensors() override { return false; }
size_t RequestedSize(const void* ptr) override { return 0; }
size_t AllocatedSize(const void* ptr) override { return 0; }
int64 AllocationId(const void* ptr) override { return 0; }
size_t AllocatedSizeSlow(const void* ptr) override { return 0; }
string Name() override;
private:
mutex mu_;
ScopedAllocator* scoped_allocator_;
int32 field_index_;
bool allocated_ GUARDED_BY(mu_);
bool deallocated_ GUARDED_BY(mu_);
bool in_table_ GUARDED_BY(mu_);
};
} // namespace tensorflow
#endif // TENSORFLOW_CORE_COMMON_RUNTIME_SCOPED_ALLOCATOR_H_
@@ -0,0 +1,110 @@
/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
#ifndef TENSORFLOW_CORE_COMMON_RUNTIME_SCOPED_ALLOCATOR_MGR_H_
#define TENSORFLOW_CORE_COMMON_RUNTIME_SCOPED_ALLOCATOR_MGR_H_
#include <string>
#include <unordered_map>
#include "tensorflow/core/common_runtime/scoped_allocator.h"
#include "tensorflow/core/lib/core/refcount.h"
#include "tensorflow/core/lib/core/status.h"
#include "tensorflow/core/platform/mutex.h"
namespace tensorflow {
class ScopedAllocatorMgr;
// At most one of these exists per <device, step_id> pair.
// A Ref is held by every ScopedAllocator and also by the ScopedAllocatorMgr.
class ScopedAllocatorContainer : public core::RefCounted {
public:
// Establishes a reachable ScopedAllocator.
Status AddScopedAllocator(
const Tensor& backing_tensor, int32 scope_id, const string& scope_name,
const gtl::ArraySlice<ScopedAllocator::Field>& fields,
int32 expected_call_count);
ScopedAllocatorInstance* GetInstance(int32 scope_id);
ScopedAllocator* GetAllocator(int32 scope_id);
// Retire the scope_id.
void Drop(int32 scope_id, ScopedAllocator* sa);
protected:
friend class ScopedAllocatorMgr;
ScopedAllocatorContainer(const ScopedAllocatorMgr* mgr, int64 step_id)
: mgr_(mgr), step_id_(step_id) {}
~ScopedAllocatorContainer();
private:
const ScopedAllocatorMgr* mgr_;
int64 step_id_;
mutex mu_;
struct SAField {
int32 field_index;
union {
ScopedAllocator* scoped_allocator;
ScopedAllocatorInstance* instance;
};
SAField(int32 fi, ScopedAllocatorInstance* sai)
: field_index(fi), instance(sai) {}
SAField(int32 fi, ScopedAllocator* sa)
: field_index(fi), scoped_allocator(sa) {}
SAField()
: field_index(ScopedAllocator::kBackingIndex),
scoped_allocator(nullptr) {}
};
std::unordered_map<int32, SAField> allocators_ GUARDED_BY(mu_);
};
// At most one of these exists per device.
class ScopedAllocatorMgr {
public:
explicit ScopedAllocatorMgr(const string& device_name)
: device_name_(device_name) {}
~ScopedAllocatorMgr();
ScopedAllocatorContainer* GetContainer(int64 step_id);
// Establishes a reachable ScopedAllocator.
Status AddScopedAllocator(
const Tensor& backing_tensor, int64 step_id, int32 scope_id,
const string& scope_name,
const gtl::ArraySlice<ScopedAllocator::Field>& fields,
int32 expected_call_count);
void Cleanup(int64 step_id);
// Populate the bytes and offset members of Field. Instance allocaters get
// consecutive scope_id values following that of the base ScopedAllocator.
// Returns the total number of bytes required to be allocated in the
// backing tensor, for convenience. (The same value can be obtained
// by summing offset and bytes in the last field.)
static size_t PopulateFields(int32 scope_id,
const gtl::ArraySlice<TensorShape>& shapes,
const DataType dtype,
std::vector<ScopedAllocator::Field>* fields);
const string& device_name() const { return device_name_; }
private:
string device_name_;
mutex mu_;
std::unordered_map<int64, ScopedAllocatorContainer*> per_step_map_
GUARDED_BY(mu_);
};
} // namespace tensorflow
#endif // TENSORFLOW_CORE_COMMON_RUNTIME_SCOPED_ALLOCATOR_MGR_H_
@@ -0,0 +1,76 @@
/* Copyright 2015 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
#ifndef TENSORFLOW_CORE_COMMON_RUNTIME_SESSION_FACTORY_H_
#define TENSORFLOW_CORE_COMMON_RUNTIME_SESSION_FACTORY_H_
#include <string>
#include "tensorflow/core/lib/core/errors.h"
#include "tensorflow/core/lib/core/status.h"
#include "tensorflow/core/lib/gtl/array_slice.h"
#include "tensorflow/core/platform/types.h"
namespace tensorflow {
class Session;
struct SessionOptions;
class SessionFactory {
public:
// Creates a new session and stores it in *out_session, or fails with an error
// status if the Session could not be created. Caller takes ownership of
// *out_session if this returns Status::OK().
virtual Status NewSession(const SessionOptions& options,
Session** out_session) = 0;
virtual bool AcceptsOptions(const SessionOptions& options) = 0;
// Abort and close all existing sessions, disconnecting their resources from
// future sessions.
//
// Reset() allows misbehaving or slow sessions to be aborted and closed, and
// causes their resources eventually to be released. Reset() does not wait
// for the computations in old sessions to cease; it merely starts the
// process of tearing them down. However, if a new session is started after
// a Reset(), the new session is isolated from changes that old sessions
// (started prior to the Reset()) may continue to make to resources, provided
// all those resources are in containers listed in "containers".
//
// Old sessions may continue to have side-effects on resources not in
// containers listed in "containers", and thus may affect future
// sessions' results in ways that are hard to predict. Thus, if well-defined
// behavior is desired, is it recommended that all containers be listed in
// "containers".
//
// If the "containers" vector is empty, the default container is assumed.
// If the "containers" vector is non-empty, the default container should be
// listed explicitly.
//
// Sessions that support resource containers should override this function.
virtual Status Reset(const SessionOptions& options,
const std::vector<string>& containers) {
return errors::Unimplemented("Reset()");
}
virtual ~SessionFactory() {}
static void Register(const string& runtime_type, SessionFactory* factory);
static Status GetFactory(const SessionOptions& options,
SessionFactory** out_factory);
};
} // namespace tensorflow
#endif // TENSORFLOW_CORE_COMMON_RUNTIME_SESSION_FACTORY_H_
@@ -0,0 +1,306 @@
/* Copyright 2016 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
#ifndef TENSORFLOW_CORE_COMMON_RUNTIME_SHAPE_REFINER_H_
#define TENSORFLOW_CORE_COMMON_RUNTIME_SHAPE_REFINER_H_
#include <vector>
#include "tensorflow/core/common_runtime/graph_runner.h"
#include "tensorflow/core/framework/function.pb.h"
#include "tensorflow/core/framework/shape_inference.h"
#include "tensorflow/core/graph/graph.h"
#include "tensorflow/core/lib/core/status.h"
#include "tensorflow/core/platform/macros.h"
namespace tensorflow {
namespace grappler {
class GraphProperties;
}
// This class stores extra inference information in addition to
// InferenceContext, such as inference tree for user-defined functions and node
// input and output types.
class ExtendedInferenceContext {
public:
ExtendedInferenceContext(
std::unique_ptr<shape_inference::InferenceContext> ic, const Node* node)
: inference_context_(std::move(ic)) {
input_types_.reserve(node->num_inputs());
for (int i = 0; i < node->num_inputs(); i++) {
input_types_.push_back(node->input_type(i));
}
output_types_.reserve(node->num_outputs());
for (int i = 0; i < node->num_outputs(); i++) {
output_types_.push_back(node->output_type(i));
}
}
const std::unordered_map<string, std::unique_ptr<ExtendedInferenceContext>>&
nested_inferences() const {
return nested_inferences_;
}
DataType input_type(int64 idx) const { return input_types_[idx]; }
DataType output_type(int64 idx) const { return output_types_[idx]; }
shape_inference::InferenceContext* get_context() {
return inference_context_.get();
}
// Sets nested inference info.
// For composite ops (user-defined functions) only.
// Inference for trivial ops must not call this setter.
void set_nested_inferences(
std::unordered_map<string, std::unique_ptr<ExtendedInferenceContext>>
inferences) {
nested_inferences_ = std::move(inferences);
}
private:
std::unique_ptr<shape_inference::InferenceContext> inference_context_;
std::vector<DataType> input_types_;
std::vector<DataType> output_types_;
// Nested inferences for composite ops (user-defined functions).
// Mapping key is nested node name.
// For trivial ops this map must be empty.
std::unordered_map<string, std::unique_ptr<ExtendedInferenceContext>>
nested_inferences_;
TF_DISALLOW_COPY_AND_ASSIGN(ExtendedInferenceContext);
};
// ShapeRefiner performs shape inference for TensorFlow Graphs. It is
// responsible for instantiating InferenceContext objects for each
// Node in the Graph, and providing/storing the 'input_tensor' Tensors
// used by Shape Inference functions, when available at graph
// construction time.
class ShapeRefiner {
public:
ShapeRefiner(int graph_def_version, const OpRegistryInterface* ops);
// Same as ShapeRefiner(versions.producer(), ops)
ShapeRefiner(const VersionDef& versions, const OpRegistryInterface* ops);
~ShapeRefiner();
// Performs validation of 'node' and runs 'node's shape function,
// storing its shape outputs.
//
// All inputs of 'node' must be added to ShapeRefiner prior to
// adding 'node'.
//
// Returns an error if:
// - the shape function for 'node' was not registered.
// - 'node' was added before its inputs.
// - The shape inference function returns an error.
Status AddNode(const Node* node);
// Sets 'node's 'output_port' output to have shape 'shape'.
//
// Returns an error if 'node' was not previously added to this
// object, if 'output_port' is invalid, or if 'shape' is
// not compatible with the existing shape of the output.
Status SetShape(const Node* node, int output_port,
shape_inference::ShapeHandle shape);
// Update the input shapes of node in case the shapes of the fan-ins of 'node'
// have themselves been modified (For example, in case of incremental shape
// refinement). If 'relax' is true, a new shape with the broadest set of
// information will be set as the new input (see InferenceContext::RelaxInput
// for full details and examples). Sets refined to true if any shapes have
// changed (in their string representations). Note that shapes may have been
// updated to newer versions (but with identical string representations) even
// if <*refined> is set to false.
Status UpdateNode(const Node* node, bool relax, bool* refined);
// Returns the InferenceContext for 'node', if present.
shape_inference::InferenceContext* GetContext(const Node* node) const {
auto it = node_to_context_.find(node);
if (it == node_to_context_.end()) {
return nullptr;
}
return it->second->get_context();
}
// Returns the ExtendedInferenceContext for 'node', if present.
ExtendedInferenceContext* GetExtendedContext(const Node* node) const {
auto it = node_to_context_.find(node);
if (it == node_to_context_.end()) {
return nullptr;
}
return it->second.get();
}
// Getters and setters for graph_def_version_.
int32 graph_def_version() const { return graph_def_version_; }
void set_graph_def_version(int32 version) { graph_def_version_ = version; }
void set_require_shape_inference_fns(bool require_shape_inference_fns) {
require_shape_inference_fns_ = require_shape_inference_fns;
}
void set_disable_constant_propagation(bool disable) {
disable_constant_propagation_ = disable;
}
// Set function library to enable function shape inference.
// Without function library, function inference always yields unknown shapes.
// With this enabled, shape inference can take more time since it descends
// into all function calls. It doesn't do inference once for each function
// definition, but once for each function call.
// The function library must outlive the shape refiner.
void set_function_library_for_shape_inference(
const tensorflow::FunctionLibraryDefinition* lib) {
function_library_ = lib;
}
bool function_shape_inference_supported() const {
return function_library_ != nullptr;
}
// Call this to keep nested shapes information for user-defined functions:
// nested inferences will be available on the ExtendedInferenceContext for
// each function node, forming a tree of shape inferences corresponding to the
// tree of nested function calls. By default this setting is disabled, and
// only the shapes for the top-level function node will be reported on the
// InferenceContext for each function node, to reduce memory usage.
//
// This flag has no effect when the function inference is not enabled via
// set_function_library_for_shape_inference.
void set_keep_nested_shape_inferences() {
keep_nested_shape_inferences_ = true;
}
private:
friend class ShapeRefinerTest;
friend class ::tensorflow::grappler::GraphProperties;
// Returns true if the ranks and all dimensions of <s0> and <s1> are either
// equal in value or both unknown.
static bool SameDefinedShape(shape_inference::InferenceContext* c,
shape_inference::ShapeHandle s0,
shape_inference::ShapeHandle s1);
// Returns true if the shapes and types stored in <*existing> are identical in
// value to the shapes and types in <*updated>.
static bool IsUpdatedShapesOrTypes(
shape_inference::InferenceContext* c,
const std::vector<shape_inference::ShapeAndType>& existing,
const std::vector<shape_inference::ShapeAndType>& updated);
// Performs shape inference for the given function_def within the
// given outer_context. Internally it instantiates the function as a graph
// and runs shape inference recursively on it with the input shapes provided
// by the outer_context.
//
// Returns an error if:
// - number of inputs/outputs on outer_context doesn't match the function_def
//
// On success:
// - outer_context will contain output shapes inferred from input shapes
// - outer_context will contain nested inferences collection, iff
// keep_nested_shapes is true
Status InferShapesForFunction(const tensorflow::FunctionDef* function_def,
bool keep_nested_shapes,
ExtendedInferenceContext* outer_context);
// Attempts to evaluate the 'dst_idx'-th input to 'node'. If the input edge
// value can be evaluated, 'evaluated' is set to true and the value returned
// in 'result'. Otherwise 'evaluated' is set to false.
Status EvaluateConstantTensorForEdge(const Node* node, int dst_idx,
bool* evaluated, Tensor* result);
// Wrapper around EvaluateConstantTensorForEdge for scalar int32/int64 input
// tensors. The caller is responsible for checking that the specified edge is
// scalar and int32 or int64.
Status EvaluateConstantIntScalarEdge(const Node* node, int dst_idx,
bool* evaluated, int64* result);
// This function tries to materialize as much information about the 'node''s
// dst_idx input as a statically computable shape, and the result may be
// partially known, depending on what is statically inferable.
//
// This is called when node.input[dst_idx] is a tensor that is used to define
// the shape of some other tensor (e.g., the second argument to Reshape is a
// <shape> tensor, where each element of the shape tensor is a dimension of
// the target tensor). It returns in <result> a shape for that input.
//
// Unlike simply resolving node.input[dst_idx] to a constant and then
// converting that to a shape, this function can return a partial shape. This
// is useful for cases where the shape tensor is only partially defined, such
// as with calls for: reshape(x, shape(y)) where shape(y) is partially
// defined.
//
// The implementation has op implementations for ops commonly called on shape
// tensors, and the implementations are specialized to shape tensors (namely,
// the output is a vector).
//
// <target_context> is used when creating new DimensionHandle and ShapeHandle
// objects.
Status ConstantPartialShape(shape_inference::InferenceContext* target_context,
const Node* node, int dst_idx,
shape_inference::ShapeHandle* result);
// Implementation of ConstantPartialShape for StridedSlice nodes.
Status PartialStridedSliceShape(Node* slice_node,
shape_inference::InferenceContext* ctx,
shape_inference::ShapeHandle* result);
Status RunShapeFn(const Node* node, const OpRegistrationData* op_reg_data,
ExtendedInferenceContext* ec);
int32 graph_def_version_;
const OpRegistryInterface* const ops_registry_;
// The lifetime of the tensors are bound to the runner, so it should be the
// deleted after the tensors.
GraphRunner graph_runner_;
// Stores a map from a node to its ExtendedInferenceContext.
std::unordered_map<const Node*, std::unique_ptr<ExtendedInferenceContext>>
node_to_context_;
// Holds a cache from 'tensor name' to the tensor that is
// evaluatable as a constant expression. This reduces repeated
// execution of the entire constant subgraph as a graph is being
// built up. This could be changed to some kind of size-based LRU
// cache to avoid consuming too much memory, if that eventually
// becomes a concern.
//
// Only tensors less than 1KiB are currently stored in the cache.
static constexpr int64 kMaxTensorSize = 1024;
std::unordered_map<string, Tensor> const_tensor_map_;
bool require_shape_inference_fns_ = true;
bool disable_constant_propagation_ = false;
// Function library is optional, but has to be set to enable function
// shape inference.
const tensorflow::FunctionLibraryDefinition* function_library_ = nullptr;
// Determines whether to keep the nested shape inference info for user-
// defined functions. By default that info is discarded to save memory.
bool keep_nested_shape_inferences_ = false;
// Cache the graph corresponding to each functin definition for which shapes
// are refined.
std::unordered_map<const FunctionDef*, std::unique_ptr<const Graph>>
functions_;
TF_DISALLOW_COPY_AND_ASSIGN(ShapeRefiner);
};
} // namespace tensorflow
#endif // TENSORFLOW_CORE_COMMON_RUNTIME_SHAPE_REFINER_H_
@@ -0,0 +1,83 @@
/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
#ifndef TENSORFLOW_CORE_COMMON_RUNTIME_SINGLE_THREADED_CPU_DEVICE_H_
#define TENSORFLOW_CORE_COMMON_RUNTIME_SINGLE_THREADED_CPU_DEVICE_H_
#define EIGEN_USE_THREADS
#include "third_party/eigen3/unsupported/Eigen/CXX11/Tensor"
#include "tensorflow/core/common_runtime/device.h"
#include "tensorflow/core/common_runtime/eigen_thread_pool.h"
#include "tensorflow/core/framework/allocator.h"
#include "tensorflow/core/framework/tensor.pb.h"
#include "tensorflow/core/lib/core/threadpool.h"
namespace tensorflow {
class Env;
// A simple single-threaded CPU device. This can be used to run inexpensive
// computations. In particular, using this avoids initializing the global thread
// pools in LocalDevice.
class SingleThreadedCpuDevice : public Device {
public:
SingleThreadedCpuDevice(Env* env)
: Device(env, Device::BuildDeviceAttributes("/device:CPU:0", DEVICE_CPU,
Bytes(256 << 20),
DeviceLocality())) {
eigen_worker_threads_.num_threads = 1;
eigen_worker_threads_.workers = new thread::ThreadPool(
env, "graph_runner", eigen_worker_threads_.num_threads);
eigen_threadpool_wrapper_.reset(
new EigenThreadPoolWrapper(eigen_worker_threads_.workers));
eigen_device_.reset(new Eigen::ThreadPoolDevice(
eigen_threadpool_wrapper_.get(), eigen_worker_threads_.num_threads));
set_tensorflow_cpu_worker_threads(&eigen_worker_threads_);
set_eigen_cpu_device(eigen_device_.get());
}
~SingleThreadedCpuDevice() override {
eigen_threadpool_wrapper_.reset();
eigen_device_.reset();
delete eigen_worker_threads_.workers;
}
Status Sync() override { return Status::OK(); }
Status MakeTensorFromProto(const TensorProto& tensor_proto,
const AllocatorAttributes alloc_attrs,
Tensor* tensor) override {
Tensor parsed(tensor_proto.dtype());
if (!parsed.FromProto(cpu_allocator(), tensor_proto)) {
return errors::InvalidArgument("Cannot parse tensor from tensor_proto.");
}
*tensor = parsed;
return Status::OK();
}
Allocator* GetAllocator(AllocatorAttributes attr) override {
return cpu_allocator();
}
private:
DeviceBase::CpuWorkerThreads eigen_worker_threads_;
std::unique_ptr<Eigen::ThreadPoolInterface> eigen_threadpool_wrapper_;
std::unique_ptr<Eigen::ThreadPoolDevice> eigen_device_;
};
} // namespace tensorflow
#endif // TENSORFLOW_CORE_COMMON_RUNTIME_SINGLE_THREADED_CPU_DEVICE_H_
@@ -0,0 +1,64 @@
/* Copyright 2016 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
#ifndef TENSORFLOW_CORE_COMMON_RUNTIME_STATS_PUBLISHER_INTERFACE_H_
#define TENSORFLOW_CORE_COMMON_RUNTIME_STATS_PUBLISHER_INTERFACE_H_
#include "tensorflow/core/common_runtime/build_graph_options.h"
#include "tensorflow/core/common_runtime/profile_handler.h"
#include "tensorflow/core/protobuf/config.pb.h"
#include "tensorflow/core/public/session_options.h"
namespace tensorflow {
// StatsPublisherInterface describes objects that publish information exported
// by Sessions.
// NOTE: This interface is experimental and subject to change.
// Implementations must be thread-safe.
class StatsPublisherInterface {
public:
// PublishStatsProto publishes step_stats.
// When PublishStatsProto is called multiple times, only the step_stats
// corresponding to the latest call will be published.
virtual void PublishStatsProto(const StepStats& step_stats) = 0;
// PublishGraphProto publishes the graph_defs corresponding to each partition
// in the session.
// When PublishGraphProto is called multiple times, only the graph_defs
// corresponding to the latest call will be published.
virtual void PublishGraphProto(
const std::vector<const GraphDef*>& graph_defs) = 0;
// Returns a profile handler for the given step based on the execution_count
// and RunOptions.
//
// This method may return a null pointer, if no handler was created.
virtual std::unique_ptr<ProfileHandler> GetProfileHandler(
uint64 step, int64 execution_count, const RunOptions& ropts) = 0;
virtual ~StatsPublisherInterface() {}
};
typedef std::function<std::unique_ptr<StatsPublisherInterface>(
const string&, const BuildGraphOptions&, const SessionOptions&)>
StatsPublisherFactory;
std::unique_ptr<StatsPublisherInterface> CreateNoOpStatsPublisher(
const string& session, const BuildGraphOptions& bopts,
const SessionOptions& sopts);
} // namespace tensorflow
#endif // TENSORFLOW_CORE_COMMON_RUNTIME_STATS_PUBLISHER_INTERFACE_H_
@@ -0,0 +1,200 @@
/* Copyright 2015 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
#ifndef TENSORFLOW_CORE_COMMON_RUNTIME_STEP_STATS_COLLECTOR_H_
#define TENSORFLOW_CORE_COMMON_RUNTIME_STEP_STATS_COLLECTOR_H_
#include <memory>
#include <unordered_map>
#include <vector>
#include "tensorflow/core/framework/step_stats.pb.h"
#include "tensorflow/core/framework/tensor_reference.h"
#include "tensorflow/core/lib/gtl/inlined_vector.h"
#include "tensorflow/core/platform/env.h"
#include "tensorflow/core/platform/mutex.h"
#include "tensorflow/core/platform/thread_annotations.h"
#include "tensorflow/core/platform/types.h"
namespace tensorflow {
class Allocator;
class AllocatorMemoryUsed;
class CostModelManager;
class Graph;
class Node;
class NodeExecStats;
class OpKernelContext;
class StepStats;
class StepStatsCollector;
class Tensor;
class TrackingAllocator;
// Statistics collection interface for individual node execution.
//
// See `NodeExecStatsWrapper` for a concrete implementation of this interface
// that interfaces with the `Session` layer.
class NodeExecStatsInterface {
public:
virtual ~NodeExecStatsInterface() {}
// Called when the statistics collection for the node has finished. Once this
// method is called, the caller should not make assumptions about the validity
// of this object.
virtual void Done(const string& device) = 0;
// Called immediately after this node starts being processed by the executor.
virtual void RecordExecutorStarted() = 0;
// Called immediately before this node's `Compute()` or `ComputeAsync()`
// method is called.
virtual void RecordComputeStarted() = 0;
// Called immediately after this node's `Compute()` method returned (or, for
// asynchronous operations, the callback passed to its `ComputeAsync()` method
// was called).
virtual void RecordComputeEnded() = 0;
// Called immediately after this executor finishes processing this node.
virtual void RecordExecutorEnded() = 0;
// Records information about the memory allocated during the execution of this
// node.
virtual void SetMemory(OpKernelContext* ctx) = 0;
// Records information about the tensor produced by this node at the given
// output slot.
virtual void SetOutput(int slot, const Tensor* tensor) = 0;
// Records information about the tensors that were accessed during the
// execution of this node.
virtual void SetReferencedTensors(const TensorReferenceVector& tensors) = 0;
// Records the absolute time in nanoseconds at which this node became
// runnable (i.e. was scheduled for execution).
virtual void SetScheduled(int64 nanos) = 0;
};
// Wraps NodeExecStats and adds allocation to it.
class NodeExecStatsWrapper : public NodeExecStatsInterface {
public:
// Does not take ownership of `node` or `step_stats_collector`.
NodeExecStatsWrapper(const Node* node,
StepStatsCollector* step_stats_collector);
// Takes ownership of 'stats' but not `node` or `step_stats_collector`.
NodeExecStatsWrapper(std::unique_ptr<NodeExecStats> stats, const Node* node,
StepStatsCollector* step_stats_collector);
// Destructor calls Finalize() to release the TrackingAllocators.
~NodeExecStatsWrapper() { Finalize(); }
void Done(const string& device) override;
void RecordExecutorStarted() override;
void RecordComputeStarted() override;
void RecordComputeEnded() override;
void RecordExecutorEnded() override;
void SetMemory(OpKernelContext* ctx) override;
void SetOutput(int slot, const Tensor* tensor) override;
void SetReferencedTensors(const TensorReferenceVector& tensors) override;
void SetScheduled(int64 nanos) override;
private:
friend class StepStatsCollector;
NodeExecStats* stats() { return stats_.get(); }
// Populates stats_ and releases TrackingAllocator.
void Finalize();
// Does not take ownership of the `allocator`.
// Takes ownership of `tracking_allocator`.
void AddAllocation(Allocator* allocator,
TrackingAllocator* tracking_allocator);
gtl::InlinedVector<std::pair<AllocatorMemoryUsed*, TrackingAllocator*>, 2>
allocations_;
std::unique_ptr<NodeExecStats> stats_;
const Node* const node_; // Not owned.
StepStatsCollector* const step_stats_collector_; // Not owned.
};
// Statistics collection interface for step execution.
//
// See `StepStatsCollector` for a concrete implementation of this interface
// that interfaces with the `Session` layer.
class StepStatsCollectorInterface {
public:
virtual ~StepStatsCollectorInterface() {}
// Creates an instance of `NodeExecStatsInterface` that should be used for
// collecting statistics about individual node execution.
virtual NodeExecStatsInterface* CreateNodeExecStats(const Node* node) = 0;
// Generates a string reporting the currently used memory based
// on ResourceExhausted OOM `err` message.
// `err` message needs to contain device name and allocator name, e.g.:
// "ResourceExhaustedError: OOM when allocating tensor ...
// on /job:localhost/replica:0/task:0/device:GPU:0 by allocator GPU_0_bfc"
virtual string ReportAllocsOnResourceExhausted(const string& err) = 0;
};
// StepStatsCollector manages the collection of a StepStats object.
// The StepStats object holds multiple DeviceStats.
// Each DeviceStats object holds multiple NodeExecStats.
class StepStatsCollector : public StepStatsCollectorInterface {
public:
// Does not take ownership of `step_stats`.
explicit StepStatsCollector(StepStats* step_stats);
// BuildCostModel builds or updates a CostModel managed by cost_model_manager,
// using the currently collected DeviceStats associated with the devices in
// device_map.
void BuildCostModel(
CostModelManager* cost_model_manager,
const std::unordered_map<string, const Graph*>& device_map);
// Saves node statistics to the DeviceStats object associated with device.
// Should be called before Finalize.
void Save(const string& device, NodeExecStats* node_stats_pb);
void Save(const string& device, NodeExecStatsWrapper* node_stats);
NodeExecStatsInterface* CreateNodeExecStats(const Node* node) override;
string ReportAllocsOnResourceExhausted(const string& err) override;
// The following 2 Finalize methods populate the StepStats passed
// from the constructor. Calling it more than once won't have any effect.
// User shouldn't call Save() methods after Finalize.
void Finalize();
// swaps the content of StepStats* from constructor with 'ss'.
void FinalizeAndSwap(StepStats* step_stats);
private:
// TODO(suharshs): Make this configurable if its not possible to find a value
// that works for all cases.
static const uint64 kMaxCollectedNodes = 1 << 20;
typedef std::vector<std::unique_ptr<NodeExecStatsWrapper>> NodeStatsVector;
void FinalizeInternal() EXCLUSIVE_LOCKS_REQUIRED(mu_);
mutex mu_;
bool finalized_ GUARDED_BY(mu_);
std::unordered_map<string, NodeStatsVector> dev_stats_ GUARDED_BY(mu_);
StepStats* step_stats_ GUARDED_BY(mu_);
uint64 collected_nodes_ GUARDED_BY(mu_) = 0;
};
} // namespace tensorflow
#endif // TENSORFLOW_CORE_COMMON_RUNTIME_STEP_STATS_COLLECTOR_H_
@@ -0,0 +1,75 @@
/* Copyright 2016 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
#if !TENSORFLOW_USE_SYCL
#error This file must only be included when building TensorFlow with SYCL support
#endif
#ifndef TENSORFLOW_CORE_COMMON_RUNTIME_SYCL_SYCL_ALLOCATOR_H_
#define TENSORFLOW_CORE_COMMON_RUNTIME_SYCL_SYCL_ALLOCATOR_H_
#include "third_party/eigen3/unsupported/Eigen/CXX11/Tensor"
#include "tensorflow/core/framework/allocator.h"
#include "tensorflow/core/platform/mutex.h"
#include "tensorflow/core/platform/types.h"
namespace tensorflow {
class SYCLAllocator : public Allocator {
public:
SYCLAllocator(Eigen::QueueInterface* queue);
virtual ~SYCLAllocator() override;
string Name() override;
void* AllocateRaw(size_t alignment, size_t num_bytes) override;
void DeallocateRaw(void* ptr) override;
virtual bool ShouldAllocateEmptyTensors() override final { return true; }
void Synchronize() {
mutex_lock lock(mu_);
if (sycl_device_) {
sycl_device_->synchronize();
}
}
bool Ok() { return sycl_device_ && sycl_device_->ok(); }
void GetStats(AllocatorStats* stats) override;
void ClearStats() override;
// The SYCL buffers keep track of their size, so we already have tracking.
bool TracksAllocationSizes() override { return true; }
// Get the size of the corresponding SYCL buffer.
// Implementing this also provides an implementation of
// AllocatedSize(void* ptr) by default.
size_t RequestedSize(void* ptr) override;
Eigen::SyclDevice* getSyclDevice() { return sycl_device_; }
// Clear the SYCL device used by the Allocator
void ClearSYCLDevice() {
mutex_lock lock(mu_);
if (sycl_device_) {
delete sycl_device_;
sycl_device_ = nullptr;
}
}
private:
mutable mutex mu_;
Eigen::SyclDevice* sycl_device_ GUARDED_BY(mu_); // owned
AllocatorStats stats_ GUARDED_BY(mu_);
TF_DISALLOW_COPY_AND_ASSIGN(SYCLAllocator);
};
} // namespace tensorflow
#endif // TENSORFLOW_CORE_COMMON_RUNTIME_SYCL_SYCL_ALLOCATOR_H_
@@ -0,0 +1,232 @@
/* Copyright 2016 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
#if !TENSORFLOW_USE_SYCL
#error This file must only be included when building TensorFlow with SYCL support
#endif
#ifndef TENSORFLOW_CORE_COMMON_RUNTIME_SYCL_SYCL_DEVICE_H_
#define TENSORFLOW_CORE_COMMON_RUNTIME_SYCL_SYCL_DEVICE_H_
#include "tensorflow/core/common_runtime/local_device.h"
#include "tensorflow/core/common_runtime/sycl/sycl_allocator.h"
#include "tensorflow/core/common_runtime/sycl/sycl_device_context.h"
#include "tensorflow/core/public/session_options.h"
namespace tensorflow {
class GSYCLInterface {
std::vector<Eigen::QueueInterface*> m_queue_interface_; // owned
std::vector<Allocator*> m_cpu_allocator_; // not owned
std::vector<SYCLAllocator*> m_sycl_allocator_; // owned
std::vector<SYCLDeviceContext*> m_sycl_context_; // ref counted
GSYCLInterface() {
bool found_device = false;
auto device_list = Eigen::get_sycl_supported_devices();
// Obtain list of supported devices from Eigen
for (const auto& device : device_list) {
if (device.is_gpu()) {
// returns first found GPU
AddDevice(device);
found_device = true;
}
}
if (!found_device) {
// Currently Intel GPU is not supported
LOG(WARNING) << "No OpenCL GPU found that is supported by "
<< "ComputeCpp/triSYCL, trying OpenCL CPU";
}
for (const auto& device : device_list) {
if (device.is_cpu()) {
// returns first found CPU
AddDevice(device);
found_device = true;
}
}
if (!found_device) {
LOG(WARNING) << "No OpenCL CPU found that is supported by "
<< "ComputeCpp/triSYCL, checking for host sycl device";
}
for (const auto& device : device_list) {
// triSYCL only supports the host device for now
if (device.is_host()) {
LOG(WARNING) << "Found SYCL host device";
AddDevice(device);
found_device = true;
}
}
if (!found_device) {
// Currently Intel GPU is not supported
LOG(FATAL) << "No SYCL host and no OpenCL GPU nor CPU"
<< " supported by ComputeCPP/triSYCL was found";
} else {
LOG(INFO) << "Found following OpenCL devices:";
for (int i = 0; i < device_list.size(); i++) {
LOG(INFO) << GetShortDeviceDescription(i);
}
}
}
~GSYCLInterface() {
m_cpu_allocator_.clear();
for (auto p : m_sycl_allocator_) {
p->Synchronize();
p->ClearSYCLDevice();
// Cannot delete the Allocator instances, as the Allocator lifetime
// needs to exceed any Tensor created by it. There is no way of
// knowing when all Tensors have been deallocated, as they are
// RefCounted and wait until all instances of a Tensor have been
// destroyed before calling Allocator.Deallocate. This could happen at
// program exit, which can set up a race condition between destroying
// Tensors and Allocators when the program is cleaning up.
}
m_sycl_allocator_.clear();
for (auto p : m_sycl_context_) {
p->Unref();
}
m_sycl_context_.clear();
for (auto p : m_queue_interface_) {
p->deallocate_all();
delete p;
}
m_queue_interface_.clear();
}
void AddDevice(const cl::sycl::device& d) {
m_queue_interface_.push_back(new Eigen::QueueInterface(d));
m_cpu_allocator_.push_back(cpu_allocator());
m_sycl_allocator_.push_back(new SYCLAllocator(m_queue_interface_.back()));
m_sycl_context_.push_back(new SYCLDeviceContext());
}
public:
static const GSYCLInterface* instance() {
// c++11 guarantees that this will be constructed in a thread safe way
static const GSYCLInterface instance;
return &instance;
}
Eigen::QueueInterface* GetQueueInterface(size_t i = 0) const {
if (!m_queue_interface_.empty()) {
return m_queue_interface_[i];
} else {
std::cerr << "No cl::sycl::device has been added" << std::endl;
return nullptr;
}
}
SYCLAllocator* GetSYCLAllocator(size_t i = 0) const {
if (!m_sycl_allocator_.empty()) {
return m_sycl_allocator_[i];
} else {
std::cerr << "No cl::sycl::device has been added" << std::endl;
return nullptr;
}
}
Allocator* GetCPUAllocator(size_t i = 0) const {
if (!m_cpu_allocator_.empty()) {
return m_cpu_allocator_[i];
} else {
std::cerr << "No cl::sycl::device has been added" << std::endl;
return nullptr;
}
}
SYCLDeviceContext* GetSYCLContext(size_t i = 0) const {
if (!m_sycl_context_.empty()) {
return m_sycl_context_[i];
} else {
std::cerr << "No cl::sycl::device has been added" << std::endl;
return nullptr;
}
}
string GetShortDeviceDescription(int device_id = 0) const {
Eigen::QueueInterface* queue_ptr = GetQueueInterface(device_id);
if (!queue_ptr) {
LOG(ERROR)
<< "Device name cannot be given after Eigen QueueInterface destroyed";
return "";
}
auto device = queue_ptr->sycl_queue().get_device();
auto name = device.get_info<cl::sycl::info::device::name>();
auto vendor = device.get_info<cl::sycl::info::device::vendor>();
auto profile = device.get_info<cl::sycl::info::device::profile>();
std::string type;
if (device.is_host()) {
type = "Host";
} else if (device.is_cpu()) {
type = "CPU";
} else if (device.is_gpu()) {
type = "GPU";
} else if (device.is_accelerator()) {
type = "Accelerator";
} else {
type = "Unknown";
}
return strings::StrCat(
"id: ", device_id, ", type: ", type, ", name: ", name.c_str(),
", vendor: ", vendor.c_str(), ", profile: ", profile.c_str());
}
};
class SYCLDevice : public LocalDevice {
public:
SYCLDevice(const SessionOptions& options, const string& name,
Bytes memory_limit, const DeviceLocality& locality,
const string& physical_device_desc, SYCLAllocator* sycl_allocator,
Allocator* cpu_allocator, SYCLDeviceContext* ctx)
: LocalDevice(options, Device::BuildDeviceAttributes(
name, DEVICE_SYCL, memory_limit, locality,
physical_device_desc)),
cpu_allocator_(cpu_allocator),
sycl_allocator_(sycl_allocator),
device_context_(ctx) {
set_eigen_sycl_device(sycl_allocator->getSyclDevice());
}
~SYCLDevice() override;
void Compute(OpKernel* op_kernel, OpKernelContext* context) override;
Allocator* GetAllocator(AllocatorAttributes attr) override;
Status MakeTensorFromProto(const TensorProto& tensor_proto,
const AllocatorAttributes alloc_attrs,
Tensor* tensor) override;
Status FillContextMap(const Graph* graph,
DeviceContextMap* device_context_map) override;
Status Sync() override;
private:
Allocator* cpu_allocator_; // not owned
SYCLAllocator* sycl_allocator_; // not owned
SYCLDeviceContext* device_context_; // not owned
};
} // namespace tensorflow
#endif // TENSORFLOW_CORE_COMMON_RUNTIME_SYCL_SYCL_DEVICE_H_
@@ -0,0 +1,45 @@
/* Copyright 2016 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
#if !TENSORFLOW_USE_SYCL
#error This file must only be included when building TensorFlow with SYCL support
#endif
#ifndef TENSORFLOW_COMMON_RUNTIME_SYCL_SYCL_DEVICE_CONTEXT_H_
#define TENSORFLOW_COMMON_RUNTIME_SYCL_SYCL_DEVICE_CONTEXT_H_
#include "tensorflow/core/common_runtime/device.h"
#include "tensorflow/core/framework/device_base.h"
namespace tensorflow {
class SYCLDeviceContext : public DeviceContext {
public:
SYCLDeviceContext() {}
~SYCLDeviceContext() override {}
void CopyCPUTensorToDevice(const Tensor *cpu_tensor, Device *device,
Tensor *device_tensor,
StatusCallback done) const override;
void CopyDeviceTensorToCPU(const Tensor *device_tensor, StringPiece edge_name,
Device *device, Tensor *cpu_tensor,
StatusCallback done) override;
};
} // namespace tensorflow
#endif // TENSORFLOW_COMMON_RUNTIME_SYCL_SYCL_DEVICE_CONTEXT_H_
@@ -0,0 +1,80 @@
/* Copyright 2017 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
#if !TENSORFLOW_USE_SYCL
#error This file must only be included when building TensorFlow with SYCL support
#endif
#ifndef TENSORFLOW_CORE_COMMON_RUNTIME_SYCL_SYCL_UTIL_H_
#define TENSORFLOW_CORE_COMMON_RUNTIME_SYCL_SYCL_UTIL_H_
#include "third_party/eigen3/unsupported/Eigen/CXX11/Tensor"
#include "tensorflow/core/common_runtime/device.h"
// For DMA helper
#include "tensorflow/core/common_runtime/dma_helper.h"
#include "tensorflow/core/framework/tensor.h"
namespace tensorflow {
inline void const* GetBase(const Tensor* src) { return DMAHelper::base(src); }
inline void* GetBase(Tensor* dst) { return DMAHelper::base(dst); }
inline void SYCLmemcpy(Eigen::SyclDevice const& device,
Tensor const& src_tensor, Tensor* dst_tensor) {
const size_t size = src_tensor.TotalBytes();
void* dst_ptr = GetBase(dst_tensor);
void const* src_ptr = GetBase(&src_tensor);
#define COPY_WITH_TYPE(T) \
device.memcpy(dst_ptr, static_cast<T const*>(src_ptr), size);
switch (src_tensor.dtype()) {
case DT_COMPLEX128:
COPY_WITH_TYPE(cl::sycl::cl_ulong2);
break;
case DT_DOUBLE:
case DT_COMPLEX64:
case DT_INT64:
COPY_WITH_TYPE(cl::sycl::cl_ulong);
break;
case DT_FLOAT:
case DT_INT32:
case DT_QINT32:
COPY_WITH_TYPE(cl::sycl::cl_uint);
break;
case DT_INT16:
case DT_UINT16:
case DT_BFLOAT16:
case DT_QINT16:
case DT_QUINT16:
case DT_HALF:
COPY_WITH_TYPE(cl::sycl::cl_ushort);
break;
case DT_BOOL:
COPY_WITH_TYPE(bool);
break;
case DT_UINT8:
case DT_INT8:
case DT_QINT8:
case DT_QUINT8:
COPY_WITH_TYPE(cl::sycl::cl_uchar);
break;
default:
LOG(FATAL) << "Unknown data type " << src_tensor.dtype();
break;
}
#undef COPY_WITH_TYPE
}
} // namespace tensorflow
#endif // TENSORFLOW_CORE_COMMON_RUNTIME_SYCL_SYCL_UTIL_H_
@@ -0,0 +1,51 @@
/* Copyright 2015 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
#ifndef TENSORFLOW_CORE_COMMON_RUNTIME_THREADPOOL_DEVICE_H_
#define TENSORFLOW_CORE_COMMON_RUNTIME_THREADPOOL_DEVICE_H_
#include "tensorflow/core/common_runtime/device_factory.h"
#include "tensorflow/core/common_runtime/local_device.h"
namespace tensorflow {
// CPU device implementation.
class ThreadPoolDevice : public LocalDevice {
public:
ThreadPoolDevice(const SessionOptions& options, const string& name,
Bytes memory_limit, const DeviceLocality& locality,
Allocator* allocator);
~ThreadPoolDevice() override;
Allocator* GetAllocator(AllocatorAttributes attr) override;
Allocator* GetScopedAllocator(AllocatorAttributes attr,
int64 step_id) override;
ScopedAllocatorMgr* GetScopedAllocatorMgr() const override {
return scoped_allocator_mgr_.get();
}
Status MakeTensorFromProto(const TensorProto& tensor_proto,
const AllocatorAttributes alloc_attrs,
Tensor* tensor) override;
Status Sync() override { return Status::OK(); }
private:
Allocator* allocator_; // Not owned
std::unique_ptr<ScopedAllocatorMgr> scoped_allocator_mgr_;
};
} // namespace tensorflow
#endif // TENSORFLOW_CORE_COMMON_RUNTIME_THREADPOOL_DEVICE_H_
@@ -0,0 +1,126 @@
/* Copyright 2016 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
#ifndef TENSORFLOW_CORE_DEBUG_DEBUG_GRAPH_UTILS_H_
#define TENSORFLOW_CORE_DEBUG_DEBUG_GRAPH_UTILS_H_
#include <unordered_map>
#include <vector>
#include "tensorflow/core/common_runtime/debugger_state_interface.h"
#include "tensorflow/core/common_runtime/device.h"
#include "tensorflow/core/graph/graph.h"
#include "tensorflow/core/lib/core/status.h"
#include "tensorflow/core/platform/protobuf.h"
#include "tensorflow/core/protobuf/debug.pb.h"
namespace tensorflow {
class DebugNodeInserter {
public:
// EXPERIMENTAL: Insert special debug ops (e.g., DebugIdentity) to graph for
// debugging. Currently, such ops need to take exactly one input and has the
// string attribute "tensor_name" to indicate what tensor it watches.
// For example, before the node insertion, the graph may look like:
//
// A:0 -----------1----------> B
// |
// ---------2-----------> C
//
// wherein the output slot 0 of node A feeds as the input to nodes B through
// edge 1 and to node C through edge 2.
// After the node insertion, assuming both B and C have non-Ref input, the
// graph becomes:
// A:0 ---3---> Copy -----------4----------> B
// |
// ---------5--------> C
// |
// ---------6--------> X
//
// If a node (e.g., B) has Ref input, the graph becomes:
//
// --------------------------------> B
// |
// A:0 ---3-----> Copy -----------4----------> C
// |
// -----------5--------> X
//
// In other words, we do not feed Refs to deep-copies to downstream nodes.
//
// Copy is the inserted deep-copy node that copies the input tensor on-device
// (e.g., CPU-to-CPU or GPU-to-GPU deep copy) that reduces the likelihood of
// racy updates during the debug watches. X is the newly created debug node
// that transforms the input (copy of the watched tensor) into a debug signal.
//
// DebugIdentity is the simplest debugging paradigm, in which the debug signal
// (i.e., X:0) equals the tensor itself. More sophisticated debug ops can be
// used to transform the tensor into other debug signals. An example is the
// DebugNanCounter op.
//
// If the nodes (A, B and C) are located on GPU and the edges from A to B or C
// is HOST_MEMORY, then the CopyHost op will be used instead of the Copy op.
static Status InsertNodes(
const protobuf::RepeatedPtrField<DebugTensorWatch>& watches, Graph* graph,
Device* device);
// Set the parallel_iterations attribute of TensorFlow while loops
// (specifically the nodes for which IsEnter() returns true) to 1 to prevent
// any node from being executed multiple times concurrently and
// generating temporally-overlapping debug Tensor dumps.
static void DeparallelizeWhileLoops(Graph* graph, Device* device);
// Get canonical name of a copy node.
static const string GetCopyNodeName(const string& node_name,
const int output_slot);
// Get canonical name of a debug node.
static const string GetDebugNodeName(const string& tensor_name,
const int debug_op_num,
const string& debug_op_name);
private:
static Status CreateCopyNode(
Graph* graph, const DeviceType device_type, const bool is_host_memory,
const string& src_node_name, const int src_output, const DataType src_dt,
const string& tensor_name, const std::vector<string>& debug_ops,
const std::vector<string>& debug_urls, Node** copy_node);
// Parse the debug_op_name string to extract proper op name and attributes.
// debug_op_name can be the proper op name only, e.g., "DebugNumericSummary".
// It can also contain customizable keys and values. Each key-value pair is
// connected with an equal sign ("="). Multiple key-value pairs are separated
// with semicolons (";"), which optional whitespace in between, e.g.,
// "DebugNumericSummary(mute_if_healthy=true, lower_bound=-100.0)".
static Status ParseDebugOpName(
const string& debug_op_name, string* debug_op_name_proper,
std::unordered_map<string, string>* attributes);
static Status SetDebugNodeAttributes(
Node* debug_node, const std::unordered_map<string, string>& attributes);
static Status CreateDebugNode(Graph* graph, const Device& device,
const string& src_copy_node_name,
const DataType src_dt,
const string& tensor_name,
const std::vector<string>& debug_urls,
const int debug_op_num,
const string& debug_op_name, Node** debug_node);
// TODO(cais): Cut down the number of args to this method.
friend class DebugGraphUtilsTest;
};
} // namespace tensorflow
#endif // TENSORFLOW_CORE_DEBUG_DEBUG_GRAPH_UTILS_H_
@@ -0,0 +1,539 @@
// Generated by the protocol buffer compiler. DO NOT EDIT!
// source: tensorflow/core/example/example.proto
#ifndef PROTOBUF_INCLUDED_tensorflow_2fcore_2fexample_2fexample_2eproto
#define PROTOBUF_INCLUDED_tensorflow_2fcore_2fexample_2fexample_2eproto
#include <string>
#include <google/protobuf/stubs/common.h>
#if GOOGLE_PROTOBUF_VERSION < 3006000
#error This file was generated by a newer version of protoc which is
#error incompatible with your Protocol Buffer headers. Please update
#error your headers.
#endif
#if 3006000 < GOOGLE_PROTOBUF_MIN_PROTOC_VERSION
#error This file was generated by an older version of protoc which is
#error incompatible with your Protocol Buffer headers. Please
#error regenerate this file with a newer version of protoc.
#endif
#include <google/protobuf/io/coded_stream.h>
#include <google/protobuf/arena.h>
#include <google/protobuf/arenastring.h>
#include <google/protobuf/generated_message_table_driven.h>
#include <google/protobuf/generated_message_util.h>
#include <google/protobuf/inlined_string_field.h>
#include <google/protobuf/metadata.h>
#include <google/protobuf/message.h>
#include <google/protobuf/repeated_field.h> // IWYU pragma: export
#include <google/protobuf/extension_set.h> // IWYU pragma: export
#include <google/protobuf/unknown_field_set.h>
#include "tensorflow/core/example/feature.pb.h"
// @@protoc_insertion_point(includes)
#define PROTOBUF_INTERNAL_EXPORT_protobuf_tensorflow_2fcore_2fexample_2fexample_2eproto
namespace protobuf_tensorflow_2fcore_2fexample_2fexample_2eproto {
// Internal implementation detail -- do not use these members.
struct TableStruct {
static const ::google::protobuf::internal::ParseTableField entries[];
static const ::google::protobuf::internal::AuxillaryParseTableField aux[];
static const ::google::protobuf::internal::ParseTable schema[2];
static const ::google::protobuf::internal::FieldMetadata field_metadata[];
static const ::google::protobuf::internal::SerializationTable serialization_table[];
static const ::google::protobuf::uint32 offsets[];
};
void AddDescriptors();
} // namespace protobuf_tensorflow_2fcore_2fexample_2fexample_2eproto
namespace tensorflow {
class Example;
class ExampleDefaultTypeInternal;
extern ExampleDefaultTypeInternal _Example_default_instance_;
class SequenceExample;
class SequenceExampleDefaultTypeInternal;
extern SequenceExampleDefaultTypeInternal _SequenceExample_default_instance_;
} // namespace tensorflow
namespace google {
namespace protobuf {
template<> ::tensorflow::Example* Arena::CreateMaybeMessage<::tensorflow::Example>(Arena*);
template<> ::tensorflow::SequenceExample* Arena::CreateMaybeMessage<::tensorflow::SequenceExample>(Arena*);
} // namespace protobuf
} // namespace google
namespace tensorflow {
// ===================================================================
class Example : public ::google::protobuf::Message /* @@protoc_insertion_point(class_definition:tensorflow.Example) */ {
public:
Example();
virtual ~Example();
Example(const Example& from);
inline Example& operator=(const Example& from) {
CopyFrom(from);
return *this;
}
#if LANG_CXX11
Example(Example&& from) noexcept
: Example() {
*this = ::std::move(from);
}
inline Example& operator=(Example&& from) noexcept {
if (GetArenaNoVirtual() == from.GetArenaNoVirtual()) {
if (this != &from) InternalSwap(&from);
} else {
CopyFrom(from);
}
return *this;
}
#endif
inline ::google::protobuf::Arena* GetArena() const final {
return GetArenaNoVirtual();
}
inline void* GetMaybeArenaPointer() const final {
return MaybeArenaPtr();
}
static const ::google::protobuf::Descriptor* descriptor();
static const Example& default_instance();
static void InitAsDefaultInstance(); // FOR INTERNAL USE ONLY
static inline const Example* internal_default_instance() {
return reinterpret_cast<const Example*>(
&_Example_default_instance_);
}
static constexpr int kIndexInFileMessages =
0;
void UnsafeArenaSwap(Example* other);
void Swap(Example* other);
friend void swap(Example& a, Example& b) {
a.Swap(&b);
}
// implements Message ----------------------------------------------
inline Example* New() const final {
return CreateMaybeMessage<Example>(NULL);
}
Example* New(::google::protobuf::Arena* arena) const final {
return CreateMaybeMessage<Example>(arena);
}
void CopyFrom(const ::google::protobuf::Message& from) final;
void MergeFrom(const ::google::protobuf::Message& from) final;
void CopyFrom(const Example& from);
void MergeFrom(const Example& from);
void Clear() final;
bool IsInitialized() const final;
size_t ByteSizeLong() const final;
bool MergePartialFromCodedStream(
::google::protobuf::io::CodedInputStream* input) final;
void SerializeWithCachedSizes(
::google::protobuf::io::CodedOutputStream* output) const final;
::google::protobuf::uint8* InternalSerializeWithCachedSizesToArray(
bool deterministic, ::google::protobuf::uint8* target) const final;
int GetCachedSize() const final { return _cached_size_.Get(); }
private:
void SharedCtor();
void SharedDtor();
void SetCachedSize(int size) const final;
void InternalSwap(Example* other);
protected:
explicit Example(::google::protobuf::Arena* arena);
private:
static void ArenaDtor(void* object);
inline void RegisterArenaDtor(::google::protobuf::Arena* arena);
private:
inline ::google::protobuf::Arena* GetArenaNoVirtual() const {
return _internal_metadata_.arena();
}
inline void* MaybeArenaPtr() const {
return _internal_metadata_.raw_arena_ptr();
}
public:
::google::protobuf::Metadata GetMetadata() const final;
// nested types ----------------------------------------------------
// accessors -------------------------------------------------------
// .tensorflow.Features features = 1;
bool has_features() const;
void clear_features();
static const int kFeaturesFieldNumber = 1;
private:
const ::tensorflow::Features& _internal_features() const;
public:
const ::tensorflow::Features& features() const;
::tensorflow::Features* release_features();
::tensorflow::Features* mutable_features();
void set_allocated_features(::tensorflow::Features* features);
void unsafe_arena_set_allocated_features(
::tensorflow::Features* features);
::tensorflow::Features* unsafe_arena_release_features();
// @@protoc_insertion_point(class_scope:tensorflow.Example)
private:
::google::protobuf::internal::InternalMetadataWithArena _internal_metadata_;
template <typename T> friend class ::google::protobuf::Arena::InternalHelper;
typedef void InternalArenaConstructable_;
typedef void DestructorSkippable_;
::tensorflow::Features* features_;
mutable ::google::protobuf::internal::CachedSize _cached_size_;
friend struct ::protobuf_tensorflow_2fcore_2fexample_2fexample_2eproto::TableStruct;
};
// -------------------------------------------------------------------
class SequenceExample : public ::google::protobuf::Message /* @@protoc_insertion_point(class_definition:tensorflow.SequenceExample) */ {
public:
SequenceExample();
virtual ~SequenceExample();
SequenceExample(const SequenceExample& from);
inline SequenceExample& operator=(const SequenceExample& from) {
CopyFrom(from);
return *this;
}
#if LANG_CXX11
SequenceExample(SequenceExample&& from) noexcept
: SequenceExample() {
*this = ::std::move(from);
}
inline SequenceExample& operator=(SequenceExample&& from) noexcept {
if (GetArenaNoVirtual() == from.GetArenaNoVirtual()) {
if (this != &from) InternalSwap(&from);
} else {
CopyFrom(from);
}
return *this;
}
#endif
inline ::google::protobuf::Arena* GetArena() const final {
return GetArenaNoVirtual();
}
inline void* GetMaybeArenaPointer() const final {
return MaybeArenaPtr();
}
static const ::google::protobuf::Descriptor* descriptor();
static const SequenceExample& default_instance();
static void InitAsDefaultInstance(); // FOR INTERNAL USE ONLY
static inline const SequenceExample* internal_default_instance() {
return reinterpret_cast<const SequenceExample*>(
&_SequenceExample_default_instance_);
}
static constexpr int kIndexInFileMessages =
1;
void UnsafeArenaSwap(SequenceExample* other);
void Swap(SequenceExample* other);
friend void swap(SequenceExample& a, SequenceExample& b) {
a.Swap(&b);
}
// implements Message ----------------------------------------------
inline SequenceExample* New() const final {
return CreateMaybeMessage<SequenceExample>(NULL);
}
SequenceExample* New(::google::protobuf::Arena* arena) const final {
return CreateMaybeMessage<SequenceExample>(arena);
}
void CopyFrom(const ::google::protobuf::Message& from) final;
void MergeFrom(const ::google::protobuf::Message& from) final;
void CopyFrom(const SequenceExample& from);
void MergeFrom(const SequenceExample& from);
void Clear() final;
bool IsInitialized() const final;
size_t ByteSizeLong() const final;
bool MergePartialFromCodedStream(
::google::protobuf::io::CodedInputStream* input) final;
void SerializeWithCachedSizes(
::google::protobuf::io::CodedOutputStream* output) const final;
::google::protobuf::uint8* InternalSerializeWithCachedSizesToArray(
bool deterministic, ::google::protobuf::uint8* target) const final;
int GetCachedSize() const final { return _cached_size_.Get(); }
private:
void SharedCtor();
void SharedDtor();
void SetCachedSize(int size) const final;
void InternalSwap(SequenceExample* other);
protected:
explicit SequenceExample(::google::protobuf::Arena* arena);
private:
static void ArenaDtor(void* object);
inline void RegisterArenaDtor(::google::protobuf::Arena* arena);
private:
inline ::google::protobuf::Arena* GetArenaNoVirtual() const {
return _internal_metadata_.arena();
}
inline void* MaybeArenaPtr() const {
return _internal_metadata_.raw_arena_ptr();
}
public:
::google::protobuf::Metadata GetMetadata() const final;
// nested types ----------------------------------------------------
// accessors -------------------------------------------------------
// .tensorflow.Features context = 1;
bool has_context() const;
void clear_context();
static const int kContextFieldNumber = 1;
private:
const ::tensorflow::Features& _internal_context() const;
public:
const ::tensorflow::Features& context() const;
::tensorflow::Features* release_context();
::tensorflow::Features* mutable_context();
void set_allocated_context(::tensorflow::Features* context);
void unsafe_arena_set_allocated_context(
::tensorflow::Features* context);
::tensorflow::Features* unsafe_arena_release_context();
// .tensorflow.FeatureLists feature_lists = 2;
bool has_feature_lists() const;
void clear_feature_lists();
static const int kFeatureListsFieldNumber = 2;
private:
const ::tensorflow::FeatureLists& _internal_feature_lists() const;
public:
const ::tensorflow::FeatureLists& feature_lists() const;
::tensorflow::FeatureLists* release_feature_lists();
::tensorflow::FeatureLists* mutable_feature_lists();
void set_allocated_feature_lists(::tensorflow::FeatureLists* feature_lists);
void unsafe_arena_set_allocated_feature_lists(
::tensorflow::FeatureLists* feature_lists);
::tensorflow::FeatureLists* unsafe_arena_release_feature_lists();
// @@protoc_insertion_point(class_scope:tensorflow.SequenceExample)
private:
::google::protobuf::internal::InternalMetadataWithArena _internal_metadata_;
template <typename T> friend class ::google::protobuf::Arena::InternalHelper;
typedef void InternalArenaConstructable_;
typedef void DestructorSkippable_;
::tensorflow::Features* context_;
::tensorflow::FeatureLists* feature_lists_;
mutable ::google::protobuf::internal::CachedSize _cached_size_;
friend struct ::protobuf_tensorflow_2fcore_2fexample_2fexample_2eproto::TableStruct;
};
// ===================================================================
// ===================================================================
#ifdef __GNUC__
#pragma GCC diagnostic push
#pragma GCC diagnostic ignored "-Wstrict-aliasing"
#endif // __GNUC__
// Example
// .tensorflow.Features features = 1;
inline bool Example::has_features() const {
return this != internal_default_instance() && features_ != NULL;
}
inline const ::tensorflow::Features& Example::_internal_features() const {
return *features_;
}
inline const ::tensorflow::Features& Example::features() const {
const ::tensorflow::Features* p = features_;
// @@protoc_insertion_point(field_get:tensorflow.Example.features)
return p != NULL ? *p : *reinterpret_cast<const ::tensorflow::Features*>(
&::tensorflow::_Features_default_instance_);
}
inline ::tensorflow::Features* Example::release_features() {
// @@protoc_insertion_point(field_release:tensorflow.Example.features)
::tensorflow::Features* temp = features_;
if (GetArenaNoVirtual() != NULL) {
temp = ::google::protobuf::internal::DuplicateIfNonNull(temp);
}
features_ = NULL;
return temp;
}
inline ::tensorflow::Features* Example::unsafe_arena_release_features() {
// @@protoc_insertion_point(field_unsafe_arena_release:tensorflow.Example.features)
::tensorflow::Features* temp = features_;
features_ = NULL;
return temp;
}
inline ::tensorflow::Features* Example::mutable_features() {
if (features_ == NULL) {
auto* p = CreateMaybeMessage<::tensorflow::Features>(GetArenaNoVirtual());
features_ = p;
}
// @@protoc_insertion_point(field_mutable:tensorflow.Example.features)
return features_;
}
inline void Example::set_allocated_features(::tensorflow::Features* features) {
::google::protobuf::Arena* message_arena = GetArenaNoVirtual();
if (message_arena == NULL) {
delete reinterpret_cast< ::google::protobuf::MessageLite*>(features_);
}
if (features) {
::google::protobuf::Arena* submessage_arena =
reinterpret_cast<::google::protobuf::MessageLite*>(features)->GetArena();
if (message_arena != submessage_arena) {
features = ::google::protobuf::internal::GetOwnedMessage(
message_arena, features, submessage_arena);
}
} else {
}
features_ = features;
// @@protoc_insertion_point(field_set_allocated:tensorflow.Example.features)
}
// -------------------------------------------------------------------
// SequenceExample
// .tensorflow.Features context = 1;
inline bool SequenceExample::has_context() const {
return this != internal_default_instance() && context_ != NULL;
}
inline const ::tensorflow::Features& SequenceExample::_internal_context() const {
return *context_;
}
inline const ::tensorflow::Features& SequenceExample::context() const {
const ::tensorflow::Features* p = context_;
// @@protoc_insertion_point(field_get:tensorflow.SequenceExample.context)
return p != NULL ? *p : *reinterpret_cast<const ::tensorflow::Features*>(
&::tensorflow::_Features_default_instance_);
}
inline ::tensorflow::Features* SequenceExample::release_context() {
// @@protoc_insertion_point(field_release:tensorflow.SequenceExample.context)
::tensorflow::Features* temp = context_;
if (GetArenaNoVirtual() != NULL) {
temp = ::google::protobuf::internal::DuplicateIfNonNull(temp);
}
context_ = NULL;
return temp;
}
inline ::tensorflow::Features* SequenceExample::unsafe_arena_release_context() {
// @@protoc_insertion_point(field_unsafe_arena_release:tensorflow.SequenceExample.context)
::tensorflow::Features* temp = context_;
context_ = NULL;
return temp;
}
inline ::tensorflow::Features* SequenceExample::mutable_context() {
if (context_ == NULL) {
auto* p = CreateMaybeMessage<::tensorflow::Features>(GetArenaNoVirtual());
context_ = p;
}
// @@protoc_insertion_point(field_mutable:tensorflow.SequenceExample.context)
return context_;
}
inline void SequenceExample::set_allocated_context(::tensorflow::Features* context) {
::google::protobuf::Arena* message_arena = GetArenaNoVirtual();
if (message_arena == NULL) {
delete reinterpret_cast< ::google::protobuf::MessageLite*>(context_);
}
if (context) {
::google::protobuf::Arena* submessage_arena =
reinterpret_cast<::google::protobuf::MessageLite*>(context)->GetArena();
if (message_arena != submessage_arena) {
context = ::google::protobuf::internal::GetOwnedMessage(
message_arena, context, submessage_arena);
}
} else {
}
context_ = context;
// @@protoc_insertion_point(field_set_allocated:tensorflow.SequenceExample.context)
}
// .tensorflow.FeatureLists feature_lists = 2;
inline bool SequenceExample::has_feature_lists() const {
return this != internal_default_instance() && feature_lists_ != NULL;
}
inline const ::tensorflow::FeatureLists& SequenceExample::_internal_feature_lists() const {
return *feature_lists_;
}
inline const ::tensorflow::FeatureLists& SequenceExample::feature_lists() const {
const ::tensorflow::FeatureLists* p = feature_lists_;
// @@protoc_insertion_point(field_get:tensorflow.SequenceExample.feature_lists)
return p != NULL ? *p : *reinterpret_cast<const ::tensorflow::FeatureLists*>(
&::tensorflow::_FeatureLists_default_instance_);
}
inline ::tensorflow::FeatureLists* SequenceExample::release_feature_lists() {
// @@protoc_insertion_point(field_release:tensorflow.SequenceExample.feature_lists)
::tensorflow::FeatureLists* temp = feature_lists_;
if (GetArenaNoVirtual() != NULL) {
temp = ::google::protobuf::internal::DuplicateIfNonNull(temp);
}
feature_lists_ = NULL;
return temp;
}
inline ::tensorflow::FeatureLists* SequenceExample::unsafe_arena_release_feature_lists() {
// @@protoc_insertion_point(field_unsafe_arena_release:tensorflow.SequenceExample.feature_lists)
::tensorflow::FeatureLists* temp = feature_lists_;
feature_lists_ = NULL;
return temp;
}
inline ::tensorflow::FeatureLists* SequenceExample::mutable_feature_lists() {
if (feature_lists_ == NULL) {
auto* p = CreateMaybeMessage<::tensorflow::FeatureLists>(GetArenaNoVirtual());
feature_lists_ = p;
}
// @@protoc_insertion_point(field_mutable:tensorflow.SequenceExample.feature_lists)
return feature_lists_;
}
inline void SequenceExample::set_allocated_feature_lists(::tensorflow::FeatureLists* feature_lists) {
::google::protobuf::Arena* message_arena = GetArenaNoVirtual();
if (message_arena == NULL) {
delete reinterpret_cast< ::google::protobuf::MessageLite*>(feature_lists_);
}
if (feature_lists) {
::google::protobuf::Arena* submessage_arena =
reinterpret_cast<::google::protobuf::MessageLite*>(feature_lists)->GetArena();
if (message_arena != submessage_arena) {
feature_lists = ::google::protobuf::internal::GetOwnedMessage(
message_arena, feature_lists, submessage_arena);
}
} else {
}
feature_lists_ = feature_lists;
// @@protoc_insertion_point(field_set_allocated:tensorflow.SequenceExample.feature_lists)
}
#ifdef __GNUC__
#pragma GCC diagnostic pop
#endif // __GNUC__
// -------------------------------------------------------------------
// @@protoc_insertion_point(namespace_scope)
} // namespace tensorflow
// @@protoc_insertion_point(global_scope)
#endif // PROTOBUF_INCLUDED_tensorflow_2fcore_2fexample_2fexample_2eproto
@@ -0,0 +1,34 @@
// GENERATED FILE - DO NOT MODIFY
#ifndef tensorflow_core_example_example_proto_IMPL_H_
#define tensorflow_core_example_example_proto_IMPL_H_
#include "tensorflow/core/example/example.pb.h"
#include "tensorflow/core/example/example.pb_text.h"
#include "tensorflow/core/example/feature.pb.h"
#include "tensorflow/core/example/feature.pb_text-impl.h"
#include "tensorflow/core/lib/strings/proto_text_util.h"
#include "tensorflow/core/lib/strings/scanner.h"
namespace tensorflow {
namespace internal {
void AppendProtoDebugString(
::tensorflow::strings::ProtoTextOutput* o,
const ::tensorflow::Example& msg);
bool ProtoParseFromScanner(
::tensorflow::strings::Scanner* scanner, bool nested, bool close_curly,
::tensorflow::Example* msg);
void AppendProtoDebugString(
::tensorflow::strings::ProtoTextOutput* o,
const ::tensorflow::SequenceExample& msg);
bool ProtoParseFromScanner(
::tensorflow::strings::Scanner* scanner, bool nested, bool close_curly,
::tensorflow::SequenceExample* msg);
} // namespace internal
} // namespace tensorflow
#endif // tensorflow_core_example_example_proto_IMPL_H_
@@ -0,0 +1,34 @@
// GENERATED FILE - DO NOT MODIFY
#ifndef tensorflow_core_example_example_proto_H_
#define tensorflow_core_example_example_proto_H_
#include "tensorflow/core/example/example.pb.h"
#include "tensorflow/core/platform/macros.h"
#include "tensorflow/core/platform/protobuf.h"
#include "tensorflow/core/platform/types.h"
namespace tensorflow {
// Message-text conversion for tensorflow.Example
string ProtoDebugString(
const ::tensorflow::Example& msg);
string ProtoShortDebugString(
const ::tensorflow::Example& msg);
bool ProtoParseFromString(
const string& s,
::tensorflow::Example* msg)
TF_MUST_USE_RESULT;
// Message-text conversion for tensorflow.SequenceExample
string ProtoDebugString(
const ::tensorflow::SequenceExample& msg);
string ProtoShortDebugString(
const ::tensorflow::SequenceExample& msg);
bool ProtoParseFromString(
const string& s,
::tensorflow::SequenceExample* msg)
TF_MUST_USE_RESULT;
} // namespace tensorflow
#endif // tensorflow_core_example_example_proto_H_
@@ -0,0 +1,56 @@
/* Copyright 2016 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
#ifndef TENSORFLOW_CORE_EXAMPLE_EXAMPLE_PARSER_CONFIGURATION_H_
#define TENSORFLOW_CORE_EXAMPLE_EXAMPLE_PARSER_CONFIGURATION_H_
#include <string>
#include <vector>
#include "tensorflow/core/example/example.pb.h"
#include "tensorflow/core/example/example_parser_configuration.pb.h"
#include "tensorflow/core/framework/allocator.h"
#include "tensorflow/core/framework/graph.pb.h"
#include "tensorflow/core/framework/tensor.h"
#include "tensorflow/core/framework/types.h"
#include "tensorflow/core/platform/types.h"
#include "tensorflow/core/public/session.h"
#include "tensorflow/core/util/example_proto_helper.h"
#include "tensorflow/core/util/sparse/sparse_tensor.h"
// This is a set of helper methods that will make it possible to share
// tensorflow::Example proto Tensor conversion code inside the ExampleParserOp
// OpKernel as well as in external code.
namespace tensorflow {
// Given a graph and the node_name of a ParseExample op,
// extract the FixedLenFeature/VarLenFeature configurations.
Status ExtractExampleParserConfiguration(
const tensorflow::GraphDef& graph, const string& node_name,
tensorflow::Session* session,
std::vector<FixedLenFeature>* fixed_len_features,
std::vector<VarLenFeature>* var_len_features);
// Given a config proto, ostensibly extracted via python,
// fill a vector of C++ structs suitable for calling
// the tensorflow.Example -> Tensor conversion code.
Status ExampleParserConfigurationProtoToFeatureVectors(
const ExampleParserConfiguration& config_proto,
std::vector<FixedLenFeature>* fixed_len_features,
std::vector<VarLenFeature>* var_len_features);
} // namespace tensorflow
#endif // TENSORFLOW_CORE_EXAMPLE_EXAMPLE_PARSER_CONFIGURATION_H_
File diff suppressed because it is too large. Load diff
File diff suppressed because it is too large. Load diff
@@ -0,0 +1,67 @@
// GENERATED FILE - DO NOT MODIFY
#ifndef tensorflow_core_example_feature_proto_IMPL_H_
#define tensorflow_core_example_feature_proto_IMPL_H_
#include "tensorflow/core/example/feature.pb.h"
#include "tensorflow/core/example/feature.pb_text.h"
#include "tensorflow/core/lib/strings/proto_text_util.h"
#include "tensorflow/core/lib/strings/scanner.h"
namespace tensorflow {
namespace internal {
void AppendProtoDebugString(
::tensorflow::strings::ProtoTextOutput* o,
const ::tensorflow::BytesList& msg);
bool ProtoParseFromScanner(
::tensorflow::strings::Scanner* scanner, bool nested, bool close_curly,
::tensorflow::BytesList* msg);
void AppendProtoDebugString(
::tensorflow::strings::ProtoTextOutput* o,
const ::tensorflow::FloatList& msg);
bool ProtoParseFromScanner(
::tensorflow::strings::Scanner* scanner, bool nested, bool close_curly,
::tensorflow::FloatList* msg);
void AppendProtoDebugString(
::tensorflow::strings::ProtoTextOutput* o,
const ::tensorflow::Int64List& msg);
bool ProtoParseFromScanner(
::tensorflow::strings::Scanner* scanner, bool nested, bool close_curly,
::tensorflow::Int64List* msg);
void AppendProtoDebugString(
::tensorflow::strings::ProtoTextOutput* o,
const ::tensorflow::Feature& msg);
bool ProtoParseFromScanner(
::tensorflow::strings::Scanner* scanner, bool nested, bool close_curly,
::tensorflow::Feature* msg);
void AppendProtoDebugString(
::tensorflow::strings::ProtoTextOutput* o,
const ::tensorflow::Features& msg);
bool ProtoParseFromScanner(
::tensorflow::strings::Scanner* scanner, bool nested, bool close_curly,
::tensorflow::Features* msg);
void AppendProtoDebugString(
::tensorflow::strings::ProtoTextOutput* o,
const ::tensorflow::FeatureList& msg);
bool ProtoParseFromScanner(
::tensorflow::strings::Scanner* scanner, bool nested, bool close_curly,
::tensorflow::FeatureList* msg);
void AppendProtoDebugString(
::tensorflow::strings::ProtoTextOutput* o,
const ::tensorflow::FeatureLists& msg);
bool ProtoParseFromScanner(
::tensorflow::strings::Scanner* scanner, bool nested, bool close_curly,
::tensorflow::FeatureLists* msg);
} // namespace internal
} // namespace tensorflow
#endif // tensorflow_core_example_feature_proto_IMPL_H_
@@ -0,0 +1,84 @@
// GENERATED FILE - DO NOT MODIFY
#ifndef tensorflow_core_example_feature_proto_H_
#define tensorflow_core_example_feature_proto_H_
#include "tensorflow/core/example/feature.pb.h"
#include "tensorflow/core/platform/macros.h"
#include "tensorflow/core/platform/protobuf.h"
#include "tensorflow/core/platform/types.h"
namespace tensorflow {
// Message-text conversion for tensorflow.BytesList
string ProtoDebugString(
const ::tensorflow::BytesList& msg);
string ProtoShortDebugString(
const ::tensorflow::BytesList& msg);
bool ProtoParseFromString(
const string& s,
::tensorflow::BytesList* msg)
TF_MUST_USE_RESULT;
// Message-text conversion for tensorflow.FloatList
string ProtoDebugString(
const ::tensorflow::FloatList& msg);
string ProtoShortDebugString(
const ::tensorflow::FloatList& msg);
bool ProtoParseFromString(
const string& s,
::tensorflow::FloatList* msg)
TF_MUST_USE_RESULT;
// Message-text conversion for tensorflow.Int64List
string ProtoDebugString(
const ::tensorflow::Int64List& msg);
string ProtoShortDebugString(
const ::tensorflow::Int64List& msg);
bool ProtoParseFromString(
const string& s,
::tensorflow::Int64List* msg)
TF_MUST_USE_RESULT;
// Message-text conversion for tensorflow.Feature
string ProtoDebugString(
const ::tensorflow::Feature& msg);
string ProtoShortDebugString(
const ::tensorflow::Feature& msg);
bool ProtoParseFromString(
const string& s,
::tensorflow::Feature* msg)
TF_MUST_USE_RESULT;
// Message-text conversion for tensorflow.Features
string ProtoDebugString(
const ::tensorflow::Features& msg);
string ProtoShortDebugString(
const ::tensorflow::Features& msg);
bool ProtoParseFromString(
const string& s,
::tensorflow::Features* msg)
TF_MUST_USE_RESULT;
// Message-text conversion for tensorflow.FeatureList
string ProtoDebugString(
const ::tensorflow::FeatureList& msg);
string ProtoShortDebugString(
const ::tensorflow::FeatureList& msg);
bool ProtoParseFromString(
const string& s,
::tensorflow::FeatureList* msg)
TF_MUST_USE_RESULT;
// Message-text conversion for tensorflow.FeatureLists
string ProtoDebugString(
const ::tensorflow::FeatureLists& msg);
string ProtoShortDebugString(
const ::tensorflow::FeatureLists& msg);
bool ProtoParseFromString(
const string& s,
::tensorflow::FeatureLists* msg)
TF_MUST_USE_RESULT;
} // namespace tensorflow
#endif // tensorflow_core_example_feature_proto_H_
@@ -0,0 +1,326 @@
/* Copyright 2016 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
// A set of lightweight wrappers which simplify access to Feature protos.
//
// TensorFlow Example proto uses associative maps on top of oneof fields.
// SequenceExample proto uses associative map of FeatureList.
// So accessing feature values is not very convenient.
//
// For example, to read a first value of integer feature "tag":
// int id = example.features().feature().at("tag").int64_list().value(0)
//
// to add a value:
// auto features = example->mutable_features();
// (*features->mutable_feature())["tag"].mutable_int64_list()->add_value(id)
//
// For float features you have to use float_list, for string - bytes_list.
//
// To do the same with this library:
// int id = GetFeatureValues<int64>("tag", example).Get(0);
// GetFeatureValues<int64>("tag", &example)->Add(id);
//
// Modification of bytes features is slightly different:
// auto tag = GetFeatureValues<string>("tag", &example);
// *tag->Add() = "lorem ipsum";
//
// To copy multiple values into a feature:
// AppendFeatureValues({1,2,3}, "tag", &example);
//
// GetFeatureValues gives you access to underlying data - RepeatedField object
// (RepeatedPtrField for byte list). So refer to its documentation of
// RepeatedField for full list of supported methods.
//
// NOTE: Due to the nature of oneof proto fields setting a feature of one type
// automatically clears all values stored as another type with the same feature
// key.
//
// This library also has tools to work with SequenceExample protos.
//
// To get a value from SequenceExample.context:
// int id = GetFeatureValues<protobuf_int64>("tag", se.context()).Get(0);
// To add a value to the context:
// GetFeatureValues<protobuf_int64>("tag", se.mutable_context())->Add(42);
//
// To add values to feature_lists:
// AppendFeatureValues({4.0},
// GetFeatureList("images", &se)->Add());
// AppendFeatureValues({5.0, 3.0},
// GetFeatureList("images", &se)->Add());
// This will create a feature list keyed as "images" with two features:
// feature_lists {
// feature_list {
// key: "images"
// value {
// feature { float_list { value: [4.0] } }
// feature { float_list { value: [5.0, 3.0] } }
// }
// } }
//
// Functions exposed by this library:
// HasFeature<[FeatureType]>(key, proto) -> bool
// Returns true if a feature with the specified key, and optionally
// FeatureType, belongs to the Features or Example proto.
// HasFeatureList(key, sequence_example) -> bool
// Returns true if SequenceExample has a feature_list with the key.
// GetFeatureValues<FeatureType>(key, proto) -> RepeatedField<FeatureType>
// Returns values for the specified key and the FeatureType.
// Supported types for the proto: Example, Features.
// GetFeatureList(key, sequence_example) -> RepeatedPtrField<Feature>
// Returns Feature protos associated with a key.
// AppendFeatureValues(begin, end, feature)
// AppendFeatureValues(container or initializer_list, feature)
// Copies values into a Feature.
// AppendFeatureValues(begin, end, key, proto)
// AppendFeatureValues(container or initializer_list, key, proto)
// Copies values into Features and Example protos with the specified key.
//
// Auxiliary functions, it is unlikely you'll need to use them directly:
// GetFeatures(proto) -> Features
// A convenience function to get Features proto.
// Supported types for the proto: Example, Features.
// GetFeature(key, proto) -> Feature*
// Returns a Feature proto for the specified key, creates a new if
// necessary. Supported types for the proto: Example, Features.
// GetFeatureValues<FeatureType>(feature) -> RepeatedField<FeatureType>
// Returns values of the feature for the FeatureType.
#ifndef TENSORFLOW_CORE_EXAMPLE_FEATURE_UTIL_H_
#define TENSORFLOW_CORE_EXAMPLE_FEATURE_UTIL_H_
#include <iterator>
#include <type_traits>
#include "absl/base/macros.h"
#include "tensorflow/core/example/example.pb.h"
#include "tensorflow/core/example/feature.pb.h"
#include "tensorflow/core/lib/core/stringpiece.h"
#include "tensorflow/core/platform/protobuf.h"
#include "tensorflow/core/platform/types.h"
namespace tensorflow {
namespace internal {
// TODO(gorban): Update all clients in a followup CL.
// Returns a reference to a feature corresponding to the name.
// Note: it will create a new Feature if it is missing in the example.
ABSL_DEPRECATED("Use GetFeature instead.")
Feature& ExampleFeature(const string& name, Example* example);
// Specializations of RepeatedFieldTrait define a type of RepeatedField
// corresponding to a selected feature type.
template <typename FeatureType>
struct RepeatedFieldTrait;
template <>
struct RepeatedFieldTrait<protobuf_int64> {
using Type = protobuf::RepeatedField<protobuf_int64>;
};
template <>
struct RepeatedFieldTrait<float> {
using Type = protobuf::RepeatedField<float>;
};
template <>
struct RepeatedFieldTrait<string> {
using Type = protobuf::RepeatedPtrField<string>;
};
// Specializations of FeatureTrait define a type of feature corresponding to a
// selected value type.
template <typename ValueType, class Enable = void>
struct FeatureTrait;
template <typename ValueType>
struct FeatureTrait<ValueType, typename std::enable_if<
std::is_integral<ValueType>::value>::type> {
using Type = protobuf_int64;
};
template <typename ValueType>
struct FeatureTrait<
ValueType,
typename std::enable_if<std::is_floating_point<ValueType>::value>::type> {
using Type = float;
};
template <typename T>
struct is_string
: public std::integral_constant<
bool,
std::is_same<char*, typename std::decay<T>::type>::value ||
std::is_same<const char*, typename std::decay<T>::type>::value> {
};
template <>
struct is_string<string> : std::true_type {};
template <>
struct is_string<::tensorflow::StringPiece> : std::true_type {};
template <typename ValueType>
struct FeatureTrait<
ValueType, typename std::enable_if<is_string<ValueType>::value>::type> {
using Type = string;
};
} // namespace internal
// Returns true if sequence_example has a feature_list with the specified key.
bool HasFeatureList(const string& key, const SequenceExample& sequence_example);
template <typename T>
struct TypeHasFeatures : std::false_type {};
template <>
struct TypeHasFeatures<Example> : std::true_type {};
template <>
struct TypeHasFeatures<Features> : std::true_type {};
// A family of template functions to return mutable Features proto from a
// container proto. Supported ProtoTypes: Example, Features.
template <typename ProtoType>
typename std::enable_if<TypeHasFeatures<ProtoType>::value, Features*>::type
GetFeatures(ProtoType* proto);
template <typename ProtoType>
typename std::enable_if<TypeHasFeatures<ProtoType>::value,
const Features&>::type
GetFeatures(const ProtoType& proto);
// Base declaration of a family of template functions to return a read only
// repeated field of feature values.
template <typename FeatureType>
const typename internal::RepeatedFieldTrait<FeatureType>::Type&
GetFeatureValues(const Feature& feature);
// Returns a read only repeated field corresponding to a feature with the
// specified name and FeatureType. Supported ProtoTypes: Example, Features.
template <typename FeatureType, typename ProtoType>
const typename internal::RepeatedFieldTrait<FeatureType>::Type&
GetFeatureValues(const string& key, const ProtoType& proto) {
return GetFeatureValues<FeatureType>(GetFeatures(proto).feature().at(key));
}
// Returns a mutable repeated field of a feature values.
template <typename FeatureType>
typename internal::RepeatedFieldTrait<FeatureType>::Type* GetFeatureValues(
Feature* feature);
// Returns a mutable repeated field corresponding to a feature with the
// specified name and FeatureType. Supported ProtoTypes: Example, Features.
template <typename FeatureType, typename ProtoType>
typename internal::RepeatedFieldTrait<FeatureType>::Type* GetFeatureValues(
const string& key, ProtoType* proto) {
::tensorflow::Feature& feature =
(*GetFeatures(proto)->mutable_feature())[key];
return GetFeatureValues<FeatureType>(&feature);
}
// Returns a Feature proto for the specified key, creates a new if necessary.
// Supported types for the proto: Example, Features.
template <typename ProtoType>
Feature* GetFeature(const string& key, ProtoType* proto) {
return &(*GetFeatures(proto)->mutable_feature())[key];
}
// Returns a repeated field with features corresponding to a feature_list key.
const protobuf::RepeatedPtrField<Feature>& GetFeatureList(
const string& key, const SequenceExample& sequence_example);
// Returns a mutable repeated field with features corresponding to a
// feature_list key. It will create a new FeatureList if necessary.
protobuf::RepeatedPtrField<Feature>* GetFeatureList(
const string& feature_list_key, SequenceExample* sequence_example);
template <typename IteratorType>
void AppendFeatureValues(IteratorType first, IteratorType last,
Feature* feature) {
using FeatureType = typename internal::FeatureTrait<
typename std::iterator_traits<IteratorType>::value_type>::Type;
std::copy(first, last,
protobuf::RepeatedFieldBackInserter(
GetFeatureValues<FeatureType>(feature)));
}
template <typename ValueType>
void AppendFeatureValues(std::initializer_list<ValueType> container,
Feature* feature) {
AppendFeatureValues(container.begin(), container.end(), feature);
}
template <typename ContainerType>
void AppendFeatureValues(const ContainerType& container, Feature* feature) {
using IteratorType = typename ContainerType::const_iterator;
AppendFeatureValues<IteratorType>(container.begin(), container.end(),
feature);
}
// Copies elements from the range, defined by [first, last) into the feature
// obtainable from the (proto, key) combination.
template <typename IteratorType, typename ProtoType>
void AppendFeatureValues(IteratorType first, IteratorType last,
const string& key, ProtoType* proto) {
AppendFeatureValues(first, last, GetFeature(key, GetFeatures(proto)));
}
// Copies all elements from the container into a feature.
template <typename ContainerType, typename ProtoType>
void AppendFeatureValues(const ContainerType& container, const string& key,
ProtoType* proto) {
using IteratorType = typename ContainerType::const_iterator;
AppendFeatureValues<IteratorType>(container.begin(), container.end(), key,
proto);
}
// Copies all elements from the initializer list into a Feature contained by
// Features or Example proto.
template <typename ValueType, typename ProtoType>
void AppendFeatureValues(std::initializer_list<ValueType> container,
const string& key, ProtoType* proto) {
using IteratorType =
typename std::initializer_list<ValueType>::const_iterator;
AppendFeatureValues<IteratorType>(container.begin(), container.end(), key,
proto);
}
// Returns true if a feature with the specified key belongs to the Features.
// The template parameter pack accepts zero or one template argument - which
// is FeatureType. If the FeatureType not specified (zero template arguments)
// the function will not check the feature type. Otherwise it will return false
// if the feature has a wrong type.
template <typename... FeatureType>
bool HasFeature(const string& key, const Features& features);
// Returns true if a feature with the specified key belongs to the Example.
// Doesn't check feature type if used without FeatureType, otherwise the
// specialized versions return false if the feature has a wrong type.
template <typename... FeatureType>
bool HasFeature(const string& key, const Example& example) {
return HasFeature<FeatureType...>(key, GetFeatures(example));
}
// TODO(gorban): update all clients in a followup CL.
template <typename... FeatureType>
ABSL_DEPRECATED("Use HasFeature instead.")
bool ExampleHasFeature(const string& key, const Example& example) {
return HasFeature<FeatureType...>(key, example);
}
} // namespace tensorflow
#endif // TENSORFLOW_CORE_EXAMPLE_FEATURE_UTIL_H_
@@ -0,0 +1,396 @@
// Generated by the protocol buffer compiler. DO NOT EDIT!
// source: tensorflow/core/framework/allocation_description.proto
#ifndef PROTOBUF_INCLUDED_tensorflow_2fcore_2fframework_2fallocation_5fdescription_2eproto
#define PROTOBUF_INCLUDED_tensorflow_2fcore_2fframework_2fallocation_5fdescription_2eproto
#include <string>
#include <google/protobuf/stubs/common.h>
#if GOOGLE_PROTOBUF_VERSION < 3006000
#error This file was generated by a newer version of protoc which is
#error incompatible with your Protocol Buffer headers. Please update
#error your headers.
#endif
#if 3006000 < GOOGLE_PROTOBUF_MIN_PROTOC_VERSION
#error This file was generated by an older version of protoc which is
#error incompatible with your Protocol Buffer headers. Please
#error regenerate this file with a newer version of protoc.
#endif
#include <google/protobuf/io/coded_stream.h>
#include <google/protobuf/arena.h>
#include <google/protobuf/arenastring.h>
#include <google/protobuf/generated_message_table_driven.h>
#include <google/protobuf/generated_message_util.h>
#include <google/protobuf/inlined_string_field.h>
#include <google/protobuf/metadata.h>
#include <google/protobuf/message.h>
#include <google/protobuf/repeated_field.h> // IWYU pragma: export
#include <google/protobuf/extension_set.h> // IWYU pragma: export
#include <google/protobuf/unknown_field_set.h>
// @@protoc_insertion_point(includes)
#define PROTOBUF_INTERNAL_EXPORT_protobuf_tensorflow_2fcore_2fframework_2fallocation_5fdescription_2eproto
namespace protobuf_tensorflow_2fcore_2fframework_2fallocation_5fdescription_2eproto {
// Internal implementation detail -- do not use these members.
struct TableStruct {
static const ::google::protobuf::internal::ParseTableField entries[];
static const ::google::protobuf::internal::AuxillaryParseTableField aux[];
static const ::google::protobuf::internal::ParseTable schema[1];
static const ::google::protobuf::internal::FieldMetadata field_metadata[];
static const ::google::protobuf::internal::SerializationTable serialization_table[];
static const ::google::protobuf::uint32 offsets[];
};
void AddDescriptors();
} // namespace protobuf_tensorflow_2fcore_2fframework_2fallocation_5fdescription_2eproto
namespace tensorflow {
class AllocationDescription;
class AllocationDescriptionDefaultTypeInternal;
extern AllocationDescriptionDefaultTypeInternal _AllocationDescription_default_instance_;
} // namespace tensorflow
namespace google {
namespace protobuf {
template<> ::tensorflow::AllocationDescription* Arena::CreateMaybeMessage<::tensorflow::AllocationDescription>(Arena*);
} // namespace protobuf
} // namespace google
namespace tensorflow {
// ===================================================================
class AllocationDescription : public ::google::protobuf::Message /* @@protoc_insertion_point(class_definition:tensorflow.AllocationDescription) */ {
public:
AllocationDescription();
virtual ~AllocationDescription();
AllocationDescription(const AllocationDescription& from);
inline AllocationDescription& operator=(const AllocationDescription& from) {
CopyFrom(from);
return *this;
}
#if LANG_CXX11
AllocationDescription(AllocationDescription&& from) noexcept
: AllocationDescription() {
*this = ::std::move(from);
}
inline AllocationDescription& operator=(AllocationDescription&& from) noexcept {
if (GetArenaNoVirtual() == from.GetArenaNoVirtual()) {
if (this != &from) InternalSwap(&from);
} else {
CopyFrom(from);
}
return *this;
}
#endif
inline ::google::protobuf::Arena* GetArena() const final {
return GetArenaNoVirtual();
}
inline void* GetMaybeArenaPointer() const final {
return MaybeArenaPtr();
}
static const ::google::protobuf::Descriptor* descriptor();
static const AllocationDescription& default_instance();
static void InitAsDefaultInstance(); // FOR INTERNAL USE ONLY
static inline const AllocationDescription* internal_default_instance() {
return reinterpret_cast<const AllocationDescription*>(
&_AllocationDescription_default_instance_);
}
static constexpr int kIndexInFileMessages =
0;
void UnsafeArenaSwap(AllocationDescription* other);
void Swap(AllocationDescription* other);
friend void swap(AllocationDescription& a, AllocationDescription& b) {
a.Swap(&b);
}
// implements Message ----------------------------------------------
inline AllocationDescription* New() const final {
return CreateMaybeMessage<AllocationDescription>(NULL);
}
AllocationDescription* New(::google::protobuf::Arena* arena) const final {
return CreateMaybeMessage<AllocationDescription>(arena);
}
void CopyFrom(const ::google::protobuf::Message& from) final;
void MergeFrom(const ::google::protobuf::Message& from) final;
void CopyFrom(const AllocationDescription& from);
void MergeFrom(const AllocationDescription& from);
void Clear() final;
bool IsInitialized() const final;
size_t ByteSizeLong() const final;
bool MergePartialFromCodedStream(
::google::protobuf::io::CodedInputStream* input) final;
void SerializeWithCachedSizes(
::google::protobuf::io::CodedOutputStream* output) const final;
::google::protobuf::uint8* InternalSerializeWithCachedSizesToArray(
bool deterministic, ::google::protobuf::uint8* target) const final;
int GetCachedSize() const final { return _cached_size_.Get(); }
private:
void SharedCtor();
void SharedDtor();
void SetCachedSize(int size) const final;
void InternalSwap(AllocationDescription* other);
protected:
explicit AllocationDescription(::google::protobuf::Arena* arena);
private:
static void ArenaDtor(void* object);
inline void RegisterArenaDtor(::google::protobuf::Arena* arena);
private:
inline ::google::protobuf::Arena* GetArenaNoVirtual() const {
return _internal_metadata_.arena();
}
inline void* MaybeArenaPtr() const {
return _internal_metadata_.raw_arena_ptr();
}
public:
::google::protobuf::Metadata GetMetadata() const final;
// nested types ----------------------------------------------------
// accessors -------------------------------------------------------
// string allocator_name = 3;
void clear_allocator_name();
static const int kAllocatorNameFieldNumber = 3;
const ::std::string& allocator_name() const;
void set_allocator_name(const ::std::string& value);
#if LANG_CXX11
void set_allocator_name(::std::string&& value);
#endif
void set_allocator_name(const char* value);
void set_allocator_name(const char* value, size_t size);
::std::string* mutable_allocator_name();
::std::string* release_allocator_name();
void set_allocated_allocator_name(::std::string* allocator_name);
PROTOBUF_RUNTIME_DEPRECATED("The unsafe_arena_ accessors for"
" string fields are deprecated and will be removed in a"
" future release.")
::std::string* unsafe_arena_release_allocator_name();
PROTOBUF_RUNTIME_DEPRECATED("The unsafe_arena_ accessors for"
" string fields are deprecated and will be removed in a"
" future release.")
void unsafe_arena_set_allocated_allocator_name(
::std::string* allocator_name);
// int64 requested_bytes = 1;
void clear_requested_bytes();
static const int kRequestedBytesFieldNumber = 1;
::google::protobuf::int64 requested_bytes() const;
void set_requested_bytes(::google::protobuf::int64 value);
// int64 allocated_bytes = 2;
void clear_allocated_bytes();
static const int kAllocatedBytesFieldNumber = 2;
::google::protobuf::int64 allocated_bytes() const;
void set_allocated_bytes(::google::protobuf::int64 value);
// int64 allocation_id = 4;
void clear_allocation_id();
static const int kAllocationIdFieldNumber = 4;
::google::protobuf::int64 allocation_id() const;
void set_allocation_id(::google::protobuf::int64 value);
// uint64 ptr = 6;
void clear_ptr();
static const int kPtrFieldNumber = 6;
::google::protobuf::uint64 ptr() const;
void set_ptr(::google::protobuf::uint64 value);
// bool has_single_reference = 5;
void clear_has_single_reference();
static const int kHasSingleReferenceFieldNumber = 5;
bool has_single_reference() const;
void set_has_single_reference(bool value);
// @@protoc_insertion_point(class_scope:tensorflow.AllocationDescription)
private:
::google::protobuf::internal::InternalMetadataWithArena _internal_metadata_;
template <typename T> friend class ::google::protobuf::Arena::InternalHelper;
typedef void InternalArenaConstructable_;
typedef void DestructorSkippable_;
::google::protobuf::internal::ArenaStringPtr allocator_name_;
::google::protobuf::int64 requested_bytes_;
::google::protobuf::int64 allocated_bytes_;
::google::protobuf::int64 allocation_id_;
::google::protobuf::uint64 ptr_;
bool has_single_reference_;
mutable ::google::protobuf::internal::CachedSize _cached_size_;
friend struct ::protobuf_tensorflow_2fcore_2fframework_2fallocation_5fdescription_2eproto::TableStruct;
};
// ===================================================================
// ===================================================================
#ifdef __GNUC__
#pragma GCC diagnostic push
#pragma GCC diagnostic ignored "-Wstrict-aliasing"
#endif // __GNUC__
// AllocationDescription
// int64 requested_bytes = 1;
inline void AllocationDescription::clear_requested_bytes() {
requested_bytes_ = GOOGLE_LONGLONG(0);
}
inline ::google::protobuf::int64 AllocationDescription::requested_bytes() const {
// @@protoc_insertion_point(field_get:tensorflow.AllocationDescription.requested_bytes)
return requested_bytes_;
}
inline void AllocationDescription::set_requested_bytes(::google::protobuf::int64 value) {
requested_bytes_ = value;
// @@protoc_insertion_point(field_set:tensorflow.AllocationDescription.requested_bytes)
}
// int64 allocated_bytes = 2;
inline void AllocationDescription::clear_allocated_bytes() {
allocated_bytes_ = GOOGLE_LONGLONG(0);
}
inline ::google::protobuf::int64 AllocationDescription::allocated_bytes() const {
// @@protoc_insertion_point(field_get:tensorflow.AllocationDescription.allocated_bytes)
return allocated_bytes_;
}
inline void AllocationDescription::set_allocated_bytes(::google::protobuf::int64 value) {
allocated_bytes_ = value;
// @@protoc_insertion_point(field_set:tensorflow.AllocationDescription.allocated_bytes)
}
// string allocator_name = 3;
inline void AllocationDescription::clear_allocator_name() {
allocator_name_.ClearToEmpty(&::google::protobuf::internal::GetEmptyStringAlreadyInited(), GetArenaNoVirtual());
}
inline const ::std::string& AllocationDescription::allocator_name() const {
// @@protoc_insertion_point(field_get:tensorflow.AllocationDescription.allocator_name)
return allocator_name_.Get();
}
inline void AllocationDescription::set_allocator_name(const ::std::string& value) {
allocator_name_.Set(&::google::protobuf::internal::GetEmptyStringAlreadyInited(), value, GetArenaNoVirtual());
// @@protoc_insertion_point(field_set:tensorflow.AllocationDescription.allocator_name)
}
#if LANG_CXX11
inline void AllocationDescription::set_allocator_name(::std::string&& value) {
allocator_name_.Set(
&::google::protobuf::internal::GetEmptyStringAlreadyInited(), ::std::move(value), GetArenaNoVirtual());
// @@protoc_insertion_point(field_set_rvalue:tensorflow.AllocationDescription.allocator_name)
}
#endif
inline void AllocationDescription::set_allocator_name(const char* value) {
GOOGLE_DCHECK(value != NULL);
allocator_name_.Set(&::google::protobuf::internal::GetEmptyStringAlreadyInited(), ::std::string(value),
GetArenaNoVirtual());
// @@protoc_insertion_point(field_set_char:tensorflow.AllocationDescription.allocator_name)
}
inline void AllocationDescription::set_allocator_name(const char* value,
size_t size) {
allocator_name_.Set(&::google::protobuf::internal::GetEmptyStringAlreadyInited(), ::std::string(
reinterpret_cast<const char*>(value), size), GetArenaNoVirtual());
// @@protoc_insertion_point(field_set_pointer:tensorflow.AllocationDescription.allocator_name)
}
inline ::std::string* AllocationDescription::mutable_allocator_name() {
// @@protoc_insertion_point(field_mutable:tensorflow.AllocationDescription.allocator_name)
return allocator_name_.Mutable(&::google::protobuf::internal::GetEmptyStringAlreadyInited(), GetArenaNoVirtual());
}
inline ::std::string* AllocationDescription::release_allocator_name() {
// @@protoc_insertion_point(field_release:tensorflow.AllocationDescription.allocator_name)
return allocator_name_.Release(&::google::protobuf::internal::GetEmptyStringAlreadyInited(), GetArenaNoVirtual());
}
inline void AllocationDescription::set_allocated_allocator_name(::std::string* allocator_name) {
if (allocator_name != NULL) {
} else {
}
allocator_name_.SetAllocated(&::google::protobuf::internal::GetEmptyStringAlreadyInited(), allocator_name,
GetArenaNoVirtual());
// @@protoc_insertion_point(field_set_allocated:tensorflow.AllocationDescription.allocator_name)
}
inline ::std::string* AllocationDescription::unsafe_arena_release_allocator_name() {
// @@protoc_insertion_point(field_unsafe_arena_release:tensorflow.AllocationDescription.allocator_name)
GOOGLE_DCHECK(GetArenaNoVirtual() != NULL);
return allocator_name_.UnsafeArenaRelease(&::google::protobuf::internal::GetEmptyStringAlreadyInited(),
GetArenaNoVirtual());
}
inline void AllocationDescription::unsafe_arena_set_allocated_allocator_name(
::std::string* allocator_name) {
GOOGLE_DCHECK(GetArenaNoVirtual() != NULL);
if (allocator_name != NULL) {
} else {
}
allocator_name_.UnsafeArenaSetAllocated(&::google::protobuf::internal::GetEmptyStringAlreadyInited(),
allocator_name, GetArenaNoVirtual());
// @@protoc_insertion_point(field_unsafe_arena_set_allocated:tensorflow.AllocationDescription.allocator_name)
}
// int64 allocation_id = 4;
inline void AllocationDescription::clear_allocation_id() {
allocation_id_ = GOOGLE_LONGLONG(0);
}
inline ::google::protobuf::int64 AllocationDescription::allocation_id() const {
// @@protoc_insertion_point(field_get:tensorflow.AllocationDescription.allocation_id)
return allocation_id_;
}
inline void AllocationDescription::set_allocation_id(::google::protobuf::int64 value) {
allocation_id_ = value;
// @@protoc_insertion_point(field_set:tensorflow.AllocationDescription.allocation_id)
}
// bool has_single_reference = 5;
inline void AllocationDescription::clear_has_single_reference() {
has_single_reference_ = false;
}
inline bool AllocationDescription::has_single_reference() const {
// @@protoc_insertion_point(field_get:tensorflow.AllocationDescription.has_single_reference)
return has_single_reference_;
}
inline void AllocationDescription::set_has_single_reference(bool value) {
has_single_reference_ = value;
// @@protoc_insertion_point(field_set:tensorflow.AllocationDescription.has_single_reference)
}
// uint64 ptr = 6;
inline void AllocationDescription::clear_ptr() {
ptr_ = GOOGLE_ULONGLONG(0);
}
inline ::google::protobuf::uint64 AllocationDescription::ptr() const {
// @@protoc_insertion_point(field_get:tensorflow.AllocationDescription.ptr)
return ptr_;
}
inline void AllocationDescription::set_ptr(::google::protobuf::uint64 value) {
ptr_ = value;
// @@protoc_insertion_point(field_set:tensorflow.AllocationDescription.ptr)
}
#ifdef __GNUC__
#pragma GCC diagnostic pop
#endif // __GNUC__
// @@protoc_insertion_point(namespace_scope)
} // namespace tensorflow
// @@protoc_insertion_point(global_scope)
#endif // PROTOBUF_INCLUDED_tensorflow_2fcore_2fframework_2fallocation_5fdescription_2eproto
@@ -0,0 +1,25 @@
// GENERATED FILE - DO NOT MODIFY
#ifndef tensorflow_core_framework_allocation_description_proto_IMPL_H_
#define tensorflow_core_framework_allocation_description_proto_IMPL_H_
#include "tensorflow/core/framework/allocation_description.pb.h"
#include "tensorflow/core/framework/allocation_description.pb_text.h"
#include "tensorflow/core/lib/strings/proto_text_util.h"
#include "tensorflow/core/lib/strings/scanner.h"
namespace tensorflow {
namespace internal {
void AppendProtoDebugString(
::tensorflow::strings::ProtoTextOutput* o,
const ::tensorflow::AllocationDescription& msg);
bool ProtoParseFromScanner(
::tensorflow::strings::Scanner* scanner, bool nested, bool close_curly,
::tensorflow::AllocationDescription* msg);
} // namespace internal
} // namespace tensorflow
#endif // tensorflow_core_framework_allocation_description_proto_IMPL_H_
@@ -0,0 +1,24 @@
// GENERATED FILE - DO NOT MODIFY
#ifndef tensorflow_core_framework_allocation_description_proto_H_
#define tensorflow_core_framework_allocation_description_proto_H_
#include "tensorflow/core/framework/allocation_description.pb.h"
#include "tensorflow/core/platform/macros.h"
#include "tensorflow/core/platform/protobuf.h"
#include "tensorflow/core/platform/types.h"
namespace tensorflow {
// Message-text conversion for tensorflow.AllocationDescription
string ProtoDebugString(
const ::tensorflow::AllocationDescription& msg);
string ProtoShortDebugString(
const ::tensorflow::AllocationDescription& msg);
bool ProtoParseFromString(
const string& s,
::tensorflow::AllocationDescription* msg)
TF_MUST_USE_RESULT;
} // namespace tensorflow
#endif // tensorflow_core_framework_allocation_description_proto_H_
@@ -0,0 +1,425 @@
/* Copyright 2015 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
#ifndef TENSORFLOW_CORE_FRAMEWORK_ALLOCATOR_H_
#define TENSORFLOW_CORE_FRAMEWORK_ALLOCATOR_H_
#include <stdlib.h>
#include <limits>
#include "tensorflow/core/framework/numeric_types.h"
#include "tensorflow/core/framework/resource_handle.h"
#include "tensorflow/core/framework/type_traits.h"
#include "tensorflow/core/platform/logging.h"
#include "tensorflow/core/platform/mutex.h"
#include "tensorflow/core/platform/types.h"
namespace tensorflow {
class Variant;
// Attributes for a single allocation call. Different calls to the same
// allocator could potentially have different allocation attributes.
struct AllocationAttributes {
// If the first attempt to allocate the memory fails, the allocation
// should return immediately without retrying.
// An example use case is optional scratch spaces where a failure
// has only performance impact.
bool no_retry_on_failure = false;
// If a Tensor is allocated without the following set to true, then
// it is logged as an unknown allocation. During execution Tensors
// should be allocated through the OpKernelContext which records
// which Op is performing the allocation, and sets this flag to
// true.
bool allocation_will_be_logged = false;
};
// Runtime statistics collected by an allocator.
struct AllocatorStats {
int64 num_allocs; // Number of allocations.
int64 bytes_in_use; // Number of bytes in use.
int64 max_bytes_in_use; // The maximum bytes in use.
int64 max_alloc_size; // The max single allocation seen.
// The upper limit what the allocator can allocate, if such a limit
// is known. Certain allocator may return 0 to indicate the limit is
// unknown.
int64 bytes_limit;
AllocatorStats() { Clear(); }
void Clear();
string DebugString() const;
};
// Allocator is an abstract interface for allocating and deallocating
// device memory.
class Allocator {
public:
// Align to 64 byte boundary.
static constexpr size_t kAllocatorAlignment = 64;
virtual ~Allocator();
// Return a string identifying this allocator
virtual string Name() = 0;
// Return an uninitialized block of memory that is "num_bytes" bytes
// in size. The returned pointer is guaranteed to be aligned to a
// multiple of "alignment" bytes.
// REQUIRES: "alignment" is a power of 2.
virtual void* AllocateRaw(size_t alignment, size_t num_bytes) = 0;
// Return an uninitialized block of memory that is "num_bytes" bytes
// in size with specified allocation attributes. The returned pointer is
// guaranteed to be aligned to a multiple of "alignment" bytes.
// REQUIRES: "alignment" is a power of 2.
virtual void* AllocateRaw(size_t alignment, size_t num_bytes,
const AllocationAttributes& allocation_attr) {
// The default behavior is to use the implementation without any allocation
// attributes.
return AllocateRaw(alignment, num_bytes);
}
// Deallocate a block of memory pointer to by "ptr"
// REQUIRES: "ptr" was previously returned by a call to AllocateRaw
virtual void DeallocateRaw(void* ptr) = 0;
// Convenience functions to do typed allocation. C++ constructors
// and destructors are invoked for complex types if necessary,
// depending on the concrete Allocator implementation. May return
// NULL if the tensor has too many elements to represent in a single
// allocation.
template <typename T>
T* Allocate(size_t num_elements) {
return Allocate<T>(num_elements, AllocationAttributes());
}
template <typename T>
T* Allocate(size_t num_elements,
const AllocationAttributes& allocation_attr) {
// TODO(jeff): Do we need to allow clients to pass in alignment
// requirements?
if (num_elements > (std::numeric_limits<size_t>::max() / sizeof(T))) {
return NULL;
}
void* p = AllocateRaw(kAllocatorAlignment, sizeof(T) * num_elements,
allocation_attr);
T* typed_p = reinterpret_cast<T*>(p);
if (typed_p) RunCtor<T>(typed_p, num_elements);
return typed_p;
}
template <typename T>
void Deallocate(T* ptr, size_t num_elements) {
if (ptr) {
RunDtor<T>(ptr, num_elements);
DeallocateRaw(ptr);
}
}
// Returns true if this allocator tracks the sizes of allocations.
// RequestedSize and AllocatedSize must be overridden if
// TracksAllocationSizes is overridden to return true.
virtual bool TracksAllocationSizes() { return false; }
// Returns true if this allocator requires tensors with 0 elements
// to allocate buffers. This is false for most allocators, but may
// be used by special-case allocators that want to track tensor
// usage.
virtual bool ShouldAllocateEmptyTensors() { return false; }
// Returns the user-requested size of the data allocated at
// 'ptr'. Note that the actual buffer allocated might be larger
// than requested, but this function returns the size requested by
// the user.
//
// REQUIRES: TracksAllocationSizes() is true.
//
// REQUIRES: 'ptr!=nullptr' and points to a buffer previously
// allocated by this allocator.
virtual size_t RequestedSize(const void* ptr) {
CHECK(false) << "allocator doesn't track sizes";
return size_t(0);
}
// Returns the allocated size of the buffer at 'ptr' if known,
// otherwise returns RequestedSize(ptr). AllocatedSize(ptr) is
// guaranteed to be >= RequestedSize(ptr).
//
// REQUIRES: TracksAllocationSizes() is true.
//
// REQUIRES: 'ptr!=nullptr' and points to a buffer previously
// allocated by this allocator.
virtual size_t AllocatedSize(const void* ptr) { return RequestedSize(ptr); }
// Returns either 0 or an identifier assigned to the buffer at 'ptr'
// when the buffer was returned by AllocateRaw. If non-zero, the
// identifier differs from every other ID assigned by this
// allocator.
//
// REQUIRES: TracksAllocationSizes() is true.
//
// REQUIRES: 'ptr!=nullptr' and points to a buffer previously
// allocated by this allocator.
virtual int64 AllocationId(const void* ptr) { return 0; }
// Returns the allocated size of the buffer at 'ptr' if known,
// otherwise returns 0. This method can be called when
// TracksAllocationSizes() is false, but can be extremely slow.
//
// REQUIRES: 'ptr!=nullptr' and points to a buffer previously
// allocated by this allocator.
virtual size_t AllocatedSizeSlow(const void* ptr) {
if (TracksAllocationSizes()) {
return AllocatedSize(ptr);
}
return 0;
}
// Fills in 'stats' with statistics collected by this allocator.
virtual void GetStats(AllocatorStats* stats) { stats->Clear(); }
// Clears the internal stats except for the `in_use` field.
virtual void ClearStats() {}
private:
// No constructors or destructors are run for simple types
template <typename T>
void RunCtor(T* p, size_t n) {
static_assert(is_simple_type<T>::value, "T is not a simple type.");
}
template <typename T>
void RunDtor(T* p, size_t n) {}
// custom constructors and destructors that can be overridden for
// non-standard allocators
// Runs string's default constructor for p[0], p[1], ..., p[n-1].
virtual void RunStringCtor(string* p, size_t n) {
for (size_t i = 0; i < n; ++p, ++i) new (p) string();
}
// Runs string's default destructor for p[0], p[1], ..., p[n-1].
virtual void RunStringDtor(string* p, size_t n) {
for (size_t i = 0; i < n; ++p, ++i) p->~string();
}
virtual void RunResourceCtor(ResourceHandle* p, size_t n) {
for (size_t i = 0; i < n; ++p, ++i) new (p) ResourceHandle();
}
// Runs string's default destructor for p[0], p[1], ..., p[n-1].
virtual void RunResourceDtor(ResourceHandle* p, size_t n) {
for (size_t i = 0; i < n; ++p, ++i) p->~ResourceHandle();
}
virtual void RunVariantCtor(Variant* p, size_t n);
virtual void RunVariantDtor(Variant* p, size_t n);
// TODO(jeff): Maybe provide some interface to give info about
// current allocation state (total number of bytes available for
// allocation, number of bytes free on device, etc.)
};
// Allocator-specific constructors and destructors are used for
// strings
template <>
inline void Allocator::RunCtor(string* p, size_t n) {
RunStringCtor(p, n);
}
template <>
inline void Allocator::RunDtor(string* p, size_t n) {
RunStringDtor(p, n);
}
template <>
inline void Allocator::RunCtor(ResourceHandle* p, size_t n) {
RunResourceCtor(p, n);
}
template <>
inline void Allocator::RunDtor(ResourceHandle* p, size_t n) {
RunResourceDtor(p, n);
}
template <>
inline void Allocator::RunCtor(Variant* p, size_t n) {
RunVariantCtor(p, n);
}
template <>
inline void Allocator::RunDtor(Variant* p, size_t n) {
RunVariantDtor(p, n);
}
// An implementation of Allocator that delegates all calls to another Allocator.
//
// Useful to clients who want to override part of the functionality of another
// allocator.
class AllocatorWrapper : public Allocator {
public:
explicit AllocatorWrapper(Allocator* wrapped) : wrapped_(wrapped) {}
~AllocatorWrapper() override {}
// Returns the wrapped allocator to which all calls are delegated.
Allocator* wrapped() const { return wrapped_; }
string Name() override { return wrapped_->Name(); }
void* AllocateRaw(size_t alignment, size_t num_bytes) override {
return wrapped_->AllocateRaw(alignment, num_bytes);
}
void* AllocateRaw(size_t alignment, size_t num_bytes,
const AllocationAttributes& allocation_attr) override {
return wrapped_->AllocateRaw(alignment, num_bytes, allocation_attr);
}
void DeallocateRaw(void* ptr) override { wrapped_->DeallocateRaw(ptr); }
bool TracksAllocationSizes() override {
return wrapped_->TracksAllocationSizes();
}
bool ShouldAllocateEmptyTensors() override {
return wrapped_->TracksAllocationSizes();
}
size_t RequestedSize(const void* ptr) override {
return wrapped_->RequestedSize(ptr);
}
size_t AllocatedSize(const void* ptr) override {
return wrapped_->AllocatedSize(ptr);
}
int64 AllocationId(const void* ptr) override {
return wrapped_->AllocationId(ptr);
}
size_t AllocatedSizeSlow(const void* ptr) override {
return wrapped_->AllocatedSizeSlow(ptr);
}
private:
Allocator* const wrapped_;
};
// A tensorflow Op may need access to different kinds of memory that
// are not simply a function of the device to which the Op has been
// assigned. For example, an Op executing on a GPU may still need
// to allocate CPU RAM for some purpose. Internal to the tensorflow
// runtime we may choose to allocate CPU ram from special regions
// that have been prepared for higher performance in some use
// contexts, e.g. doing DMA with particular devices. For these
// reasons, the Device interface does not expose just one memory
// Allocator, but instead provides an accessor that takes a
// specification of the desired memory attributes in order to select
// an Allocator.
//
// Example use:
// // Allocator for ordinary device memory:
// Allocator* a = allocator(AllocatorAttributes());
// ...
// // Allocator for CPU RAM, regardless of where Op is executing:
// AllocatorAttributes attr;
// attr.set_on_host(true);
// Allocator* a = allocator(attr);
struct AllocatorAttributes {
void set_on_host(bool v) { value |= (static_cast<int>(v)); }
bool on_host() const { return value & 0x1; }
void set_nic_compatible(bool v) { value |= (static_cast<int>(v) << 1); }
bool nic_compatible() const { return value & (0x1 << 1); }
void set_gpu_compatible(bool v) { value |= (static_cast<int>(v) << 2); }
bool gpu_compatible() const { return value & (0x1 << 2); }
void Merge(AllocatorAttributes other) {
value |= other.value;
scope_id = (scope_id > 0 && other.scope_id == 0)
? scope_id
: ((scope_id == 0) ? other.scope_id : 0);
}
// Returns true if the fields set in *this is a subset of or equal to
// those set in other.
bool IsEqualOrLessRestrictiveThan(const AllocatorAttributes& other) const {
return (value | other.value) == other.value;
}
// NOTE: The upper 8 bits of the value are reserved for
// device-specific uses. Implementors of a device can interpret these
// upper 8 bits in device-specific ways, and ops implemented for those
// devices are responsible for setting those 8 bits appropriately.
uint32 value = 0;
// EXPERIMENTAL: If this is greater than zero, then allocation is delegated to
// a named special-purpose allocator on the same device.
int32 scope_id = 0;
};
// Returns a trivial implementation of Allocator, which is a process singleton.
// Access through this function is only intended for use in tests and auxiliary
// processing. Performance sensitive uses should always obtain allocators from
// ProcessState.
Allocator* cpu_allocator();
// If 'enable' is true, the default CPU allocator implementation will collect
// AllocatorStats. By default, it's disabled.
void EnableCPUAllocatorStats(bool enable);
// If 'enable' is true, the default CPU allocator implementation will collect
// full statistics. By default, it's disabled.
void EnableCPUAllocatorFullStats(bool enable);
// An object that does the underlying suballoc/free of memory for a higher-level
// allocator. The expectation is that the higher-level allocator is doing some
// kind of cache or pool management so that it will call SubAllocator::Alloc and
// Free relatively infrequently, compared to the number of times its own
// AllocateRaw and Free methods are called.
class SubAllocator {
public:
// Visitor gets called with a pointer to a memory area and its
// size in bytes. The index value will be numa_node for a CPU
// allocator and GPU id for a GPU allocator.
typedef std::function<void(void*, int index, size_t)> Visitor;
SubAllocator(const std::vector<Visitor>& alloc_visitors,
const std::vector<Visitor>& free_visitors);
virtual ~SubAllocator() {}
virtual void* Alloc(size_t alignment, size_t num_bytes) = 0;
virtual void Free(void* ptr, size_t num_bytes) = 0;
protected:
// Implementation of Alloc() method must call this on newly allocated
// value.
void VisitAlloc(void* ptr, int index, size_t num_bytes);
// Implementation of Free() method must call this on value to be
// freed immediately before deallocation.
void VisitFree(void* ptr, int index, size_t num_bytes);
const std::vector<Visitor> alloc_visitors_;
const std::vector<Visitor> free_visitors_;
};
} // namespace tensorflow
#endif // TENSORFLOW_CORE_FRAMEWORK_ALLOCATOR_H_
@@ -0,0 +1,122 @@
/* Copyright 2017 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
// Classes to maintain a static registry of memory allocator factories.
#ifndef TENSORFLOW_CORE_FRAMEWORK_ALLOCATOR_REGISTRY_H_
#define TENSORFLOW_CORE_FRAMEWORK_ALLOCATOR_REGISTRY_H_
#include <string>
#include <vector>
#include "tensorflow/core/framework/allocator.h"
#include "tensorflow/core/platform/mutex.h"
#include "tensorflow/core/platform/numa.h"
namespace tensorflow {
class AllocatorFactory {
public:
virtual ~AllocatorFactory() {}
// Returns true if the factory will create a functionally different
// SubAllocator for different (legal) values of numa_node.
virtual bool NumaEnabled() { return false; }
// Create an Allocator.
virtual Allocator* CreateAllocator() = 0;
// Create a SubAllocator. If NumaEnabled() is true, then returned SubAllocator
// will allocate memory local to numa_node. If numa_node == kNUMANoAffinity
// then allocated memory is not specific to any NUMA node.
virtual SubAllocator* CreateSubAllocator(int numa_node) = 0;
};
// A singleton registry of AllocatorFactories.
//
// Allocators should be obtained through ProcessState or cpu_allocator()
// (deprecated), not directly through this interface. The purpose of this
// registry is to allow link-time discovery of multiple AllocatorFactories among
// which ProcessState will obtain the best fit at startup.
class AllocatorFactoryRegistry {
public:
AllocatorFactoryRegistry() {}
~AllocatorFactoryRegistry() {}
void Register(const char* source_file, int source_line, const string& name,
int priority, AllocatorFactory* factory);
// Returns 'best fit' Allocator. Find the factory with the highest priority
// and return an allocator constructed by it. If multiple factories have
// been registered with the same priority, picks one by unspecified criteria.
Allocator* GetAllocator();
// Returns 'best fit' SubAllocator. First look for the highest priority
// factory that is NUMA-enabled. If none is registered, fall back to the
// highest priority non-NUMA-enabled factory. If NUMA-enabled, return a
// SubAllocator specific to numa_node, otherwise return a NUMA-insensitive
// SubAllocator.
SubAllocator* GetSubAllocator(int numa_node);
// Returns the singleton value.
static AllocatorFactoryRegistry* singleton();
private:
mutex mu_;
bool first_alloc_made_ = false;
struct FactoryEntry {
const char* source_file;
int source_line;
string name;
int priority;
std::unique_ptr<AllocatorFactory> factory;
std::unique_ptr<Allocator> allocator;
// Index 0 corresponds to kNUMANoAffinity, other indices are (numa_node +
// 1).
std::vector<std::unique_ptr<SubAllocator>> sub_allocators;
};
std::vector<FactoryEntry> factories_ GUARDED_BY(mu_);
// Returns any FactoryEntry registered under 'name' and 'priority',
// or 'nullptr' if none found.
const FactoryEntry* FindEntry(const string& name, int priority) const
EXCLUSIVE_LOCKS_REQUIRED(mu_);
TF_DISALLOW_COPY_AND_ASSIGN(AllocatorFactoryRegistry);
};
class AllocatorFactoryRegistration {
public:
AllocatorFactoryRegistration(const char* file, int line, const string& name,
int priority, AllocatorFactory* factory) {
AllocatorFactoryRegistry::singleton()->Register(file, line, name, priority,
factory);
}
};
#define REGISTER_MEM_ALLOCATOR(name, priority, factory) \
REGISTER_MEM_ALLOCATOR_UNIQ_HELPER(__COUNTER__, __FILE__, __LINE__, name, \
priority, factory)
#define REGISTER_MEM_ALLOCATOR_UNIQ_HELPER(ctr, file, line, name, priority, \
factory) \
REGISTER_MEM_ALLOCATOR_UNIQ(ctr, file, line, name, priority, factory)
#define REGISTER_MEM_ALLOCATOR_UNIQ(ctr, file, line, name, priority, factory) \
static AllocatorFactoryRegistration allocator_factory_reg_##ctr( \
file, line, name, priority, new factory)
} // namespace tensorflow
#endif // TENSORFLOW_CORE_FRAMEWORK_ALLOCATOR_REGISTRY_H_
File diff suppressed because it is too large. Load diff
@@ -0,0 +1,63 @@
// GENERATED FILE - DO NOT MODIFY
#ifndef tensorflow_core_framework_api_def_proto_IMPL_H_
#define tensorflow_core_framework_api_def_proto_IMPL_H_
#include "tensorflow/core/framework/api_def.pb.h"
#include "tensorflow/core/framework/api_def.pb_text.h"
#include "tensorflow/core/framework/attr_value.pb.h"
#include "tensorflow/core/framework/attr_value.pb_text-impl.h"
#include "tensorflow/core/framework/resource_handle.pb.h"
#include "tensorflow/core/framework/resource_handle.pb_text-impl.h"
#include "tensorflow/core/framework/tensor.pb.h"
#include "tensorflow/core/framework/tensor.pb_text-impl.h"
#include "tensorflow/core/framework/tensor_shape.pb.h"
#include "tensorflow/core/framework/tensor_shape.pb_text-impl.h"
#include "tensorflow/core/framework/types.pb.h"
#include "tensorflow/core/framework/types.pb_text-impl.h"
#include "tensorflow/core/lib/strings/proto_text_util.h"
#include "tensorflow/core/lib/strings/scanner.h"
namespace tensorflow {
namespace internal {
void AppendProtoDebugString(
::tensorflow::strings::ProtoTextOutput* o,
const ::tensorflow::ApiDef_Endpoint& msg);
bool ProtoParseFromScanner(
::tensorflow::strings::Scanner* scanner, bool nested, bool close_curly,
::tensorflow::ApiDef_Endpoint* msg);
void AppendProtoDebugString(
::tensorflow::strings::ProtoTextOutput* o,
const ::tensorflow::ApiDef_Arg& msg);
bool ProtoParseFromScanner(
::tensorflow::strings::Scanner* scanner, bool nested, bool close_curly,
::tensorflow::ApiDef_Arg* msg);
void AppendProtoDebugString(
::tensorflow::strings::ProtoTextOutput* o,
const ::tensorflow::ApiDef_Attr& msg);
bool ProtoParseFromScanner(
::tensorflow::strings::Scanner* scanner, bool nested, bool close_curly,
::tensorflow::ApiDef_Attr* msg);
void AppendProtoDebugString(
::tensorflow::strings::ProtoTextOutput* o,
const ::tensorflow::ApiDef& msg);
bool ProtoParseFromScanner(
::tensorflow::strings::Scanner* scanner, bool nested, bool close_curly,
::tensorflow::ApiDef* msg);
void AppendProtoDebugString(
::tensorflow::strings::ProtoTextOutput* o,
const ::tensorflow::ApiDefs& msg);
bool ProtoParseFromScanner(
::tensorflow::strings::Scanner* scanner, bool nested, bool close_curly,
::tensorflow::ApiDefs* msg);
} // namespace internal
} // namespace tensorflow
#endif // tensorflow_core_framework_api_def_proto_IMPL_H_
@@ -0,0 +1,68 @@
// GENERATED FILE - DO NOT MODIFY
#ifndef tensorflow_core_framework_api_def_proto_H_
#define tensorflow_core_framework_api_def_proto_H_
#include "tensorflow/core/framework/api_def.pb.h"
#include "tensorflow/core/platform/macros.h"
#include "tensorflow/core/platform/protobuf.h"
#include "tensorflow/core/platform/types.h"
namespace tensorflow {
// Enum text output for tensorflow.ApiDef.Visibility
const char* EnumName_ApiDef_Visibility(
::tensorflow::ApiDef_Visibility value);
// Message-text conversion for tensorflow.ApiDef.Endpoint
string ProtoDebugString(
const ::tensorflow::ApiDef_Endpoint& msg);
string ProtoShortDebugString(
const ::tensorflow::ApiDef_Endpoint& msg);
bool ProtoParseFromString(
const string& s,
::tensorflow::ApiDef_Endpoint* msg)
TF_MUST_USE_RESULT;
// Message-text conversion for tensorflow.ApiDef.Arg
string ProtoDebugString(
const ::tensorflow::ApiDef_Arg& msg);
string ProtoShortDebugString(
const ::tensorflow::ApiDef_Arg& msg);
bool ProtoParseFromString(
const string& s,
::tensorflow::ApiDef_Arg* msg)
TF_MUST_USE_RESULT;
// Message-text conversion for tensorflow.ApiDef.Attr
string ProtoDebugString(
const ::tensorflow::ApiDef_Attr& msg);
string ProtoShortDebugString(
const ::tensorflow::ApiDef_Attr& msg);
bool ProtoParseFromString(
const string& s,
::tensorflow::ApiDef_Attr* msg)
TF_MUST_USE_RESULT;
// Message-text conversion for tensorflow.ApiDef
string ProtoDebugString(
const ::tensorflow::ApiDef& msg);
string ProtoShortDebugString(
const ::tensorflow::ApiDef& msg);
bool ProtoParseFromString(
const string& s,
::tensorflow::ApiDef* msg)
TF_MUST_USE_RESULT;
// Message-text conversion for tensorflow.ApiDefs
string ProtoDebugString(
const ::tensorflow::ApiDefs& msg);
string ProtoShortDebugString(
const ::tensorflow::ApiDefs& msg);
bool ProtoParseFromString(
const string& s,
::tensorflow::ApiDefs* msg)
TF_MUST_USE_RESULT;
} // namespace tensorflow
#endif // tensorflow_core_framework_api_def_proto_H_
File diff suppressed because it is too large. Load diff
@@ -0,0 +1,47 @@
// GENERATED FILE - DO NOT MODIFY
#ifndef tensorflow_core_framework_attr_value_proto_IMPL_H_
#define tensorflow_core_framework_attr_value_proto_IMPL_H_
#include "tensorflow/core/framework/attr_value.pb.h"
#include "tensorflow/core/framework/attr_value.pb_text.h"
#include "tensorflow/core/framework/resource_handle.pb.h"
#include "tensorflow/core/framework/resource_handle.pb_text-impl.h"
#include "tensorflow/core/framework/tensor.pb.h"
#include "tensorflow/core/framework/tensor.pb_text-impl.h"
#include "tensorflow/core/framework/tensor_shape.pb.h"
#include "tensorflow/core/framework/tensor_shape.pb_text-impl.h"
#include "tensorflow/core/framework/types.pb.h"
#include "tensorflow/core/framework/types.pb_text-impl.h"
#include "tensorflow/core/lib/strings/proto_text_util.h"
#include "tensorflow/core/lib/strings/scanner.h"
namespace tensorflow {
namespace internal {
void AppendProtoDebugString(
::tensorflow::strings::ProtoTextOutput* o,
const ::tensorflow::AttrValue_ListValue& msg);
bool ProtoParseFromScanner(
::tensorflow::strings::Scanner* scanner, bool nested, bool close_curly,
::tensorflow::AttrValue_ListValue* msg);
void AppendProtoDebugString(
::tensorflow::strings::ProtoTextOutput* o,
const ::tensorflow::AttrValue& msg);
bool ProtoParseFromScanner(
::tensorflow::strings::Scanner* scanner, bool nested, bool close_curly,
::tensorflow::AttrValue* msg);
void AppendProtoDebugString(
::tensorflow::strings::ProtoTextOutput* o,
const ::tensorflow::NameAttrList& msg);
bool ProtoParseFromScanner(
::tensorflow::strings::Scanner* scanner, bool nested, bool close_curly,
::tensorflow::NameAttrList* msg);
} // namespace internal
} // namespace tensorflow
#endif // tensorflow_core_framework_attr_value_proto_IMPL_H_
@@ -0,0 +1,44 @@
// GENERATED FILE - DO NOT MODIFY
#ifndef tensorflow_core_framework_attr_value_proto_H_
#define tensorflow_core_framework_attr_value_proto_H_
#include "tensorflow/core/framework/attr_value.pb.h"
#include "tensorflow/core/platform/macros.h"
#include "tensorflow/core/platform/protobuf.h"
#include "tensorflow/core/platform/types.h"
namespace tensorflow {
// Message-text conversion for tensorflow.AttrValue.ListValue
string ProtoDebugString(
const ::tensorflow::AttrValue_ListValue& msg);
string ProtoShortDebugString(
const ::tensorflow::AttrValue_ListValue& msg);
bool ProtoParseFromString(
const string& s,
::tensorflow::AttrValue_ListValue* msg)
TF_MUST_USE_RESULT;
// Message-text conversion for tensorflow.AttrValue
string ProtoDebugString(
const ::tensorflow::AttrValue& msg);
string ProtoShortDebugString(
const ::tensorflow::AttrValue& msg);
bool ProtoParseFromString(
const string& s,
::tensorflow::AttrValue* msg)
TF_MUST_USE_RESULT;
// Message-text conversion for tensorflow.NameAttrList
string ProtoDebugString(
const ::tensorflow::NameAttrList& msg);
string ProtoShortDebugString(
const ::tensorflow::NameAttrList& msg);
bool ProtoParseFromString(
const string& s,
::tensorflow::NameAttrList* msg)
TF_MUST_USE_RESULT;
} // namespace tensorflow
#endif // tensorflow_core_framework_attr_value_proto_H_
@@ -0,0 +1,129 @@
/* Copyright 2015 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
#ifndef TENSORFLOW_CORE_FRAMEWORK_ATTR_VALUE_UTIL_H_
#define TENSORFLOW_CORE_FRAMEWORK_ATTR_VALUE_UTIL_H_
#include <functional>
#include <string>
#include <vector>
#include "tensorflow/core/framework/partial_tensor_shape.h"
#include "tensorflow/core/framework/tensor.h"
#include "tensorflow/core/framework/tensor_shape.h"
#include "tensorflow/core/framework/types.h"
#include "tensorflow/core/lib/core/status.h"
#include "tensorflow/core/lib/core/stringpiece.h"
#include "tensorflow/core/lib/gtl/array_slice.h"
namespace tensorflow {
// Forward declare protos so their symbols can be removed from .so exports
class AttrValue;
class NameAttrList;
// A human-readable rendering of attr_value, that is more concise than a
// text-format proto.
string SummarizeAttrValue(const AttrValue& attr_value);
// Generates an error if attr_value doesn't have the indicated attr type.
Status AttrValueHasType(const AttrValue& attr_value, StringPiece type);
// Converts a text proto value from "text" into the field of *out
// indicated by "type" (e.g. from the type field of an AttrDef).
// Examples:
// * If type:"int" and text:"-14", then *out is set to "i: -14"
// * If type:"list(string)" and text:"['foo', 'bar']",
// then *out is set to "list { s: ['foo', 'bar'] }"
// Returns true on success.
bool ParseAttrValue(StringPiece type, StringPiece text, AttrValue* out);
// Sets *out based on the type of value.
void SetAttrValue(const string& value, AttrValue* out);
void SetAttrValue(const char* value, AttrValue* out);
void SetAttrValue(StringPiece value, AttrValue* out);
void SetAttrValue(int64 value, AttrValue* out);
void SetAttrValue(int32 value, AttrValue* out);
void SetAttrValue(float value, AttrValue* out);
void SetAttrValue(double value, AttrValue* out);
void SetAttrValue(bool value, AttrValue* out);
void SetAttrValue(DataType value, AttrValue* out);
void SetAttrValue(const TensorShape& value, AttrValue* out);
void SetAttrValue(const TensorShapeProto& value, AttrValue* out);
void SetAttrValue(const PartialTensorShape& value, AttrValue* out);
void SetAttrValue(const Tensor& value, AttrValue* out);
void SetAttrValue(const TensorProto& value, AttrValue* out);
void SetAttrValue(const NameAttrList& value, AttrValue* out);
void SetAttrValue(gtl::ArraySlice<string> value, AttrValue* out);
void SetAttrValue(gtl::ArraySlice<const char*> value, AttrValue* out);
void SetAttrValue(gtl::ArraySlice<StringPiece> value, AttrValue* out);
void SetAttrValue(gtl::ArraySlice<int64> value, AttrValue* out);
void SetAttrValue(gtl::ArraySlice<int32> value, AttrValue* out);
void SetAttrValue(gtl::ArraySlice<float> value, AttrValue* out);
void SetAttrValue(gtl::ArraySlice<double> value, AttrValue* out);
void SetAttrValue(gtl::ArraySlice<bool> value, AttrValue* out);
void SetAttrValue(const std::vector<bool>& value, AttrValue* out);
void SetAttrValue(std::initializer_list<bool> value, AttrValue* out);
void SetAttrValue(DataTypeSlice value, AttrValue* out);
void SetAttrValue(gtl::ArraySlice<TensorShape> value, AttrValue* out);
void SetAttrValue(gtl::ArraySlice<TensorShapeProto> value, AttrValue* out);
void SetAttrValue(gtl::ArraySlice<PartialTensorShape> value, AttrValue* out);
void SetAttrValue(gtl::ArraySlice<Tensor> value, AttrValue* out);
void SetAttrValue(gtl::ArraySlice<TensorProto> value, AttrValue* out);
void SetAttrValue(gtl::ArraySlice<NameAttrList> value, AttrValue* out);
void SetAttrValue(const AttrValue& value, AttrValue* out);
// Returns true if a and b have the same value.
bool AreAttrValuesEqual(const AttrValue& a, const AttrValue& b);
// Returns a hash of `a` that is consistent with AreAttrValuesEqual. In other
// words, if two AttrValues compare equal according to AreAttrValuesEqual,
// they will have the same hash value.
// Similarly to protobuf deterministic serialization, hash value is
// guaranteed to be stable only for a given binary. In particular, one should
// probably not persist the returned value.
uint64 AttrValueHash(const AttrValue& a);
// WARNING: Equality check might return false-negative for large (> 32mb)
// tensors defined with different TensorProto representations.
//
// A pair of consistent hash and equals functions that are guaranteed to be fast
// with AttrValues that potentially can have very large Tensors (larger than
// 32mb) defined by TensorProto. If large identical Tensors are defined using
// different representations (e.g. one with tensor content, and second with
// bool_val), they will have different hash code and equals will return false.
// Small (less than 32mb) tensors with different TensorProto representations
// hashed/compared by their tensor content.
uint64 FastAttrValueHash(const AttrValue& a);
bool FastAreAttrValuesEqual(const AttrValue& a, const AttrValue& b);
// Returns true if "val" has a placeholder.
bool HasPlaceHolder(const AttrValue& val);
// SubstitutePlaceholders recursively replaces placeholders in 'value'
// with an attr value by calling SubstituteFunc. Returns true iff all
// placeholders in "value" are replaced with a value.
//
// SubstituteFunc is given a placeholder string. If the placeholder is
// unknown, SubstituteFunc returns false. Otherwise, overwrites the
// attr value and returns true.
using SubstituteFunc = std::function<bool(const string&, AttrValue*)>;
bool SubstitutePlaceholders(const SubstituteFunc& substitute, AttrValue* value);
} // namespace tensorflow
#endif // TENSORFLOW_CORE_FRAMEWORK_ATTR_VALUE_UTIL_H_
@@ -0,0 +1,63 @@
/* Copyright 2015 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
#ifndef TENSORFLOW_CORE_FRAMEWORK_BFLOAT16_H_
#define TENSORFLOW_CORE_FRAMEWORK_BFLOAT16_H_
#include "tensorflow/core/framework/numeric_types.h"
#include "tensorflow/core/platform/byte_order.h"
#include "tensorflow/core/platform/types.h"
#if defined(PLATFORM_WINDOWS)
#include "tensorflow/core/platform/windows/cpu_info.h"
#endif
// Compact 16-bit encoding of floating point numbers. This representation uses
// 1 bit for the sign, 8 bits for the exponent and 7 bits for the mantissa. It
// is assumed that floats are in IEEE 754 format so the representation is just
// bits 16-31 of a single precision float.
//
// NOTE: The IEEE floating point standard defines a float16 format that
// is different than this format (it has fewer bits of exponent and more
// bits of mantissa). We don't use that format here because conversion
// to/from 32-bit floats is more complex for that format, and the
// conversion for this format is very simple.
//
// Because of the existing IEEE float16 type, we do not name our representation
// "float16" but just use "uint16".
//
// <-----our 16bits float------->
// s e e e e e e e e f f f f f f f f f f f f f f f f f f f f f f f
// <------------------------------float-------------------------->
// 3 3 2 2 1 1 0
// 1 0 3 2 5 4 0
//
//
// This type only supports conversion back and forth with float.
//
// This file must be compilable by nvcc.
//
// The type is defined in framework/numeric_types.h.
namespace tensorflow {
// Conversion routines between an array of float and bfloat16 of
// "size".
void FloatToBFloat16(const float* src, bfloat16* dst, int64 size);
void BFloat16ToFloat(const bfloat16* src, float* dst, int64 size);
} // namespace tensorflow
#endif // TENSORFLOW_CORE_FRAMEWORK_BFLOAT16_H_
@@ -0,0 +1,146 @@
/* Copyright 2015 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
#ifndef TENSORFLOW_CORE_FRAMEWORK_CANCELLATION_H_
#define TENSORFLOW_CORE_FRAMEWORK_CANCELLATION_H_
#include <atomic>
#include <functional>
#include "tensorflow/core/lib/core/notification.h"
#include "tensorflow/core/lib/core/status.h"
#include "tensorflow/core/lib/gtl/flatmap.h"
#include "tensorflow/core/lib/hash/hash.h"
#include "tensorflow/core/platform/mutex.h"
#include "tensorflow/core/platform/thread_annotations.h"
#include "tensorflow/core/platform/types.h"
namespace tensorflow {
// A token that can be used to register and deregister a
// CancelCallback with a CancellationManager.
//
// CancellationToken values must be created by a call to
// CancellationManager::get_cancellation_token.
typedef int64 CancellationToken;
// A callback that is invoked when a step is canceled.
//
// NOTE(mrry): See caveats about CancelCallback implementations in the
// comment for CancellationManager::RegisterCallback.
typedef std::function<void()> CancelCallback;
class CancellationManager {
public:
// A value that won't be returned by get_cancellation_token().
static const CancellationToken kInvalidToken;
CancellationManager();
~CancellationManager();
// Run all callbacks associated with this manager.
void StartCancel();
// Returns true iff StartCancel() has been called.
bool IsCancelled() { return is_cancelled_.load(std::memory_order_acquire); }
// Returns a token that must be used in calls to RegisterCallback
// and DeregisterCallback.
CancellationToken get_cancellation_token();
// Attempts to register the given callback to be invoked when this
// manager is cancelled. Returns true if the callback was
// registered; returns false if this manager was already cancelled,
// and the callback was not registered.
//
// If this method returns false, it is the caller's responsibility
// to perform any cancellation cleanup.
//
// This method is tricky to use correctly. The following usage pattern
// is recommended:
//
// class ObjectWithCancellableOperation {
// mutex mu_;
// void CancellableOperation(CancellationManager* cm,
// std::function<void(Status)> callback) {
// bool already_cancelled;
// CancellationToken token = cm->get_cancellation_token();
// {
// mutex_lock(mu_);
// already_cancelled = !cm->RegisterCallback(
// [this, token]() { Cancel(token); });
// if (!already_cancelled) {
// // Issue asynchronous operation. Associate the pending operation
// // with `token` in some object state, or provide another way for
// // the Cancel method to look up the operation for cancellation.
// // Ensure that `cm->DeregisterCallback(token)` is called without
// // holding `mu_`, before `callback` is invoked.
// // ...
// }
// }
// if (already_cancelled) {
// callback(errors::Cancelled("Operation was cancelled"));
// }
// }
//
// void Cancel(CancellationToken token) {
// mutex_lock(mu_);
// // Take action to cancel the operation with the given cancellation
// // token.
// }
//
// NOTE(mrry): The caller should take care that (i) the calling code
// is robust to `callback` being invoked asynchronously (e.g. from
// another thread), (ii) `callback` is deregistered by a call to
// this->DeregisterCallback(token) when the operation completes
// successfully, and (iii) `callback` does not invoke any method
// on this cancellation manager. Furthermore, it is important that
// the eventual caller of the complementary DeregisterCallback does not
// hold any mutexes that are required by `callback`.
bool RegisterCallback(CancellationToken token, CancelCallback callback);
// Deregister the callback that, when registered, was associated
// with the given cancellation token. Returns true iff the callback
// was deregistered and will not be invoked; otherwise returns false
// after the callback has been invoked, blocking if necessary.
//
// NOTE(mrry): This method may block if cancellation is in progress.
// The caller of this method must not hold any mutexes that are required
// to invoke any cancellation callback that has been registered with this
// cancellation manager.
bool DeregisterCallback(CancellationToken token);
// Deregister the callback that, when registered, was associated
// with the given cancellation token. Returns true iff the callback
// was deregistered and will not be invoked; otherwise returns false
// immediately, with no guarantee that the callback has completed.
//
// This method is guaranteed to return true if StartCancel has not been
// called.
bool TryDeregisterCallback(CancellationToken token);
private:
bool is_cancelling_;
std::atomic_bool is_cancelled_;
mutex mu_;
Notification cancelled_notification_;
CancellationToken next_cancellation_token_ GUARDED_BY(mu_);
gtl::FlatMap<CancellationToken, CancelCallback> callbacks_ GUARDED_BY(mu_);
};
} // namespace tensorflow
#endif // TENSORFLOW_CORE_FRAMEWORK_CANCELLATION_H_
@@ -0,0 +1,418 @@
/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
#ifndef TENSORFLOW_CORE_FRAMEWORK_COLLECTIVE_H_
#define TENSORFLOW_CORE_FRAMEWORK_COLLECTIVE_H_
#include <string>
#include <vector>
#include "tensorflow/core/framework/device_attributes.pb.h"
#include "tensorflow/core/framework/device_base.h"
#include "tensorflow/core/framework/op_kernel.h"
#include "tensorflow/core/lib/core/refcount.h"
#include "tensorflow/core/lib/core/status.h"
namespace tensorflow {
class BufRendezvous;
class CancellationManager;
class CompleteGroupRequest;
class CompleteGroupResponse;
class CompleteInstanceRequest;
class CompleteInstanceResponse;
class Device;
class DeviceMgr;
class GetStepSequenceRequest;
class GetStepSequenceResponse;
class Op;
class Tensor;
// Types of supported collective operations.
enum CollectiveType {
REDUCTION_COLLECTIVE = 0,
BROADCAST_COLLECTIVE,
UNDEFINED_COLLECTIVE,
};
// Data common to all members of a device group.
// All members share the same device set but its order is
// particular to an instance so it is stored there.
struct CollGroupParams {
int32 group_key;
int32 group_size;
DeviceType device_type;
int32 num_tasks; // number of distinct tasks in group
string ToString() const;
CollGroupParams()
: group_key(0), group_size(0), device_type(DEVICE_CPU), num_tasks(0) {}
};
// The best implementation of a collective op depends on many factors
// including the number of devices involved, the topology of
// interconnects between them and the sizes of inputs. This structure
// is used in generating and representing data movement choreography
// for each specific algorithm, hence it does not have a single, fixed
// interpretation. On first execution the runtime will update this
// structure with decisions that will guide all subsequent executions.
struct CollImplDetails {
string collective_name;
std::vector<std::vector<int>> subdiv_permutations;
std::vector<int> subdiv_offsets;
std::vector<int> subdiv_source_rank; // rank of source in each subdiv
};
// Data common to all members of a collective instance.
struct CollInstanceParams {
// Identifies all participating graph nodes.
int32 instance_key = -1;
CollectiveType type = UNDEFINED_COLLECTIVE;
DataType data_type = DT_FLOAT;
TensorShape shape = {0};
// Fully qualified name of device for each member, in default rank order.
std::vector<string> device_names;
// Task name prefix of corresponding device name.
std::vector<string> task_names;
// True if every task has the same number of devices.
bool same_num_devices_per_task = false;
CollImplDetails impl_details;
string ToString() const;
CollInstanceParams& operator=(const struct CollInstanceParams& other);
};
// Data common to all instance members in the same task.
struct CollTaskParams {
// True for devices that are local to the process, i.e. no RPC needed.
std::vector<bool> is_local;
string ToString() const;
};
// Unique to a single CollectiveOp node.
struct CollectiveParams {
CollGroupParams group;
CollInstanceParams instance;
CollTaskParams task;
string name = ""; // node name used only for log or error messages
int default_rank = -1; // index of this op within device_names
bool is_source = false; // broadcast only
int source_rank = -1; // broadcast only
// Rank of this device in each subdivision permutation.
std::vector<int> subdiv_rank;
std::unique_ptr<OpKernel> merge_op; // reduction only
std::unique_ptr<OpKernel> final_op; // reduction only
string ToString() const;
};
class CollectiveExecutor;
// Interface that provides resolution of device localities.
class DeviceResolverInterface {
public:
virtual ~DeviceResolverInterface() {}
// Collects DeviceLocality protobufs from all of the devices identified
// in 'col_params'.
virtual void GetDeviceLocalitiesAsync(const CollInstanceParams& inst_params,
std::vector<DeviceLocality>* localities,
const StatusCallback& done) = 0;
// Populate *locality with the DeviceLocality of the specified
// device.
virtual void GetLocalityAsync(const string& device, const string& task,
DeviceLocality* locality,
const StatusCallback& done) = 0;
// Clear the cache of device data belonging
// to the specified task.
virtual void ClearTask(const string& task) = 0;
};
// Interface that provides resolution of shared CollectiveParams fields.
class ParamResolverInterface {
public:
virtual ~ParamResolverInterface() {}
// Called by each collective op at first execution in order to fill out
// the CollectiveParams structure with data gathered from the full
// (maybe distributed) collection of peer nodes.
virtual void CompleteParamsAsync(const string& device, CollectiveParams* cp,
CancellationManager* cancel_mgr,
const StatusCallback& done) = 0;
// Used within a distributed implementation to discover/verify
// data shared across a device group.
virtual void CompleteGroupAsync(const CompleteGroupRequest* request,
CompleteGroupResponse* response,
CancellationManager* cancel_mgr,
const StatusCallback& done) = 0;
// Used within a distributed implementation to discover/verify data
// shared across an instance group.
virtual void CompleteInstanceAsync(const CompleteInstanceRequest* request,
CompleteInstanceResponse* response,
CancellationManager* cancel_mgr,
const StatusCallback& done) = 0;
};
// Graphs which utilize Collective Ops in a common instance must
// execute with identical step_ids even if they are disjoint graphs
// run by otherwise independent tasks. This interface supplies
// coordinated step_ids to use in such cases.
class StepSequenceInterface {
public:
virtual ~StepSequenceInterface() {}
// Used with a distributed implementation to coordinate step_id
// sequences across tasks.
virtual void GetStepSequenceAsync(const GetStepSequenceRequest* request,
GetStepSequenceResponse* response,
const StatusCallback& done) = 0;
// Refresh the local per-graph_key step_id sequence from collective
// group leader, if applicable.
virtual void RefreshStepIdSequenceAsync(int64 graph_key,
const StatusCallback& done) = 0;
// Returns the step_id that should be used for initiating a new execution
// on the specified graph. May return the same step_id multiple times if
// RetireStepId or RefreshStepIdReservation is not called.
virtual int64 NextStepId(int64 graph_key) = 0;
// Reports that execution of the given step has completed successfully.
// Should be called immediately after a step completes with OK status,
// prior to calling NextStepId(). If the step fails, don't call.
virtual void RetireStepId(int64 graph_key, int64 step_id) = 0;
};
// Interface that provides access to per-step CollectiveExecutor
// instances and various distributed resolution capabilities.
class CollectiveExecutorMgrInterface : public StepSequenceInterface {
public:
virtual ~CollectiveExecutorMgrInterface() {}
// Returns the step-specific CollectiveExecutor, creating if one does not
// already exist. The caller assumes ownership of one Ref on the object.
virtual CollectiveExecutor* FindOrCreate(int64 step_id) = 0;
// If there is a CollectiveExecutor for step_id, remove it from the
// table.
virtual void Cleanup(int64 step_id) = 0;
virtual ParamResolverInterface* GetParamResolver() const = 0;
virtual DeviceResolverInterface* GetDeviceResolver() const = 0;
};
// Interface that a Collective Op implementation uses to exchange data
// with peers. Note that data exchange is currently limited to types
// for which DMAHelper::CanUseDMA() returns true, i.e. dense numeric
// types.
class PeerAccessInterface {
public:
virtual ~PeerAccessInterface() {}
virtual void RecvFromPeer(const string& peer_device, const string& peer_task,
bool peer_is_local, const string& key,
Device* to_device, DeviceContext* to_device_ctx,
const AllocatorAttributes& to_alloc_attr,
Tensor* to_tensor,
const DeviceLocality& client_locality,
int dev_to_dev_stream_index,
const StatusCallback& done) = 0;
virtual void PostToPeer(const string& peer_device, const string& peer_task,
const string& key, Device* from_device,
DeviceContext* from_device_ctx,
const AllocatorAttributes& from_alloc_attr,
const Tensor* from_tensor,
const DeviceLocality& client_locality,
const StatusCallback& done) = 0;
};
class PerStepCollectiveRemoteAccess;
// A step-specific object that can execute a collective operation completely
// described by a CollectiveParams object.
class CollectiveExecutor : public PeerAccessInterface, public core::RefCounted {
public:
virtual void StartAbort(const Status& s) {}
virtual void ExecuteAsync(OpKernelContext* ctx,
const CollectiveParams& col_params,
const string& exec_key, StatusCallback done) {
done(errors::Internal(
"A collective Op has been called in a context in which "
"a CollectiveExecutor has not been provided."));
}
virtual void CompleteParamsAsync(const string& device, CollectiveParams* cp,
CancellationManager* cancel_mgr,
StatusCallback done) {
cem_->GetParamResolver()->CompleteParamsAsync(device, cp, cancel_mgr, done);
}
virtual PerStepCollectiveRemoteAccess* remote_access() { return nullptr; }
// Used to designate an invalid group or instance key.
static int64 kInvalidId;
// Lexically scoped handle for Ref.
class Handle {
public:
explicit Handle(CollectiveExecutor* ce, bool inherit_ref) : ce_(ce) {
if (!inherit_ref) ce->Ref();
}
~Handle() { ce_->Unref(); }
CollectiveExecutor* get() const { return ce_; }
private:
CollectiveExecutor* ce_;
};
protected:
explicit CollectiveExecutor(CollectiveExecutorMgrInterface* cem)
: cem_(cem) {}
// For use only by derived classes
static OpKernelContext::Params* CtxParams(OpKernelContext* ctx);
CollectiveExecutorMgrInterface* cem_;
TF_DISALLOW_COPY_AND_ASSIGN(CollectiveExecutor);
};
// Interface of a helper object that provides a CollectiveExecutor with
// all of the remote access it needs.
class CollectiveRemoteAccess : public PeerAccessInterface,
public DeviceResolverInterface {
public:
virtual ~CollectiveRemoteAccess() {}
virtual BufRendezvous* buf_rendezvous() = 0;
};
// A per-step version of CollectiveRemoteAccess that cleans up outstanding
// communications in case step execution is abandoned.
class PerStepCollectiveRemoteAccess : public CollectiveRemoteAccess {
public:
virtual ~PerStepCollectiveRemoteAccess() {}
virtual void StartAbort(const Status& s) = 0;
};
class CollectiveContext {
public:
CollectiveContext(CollectiveExecutor* col_exec, const DeviceMgr* dev_mgr,
OpKernelContext* ctx, OpKernelContext::Params* op_params,
const CollectiveParams& col_params, const string& exec_key,
int64 step_id, const Tensor* input, Tensor* output);
virtual ~CollectiveContext() = default;
CollectiveExecutor* col_exec; // Not owned
const DeviceMgr* dev_mgr; // Not owned
OpKernelContext* op_ctx; // Not owned
OpKernelContext::Params* op_params; // Not owned
const CollectiveParams& col_params;
const string exec_key;
const int64 step_id;
const Tensor* input; // Not owned
Tensor* output; // Not owned
Device* device; // The device for which this instance labors
const string device_name;
DeviceLocality device_locality;
};
// Interface of a Collective Op implementation. Each specific CollectiveOp will
// implement this interface and register the implementation via the
// CollectiveRegistry detailed below. See common_runtime/ring_reducer and
// common_runtime/hierarchical_tree_broadcaster for examples.
class CollectiveImplementationInterface {
public:
virtual ~CollectiveImplementationInterface() = default;
// Initializes the portions of `col_params` specific to this
// implementation. Called exactly once for every Collective instance during
// the CollectiveParams resolution process when the graph is first executed.
// NOTE(ayushd): This is effectively a static function because it modifies the
// `col_params` passed in and should not manipulate any data members. However
// because it is virtual and needs to be implemented by every derived class we
// do not mark it as static.
virtual Status InitializeCollectiveParams(CollectiveParams* col_params) = 0;
// Prepares the CollectiveContext for executing this CollectiveImplementation.
// Called from CollectiveExecutor right before calling Run(). The
// CollectiveContext passed in must outlive the CollectiveImplementation
// object.
virtual Status InitializeCollectiveContext(CollectiveContext* col_ctx) = 0;
// Processes and moves data according to the logic of this Collective
// implementation. Relies on appropriate initialization of op-specific
// CollectiveParams in InitializeCollectiveParams(), as well as appropriate
// context initialization in InitializeCollectiveContext().
virtual void Run(StatusCallback done) = 0;
};
// Static-methods only class for registering and looking up collective
// implementations.
class CollectiveRegistry {
public:
using Factory = std::function<CollectiveImplementationInterface*()>;
// Looks up a previously registered CollectiveImplementation under
// `collective_name`. If found, creates an instance of the implementation and
// assign to `implementation`.
static Status Lookup(const string& collective_name,
CollectiveImplementationInterface** implementation);
// Looks up a previously registered CollectiveImplementation under
// `collective_name`. If found, returns the static instance of this
// implementation via `implementation`. This instance should only be used to
// call InitializateCollectiveParams.
static Status LookupParamResolverInstance(
const string& collective_name,
CollectiveImplementationInterface** implementation);
// Returns all registered collective implementations.
static void GetAll(
std::vector<CollectiveImplementationInterface*>* implementations);
private:
friend class CollectiveRegistration;
// Registers a CollectiveImplementation with name `collective_name` and
// factory `factory`. The latter is a function used to create instances of
// the CollectiveImplementation. Also creates a static instance of the
// implementation - this instance is used during param resolution and should
// only be used to call InitializeCollectiveParams.
static Status Register(const string& collective_name, Factory factory);
static Status LookupHelper(const string& collective_name,
CollectiveImplementationInterface** implementation,
bool param_resolver);
};
// Class used to call CollectiveRegistry::Register. This should only be used to
// create a global static object.
class CollectiveRegistration {
public:
CollectiveRegistration(const string& collective_name,
CollectiveRegistry::Factory factory) {
TF_CHECK_OK(CollectiveRegistry::Register(collective_name, factory));
}
};
#define REGISTER_COLLECTIVE(name, implementation) \
static CollectiveRegistration register_##name##_collective( \
#name, []() { return new implementation; });
} // namespace tensorflow
#endif // TENSORFLOW_CORE_FRAMEWORK_COLLECTIVE_H_
@@ -0,0 +1,317 @@
/* Copyright 2016 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
#ifndef TENSORFLOW_CORE_FRAMEWORK_COMMON_SHAPE_FNS_H_
#define TENSORFLOW_CORE_FRAMEWORK_COMMON_SHAPE_FNS_H_
#include <array>
#include "tensorflow/core/framework/shape_inference.h"
#include "tensorflow/core/util/padding.h"
#include "tensorflow/core/util/tensor_format.h"
namespace tensorflow {
// GetWindowedOutputSize(): Given an input tensor, kernel, stride and padding
// type, the function computes the output and padding dimensions.
//
// For example, ignoring batches or multiple features, a 1D convolution
// takes as input a 1D tensor of shape (H), and convolves it with a filter of
// shape (K).
//
// It also takes in a few additional parameters:
//
// Stride (S): the stride with which we apply the filters. This is the offset
// between locations where we apply the filters. A larger stride
// means that the output will be spatially smaller.
//
// Padding (P): the padding we apply to the input tensor along each
// dimension. This is usually used to make sure that the spatial dimensions
// do not shrink when we progress with convolutions. Two types of padding are
// often used:
// SAME: the pad value is computed so that the output will have size H/S.
// VALID: no padding is carried out.
// The padded area is zero-filled.
//
// The output dimensions for convolution and many other operations, when given
// all the parameters above, are as follows:
// - When Padding = SAME: the output size is (H'), where
// H' = ceil(float(H) / float(S))
// where ceil is the ceiling function. The number of padded cells
// is computed as:
// Pc = ((H' - 1) * S + K - H) / 2
// When the stride is 1, the expression simplifies to
// H' = H, Pc = (K-1)/2.
// This is where SAME comes from - the output has the same size as the input
// has.
//
// - When Padding = VALID: the output size is computed as
// H' = ceil(float(H - K + 1) / float(S))
// and the number of padded cells is always zero.
// When the stride is 1, the expression simplifies to
// H' = H-K+1.
//
// For convolution, mathematically, the output value at location (r')
// is the inner product of two vectors: the chunk of input at
// ((r'*S-Pr) : (r'*S-Pr+K)),
// and the filter.
//
// For 2D and 3D convolutions, the spatial dimensions are orthogonal, so the
// size and padding of each spatial dimension can be computed by calling
// GetWindowedOutputSize separately for each dimension.
//
Status GetWindowedOutputSize(int64 input_size, int64 filter_size, int64 stride,
Padding padding_type, int64* output_size,
int64* padding_size);
// The V2 version computes the same outputs with arbitrary dilation_rate.
// The output dimensions are computed as follows:
// - When adding dilation_rate (D), we compute an effective filter size (K'):
// K' = (K - 1) * D + 1
// - When Padding = SAME: the output size is (H'), where
// H' = ceil(float(H) / float(S))
// where ceil is the ceiling function. The number of padded cells
// is computed as:
// Pc = ((H' - 1) * S + K' - H) / 2
// When the stride is 1, the expression simplifies to
// H' = H, Pc = (K'-1)/2.
// This is where SAME comes from - the output has the same size as the input
// has.
//
// - When Padding = VALID: the output size is computed as
// H' = ceil(float(H - K' + 1) / float(S))
// and the number of padded cells is always zero.
// When the stride is 1, the expression simplifies to
// H' = H-K'+1.
//
// TODO(b/67112639): Merge V2 versions and the original versions eventually.
Status GetWindowedOutputSizeV2(int64 input_size, int64 filter_size,
int64 dilation_rate, int64 stride,
Padding padding_type, int64* output_size,
int64* padding_size);
// Returns the same output dimensions as in GetWindowedOutputSize, but returns
// verbose padding dimensions (before/after). Any excess padding
// (caused by an odd padding size value) is added to the 'padding_after'
// dimension.
Status GetWindowedOutputSizeVerbose(int64 input_size, int64 filter_size,
int64 stride, Padding padding_type,
int64* output_size, int64* padding_before,
int64* padding_after);
// The V2 version computes the same outputs with arbitrary dilation_rate. For
// detailed equations, refer to the comments for GetWindowedOutputSizeV2().
Status GetWindowedOutputSizeVerboseV2(int64 input_size, int64 filter_size,
int64 dilation_rate, int64 stride,
Padding padding_type, int64* output_size,
int64* padding_before,
int64* padding_after);
// Given an input tensor, kernel, stride and padding type, populates the 3D size
// of the output tensor and padding to be applied to the input tensor at the
// lower end of every dimension. Use for 3D convolutions, where the input data
// is padded with zeros, as well as for 3D avg/max pooling, where the input data
// is padded with invalid values that are not considered for pooling.
Status Get3dOutputSize(const std::array<int64, 3>& input,
const std::array<int64, 3>& window,
const std::array<int64, 3>& strides,
Padding padding_type, std::array<int64, 3>* output_ptr,
std::array<int64, 3>* padding_ptr);
// The V2 version computes the same outputs with arbitrary dilation_rate. For
// detailed equations, refer to the comments for GetWindowedOutputSizeV2().
Status Get3dOutputSizeV2(const std::array<int64, 3>& input,
const std::array<int64, 3>& window,
const std::array<int64, 3>& dilations,
const std::array<int64, 3>& strides,
Padding padding_type, std::array<int64, 3>* output_ptr,
std::array<int64, 3>* padding_ptr);
namespace shape_inference {
// Like GetWindowedOutputSize, but deals with DimensionHandles.
Status GetWindowedOutputSizeFromDims(InferenceContext* c,
DimensionHandle input_size,
DimensionOrConstant filter_size,
int64 stride, Padding padding_type,
DimensionHandle* output_size);
// The V2 version computes the same outputs with arbitrary dilation_rate. For
// detailed equations, refer to the comments for GetWindowedOutputSizeV2().
Status GetWindowedOutputSizeFromDimsV2(InferenceContext* c,
DimensionHandle input_size,
DimensionOrConstant filter_size,
int64 dilation_rate, int64 stride,
Padding padding_type,
DimensionHandle* output_size);
// Transfers shape of input(0) to output(0).
Status UnchangedShape(shape_inference::InferenceContext* c);
// Transfers shape of input(0) to output(0), after asserting its rank is <rank>.
inline Status UnchangedShapeWithRank(shape_inference::InferenceContext* c,
int32 rank) {
ShapeHandle out;
TF_RETURN_IF_ERROR(c->WithRank(c->input(0), rank, &out));
c->set_output(0, out);
return Status::OK();
}
// Transfers shape of input(0) to output(0), after asserting its rank >= <rank>.
inline Status UnchangedShapeWithRankAtLeast(
shape_inference::InferenceContext* c, int32 rank) {
ShapeHandle out;
TF_RETURN_IF_ERROR(c->WithRankAtLeast(c->input(0), rank, &out));
c->set_output(0, out);
return Status::OK();
}
// Transfers shape of input(0) to output(0), after asserting its rank <= <rank>.
inline Status UnchangedShapeWithRankAtMost(shape_inference::InferenceContext* c,
int32 rank) {
ShapeHandle out;
TF_RETURN_IF_ERROR(c->WithRankAtMost(c->input(0), rank, &out));
c->set_output(0, out);
return Status::OK();
}
// Shape function for use with ops no outputs.
inline Status NoOutputs(shape_inference::InferenceContext* c) {
return Status::OK();
}
// Shape function for ops that output a single scalar value.
inline Status ScalarShape(shape_inference::InferenceContext* c) {
c->set_output(0, c->Scalar());
return Status::OK();
}
// Shape function for binary ops where both inputs and the output match.
inline Status MergeBothInputsShapeFn(InferenceContext* c) {
ShapeHandle out;
TF_RETURN_IF_ERROR(c->Merge(c->input(0), c->input(1), &out));
c->set_output(0, out);
return Status::OK();
}
// Returns a new shape with the specified dims arranged in the specified
// format. The returned value is owned by this context.
// Note: if format = "FORMAT_NCHW_VECT_C" then C represents the outer_depth.
Status MakeShapeFromFormat(TensorFormat format, DimensionOrConstant N,
const std::vector<DimensionOrConstant>& spatial,
DimensionOrConstant C, ShapeHandle* out,
shape_inference::InferenceContext* context);
// Shape function for MatMul-like operations.
Status MatMulShape(shape_inference::InferenceContext* c);
// Shape function for BiasAdd-like operations.
Status BiasAddShape(shape_inference::InferenceContext* c);
// Shape function for BiasAddGrad-like operations.
Status BiasAddGradShape(shape_inference::InferenceContext* c);
// Shape function for Conv2D-like operations.
Status Conv2DShape(shape_inference::InferenceContext* c);
// Shape function for Conv3D-like operations.
Status Conv3DShape(shape_inference::InferenceContext* c);
// Shape function for DepthwiseConv2D-like operations.
Status DepthwiseConv2DNativeShape(shape_inference::InferenceContext* c);
// Shape function for AvgPool-like operations.
Status AvgPoolShape(shape_inference::InferenceContext* c);
// Shape function for FusedBatchNorm and FusedBatchNormV2 operations.
Status FusedBatchNormShape(shape_inference::InferenceContext* c);
// Shape function for FusedBatchNormGrad and FusedBatchNormGradV2 operations.
Status FusedBatchNormGradShape(shape_inference::InferenceContext* c);
// Shape function for MaxPool-like operations.
Status MaxPoolShape(shape_inference::InferenceContext* c);
// Shape function for MaxPoolV2-like operations.
Status MaxPoolV2Shape(shape_inference::InferenceContext* c, int num_inputs);
// Shape function for 3D Pooling operations.
Status Pool3DShape(shape_inference::InferenceContext* c);
// Shape function for use with ops whose output shapes are unknown.
Status UnknownShape(shape_inference::InferenceContext* c);
// Shape function for reduction operations.
Status ReductionShape(shape_inference::InferenceContext* c);
// Shape function for concat operations.
// <num_inputs_to_concat> is the number of inputs to concatenate and are taken
// from inputs
// [1,num_inputs_to_concat] of the op. Input 0 is the concat_dim input.
Status ConcatShape(shape_inference::InferenceContext* c,
int num_inputs_to_concat);
// Shape function for concat operations.
Status ConcatV2Shape(shape_inference::InferenceContext* c);
// Shape function for binary operators that broadcast their inputs
// and with output to output_index.
// Note: out cannot be NULL.
Status BroadcastBinaryOpOutputShapeFnHelper(InferenceContext* c,
ShapeHandle shape_x,
ShapeHandle shape_y,
ShapeHandle* out);
// Shape function for binary operators that broadcast their inputs
// and with output to output_index.
inline Status BroadcastBinaryOpOutputShapeFn(InferenceContext* c,
int output_index) {
ShapeHandle out;
TF_RETURN_IF_ERROR(
BroadcastBinaryOpOutputShapeFnHelper(c, c->input(0), c->input(1), &out));
c->set_output(output_index, out);
return Status::OK();
}
// Shape function for binary operators that broadcast their inputs.
// Tested by ops/math_ops_test.cc.
inline Status BroadcastBinaryOpShapeFn(InferenceContext* c) {
return BroadcastBinaryOpOutputShapeFn(c, 0);
}
// Shape function for random operations.
Status RandomShape(shape_inference::InferenceContext* c);
// Shape function for Slice opertaions.
Status SliceShape(shape_inference::InferenceContext* c);
// Validates the 3 component tensors of a sparse tensor have the proper
// shapes. This mimics SparseTensor.__init__ in python/framework/ops.py.
Status ValidateSparseTensor(InferenceContext* c, ShapeHandle indices_shape,
ShapeHandle values_shape, ShapeHandle shape_shape);
// Shape function for ScatterNd update/add/sub/... operations.
Status ScatterNdUpdateShape(InferenceContext* c);
// Shape function for ops with an explicit "shape" attribute.
Status ExplicitShape(InferenceContext* c);
// Shape function for multiple-output ops with an explicit "shapes" attribute.
Status ExplicitShapes(InferenceContext* c);
} // namespace shape_inference
} // namespace tensorflow
#endif // TENSORFLOW_CORE_FRAMEWORK_COMMON_SHAPE_FNS_H_
@@ -0,0 +1,58 @@
/* Copyright 2015 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
#ifndef TENSORFLOW_CORE_FRAMEWORK_CONTROL_FLOW_H_
#define TENSORFLOW_CORE_FRAMEWORK_CONTROL_FLOW_H_
#include "tensorflow/core/lib/hash/hash.h"
#include "tensorflow/core/platform/logging.h"
#include "tensorflow/core/platform/types.h"
namespace tensorflow {
const uint64 kIllegalFrameId = ~0uLL;
const int64 kIllegalIterId = -1;
// For the purpose of control flow, every tensor produced by TensorFlow is
// conceptually tagged by a 'FrameAndIter'. FrameAndIter consists of a
// 'frame_id' and an 'iter_id'. The tensor value it represents is produced
// in the frame with frame_id at the iteration of iter_id.
struct FrameAndIter {
uint64 frame_id = kIllegalFrameId;
int64 iter_id = kIllegalIterId;
FrameAndIter() {}
FrameAndIter(uint64 frame, int64 iter) {
frame_id = frame;
iter_id = iter;
}
bool operator==(const FrameAndIter& other) const {
return (frame_id == other.frame_id && iter_id == other.iter_id);
}
};
struct FrameAndIterHash {
size_t operator()(const FrameAndIter& key) const {
// Make sure there are no padding bytes that we don't want
CHECK_EQ(sizeof(uint64) + sizeof(int64), sizeof(FrameAndIter));
return Hash64(reinterpret_cast<const char*>(&key), sizeof(FrameAndIter));
}
};
} // namespace tensorflow
#endif // TENSORFLOW_CORE_FRAMEWORK_CONTROL_FLOW_H_
Loaded 100 of 1283 files, more files were not shown because too many files have changed in this diff. Show more