This commit is contained in:
Kevin Darby committed 2019-01-02 17:04:10 -05:00
1 parent eb1b42a5e8
commit 458eb2aef1
1283 files changed
+323373

No files matched your search

@@ -0,0 +1,539 @@
// Generated by the protocol buffer compiler. DO NOT EDIT!
// source: tensorflow/core/example/example.proto
#ifndef PROTOBUF_INCLUDED_tensorflow_2fcore_2fexample_2fexample_2eproto
#define PROTOBUF_INCLUDED_tensorflow_2fcore_2fexample_2fexample_2eproto
#include <string>
#include <google/protobuf/stubs/common.h>
#if GOOGLE_PROTOBUF_VERSION < 3006000
#error This file was generated by a newer version of protoc which is
#error incompatible with your Protocol Buffer headers. Please update
#error your headers.
#endif
#if 3006000 < GOOGLE_PROTOBUF_MIN_PROTOC_VERSION
#error This file was generated by an older version of protoc which is
#error incompatible with your Protocol Buffer headers. Please
#error regenerate this file with a newer version of protoc.
#endif
#include <google/protobuf/io/coded_stream.h>
#include <google/protobuf/arena.h>
#include <google/protobuf/arenastring.h>
#include <google/protobuf/generated_message_table_driven.h>
#include <google/protobuf/generated_message_util.h>
#include <google/protobuf/inlined_string_field.h>
#include <google/protobuf/metadata.h>
#include <google/protobuf/message.h>
#include <google/protobuf/repeated_field.h> // IWYU pragma: export
#include <google/protobuf/extension_set.h> // IWYU pragma: export
#include <google/protobuf/unknown_field_set.h>
#include "tensorflow/core/example/feature.pb.h"
// @@protoc_insertion_point(includes)
#define PROTOBUF_INTERNAL_EXPORT_protobuf_tensorflow_2fcore_2fexample_2fexample_2eproto
namespace protobuf_tensorflow_2fcore_2fexample_2fexample_2eproto {
// Internal implementation detail -- do not use these members.
struct TableStruct {
static const ::google::protobuf::internal::ParseTableField entries[];
static const ::google::protobuf::internal::AuxillaryParseTableField aux[];
static const ::google::protobuf::internal::ParseTable schema[2];
static const ::google::protobuf::internal::FieldMetadata field_metadata[];
static const ::google::protobuf::internal::SerializationTable serialization_table[];
static const ::google::protobuf::uint32 offsets[];
};
void AddDescriptors();
} // namespace protobuf_tensorflow_2fcore_2fexample_2fexample_2eproto
namespace tensorflow {
class Example;
class ExampleDefaultTypeInternal;
extern ExampleDefaultTypeInternal _Example_default_instance_;
class SequenceExample;
class SequenceExampleDefaultTypeInternal;
extern SequenceExampleDefaultTypeInternal _SequenceExample_default_instance_;
} // namespace tensorflow
namespace google {
namespace protobuf {
template<> ::tensorflow::Example* Arena::CreateMaybeMessage<::tensorflow::Example>(Arena*);
template<> ::tensorflow::SequenceExample* Arena::CreateMaybeMessage<::tensorflow::SequenceExample>(Arena*);
} // namespace protobuf
} // namespace google
namespace tensorflow {
// ===================================================================
class Example : public ::google::protobuf::Message /* @@protoc_insertion_point(class_definition:tensorflow.Example) */ {
public:
Example();
virtual ~Example();
Example(const Example& from);
inline Example& operator=(const Example& from) {
CopyFrom(from);
return *this;
}
#if LANG_CXX11
Example(Example&& from) noexcept
: Example() {
*this = ::std::move(from);
}
inline Example& operator=(Example&& from) noexcept {
if (GetArenaNoVirtual() == from.GetArenaNoVirtual()) {
if (this != &from) InternalSwap(&from);
} else {
CopyFrom(from);
}
return *this;
}
#endif
inline ::google::protobuf::Arena* GetArena() const final {
return GetArenaNoVirtual();
}
inline void* GetMaybeArenaPointer() const final {
return MaybeArenaPtr();
}
static const ::google::protobuf::Descriptor* descriptor();
static const Example& default_instance();
static void InitAsDefaultInstance(); // FOR INTERNAL USE ONLY
static inline const Example* internal_default_instance() {
return reinterpret_cast<const Example*>(
&_Example_default_instance_);
}
static constexpr int kIndexInFileMessages =
0;
void UnsafeArenaSwap(Example* other);
void Swap(Example* other);
friend void swap(Example& a, Example& b) {
a.Swap(&b);
}
// implements Message ----------------------------------------------
inline Example* New() const final {
return CreateMaybeMessage<Example>(NULL);
}
Example* New(::google::protobuf::Arena* arena) const final {
return CreateMaybeMessage<Example>(arena);
}
void CopyFrom(const ::google::protobuf::Message& from) final;
void MergeFrom(const ::google::protobuf::Message& from) final;
void CopyFrom(const Example& from);
void MergeFrom(const Example& from);
void Clear() final;
bool IsInitialized() const final;
size_t ByteSizeLong() const final;
bool MergePartialFromCodedStream(
::google::protobuf::io::CodedInputStream* input) final;
void SerializeWithCachedSizes(
::google::protobuf::io::CodedOutputStream* output) const final;
::google::protobuf::uint8* InternalSerializeWithCachedSizesToArray(
bool deterministic, ::google::protobuf::uint8* target) const final;
int GetCachedSize() const final { return _cached_size_.Get(); }
private:
void SharedCtor();
void SharedDtor();
void SetCachedSize(int size) const final;
void InternalSwap(Example* other);
protected:
explicit Example(::google::protobuf::Arena* arena);
private:
static void ArenaDtor(void* object);
inline void RegisterArenaDtor(::google::protobuf::Arena* arena);
private:
inline ::google::protobuf::Arena* GetArenaNoVirtual() const {
return _internal_metadata_.arena();
}
inline void* MaybeArenaPtr() const {
return _internal_metadata_.raw_arena_ptr();
}
public:
::google::protobuf::Metadata GetMetadata() const final;
// nested types ----------------------------------------------------
// accessors -------------------------------------------------------
// .tensorflow.Features features = 1;
bool has_features() const;
void clear_features();
static const int kFeaturesFieldNumber = 1;
private:
const ::tensorflow::Features& _internal_features() const;
public:
const ::tensorflow::Features& features() const;
::tensorflow::Features* release_features();
::tensorflow::Features* mutable_features();
void set_allocated_features(::tensorflow::Features* features);
void unsafe_arena_set_allocated_features(
::tensorflow::Features* features);
::tensorflow::Features* unsafe_arena_release_features();
// @@protoc_insertion_point(class_scope:tensorflow.Example)
private:
::google::protobuf::internal::InternalMetadataWithArena _internal_metadata_;
template <typename T> friend class ::google::protobuf::Arena::InternalHelper;
typedef void InternalArenaConstructable_;
typedef void DestructorSkippable_;
::tensorflow::Features* features_;
mutable ::google::protobuf::internal::CachedSize _cached_size_;
friend struct ::protobuf_tensorflow_2fcore_2fexample_2fexample_2eproto::TableStruct;
};
// -------------------------------------------------------------------
class SequenceExample : public ::google::protobuf::Message /* @@protoc_insertion_point(class_definition:tensorflow.SequenceExample) */ {
public:
SequenceExample();
virtual ~SequenceExample();
SequenceExample(const SequenceExample& from);
inline SequenceExample& operator=(const SequenceExample& from) {
CopyFrom(from);
return *this;
}
#if LANG_CXX11
SequenceExample(SequenceExample&& from) noexcept
: SequenceExample() {
*this = ::std::move(from);
}
inline SequenceExample& operator=(SequenceExample&& from) noexcept {
if (GetArenaNoVirtual() == from.GetArenaNoVirtual()) {
if (this != &from) InternalSwap(&from);
} else {
CopyFrom(from);
}
return *this;
}
#endif
inline ::google::protobuf::Arena* GetArena() const final {
return GetArenaNoVirtual();
}
inline void* GetMaybeArenaPointer() const final {
return MaybeArenaPtr();
}
static const ::google::protobuf::Descriptor* descriptor();
static const SequenceExample& default_instance();
static void InitAsDefaultInstance(); // FOR INTERNAL USE ONLY
static inline const SequenceExample* internal_default_instance() {
return reinterpret_cast<const SequenceExample*>(
&_SequenceExample_default_instance_);
}
static constexpr int kIndexInFileMessages =
1;
void UnsafeArenaSwap(SequenceExample* other);
void Swap(SequenceExample* other);
friend void swap(SequenceExample& a, SequenceExample& b) {
a.Swap(&b);
}
// implements Message ----------------------------------------------
inline SequenceExample* New() const final {
return CreateMaybeMessage<SequenceExample>(NULL);
}
SequenceExample* New(::google::protobuf::Arena* arena) const final {
return CreateMaybeMessage<SequenceExample>(arena);
}
void CopyFrom(const ::google::protobuf::Message& from) final;
void MergeFrom(const ::google::protobuf::Message& from) final;
void CopyFrom(const SequenceExample& from);
void MergeFrom(const SequenceExample& from);
void Clear() final;
bool IsInitialized() const final;
size_t ByteSizeLong() const final;
bool MergePartialFromCodedStream(
::google::protobuf::io::CodedInputStream* input) final;
void SerializeWithCachedSizes(
::google::protobuf::io::CodedOutputStream* output) const final;
::google::protobuf::uint8* InternalSerializeWithCachedSizesToArray(
bool deterministic, ::google::protobuf::uint8* target) const final;
int GetCachedSize() const final { return _cached_size_.Get(); }
private:
void SharedCtor();
void SharedDtor();
void SetCachedSize(int size) const final;
void InternalSwap(SequenceExample* other);
protected:
explicit SequenceExample(::google::protobuf::Arena* arena);
private:
static void ArenaDtor(void* object);
inline void RegisterArenaDtor(::google::protobuf::Arena* arena);
private:
inline ::google::protobuf::Arena* GetArenaNoVirtual() const {
return _internal_metadata_.arena();
}
inline void* MaybeArenaPtr() const {
return _internal_metadata_.raw_arena_ptr();
}
public:
::google::protobuf::Metadata GetMetadata() const final;
// nested types ----------------------------------------------------
// accessors -------------------------------------------------------
// .tensorflow.Features context = 1;
bool has_context() const;
void clear_context();
static const int kContextFieldNumber = 1;
private:
const ::tensorflow::Features& _internal_context() const;
public:
const ::tensorflow::Features& context() const;
::tensorflow::Features* release_context();
::tensorflow::Features* mutable_context();
void set_allocated_context(::tensorflow::Features* context);
void unsafe_arena_set_allocated_context(
::tensorflow::Features* context);
::tensorflow::Features* unsafe_arena_release_context();
// .tensorflow.FeatureLists feature_lists = 2;
bool has_feature_lists() const;
void clear_feature_lists();
static const int kFeatureListsFieldNumber = 2;
private:
const ::tensorflow::FeatureLists& _internal_feature_lists() const;
public:
const ::tensorflow::FeatureLists& feature_lists() const;
::tensorflow::FeatureLists* release_feature_lists();
::tensorflow::FeatureLists* mutable_feature_lists();
void set_allocated_feature_lists(::tensorflow::FeatureLists* feature_lists);
void unsafe_arena_set_allocated_feature_lists(
::tensorflow::FeatureLists* feature_lists);
::tensorflow::FeatureLists* unsafe_arena_release_feature_lists();
// @@protoc_insertion_point(class_scope:tensorflow.SequenceExample)
private:
::google::protobuf::internal::InternalMetadataWithArena _internal_metadata_;
template <typename T> friend class ::google::protobuf::Arena::InternalHelper;
typedef void InternalArenaConstructable_;
typedef void DestructorSkippable_;
::tensorflow::Features* context_;
::tensorflow::FeatureLists* feature_lists_;
mutable ::google::protobuf::internal::CachedSize _cached_size_;
friend struct ::protobuf_tensorflow_2fcore_2fexample_2fexample_2eproto::TableStruct;
};
// ===================================================================
// ===================================================================
#ifdef __GNUC__
#pragma GCC diagnostic push
#pragma GCC diagnostic ignored "-Wstrict-aliasing"
#endif // __GNUC__
// Example
// .tensorflow.Features features = 1;
inline bool Example::has_features() const {
return this != internal_default_instance() && features_ != NULL;
}
inline const ::tensorflow::Features& Example::_internal_features() const {
return *features_;
}
inline const ::tensorflow::Features& Example::features() const {
const ::tensorflow::Features* p = features_;
// @@protoc_insertion_point(field_get:tensorflow.Example.features)
return p != NULL ? *p : *reinterpret_cast<const ::tensorflow::Features*>(
&::tensorflow::_Features_default_instance_);
}
inline ::tensorflow::Features* Example::release_features() {
// @@protoc_insertion_point(field_release:tensorflow.Example.features)
::tensorflow::Features* temp = features_;
if (GetArenaNoVirtual() != NULL) {
temp = ::google::protobuf::internal::DuplicateIfNonNull(temp);
}
features_ = NULL;
return temp;
}
inline ::tensorflow::Features* Example::unsafe_arena_release_features() {
// @@protoc_insertion_point(field_unsafe_arena_release:tensorflow.Example.features)
::tensorflow::Features* temp = features_;
features_ = NULL;
return temp;
}
inline ::tensorflow::Features* Example::mutable_features() {
if (features_ == NULL) {
auto* p = CreateMaybeMessage<::tensorflow::Features>(GetArenaNoVirtual());
features_ = p;
}
// @@protoc_insertion_point(field_mutable:tensorflow.Example.features)
return features_;
}
inline void Example::set_allocated_features(::tensorflow::Features* features) {
::google::protobuf::Arena* message_arena = GetArenaNoVirtual();
if (message_arena == NULL) {
delete reinterpret_cast< ::google::protobuf::MessageLite*>(features_);
}
if (features) {
::google::protobuf::Arena* submessage_arena =
reinterpret_cast<::google::protobuf::MessageLite*>(features)->GetArena();
if (message_arena != submessage_arena) {
features = ::google::protobuf::internal::GetOwnedMessage(
message_arena, features, submessage_arena);
}
} else {
}
features_ = features;
// @@protoc_insertion_point(field_set_allocated:tensorflow.Example.features)
}
// -------------------------------------------------------------------
// SequenceExample
// .tensorflow.Features context = 1;
inline bool SequenceExample::has_context() const {
return this != internal_default_instance() && context_ != NULL;
}
inline const ::tensorflow::Features& SequenceExample::_internal_context() const {
return *context_;
}
inline const ::tensorflow::Features& SequenceExample::context() const {
const ::tensorflow::Features* p = context_;
// @@protoc_insertion_point(field_get:tensorflow.SequenceExample.context)
return p != NULL ? *p : *reinterpret_cast<const ::tensorflow::Features*>(
&::tensorflow::_Features_default_instance_);
}
inline ::tensorflow::Features* SequenceExample::release_context() {
// @@protoc_insertion_point(field_release:tensorflow.SequenceExample.context)
::tensorflow::Features* temp = context_;
if (GetArenaNoVirtual() != NULL) {
temp = ::google::protobuf::internal::DuplicateIfNonNull(temp);
}
context_ = NULL;
return temp;
}
inline ::tensorflow::Features* SequenceExample::unsafe_arena_release_context() {
// @@protoc_insertion_point(field_unsafe_arena_release:tensorflow.SequenceExample.context)
::tensorflow::Features* temp = context_;
context_ = NULL;
return temp;
}
inline ::tensorflow::Features* SequenceExample::mutable_context() {
if (context_ == NULL) {
auto* p = CreateMaybeMessage<::tensorflow::Features>(GetArenaNoVirtual());
context_ = p;
}
// @@protoc_insertion_point(field_mutable:tensorflow.SequenceExample.context)
return context_;
}
inline void SequenceExample::set_allocated_context(::tensorflow::Features* context) {
::google::protobuf::Arena* message_arena = GetArenaNoVirtual();
if (message_arena == NULL) {
delete reinterpret_cast< ::google::protobuf::MessageLite*>(context_);
}
if (context) {
::google::protobuf::Arena* submessage_arena =
reinterpret_cast<::google::protobuf::MessageLite*>(context)->GetArena();
if (message_arena != submessage_arena) {
context = ::google::protobuf::internal::GetOwnedMessage(
message_arena, context, submessage_arena);
}
} else {
}
context_ = context;
// @@protoc_insertion_point(field_set_allocated:tensorflow.SequenceExample.context)
}
// .tensorflow.FeatureLists feature_lists = 2;
inline bool SequenceExample::has_feature_lists() const {
return this != internal_default_instance() && feature_lists_ != NULL;
}
inline const ::tensorflow::FeatureLists& SequenceExample::_internal_feature_lists() const {
return *feature_lists_;
}
inline const ::tensorflow::FeatureLists& SequenceExample::feature_lists() const {
const ::tensorflow::FeatureLists* p = feature_lists_;
// @@protoc_insertion_point(field_get:tensorflow.SequenceExample.feature_lists)
return p != NULL ? *p : *reinterpret_cast<const ::tensorflow::FeatureLists*>(
&::tensorflow::_FeatureLists_default_instance_);
}
inline ::tensorflow::FeatureLists* SequenceExample::release_feature_lists() {
// @@protoc_insertion_point(field_release:tensorflow.SequenceExample.feature_lists)
::tensorflow::FeatureLists* temp = feature_lists_;
if (GetArenaNoVirtual() != NULL) {
temp = ::google::protobuf::internal::DuplicateIfNonNull(temp);
}
feature_lists_ = NULL;
return temp;
}
inline ::tensorflow::FeatureLists* SequenceExample::unsafe_arena_release_feature_lists() {
// @@protoc_insertion_point(field_unsafe_arena_release:tensorflow.SequenceExample.feature_lists)
::tensorflow::FeatureLists* temp = feature_lists_;
feature_lists_ = NULL;
return temp;
}
inline ::tensorflow::FeatureLists* SequenceExample::mutable_feature_lists() {
if (feature_lists_ == NULL) {
auto* p = CreateMaybeMessage<::tensorflow::FeatureLists>(GetArenaNoVirtual());
feature_lists_ = p;
}
// @@protoc_insertion_point(field_mutable:tensorflow.SequenceExample.feature_lists)
return feature_lists_;
}
inline void SequenceExample::set_allocated_feature_lists(::tensorflow::FeatureLists* feature_lists) {
::google::protobuf::Arena* message_arena = GetArenaNoVirtual();
if (message_arena == NULL) {
delete reinterpret_cast< ::google::protobuf::MessageLite*>(feature_lists_);
}
if (feature_lists) {
::google::protobuf::Arena* submessage_arena =
reinterpret_cast<::google::protobuf::MessageLite*>(feature_lists)->GetArena();
if (message_arena != submessage_arena) {
feature_lists = ::google::protobuf::internal::GetOwnedMessage(
message_arena, feature_lists, submessage_arena);
}
} else {
}
feature_lists_ = feature_lists;
// @@protoc_insertion_point(field_set_allocated:tensorflow.SequenceExample.feature_lists)
}
#ifdef __GNUC__
#pragma GCC diagnostic pop
#endif // __GNUC__
// -------------------------------------------------------------------
// @@protoc_insertion_point(namespace_scope)
} // namespace tensorflow
// @@protoc_insertion_point(global_scope)
#endif // PROTOBUF_INCLUDED_tensorflow_2fcore_2fexample_2fexample_2eproto
@@ -0,0 +1,34 @@
// GENERATED FILE - DO NOT MODIFY
#ifndef tensorflow_core_example_example_proto_IMPL_H_
#define tensorflow_core_example_example_proto_IMPL_H_
#include "tensorflow/core/example/example.pb.h"
#include "tensorflow/core/example/example.pb_text.h"
#include "tensorflow/core/example/feature.pb.h"
#include "tensorflow/core/example/feature.pb_text-impl.h"
#include "tensorflow/core/lib/strings/proto_text_util.h"
#include "tensorflow/core/lib/strings/scanner.h"
namespace tensorflow {
namespace internal {
void AppendProtoDebugString(
::tensorflow::strings::ProtoTextOutput* o,
const ::tensorflow::Example& msg);
bool ProtoParseFromScanner(
::tensorflow::strings::Scanner* scanner, bool nested, bool close_curly,
::tensorflow::Example* msg);
void AppendProtoDebugString(
::tensorflow::strings::ProtoTextOutput* o,
const ::tensorflow::SequenceExample& msg);
bool ProtoParseFromScanner(
::tensorflow::strings::Scanner* scanner, bool nested, bool close_curly,
::tensorflow::SequenceExample* msg);
} // namespace internal
} // namespace tensorflow
#endif // tensorflow_core_example_example_proto_IMPL_H_
@@ -0,0 +1,34 @@
// GENERATED FILE - DO NOT MODIFY
#ifndef tensorflow_core_example_example_proto_H_
#define tensorflow_core_example_example_proto_H_
#include "tensorflow/core/example/example.pb.h"
#include "tensorflow/core/platform/macros.h"
#include "tensorflow/core/platform/protobuf.h"
#include "tensorflow/core/platform/types.h"
namespace tensorflow {
// Message-text conversion for tensorflow.Example
string ProtoDebugString(
const ::tensorflow::Example& msg);
string ProtoShortDebugString(
const ::tensorflow::Example& msg);
bool ProtoParseFromString(
const string& s,
::tensorflow::Example* msg)
TF_MUST_USE_RESULT;
// Message-text conversion for tensorflow.SequenceExample
string ProtoDebugString(
const ::tensorflow::SequenceExample& msg);
string ProtoShortDebugString(
const ::tensorflow::SequenceExample& msg);
bool ProtoParseFromString(
const string& s,
::tensorflow::SequenceExample* msg)
TF_MUST_USE_RESULT;
} // namespace tensorflow
#endif // tensorflow_core_example_example_proto_H_
@@ -0,0 +1,56 @@
/* Copyright 2016 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
#ifndef TENSORFLOW_CORE_EXAMPLE_EXAMPLE_PARSER_CONFIGURATION_H_
#define TENSORFLOW_CORE_EXAMPLE_EXAMPLE_PARSER_CONFIGURATION_H_
#include <string>
#include <vector>
#include "tensorflow/core/example/example.pb.h"
#include "tensorflow/core/example/example_parser_configuration.pb.h"
#include "tensorflow/core/framework/allocator.h"
#include "tensorflow/core/framework/graph.pb.h"
#include "tensorflow/core/framework/tensor.h"
#include "tensorflow/core/framework/types.h"
#include "tensorflow/core/platform/types.h"
#include "tensorflow/core/public/session.h"
#include "tensorflow/core/util/example_proto_helper.h"
#include "tensorflow/core/util/sparse/sparse_tensor.h"
// This is a set of helper methods that will make it possible to share
// tensorflow::Example proto Tensor conversion code inside the ExampleParserOp
// OpKernel as well as in external code.
namespace tensorflow {
// Given a graph and the node_name of a ParseExample op,
// extract the FixedLenFeature/VarLenFeature configurations.
Status ExtractExampleParserConfiguration(
const tensorflow::GraphDef& graph, const string& node_name,
tensorflow::Session* session,
std::vector<FixedLenFeature>* fixed_len_features,
std::vector<VarLenFeature>* var_len_features);
// Given a config proto, ostensibly extracted via python,
// fill a vector of C++ structs suitable for calling
// the tensorflow.Example -> Tensor conversion code.
Status ExampleParserConfigurationProtoToFeatureVectors(
const ExampleParserConfiguration& config_proto,
std::vector<FixedLenFeature>* fixed_len_features,
std::vector<VarLenFeature>* var_len_features);
} // namespace tensorflow
#endif // TENSORFLOW_CORE_EXAMPLE_EXAMPLE_PARSER_CONFIGURATION_H_
File diff suppressed because it is too large. Load diff
File diff suppressed because it is too large. Load diff
@@ -0,0 +1,67 @@
// GENERATED FILE - DO NOT MODIFY
#ifndef tensorflow_core_example_feature_proto_IMPL_H_
#define tensorflow_core_example_feature_proto_IMPL_H_
#include "tensorflow/core/example/feature.pb.h"
#include "tensorflow/core/example/feature.pb_text.h"
#include "tensorflow/core/lib/strings/proto_text_util.h"
#include "tensorflow/core/lib/strings/scanner.h"
namespace tensorflow {
namespace internal {
void AppendProtoDebugString(
::tensorflow::strings::ProtoTextOutput* o,
const ::tensorflow::BytesList& msg);
bool ProtoParseFromScanner(
::tensorflow::strings::Scanner* scanner, bool nested, bool close_curly,
::tensorflow::BytesList* msg);
void AppendProtoDebugString(
::tensorflow::strings::ProtoTextOutput* o,
const ::tensorflow::FloatList& msg);
bool ProtoParseFromScanner(
::tensorflow::strings::Scanner* scanner, bool nested, bool close_curly,
::tensorflow::FloatList* msg);
void AppendProtoDebugString(
::tensorflow::strings::ProtoTextOutput* o,
const ::tensorflow::Int64List& msg);
bool ProtoParseFromScanner(
::tensorflow::strings::Scanner* scanner, bool nested, bool close_curly,
::tensorflow::Int64List* msg);
void AppendProtoDebugString(
::tensorflow::strings::ProtoTextOutput* o,
const ::tensorflow::Feature& msg);
bool ProtoParseFromScanner(
::tensorflow::strings::Scanner* scanner, bool nested, bool close_curly,
::tensorflow::Feature* msg);
void AppendProtoDebugString(
::tensorflow::strings::ProtoTextOutput* o,
const ::tensorflow::Features& msg);
bool ProtoParseFromScanner(
::tensorflow::strings::Scanner* scanner, bool nested, bool close_curly,
::tensorflow::Features* msg);
void AppendProtoDebugString(
::tensorflow::strings::ProtoTextOutput* o,
const ::tensorflow::FeatureList& msg);
bool ProtoParseFromScanner(
::tensorflow::strings::Scanner* scanner, bool nested, bool close_curly,
::tensorflow::FeatureList* msg);
void AppendProtoDebugString(
::tensorflow::strings::ProtoTextOutput* o,
const ::tensorflow::FeatureLists& msg);
bool ProtoParseFromScanner(
::tensorflow::strings::Scanner* scanner, bool nested, bool close_curly,
::tensorflow::FeatureLists* msg);
} // namespace internal
} // namespace tensorflow
#endif // tensorflow_core_example_feature_proto_IMPL_H_
@@ -0,0 +1,84 @@
// GENERATED FILE - DO NOT MODIFY
#ifndef tensorflow_core_example_feature_proto_H_
#define tensorflow_core_example_feature_proto_H_
#include "tensorflow/core/example/feature.pb.h"
#include "tensorflow/core/platform/macros.h"
#include "tensorflow/core/platform/protobuf.h"
#include "tensorflow/core/platform/types.h"
namespace tensorflow {
// Message-text conversion for tensorflow.BytesList
string ProtoDebugString(
const ::tensorflow::BytesList& msg);
string ProtoShortDebugString(
const ::tensorflow::BytesList& msg);
bool ProtoParseFromString(
const string& s,
::tensorflow::BytesList* msg)
TF_MUST_USE_RESULT;
// Message-text conversion for tensorflow.FloatList
string ProtoDebugString(
const ::tensorflow::FloatList& msg);
string ProtoShortDebugString(
const ::tensorflow::FloatList& msg);
bool ProtoParseFromString(
const string& s,
::tensorflow::FloatList* msg)
TF_MUST_USE_RESULT;
// Message-text conversion for tensorflow.Int64List
string ProtoDebugString(
const ::tensorflow::Int64List& msg);
string ProtoShortDebugString(
const ::tensorflow::Int64List& msg);
bool ProtoParseFromString(
const string& s,
::tensorflow::Int64List* msg)
TF_MUST_USE_RESULT;
// Message-text conversion for tensorflow.Feature
string ProtoDebugString(
const ::tensorflow::Feature& msg);
string ProtoShortDebugString(
const ::tensorflow::Feature& msg);
bool ProtoParseFromString(
const string& s,
::tensorflow::Feature* msg)
TF_MUST_USE_RESULT;
// Message-text conversion for tensorflow.Features
string ProtoDebugString(
const ::tensorflow::Features& msg);
string ProtoShortDebugString(
const ::tensorflow::Features& msg);
bool ProtoParseFromString(
const string& s,
::tensorflow::Features* msg)
TF_MUST_USE_RESULT;
// Message-text conversion for tensorflow.FeatureList
string ProtoDebugString(
const ::tensorflow::FeatureList& msg);
string ProtoShortDebugString(
const ::tensorflow::FeatureList& msg);
bool ProtoParseFromString(
const string& s,
::tensorflow::FeatureList* msg)
TF_MUST_USE_RESULT;
// Message-text conversion for tensorflow.FeatureLists
string ProtoDebugString(
const ::tensorflow::FeatureLists& msg);
string ProtoShortDebugString(
const ::tensorflow::FeatureLists& msg);
bool ProtoParseFromString(
const string& s,
::tensorflow::FeatureLists* msg)
TF_MUST_USE_RESULT;
} // namespace tensorflow
#endif // tensorflow_core_example_feature_proto_H_
@@ -0,0 +1,326 @@
/* Copyright 2016 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
// A set of lightweight wrappers which simplify access to Feature protos.
//
// TensorFlow Example proto uses associative maps on top of oneof fields.
// SequenceExample proto uses associative map of FeatureList.
// So accessing feature values is not very convenient.
//
// For example, to read a first value of integer feature "tag":
// int id = example.features().feature().at("tag").int64_list().value(0)
//
// to add a value:
// auto features = example->mutable_features();
// (*features->mutable_feature())["tag"].mutable_int64_list()->add_value(id)
//
// For float features you have to use float_list, for string - bytes_list.
//
// To do the same with this library:
// int id = GetFeatureValues<int64>("tag", example).Get(0);
// GetFeatureValues<int64>("tag", &example)->Add(id);
//
// Modification of bytes features is slightly different:
// auto tag = GetFeatureValues<string>("tag", &example);
// *tag->Add() = "lorem ipsum";
//
// To copy multiple values into a feature:
// AppendFeatureValues({1,2,3}, "tag", &example);
//
// GetFeatureValues gives you access to underlying data - RepeatedField object
// (RepeatedPtrField for byte list). So refer to its documentation of
// RepeatedField for full list of supported methods.
//
// NOTE: Due to the nature of oneof proto fields setting a feature of one type
// automatically clears all values stored as another type with the same feature
// key.
//
// This library also has tools to work with SequenceExample protos.
//
// To get a value from SequenceExample.context:
// int id = GetFeatureValues<protobuf_int64>("tag", se.context()).Get(0);
// To add a value to the context:
// GetFeatureValues<protobuf_int64>("tag", se.mutable_context())->Add(42);
//
// To add values to feature_lists:
// AppendFeatureValues({4.0},
// GetFeatureList("images", &se)->Add());
// AppendFeatureValues({5.0, 3.0},
// GetFeatureList("images", &se)->Add());
// This will create a feature list keyed as "images" with two features:
// feature_lists {
// feature_list {
// key: "images"
// value {
// feature { float_list { value: [4.0] } }
// feature { float_list { value: [5.0, 3.0] } }
// }
// } }
//
// Functions exposed by this library:
// HasFeature<[FeatureType]>(key, proto) -> bool
// Returns true if a feature with the specified key, and optionally
// FeatureType, belongs to the Features or Example proto.
// HasFeatureList(key, sequence_example) -> bool
// Returns true if SequenceExample has a feature_list with the key.
// GetFeatureValues<FeatureType>(key, proto) -> RepeatedField<FeatureType>
// Returns values for the specified key and the FeatureType.
// Supported types for the proto: Example, Features.
// GetFeatureList(key, sequence_example) -> RepeatedPtrField<Feature>
// Returns Feature protos associated with a key.
// AppendFeatureValues(begin, end, feature)
// AppendFeatureValues(container or initializer_list, feature)
// Copies values into a Feature.
// AppendFeatureValues(begin, end, key, proto)
// AppendFeatureValues(container or initializer_list, key, proto)
// Copies values into Features and Example protos with the specified key.
//
// Auxiliary functions, it is unlikely you'll need to use them directly:
// GetFeatures(proto) -> Features
// A convenience function to get Features proto.
// Supported types for the proto: Example, Features.
// GetFeature(key, proto) -> Feature*
// Returns a Feature proto for the specified key, creates a new if
// necessary. Supported types for the proto: Example, Features.
// GetFeatureValues<FeatureType>(feature) -> RepeatedField<FeatureType>
// Returns values of the feature for the FeatureType.
#ifndef TENSORFLOW_CORE_EXAMPLE_FEATURE_UTIL_H_
#define TENSORFLOW_CORE_EXAMPLE_FEATURE_UTIL_H_
#include <iterator>
#include <type_traits>
#include "absl/base/macros.h"
#include "tensorflow/core/example/example.pb.h"
#include "tensorflow/core/example/feature.pb.h"
#include "tensorflow/core/lib/core/stringpiece.h"
#include "tensorflow/core/platform/protobuf.h"
#include "tensorflow/core/platform/types.h"
namespace tensorflow {
namespace internal {
// TODO(gorban): Update all clients in a followup CL.
// Returns a reference to a feature corresponding to the name.
// Note: it will create a new Feature if it is missing in the example.
ABSL_DEPRECATED("Use GetFeature instead.")
Feature& ExampleFeature(const string& name, Example* example);
// Specializations of RepeatedFieldTrait define a type of RepeatedField
// corresponding to a selected feature type.
template <typename FeatureType>
struct RepeatedFieldTrait;
template <>
struct RepeatedFieldTrait<protobuf_int64> {
using Type = protobuf::RepeatedField<protobuf_int64>;
};
template <>
struct RepeatedFieldTrait<float> {
using Type = protobuf::RepeatedField<float>;
};
template <>
struct RepeatedFieldTrait<string> {
using Type = protobuf::RepeatedPtrField<string>;
};
// Specializations of FeatureTrait define a type of feature corresponding to a
// selected value type.
template <typename ValueType, class Enable = void>
struct FeatureTrait;
template <typename ValueType>
struct FeatureTrait<ValueType, typename std::enable_if<
std::is_integral<ValueType>::value>::type> {
using Type = protobuf_int64;
};
template <typename ValueType>
struct FeatureTrait<
ValueType,
typename std::enable_if<std::is_floating_point<ValueType>::value>::type> {
using Type = float;
};
template <typename T>
struct is_string
: public std::integral_constant<
bool,
std::is_same<char*, typename std::decay<T>::type>::value ||
std::is_same<const char*, typename std::decay<T>::type>::value> {
};
template <>
struct is_string<string> : std::true_type {};
template <>
struct is_string<::tensorflow::StringPiece> : std::true_type {};
template <typename ValueType>
struct FeatureTrait<
ValueType, typename std::enable_if<is_string<ValueType>::value>::type> {
using Type = string;
};
} // namespace internal
// Returns true if sequence_example has a feature_list with the specified key.
bool HasFeatureList(const string& key, const SequenceExample& sequence_example);
template <typename T>
struct TypeHasFeatures : std::false_type {};
template <>
struct TypeHasFeatures<Example> : std::true_type {};
template <>
struct TypeHasFeatures<Features> : std::true_type {};
// A family of template functions to return mutable Features proto from a
// container proto. Supported ProtoTypes: Example, Features.
template <typename ProtoType>
typename std::enable_if<TypeHasFeatures<ProtoType>::value, Features*>::type
GetFeatures(ProtoType* proto);
template <typename ProtoType>
typename std::enable_if<TypeHasFeatures<ProtoType>::value,
const Features&>::type
GetFeatures(const ProtoType& proto);
// Base declaration of a family of template functions to return a read only
// repeated field of feature values.
template <typename FeatureType>
const typename internal::RepeatedFieldTrait<FeatureType>::Type&
GetFeatureValues(const Feature& feature);
// Returns a read only repeated field corresponding to a feature with the
// specified name and FeatureType. Supported ProtoTypes: Example, Features.
template <typename FeatureType, typename ProtoType>
const typename internal::RepeatedFieldTrait<FeatureType>::Type&
GetFeatureValues(const string& key, const ProtoType& proto) {
return GetFeatureValues<FeatureType>(GetFeatures(proto).feature().at(key));
}
// Returns a mutable repeated field of a feature values.
template <typename FeatureType>
typename internal::RepeatedFieldTrait<FeatureType>::Type* GetFeatureValues(
Feature* feature);
// Returns a mutable repeated field corresponding to a feature with the
// specified name and FeatureType. Supported ProtoTypes: Example, Features.
template <typename FeatureType, typename ProtoType>
typename internal::RepeatedFieldTrait<FeatureType>::Type* GetFeatureValues(
const string& key, ProtoType* proto) {
::tensorflow::Feature& feature =
(*GetFeatures(proto)->mutable_feature())[key];
return GetFeatureValues<FeatureType>(&feature);
}
// Returns a Feature proto for the specified key, creates a new if necessary.
// Supported types for the proto: Example, Features.
template <typename ProtoType>
Feature* GetFeature(const string& key, ProtoType* proto) {
return &(*GetFeatures(proto)->mutable_feature())[key];
}
// Returns a repeated field with features corresponding to a feature_list key.
const protobuf::RepeatedPtrField<Feature>& GetFeatureList(
const string& key, const SequenceExample& sequence_example);
// Returns a mutable repeated field with features corresponding to a
// feature_list key. It will create a new FeatureList if necessary.
protobuf::RepeatedPtrField<Feature>* GetFeatureList(
const string& feature_list_key, SequenceExample* sequence_example);
template <typename IteratorType>
void AppendFeatureValues(IteratorType first, IteratorType last,
Feature* feature) {
using FeatureType = typename internal::FeatureTrait<
typename std::iterator_traits<IteratorType>::value_type>::Type;
std::copy(first, last,
protobuf::RepeatedFieldBackInserter(
GetFeatureValues<FeatureType>(feature)));
}
template <typename ValueType>
void AppendFeatureValues(std::initializer_list<ValueType> container,
Feature* feature) {
AppendFeatureValues(container.begin(), container.end(), feature);
}
template <typename ContainerType>
void AppendFeatureValues(const ContainerType& container, Feature* feature) {
using IteratorType = typename ContainerType::const_iterator;
AppendFeatureValues<IteratorType>(container.begin(), container.end(),
feature);
}
// Copies elements from the range, defined by [first, last) into the feature
// obtainable from the (proto, key) combination.
template <typename IteratorType, typename ProtoType>
void AppendFeatureValues(IteratorType first, IteratorType last,
const string& key, ProtoType* proto) {
AppendFeatureValues(first, last, GetFeature(key, GetFeatures(proto)));
}
// Copies all elements from the container into a feature.
template <typename ContainerType, typename ProtoType>
void AppendFeatureValues(const ContainerType& container, const string& key,
ProtoType* proto) {
using IteratorType = typename ContainerType::const_iterator;
AppendFeatureValues<IteratorType>(container.begin(), container.end(), key,
proto);
}
// Copies all elements from the initializer list into a Feature contained by
// Features or Example proto.
template <typename ValueType, typename ProtoType>
void AppendFeatureValues(std::initializer_list<ValueType> container,
const string& key, ProtoType* proto) {
using IteratorType =
typename std::initializer_list<ValueType>::const_iterator;
AppendFeatureValues<IteratorType>(container.begin(), container.end(), key,
proto);
}
// Returns true if a feature with the specified key belongs to the Features.
// The template parameter pack accepts zero or one template argument - which
// is FeatureType. If the FeatureType not specified (zero template arguments)
// the function will not check the feature type. Otherwise it will return false
// if the feature has a wrong type.
template <typename... FeatureType>
bool HasFeature(const string& key, const Features& features);
// Returns true if a feature with the specified key belongs to the Example.
// Doesn't check feature type if used without FeatureType, otherwise the
// specialized versions return false if the feature has a wrong type.
template <typename... FeatureType>
bool HasFeature(const string& key, const Example& example) {
return HasFeature<FeatureType...>(key, GetFeatures(example));
}
// TODO(gorban): update all clients in a followup CL.
template <typename... FeatureType>
ABSL_DEPRECATED("Use HasFeature instead.")
bool ExampleHasFeature(const string& key, const Example& example) {
return HasFeature<FeatureType...>(key, example);
}
} // namespace tensorflow
#endif // TENSORFLOW_CORE_EXAMPLE_FEATURE_UTIL_H_