Upgrade linux protobuf from 3.19.1 -> 21.12

This commit is contained in:
John Colvin committed 2023-03-18 15:31:41 -04:00
1 parent 4a3e40193e
commit 01a4bb004a
554 files changed
+34981 -37969

No files matched your search

@@ -42,7 +42,7 @@
// performance-minded Python code leverage the fast C++ implementation
// directly.
#include <google/protobuf/compiler/python/python_generator.h>
#include <google/protobuf/compiler/python/generator.h>
#include <algorithm>
#include <limits>
@@ -54,13 +54,15 @@
#include <google/protobuf/stubs/logging.h>
#include <google/protobuf/stubs/common.h>
#include <google/protobuf/stubs/strutil.h>
#include <google/protobuf/stubs/stringprintf.h>
#include <google/protobuf/stubs/substitute.h>
#include <google/protobuf/compiler/python/helpers.h>
#include <google/protobuf/compiler/python/pyi_generator.h>
#include <google/protobuf/descriptor.h>
#include <google/protobuf/descriptor.pb.h>
#include <google/protobuf/io/printer.h>
#include <google/protobuf/io/zero_copy_stream.h>
#include <google/protobuf/descriptor.h>
#include <google/protobuf/stubs/strutil.h>
#include <google/protobuf/stubs/substitute.h>
namespace google {
namespace protobuf {
@@ -68,16 +70,6 @@ namespace compiler {
namespace python {
namespace {
// Returns the Python module name expected for a given .proto filename.
std::string ModuleName(const std::string& filename) {
std::string basename = StripProto(filename);
ReplaceCharacters(&basename, "-", '_');
ReplaceCharacters(&basename, "/", '.');
return basename + "_pb2";
}
// Returns the alias we assign to the module of the given .proto filename
// when importing. See testPackageInitializationImport in
// net/proto2/python/internal/reflection_test.py
@@ -92,78 +84,13 @@ std::string ModuleAlias(const std::string& filename) {
return module_name;
}
// Keywords reserved by the Python language.
const char* const kKeywords[] = {
"False", "None", "True", "and", "as", "assert",
"async", "await", "break", "class", "continue", "def",
"del", "elif", "else", "except", "finally", "for",
"from", "global", "if", "import", "in", "is",
"lambda", "nonlocal", "not", "or", "pass", "raise",
"return", "try", "while", "with", "yield", "print",
};
const char* const* kKeywordsEnd =
kKeywords + (sizeof(kKeywords) / sizeof(kKeywords[0]));
bool ContainsPythonKeyword(const std::string& module_name) {
std::vector<std::string> tokens = Split(module_name, ".");
for (int i = 0; i < tokens.size(); ++i) {
if (std::find(kKeywords, kKeywordsEnd, tokens[i]) != kKeywordsEnd) {
return true;
}
}
return false;
}
inline bool IsPythonKeyword(const std::string& name) {
return (std::find(kKeywords, kKeywordsEnd, name) != kKeywordsEnd);
}
std::string ResolveKeyword(const std::string& name) {
if (IsPythonKeyword(name)) {
return "globals()['" + name + "']";
}
return name;
}
// Returns the name of all containing types for descriptor,
// in order from outermost to innermost, followed by descriptor's
// own name. Each name is separated by |separator|.
template <typename DescriptorT>
std::string NamePrefixedWithNestedTypes(const DescriptorT& descriptor,
const std::string& separator) {
std::string name = descriptor.name();
const Descriptor* parent = descriptor.containing_type();
if (parent != nullptr) {
std::string prefix = NamePrefixedWithNestedTypes(*parent, separator);
if (separator == "." && IsPythonKeyword(name)) {
return "getattr(" + prefix + ", '" + name + "')";
} else {
return prefix + separator + name;
}
}
if (separator == ".") {
name = ResolveKeyword(name);
}
return name;
}
// Name of the class attribute where we store the Python
// descriptor.Descriptor instance for the generated class.
// Must stay consistent with the _DESCRIPTOR_KEY constant
// in proto2/public/reflection.py.
const char kDescriptorKey[] = "DESCRIPTOR";
// Does the file have top-level enums?
inline bool HasTopLevelEnums(const FileDescriptor* file) {
return file->enum_type_count() > 0;
}
// Should we generate generic services for this file?
inline bool HasGenericServices(const FileDescriptor* file) {
return file->service_count() > 0 && file->options().py_generic_services();
}
// Prints the common boilerplate needed at the top of every .py
// file output by this generator.
void PrintTopBoilerplate(io::Printer* printer, const FileDescriptor* file,
bool descriptor_proto) {
@@ -174,27 +101,16 @@ void PrintTopBoilerplate(io::Printer* printer, const FileDescriptor* file,
"# source: $filename$\n"
"\"\"\"Generated protocol buffer code.\"\"\"\n",
"filename", file->name());
if (HasTopLevelEnums(file)) {
printer->Print(
"from google.protobuf.internal import enum_type_wrapper\n");
}
printer->Print(
"from google.protobuf.internal import builder as _builder\n"
"from google.protobuf import descriptor as _descriptor\n"
"from google.protobuf import descriptor_pool as "
"_descriptor_pool\n"
"from google.protobuf import message as _message\n"
"from google.protobuf import reflection as _reflection\n"
"from google.protobuf import symbol_database as "
"_symbol_database\n");
if (HasGenericServices(file)) {
printer->Print(
"from google.protobuf import service as _service\n"
"from google.protobuf import service_reflection\n");
}
printer->Print(
"# @@protoc_insertion_point(imports)\n\n"
"_sym_db = _symbol_database.Default()\n");
printer->Print("# @@protoc_insertion_point(imports)\n\n");
printer->Print("_sym_db = _symbol_database.Default()\n");
printer->Print("\n\n");
}
@@ -309,6 +225,11 @@ bool Generator::Generate(const FileDescriptor* file,
for (int i = 0; i < options.size(); i++) {
if (options[i].first == "cpp_generated_lib_linked") {
cpp_generated_lib_linked = true;
} else if (options[i].first == "pyi_out") {
python::PyiGenerator pyi_generator;
if (!pyi_generator.Generate(file, "", context, error)) {
return false;
}
} else {
*error = "Unknown generator option: " + options[i].first;
return false;
@@ -324,11 +245,8 @@ bool Generator::Generate(const FileDescriptor* file,
// to have any mutable members. Then it is implicitly thread-safe.
MutexLock lock(&mutex_);
file_ = file;
std::string module_name = ModuleName(file->name());
std::string filename = module_name;
ReplaceCharacters(&filename, ".", '/');
filename += ".py";
std::string filename = GetFileName(file, ".py");
pure_python_workable_ = !cpp_generated_lib_linked;
if (HasPrefixString(file->name(), "google/protobuf/")) {
pure_python_workable_ = true;
@@ -349,15 +267,13 @@ bool Generator::Generate(const FileDescriptor* file,
PrintImports();
}
PrintFileDescriptor();
PrintTopLevelEnums();
PrintTopLevelExtensions();
if (pure_python_workable_) {
if (GeneratingDescriptorProto()) {
printer_->Print("if _descriptor._USE_C_DESCRIPTORS == False:\n");
printer_->Indent();
// Create enums before message descriptors
PrintAllNestedEnumsInFile(StripPrintDescriptor::kCreate);
PrintMessageDescriptors(StripPrintDescriptor::kCreate);
PrintAllNestedEnumsInFile();
PrintMessageDescriptors();
FixForeignFieldsInDescriptors();
printer_->Outdent();
printer_->Print("else:\n");
@@ -365,16 +281,18 @@ bool Generator::Generate(const FileDescriptor* file,
}
// Find the message descriptors first and then use the message
// descriptor to find enums.
PrintMessageDescriptors(StripPrintDescriptor::kFind);
PrintAllNestedEnumsInFile(StripPrintDescriptor::kFind);
printer_->Print(
"_builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, globals())\n");
if (GeneratingDescriptorProto()) {
printer_->Outdent();
}
}
PrintMessages();
std::string module_name = ModuleName(file->name());
printer_->Print(
"_builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, '$module_name$', "
"globals())\n",
"module_name", module_name);
if (pure_python_workable_) {
PrintServiceDescriptors();
printer.Print("if _descriptor._USE_C_DESCRIPTORS == False:\n");
printer_->Indent();
@@ -395,7 +313,9 @@ bool Generator::Generate(const FileDescriptor* file,
printer_->Outdent();
}
if (HasGenericServices(file)) {
PrintServices();
printer_->Print(
"_builder.BuildServices(DESCRIPTOR, '$module_name$', globals())\n",
"module_name", module_name);
}
printer.Print("# @@protoc_insertion_point(module_scope)\n");
@@ -403,7 +323,6 @@ bool Generator::Generate(const FileDescriptor* file,
return !printer.failed();
}
// Prints Python imports for all modules imported by |file|.
void Generator::PrintImports() const {
for (int i = 0; i < file_->dependency_count(); ++i) {
@@ -516,47 +435,17 @@ void Generator::PrintFileDescriptor() const {
printer_->Print("\n");
}
// Prints descriptors and module-level constants for all top-level
// enums defined in |file|.
void Generator::PrintTopLevelEnums() const {
std::vector<std::pair<std::string, int> > top_level_enum_values;
for (int i = 0; i < file_->enum_type_count(); ++i) {
const EnumDescriptor& enum_descriptor = *file_->enum_type(i);
PrintFindEnum(enum_descriptor);
printer_->Print(
"$name$ = "
"enum_type_wrapper.EnumTypeWrapper($descriptor_name$)",
"name", ResolveKeyword(enum_descriptor.name()), "descriptor_name",
ModuleLevelDescriptorName(enum_descriptor));
printer_->Print("\n");
for (int j = 0; j < enum_descriptor.value_count(); ++j) {
const EnumValueDescriptor& value_descriptor = *enum_descriptor.value(j);
top_level_enum_values.push_back(
std::make_pair(value_descriptor.name(), value_descriptor.number()));
}
}
for (int i = 0; i < top_level_enum_values.size(); ++i) {
printer_->Print("$name$ = $value$\n", "name",
ResolveKeyword(top_level_enum_values[i].first), "value",
StrCat(top_level_enum_values[i].second));
}
printer_->Print("\n");
}
// Prints all enums contained in all message types in |file|.
void Generator::PrintAllNestedEnumsInFile(
StripPrintDescriptor print_mode) const {
void Generator::PrintAllNestedEnumsInFile() const {
for (int i = 0; i < file_->message_type_count(); ++i) {
PrintNestedEnums(*file_->message_type(i), print_mode);
PrintNestedEnums(*file_->message_type(i));
}
}
// Prints a Python statement assigning the appropriate module-level
// enum name to a Python EnumDescriptor object equivalent to
// enum_descriptor.
void Generator::PrintCreateEnum(const EnumDescriptor& enum_descriptor) const {
void Generator::PrintEnum(const EnumDescriptor& enum_descriptor) const {
std::map<std::string, std::string> m;
std::string module_level_descriptor_name =
ModuleLevelDescriptorName(enum_descriptor);
@@ -600,68 +489,23 @@ void Generator::PrintCreateEnum(const EnumDescriptor& enum_descriptor) const {
printer_->Print("\n");
}
void Generator::PrintFindEnum(const EnumDescriptor& enum_descriptor) const {
std::map<std::string, std::string> m;
m["descriptor_name"] = ModuleLevelDescriptorName(enum_descriptor);
m["name"] = enum_descriptor.name();
m["file"] = kDescriptorKey;
if (enum_descriptor.containing_type()) {
m["containing_type"] =
ModuleLevelDescriptorName(*enum_descriptor.containing_type());
printer_->Print(m,
"$descriptor_name$ = "
"$containing_type$.enum_types_by_name['$name$']\n");
} else {
printer_->Print(
m, "$descriptor_name$ = $file$.enum_types_by_name['$name$']\n");
}
}
// Recursively prints enums in nested types within descriptor, then
// prints enums contained at the top level in descriptor.
void Generator::PrintNestedEnums(const Descriptor& descriptor,
StripPrintDescriptor print_mode) const {
void Generator::PrintNestedEnums(const Descriptor& descriptor) const {
for (int i = 0; i < descriptor.nested_type_count(); ++i) {
PrintNestedEnums(*descriptor.nested_type(i), print_mode);
PrintNestedEnums(*descriptor.nested_type(i));
}
for (int i = 0; i < descriptor.enum_type_count(); ++i) {
if (print_mode == StripPrintDescriptor::kCreate) {
PrintCreateEnum(*descriptor.enum_type(i));
} else {
PrintFindEnum(*descriptor.enum_type(i));
}
PrintEnum(*descriptor.enum_type(i));
}
}
void Generator::PrintTopLevelExtensions() const {
for (int i = 0; i < file_->extension_count(); ++i) {
const FieldDescriptor& extension_field = *file_->extension(i);
std::string constant_name = extension_field.name() + "_FIELD_NUMBER";
ToUpper(&constant_name);
printer_->Print("$constant_name$ = $number$\n", "constant_name",
constant_name, "number",
StrCat(extension_field.number()));
printer_->Print(
"$resolved_name$ = "
"$file$.extensions_by_name['$name$']\n",
"resolved_name", ResolveKeyword(extension_field.name()), "file",
kDescriptorKey, "name", extension_field.name());
}
printer_->Print("\n");
}
// Prints Python equivalents of all Descriptors in |file|.
void Generator::PrintMessageDescriptors(StripPrintDescriptor print_mode) const {
if (print_mode == StripPrintDescriptor::kCreate) {
for (int i = 0; i < file_->message_type_count(); ++i) {
PrintCreateDescriptor(*file_->message_type(i));
printer_->Print("\n");
}
} else {
for (int i = 0; i < file_->message_type_count(); ++i) {
PrintFindDescriptor(*file_->message_type(i));
}
void Generator::PrintMessageDescriptors() const {
for (int i = 0; i < file_->message_type_count(); ++i) {
PrintDescriptor(*file_->message_type(i));
printer_->Print("\n");
}
}
@@ -730,14 +574,13 @@ void Generator::PrintServiceStub(const ServiceDescriptor& descriptor) const {
// to a Python Descriptor object for message_descriptor.
//
// Mutually recursive with PrintNestedDescriptors().
void Generator::PrintCreateDescriptor(
const Descriptor& message_descriptor) const {
void Generator::PrintDescriptor(const Descriptor& message_descriptor) const {
std::map<std::string, std::string> m;
m["name"] = message_descriptor.name();
m["full_name"] = message_descriptor.full_name();
m["file"] = kDescriptorKey;
PrintNestedDescriptors(message_descriptor, StripPrintDescriptor::kCreate);
PrintNestedDescriptors(message_descriptor);
printer_->Print("\n");
printer_->Print("$descriptor_name$ = _descriptor.Descriptor(\n",
@@ -823,41 +666,14 @@ void Generator::PrintCreateDescriptor(
printer_->Print(")\n");
}
void Generator::PrintFindDescriptor(
const Descriptor& message_descriptor) const {
std::map<std::string, std::string> m;
m["descriptor_name"] = ModuleLevelDescriptorName(message_descriptor);
m["name"] = message_descriptor.name();
if (message_descriptor.containing_type()) {
m["containing_type"] =
ModuleLevelDescriptorName(*message_descriptor.containing_type());
printer_->Print(m,
"$descriptor_name$ = "
"$containing_type$.nested_types_by_name['$name$']\n");
} else {
m["file"] = kDescriptorKey;
printer_->Print(
m, "$descriptor_name$ = $file$.message_types_by_name['$name$']\n");
}
PrintNestedDescriptors(message_descriptor, StripPrintDescriptor::kFind);
}
// Prints Python Descriptor objects for all nested types contained in
// message_descriptor.
//
// Mutually recursive with PrintDescriptor().
void Generator::PrintNestedDescriptors(const Descriptor& containing_descriptor,
StripPrintDescriptor print_mode) const {
if (print_mode == StripPrintDescriptor::kCreate) {
for (int i = 0; i < containing_descriptor.nested_type_count(); ++i) {
PrintCreateDescriptor(*containing_descriptor.nested_type(i));
}
} else {
for (int i = 0; i < containing_descriptor.nested_type_count(); ++i) {
PrintFindDescriptor(*containing_descriptor.nested_type(i));
}
void Generator::PrintNestedDescriptors(
const Descriptor& containing_descriptor) const {
for (int i = 0; i < containing_descriptor.nested_type_count(); ++i) {
PrintDescriptor(*containing_descriptor.nested_type(i));
}
}
@@ -1463,7 +1279,7 @@ void Generator::FixOptionsForEnum(const EnumDescriptor& enum_descriptor) const {
if (value_options != "None") {
PrintDescriptorOptionsFixingCode(
StringPrintf("%s.values_by_name[\"%s\"]", descriptor_name.c_str(),
value_descriptor.name().c_str()),
value_descriptor.name().c_str()),
value_options, printer_);
}
}
@@ -0,0 +1,185 @@
// Protocol Buffers - Google's data interchange format
// Copyright 2008 Google Inc. All rights reserved.
// https://developers.google.com/protocol-buffers/
//
// Redistribution and use in source and binary forms, with or without
// modification, are permitted provided that the following conditions are
// met:
//
// * Redistributions of source code must retain the above copyright
// notice, this list of conditions and the following disclaimer.
// * Redistributions in binary form must reproduce the above
// copyright notice, this list of conditions and the following disclaimer
// in the documentation and/or other materials provided with the
// distribution.
// * Neither the name of Google Inc. nor the names of its
// contributors may be used to endorse or promote products derived from
// this software without specific prior written permission.
//
// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
// Author: robinson@google.com (Will Robinson)
//
// Generates Python code for a given .proto file.
#ifndef GOOGLE_PROTOBUF_COMPILER_PYTHON_GENERATOR_H__
#define GOOGLE_PROTOBUF_COMPILER_PYTHON_GENERATOR_H__
#include <string>
#include <google/protobuf/stubs/mutex.h>
#include <google/protobuf/compiler/code_generator.h>
// Must be included last.
#include <google/protobuf/port_def.inc>
namespace google {
namespace protobuf {
class Descriptor;
class EnumDescriptor;
class EnumValueDescriptor;
class FieldDescriptor;
class OneofDescriptor;
class ServiceDescriptor;
namespace io {
class Printer;
}
namespace compiler {
namespace python {
// CodeGenerator implementation for generated Python protocol buffer classes.
// If you create your own protocol compiler binary and you want it to support
// Python output, you can do so by registering an instance of this
// CodeGenerator with the CommandLineInterface in your main() function.
class PROTOC_EXPORT Generator : public CodeGenerator {
public:
Generator();
~Generator() override;
// CodeGenerator methods.
bool Generate(const FileDescriptor* file, const std::string& parameter,
GeneratorContext* generator_context,
std::string* error) const override;
uint64_t GetSupportedFeatures() const override;
private:
void PrintImports() const;
void PrintFileDescriptor() const;
void PrintAllNestedEnumsInFile() const;
void PrintNestedEnums(const Descriptor& descriptor) const;
void PrintEnum(const EnumDescriptor& enum_descriptor) const;
void PrintFieldDescriptor(const FieldDescriptor& field,
bool is_extension) const;
void PrintFieldDescriptorsInDescriptor(
const Descriptor& message_descriptor, bool is_extension,
const std::string& list_variable_name, int (Descriptor::*CountFn)() const,
const FieldDescriptor* (Descriptor::*GetterFn)(int)const) const;
void PrintFieldsInDescriptor(const Descriptor& message_descriptor) const;
void PrintExtensionsInDescriptor(const Descriptor& message_descriptor) const;
void PrintMessageDescriptors() const;
void PrintDescriptor(const Descriptor& message_descriptor) const;
void PrintNestedDescriptors(const Descriptor& containing_descriptor) const;
void PrintMessages() const;
void PrintMessage(const Descriptor& message_descriptor,
const std::string& prefix,
std::vector<std::string>* to_register,
bool is_nested) const;
void PrintNestedMessages(const Descriptor& containing_descriptor,
const std::string& prefix,
std::vector<std::string>* to_register) const;
void FixForeignFieldsInDescriptors() const;
void FixForeignFieldsInDescriptor(
const Descriptor& descriptor,
const Descriptor* containing_descriptor) const;
void FixForeignFieldsInField(const Descriptor* containing_type,
const FieldDescriptor& field,
const std::string& python_dict_name) const;
void AddMessageToFileDescriptor(const Descriptor& descriptor) const;
void AddEnumToFileDescriptor(const EnumDescriptor& descriptor) const;
void AddExtensionToFileDescriptor(const FieldDescriptor& descriptor) const;
void AddServiceToFileDescriptor(const ServiceDescriptor& descriptor) const;
std::string FieldReferencingExpression(
const Descriptor* containing_type, const FieldDescriptor& field,
const std::string& python_dict_name) const;
template <typename DescriptorT>
void FixContainingTypeInDescriptor(
const DescriptorT& descriptor,
const Descriptor* containing_descriptor) const;
void FixForeignFieldsInExtensions() const;
void FixForeignFieldsInExtension(
const FieldDescriptor& extension_field) const;
void FixForeignFieldsInNestedExtensions(const Descriptor& descriptor) const;
void PrintServices() const;
void PrintServiceDescriptors() const;
void PrintServiceDescriptor(const ServiceDescriptor& descriptor) const;
void PrintServiceClass(const ServiceDescriptor& descriptor) const;
void PrintServiceStub(const ServiceDescriptor& descriptor) const;
void PrintDescriptorKeyAndModuleName(
const ServiceDescriptor& descriptor) const;
void PrintEnumValueDescriptor(const EnumValueDescriptor& descriptor) const;
std::string OptionsValue(const std::string& serialized_options) const;
bool GeneratingDescriptorProto() const;
template <typename DescriptorT>
std::string ModuleLevelDescriptorName(const DescriptorT& descriptor) const;
std::string ModuleLevelMessageName(const Descriptor& descriptor) const;
std::string ModuleLevelServiceDescriptorName(
const ServiceDescriptor& descriptor) const;
template <typename DescriptorT, typename DescriptorProtoT>
void PrintSerializedPbInterval(const DescriptorT& descriptor,
DescriptorProtoT& proto,
const std::string& name) const;
void FixAllDescriptorOptions() const;
void FixOptionsForField(const FieldDescriptor& field) const;
void FixOptionsForOneof(const OneofDescriptor& oneof) const;
void FixOptionsForEnum(const EnumDescriptor& descriptor) const;
void FixOptionsForService(const ServiceDescriptor& descriptor) const;
void FixOptionsForMessage(const Descriptor& descriptor) const;
void SetSerializedPbInterval() const;
void SetMessagePbInterval(const Descriptor& descriptor) const;
void CopyPublicDependenciesAliases(const std::string& copy_from,
const FileDescriptor* file) const;
// Very coarse-grained lock to ensure that Generate() is reentrant.
// Guards file_, printer_ and file_descriptor_serialized_.
mutable Mutex mutex_;
mutable const FileDescriptor* file_; // Set in Generate(). Under mutex_.
mutable std::string file_descriptor_serialized_;
mutable io::Printer* printer_; // Set in Generate(). Under mutex_.
mutable bool pure_python_workable_;
GOOGLE_DISALLOW_EVIL_CONSTRUCTORS(Generator);
};
} // namespace python
} // namespace compiler
} // namespace protobuf
} // namespace google
#include <google/protobuf/port_undef.inc>
#endif // GOOGLE_PROTOBUF_COMPILER_PYTHON_GENERATOR_H__
@@ -0,0 +1,131 @@
// Protocol Buffers - Google's data interchange format
// Copyright 2008 Google Inc. All rights reserved.
// https://developers.google.com/protocol-buffers/
//
// Redistribution and use in source and binary forms, with or without
// modification, are permitted provided that the following conditions are
// met:
//
// * Redistributions of source code must retain the above copyright
// notice, this list of conditions and the following disclaimer.
// * Redistributions in binary form must reproduce the above
// copyright notice, this list of conditions and the following disclaimer
// in the documentation and/or other materials provided with the
// distribution.
// * Neither the name of Google Inc. nor the names of its
// contributors may be used to endorse or promote products derived from
// this software without specific prior written permission.
//
// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#include <google/protobuf/compiler/python/helpers.h>
#include <algorithm>
#include <google/protobuf/stubs/strutil.h>
#include <google/protobuf/compiler/code_generator.h>
#include <google/protobuf/descriptor.h>
#include <google/protobuf/descriptor.pb.h>
namespace google {
namespace protobuf {
namespace compiler {
namespace python {
// Returns the Python module name expected for a given .proto filename.
std::string ModuleName(const std::string& filename) {
std::string basename = StripProto(filename);
ReplaceCharacters(&basename, "-", '_');
ReplaceCharacters(&basename, "/", '.');
return basename + "_pb2";
}
std::string StrippedModuleName(const std::string& filename) {
std::string module_name = ModuleName(filename);
return module_name;
}
// Keywords reserved by the Python language.
const char* const kKeywords[] = {
"False", "None", "True", "and", "as", "assert",
"async", "await", "break", "class", "continue", "def",
"del", "elif", "else", "except", "finally", "for",
"from", "global", "if", "import", "in", "is",
"lambda", "nonlocal", "not", "or", "pass", "raise",
"return", "try", "while", "with", "yield",
};
const char* const* kKeywordsEnd =
kKeywords + (sizeof(kKeywords) / sizeof(kKeywords[0]));
bool ContainsPythonKeyword(const std::string& module_name) {
std::vector<std::string> tokens = Split(module_name, ".");
for (int i = 0; i < static_cast<int>(tokens.size()); ++i) {
if (std::find(kKeywords, kKeywordsEnd, tokens[i]) != kKeywordsEnd) {
return true;
}
}
return false;
}
bool IsPythonKeyword(const std::string& name) {
return (std::find(kKeywords, kKeywordsEnd, name) != kKeywordsEnd);
}
std::string ResolveKeyword(const std::string& name) {
if (IsPythonKeyword(name)) {
return "globals()['" + name + "']";
}
return name;
}
std::string GetFileName(const FileDescriptor* file_des,
const std::string& suffix) {
std::string module_name = ModuleName(file_des->name());
std::string filename = module_name;
ReplaceCharacters(&filename, ".", '/');
filename += suffix;
return filename;
}
bool HasGenericServices(const FileDescriptor* file) {
return file->service_count() > 0 && file->options().py_generic_services();
}
template <typename DescriptorT>
std::string NamePrefixedWithNestedTypes(const DescriptorT& descriptor,
const std::string& separator) {
std::string name = descriptor.name();
const Descriptor* parent = descriptor.containing_type();
if (parent != nullptr) {
std::string prefix = NamePrefixedWithNestedTypes(*parent, separator);
if (separator == "." && IsPythonKeyword(name)) {
return "getattr(" + prefix + ", '" + name + "')";
} else {
return prefix + separator + name;
}
}
if (separator == ".") {
name = ResolveKeyword(name);
}
return name;
}
template std::string NamePrefixedWithNestedTypes<Descriptor>(
const Descriptor& descriptor, const std::string& separator);
template std::string NamePrefixedWithNestedTypes<EnumDescriptor>(
const EnumDescriptor& descriptor, const std::string& separator);
} // namespace python
} // namespace compiler
} // namespace protobuf
} // namespace google
@@ -0,0 +1,62 @@
// Protocol Buffers - Google's data interchange format
// Copyright 2008 Google Inc. All rights reserved.
// https://developers.google.com/protocol-buffers/
//
// Redistribution and use in source and binary forms, with or without
// modification, are permitted provided that the following conditions are
// met:
//
// * Redistributions of source code must retain the above copyright
// notice, this list of conditions and the following disclaimer.
// * Redistributions in binary form must reproduce the above
// copyright notice, this list of conditions and the following disclaimer
// in the documentation and/or other materials provided with the
// distribution.
// * Neither the name of Google Inc. nor the names of its
// contributors may be used to endorse or promote products derived from
// this software without specific prior written permission.
//
// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#ifndef GOOGLE_PROTOBUF_COMPILER_PYTHON_HELPERS_H__
#define GOOGLE_PROTOBUF_COMPILER_PYTHON_HELPERS_H__
#include <string>
#include <google/protobuf/descriptor.h>
namespace google {
namespace protobuf {
namespace compiler {
namespace python {
std::string ModuleName(const std::string& filename);
std::string StrippedModuleName(const std::string& filename);
bool ContainsPythonKeyword(const std::string& module_name);
bool IsPythonKeyword(const std::string& name);
std::string ResolveKeyword(const std::string& name);
std::string GetFileName(const FileDescriptor* file_des,
const std::string& suffix);
bool HasGenericServices(const FileDescriptor* file);
template <typename DescriptorT>
std::string NamePrefixedWithNestedTypes(const DescriptorT& descriptor,
const std::string& separator);
} // namespace python
} // namespace compiler
} // namespace protobuf
} // namespace google
#endif // GOOGLE_PROTOBUF_COMPILER_PYTHON_HELPERS_H__
@@ -29,17 +29,14 @@
// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
// Author: kenton@google.com (Kenton Varda)
//
// TODO(kenton): Share code with the versions of this test in other languages?
// It seemed like parameterizing it would add more complexity than it is
// worth.
#include <memory>
#include <string>
#include <google/protobuf/testing/file.h>
#include <google/protobuf/testing/file.h>
#include <google/protobuf/compiler/command_line_interface.h>
#include <google/protobuf/compiler/python/python_generator.h>
#include <google/protobuf/compiler/python/generator.h>
#include <google/protobuf/io/printer.h>
#include <google/protobuf/io/zero_copy_stream.h>
#include <google/protobuf/stubs/strutil.h>
@@ -55,11 +52,10 @@ namespace {
class TestGenerator : public CodeGenerator {
public:
TestGenerator() {}
~TestGenerator() {}
~TestGenerator() override {}
virtual bool Generate(const FileDescriptor* file,
const std::string& parameter, GeneratorContext* context,
std::string* error) const {
bool Generate(const FileDescriptor* file, const std::string& parameter,
GeneratorContext* context, std::string* error) const override {
TryInsert("test_pb2.py", "imports", context);
TryInsert("test_pb2.py", "module_scope", context);
TryInsert("test_pb2.py", "class_scope:foo.Bar", context);
@@ -77,37 +73,6 @@ class TestGenerator : public CodeGenerator {
}
};
// This test verifies that all the expected insertion points exist. It does
// not verify that they are correctly-placed; that would require actually
// compiling the output which is a bit more than I care to do for this test.
TEST(PythonPluginTest, PluginTest) {
GOOGLE_CHECK_OK(File::SetContents(TestTempDir() + "/test.proto",
"syntax = \"proto2\";\n"
"package foo;\n"
"message Bar {\n"
" message Baz {}\n"
"}\n",
true));
compiler::CommandLineInterface cli;
cli.SetInputsAreProtoPathRelative(true);
python::Generator python_generator;
TestGenerator test_generator;
cli.RegisterGenerator("--python_out", &python_generator, "");
cli.RegisterGenerator("--test_out", &test_generator, "");
std::string proto_path = "-I" + TestTempDir();
std::string python_out = "--python_out=" + TestTempDir();
std::string test_out = "--test_out=" + TestTempDir();
const char* argv[] = {"protoc", proto_path.c_str(), python_out.c_str(),
test_out.c_str(), "test.proto"};
EXPECT_EQ(0, cli.Run(5, argv));
}
// This test verifies that the generated Python output uses regular imports (as
// opposed to importlib) in the usual case where the .proto file paths do not
// not contain any Python keywords.
TEST(PythonPluginTest, ImportTest) {
@@ -0,0 +1,636 @@
// Protocol Buffers - Google's data interchange format
// Copyright 2008 Google Inc. All rights reserved.
// https://developers.google.com/protocol-buffers/
//
// Redistribution and use in source and binary forms, with or without
// modification, are permitted provided that the following conditions are
// met:
//
// * Redistributions of source code must retain the above copyright
// notice, this list of conditions and the following disclaimer.
// * Redistributions in binary form must reproduce the above
// copyright notice, this list of conditions and the following disclaimer
// in the documentation and/or other materials provided with the
// distribution.
// * Neither the name of Google Inc. nor the names of its
// contributors may be used to endorse or promote products derived from
// this software without specific prior written permission.
//
// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#include <google/protobuf/compiler/python/pyi_generator.h>
#include <string>
#include <google/protobuf/stubs/strutil.h>
#include <google/protobuf/compiler/python/helpers.h>
#include <google/protobuf/descriptor.h>
#include <google/protobuf/descriptor.pb.h>
#include <google/protobuf/io/printer.h>
#include <google/protobuf/io/zero_copy_stream.h>
namespace google {
namespace protobuf {
namespace compiler {
namespace python {
template <typename DescriptorT>
struct SortByName {
bool operator()(const DescriptorT* l, const DescriptorT* r) const {
return l->name() < r->name();
}
};
PyiGenerator::PyiGenerator() : file_(nullptr) {}
PyiGenerator::~PyiGenerator() {}
void PyiGenerator::PrintItemMap(
const std::map<std::string, std::string>& item_map) const {
for (const auto& entry : item_map) {
printer_->Print("$key$: $value$\n", "key", entry.first, "value",
entry.second);
}
}
template <typename DescriptorT>
std::string PyiGenerator::ModuleLevelName(
const DescriptorT& descriptor,
const std::map<std::string, std::string>& import_map) const {
std::string name = NamePrefixedWithNestedTypes(descriptor, ".");
if (descriptor.file() != file_) {
std::string module_alias;
std::string filename = descriptor.file()->name();
if (import_map.find(filename) == import_map.end()) {
std::string module_name = ModuleName(descriptor.file()->name());
std::vector<std::string> tokens = Split(module_name, ".");
module_alias = "_" + tokens.back();
} else {
module_alias = import_map.at(filename);
}
name = module_alias + "." + name;
}
return name;
}
struct ImportModules {
bool has_repeated = false; // _containers
bool has_iterable = false; // typing.Iterable
bool has_messages = false; // _message
bool has_enums = false; // _enum_type_wrapper
bool has_extendable = false; // _python_message
bool has_mapping = false; // typing.Mapping
bool has_optional = false; // typing.Optional
bool has_union = false; // typing.Union
bool has_well_known_type = false;
};
// Checks whether a descriptor name matches a well-known type.
bool IsWellKnownType(const std::string& name) {
// LINT.IfChange(wktbases)
return (name == "google.protobuf.Any" ||
name == "google.protobuf.Duration" ||
name == "google.protobuf.FieldMask" ||
name == "google.protobuf.ListValue" ||
name == "google.protobuf.Struct" ||
name == "google.protobuf.Timestamp");
// LINT.ThenChange(//depot/google3/net/proto2/python/internal/well_known_types.py:wktbases)
}
// Checks what modules should be imported for this message
// descriptor.
void CheckImportModules(const Descriptor* descriptor,
ImportModules* import_modules) {
if (descriptor->extension_range_count() > 0) {
import_modules->has_extendable = true;
}
if (descriptor->enum_type_count() > 0) {
import_modules->has_enums = true;
}
if (IsWellKnownType(descriptor->full_name())) {
import_modules->has_well_known_type = true;
}
for (int i = 0; i < descriptor->field_count(); ++i) {
const FieldDescriptor* field = descriptor->field(i);
if (IsPythonKeyword(field->name())) {
continue;
}
import_modules->has_optional = true;
if (field->is_repeated()) {
import_modules->has_repeated = true;
}
if (field->is_map()) {
import_modules->has_mapping = true;
const FieldDescriptor* value_des = field->message_type()->field(1);
if (value_des->cpp_type() == FieldDescriptor::CPPTYPE_MESSAGE ||
value_des->cpp_type() == FieldDescriptor::CPPTYPE_ENUM) {
import_modules->has_union = true;
}
} else {
if (field->is_repeated()) {
import_modules->has_iterable = true;
}
if (field->cpp_type() == FieldDescriptor::CPPTYPE_MESSAGE) {
import_modules->has_union = true;
import_modules->has_mapping = true;
}
if (field->cpp_type() == FieldDescriptor::CPPTYPE_ENUM) {
import_modules->has_union = true;
}
}
}
for (int i = 0; i < descriptor->nested_type_count(); ++i) {
CheckImportModules(descriptor->nested_type(i), import_modules);
}
}
void PyiGenerator::PrintImportForDescriptor(
const FileDescriptor& desc,
std::map<std::string, std::string>* import_map,
std::set<std::string>* seen_aliases) const {
const std::string& filename = desc.name();
std::string module_name = StrippedModuleName(filename);
size_t last_dot_pos = module_name.rfind('.');
std::string import_statement;
if (last_dot_pos == std::string::npos) {
import_statement = "import " + module_name;
} else {
import_statement = "from " + module_name.substr(0, last_dot_pos) +
" import " + module_name.substr(last_dot_pos + 1);
module_name = module_name.substr(last_dot_pos + 1);
}
std::string alias = "_" + module_name;
// Generate a unique alias by adding _1 suffixes until we get an unused alias.
while (seen_aliases->find(alias) != seen_aliases->end()) {
alias = alias + "_1";
}
printer_->Print("$statement$ as $alias$\n", "statement",
import_statement, "alias", alias);
(*import_map)[filename] = alias;
seen_aliases->insert(alias);
}
void PyiGenerator::PrintImports(
std::map<std::string, std::string>* item_map,
std::map<std::string, std::string>* import_map) const {
// Prints imported dependent _pb2 files.
std::set<std::string> seen_aliases;
for (int i = 0; i < file_->dependency_count(); ++i) {
const FileDescriptor* dep = file_->dependency(i);
PrintImportForDescriptor(*dep, import_map, &seen_aliases);
for (int j = 0; j < dep->public_dependency_count(); ++j) {
PrintImportForDescriptor(
*dep->public_dependency(j), import_map, &seen_aliases);
}
}
// Checks what modules should be imported.
ImportModules import_modules;
if (file_->message_type_count() > 0) {
import_modules.has_messages = true;
}
if (file_->enum_type_count() > 0) {
import_modules.has_enums = true;
}
for (int i = 0; i < file_->message_type_count(); i++) {
CheckImportModules(file_->message_type(i), &import_modules);
}
// Prints modules (e.g. _containers, _messages, typing) that are
// required in the proto file.
if (import_modules.has_repeated) {
printer_->Print(
"from google.protobuf.internal import containers as "
"_containers\n");
}
if (import_modules.has_enums) {
printer_->Print(
"from google.protobuf.internal import enum_type_wrapper"
" as _enum_type_wrapper\n");
}
if (import_modules.has_extendable) {
printer_->Print(
"from google.protobuf.internal import python_message"
" as _python_message\n");
}
if (import_modules.has_well_known_type) {
printer_->Print(
"from google.protobuf.internal import well_known_types"
" as _well_known_types\n");
}
printer_->Print(
"from google.protobuf import"
" descriptor as _descriptor\n");
if (import_modules.has_messages) {
printer_->Print(
"from google.protobuf import message as _message\n");
}
if (HasGenericServices(file_)) {
printer_->Print(
"from google.protobuf import service as"
" _service\n");
}
printer_->Print("from typing import ");
printer_->Print("ClassVar as _ClassVar");
if (import_modules.has_iterable) {
printer_->Print(", Iterable as _Iterable");
}
if (import_modules.has_mapping) {
printer_->Print(", Mapping as _Mapping");
}
if (import_modules.has_optional) {
printer_->Print(", Optional as _Optional");
}
if (import_modules.has_union) {
printer_->Print(", Union as _Union");
}
printer_->Print("\n\n");
// Public imports
for (int i = 0; i < file_->public_dependency_count(); ++i) {
const FileDescriptor* public_dep = file_->public_dependency(i);
std::string module_name = StrippedModuleName(public_dep->name());
// Top level messages in public imports
for (int i = 0; i < public_dep->message_type_count(); ++i) {
printer_->Print("from $module$ import $message_class$\n", "module",
module_name, "message_class",
public_dep->message_type(i)->name());
}
// Top level enums for public imports
for (int i = 0; i < public_dep->enum_type_count(); ++i) {
printer_->Print("from $module$ import $enum_class$\n", "module",
module_name, "enum_class",
public_dep->enum_type(i)->name());
}
// Enum values for public imports
for (int i = 0; i < public_dep->enum_type_count(); ++i) {
const EnumDescriptor* enum_descriptor = public_dep->enum_type(i);
for (int j = 0; j < enum_descriptor->value_count(); ++j) {
(*item_map)[enum_descriptor->value(j)->name()] =
ModuleLevelName(*enum_descriptor, *import_map);
}
}
// Top level extensions for public imports
AddExtensions(*public_dep, item_map);
}
}
void PyiGenerator::PrintEnum(const EnumDescriptor& enum_descriptor) const {
std::string enum_name = enum_descriptor.name();
printer_->Print(
"class $enum_name$(int, metaclass=_enum_type_wrapper.EnumTypeWrapper):\n"
" __slots__ = []\n",
"enum_name", enum_name);
}
// Adds enum value to item map which will be ordered and printed later.
void PyiGenerator::AddEnumValue(
const EnumDescriptor& enum_descriptor,
std::map<std::string, std::string>* item_map,
const std::map<std::string, std::string>& import_map) const {
// enum values
std::string module_enum_name = ModuleLevelName(enum_descriptor, import_map);
for (int j = 0; j < enum_descriptor.value_count(); ++j) {
const EnumValueDescriptor* value_descriptor = enum_descriptor.value(j);
(*item_map)[value_descriptor->name()] = module_enum_name;
}
}
// Prints top level enums
void PyiGenerator::PrintTopLevelEnums() const {
for (int i = 0; i < file_->enum_type_count(); ++i) {
printer_->Print("\n");
PrintEnum(*file_->enum_type(i));
}
}
// Add top level extensions to item_map which will be ordered and
// printed later.
template <typename DescriptorT>
void PyiGenerator::AddExtensions(
const DescriptorT& descriptor,
std::map<std::string, std::string>* item_map) const {
for (int i = 0; i < descriptor.extension_count(); ++i) {
const FieldDescriptor* extension_field = descriptor.extension(i);
std::string constant_name = extension_field->name() + "_FIELD_NUMBER";
ToUpper(&constant_name);
(*item_map)[constant_name] = "_ClassVar[int]";
(*item_map)[extension_field->name()] = "_descriptor.FieldDescriptor";
}
}
// Returns the string format of a field's cpp_type
std::string PyiGenerator::GetFieldType(
const FieldDescriptor& field_des, const Descriptor& containing_des,
const std::map<std::string, std::string>& import_map) const {
switch (field_des.cpp_type()) {
case FieldDescriptor::CPPTYPE_INT32:
case FieldDescriptor::CPPTYPE_UINT32:
case FieldDescriptor::CPPTYPE_INT64:
case FieldDescriptor::CPPTYPE_UINT64:
return "int";
case FieldDescriptor::CPPTYPE_DOUBLE:
case FieldDescriptor::CPPTYPE_FLOAT:
return "float";
case FieldDescriptor::CPPTYPE_BOOL:
return "bool";
case FieldDescriptor::CPPTYPE_ENUM:
return ModuleLevelName(*field_des.enum_type(), import_map);
case FieldDescriptor::CPPTYPE_STRING:
if (field_des.type() == FieldDescriptor::TYPE_STRING) {
return "str";
} else {
return "bytes";
}
case FieldDescriptor::CPPTYPE_MESSAGE: {
// If the field is inside a nested message and the nested message has the
// same name as a top-level message, then we need to prefix the field type
// with the module name for disambiguation.
std::string name = ModuleLevelName(*field_des.message_type(), import_map);
if ((containing_des.containing_type() != nullptr &&
name == containing_des.name())) {
std::string module = ModuleName(field_des.file()->name());
name = module + "." + name;
}
return name;
}
default:
GOOGLE_LOG(FATAL) << "Unsupported field type.";
}
return "";
}
void PyiGenerator::PrintMessage(
const Descriptor& message_descriptor, bool is_nested,
const std::map<std::string, std::string>& import_map) const {
if (!is_nested) {
printer_->Print("\n");
}
std::string class_name = message_descriptor.name();
std::string extra_base;
// A well-known type needs to inherit from its corresponding base class in
// net/proto2/python/internal/well_known_types.
if (IsWellKnownType(message_descriptor.full_name())) {
extra_base = ", _well_known_types." + message_descriptor.name();
} else {
extra_base = "";
}
printer_->Print("class $class_name$(_message.Message$extra_base$):\n",
"class_name", class_name, "extra_base", extra_base);
printer_->Indent();
printer_->Indent();
std::vector<const FieldDescriptor*> fields;
fields.reserve(message_descriptor.field_count());
for (int i = 0; i < message_descriptor.field_count(); ++i) {
fields.push_back(message_descriptor.field(i));
}
std::sort(fields.begin(), fields.end(), SortByName<FieldDescriptor>());
// Prints slots
printer_->Print("__slots__ = [", "class_name", class_name);
bool first_item = true;
for (const auto& field_des : fields) {
if (IsPythonKeyword(field_des->name())) {
continue;
}
if (first_item) {
first_item = false;
} else {
printer_->Print(", ");
}
printer_->Print("\"$field_name$\"", "field_name", field_des->name());
}
printer_->Print("]\n");
std::map<std::string, std::string> item_map;
// Prints Extensions for extendable messages
if (message_descriptor.extension_range_count() > 0) {
item_map["Extensions"] = "_python_message._ExtensionDict";
}
// Prints nested enums
std::vector<const EnumDescriptor*> nested_enums;
nested_enums.reserve(message_descriptor.enum_type_count());
for (int i = 0; i < message_descriptor.enum_type_count(); ++i) {
nested_enums.push_back(message_descriptor.enum_type(i));
}
std::sort(nested_enums.begin(), nested_enums.end(),
SortByName<EnumDescriptor>());
for (const auto& entry : nested_enums) {
PrintEnum(*entry);
// Adds enum value to item_map which will be ordered and printed later
AddEnumValue(*entry, &item_map, import_map);
}
// Prints nested messages
std::vector<const Descriptor*> nested_messages;
nested_messages.reserve(message_descriptor.nested_type_count());
for (int i = 0; i < message_descriptor.nested_type_count(); ++i) {
nested_messages.push_back(message_descriptor.nested_type(i));
}
std::sort(nested_messages.begin(), nested_messages.end(),
SortByName<Descriptor>());
for (const auto& entry : nested_messages) {
PrintMessage(*entry, true, import_map);
}
// Adds extensions to item_map which will be ordered and printed later
AddExtensions(message_descriptor, &item_map);
// Adds field number and field descriptor to item_map
for (int i = 0; i < message_descriptor.field_count(); ++i) {
const FieldDescriptor& field_des = *message_descriptor.field(i);
item_map[ToUpper(field_des.name()) + "_FIELD_NUMBER"] =
"_ClassVar[int]";
if (IsPythonKeyword(field_des.name())) {
continue;
}
std::string field_type = "";
if (field_des.is_map()) {
const FieldDescriptor* key_des = field_des.message_type()->field(0);
const FieldDescriptor* value_des = field_des.message_type()->field(1);
field_type = (value_des->cpp_type() == FieldDescriptor::CPPTYPE_MESSAGE
? "_containers.MessageMap["
: "_containers.ScalarMap[");
field_type += GetFieldType(*key_des, message_descriptor, import_map);
field_type += ", ";
field_type += GetFieldType(*value_des, message_descriptor, import_map);
} else {
if (field_des.is_repeated()) {
field_type = (field_des.cpp_type() == FieldDescriptor::CPPTYPE_MESSAGE
? "_containers.RepeatedCompositeFieldContainer["
: "_containers.RepeatedScalarFieldContainer[");
}
field_type += GetFieldType(field_des, message_descriptor, import_map);
}
if (field_des.is_repeated()) {
field_type += "]";
}
item_map[field_des.name()] = field_type;
}
// Prints all items in item_map
PrintItemMap(item_map);
// Prints __init__
printer_->Print("def __init__(self");
bool has_key_words = false;
bool is_first = true;
for (int i = 0; i < message_descriptor.field_count(); ++i) {
const FieldDescriptor* field_des = message_descriptor.field(i);
if (IsPythonKeyword(field_des->name())) {
has_key_words = true;
continue;
}
std::string field_name = field_des->name();
if (is_first && field_name == "self") {
// See b/144146793 for an example of real code that generates a (self,
// self) method signature. Since repeating a parameter name is illegal in
// Python, we rename the duplicate self.
field_name = "self_";
}
is_first = false;
printer_->Print(", $field_name$: ", "field_name", field_name);
if (field_des->is_repeated() ||
field_des->cpp_type() != FieldDescriptor::CPPTYPE_BOOL) {
printer_->Print("_Optional[");
}
if (field_des->is_map()) {
const Descriptor* map_entry = field_des->message_type();
printer_->Print(
"_Mapping[$key_type$, $value_type$]", "key_type",
GetFieldType(*map_entry->field(0), message_descriptor, import_map),
"value_type",
GetFieldType(*map_entry->field(1), message_descriptor, import_map));
} else {
if (field_des->is_repeated()) {
printer_->Print("_Iterable[");
}
if (field_des->cpp_type() == FieldDescriptor::CPPTYPE_MESSAGE) {
printer_->Print(
"_Union[$type_name$, _Mapping]", "type_name",
GetFieldType(*field_des, message_descriptor, import_map));
} else {
if (field_des->cpp_type() == FieldDescriptor::CPPTYPE_ENUM) {
printer_->Print("_Union[$type_name$, str]", "type_name",
ModuleLevelName(*field_des->enum_type(), import_map));
} else {
printer_->Print(
"$type_name$", "type_name",
GetFieldType(*field_des, message_descriptor, import_map));
}
}
if (field_des->is_repeated()) {
printer_->Print("]");
}
}
if (field_des->is_repeated() ||
field_des->cpp_type() != FieldDescriptor::CPPTYPE_BOOL) {
printer_->Print("]");
}
printer_->Print(" = ...");
}
if (has_key_words) {
printer_->Print(", **kwargs");
}
printer_->Print(") -> None: ...\n");
printer_->Outdent();
printer_->Outdent();
}
void PyiGenerator::PrintMessages(
const std::map<std::string, std::string>& import_map) const {
// Deterministically order the descriptors.
std::vector<const Descriptor*> messages;
messages.reserve(file_->message_type_count());
for (int i = 0; i < file_->message_type_count(); ++i) {
messages.push_back(file_->message_type(i));
}
std::sort(messages.begin(), messages.end(), SortByName<Descriptor>());
for (const auto& entry : messages) {
PrintMessage(*entry, false, import_map);
}
}
void PyiGenerator::PrintServices() const {
std::vector<const ServiceDescriptor*> services;
services.reserve(file_->service_count());
for (int i = 0; i < file_->service_count(); ++i) {
services.push_back(file_->service(i));
}
std::sort(services.begin(), services.end(), SortByName<ServiceDescriptor>());
// Prints $Service$ and $Service$_Stub classes
for (const auto& entry : services) {
printer_->Print("\n");
printer_->Print(
"class $service_name$(_service.service): ...\n\n"
"class $service_name$_Stub($service_name$): ...\n",
"service_name", entry->name());
}
}
bool PyiGenerator::Generate(const FileDescriptor* file,
const std::string& parameter,
GeneratorContext* context,
std::string* error) const {
MutexLock lock(&mutex_);
// Calculate file name.
file_ = file;
std::string filename =
parameter.empty() ? GetFileName(file, ".pyi") : parameter;
std::unique_ptr<io::ZeroCopyOutputStream> output(context->Open(filename));
GOOGLE_CHECK(output.get());
io::Printer printer(output.get(), '$');
printer_ = &printer;
// item map will store "DESCRIPTOR", top level extensions, top level enum
// values. The items will be sorted and printed later.
std::map<std::string, std::string> item_map;
// Adds "DESCRIPTOR" into item_map.
item_map["DESCRIPTOR"] = "_descriptor.FileDescriptor";
// import_map will be a mapping from filename to module alias, e.g.
// "google3/foo/bar.py" -> "_bar"
std::map<std::string, std::string> import_map;
PrintImports(&item_map, &import_map);
// Adds top level enum values to item_map.
for (int i = 0; i < file_->enum_type_count(); ++i) {
AddEnumValue(*file_->enum_type(i), &item_map, import_map);
}
// Adds top level extensions to item_map.
AddExtensions(*file_, &item_map);
// Prints item map
PrintItemMap(item_map);
PrintMessages(import_map);
PrintTopLevelEnums();
if (HasGenericServices(file)) {
PrintServices();
}
return true;
}
} // namespace python
} // namespace compiler
} // namespace protobuf
} // namespace google
@@ -0,0 +1,120 @@
// Protocol Buffers - Google's data interchange format
// Copyright 2008 Google Inc. All rights reserved.
// https://developers.google.com/protocol-buffers/
//
// Redistribution and use in source and binary forms, with or without
// modification, are permitted provided that the following conditions are
// met:
//
// * Redistributions of source code must retain the above copyright
// notice, this list of conditions and the following disclaimer.
// * Redistributions in binary form must reproduce the above
// copyright notice, this list of conditions and the following disclaimer
// in the documentation and/or other materials provided with the
// distribution.
// * Neither the name of Google Inc. nor the names of its
// contributors may be used to endorse or promote products derived from
// this software without specific prior written permission.
//
// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
// Author: jieluo@google.com (Jie Luo)
//
// Generates Python stub (.pyi) for a given .proto file.
#ifndef GOOGLE_PROTOBUF_COMPILER_PYTHON_PYI_GENERATOR_H__
#define GOOGLE_PROTOBUF_COMPILER_PYTHON_PYI_GENERATOR_H__
#include <map>
#include <set>
#include <string>
#include <google/protobuf/stubs/mutex.h>
#include <google/protobuf/compiler/code_generator.h>
// Must be included last.
#include <google/protobuf/port_def.inc>
namespace google {
namespace protobuf {
class Descriptor;
class EnumDescriptor;
class FieldDescriptor;
class MethodDescriptor;
class ServiceDescriptor;
namespace io {
class Printer;
}
namespace compiler {
namespace python {
class PROTOC_EXPORT PyiGenerator : public google::protobuf::compiler::CodeGenerator {
public:
PyiGenerator();
~PyiGenerator() override;
// CodeGenerator methods.
uint64_t GetSupportedFeatures() const override {
// Code generators must explicitly support proto3 optional.
return CodeGenerator::FEATURE_PROTO3_OPTIONAL;
}
bool Generate(const FileDescriptor* file, const std::string& parameter,
GeneratorContext* generator_context,
std::string* error) const override;
private:
void PrintImportForDescriptor(const FileDescriptor& desc,
std::map<std::string, std::string>* import_map,
std::set<std::string>* seen_aliases) const;
void PrintImports(std::map<std::string, std::string>* item_map,
std::map<std::string, std::string>* import_map) const;
void PrintEnum(const EnumDescriptor& enum_descriptor) const;
void AddEnumValue(const EnumDescriptor& enum_descriptor,
std::map<std::string, std::string>* item_map,
const std::map<std::string, std::string>& import_map) const;
void PrintTopLevelEnums() const;
template <typename DescriptorT>
void AddExtensions(const DescriptorT& descriptor,
std::map<std::string, std::string>* item_map) const;
void PrintMessages(
const std::map<std::string, std::string>& import_map) const;
void PrintMessage(const Descriptor& message_descriptor, bool is_nested,
const std::map<std::string, std::string>& import_map) const;
void PrintServices() const;
void PrintItemMap(const std::map<std::string, std::string>& item_map) const;
std::string GetFieldType(
const FieldDescriptor& field_des, const Descriptor& containing_des,
const std::map<std::string, std::string>& import_map) const;
template <typename DescriptorT>
std::string ModuleLevelName(
const DescriptorT& descriptor,
const std::map<std::string, std::string>& import_map) const;
// Very coarse-grained lock to ensure that Generate() is reentrant.
// Guards file_ and printer_.
mutable Mutex mutex_;
mutable const FileDescriptor* file_; // Set in Generate(). Under mutex_.
mutable io::Printer* printer_; // Set in Generate(). Under mutex_.
GOOGLE_DISALLOW_EVIL_CONSTRUCTORS(PyiGenerator);
};
} // namespace python
} // namespace compiler
} // namespace protobuf
} // namespace google
#include <google/protobuf/port_undef.inc>
#endif // GOOGLE_PROTOBUF_COMPILER_PYTHON_PYI_GENERATOR_H__
@@ -1,193 +1,6 @@
// Protocol Buffers - Google's data interchange format
// Copyright 2008 Google Inc. All rights reserved.
// https://developers.google.com/protocol-buffers/
//
// Redistribution and use in source and binary forms, with or without
// modification, are permitted provided that the following conditions are
// met:
//
// * Redistributions of source code must retain the above copyright
// notice, this list of conditions and the following disclaimer.
// * Redistributions in binary form must reproduce the above
// copyright notice, this list of conditions and the following disclaimer
// in the documentation and/or other materials provided with the
// distribution.
// * Neither the name of Google Inc. nor the names of its
// contributors may be used to endorse or promote products derived from
// this software without specific prior written permission.
//
// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#ifndef GOOGLE_PROTOBUF_COMPILER_PYTHON_PYTHON_GENERATOR_H_
#define GOOGLE_PROTOBUF_COMPILER_PYTHON_PYTHON_GENERATOR_H_
// Author: robinson@google.com (Will Robinson)
//
// Generates Python code for a given .proto file.
#include <google/protobuf/compiler/python/generator.h>
#ifndef GOOGLE_PROTOBUF_COMPILER_PYTHON_GENERATOR_H__
#define GOOGLE_PROTOBUF_COMPILER_PYTHON_GENERATOR_H__
#include <string>
#include <google/protobuf/compiler/code_generator.h>
#include <google/protobuf/stubs/mutex.h>
#include <google/protobuf/port_def.inc>
namespace google {
namespace protobuf {
class Descriptor;
class EnumDescriptor;
class EnumValueDescriptor;
class FieldDescriptor;
class OneofDescriptor;
class ServiceDescriptor;
namespace io {
class Printer;
}
namespace compiler {
namespace python {
enum class StripPrintDescriptor { kCreate, kFind };
// CodeGenerator implementation for generated Python protocol buffer classes.
// If you create your own protocol compiler binary and you want it to support
// Python output, you can do so by registering an instance of this
// CodeGenerator with the CommandLineInterface in your main() function.
class PROTOC_EXPORT Generator : public CodeGenerator {
public:
Generator();
virtual ~Generator();
// CodeGenerator methods.
bool Generate(const FileDescriptor* file, const std::string& parameter,
GeneratorContext* generator_context,
std::string* error) const override;
uint64_t GetSupportedFeatures() const override;
private:
void PrintImports() const;
void PrintFileDescriptor() const;
void PrintTopLevelEnums() const;
void PrintAllNestedEnumsInFile(StripPrintDescriptor print_mode) const;
void PrintNestedEnums(const Descriptor& descriptor,
StripPrintDescriptor print_mode) const;
void PrintCreateEnum(const EnumDescriptor& enum_descriptor) const;
void PrintFindEnum(const EnumDescriptor& enum_descriptor) const;
void PrintTopLevelExtensions() const;
void PrintFieldDescriptor(const FieldDescriptor& field,
bool is_extension) const;
void PrintFieldDescriptorsInDescriptor(
const Descriptor& message_descriptor, bool is_extension,
const std::string& list_variable_name, int (Descriptor::*CountFn)() const,
const FieldDescriptor* (Descriptor::*GetterFn)(int)const) const;
void PrintFieldsInDescriptor(const Descriptor& message_descriptor) const;
void PrintExtensionsInDescriptor(const Descriptor& message_descriptor) const;
void PrintMessageDescriptors(StripPrintDescriptor print_mode) const;
void PrintCreateDescriptor(const Descriptor& message_descriptor) const;
void PrintFindDescriptor(const Descriptor& message_descriptor) const;
void PrintNestedDescriptors(const Descriptor& containing_descriptor,
StripPrintDescriptor print_mode) const;
void PrintMessages() const;
void PrintMessage(const Descriptor& message_descriptor,
const std::string& prefix,
std::vector<std::string>* to_register,
bool is_nested) const;
void PrintNestedMessages(const Descriptor& containing_descriptor,
const std::string& prefix,
std::vector<std::string>* to_register) const;
void FixForeignFieldsInDescriptors() const;
void FixForeignFieldsInDescriptor(
const Descriptor& descriptor,
const Descriptor* containing_descriptor) const;
void FixForeignFieldsInField(const Descriptor* containing_type,
const FieldDescriptor& field,
const std::string& python_dict_name) const;
void AddMessageToFileDescriptor(const Descriptor& descriptor) const;
void AddEnumToFileDescriptor(const EnumDescriptor& descriptor) const;
void AddExtensionToFileDescriptor(const FieldDescriptor& descriptor) const;
void AddServiceToFileDescriptor(const ServiceDescriptor& descriptor) const;
std::string FieldReferencingExpression(
const Descriptor* containing_type, const FieldDescriptor& field,
const std::string& python_dict_name) const;
template <typename DescriptorT>
void FixContainingTypeInDescriptor(
const DescriptorT& descriptor,
const Descriptor* containing_descriptor) const;
void FixForeignFieldsInExtensions() const;
void FixForeignFieldsInExtension(
const FieldDescriptor& extension_field) const;
void FixForeignFieldsInNestedExtensions(const Descriptor& descriptor) const;
void PrintServices() const;
void PrintServiceDescriptors() const;
void PrintServiceDescriptor(const ServiceDescriptor& descriptor) const;
void PrintServiceClass(const ServiceDescriptor& descriptor) const;
void PrintServiceStub(const ServiceDescriptor& descriptor) const;
void PrintDescriptorKeyAndModuleName(
const ServiceDescriptor& descriptor) const;
void PrintEnumValueDescriptor(const EnumValueDescriptor& descriptor) const;
std::string OptionsValue(const std::string& serialized_options) const;
bool GeneratingDescriptorProto() const;
template <typename DescriptorT>
std::string ModuleLevelDescriptorName(const DescriptorT& descriptor) const;
std::string ModuleLevelMessageName(const Descriptor& descriptor) const;
std::string ModuleLevelServiceDescriptorName(
const ServiceDescriptor& descriptor) const;
template <typename DescriptorT, typename DescriptorProtoT>
void PrintSerializedPbInterval(const DescriptorT& descriptor,
DescriptorProtoT& proto,
const std::string& name) const;
void FixAllDescriptorOptions() const;
void FixOptionsForField(const FieldDescriptor& field) const;
void FixOptionsForOneof(const OneofDescriptor& oneof) const;
void FixOptionsForEnum(const EnumDescriptor& descriptor) const;
void FixOptionsForService(const ServiceDescriptor& descriptor) const;
void FixOptionsForMessage(const Descriptor& descriptor) const;
void SetSerializedPbInterval() const;
void SetMessagePbInterval(const Descriptor& descriptor) const;
void CopyPublicDependenciesAliases(const std::string& copy_from,
const FileDescriptor* file) const;
// Very coarse-grained lock to ensure that Generate() is reentrant.
// Guards file_, printer_ and file_descriptor_serialized_.
mutable Mutex mutex_;
mutable const FileDescriptor* file_; // Set in Generate(). Under mutex_.
mutable std::string file_descriptor_serialized_;
mutable io::Printer* printer_; // Set in Generate(). Under mutex_.
mutable bool pure_python_workable_;
GOOGLE_DISALLOW_EVIL_CONSTRUCTORS(Generator);
};
} // namespace python
} // namespace compiler
} // namespace protobuf
} // namespace google
#include <google/protobuf/port_undef.inc>
#endif // GOOGLE_PROTOBUF_COMPILER_PYTHON_GENERATOR_H__
#endif // GOOGLE_PROTOBUF_COMPILER_PYTHON_PYTHON_GENERATOR_H_