Upgrade linux protobuf

This commit is contained in:
John Colvin committed 2021-12-12 15:56:55 -05:00
1 parent 79f2f356d5
commit 0f3130fb47
689 files changed
+141922 -132057

No files matched your search

@@ -62,26 +62,26 @@ namespace java {
namespace {
struct FieldDescriptorCompare {
bool operator ()(const FieldDescriptor* f1, const FieldDescriptor* f2) const {
if(f1 == NULL) {
bool operator()(const FieldDescriptor* f1, const FieldDescriptor* f2) const {
if (f1 == NULL) {
return false;
}
if(f2 == NULL) {
if (f2 == NULL) {
return true;
}
return f1->full_name() < f2->full_name();
}
};
typedef std::set<const FieldDescriptor*, FieldDescriptorCompare> FieldDescriptorSet;
typedef std::set<const FieldDescriptor*, FieldDescriptorCompare>
FieldDescriptorSet;
// Recursively searches the given message to collect extensions.
// Returns true if all the extensions can be recognized. The extensions will be
// appended in to the extensions parameter.
// Returns false when there are unknown fields, in which case the data in the
// extensions output parameter is not reliable and should be discarded.
bool CollectExtensions(const Message& message,
FieldDescriptorSet* extensions) {
bool CollectExtensions(const Message& message, FieldDescriptorSet* extensions) {
const Reflection* reflection = message.GetReflection();
// There are unknown fields that could be extensions, thus this call fails.
@@ -91,14 +91,16 @@ bool CollectExtensions(const Message& message,
reflection->ListFields(message, &fields);
for (int i = 0; i < fields.size(); i++) {
if (fields[i]->is_extension()) extensions->insert(fields[i]);
if (fields[i]->is_extension()) {
extensions->insert(fields[i]);
}
if (GetJavaType(fields[i]) == JAVATYPE_MESSAGE) {
if (fields[i]->is_repeated()) {
int size = reflection->FieldSize(message, fields[i]);
for (int j = 0; j < size; j++) {
const Message& sub_message =
reflection->GetRepeatedMessage(message, fields[i], j);
reflection->GetRepeatedMessage(message, fields[i], j);
if (!CollectExtensions(sub_message, extensions)) return false;
}
} else {
@@ -119,7 +121,7 @@ bool CollectExtensions(const Message& message,
void CollectExtensions(const FileDescriptorProto& file_proto,
const DescriptorPool& alternate_pool,
FieldDescriptorSet* extensions,
const string& file_data) {
const std::string& file_data) {
if (!CollectExtensions(file_proto, extensions)) {
// There are unknown fields in the file_proto, which are probably
// extensions. We need to parse the data into a dynamic message based on the
@@ -159,11 +161,9 @@ void CollectExtensions(const FileDescriptorProto& file_proto,
// extend our methods by simply chaining them to another method
// with a tail call. This inserts the sequence call-next-method,
// end this one, begin-next-method as needed.
void MaybeRestartJavaMethod(io::Printer* printer,
int *bytecode_estimate,
int *method_num,
const char *chain_statement,
const char *method_decl) {
void MaybeRestartJavaMethod(io::Printer* printer, int* bytecode_estimate,
int* method_num, const char* chain_statement,
const char* method_decl) {
// The goal here is to stay under 64K bytes of jvm bytecode/method,
// since otherwise we hit a hardcoded limit in the jvm and javac will
// then fail with the error "code too large". This limit lets our
@@ -172,10 +172,10 @@ void MaybeRestartJavaMethod(io::Printer* printer,
if ((*bytecode_estimate) > bytesPerMethod) {
++(*method_num);
printer->Print(chain_statement, "method_num", SimpleItoa(*method_num));
printer->Print(chain_statement, "method_num", StrCat(*method_num));
printer->Outdent();
printer->Print("}\n");
printer->Print(method_decl, "method_num", SimpleItoa(*method_num));
printer->Print(method_decl, "method_num", StrCat(*method_num));
printer->Indent();
*bytecode_estimate = 0;
}
@@ -193,8 +193,7 @@ FileGenerator::FileGenerator(const FileDescriptor* file, const Options& options,
options_(options),
immutable_api_(immutable_api) {
classname_ = name_resolver_->GetFileClassName(file, immutable_api);
generator_factory_.reset(
new ImmutableGeneratorFactory(context_.get()));
generator_factory_.reset(new ImmutableGeneratorFactory(context_.get()));
for (int i = 0; i < file_->message_type_count(); ++i) {
message_generators_[i].reset(
generator_factory_->NewMessageGenerator(file_->message_type(i)));
@@ -207,31 +206,49 @@ FileGenerator::FileGenerator(const FileDescriptor* file, const Options& options,
FileGenerator::~FileGenerator() {}
bool FileGenerator::Validate(string* error) {
bool FileGenerator::Validate(std::string* error) {
// Check that no class name matches the file's class name. This is a common
// problem that leads to Java compile errors that can be hard to understand.
// It's especially bad when using the java_multiple_files, since we would
// end up overwriting the outer class with one of the inner ones.
if (name_resolver_->HasConflictingClassName(file_, classname_)) {
if (name_resolver_->HasConflictingClassName(file_, classname_,
NameEquality::EXACT_EQUAL)) {
error->assign(file_->name());
error->append(
": Cannot generate Java output because the file's outer class name, \"");
": Cannot generate Java output because the file's outer class name, "
"\"");
error->append(classname_);
error->append(
"\", matches the name of one of the types declared inside it. "
"Please either rename the type or use the java_outer_classname "
"option to specify a different outer class name for the .proto file.");
"\", matches the name of one of the types declared inside it. "
"Please either rename the type or use the java_outer_classname "
"option to specify a different outer class name for the .proto file.");
return false;
}
// Similar to the check above, but ignore the case this time. This is not a
// problem on Linux, but will lead to Java compile errors on Windows / Mac
// because filenames are case-insensitive on those platforms.
if (name_resolver_->HasConflictingClassName(
file_, classname_, NameEquality::EQUAL_IGNORE_CASE)) {
GOOGLE_LOG(WARNING)
<< file_->name() << ": The file's outer class name, \"" << classname_
<< "\", matches the name of one of the types declared inside it when "
<< "case is ignored. This can cause compilation issues on Windows / "
<< "MacOS. Please either rename the type or use the "
<< "java_outer_classname option to specify a different outer class "
<< "name for the .proto file to be safe.";
}
// Print a warning if optimize_for = LITE_RUNTIME is used.
if (file_->options().optimize_for() == FileOptions::LITE_RUNTIME) {
if (file_->options().optimize_for() == FileOptions::LITE_RUNTIME &&
!options_.enforce_lite) {
GOOGLE_LOG(WARNING)
<< "The optimize_for = LITE_RUNTIME option is no longer supported by "
<< "protobuf Java code generator and may generate broken code. It "
<< "will be ignored by protoc in the future and protoc will always "
<< "protobuf Java code generator and is ignored--protoc will always "
<< "generate full runtime code for Java. To use Java Lite runtime, "
<< "users should use the Java Lite plugin instead. See:\n"
<< " https://github.com/google/protobuf/blob/master/java/lite.md";
<< " "
"https://github.com/protocolbuffers/protobuf/blob/master/java/"
"lite.md";
}
return true;
}
@@ -240,33 +257,33 @@ void FileGenerator::Generate(io::Printer* printer) {
// We don't import anything because we refer to all classes by their
// fully-qualified names in the generated source.
printer->Print(
"// Generated by the protocol buffer compiler. DO NOT EDIT!\n"
"// source: $filename$\n"
"\n",
"filename", file_->name());
"// Generated by the protocol buffer compiler. DO NOT EDIT!\n"
"// source: $filename$\n"
"\n",
"filename", file_->name());
if (!java_package_.empty()) {
printer->Print(
"package $package$;\n"
"\n",
"package", java_package_);
"package $package$;\n"
"\n",
"package", java_package_);
}
PrintGeneratedAnnotation(
printer, '$', options_.annotate_code ? classname_ + ".java.pb.meta" : "");
printer->Print(
"$deprecation$public final class $classname$ {\n"
" private $ctor$() {}\n",
"deprecation", file_->options().deprecated() ?
"@java.lang.Deprecated " : "",
"classname", classname_,
"ctor", classname_);
"deprecation",
file_->options().deprecated() ? "@java.lang.Deprecated " : "",
"classname", classname_, "ctor", classname_);
printer->Annotate("classname", file_->name());
printer->Indent();
// -----------------------------------------------------------------
printer->Print(
"public static void registerAllExtensions(\n"
" com.google.protobuf.ExtensionRegistryLite registry) {\n");
"public static void registerAllExtensions(\n"
" com.google.protobuf.ExtensionRegistryLite registry) {\n");
printer->Indent();
@@ -279,8 +296,7 @@ void FileGenerator::Generate(io::Printer* printer) {
}
printer->Outdent();
printer->Print(
"}\n");
printer->Print("}\n");
if (HasDescriptorMethods(file_, context_->EnforceLite())) {
// Overload registerAllExtensions for the non-lite usage to
// redundantly maintain the original signature (this is
@@ -288,12 +304,12 @@ void FileGenerator::Generate(io::Printer* printer) {
// ExtensionRegistry in the non-lite usage). Intent is
// to remove this in the future.
printer->Print(
"\n"
"public static void registerAllExtensions(\n"
" com.google.protobuf.ExtensionRegistry registry) {\n"
" registerAllExtensions(\n"
" (com.google.protobuf.ExtensionRegistryLite) registry);\n"
"}\n");
"\n"
"public static void registerAllExtensions(\n"
" com.google.protobuf.ExtensionRegistry registry) {\n"
" registerAllExtensions(\n"
" (com.google.protobuf.ExtensionRegistryLite) registry);\n"
"}\n");
}
// -----------------------------------------------------------------
@@ -345,29 +361,27 @@ void FileGenerator::Generate(io::Printer* printer) {
GenerateDescriptorInitializationCodeForMutable(printer);
}
} else {
printer->Print(
"static {\n");
printer->Print("static {\n");
printer->Indent();
int bytecode_estimate = 0;
int method_num = 0;
for (int i = 0; i < file_->message_type_count(); i++) {
bytecode_estimate += message_generators_[i]->GenerateStaticVariableInitializers(printer);
bytecode_estimate +=
message_generators_[i]->GenerateStaticVariableInitializers(printer);
MaybeRestartJavaMethod(
printer,
&bytecode_estimate, &method_num,
"_clinit_autosplit_$method_num$();\n",
"private static void _clinit_autosplit_$method_num$() {\n");
printer, &bytecode_estimate, &method_num,
"_clinit_autosplit_$method_num$();\n",
"private static void _clinit_autosplit_$method_num$() {\n");
}
printer->Outdent();
printer->Print(
"}\n");
printer->Print("}\n");
}
printer->Print(
"\n"
"// @@protoc_insertion_point(outer_class_scope)\n");
"\n"
"// @@protoc_insertion_point(outer_class_scope)\n");
printer->Outdent();
printer->Print("}\n");
@@ -376,15 +390,15 @@ void FileGenerator::Generate(io::Printer* printer) {
void FileGenerator::GenerateDescriptorInitializationCodeForImmutable(
io::Printer* printer) {
printer->Print(
"public static com.google.protobuf.Descriptors.FileDescriptor\n"
" getDescriptor() {\n"
" return descriptor;\n"
"}\n"
"private static $final$ com.google.protobuf.Descriptors.FileDescriptor\n"
" descriptor;\n"
"static {\n",
// TODO(dweis): Mark this as final.
"final", "");
"public static com.google.protobuf.Descriptors.FileDescriptor\n"
" getDescriptor() {\n"
" return descriptor;\n"
"}\n"
"private static $final$ com.google.protobuf.Descriptors.FileDescriptor\n"
" descriptor;\n"
"static {\n",
// TODO(dweis): Mark this as final.
"final", "");
printer->Indent();
SharedCodeGenerator shared_code_generator(file_, options_);
@@ -394,20 +408,20 @@ void FileGenerator::GenerateDescriptorInitializationCodeForImmutable(
int method_num = 0;
for (int i = 0; i < file_->message_type_count(); i++) {
bytecode_estimate += message_generators_[i]->GenerateStaticVariableInitializers(printer);
bytecode_estimate +=
message_generators_[i]->GenerateStaticVariableInitializers(printer);
MaybeRestartJavaMethod(
printer,
&bytecode_estimate, &method_num,
"_clinit_autosplit_dinit_$method_num$();\n",
"private static void _clinit_autosplit_dinit_$method_num$() {\n");
printer, &bytecode_estimate, &method_num,
"_clinit_autosplit_dinit_$method_num$();\n",
"private static void _clinit_autosplit_dinit_$method_num$() {\n");
}
for (int i = 0; i < file_->extension_count(); i++) {
bytecode_estimate += extension_generators_[i]->GenerateNonNestedInitializationCode(printer);
bytecode_estimate +=
extension_generators_[i]->GenerateNonNestedInitializationCode(printer);
MaybeRestartJavaMethod(
printer,
&bytecode_estimate, &method_num,
"_clinit_autosplit_dinit_$method_num$();\n",
"private static void _clinit_autosplit_dinit_$method_num$() {\n");
printer, &bytecode_estimate, &method_num,
"_clinit_autosplit_dinit_$method_num$();\n",
"private static void _clinit_autosplit_dinit_$method_num$() {\n");
}
// Proto compiler builds a DescriptorPool, which holds all the descriptors to
@@ -427,7 +441,7 @@ void FileGenerator::GenerateDescriptorInitializationCodeForImmutable(
// reflections to find all extension fields
FileDescriptorProto file_proto;
file_->CopyTo(&file_proto);
string file_data;
std::string file_data;
file_proto.SerializeToString(&file_data);
FieldDescriptorSet extensions;
CollectExtensions(file_proto, *file_->pool(), &extensions, file_data);
@@ -436,56 +450,54 @@ void FileGenerator::GenerateDescriptorInitializationCodeForImmutable(
// Must construct an ExtensionRegistry containing all existing extensions
// and use it to parse the descriptor data again to recognize extensions.
printer->Print(
"com.google.protobuf.ExtensionRegistry registry =\n"
" com.google.protobuf.ExtensionRegistry.newInstance();\n");
"com.google.protobuf.ExtensionRegistry registry =\n"
" com.google.protobuf.ExtensionRegistry.newInstance();\n");
FieldDescriptorSet::iterator it;
for (it = extensions.begin(); it != extensions.end(); it++) {
std::unique_ptr<ExtensionGenerator> generator(
generator_factory_->NewExtensionGenerator(*it));
bytecode_estimate += generator->GenerateRegistrationCode(printer);
MaybeRestartJavaMethod(
printer,
&bytecode_estimate, &method_num,
"_clinit_autosplit_dinit_$method_num$(registry);\n",
"private static void _clinit_autosplit_dinit_$method_num$(\n"
" com.google.protobuf.ExtensionRegistry registry) {\n");
printer, &bytecode_estimate, &method_num,
"_clinit_autosplit_dinit_$method_num$(registry);\n",
"private static void _clinit_autosplit_dinit_$method_num$(\n"
" com.google.protobuf.ExtensionRegistry registry) {\n");
}
printer->Print(
"com.google.protobuf.Descriptors.FileDescriptor\n"
" .internalUpdateFileDescriptor(descriptor, registry);\n");
"com.google.protobuf.Descriptors.FileDescriptor\n"
" .internalUpdateFileDescriptor(descriptor, registry);\n");
}
// Force descriptor initialization of all dependencies.
for (int i = 0; i < file_->dependency_count(); i++) {
if (ShouldIncludeDependency(file_->dependency(i), true)) {
string dependency =
std::string dependency =
name_resolver_->GetImmutableClassName(file_->dependency(i));
printer->Print(
"$dependency$.getDescriptor();\n",
"dependency", dependency);
printer->Print("$dependency$.getDescriptor();\n", "dependency",
dependency);
}
}
printer->Outdent();
printer->Print(
"}\n");
printer->Print("}\n");
}
void FileGenerator::GenerateDescriptorInitializationCodeForMutable(io::Printer* printer) {
void FileGenerator::GenerateDescriptorInitializationCodeForMutable(
io::Printer* printer) {
printer->Print(
"public static com.google.protobuf.Descriptors.FileDescriptor\n"
" getDescriptor() {\n"
" return descriptor;\n"
"}\n"
"private static final com.google.protobuf.Descriptors.FileDescriptor\n"
" descriptor;\n"
"static {\n");
"public static com.google.protobuf.Descriptors.FileDescriptor\n"
" getDescriptor() {\n"
" return descriptor;\n"
"}\n"
"private static final com.google.protobuf.Descriptors.FileDescriptor\n"
" descriptor;\n"
"static {\n");
printer->Indent();
printer->Print(
"descriptor = $immutable_package$.$descriptor_classname$.descriptor;\n",
"immutable_package", FileJavaPackage(file_, true),
"descriptor_classname", name_resolver_->GetDescriptorClassName(file_));
"descriptor = $immutable_package$.$descriptor_classname$.descriptor;\n",
"immutable_package", FileJavaPackage(file_, true), "descriptor_classname",
name_resolver_->GetDescriptorClassName(file_));
for (int i = 0; i < file_->message_type_count(); i++) {
message_generators_[i]->GenerateStaticVariableInitializers(printer);
@@ -498,7 +510,7 @@ void FileGenerator::GenerateDescriptorInitializationCodeForMutable(io::Printer*
// custom options are only represented with immutable messages.
FileDescriptorProto file_proto;
file_->CopyTo(&file_proto);
string file_data;
std::string file_data;
file_proto.SerializeToString(&file_data);
FieldDescriptorSet extensions;
CollectExtensions(file_proto, *file_->pool(), &extensions, file_data);
@@ -512,7 +524,7 @@ void FileGenerator::GenerateDescriptorInitializationCodeForMutable(io::Printer*
// we want the mutable code to be independent from the immutable code
// at compile time. It is required to implement dual-compile for
// mutable and immutable API in blaze.
" java.lang.Class immutableClass = java.lang.Class.forName(\n"
" java.lang.Class<?> immutableClass = java.lang.Class.forName(\n"
" \"$immutable_classname$\");\n"
"} catch (java.lang.ClassNotFoundException e) {\n",
"immutable_classname", name_resolver_->GetImmutableClassName(file_));
@@ -527,7 +539,7 @@ void FileGenerator::GenerateDescriptorInitializationCodeForMutable(io::Printer*
FieldDescriptorSet::iterator it;
for (it = extensions.begin(); it != extensions.end(); it++) {
const FieldDescriptor* field = *it;
string scope;
std::string scope;
if (field->extension_scope() != NULL) {
scope = name_resolver_->GetMutableClassName(field->extension_scope()) +
".getDescriptor()";
@@ -545,11 +557,11 @@ void FileGenerator::GenerateDescriptorInitializationCodeForMutable(io::Printer*
" $scope$.getExtensions().get($index$),\n"
" (com.google.protobuf.Message) defaultExtensionInstance);\n"
"}\n",
"scope", scope, "index", SimpleItoa(field->index()), "class",
"scope", scope, "index", StrCat(field->index()), "class",
name_resolver_->GetImmutableClassName(field->message_type()));
} else {
printer->Print("registry.add($scope$.getExtensions().get($index$));\n",
"scope", scope, "index", SimpleItoa(field->index()));
"scope", scope, "index", StrCat(field->index()));
}
}
printer->Print(
@@ -563,32 +575,29 @@ void FileGenerator::GenerateDescriptorInitializationCodeForMutable(io::Printer*
// Force descriptor initialization of all dependencies.
for (int i = 0; i < file_->dependency_count(); i++) {
if (ShouldIncludeDependency(file_->dependency(i), false)) {
string dependency = name_resolver_->GetMutableClassName(
file_->dependency(i));
printer->Print(
"$dependency$.getDescriptor();\n",
"dependency", dependency);
std::string dependency =
name_resolver_->GetMutableClassName(file_->dependency(i));
printer->Print("$dependency$.getDescriptor();\n", "dependency",
dependency);
}
}
printer->Outdent();
printer->Print(
"}\n");
printer->Print("}\n");
}
template <typename GeneratorClass, typename DescriptorClass>
static void GenerateSibling(const string& package_dir,
const string& java_package,
const DescriptorClass* descriptor,
GeneratorContext* context,
std::vector<string>* file_list, bool annotate_code,
std::vector<string>* annotation_list,
const string& name_suffix,
GeneratorClass* generator,
void (GeneratorClass::*pfn)(io::Printer* printer)) {
string filename = package_dir + descriptor->name() + name_suffix + ".java";
static void GenerateSibling(
const std::string& package_dir, const std::string& java_package,
const DescriptorClass* descriptor, GeneratorContext* context,
std::vector<std::string>* file_list, bool annotate_code,
std::vector<std::string>* annotation_list, const std::string& name_suffix,
GeneratorClass* generator,
void (GeneratorClass::*pfn)(io::Printer* printer)) {
std::string filename =
package_dir + descriptor->name() + name_suffix + ".java";
file_list->push_back(filename);
string info_full_path = filename + ".pb.meta";
std::string info_full_path = filename + ".pb.meta";
GeneratedCodeInfo annotations;
io::AnnotationProtoCollector<GeneratedCodeInfo> annotation_collector(
&annotations);
@@ -598,15 +607,15 @@ static void GenerateSibling(const string& package_dir,
annotate_code ? &annotation_collector : NULL);
printer.Print(
"// Generated by the protocol buffer compiler. DO NOT EDIT!\n"
"// source: $filename$\n"
"\n",
"filename", descriptor->file()->name());
"// Generated by the protocol buffer compiler. DO NOT EDIT!\n"
"// source: $filename$\n"
"\n",
"filename", descriptor->file()->name());
if (!java_package.empty()) {
printer.Print(
"package $package$;\n"
"\n",
"package", java_package);
"package $package$;\n"
"\n",
"package", java_package);
}
(generator->*pfn)(&printer);
@@ -619,10 +628,10 @@ static void GenerateSibling(const string& package_dir,
}
}
void FileGenerator::GenerateSiblings(const string& package_dir,
GeneratorContext* context,
std::vector<string>* file_list,
std::vector<string>* annotation_list) {
void FileGenerator::GenerateSiblings(
const std::string& package_dir, GeneratorContext* context,
std::vector<std::string>* file_list,
std::vector<std::string>* annotation_list) {
if (MultipleJavaFiles(file_, immutable_api_)) {
for (int i = 0; i < file_->enum_type_count(); i++) {
if (HasDescriptorMethods(file_, context_->EnforceLite())) {
@@ -666,8 +675,56 @@ void FileGenerator::GenerateSiblings(const string& package_dir,
}
}
bool FileGenerator::ShouldIncludeDependency(
const FileDescriptor* descriptor, bool immutable_api) {
std::string FileGenerator::GetKotlinClassname() {
return name_resolver_->GetFileClassName(file_, immutable_api_, true);
}
void FileGenerator::GenerateKotlinSiblings(
const std::string& package_dir, GeneratorContext* context,
std::vector<std::string>* file_list,
std::vector<std::string>* annotation_list) {
for (int i = 0; i < file_->message_type_count(); i++) {
const Descriptor* descriptor = file_->message_type(i);
MessageGenerator* generator = message_generators_[i].get();
auto open_file = [context](const std::string& filename) {
return std::unique_ptr<io::ZeroCopyOutputStream>(context->Open(filename));
};
std::string filename = package_dir + descriptor->name() + "Kt.kt";
file_list->push_back(filename);
std::string info_full_path = filename + ".pb.meta";
GeneratedCodeInfo annotations;
io::AnnotationProtoCollector<GeneratedCodeInfo> annotation_collector(
&annotations);
auto output = open_file(filename);
io::Printer printer(
output.get(), '$',
options_.annotate_code ? &annotation_collector : nullptr);
printer.Print(
"//Generated by the protocol buffer compiler. DO NOT EDIT!\n"
"// source: $filename$\n"
"\n",
"filename", descriptor->file()->name());
if (!java_package_.empty()) {
printer.Print(
"package $package$;\n"
"\n",
"package", java_package_);
}
generator->GenerateKotlinMembers(&printer);
generator->GenerateTopLevelKotlinMembers(&printer);
if (options_.annotate_code) {
auto info_output = open_file(info_full_path);
annotations.SerializeToZeroCopyStream(info_output.get());
annotation_list->push_back(info_full_path);
}
}
}
bool FileGenerator::ShouldIncludeDependency(const FileDescriptor* descriptor,
bool immutable_api) {
return true;
}