Upgrade linux protobuf from 3.19.1 -> 21.12

This commit is contained in:
John Colvin committed 2023-03-18 15:31:41 -04:00
1 parent 4a3e40193e
commit 01a4bb004a
554 files changed
+34981 -37969

No files matched your search

+172 -116
View File
@@ -95,7 +95,8 @@ def MessageToJson(
sort_keys=False,
use_integers_for_enums=False,
descriptor_pool=None,
float_precision=None):
float_precision=None,
ensure_ascii=True):
"""Converts protobuf message to JSON format.
Args:
@@ -114,6 +115,8 @@ def MessageToJson(
descriptor_pool: A Descriptor Pool for resolving types. If None use the
default.
float_precision: If set, use this to specify float field valid digits.
ensure_ascii: If True, strings with non-ASCII characters are escaped.
If False, Unicode strings are returned unchanged.
Returns:
A string containing the JSON formatted protocol buffer message.
@@ -124,7 +127,7 @@ def MessageToJson(
use_integers_for_enums,
descriptor_pool,
float_precision=float_precision)
return printer.ToJsonString(message, indent, sort_keys)
return printer.ToJsonString(message, indent, sort_keys, ensure_ascii)
def MessageToDict(
@@ -190,9 +193,10 @@ class _Printer(object):
else:
self.float_format = None
def ToJsonString(self, message, indent, sort_keys):
def ToJsonString(self, message, indent, sort_keys, ensure_ascii):
js = self._MessageToJsonObject(message)
return json.dumps(js, indent=indent, sort_keys=sort_keys)
return json.dumps(
js, indent=indent, sort_keys=sort_keys, ensure_ascii=ensure_ascii)
def _MessageToJsonObject(self, message):
"""Converts message to an object according to Proto3 JSON Specification."""
@@ -395,12 +399,16 @@ def _CreateMessageFromTypeUrl(type_url, descriptor_pool):
message_descriptor = pool.FindMessageTypeByName(type_name)
except KeyError:
raise TypeError(
'Can not find message descriptor by type_url: {0}.'.format(type_url))
'Can not find message descriptor by type_url: {0}'.format(type_url))
message_class = db.GetPrototype(message_descriptor)
return message_class()
def Parse(text, message, ignore_unknown_fields=False, descriptor_pool=None):
def Parse(text,
message,
ignore_unknown_fields=False,
descriptor_pool=None,
max_recursion_depth=100):
"""Parses a JSON representation of a protocol message into a message.
Args:
@@ -408,7 +416,10 @@ def Parse(text, message, ignore_unknown_fields=False, descriptor_pool=None):
message: A protocol buffer message to merge into.
ignore_unknown_fields: If True, do not raise errors for unknown fields.
descriptor_pool: A Descriptor Pool for resolving types. If None use the
default.
default.
max_recursion_depth: max recursion depth of JSON message to be
deserialized. JSON messages over this depth will fail to be
deserialized. Default value is 100.
Returns:
The same message passed as argument.
@@ -422,13 +433,15 @@ def Parse(text, message, ignore_unknown_fields=False, descriptor_pool=None):
js = json.loads(text, object_pairs_hook=_DuplicateChecker)
except ValueError as e:
raise ParseError('Failed to load JSON: {0}.'.format(str(e)))
return ParseDict(js, message, ignore_unknown_fields, descriptor_pool)
return ParseDict(js, message, ignore_unknown_fields, descriptor_pool,
max_recursion_depth)
def ParseDict(js_dict,
message,
ignore_unknown_fields=False,
descriptor_pool=None):
descriptor_pool=None,
max_recursion_depth=100):
"""Parses a JSON dictionary representation into a message.
Args:
@@ -437,12 +450,15 @@ def ParseDict(js_dict,
ignore_unknown_fields: If True, do not raise errors for unknown fields.
descriptor_pool: A Descriptor Pool for resolving types. If None use the
default.
max_recursion_depth: max recursion depth of JSON message to be
deserialized. JSON messages over this depth will fail to be
deserialized. Default value is 100.
Returns:
The same message passed as argument.
"""
parser = _Parser(ignore_unknown_fields, descriptor_pool)
parser.ConvertMessage(js_dict, message)
parser = _Parser(ignore_unknown_fields, descriptor_pool, max_recursion_depth)
parser.ConvertMessage(js_dict, message, '')
return message
@@ -452,35 +468,47 @@ _INT_OR_FLOAT = (int, float)
class _Parser(object):
"""JSON format parser for protocol message."""
def __init__(self, ignore_unknown_fields, descriptor_pool):
def __init__(self, ignore_unknown_fields, descriptor_pool,
max_recursion_depth):
self.ignore_unknown_fields = ignore_unknown_fields
self.descriptor_pool = descriptor_pool
self.max_recursion_depth = max_recursion_depth
self.recursion_depth = 0
def ConvertMessage(self, value, message):
def ConvertMessage(self, value, message, path):
"""Convert a JSON object into a message.
Args:
value: A JSON object.
message: A WKT or regular protocol message to record the data.
path: parent path to log parse error info.
Raises:
ParseError: In case of convert problems.
"""
self.recursion_depth += 1
if self.recursion_depth > self.max_recursion_depth:
raise ParseError('Message too deep. Max recursion depth is {0}'.format(
self.max_recursion_depth))
message_descriptor = message.DESCRIPTOR
full_name = message_descriptor.full_name
if not path:
path = message_descriptor.name
if _IsWrapperMessage(message_descriptor):
self._ConvertWrapperMessage(value, message)
self._ConvertWrapperMessage(value, message, path)
elif full_name in _WKTJSONMETHODS:
methodcaller(_WKTJSONMETHODS[full_name][1], value, message)(self)
methodcaller(_WKTJSONMETHODS[full_name][1], value, message, path)(self)
else:
self._ConvertFieldValuePair(value, message)
self._ConvertFieldValuePair(value, message, path)
self.recursion_depth -= 1
def _ConvertFieldValuePair(self, js, message):
def _ConvertFieldValuePair(self, js, message, path):
"""Convert field value pairs into regular message.
Args:
js: A JSON object to convert the field value pairs.
message: A regular protocol message to record the data.
path: parent path to log parse error info.
Raises:
ParseError: In case of problems converting.
@@ -496,8 +524,9 @@ class _Parser(object):
field = message_descriptor.fields_by_name.get(name, None)
if not field and _VALID_EXTENSION_NAME.match(name):
if not message_descriptor.is_extendable:
raise ParseError('Message type {0} does not have extensions'.format(
message_descriptor.full_name))
raise ParseError(
'Message type {0} does not have extensions at {1}'.format(
message_descriptor.full_name, path))
identifier = name[1:-1] # strip [] brackets
# pylint: disable=protected-access
field = message.Extensions._FindExtensionByName(identifier)
@@ -513,14 +542,14 @@ class _Parser(object):
if self.ignore_unknown_fields:
continue
raise ParseError(
('Message type "{0}" has no field named "{1}".\n'
' Available Fields(except extensions): {2}').format(
message_descriptor.full_name, name,
('Message type "{0}" has no field named "{1}" at "{2}".\n'
' Available Fields(except extensions): "{3}"').format(
message_descriptor.full_name, name, path,
[f.json_name for f in message_descriptor.fields]))
if name in names:
raise ParseError('Message type "{0}" should not have multiple '
'"{1}" fields.'.format(
message.DESCRIPTOR.full_name, name))
'"{1}" fields at "{2}".'.format(
message.DESCRIPTOR.full_name, name, path))
names.append(name)
value = js[name]
# Check no other oneof field is parsed.
@@ -528,8 +557,9 @@ class _Parser(object):
oneof_name = field.containing_oneof.name
if oneof_name in names:
raise ParseError('Message type "{0}" should not have multiple '
'"{1}" oneof fields.'.format(
message.DESCRIPTOR.full_name, oneof_name))
'"{1}" oneof fields at "{2}".'.format(
message.DESCRIPTOR.full_name, oneof_name,
path))
names.append(oneof_name)
if value is None:
@@ -547,42 +577,51 @@ class _Parser(object):
# Parse field value.
if _IsMapEntry(field):
message.ClearField(field.name)
self._ConvertMapFieldValue(value, message, field)
self._ConvertMapFieldValue(value, message, field,
'{0}.{1}'.format(path, name))
elif field.label == descriptor.FieldDescriptor.LABEL_REPEATED:
message.ClearField(field.name)
if not isinstance(value, list):
raise ParseError('repeated field {0} must be in [] which is '
'{1}.'.format(name, value))
'{1} at {2}'.format(name, value, path))
if field.cpp_type == descriptor.FieldDescriptor.CPPTYPE_MESSAGE:
# Repeated message field.
for item in value:
for index, item in enumerate(value):
sub_message = getattr(message, field.name).add()
# None is a null_value in Value.
if (item is None and
sub_message.DESCRIPTOR.full_name != 'google.protobuf.Value'):
raise ParseError('null is not allowed to be used as an element'
' in a repeated field.')
self.ConvertMessage(item, sub_message)
' in a repeated field at {0}.{1}[{2}]'.format(
path, name, index))
self.ConvertMessage(item, sub_message,
'{0}.{1}[{2}]'.format(path, name, index))
else:
# Repeated scalar field.
for item in value:
for index, item in enumerate(value):
if item is None:
raise ParseError('null is not allowed to be used as an element'
' in a repeated field.')
' in a repeated field at {0}.{1}[{2}]'.format(
path, name, index))
getattr(message, field.name).append(
_ConvertScalarFieldValue(item, field))
_ConvertScalarFieldValue(
item, field, '{0}.{1}[{2}]'.format(path, name, index)))
elif field.cpp_type == descriptor.FieldDescriptor.CPPTYPE_MESSAGE:
if field.is_extension:
sub_message = message.Extensions[field]
else:
sub_message = getattr(message, field.name)
sub_message.SetInParent()
self.ConvertMessage(value, sub_message)
self.ConvertMessage(value, sub_message, '{0}.{1}'.format(path, name))
else:
if field.is_extension:
message.Extensions[field] = _ConvertScalarFieldValue(value, field)
message.Extensions[field] = _ConvertScalarFieldValue(
value, field, '{0}.{1}'.format(path, name))
else:
setattr(message, field.name, _ConvertScalarFieldValue(value, field))
setattr(
message, field.name,
_ConvertScalarFieldValue(value, field,
'{0}.{1}'.format(path, name)))
except ParseError as e:
if field and field.containing_oneof is None:
raise ParseError('Failed to parse {0} field: {1}.'.format(name, e))
@@ -593,46 +632,52 @@ class _Parser(object):
except TypeError as e:
raise ParseError('Failed to parse {0} field: {1}.'.format(name, e))
def _ConvertAnyMessage(self, value, message):
def _ConvertAnyMessage(self, value, message, path):
"""Convert a JSON representation into Any message."""
if isinstance(value, dict) and not value:
return
try:
type_url = value['@type']
except KeyError:
raise ParseError('@type is missing when parsing any message.')
raise ParseError(
'@type is missing when parsing any message at {0}'.format(path))
sub_message = _CreateMessageFromTypeUrl(type_url, self.descriptor_pool)
try:
sub_message = _CreateMessageFromTypeUrl(type_url, self.descriptor_pool)
except TypeError as e:
raise ParseError('{0} at {1}'.format(e, path))
message_descriptor = sub_message.DESCRIPTOR
full_name = message_descriptor.full_name
if _IsWrapperMessage(message_descriptor):
self._ConvertWrapperMessage(value['value'], sub_message)
self._ConvertWrapperMessage(value['value'], sub_message,
'{0}.value'.format(path))
elif full_name in _WKTJSONMETHODS:
methodcaller(
_WKTJSONMETHODS[full_name][1], value['value'], sub_message)(self)
methodcaller(_WKTJSONMETHODS[full_name][1], value['value'], sub_message,
'{0}.value'.format(path))(
self)
else:
del value['@type']
self._ConvertFieldValuePair(value, sub_message)
self._ConvertFieldValuePair(value, sub_message, path)
value['@type'] = type_url
# Sets Any message
message.value = sub_message.SerializeToString()
message.type_url = type_url
def _ConvertGenericMessage(self, value, message):
def _ConvertGenericMessage(self, value, message, path):
"""Convert a JSON representation into message with FromJsonString."""
# Duration, Timestamp, FieldMask have a FromJsonString method to do the
# conversion. Users can also call the method directly.
try:
message.FromJsonString(value)
except ValueError as e:
raise ParseError(e)
raise ParseError('{0} at {1}'.format(e, path))
def _ConvertValueMessage(self, value, message):
def _ConvertValueMessage(self, value, message, path):
"""Convert a JSON representation into Value message."""
if isinstance(value, dict):
self._ConvertStructMessage(value, message.struct_value)
self._ConvertStructMessage(value, message.struct_value, path)
elif isinstance(value, list):
self. _ConvertListValueMessage(value, message.list_value)
self._ConvertListValueMessage(value, message.list_value, path)
elif value is None:
message.null_value = 0
elif isinstance(value, bool):
@@ -642,68 +687,76 @@ class _Parser(object):
elif isinstance(value, _INT_OR_FLOAT):
message.number_value = value
else:
raise ParseError('Value {0} has unexpected type {1}.'.format(
value, type(value)))
raise ParseError('Value {0} has unexpected type {1} at {2}'.format(
value, type(value), path))
def _ConvertListValueMessage(self, value, message):
def _ConvertListValueMessage(self, value, message, path):
"""Convert a JSON representation into ListValue message."""
if not isinstance(value, list):
raise ParseError(
'ListValue must be in [] which is {0}.'.format(value))
raise ParseError('ListValue must be in [] which is {0} at {1}'.format(
value, path))
message.ClearField('values')
for item in value:
self._ConvertValueMessage(item, message.values.add())
for index, item in enumerate(value):
self._ConvertValueMessage(item, message.values.add(),
'{0}[{1}]'.format(path, index))
def _ConvertStructMessage(self, value, message):
def _ConvertStructMessage(self, value, message, path):
"""Convert a JSON representation into Struct message."""
if not isinstance(value, dict):
raise ParseError(
'Struct must be in a dict which is {0}.'.format(value))
raise ParseError('Struct must be in a dict which is {0} at {1}'.format(
value, path))
# Clear will mark the struct as modified so it will be created even if
# there are no values.
message.Clear()
for key in value:
self._ConvertValueMessage(value[key], message.fields[key])
self._ConvertValueMessage(value[key], message.fields[key],
'{0}.{1}'.format(path, key))
return
def _ConvertWrapperMessage(self, value, message):
def _ConvertWrapperMessage(self, value, message, path):
"""Convert a JSON representation into Wrapper message."""
field = message.DESCRIPTOR.fields_by_name['value']
setattr(message, 'value', _ConvertScalarFieldValue(value, field))
setattr(
message, 'value',
_ConvertScalarFieldValue(value, field, path='{0}.value'.format(path)))
def _ConvertMapFieldValue(self, value, message, field):
def _ConvertMapFieldValue(self, value, message, field, path):
"""Convert map field value for a message map field.
Args:
value: A JSON object to convert the map field value.
message: A protocol message to record the converted data.
field: The descriptor of the map field to be converted.
path: parent path to log parse error info.
Raises:
ParseError: In case of convert problems.
"""
if not isinstance(value, dict):
raise ParseError(
'Map field {0} must be in a dict which is {1}.'.format(
field.name, value))
'Map field {0} must be in a dict which is {1} at {2}'.format(
field.name, value, path))
key_field = field.message_type.fields_by_name['key']
value_field = field.message_type.fields_by_name['value']
for key in value:
key_value = _ConvertScalarFieldValue(key, key_field, True)
key_value = _ConvertScalarFieldValue(key, key_field,
'{0}.key'.format(path), True)
if value_field.cpp_type == descriptor.FieldDescriptor.CPPTYPE_MESSAGE:
self.ConvertMessage(value[key], getattr(
message, field.name)[key_value])
self.ConvertMessage(value[key],
getattr(message, field.name)[key_value],
'{0}[{1}]'.format(path, key_value))
else:
getattr(message, field.name)[key_value] = _ConvertScalarFieldValue(
value[key], value_field)
value[key], value_field, path='{0}[{1}]'.format(path, key_value))
def _ConvertScalarFieldValue(value, field, require_str=False):
def _ConvertScalarFieldValue(value, field, path, require_str=False):
"""Convert a single scalar field value.
Args:
value: A scalar value to convert the scalar field value.
field: The descriptor of the field to convert.
path: parent path to log parse error info.
require_str: If True, the field value must be a str.
Returns:
@@ -712,44 +765,47 @@ def _ConvertScalarFieldValue(value, field, require_str=False):
Raises:
ParseError: In case of convert problems.
"""
if field.cpp_type in _INT_TYPES:
return _ConvertInteger(value)
elif field.cpp_type in _FLOAT_TYPES:
return _ConvertFloat(value, field)
elif field.cpp_type == descriptor.FieldDescriptor.CPPTYPE_BOOL:
return _ConvertBool(value, require_str)
elif field.cpp_type == descriptor.FieldDescriptor.CPPTYPE_STRING:
if field.type == descriptor.FieldDescriptor.TYPE_BYTES:
if isinstance(value, str):
encoded = value.encode('utf-8')
try:
if field.cpp_type in _INT_TYPES:
return _ConvertInteger(value)
elif field.cpp_type in _FLOAT_TYPES:
return _ConvertFloat(value, field)
elif field.cpp_type == descriptor.FieldDescriptor.CPPTYPE_BOOL:
return _ConvertBool(value, require_str)
elif field.cpp_type == descriptor.FieldDescriptor.CPPTYPE_STRING:
if field.type == descriptor.FieldDescriptor.TYPE_BYTES:
if isinstance(value, str):
encoded = value.encode('utf-8')
else:
encoded = value
# Add extra padding '='
padded_value = encoded + b'=' * (4 - len(encoded) % 4)
return base64.urlsafe_b64decode(padded_value)
else:
encoded = value
# Add extra padding '='
padded_value = encoded + b'=' * (4 - len(encoded) % 4)
return base64.urlsafe_b64decode(padded_value)
else:
# Checking for unpaired surrogates appears to be unreliable,
# depending on the specific Python version, so we check manually.
if _UNPAIRED_SURROGATE_PATTERN.search(value):
raise ParseError('Unpaired surrogate')
return value
elif field.cpp_type == descriptor.FieldDescriptor.CPPTYPE_ENUM:
# Convert an enum value.
enum_value = field.enum_type.values_by_name.get(value, None)
if enum_value is None:
try:
number = int(value)
enum_value = field.enum_type.values_by_number.get(number, None)
except ValueError:
raise ParseError('Invalid enum value {0} for enum type {1}.'.format(
value, field.enum_type.full_name))
# Checking for unpaired surrogates appears to be unreliable,
# depending on the specific Python version, so we check manually.
if _UNPAIRED_SURROGATE_PATTERN.search(value):
raise ParseError('Unpaired surrogate')
return value
elif field.cpp_type == descriptor.FieldDescriptor.CPPTYPE_ENUM:
# Convert an enum value.
enum_value = field.enum_type.values_by_name.get(value, None)
if enum_value is None:
if field.file.syntax == 'proto3':
# Proto3 accepts unknown enums.
return number
raise ParseError('Invalid enum value {0} for enum type {1}.'.format(
value, field.enum_type.full_name))
return enum_value.number
try:
number = int(value)
enum_value = field.enum_type.values_by_number.get(number, None)
except ValueError:
raise ParseError('Invalid enum value {0} for enum type {1}'.format(
value, field.enum_type.full_name))
if enum_value is None:
if field.file.syntax == 'proto3':
# Proto3 accepts unknown enums.
return number
raise ParseError('Invalid enum value {0} for enum type {1}'.format(
value, field.enum_type.full_name))
return enum_value.number
except ParseError as e:
raise ParseError('{0} at {1}'.format(e, path))
def _ConvertInteger(value):
@@ -765,14 +821,14 @@ def _ConvertInteger(value):
ParseError: If an integer couldn't be consumed.
"""
if isinstance(value, float) and not value.is_integer():
raise ParseError('Couldn\'t parse integer: {0}.'.format(value))
raise ParseError('Couldn\'t parse integer: {0}'.format(value))
if isinstance(value, str) and value.find(' ') != -1:
raise ParseError('Couldn\'t parse integer: "{0}".'.format(value))
raise ParseError('Couldn\'t parse integer: "{0}"'.format(value))
if isinstance(value, bool):
raise ParseError('Bool value {0} is not acceptable for '
'integer field.'.format(value))
'integer field'.format(value))
return int(value)
@@ -781,14 +837,14 @@ def _ConvertFloat(value, field):
"""Convert an floating point number."""
if isinstance(value, float):
if math.isnan(value):
raise ParseError('Couldn\'t parse NaN, use quoted "NaN" instead.')
raise ParseError('Couldn\'t parse NaN, use quoted "NaN" instead')
if math.isinf(value):
if value > 0:
raise ParseError('Couldn\'t parse Infinity or value too large, '
'use quoted "Infinity" instead.')
'use quoted "Infinity" instead')
else:
raise ParseError('Couldn\'t parse -Infinity or value too small, '
'use quoted "-Infinity" instead.')
'use quoted "-Infinity" instead')
if field.cpp_type == descriptor.FieldDescriptor.CPPTYPE_FLOAT:
# pylint: disable=protected-access
if value > type_checkers._FLOAT_MAX:
@@ -797,7 +853,7 @@ def _ConvertFloat(value, field):
if value < type_checkers._FLOAT_MIN:
raise ParseError('Float value too small')
if value == 'nan':
raise ParseError('Couldn\'t parse float "nan", use "NaN" instead.')
raise ParseError('Couldn\'t parse float "nan", use "NaN" instead')
try:
# Assume Python compatible syntax.
return float(value)
@@ -810,7 +866,7 @@ def _ConvertFloat(value, field):
elif value == _NAN:
return float('nan')
else:
raise ParseError('Couldn\'t parse float: {0}.'.format(value))
raise ParseError('Couldn\'t parse float: {0}'.format(value))
def _ConvertBool(value, require_str):
@@ -832,10 +888,10 @@ def _ConvertBool(value, require_str):
elif value == 'false':
return False
else:
raise ParseError('Expected "true" or "false", not {0}.'.format(value))
raise ParseError('Expected "true" or "false", not {0}'.format(value))
if not isinstance(value, bool):
raise ParseError('Expected true or false without quotes.')
raise ParseError('Expected true or false without quotes')
return value
_WKTJSONMETHODS = {