327 lines
14 KiB
Python
Executable File
327 lines
14 KiB
Python
Executable File
#!/usr/bin/python
|
|
import os, sys
|
|
from collections import namedtuple
|
|
|
|
DEBUG = 'DEBUG' in sys.argv
|
|
|
|
# cd to scripts working directory
|
|
abspath = os.path.abspath(__file__)
|
|
dname = os.path.dirname(abspath)
|
|
os.chdir(dname)
|
|
|
|
from google.protobuf.compiler import plugin_pb2 as plugin
|
|
from google.protobuf.descriptor_pb2 import FieldDescriptorProto
|
|
from pybars import Compiler, strlist
|
|
compiler = Compiler()
|
|
|
|
## Logging
|
|
enable_logging = DEBUG
|
|
def log(msg):
|
|
if enable_logging:
|
|
sys.stderr.write(msg)
|
|
|
|
# required to resolve custom options
|
|
# generate with: protoc apioptions.proto --python_out=temp -Id:\blue\external\protobuf\protobuf-3.0.2\src -I.
|
|
## Note:
|
|
## If this import fails with something like "'module' object has no attribute 'Default'" you need to
|
|
## use `aptitude remove python-protobuf` and `pip install protobuf`
|
|
import apioptions_pb2
|
|
|
|
def loadTemplate(lang, name):
|
|
path = os.path.join("templates", lang, name)
|
|
text = open(path).read()
|
|
utext = unicode(text, "utf-8")
|
|
return compiler.compile(utext)
|
|
|
|
def option(options, key, default=None):
|
|
if options.HasExtension(key):
|
|
return options.Extensions[key]
|
|
return default
|
|
|
|
def type_name_cpp(name_with_ns):
|
|
return name_with_ns.replace('.', '::')
|
|
|
|
#
|
|
# Map protobuf raw types to intermediate type tree
|
|
#
|
|
MethodModel = namedtuple("MethodModel", "name comment server_streaming has_complex_key key_type key_field output_type_full output_type_cpp output_type input_type_full input_type_cpp input_type skip_excel_generation lname")
|
|
KeyData = namedtuple("KeyData", "type field")
|
|
def transform_method(m, keydata):
|
|
return MethodModel(
|
|
name=m.name,
|
|
comment=option(m.options, apioptions_pb2.method_description, ""),
|
|
server_streaming=m.server_streaming,
|
|
has_complex_key = False,
|
|
key_type = keydata.type,
|
|
key_field = keydata.field,
|
|
input_type = className(m.input_type),
|
|
input_type_full = m.input_type.lstrip('.'),
|
|
input_type_cpp = type_name_cpp(m.input_type),
|
|
output_type = className(m.output_type),
|
|
output_type_full = m.output_type.lstrip('.'),
|
|
output_type_cpp = type_name_cpp(m.output_type),
|
|
skip_excel_generation = option(m.options, apioptions_pb2.skip_excel_generation, False),
|
|
lname = m.name.lower(),
|
|
)
|
|
|
|
FieldModel = namedtuple("FieldModel", "name Name nAme number type is_key proto")
|
|
def transform_field(field, msg):
|
|
return FieldModel(
|
|
name = field.name,
|
|
Name = camel_case(field.name),
|
|
nAme = js_case(field.name),
|
|
number = field.number,
|
|
type = to_type_model(field, msg),
|
|
is_key = option(field.options, apioptions_pb2.key, False),
|
|
proto = field
|
|
)
|
|
|
|
EnumModel = namedtuple("EnumModel", "name values")
|
|
EnumEntryModel = namedtuple("EnumEntryModel", "name short_name number")
|
|
def transform_enum(e):
|
|
return EnumModel(e.name, [EnumEntryModel(v.name, enum_short_name(e.name, v.name), v.number) for v in e.value])
|
|
|
|
OneofModel = namedtuple("OneofModel", "name Name")
|
|
def transform_oneof(o, msg):
|
|
return OneofModel(
|
|
name = o.name,
|
|
Name = o.name[0].upper() + o.name[1:]
|
|
)
|
|
|
|
MessageModel = namedtuple("MessageModel", "name field oneof nested_type nested_enum is_empty")
|
|
def transform_message(i):
|
|
r = MessageModel(
|
|
name = i.name,
|
|
field = [transform_field(f,i) for f in i.field],
|
|
oneof = [transform_oneof(o,i) for o in i.oneof_decl],
|
|
nested_type = [transform_message(m) for m in i.nested_type],
|
|
nested_enum = [transform_enum(e) for e in i.enum_type],
|
|
is_empty = len(i.field) == 0
|
|
)
|
|
return r
|
|
|
|
ServiceModel = namedtuple("ServiceModel", "name comment method lname")
|
|
KeyModel = namedtuple("KeyModel", "type field")
|
|
DefaultKeyModel = KeyModel("int", "id")
|
|
def transform_service(service, keymap):
|
|
return ServiceModel(
|
|
service.name,
|
|
option(service.options, apioptions_pb2.service_description, ""),
|
|
[transform_method(m, keymap.get(m.output_type, DefaultKeyModel)) for m in service.method],
|
|
service.name.lower())
|
|
|
|
FileModel = namedtuple("FileModel", "name module package namespace_csharp message_type service enum_type imports")
|
|
def transform_file(p, keymap, imports):
|
|
module_name = os.path.splitext(os.path.basename(p.name))[0]
|
|
return FileModel(
|
|
name = p.name,
|
|
module = module_name,
|
|
package = p.package, #aka namespace
|
|
namespace_csharp = p.package,
|
|
message_type = [transform_message(m) for m in p.message_type],
|
|
service = [transform_service(s, keymap) for s in p.service],
|
|
enum_type = [transform_enum(e) for e in p.enum_type],
|
|
# filter out this file from imports
|
|
imports = [i for i in imports if not i.name == p.name]
|
|
)
|
|
|
|
#
|
|
# Transform helpers
|
|
#
|
|
def className(input):
|
|
return input.split('.')[-1:][0]
|
|
|
|
# return name,is_complex,is_nullable
|
|
class TypeModel():
|
|
def __init__(self, mytype, islist=False, nullable=None, cpptype=None, jstype=None, fstype=None, isMessage=False, immutabletype=None, keytype=None, natural_key=False):
|
|
self.cstype = mytype
|
|
self.cpptype = cpptype or mytype
|
|
self.jstype = jstype or mytype
|
|
self.fstype = fstype or mytype
|
|
self.nullable_type = nullable or mytype
|
|
self.natural_key = natural_key
|
|
self.keytype = keytype
|
|
self.isdict = bool(keytype)
|
|
self.ismessage = isMessage
|
|
self.islist = islist
|
|
self.immutable = immutabletype or mytype
|
|
|
|
def to_type_model(field, msg):
|
|
types = FieldDescriptorProto
|
|
islist = field.label == FieldDescriptorProto.LABEL_REPEATED
|
|
if field.type == types.TYPE_INT32: return TypeModel("int", islist, "int?", jstype='number', natural_key=True)
|
|
elif field.type == types.TYPE_INT64: return TypeModel("long", islist, "long?", jstype='number', fstype='int64')
|
|
elif field.type == types.TYPE_SINT32: return TypeModel("int", islist, "int?", jstype='number', natural_key=True)
|
|
elif field.type == types.TYPE_SINT64: return TypeModel("long", islist, "long?", jstype='number', fstype='int64')
|
|
elif field.type == types.TYPE_UINT32: return TypeModel("uint", islist, "uint?", "unsigned", jstype='number', fstype='uint32', natural_key=True)
|
|
elif field.type == types.TYPE_UINT64: return TypeModel("ulong", islist, "ulong?", "unsigned long", jstype='number', fstype='uint64', natural_key=True)
|
|
elif field.type == types.TYPE_DOUBLE: return TypeModel("double", islist, "double?", jstype='number')
|
|
elif field.type == types.TYPE_FLOAT: return TypeModel("float", islist, "float?", jstype='number', fstype='float32')
|
|
elif field.type == types.TYPE_STRING: return TypeModel("string", islist, cpptype="std::string")
|
|
elif field.type == types.TYPE_BYTES: return TypeModel("ByteString", islist, cpptype="std::string", jstype='Uint8Array', fstype='uint8[]')
|
|
elif field.type == types.TYPE_BOOL: return TypeModel("bool", islist, "bool?", jstype='boolean', natural_key=True)
|
|
elif field.type == types.TYPE_ENUM:
|
|
nested_enums = [x.name for x in msg.enum_type]
|
|
full_name = field.type_name.lstrip('.')
|
|
fs_name = full_name
|
|
short_name = full_name.split('.')[-1:][0]
|
|
if short_name in nested_enums:
|
|
full_name = "{0}.Types.{1}".format(msg.name, short_name)
|
|
fs_name = "{0}.{1}".format(msg.name, short_name)
|
|
return TypeModel(full_name, islist, nullable=full_name+'?', cpptype=short_name, jstype='number', fstype=fs_name, natural_key=True)
|
|
elif field.type == types.TYPE_MESSAGE:
|
|
map_types = dict([(x.name, x) for x in msg.nested_type if x.options.map_entry == True])
|
|
full_name = field.type_name.lstrip('.')
|
|
short_name = full_name.split('.')[-1:][0]
|
|
immutable_postfix = "Immutable"
|
|
nullable_postfix = "?"
|
|
if full_name.startswith('google'):
|
|
immutable_postfix = ""
|
|
nullable_postfix = ""
|
|
full_name = short_name
|
|
if short_name in map_types:
|
|
t = map_types[short_name]
|
|
map_key = to_type_model([f for f in t.field if f.name == 'key'][0], msg)
|
|
map_value = to_type_model([f for f in t.field if f.name == 'value'][0], msg)
|
|
itype = map_value.cstype+immutable_postfix
|
|
return TypeModel(
|
|
map_value.cstype,
|
|
cpptype=map_value.cpptype,
|
|
jstype=map_value.jstype,
|
|
fstype=map_value.fstype,
|
|
isMessage=True,
|
|
keytype=map_key,
|
|
immutabletype=itype,
|
|
nullable=itype+nullable_postfix)
|
|
itype = full_name+immutable_postfix
|
|
return TypeModel(full_name, islist, cpptype=short_name, jstype=short_name, isMessage=True, immutabletype=itype, nullable=itype+nullable_postfix)
|
|
else:
|
|
assert(not "Failed to get type for type id: %s " % field.type)
|
|
|
|
#
|
|
# /end transform
|
|
#
|
|
|
|
|
|
def parseParams(params):
|
|
# log(params)
|
|
result = dict()
|
|
if params:
|
|
parts = params.split(',')
|
|
for p in parts:
|
|
a,b = p.split("=")
|
|
result[a] = b
|
|
if 'make_filename' in params and not 'package_name' in params:
|
|
result['package_name'] = result['make_filename'].lstrip('/').split('/')[1]
|
|
return result
|
|
|
|
def generate_insertions(params, request, response):
|
|
# Include exports definition
|
|
if os.name == 'posix':
|
|
for file in request.file_to_generate:
|
|
f1, f2 = response.file.add(), response.file.add()
|
|
f1.name = f2.name = os.path.splitext(file)[0] + ".pb.h"
|
|
f1.insertion_point = "includes"
|
|
f1.content = """#ifdef __GNUC__
|
|
#pragma GCC diagnostic push
|
|
#pragma GCC diagnostic ignored "-Weffc++"
|
|
#endif"""
|
|
f2.insertion_point = "global_scope"
|
|
f2.content = "#pragma GCC diagnostic pop"
|
|
|
|
def enum_short_name(enum_name, entry):
|
|
if enum_name.lower() == entry[0:len(enum_name)].lower():
|
|
return camel_case(entry[len(enum_name)+1:])
|
|
return camel_case(entry)
|
|
|
|
def camel_case(item):
|
|
if item.isupper() and not '_' in item: return item
|
|
return item.title().replace('_', '')
|
|
return d
|
|
|
|
def js_case(item):
|
|
if item.isupper() and not '_' in item: return item
|
|
cname = camel_case(item)
|
|
return cname[0].lower() + cname[1:]
|
|
|
|
def escape_fs_keywords(name):
|
|
if name == 'type': return "``{0}``".format(name)
|
|
return name
|
|
|
|
def generate_code(request, response):
|
|
params = parseParams(request.parameter)
|
|
if not "lang" in params:
|
|
response.error = "You must provide a language for the service generator plugin"
|
|
return
|
|
langs = params["lang"].split('+')
|
|
if 'cpp' in langs: generate_insertions(params, request, response)
|
|
if not 'no_services' in params:
|
|
current_proto_file = None
|
|
templates = []
|
|
for lang in langs:
|
|
templates += [(f, loadTemplate(lang, f)) for f in os.listdir("templates/" + lang)]
|
|
helpers = { 'upper': lambda this,item: item.upper(),
|
|
'lower': lambda this,item: item.lower(),
|
|
'cap': lambda this,item: item[0].upper() + item[1:],
|
|
'filename': lambda this,item: os.path.splitext(item)[0],
|
|
'ns-open': lambda this,item: "\n".join(["namespace {0} {{".format(x) for x in item.split('.')]),
|
|
'ns-close': lambda this,item: "}" * len(item.split('.')),
|
|
'type': lambda this,item: item.split('.')[-1:][0],
|
|
'camel': lambda _,f: camel_case(f),
|
|
'fsescape': lambda _,item: strlist(escape_fs_keywords(item))
|
|
}
|
|
keymap = {}
|
|
# Generate a map of keys for all types across files
|
|
for proto_file in request.proto_file:
|
|
for msg in proto_file.message_type:
|
|
# check eachfield
|
|
found_key = False
|
|
namekey = ".{}.{}".format(proto_file.package,msg.name)
|
|
for field in msg.field:
|
|
if option(field.options, apioptions_pb2.key, False):
|
|
found_key = True
|
|
keymap[namekey] = KeyData(to_type_model(field, msg), field.name)
|
|
break
|
|
# If no field was annotated as a key but there's a field named "id" use that
|
|
if not found_key:
|
|
for field in msg.field:
|
|
if field.name == "id":
|
|
keymap[namekey] = KeyData(to_type_model(field, msg), field.name)
|
|
break
|
|
imports = []
|
|
for proto_file in request.proto_file:
|
|
# if proto_file.name in request.file_to_generate:
|
|
imports.append(transform_file(proto_file, keymap, []))
|
|
|
|
for proto_file in request.proto_file:
|
|
if proto_file.name in request.file_to_generate:
|
|
proto_file = transform_file(proto_file, keymap, imports)
|
|
# Path on response.file name is interpreted differently by the c# and c++ code gen
|
|
filename = ('strip_out_path' in params) and os.path.split(proto_file.name)[1] or proto_file.name
|
|
source_name = os.path.splitext(filename)[0]
|
|
for name,template in templates:
|
|
f = response.file.add()
|
|
f.name = source_name + "." + name
|
|
#log("creating: " + f.name)
|
|
f.content = template(proto_file, helpers)
|
|
|
|
#### MAIN #########
|
|
if __name__ == "__main__":
|
|
if DEBUG:
|
|
data = open("data", mode='rb').read()
|
|
else:
|
|
data = sys.stdin.read()
|
|
request = plugin.CodeGeneratorRequest()
|
|
# open("data", 'wb').write(data)
|
|
request.ParseFromString(data)
|
|
response = plugin.CodeGeneratorResponse()
|
|
generate_code(request, response)
|
|
|
|
if DEBUG:
|
|
for f in response.file:
|
|
print f.content
|
|
else:
|
|
output = response.SerializeToString()
|
|
sys.stdout.write(output)
|
|
|