#!/usr/bin/python import os, sys from collections import namedtuple DEBUG = 'DEBUG' in sys.argv # cd to scripts working directory abspath = os.path.abspath(__file__) dname = os.path.dirname(abspath) os.chdir(dname) from google.protobuf.compiler import plugin_pb2 as plugin from google.protobuf.descriptor_pb2 import FieldDescriptorProto from pybars import Compiler, strlist compiler = Compiler() ## Logging enable_logging = DEBUG def log(msg): if enable_logging: sys.stderr.write(msg) # required to resolve custom options # generate with: protoc apioptions.proto --python_out=temp -Id:\blue\external\protobuf\protobuf-3.0.2\src -I. ## Note: ## If this import fails with something like "'module' object has no attribute 'Default'" you need to ## use `aptitude remove python-protobuf` and `pip install protobuf` import apioptions_pb2 def loadTemplate(lang, name): path = os.path.join("templates", lang, name) text = open(path).read() utext = unicode(text, "utf-8") return compiler.compile(utext) def option(options, key, default=None): if options.HasExtension(key): return options.Extensions[key] return default def type_name_cpp(name_with_ns): return name_with_ns.replace('.', '::') # # Map protobuf raw types to intermediate type tree # MethodModel = namedtuple("MethodModel", "name comment server_streaming has_complex_key key_type key_field output_type_full output_type_cpp output_type input_type_full input_type_cpp input_type skip_excel_generation lname") KeyData = namedtuple("KeyData", "type field") def transform_method(m, keydata): return MethodModel( name=m.name, comment=option(m.options, apioptions_pb2.method_description, ""), server_streaming=m.server_streaming, has_complex_key = False, key_type = keydata.type, key_field = keydata.field, input_type = className(m.input_type), input_type_full = m.input_type.lstrip('.'), input_type_cpp = type_name_cpp(m.input_type), output_type = className(m.output_type), output_type_full = m.output_type.lstrip('.'), output_type_cpp = type_name_cpp(m.output_type), skip_excel_generation = option(m.options, apioptions_pb2.skip_excel_generation, False), lname = m.name.lower(), ) FieldModel = namedtuple("FieldModel", "name Name nAme number type is_key proto") def transform_field(field, msg): return FieldModel( name = field.name, Name = camel_case(field.name), nAme = js_case(field.name), number = field.number, type = to_type_model(field, msg), is_key = option(field.options, apioptions_pb2.key, False), proto = field ) EnumModel = namedtuple("EnumModel", "name values") EnumEntryModel = namedtuple("EnumEntryModel", "name short_name number") def transform_enum(e): return EnumModel(e.name, [EnumEntryModel(v.name, enum_short_name(e.name, v.name), v.number) for v in e.value]) OneofModel = namedtuple("OneofModel", "name Name") def transform_oneof(o, msg): return OneofModel( name = o.name, Name = o.name[0].upper() + o.name[1:] ) MessageModel = namedtuple("MessageModel", "name field oneof nested_type nested_enum is_empty") def transform_message(i): r = MessageModel( name = i.name, field = [transform_field(f,i) for f in i.field], oneof = [transform_oneof(o,i) for o in i.oneof_decl], nested_type = [transform_message(m) for m in i.nested_type], nested_enum = [transform_enum(e) for e in i.enum_type], is_empty = len(i.field) == 0 ) return r ServiceModel = namedtuple("ServiceModel", "name comment method lname") KeyModel = namedtuple("KeyModel", "type field") DefaultKeyModel = KeyModel("int", "id") def transform_service(service, keymap): return ServiceModel( service.name, option(service.options, apioptions_pb2.service_description, ""), [transform_method(m, keymap.get(m.output_type, DefaultKeyModel)) for m in service.method], service.name.lower()) FileModel = namedtuple("FileModel", "name module package namespace_csharp message_type service enum_type imports") def transform_file(p, keymap, imports): module_name = os.path.splitext(os.path.basename(p.name))[0] return FileModel( name = p.name, module = module_name, package = p.package, #aka namespace namespace_csharp = p.package, message_type = [transform_message(m) for m in p.message_type], service = [transform_service(s, keymap) for s in p.service], enum_type = [transform_enum(e) for e in p.enum_type], # filter out this file from imports imports = [i for i in imports if not i.name == p.name] ) # # Transform helpers # def className(input): return input.split('.')[-1:][0] # return name,is_complex,is_nullable class TypeModel(): def __init__(self, mytype, islist=False, nullable=None, cpptype=None, jstype=None, fstype=None, isMessage=False, immutabletype=None, keytype=None, natural_key=False): self.cstype = mytype self.cpptype = cpptype or mytype self.jstype = jstype or mytype self.fstype = fstype or mytype self.nullable_type = nullable or mytype self.natural_key = natural_key self.keytype = keytype self.isdict = bool(keytype) self.ismessage = isMessage self.islist = islist self.immutable = immutabletype or mytype def to_type_model(field, msg): types = FieldDescriptorProto islist = field.label == FieldDescriptorProto.LABEL_REPEATED if field.type == types.TYPE_INT32: return TypeModel("int", islist, "int?", jstype='number', natural_key=True) elif field.type == types.TYPE_INT64: return TypeModel("long", islist, "long?", jstype='number', fstype='int64') elif field.type == types.TYPE_SINT32: return TypeModel("int", islist, "int?", jstype='number', natural_key=True) elif field.type == types.TYPE_SINT64: return TypeModel("long", islist, "long?", jstype='number', fstype='int64') elif field.type == types.TYPE_UINT32: return TypeModel("uint", islist, "uint?", "unsigned", jstype='number', fstype='uint32', natural_key=True) elif field.type == types.TYPE_UINT64: return TypeModel("ulong", islist, "ulong?", "unsigned long", jstype='number', fstype='uint64', natural_key=True) elif field.type == types.TYPE_DOUBLE: return TypeModel("double", islist, "double?", jstype='number') elif field.type == types.TYPE_FLOAT: return TypeModel("float", islist, "float?", jstype='number', fstype='float32') elif field.type == types.TYPE_STRING: return TypeModel("string", islist, cpptype="std::string") elif field.type == types.TYPE_BYTES: return TypeModel("ByteString", islist, cpptype="std::string", jstype='Uint8Array', fstype='uint8[]') elif field.type == types.TYPE_BOOL: return TypeModel("bool", islist, "bool?", jstype='boolean', natural_key=True) elif field.type == types.TYPE_ENUM: nested_enums = [x.name for x in msg.enum_type] full_name = field.type_name.lstrip('.') fs_name = full_name short_name = full_name.split('.')[-1:][0] if short_name in nested_enums: full_name = "{0}.Types.{1}".format(msg.name, short_name) fs_name = "{0}.{1}".format(msg.name, short_name) return TypeModel(full_name, islist, nullable=full_name+'?', cpptype=short_name, jstype='number', fstype=fs_name, natural_key=True) elif field.type == types.TYPE_MESSAGE: map_types = dict([(x.name, x) for x in msg.nested_type if x.options.map_entry == True]) full_name = field.type_name.lstrip('.') short_name = full_name.split('.')[-1:][0] immutable_postfix = "Immutable" nullable_postfix = "?" if full_name.startswith('google'): immutable_postfix = "" nullable_postfix = "" full_name = short_name if short_name in map_types: t = map_types[short_name] map_key = to_type_model([f for f in t.field if f.name == 'key'][0], msg) map_value = to_type_model([f for f in t.field if f.name == 'value'][0], msg) itype = map_value.cstype+immutable_postfix return TypeModel( map_value.cstype, cpptype=map_value.cpptype, jstype=map_value.jstype, fstype=map_value.fstype, isMessage=True, keytype=map_key, immutabletype=itype, nullable=itype+nullable_postfix) itype = full_name+immutable_postfix return TypeModel(full_name, islist, cpptype=short_name, jstype=short_name, isMessage=True, immutabletype=itype, nullable=itype+nullable_postfix) else: assert(not "Failed to get type for type id: %s " % field.type) # # /end transform # def parseParams(params): # log(params) result = dict() if params: parts = params.split(',') for p in parts: a,b = p.split("=") result[a] = b if 'make_filename' in params and not 'package_name' in params: result['package_name'] = result['make_filename'].lstrip('/').split('/')[1] return result def generate_insertions(params, request, response): # Include exports definition if os.name == 'posix': for file in request.file_to_generate: f1, f2 = response.file.add(), response.file.add() f1.name = f2.name = os.path.splitext(file)[0] + ".pb.h" f1.insertion_point = "includes" f1.content = """#ifdef __GNUC__ #pragma GCC diagnostic push #pragma GCC diagnostic ignored "-Weffc++" #endif""" f2.insertion_point = "global_scope" f2.content = "#pragma GCC diagnostic pop" def enum_short_name(enum_name, entry): if enum_name.lower() == entry[0:len(enum_name)].lower(): return camel_case(entry[len(enum_name)+1:]) return camel_case(entry) def camel_case(item): if item.isupper() and not '_' in item: return item return item.title().replace('_', '') return d def js_case(item): if item.isupper() and not '_' in item: return item cname = camel_case(item) return cname[0].lower() + cname[1:] def escape_fs_keywords(name): if name == 'type': return "``{0}``".format(name) return name def generate_code(request, response): params = parseParams(request.parameter) if not "lang" in params: response.error = "You must provide a language for the service generator plugin" return langs = params["lang"].split('+') if 'cpp' in langs: generate_insertions(params, request, response) if not 'no_services' in params: current_proto_file = None templates = [] for lang in langs: templates += [(f, loadTemplate(lang, f)) for f in os.listdir("templates/" + lang)] helpers = { 'upper': lambda this,item: item.upper(), 'lower': lambda this,item: item.lower(), 'cap': lambda this,item: item[0].upper() + item[1:], 'filename': lambda this,item: os.path.splitext(item)[0], 'ns-open': lambda this,item: "\n".join(["namespace {0} {{".format(x) for x in item.split('.')]), 'ns-close': lambda this,item: "}" * len(item.split('.')), 'type': lambda this,item: item.split('.')[-1:][0], 'camel': lambda _,f: camel_case(f), 'fsescape': lambda _,item: strlist(escape_fs_keywords(item)) } keymap = {} # Generate a map of keys for all types across files for proto_file in request.proto_file: for msg in proto_file.message_type: # check eachfield found_key = False namekey = ".{}.{}".format(proto_file.package,msg.name) for field in msg.field: if option(field.options, apioptions_pb2.key, False): found_key = True keymap[namekey] = KeyData(to_type_model(field, msg), field.name) break # If no field was annotated as a key but there's a field named "id" use that if not found_key: for field in msg.field: if field.name == "id": keymap[namekey] = KeyData(to_type_model(field, msg), field.name) break imports = [] for proto_file in request.proto_file: # if proto_file.name in request.file_to_generate: imports.append(transform_file(proto_file, keymap, [])) for proto_file in request.proto_file: if proto_file.name in request.file_to_generate: proto_file = transform_file(proto_file, keymap, imports) # Path on response.file name is interpreted differently by the c# and c++ code gen filename = ('strip_out_path' in params) and os.path.split(proto_file.name)[1] or proto_file.name source_name = os.path.splitext(filename)[0] for name,template in templates: f = response.file.add() f.name = source_name + "." + name #log("creating: " + f.name) f.content = template(proto_file, helpers) #### MAIN ######### if __name__ == "__main__": if DEBUG: data = open("data", mode='rb').read() else: data = sys.stdin.read() request = plugin.CodeGeneratorRequest() # open("data", 'wb').write(data) request.ParseFromString(data) response = plugin.CodeGeneratorResponse() generate_code(request, response) if DEBUG: for f in response.file: print f.content else: output = response.SerializeToString() sys.stdout.write(output)