| #!/usr/bin/env python | |
| # | |
| # Copyright 2007 Neal Norwitz | |
| # Portions Copyright 2007 Google Inc. | |
| # | |
| # Licensed under the Apache License, Version 2.0 (the "License"); | |
| # you may not use this file except in compliance with the License. | |
| # You may obtain a copy of the License at | |
| # | |
| # http://www.apache.org/licenses/LICENSE-2.0 | |
| # | |
| # Unless required by applicable law or agreed to in writing, software | |
| # distributed under the License is distributed on an "AS IS" BASIS, | |
| # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | |
| # See the License for the specific language governing permissions and | |
| # limitations under the License. | |
| """Generate an Abstract Syntax Tree (AST) for C++.""" | |
| __author__ = 'nnorwitz@google.com (Neal Norwitz)' | |
| # TODO: | |
| # * Tokens should never be exported, need to convert to Nodes | |
| # (return types, parameters, etc.) | |
| # * Handle static class data for templatized classes | |
| # * Handle casts (both C++ and C-style) | |
| # * Handle conditions and loops (if/else, switch, for, while/do) | |
| # | |
| # TODO much, much later: | |
| # * Handle #define | |
| # * exceptions | |
| try: | |
| # Python 3.x | |
| import builtins | |
| except ImportError: | |
| # Python 2.x | |
| import __builtin__ as builtins | |
| import sys | |
| import traceback | |
| from cpp import keywords | |
| from cpp import tokenize | |
| from cpp import utils | |
| if not hasattr(builtins, 'reversed'): | |
| # Support Python 2.3 and earlier. | |
| def reversed(seq): | |
| for i in range(len(seq)-1, -1, -1): | |
| yield seq[i] | |
| if not hasattr(builtins, 'next'): | |
| # Support Python 2.5 and earlier. | |
| def next(obj): | |
| return obj.next() | |
| VISIBILITY_PUBLIC, VISIBILITY_PROTECTED, VISIBILITY_PRIVATE = range(3) | |
| FUNCTION_NONE = 0x00 | |
| FUNCTION_CONST = 0x01 | |
| FUNCTION_VIRTUAL = 0x02 | |
| FUNCTION_PURE_VIRTUAL = 0x04 | |
| FUNCTION_CTOR = 0x08 | |
| FUNCTION_DTOR = 0x10 | |
| FUNCTION_ATTRIBUTE = 0x20 | |
| FUNCTION_UNKNOWN_ANNOTATION = 0x40 | |
| FUNCTION_THROW = 0x80 | |
| FUNCTION_OVERRIDE = 0x100 | |
| """ | |
| These are currently unused. Should really handle these properly at some point. | |
| TYPE_MODIFIER_INLINE = 0x010000 | |
| TYPE_MODIFIER_EXTERN = 0x020000 | |
| TYPE_MODIFIER_STATIC = 0x040000 | |
| TYPE_MODIFIER_CONST = 0x080000 | |
| TYPE_MODIFIER_REGISTER = 0x100000 | |
| TYPE_MODIFIER_VOLATILE = 0x200000 | |
| TYPE_MODIFIER_MUTABLE = 0x400000 | |
| TYPE_MODIFIER_MAP = { | |
| 'inline': TYPE_MODIFIER_INLINE, | |
| 'extern': TYPE_MODIFIER_EXTERN, | |
| 'static': TYPE_MODIFIER_STATIC, | |
| 'const': TYPE_MODIFIER_CONST, | |
| 'register': TYPE_MODIFIER_REGISTER, | |
| 'volatile': TYPE_MODIFIER_VOLATILE, | |
| 'mutable': TYPE_MODIFIER_MUTABLE, | |
| } | |
| """ | |
| _INTERNAL_TOKEN = 'internal' | |
| _NAMESPACE_POP = 'ns-pop' | |
| # TODO(nnorwitz): use this as a singleton for templated_types, etc | |
| # where we don't want to create a new empty dict each time. It is also const. | |
| class _NullDict(object): | |
| __contains__ = lambda self: False | |
| keys = values = items = iterkeys = itervalues = iteritems = lambda self: () | |
| # TODO(nnorwitz): move AST nodes into a separate module. | |
| class Node(object): | |
| """Base AST node.""" | |
| def __init__(self, start, end): | |
| self.start = start | |
| self.end = end | |
| def IsDeclaration(self): | |
| """Returns bool if this node is a declaration.""" | |
| return False | |
| def IsDefinition(self): | |
| """Returns bool if this node is a definition.""" | |
| return False | |
| def IsExportable(self): | |
| """Returns bool if this node exportable from a header file.""" | |
| return False | |
| def Requires(self, node): | |
| """Does this AST node require the definition of the node passed in?""" | |
| return False | |
| def XXX__str__(self): | |
| return self._StringHelper(self.__class__.__name__, '') | |
| def _StringHelper(self, name, suffix): | |
| if not utils.DEBUG: | |
| return '%s(%s)' % (name, suffix) | |
| return '%s(%d, %d, %s)' % (name, self.start, self.end, suffix) | |
| def __repr__(self): | |
| return str(self) | |
| class Define(Node): | |
| def __init__(self, start, end, name, definition): | |
| Node.__init__(self, start, end) | |
| self.name = name | |
| self.definition = definition | |
| def __str__(self): | |
| value = '%s %s' % (self.name, self.definition) | |
| return self._StringHelper(self.__class__.__name__, value) | |
| class Include(Node): | |
| def __init__(self, start, end, filename, system): | |
| Node.__init__(self, start, end) | |
| self.filename = filename | |
| self.system = system | |
| def __str__(self): | |
| fmt = '"%s"' | |
| if self.system: | |
| fmt = '<%s>' | |
| return self._StringHelper(self.__class__.__name__, fmt % self.filename) | |
| class Goto(Node): | |
| def __init__(self, start, end, label): | |
| Node.__init__(self, start, end) | |
| self.label = label | |
| def __str__(self): | |
| return self._StringHelper(self.__class__.__name__, str(self.label)) | |
| class Expr(Node): | |
| def __init__(self, start, end, expr): | |
| Node.__init__(self, start, end) | |
| self.expr = expr | |
| def Requires(self, node): | |
| # TODO(nnorwitz): impl. | |
| return False | |
| def __str__(self): | |
| return self._StringHelper(self.__class__.__name__, str(self.expr)) | |
| class Return(Expr): | |
| pass | |
| class Delete(Expr): | |
| pass | |
| class Friend(Expr): | |
| def __init__(self, start, end, expr, namespace): | |
| Expr.__init__(self, start, end, expr) | |
| self.namespace = namespace[:] | |
| class Using(Node): | |
| def __init__(self, start, end, names): | |
| Node.__init__(self, start, end) | |
| self.names = names | |
| def __str__(self): | |
| return self._StringHelper(self.__class__.__name__, str(self.names)) | |
| class Parameter(Node): | |
| def __init__(self, start, end, name, parameter_type, default): | |
| Node.__init__(self, start, end) | |
| self.name = name | |
| self.type = parameter_type | |
| self.default = default | |
| def Requires(self, node): | |
| # TODO(nnorwitz): handle namespaces, etc. | |
| return self.type.name == node.name | |
| def __str__(self): | |
| name = str(self.type) | |
| suffix = '%s %s' % (name, self.name) | |
| if self.default: | |
| suffix += ' = ' + ''.join([d.name for d in self.default]) | |
| return self._StringHelper(self.__class__.__name__, suffix) | |
| class _GenericDeclaration(Node): | |
| def __init__(self, start, end, name, namespace): | |
| Node.__init__(self, start, end) | |
| self.name = name | |
| self.namespace = namespace[:] | |
| def FullName(self): | |
| prefix = '' | |
| if self.namespace and self.namespace[-1]: | |
| prefix = '::'.join(self.namespace) + '::' | |
| return prefix + self.name | |
| def _TypeStringHelper(self, suffix): | |
| if self.namespace: | |
| names = [n or '<anonymous>' for n in self.namespace] | |
| suffix += ' in ' + '::'.join(names) | |
| return self._StringHelper(self.__class__.__name__, suffix) | |
| # TODO(nnorwitz): merge with Parameter in some way? | |
| class VariableDeclaration(_GenericDeclaration): | |
| def __init__(self, start, end, name, var_type, initial_value, namespace): | |
| _GenericDeclaration.__init__(self, start, end, name, namespace) | |
| self.type = var_type | |
| self.initial_value = initial_value | |
| def Requires(self, node): | |
| # TODO(nnorwitz): handle namespaces, etc. | |
| return self.type.name == node.name | |
| def ToString(self): | |
| """Return a string that tries to reconstitute the variable decl.""" | |
| suffix = '%s %s' % (self.type, self.name) | |
| if self.initial_value: | |
| suffix += ' = ' + self.initial_value | |
| return suffix | |
| def __str__(self): | |
| return self._StringHelper(self.__class__.__name__, self.ToString()) | |
| class Typedef(_GenericDeclaration): | |
| def __init__(self, start, end, name, alias, namespace): | |
| _GenericDeclaration.__init__(self, start, end, name, namespace) | |
| self.alias = alias | |
| def IsDefinition(self): | |
| return True | |
| def IsExportable(self): | |
| return True | |
| def Requires(self, node): | |
| # TODO(nnorwitz): handle namespaces, etc. | |
| name = node.name | |
| for token in self.alias: | |
| if token is not None and name == token.name: | |
| return True | |
| return False | |
| def __str__(self): | |
| suffix = '%s, %s' % (self.name, self.alias) | |
| return self._TypeStringHelper(suffix) | |
| class _NestedType(_GenericDeclaration): | |
| def __init__(self, start, end, name, fields, namespace): | |
| _GenericDeclaration.__init__(self, start, end, name, namespace) | |
| self.fields = fields | |
| def IsDefinition(self): | |
| return True | |
| def IsExportable(self): | |
| return True | |
| def __str__(self): | |
| suffix = '%s, {%s}' % (self.name, self.fields) | |
| return self._TypeStringHelper(suffix) | |
| class Union(_NestedType): | |
| pass | |
| class Enum(_NestedType): | |
| pass | |
| class Class(_GenericDeclaration): | |
| def __init__(self, start, end, name, bases, templated_types, body, namespace): | |
| _GenericDeclaration.__init__(self, start, end, name, namespace) | |
| self.bases = bases | |
| self.body = body | |
| self.templated_types = templated_types | |
| def IsDeclaration(self): | |
| return self.bases is None and self.body is None | |
| def IsDefinition(self): | |
| return not self.IsDeclaration() | |
| def IsExportable(self): | |
| return not self.IsDeclaration() | |
| def Requires(self, node): | |
| # TODO(nnorwitz): handle namespaces, etc. | |
| if self.bases: | |
| for token_list in self.bases: | |
| # TODO(nnorwitz): bases are tokens, do name comparision. | |
| for token in token_list: | |
| if token.name == node.name: | |
| return True | |
| # TODO(nnorwitz): search in body too. | |
| return False | |
| def __str__(self): | |
| name = self.name | |
| if self.templated_types: | |
| name += '<%s>' % self.templated_types | |
| suffix = '%s, %s, %s' % (name, self.bases, self.body) | |
| return self._TypeStringHelper(suffix) | |
| class Struct(Class): | |
| pass | |
| class Function(_GenericDeclaration): | |
| def __init__(self, start, end, name, return_type, parameters, | |
| modifiers, templated_types, body, namespace): | |
| _GenericDeclaration.__init__(self, start, end, name, namespace) | |
| converter = TypeConverter(namespace) | |
| self.return_type = converter.CreateReturnType(return_type) | |
| self.parameters = converter.ToParameters(parameters) | |
| self.modifiers = modifiers | |
| self.body = body | |
| self.templated_types = templated_types | |
| def IsDeclaration(self): | |
| return self.body is None | |
| def IsDefinition(self): | |
| return self.body is not None | |
| def IsExportable(self): | |
| if self.return_type and 'static' in self.return_type.modifiers: | |
| return False | |
| return None not in self.namespace | |
| def Requires(self, node): | |
| if self.parameters: | |
| # TODO(nnorwitz): parameters are tokens, do name comparision. | |
| for p in self.parameters: | |
| if p.name == node.name: | |
| return True | |
| # TODO(nnorwitz): search in body too. | |
| return False | |
| def __str__(self): | |
| # TODO(nnorwitz): add templated_types. | |
| suffix = ('%s %s(%s), 0x%02x, %s' % | |
| (self.return_type, self.name, self.parameters, | |
| self.modifiers, self.body)) | |
| return self._TypeStringHelper(suffix) | |
| class Method(Function): | |
| def __init__(self, start, end, name, in_class, return_type, parameters, | |
| modifiers, templated_types, body, namespace): | |
| Function.__init__(self, start, end, name, return_type, parameters, | |
| modifiers, templated_types, body, namespace) | |
| # TODO(nnorwitz): in_class could also be a namespace which can | |
| # mess up finding functions properly. | |
| self.in_class = in_class | |
| class Type(_GenericDeclaration): | |
| """Type used for any variable (eg class, primitive, struct, etc).""" | |
| def __init__(self, start, end, name, templated_types, modifiers, | |
| reference, pointer, array): | |
| """ | |
| Args: | |
| name: str name of main type | |
| templated_types: [Class (Type?)] template type info between <> | |
| modifiers: [str] type modifiers (keywords) eg, const, mutable, etc. | |
| reference, pointer, array: bools | |
| """ | |
| _GenericDeclaration.__init__(self, start, end, name, []) | |
| self.templated_types = templated_types | |
| if not name and modifiers: | |
| self.name = modifiers.pop() | |
| self.modifiers = modifiers | |
| self.reference = reference | |
| self.pointer = pointer | |
| self.array = array | |
| def __str__(self): | |
| prefix = '' | |
| if self.modifiers: | |
| prefix = ' '.join(self.modifiers) + ' ' | |
| name = str(self.name) | |
| if self.templated_types: | |
| name += '<%s>' % self.templated_types | |
| suffix = prefix + name | |
| if self.reference: | |
| suffix += '&' | |
| if self.pointer: | |
| suffix += '*' | |
| if self.array: | |
| suffix += '[]' | |
| return self._TypeStringHelper(suffix) | |
| # By definition, Is* are always False. A Type can only exist in | |
| # some sort of variable declaration, parameter, or return value. | |
| def IsDeclaration(self): | |
| return False | |
| def IsDefinition(self): | |
| return False | |
| def IsExportable(self): | |
| return False | |
| class TypeConverter(object): | |
| def __init__(self, namespace_stack): | |
| self.namespace_stack = namespace_stack | |
| def _GetTemplateEnd(self, tokens, start): | |
| count = 1 | |
| end = start | |
| while 1: | |
| token = tokens[end] | |
| end += 1 | |
| if token.name == '<': | |
| count += 1 | |
| elif token.name == '>': | |
| count -= 1 | |
| if count == 0: | |
| break | |
| return tokens[start:end-1], end | |
| def ToType(self, tokens): | |
| """Convert [Token,...] to [Class(...), ] useful for base classes. | |
| For example, code like class Foo : public Bar<x, y> { ... }; | |
| the "Bar<x, y>" portion gets converted to an AST. | |
| Returns: | |
| [Class(...), ...] | |
| """ | |
| result = [] | |
| name_tokens = [] | |
| reference = pointer = array = False | |
| def AddType(templated_types): | |
| # Partition tokens into name and modifier tokens. | |
| names = [] | |
| modifiers = [] | |
| for t in name_tokens: | |
| if keywords.IsKeyword(t.name): | |
| modifiers.append(t.name) | |
| else: | |
| names.append(t.name) | |
| name = ''.join(names) | |
| if name_tokens: | |
| result.append(Type(name_tokens[0].start, name_tokens[-1].end, | |
| name, templated_types, modifiers, | |
| reference, pointer, array)) | |
| del name_tokens[:] | |
| i = 0 | |
| end = len(tokens) | |
| while i < end: | |
| token = tokens[i] | |
| if token.name == '<': | |
| new_tokens, new_end = self._GetTemplateEnd(tokens, i+1) | |
| AddType(self.ToType(new_tokens)) | |
| # If there is a comma after the template, we need to consume | |
| # that here otherwise it becomes part of the name. | |
| i = new_end | |
| reference = pointer = array = False | |
| elif token.name == ',': | |
| AddType([]) | |
| reference = pointer = array = False | |
| elif token.name == '*': | |
| pointer = True | |
| elif token.name == '&': | |
| reference = True | |
| elif token.name == '[': | |
| pointer = True | |
| elif token.name == ']': | |
| pass | |
| else: | |
| name_tokens.append(token) | |
| i += 1 | |
| if name_tokens: | |
| # No '<' in the tokens, just a simple name and no template. | |
| AddType([]) | |
| return result | |
| def DeclarationToParts(self, parts, needs_name_removed): | |
| name = None | |
| default = [] | |
| if needs_name_removed: | |
| # Handle default (initial) values properly. | |
| for i, t in enumerate(parts): | |
| if t.name == '=': | |
| default = parts[i+1:] | |
| name = parts[i-1].name | |
| if name == ']' and parts[i-2].name == '[': | |
| name = parts[i-3].name | |
| i -= 1 | |
| parts = parts[:i-1] | |
| break | |
| else: | |
| if parts[-1].token_type == tokenize.NAME: | |
| name = parts.pop().name | |
| else: | |
| # TODO(nnorwitz): this is a hack that happens for code like | |
| # Register(Foo<T>); where it thinks this is a function call | |
| # but it's actually a declaration. | |
| name = '???' | |
| modifiers = [] | |
| type_name = [] | |
| other_tokens = [] | |
| templated_types = [] | |
| i = 0 | |
| end = len(parts) | |
| while i < end: | |
| p = parts[i] | |
| if keywords.IsKeyword(p.name): | |
| modifiers.append(p.name) | |
| elif p.name == '<': | |
| templated_tokens, new_end = self._GetTemplateEnd(parts, i+1) | |
| templated_types = self.ToType(templated_tokens) | |
| i = new_end - 1 | |
| # Don't add a spurious :: to data members being initialized. | |
| next_index = i + 1 | |
| if next_index < end and parts[next_index].name == '::': | |
| i += 1 | |
| elif p.name in ('[', ']', '='): | |
| # These are handled elsewhere. | |
| other_tokens.append(p) | |
| elif p.name not in ('*', '&', '>'): | |
| # Ensure that names have a space between them. | |
| if (type_name and type_name[-1].token_type == tokenize.NAME and | |
| p.token_type == tokenize.NAME): | |
| type_name.append(tokenize.Token(tokenize.SYNTAX, ' ', 0, 0)) | |
| type_name.append(p) | |
| else: | |
| other_tokens.append(p) | |
| i += 1 | |
| type_name = ''.join([t.name for t in type_name]) | |
| return name, type_name, templated_types, modifiers, default, other_tokens | |
| def ToParameters(self, tokens): | |
| if not tokens: | |
| return [] | |
| result = [] | |
| name = type_name = '' | |
| type_modifiers = [] | |
| pointer = reference = array = False | |
| first_token = None | |
| default = [] | |
| def AddParameter(end): | |
| if default: | |
| del default[0] # Remove flag. | |
| parts = self.DeclarationToParts(type_modifiers, True) | |
| (name, type_name, templated_types, modifiers, | |
| unused_default, unused_other_tokens) = parts | |
| parameter_type = Type(first_token.start, first_token.end, | |
| type_name, templated_types, modifiers, | |
| reference, pointer, array) | |
| p = Parameter(first_token.start, end, name, | |
| parameter_type, default) | |
| result.append(p) | |
| template_count = 0 | |
| for s in tokens: | |
| if not first_token: | |
| first_token = s | |
| if s.name == '<': | |
| template_count += 1 | |
| elif s.name == '>': | |
| template_count -= 1 | |
| if template_count > 0: | |
| type_modifiers.append(s) | |
| continue | |
| if s.name == ',': | |
| AddParameter(s.start) | |
| name = type_name = '' | |
| type_modifiers = [] | |
| pointer = reference = array = False | |
| first_token = None | |
| default = [] | |
| elif s.name == '*': | |
| pointer = True | |
| elif s.name == '&': | |
| reference = True | |
| elif s.name == '[': | |
| array = True | |
| elif s.name == ']': | |
| pass # Just don't add to type_modifiers. | |
| elif s.name == '=': | |
| # Got a default value. Add any value (None) as a flag. | |
| default.append(None) | |
| elif default: | |
| default.append(s) | |
| else: | |
| type_modifiers.append(s) | |
| AddParameter(tokens[-1].end) | |
| return result | |
| def CreateReturnType(self, return_type_seq): | |
| if not return_type_seq: | |
| return None | |
| start = return_type_seq[0].start | |
| end = return_type_seq[-1].end | |
| _, name, templated_types, modifiers, default, other_tokens = \ | |
| self.DeclarationToParts(return_type_seq, False) | |
| names = [n.name for n in other_tokens] | |
| reference = '&' in names | |
| pointer = '*' in names | |
| array = '[' in names | |
| return Type(start, end, name, templated_types, modifiers, | |
| reference, pointer, array) | |
| def GetTemplateIndices(self, names): | |
| # names is a list of strings. | |
| start = names.index('<') | |
| end = len(names) - 1 | |
| while end > 0: | |
| if names[end] == '>': | |
| break | |
| end -= 1 | |
| return start, end+1 | |
| class AstBuilder(object): | |
| def __init__(self, token_stream, filename, in_class='', visibility=None, | |
| namespace_stack=[]): | |
| self.tokens = token_stream | |
| self.filename = filename | |
| # TODO(nnorwitz): use a better data structure (deque) for the queue. | |
| # Switching directions of the "queue" improved perf by about 25%. | |
| # Using a deque should be even better since we access from both sides. | |
| self.token_queue = [] | |
| self.namespace_stack = namespace_stack[:] | |
| self.in_class = in_class | |
| if in_class is None: | |
| self.in_class_name_only = None | |
| else: | |
| self.in_class_name_only = in_class.split('::')[-1] | |
| self.visibility = visibility | |
| self.in_function = False | |
| self.current_token = None | |
| # Keep the state whether we are currently handling a typedef or not. | |
| self._handling_typedef = False | |
| self.converter = TypeConverter(self.namespace_stack) | |
| def HandleError(self, msg, token): | |
| printable_queue = list(reversed(self.token_queue[-20:])) | |
| sys.stderr.write('Got %s in %s @ %s %s\n' % | |
| (msg, self.filename, token, printable_queue)) | |
| def Generate(self): | |
| while 1: | |
| token = self._GetNextToken() | |
| if not token: | |
| break | |
| # Get the next token. | |
| self.current_token = token | |
| # Dispatch on the next token type. | |
| if token.token_type == _INTERNAL_TOKEN: | |
| if token.name == _NAMESPACE_POP: | |
| self.namespace_stack.pop() | |
| continue | |
| try: | |
| result = self._GenerateOne(token) | |
| if result is not None: | |
| yield result | |
| except: | |
| self.HandleError('exception', token) | |
| raise | |
| def _CreateVariable(self, pos_token, name, type_name, type_modifiers, | |
| ref_pointer_name_seq, templated_types, value=None): | |
| reference = '&' in ref_pointer_name_seq | |
| pointer = '*' in ref_pointer_name_seq | |
| array = '[' in ref_pointer_name_seq | |
| var_type = Type(pos_token.start, pos_token.end, type_name, | |
| templated_types, type_modifiers, | |
| reference, pointer, array) | |
| return VariableDeclaration(pos_token.start, pos_token.end, | |
| name, var_type, value, self.namespace_stack) | |
| def _GenerateOne(self, token): | |
| if token.token_type == tokenize.NAME: | |
| if (keywords.IsKeyword(token.name) and | |
| not keywords.IsBuiltinType(token.name)): | |
| method = getattr(self, 'handle_' + token.name) | |
| return method() | |
| elif token.name == self.in_class_name_only: | |
| # The token name is the same as the class, must be a ctor if | |
| # there is a paren. Otherwise, it's the return type. | |
| # Peek ahead to get the next token to figure out which. | |
| next = self._GetNextToken() | |
| self._AddBackToken(next) | |
| if next.token_type == tokenize.SYNTAX and next.name == '(': | |
| return self._GetMethod([token], FUNCTION_CTOR, None, True) | |
| # Fall through--handle like any other method. | |
| # Handle data or function declaration/definition. | |
| syntax = tokenize.SYNTAX | |
| temp_tokens, last_token = \ | |
| self._GetVarTokensUpTo(syntax, '(', ';', '{', '[') | |
| temp_tokens.insert(0, token) | |
| if last_token.name == '(': | |
| # If there is an assignment before the paren, | |
| # this is an expression, not a method. | |
| expr = bool([e for e in temp_tokens if e.name == '=']) | |
| if expr: | |
| new_temp = self._GetTokensUpTo(tokenize.SYNTAX, ';') | |
| temp_tokens.append(last_token) | |
| temp_tokens.extend(new_temp) | |
| last_token = tokenize.Token(tokenize.SYNTAX, ';', 0, 0) | |
| if last_token.name == '[': | |
| # Handle array, this isn't a method, unless it's an operator. | |
| # TODO(nnorwitz): keep the size somewhere. | |
| # unused_size = self._GetTokensUpTo(tokenize.SYNTAX, ']') | |
| temp_tokens.append(last_token) | |
| if temp_tokens[-2].name == 'operator': | |
| temp_tokens.append(self._GetNextToken()) | |
| else: | |
| temp_tokens2, last_token = \ | |
| self._GetVarTokensUpTo(tokenize.SYNTAX, ';') | |
| temp_tokens.extend(temp_tokens2) | |
| if last_token.name == ';': | |
| # Handle data, this isn't a method. | |
| parts = self.converter.DeclarationToParts(temp_tokens, True) | |
| (name, type_name, templated_types, modifiers, default, | |
| unused_other_tokens) = parts | |
| t0 = temp_tokens[0] | |
| names = [t.name for t in temp_tokens] | |
| if templated_types: | |
| start, end = self.converter.GetTemplateIndices(names) | |
| names = names[:start] + names[end:] | |
| default = ''.join([t.name for t in default]) | |
| return self._CreateVariable(t0, name, type_name, modifiers, | |
| names, templated_types, default) | |
| if last_token.name == '{': | |
| self._AddBackTokens(temp_tokens[1:]) | |
| self._AddBackToken(last_token) | |
| method_name = temp_tokens[0].name | |
| method = getattr(self, 'handle_' + method_name, None) | |
| if not method: | |
| # Must be declaring a variable. | |
| # TODO(nnorwitz): handle the declaration. | |
| return None | |
| return method() | |
| return self._GetMethod(temp_tokens, 0, None, False) | |
| elif token.token_type == tokenize.SYNTAX: | |
| if token.name == '~' and self.in_class: | |
| # Must be a dtor (probably not in method body). | |
| token = self._GetNextToken() | |
| # self.in_class can contain A::Name, but the dtor will only | |
| # be Name. Make sure to compare against the right value. | |
| if (token.token_type == tokenize.NAME and | |
| token.name == self.in_class_name_only): | |
| return self._GetMethod([token], FUNCTION_DTOR, None, True) | |
| # TODO(nnorwitz): handle a lot more syntax. | |
| elif token.token_type == tokenize.PREPROCESSOR: | |
| # TODO(nnorwitz): handle more preprocessor directives. | |
| # token starts with a #, so remove it and strip whitespace. | |
| name = token.name[1:].lstrip() | |
| if name.startswith('include'): | |
| # Remove "include". | |
| name = name[7:].strip() | |
| assert name | |
| # Handle #include \<newline> "header-on-second-line.h". | |
| if name.startswith('\\'): | |
| name = name[1:].strip() | |
| assert name[0] in '<"', token | |
| assert name[-1] in '>"', token | |
| system = name[0] == '<' | |
| filename = name[1:-1] | |
| return Include(token.start, token.end, filename, system) | |
| if name.startswith('define'): | |
| # Remove "define". | |
| name = name[6:].strip() | |
| assert name | |
| value = '' | |
| for i, c in enumerate(name): | |
| if c.isspace(): | |
| value = name[i:].lstrip() | |
| name = name[:i] | |
| break | |
| return Define(token.start, token.end, name, value) | |
| if name.startswith('if') and name[2:3].isspace(): | |
| condition = name[3:].strip() | |
| if condition.startswith('0') or condition.startswith('(0)'): | |
| self._SkipIf0Blocks() | |
| return None | |
| def _GetTokensUpTo(self, expected_token_type, expected_token): | |
| return self._GetVarTokensUpTo(expected_token_type, expected_token)[0] | |
| def _GetVarTokensUpTo(self, expected_token_type, *expected_tokens): | |
| last_token = self._GetNextToken() | |
| tokens = [] | |
| while (last_token.token_type != expected_token_type or | |
| last_token.name not in expected_tokens): | |
| tokens.append(last_token) | |
| last_token = self._GetNextToken() | |
| return tokens, last_token | |
| # TODO(nnorwitz): remove _IgnoreUpTo() it shouldn't be necesary. | |
| def _IgnoreUpTo(self, token_type, token): | |
| unused_tokens = self._GetTokensUpTo(token_type, token) | |
| def _SkipIf0Blocks(self): | |
| count = 1 | |
| while 1: | |
| token = self._GetNextToken() | |
| if token.token_type != tokenize.PREPROCESSOR: | |
| continue | |
| name = token.name[1:].lstrip() | |
| if name.startswith('endif'): | |
| count -= 1 | |
| if count == 0: | |
| break | |
| elif name.startswith('if'): | |
| count += 1 | |
| def _GetMatchingChar(self, open_paren, close_paren, GetNextToken=None): | |
| if GetNextToken is None: | |
| GetNextToken = self._GetNextToken | |
| # Assumes the current token is open_paren and we will consume | |
| # and return up to the close_paren. | |
| count = 1 | |
| token = GetNextToken() | |
| while 1: | |
| if token.token_type == tokenize.SYNTAX: | |
| if token.name == open_paren: | |
| count += 1 | |
| elif token.name == close_paren: | |
| count -= 1 | |
| if count == 0: | |
| break | |
| yield token | |
| token = GetNextToken() | |
| yield token | |
| def _GetParameters(self): | |
| return self._GetMatchingChar('(', ')') | |
| def GetScope(self): | |
| return self._GetMatchingChar('{', '}') | |
| def _GetNextToken(self): | |
| if self.token_queue: | |
| return self.token_queue.pop() | |
| return next(self.tokens) | |
| def _AddBackToken(self, token): | |
| if token.whence == tokenize.WHENCE_STREAM: | |
| token.whence = tokenize.WHENCE_QUEUE | |
| self.token_queue.insert(0, token) | |
| else: | |
| assert token.whence == tokenize.WHENCE_QUEUE, token | |
| self.token_queue.append(token) | |
| def _AddBackTokens(self, tokens): | |
| if tokens: | |
| if tokens[-1].whence == tokenize.WHENCE_STREAM: | |
| for token in tokens: | |
| token.whence = tokenize.WHENCE_QUEUE | |
| self.token_queue[:0] = reversed(tokens) | |
| else: | |
| assert tokens[-1].whence == tokenize.WHENCE_QUEUE, tokens | |
| self.token_queue.extend(reversed(tokens)) | |
| def GetName(self, seq=None): | |
| """Returns ([tokens], next_token_info).""" | |
| GetNextToken = self._GetNextToken | |
| if seq is not None: | |
| it = iter(seq) | |
| GetNextToken = lambda: next(it) | |
| next_token = GetNextToken() | |
| tokens = [] | |
| last_token_was_name = False | |
| while (next_token.token_type == tokenize.NAME or | |
| (next_token.token_type == tokenize.SYNTAX and | |
| next_token.name in ('::', '<'))): | |
| # Two NAMEs in a row means the identifier should terminate. | |
| # It's probably some sort of variable declaration. | |
| if last_token_was_name and next_token.token_type == tokenize.NAME: | |
| break | |
| last_token_was_name = next_token.token_type == tokenize.NAME | |
| tokens.append(next_token) | |
| # Handle templated names. | |
| if next_token.name == '<': | |
| tokens.extend(self._GetMatchingChar('<', '>', GetNextToken)) | |
| last_token_was_name = True | |
| next_token = GetNextToken() | |
| return tokens, next_token | |
| def GetMethod(self, modifiers, templated_types): | |
| return_type_and_name = self._GetTokensUpTo(tokenize.SYNTAX, '(') | |
| assert len(return_type_and_name) >= 1 | |
| return self._GetMethod(return_type_and_name, modifiers, templated_types, | |
| False) | |
| def _GetMethod(self, return_type_and_name, modifiers, templated_types, | |
| get_paren): | |
| template_portion = None | |
| if get_paren: | |
| token = self._GetNextToken() | |
| assert token.token_type == tokenize.SYNTAX, token | |
| if token.name == '<': | |
| # Handle templatized dtors. | |
| template_portion = [token] | |
| template_portion.extend(self._GetMatchingChar('<', '>')) | |
| token = self._GetNextToken() | |
| assert token.token_type == tokenize.SYNTAX, token | |
| assert token.name == '(', token | |
| name = return_type_and_name.pop() | |
| # Handle templatized ctors. | |
| if name.name == '>': | |
| index = 1 | |
| while return_type_and_name[index].name != '<': | |
| index += 1 | |
| template_portion = return_type_and_name[index:] + [name] | |
| del return_type_and_name[index:] | |
| name = return_type_and_name.pop() | |
| elif name.name == ']': | |
| rt = return_type_and_name | |
| assert rt[-1].name == '[', return_type_and_name | |
| assert rt[-2].name == 'operator', return_type_and_name | |
| name_seq = return_type_and_name[-2:] | |
| del return_type_and_name[-2:] | |
| name = tokenize.Token(tokenize.NAME, 'operator[]', | |
| name_seq[0].start, name.end) | |
| # Get the open paren so _GetParameters() below works. | |
| unused_open_paren = self._GetNextToken() | |
| # TODO(nnorwitz): store template_portion. | |
| return_type = return_type_and_name | |
| indices = name | |
| if return_type: | |
| indices = return_type[0] | |
| # Force ctor for templatized ctors. | |
| if name.name == self.in_class and not modifiers: | |
| modifiers |= FUNCTION_CTOR | |
| parameters = list(self._GetParameters()) | |
| del parameters[-1] # Remove trailing ')'. | |
| # Handling operator() is especially weird. | |
| if name.name == 'operator' and not parameters: | |
| token = self._GetNextToken() | |
| assert token.name == '(', token | |
| parameters = list(self._GetParameters()) | |
| del parameters[-1] # Remove trailing ')'. | |
| token = self._GetNextToken() | |
| while token.token_type == tokenize.NAME: | |
| modifier_token = token | |
| token = self._GetNextToken() | |
| if modifier_token.name == 'const': | |
| modifiers |= FUNCTION_CONST | |
| elif modifier_token.name == '__attribute__': | |
| # TODO(nnorwitz): handle more __attribute__ details. | |
| modifiers |= FUNCTION_ATTRIBUTE | |
| assert token.name == '(', token | |
| # Consume everything between the (parens). | |
| unused_tokens = list(self._GetMatchingChar('(', ')')) | |
| token = self._GetNextToken() | |
| elif modifier_token.name == 'throw': | |
| modifiers |= FUNCTION_THROW | |
| assert token.name == '(', token | |
| # Consume everything between the (parens). | |
| unused_tokens = list(self._GetMatchingChar('(', ')')) | |
| token = self._GetNextToken() | |
| elif modifier_token.name == 'override': | |
| modifiers |= FUNCTION_OVERRIDE | |
| elif modifier_token.name == modifier_token.name.upper(): | |
| # HACK(nnorwitz): assume that all upper-case names | |
| # are some macro we aren't expanding. | |
| modifiers |= FUNCTION_UNKNOWN_ANNOTATION | |
| else: | |
| self.HandleError('unexpected token', modifier_token) | |
| assert token.token_type == tokenize.SYNTAX, token | |
| # Handle ctor initializers. | |
| if token.name == ':': | |
| # TODO(nnorwitz): anything else to handle for initializer list? | |
| while token.name != ';' and token.name != '{': | |
| token = self._GetNextToken() | |
| # Handle pointer to functions that are really data but look | |
| # like method declarations. | |
| if token.name == '(': | |
| if parameters[0].name == '*': | |
| # name contains the return type. | |
| name = parameters.pop() | |
| # parameters contains the name of the data. | |
| modifiers = [p.name for p in parameters] | |
| # Already at the ( to open the parameter list. | |
| function_parameters = list(self._GetMatchingChar('(', ')')) | |
| del function_parameters[-1] # Remove trailing ')'. | |
| # TODO(nnorwitz): store the function_parameters. | |
| token = self._GetNextToken() | |
| assert token.token_type == tokenize.SYNTAX, token | |
| assert token.name == ';', token | |
| return self._CreateVariable(indices, name.name, indices.name, | |
| modifiers, '', None) | |
| # At this point, we got something like: | |
| # return_type (type::*name_)(params); | |
| # This is a data member called name_ that is a function pointer. | |
| # With this code: void (sq_type::*field_)(string&); | |
| # We get: name=void return_type=[] parameters=sq_type ... field_ | |
| # TODO(nnorwitz): is return_type always empty? | |
| # TODO(nnorwitz): this isn't even close to being correct. | |
| # Just put in something so we don't crash and can move on. | |
| real_name = parameters[-1] | |
| modifiers = [p.name for p in self._GetParameters()] | |
| del modifiers[-1] # Remove trailing ')'. | |
| return self._CreateVariable(indices, real_name.name, indices.name, | |
| modifiers, '', None) | |
| if token.name == '{': | |
| body = list(self.GetScope()) | |
| del body[-1] # Remove trailing '}'. | |
| else: | |
| body = None | |
| if token.name == '=': | |
| token = self._GetNextToken() | |
| if token.name == 'default' or token.name == 'delete': | |
| # Ignore explicitly defaulted and deleted special members | |
| # in C++11. | |
| token = self._GetNextToken() | |
| else: | |
| # Handle pure-virtual declarations. | |
| assert token.token_type == tokenize.CONSTANT, token | |
| assert token.name == '0', token | |
| modifiers |= FUNCTION_PURE_VIRTUAL | |
| token = self._GetNextToken() | |
| if token.name == '[': | |
| # TODO(nnorwitz): store tokens and improve parsing. | |
| # template <typename T, size_t N> char (&ASH(T (&seq)[N]))[N]; | |
| tokens = list(self._GetMatchingChar('[', ']')) | |
| token = self._GetNextToken() | |
| assert token.name == ';', (token, return_type_and_name, parameters) | |
| # Looks like we got a method, not a function. | |
| if len(return_type) > 2 and return_type[-1].name == '::': | |
| return_type, in_class = \ | |
| self._GetReturnTypeAndClassName(return_type) | |
| return Method(indices.start, indices.end, name.name, in_class, | |
| return_type, parameters, modifiers, templated_types, | |
| body, self.namespace_stack) | |
| return Function(indices.start, indices.end, name.name, return_type, | |
| parameters, modifiers, templated_types, body, | |
| self.namespace_stack) | |
| def _GetReturnTypeAndClassName(self, token_seq): | |
| # Splitting the return type from the class name in a method | |
| # can be tricky. For example, Return::Type::Is::Hard::To::Find(). | |
| # Where is the return type and where is the class name? | |
| # The heuristic used is to pull the last name as the class name. | |
| # This includes all the templated type info. | |
| # TODO(nnorwitz): if there is only One name like in the | |
| # example above, punt and assume the last bit is the class name. | |
| # Ignore a :: prefix, if exists so we can find the first real name. | |
| i = 0 | |
| if token_seq[0].name == '::': | |
| i = 1 | |
| # Ignore a :: suffix, if exists. | |
| end = len(token_seq) - 1 | |
| if token_seq[end-1].name == '::': | |
| end -= 1 | |
| # Make a copy of the sequence so we can append a sentinel | |
| # value. This is required for GetName will has to have some | |
| # terminating condition beyond the last name. | |
| seq_copy = token_seq[i:end] | |
| seq_copy.append(tokenize.Token(tokenize.SYNTAX, '', 0, 0)) | |
| names = [] | |
| while i < end: | |
| # Iterate through the sequence parsing out each name. | |
| new_name, next = self.GetName(seq_copy[i:]) | |
| assert new_name, 'Got empty new_name, next=%s' % next | |
| # We got a pointer or ref. Add it to the name. | |
| if next and next.token_type == tokenize.SYNTAX: | |
| new_name.append(next) | |
| names.append(new_name) | |
| i += len(new_name) | |
| # Now that we have the names, it's time to undo what we did. | |
| # Remove the sentinel value. | |
| names[-1].pop() | |
| # Flatten the token sequence for the return type. | |
| return_type = [e for seq in names[:-1] for e in seq] | |
| # The class name is the last name. | |
| class_name = names[-1] | |
| return return_type, class_name | |
| def handle_bool(self): | |
| pass | |
| def handle_char(self): | |
| pass | |
| def handle_int(self): | |
| pass | |
| def handle_long(self): | |
| pass | |
| def handle_short(self): | |
| pass | |
| def handle_double(self): | |
| pass | |
| def handle_float(self): | |
| pass | |
| def handle_void(self): | |
| pass | |
| def handle_wchar_t(self): | |
| pass | |
| def handle_unsigned(self): | |
| pass | |
| def handle_signed(self): | |
| pass | |
| def _GetNestedType(self, ctor): | |
| name = None | |
| name_tokens, token = self.GetName() | |
| if name_tokens: | |
| name = ''.join([t.name for t in name_tokens]) | |
| # Handle forward declarations. | |
| if token.token_type == tokenize.SYNTAX and token.name == ';': | |
| return ctor(token.start, token.end, name, None, | |
| self.namespace_stack) | |
| if token.token_type == tokenize.NAME and self._handling_typedef: | |
| self._AddBackToken(token) | |
| return ctor(token.start, token.end, name, None, | |
| self.namespace_stack) | |
| # Must be the type declaration. | |
| fields = list(self._GetMatchingChar('{', '}')) | |
| del fields[-1] # Remove trailing '}'. | |
| if token.token_type == tokenize.SYNTAX and token.name == '{': | |
| next = self._GetNextToken() | |
| new_type = ctor(token.start, token.end, name, fields, | |
| self.namespace_stack) | |
| # A name means this is an anonymous type and the name | |
| # is the variable declaration. | |
| if next.token_type != tokenize.NAME: | |
| return new_type | |
| name = new_type | |
| token = next | |
| # Must be variable declaration using the type prefixed with keyword. | |
| assert token.token_type == tokenize.NAME, token | |
| return self._CreateVariable(token, token.name, name, [], '', None) | |
| def handle_struct(self): | |
| # Special case the handling typedef/aliasing of structs here. | |
| # It would be a pain to handle in the class code. | |
| name_tokens, var_token = self.GetName() | |
| if name_tokens: | |
| next_token = self._GetNextToken() | |
| is_syntax = (var_token.token_type == tokenize.SYNTAX and | |
| var_token.name[0] in '*&') | |
| is_variable = (var_token.token_type == tokenize.NAME and | |
| next_token.name == ';') | |
| variable = var_token | |
| if is_syntax and not is_variable: | |
| variable = next_token | |
| temp = self._GetNextToken() | |
| if temp.token_type == tokenize.SYNTAX and temp.name == '(': | |
| # Handle methods declared to return a struct. | |
| t0 = name_tokens[0] | |
| struct = tokenize.Token(tokenize.NAME, 'struct', | |
| t0.start-7, t0.start-2) | |
| type_and_name = [struct] | |
| type_and_name.extend(name_tokens) | |
| type_and_name.extend((var_token, next_token)) | |
| return self._GetMethod(type_and_name, 0, None, False) | |
| assert temp.name == ';', (temp, name_tokens, var_token) | |
| if is_syntax or (is_variable and not self._handling_typedef): | |
| modifiers = ['struct'] | |
| type_name = ''.join([t.name for t in name_tokens]) | |
| position = name_tokens[0] | |
| return self._CreateVariable(position, variable.name, type_name, | |
| modifiers, var_token.name, None) | |
| name_tokens.extend((var_token, next_token)) | |
| self._AddBackTokens(name_tokens) | |
| else: | |
| self._AddBackToken(var_token) | |
| return self._GetClass(Struct, VISIBILITY_PUBLIC, None) | |
| def handle_union(self): | |
| return self._GetNestedType(Union) | |
| def handle_enum(self): | |
| return self._GetNestedType(Enum) | |
| def handle_auto(self): | |
| # TODO(nnorwitz): warn about using auto? Probably not since it | |
| # will be reclaimed and useful for C++0x. | |
| pass | |
| def handle_register(self): | |
| pass | |
| def handle_const(self): | |
| pass | |
| def handle_inline(self): | |
| pass | |
| def handle_extern(self): | |
| pass | |
| def handle_static(self): | |
| pass | |
| def handle_virtual(self): | |
| # What follows must be a method. | |
| token = token2 = self._GetNextToken() | |
| if token.name == 'inline': | |
| # HACK(nnorwitz): handle inline dtors by ignoring 'inline'. | |
| token2 = self._GetNextToken() | |
| if token2.token_type == tokenize.SYNTAX and token2.name == '~': | |
| return self.GetMethod(FUNCTION_VIRTUAL + FUNCTION_DTOR, None) | |
| assert token.token_type == tokenize.NAME or token.name == '::', token | |
| return_type_and_name = self._GetTokensUpTo(tokenize.SYNTAX, '(') # ) | |
| return_type_and_name.insert(0, token) | |
| if token2 is not token: | |
| return_type_and_name.insert(1, token2) | |
| return self._GetMethod(return_type_and_name, FUNCTION_VIRTUAL, | |
| None, False) | |
| def handle_volatile(self): | |
| pass | |
| def handle_mutable(self): | |
| pass | |
| def handle_public(self): | |
| assert self.in_class | |
| self.visibility = VISIBILITY_PUBLIC | |
| def handle_protected(self): | |
| assert self.in_class | |
| self.visibility = VISIBILITY_PROTECTED | |
| def handle_private(self): | |
| assert self.in_class | |
| self.visibility = VISIBILITY_PRIVATE | |
| def handle_friend(self): | |
| tokens = self._GetTokensUpTo(tokenize.SYNTAX, ';') | |
| assert tokens | |
| t0 = tokens[0] | |
| return Friend(t0.start, t0.end, tokens, self.namespace_stack) | |
| def handle_static_cast(self): | |
| pass | |
| def handle_const_cast(self): | |
| pass | |
| def handle_dynamic_cast(self): | |
| pass | |
| def handle_reinterpret_cast(self): | |
| pass | |
| def handle_new(self): | |
| pass | |
| def handle_delete(self): | |
| tokens = self._GetTokensUpTo(tokenize.SYNTAX, ';') | |
| assert tokens | |
| return Delete(tokens[0].start, tokens[0].end, tokens) | |
| def handle_typedef(self): | |
| token = self._GetNextToken() | |
| if (token.token_type == tokenize.NAME and | |
| keywords.IsKeyword(token.name)): | |
| # Token must be struct/enum/union/class. | |
| method = getattr(self, 'handle_' + token.name) | |
| self._handling_typedef = True | |
| tokens = [method()] | |
| self._handling_typedef = False | |
| else: | |
| tokens = [token] | |
| # Get the remainder of the typedef up to the semi-colon. | |
| tokens.extend(self._GetTokensUpTo(tokenize.SYNTAX, ';')) | |
| # TODO(nnorwitz): clean all this up. | |
| assert tokens | |
| name = tokens.pop() | |
| indices = name | |
| if tokens: | |
| indices = tokens[0] | |
| if not indices: | |
| indices = token | |
| if name.name == ')': | |
| # HACK(nnorwitz): Handle pointers to functions "properly". | |
| if (len(tokens) >= 4 and | |
| tokens[1].name == '(' and tokens[2].name == '*'): | |
| tokens.append(name) | |
| name = tokens[3] | |
| elif name.name == ']': | |
| # HACK(nnorwitz): Handle arrays properly. | |
| if len(tokens) >= 2: | |
| tokens.append(name) | |
| name = tokens[1] | |
| new_type = tokens | |
| if tokens and isinstance(tokens[0], tokenize.Token): | |
| new_type = self.converter.ToType(tokens)[0] | |
| return Typedef(indices.start, indices.end, name.name, | |
| new_type, self.namespace_stack) | |
| def handle_typeid(self): | |
| pass # Not needed yet. | |
| def handle_typename(self): | |
| pass # Not needed yet. | |
| def _GetTemplatedTypes(self): | |
| result = {} | |
| tokens = list(self._GetMatchingChar('<', '>')) | |
| len_tokens = len(tokens) - 1 # Ignore trailing '>'. | |
| i = 0 | |
| while i < len_tokens: | |
| key = tokens[i].name | |
| i += 1 | |
| if keywords.IsKeyword(key) or key == ',': | |
| continue | |
| type_name = default = None | |
| if i < len_tokens: | |
| i += 1 | |
| if tokens[i-1].name == '=': | |
| assert i < len_tokens, '%s %s' % (i, tokens) | |
| default, unused_next_token = self.GetName(tokens[i:]) | |
| i += len(default) | |
| else: | |
| if tokens[i-1].name != ',': | |
| # We got something like: Type variable. | |
| # Re-adjust the key (variable) and type_name (Type). | |
| key = tokens[i-1].name | |
| type_name = tokens[i-2] | |
| result[key] = (type_name, default) | |
| return result | |
| def handle_template(self): | |
| token = self._GetNextToken() | |
| assert token.token_type == tokenize.SYNTAX, token | |
| assert token.name == '<', token | |
| templated_types = self._GetTemplatedTypes() | |
| # TODO(nnorwitz): for now, just ignore the template params. | |
| token = self._GetNextToken() | |
| if token.token_type == tokenize.NAME: | |
| if token.name == 'class': | |
| return self._GetClass(Class, VISIBILITY_PRIVATE, templated_types) | |
| elif token.name == 'struct': | |
| return self._GetClass(Struct, VISIBILITY_PUBLIC, templated_types) | |
| elif token.name == 'friend': | |
| return self.handle_friend() | |
| self._AddBackToken(token) | |
| tokens, last = self._GetVarTokensUpTo(tokenize.SYNTAX, '(', ';') | |
| tokens.append(last) | |
| self._AddBackTokens(tokens) | |
| if last.name == '(': | |
| return self.GetMethod(FUNCTION_NONE, templated_types) | |
| # Must be a variable definition. | |
| return None | |
| def handle_true(self): | |
| pass # Nothing to do. | |
| def handle_false(self): | |
| pass # Nothing to do. | |
| def handle_asm(self): | |
| pass # Not needed yet. | |
| def handle_class(self): | |
| return self._GetClass(Class, VISIBILITY_PRIVATE, None) | |
| def _GetBases(self): | |
| # Get base classes. | |
| bases = [] | |
| while 1: | |
| token = self._GetNextToken() | |
| assert token.token_type == tokenize.NAME, token | |
| # TODO(nnorwitz): store kind of inheritance...maybe. | |
| if token.name not in ('public', 'protected', 'private'): | |
| # If inheritance type is not specified, it is private. | |
| # Just put the token back so we can form a name. | |
| # TODO(nnorwitz): it would be good to warn about this. | |
| self._AddBackToken(token) | |
| else: | |
| # Check for virtual inheritance. | |
| token = self._GetNextToken() | |
| if token.name != 'virtual': | |
| self._AddBackToken(token) | |
| else: | |
| # TODO(nnorwitz): store that we got virtual for this base. | |
| pass | |
| base, next_token = self.GetName() | |
| bases_ast = self.converter.ToType(base) | |
| assert len(bases_ast) == 1, bases_ast | |
| bases.append(bases_ast[0]) | |
| assert next_token.token_type == tokenize.SYNTAX, next_token | |
| if next_token.name == '{': | |
| token = next_token | |
| break | |
| # Support multiple inheritance. | |
| assert next_token.name == ',', next_token | |
| return bases, token | |
| def _GetClass(self, class_type, visibility, templated_types): | |
| class_name = None | |
| class_token = self._GetNextToken() | |
| if class_token.token_type != tokenize.NAME: | |
| assert class_token.token_type == tokenize.SYNTAX, class_token | |
| token = class_token | |
| else: | |
| # Skip any macro (e.g. storage class specifiers) after the | |
| # 'class' keyword. | |
| next_token = self._GetNextToken() | |
| if next_token.token_type == tokenize.NAME: | |
| self._AddBackToken(next_token) | |
| else: | |
| self._AddBackTokens([class_token, next_token]) | |
| name_tokens, token = self.GetName() | |
| class_name = ''.join([t.name for t in name_tokens]) | |
| bases = None | |
| if token.token_type == tokenize.SYNTAX: | |
| if token.name == ';': | |
| # Forward declaration. | |
| return class_type(class_token.start, class_token.end, | |
| class_name, None, templated_types, None, | |
| self.namespace_stack) | |
| if token.name in '*&': | |
| # Inline forward declaration. Could be method or data. | |
| name_token = self._GetNextToken() | |
| next_token = self._GetNextToken() | |
| if next_token.name == ';': | |
| # Handle data | |
| modifiers = ['class'] | |
| return self._CreateVariable(class_token, name_token.name, | |
| class_name, | |
| modifiers, token.name, None) | |
| else: | |
| # Assume this is a method. | |
| tokens = (class_token, token, name_token, next_token) | |
| self._AddBackTokens(tokens) | |
| return self.GetMethod(FUNCTION_NONE, None) | |
| if token.name == ':': | |
| bases, token = self._GetBases() | |
| body = None | |
| if token.token_type == tokenize.SYNTAX and token.name == '{': | |
| assert token.token_type == tokenize.SYNTAX, token | |
| assert token.name == '{', token | |
| ast = AstBuilder(self.GetScope(), self.filename, class_name, | |
| visibility, self.namespace_stack) | |
| body = list(ast.Generate()) | |
| if not self._handling_typedef: | |
| token = self._GetNextToken() | |
| if token.token_type != tokenize.NAME: | |
| assert token.token_type == tokenize.SYNTAX, token | |
| assert token.name == ';', token | |
| else: | |
| new_class = class_type(class_token.start, class_token.end, | |
| class_name, bases, None, | |
| body, self.namespace_stack) | |
| modifiers = [] | |
| return self._CreateVariable(class_token, | |
| token.name, new_class, | |
| modifiers, token.name, None) | |
| else: | |
| if not self._handling_typedef: | |
| self.HandleError('non-typedef token', token) | |
| self._AddBackToken(token) | |
| return class_type(class_token.start, class_token.end, class_name, | |
| bases, templated_types, body, self.namespace_stack) | |
| def handle_namespace(self): | |
| token = self._GetNextToken() | |
| # Support anonymous namespaces. | |
| name = None | |
| if token.token_type == tokenize.NAME: | |
| name = token.name | |
| token = self._GetNextToken() | |
| self.namespace_stack.append(name) | |
| assert token.token_type == tokenize.SYNTAX, token | |
| # Create an internal token that denotes when the namespace is complete. | |
| internal_token = tokenize.Token(_INTERNAL_TOKEN, _NAMESPACE_POP, | |
| None, None) | |
| internal_token.whence = token.whence | |
| if token.name == '=': | |
| # TODO(nnorwitz): handle aliasing namespaces. | |
| name, next_token = self.GetName() | |
| assert next_token.name == ';', next_token | |
| self._AddBackToken(internal_token) | |
| else: | |
| assert token.name == '{', token | |
| tokens = list(self.GetScope()) | |
| # Replace the trailing } with the internal namespace pop token. | |
| tokens[-1] = internal_token | |
| # Handle namespace with nothing in it. | |
| self._AddBackTokens(tokens) | |
| return None | |
| def handle_using(self): | |
| tokens = self._GetTokensUpTo(tokenize.SYNTAX, ';') | |
| assert tokens | |
| return Using(tokens[0].start, tokens[0].end, tokens) | |
| def handle_explicit(self): | |
| assert self.in_class | |
| # Nothing much to do. | |
| # TODO(nnorwitz): maybe verify the method name == class name. | |
| # This must be a ctor. | |
| return self.GetMethod(FUNCTION_CTOR, None) | |
| def handle_this(self): | |
| pass # Nothing to do. | |
| def handle_operator(self): | |
| # Pull off the next token(s?) and make that part of the method name. | |
| pass | |
| def handle_sizeof(self): | |
| pass | |
| def handle_case(self): | |
| pass | |
| def handle_switch(self): | |
| pass | |
| def handle_default(self): | |
| token = self._GetNextToken() | |
| assert token.token_type == tokenize.SYNTAX | |
| assert token.name == ':' | |
| def handle_if(self): | |
| pass | |
| def handle_else(self): | |
| pass | |
| def handle_return(self): | |
| tokens = self._GetTokensUpTo(tokenize.SYNTAX, ';') | |
| if not tokens: | |
| return Return(self.current_token.start, self.current_token.end, None) | |
| return Return(tokens[0].start, tokens[0].end, tokens) | |
| def handle_goto(self): | |
| tokens = self._GetTokensUpTo(tokenize.SYNTAX, ';') | |
| assert len(tokens) == 1, str(tokens) | |
| return Goto(tokens[0].start, tokens[0].end, tokens[0].name) | |
| def handle_try(self): | |
| pass # Not needed yet. | |
| def handle_catch(self): | |
| pass # Not needed yet. | |
| def handle_throw(self): | |
| pass # Not needed yet. | |
| def handle_while(self): | |
| pass | |
| def handle_do(self): | |
| pass | |
| def handle_for(self): | |
| pass | |
| def handle_break(self): | |
| self._IgnoreUpTo(tokenize.SYNTAX, ';') | |
| def handle_continue(self): | |
| self._IgnoreUpTo(tokenize.SYNTAX, ';') | |
| def BuilderFromSource(source, filename): | |
| """Utility method that returns an AstBuilder from source code. | |
| Args: | |
| source: 'C++ source code' | |
| filename: 'file1' | |
| Returns: | |
| AstBuilder | |
| """ | |
| return AstBuilder(tokenize.GetTokens(source), filename) | |
| def PrintIndentifiers(filename, should_print): | |
| """Prints all identifiers for a C++ source file. | |
| Args: | |
| filename: 'file1' | |
| should_print: predicate with signature: bool Function(token) | |
| """ | |
| source = utils.ReadFile(filename, False) | |
| if source is None: | |
| sys.stderr.write('Unable to find: %s\n' % filename) | |
| return | |
| #print('Processing %s' % actual_filename) | |
| builder = BuilderFromSource(source, filename) | |
| try: | |
| for node in builder.Generate(): | |
| if should_print(node): | |
| print(node.name) | |
| except KeyboardInterrupt: | |
| return | |
| except: | |
| pass | |
| def PrintAllIndentifiers(filenames, should_print): | |
| """Prints all identifiers for each C++ source file in filenames. | |
| Args: | |
| filenames: ['file1', 'file2', ...] | |
| should_print: predicate with signature: bool Function(token) | |
| """ | |
| for path in filenames: | |
| PrintIndentifiers(path, should_print) | |
| def main(argv): | |
| for filename in argv[1:]: | |
| source = utils.ReadFile(filename) | |
| if source is None: | |
| continue | |
| print('Processing %s' % filename) | |
| builder = BuilderFromSource(source, filename) | |
| try: | |
| entire_ast = filter(None, builder.Generate()) | |
| except KeyboardInterrupt: | |
| return | |
| except: | |
| # Already printed a warning, print the traceback and continue. | |
| traceback.print_exc() | |
| else: | |
| if utils.DEBUG: | |
| for ast in entire_ast: | |
| print(ast) | |
| if __name__ == '__main__': | |
| main(sys.argv) |