# Copyright (c) 2003-2005 Sylvain Thenault (thenault@nerim.net) # Copyright (c) 2003-2005 Logilab # # This program is free software; you can redistribute it and/or modify it under # the terms of the GNU General Public License as published by the Free Software # Foundation; either version 2 of the License, or (at your option) any later # version. # # This program is distributed in the hope that it will be useful, but WITHOUT # ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS # FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. # # You should have received a copy of the GNU General Public License along with # this program; if not, write to the Free Software Foundation, Inc., # 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. """The ASTNGBuilder makes astng from living object and / or from compiler.ast The builder is not thread safe and can't be used to parse different sources at the same time. TODO: - more complet representation on inspect build (imported modules ? use dis.dis ?) """ __author__ = "Sylvain Thenault" __revision__ = "$Id: builder.py,v 1.23 2005/01/18 17:33:06 syt Exp $" import sys from compiler import parse from inspect import isfunction, ismethod, ismethoddescriptor, isclass, \ isbuiltin, getargspec from os.path import splitext, basename, dirname, exists from logilab.common.fileutils import norm_read from logilab.common.modutils import modpath_from_file from logilab.common import astng from logilab.common.astng.raw_building import register_arguments, \ build_module, build_function, build_class def _init_module(node): """init a module node""" node.parent = None node.globals = node.locals = {} # ast NG builder ############################################################## class ASTNGBuilder: """provide astng building methods """ def __init__(self, manager): self._manager = manager self._module = None self._done = None self._stack = None self._walker = astng.ASTWalker(self) def build_from_module(self, module, mod_name=None): """build an astng from a living module instance """ node = None path = getattr(module, '__file__', None) self._module = module if path is not None: path_, ext = splitext(module.__file__) if ext in ('.py', '.pyc', '.pyo') and exists(path_ + '.py'): path = path_ + '.py' node = self.file_build(path) if node is None: # this is a built-in module # get a partial representation by introspection node = self.inspect_build(module) node.file = path node.name = mod_name or module.__name__ node.package = hasattr(module, '__path__') return node def file_build(self, path, modname=None): """build astng from a source code file (i.e. from an ast) path is expected to be a python source file """ try: data = norm_read(path, '\n') except IOError, ex: msg = 'Unable to load file %r (%s)' % (path, ex) raise astng.ASTNGBuildingException(msg) # build astng representation try: sys.path.insert(0, dirname(path)) node = self.string_build(data) finally: sys.path.pop(0) node.file = node.path = path if modname is None: try: modname = '.'.join(modpath_from_file(path)) except: modname = splitext(basename(path))[0] node.name = modname node.package = path.find('__init__') > -1 return node def string_build(self, data): """build astng from a source code stream (i.e. from an ast)""" return self.ast_build(parse(data + '\n')) # astng from ast ########################################################## def ast_build(self, node): """recurse on the ast (soon ng) to add some arguments et method """ node.pure_python = 1 self._walker.walk(node) return node def visit_module(self, node): """visit a stmt.Module node -> init node and push the corresponding object or None on the top of the stack """ self._stack = [self._module] self._par_stack = [node] node.object = self._module _init_module(node) def leave_module(self, node): """leave a stmt.Module node -> pop the last item on the stack and check the stack is empty """ self._stack.pop() assert not self._stack, 'Stack is not empty : %s' % self._stack self._par_stack.pop() assert not self._par_stack, \ 'Parent stack is not empty : %s' % self._par_stack def visit_class(self, node): """visit a stmt.Class node -> init node and push the corresponding object or None on the top of the stack """ self.visit_default(node) node.instance_attrs = {} node.basenames = [b_node.as_string() for b_node in node.bases] self._push(node) def leave_class(self, node): """leave a stmt.Class node -> pop the last item on the stack """ self.leave_default(node) self._stack.pop() def visit_function(self, node): """visit a stmt.Function node -> init node and push the corresponding object or None on the top of the stack """ self.visit_default(node) node.argnames = list(node.argnames) if node.name == '__new__': node.type = 'classmethod' self._push(node) obj = node.object if hasattr(obj, 'im_func'): node.object = obj.im_func register_arguments(node, node.argnames) #assert not obj or hasattr(node.object, 'func_defaults'), \ # '%s %s %s' % (node, node.object.__class__, dir(node.object)) def leave_function(self, node): """leave a stmt.Function node -> pop the last item on the stack """ self.leave_default(node) self._stack.pop() def visit_lambda(self, node): """visit a stmt.Lambda node -> init node locals """ self.visit_default(node) node.argnames = list(node.argnames) node.locals = {} register_arguments(node, node.argnames) def visit_global(self, node): """visit a stmt.Global node -> add declared names to locals """ self.visit_default(node) for name in node.names: node.parent.set_local(name, node) def visit_import(self, node): """visit a stmt.Import node -> add imported names to locals """ self.visit_default(node) for (name, asname) in node.names: name = asname or name node.parent.set_local(name.split('.')[0], node) def visit_from(self, node): """visit a stmt.From node -> add imported names to locals """ self.visit_default(node) # add names imported by wildcard import for (name, asname) in node.names: if name == '*': try: imported = self._manager.astng_from_module_name(node.modname) for name in imported.wildcard_import_names(): node.parent.set_local(name, node) except: import traceback traceback.print_exc() print >> sys.stderr, \ 'Unable to get imported names for %r line %s"' % ( node.modname, node.lineno) else: node.parent.set_local(asname or name, node) def visit_assign(self, node): """visit a stmt.Assign node -> check for classmethod and staticmethod """ self.visit_default(node) klass = node.parent.get_frame() if isinstance(klass, astng.Class) and \ isinstance(node.expr, astng.CallFunc) and \ isinstance(node.expr.node, astng.Name): name = node.expr.node.name if name in ('classmethod', 'staticmethod'): for ass_node in node.nodes: if isinstance(ass_node, astng.AssName): try: meth = klass.locals[ass_node.name] if isinstance(meth, astng.Function): meth.type = name else: print >> sys.stderr, 'FIXME 1', meth except Exception: print >> sys.stderr, 'FIXME 2', ass_node.name continue def visit_assname(self, node): """visit a stmt.AssName node -> add name to locals """ self.visit_default(node) if node.flags == 'OP_ASSIGN': node.parent.set_local(node.name, node) def visit_assattr(self, node): """visit a stmt.AssAttr node -> add name to locals, handle members definition """ self.visit_default(node) #node.parent.set_local(node.name, node) frame = node.get_frame() if isinstance(frame, astng.Function) and frame.is_method(): klass = frame.parent.get_frame() # are we assigning to a (new ?) instance attribute ? _self = frame.argnames[0] if isinstance(node.expr, astng.Name) and node.expr.name == _self: iattrs = klass.instance_attrs # assign if not yet existant in others if not iattrs.has_key(node.attrname): iattrs[node.attrname] = node # but always assign in __init__, except if previous assigment # already come from __init__ elif frame.name == '__init__' and not \ iattrs[node.attrname].get_frame().name == '__init__': klass.instance_attrs[node.attrname] = node def visit_default(self, node): """default visit method, handle the parent attribute """ node.parent = self._par_stack[-1] assert node.parent is not node self._par_stack.append(node) def leave_default(self, node): """default leave method, handle the parent attribute """ self._par_stack.pop() def _push(self, node): """update the stack and init some parts of the Function or Class node """ obj = getattr(self._stack[-1], node.name, None) self._stack.append(obj) node.object = obj node.locals = {} node.parent.get_frame().set_local(node.name, node) # astng from living objects ############################################### # # this is actually a really minimal representation, including only Module, # Function and Class nodes def inspect_build(self, module): """build astng from a living module (i.e. using inspect) this is used when there is no python source code available (either because it's a built-in module or because the .py is not available) """ node = build_module(module.__name__, module.__doc__) node.object = module self._done = {} if module.__name__ == 'qt': #print 'hanlding qt !' #print module.QWidget.__name__, module.QWidget.__module__ self._qt_member_build(node, module) else: self._member_build(node, module) return node def _member_build(self, node, obj): """recursive method which create a partial ast from real objects (only function, class, and method are handled) """ if self._done.has_key(obj): return node self._done[obj] = 1 modname = self._module.__name__ modfile = getattr(self._module, '__file__', None) for name in dir(obj): try: member = getattr(obj, name) except AttributeError: # damned ExtensionClass.Base, I know you're there ! continue if ismethod(member): member = member.im_func if isfunction(member): # verify this is not an imported function if member.func_code.co_filename != modfile: continue self._function_member_build(node, member) elif ismethoddescriptor(member): self._methoddescriptor_member_build(node, member) elif isbuiltin(member): # verify this is not an imported member if getattr(member, '__module__', None) != modname: continue self._builtin_member_build(node, member) elif isclass(member): # verify this is not an imported class # # /!\ some classes like ExtensionClass doesn't have a # __module__ attribute ! if getattr(member, '__module__', None) != modname: continue klass = self._class_member_build(node, member) # recursion self._member_build(klass, member) def _class_member_build(self, node, member): """create astng for a living class object""" basenames = [base.__name__ for base in member.__bases__] klass = build_class(member.__name__, basenames, member.__doc__) klass.object = member node.add_local_node(klass) return klass def _function_member_build(self, node, member): """create astng for a living function object""" args, varargs, varkw, defaults = getargspec(member) if varargs is not None: args.append(varargs) if varkw is not None: args.append(varkw) func = build_function(member.__name__, args, defaults, member.func_code.co_flags, member.__doc__) func.object = member node.add_local_node(func) def _methoddescriptor_member_build(self, node, member): """create astng for a living method descriptor object""" # FIXME get arguments ? func = build_function(member.__name__, doc=member.__doc__) func.object = member node.add_local_node(func) _builtin_member_build = _methoddescriptor_member_build ## def _builtin_member_build(self, node, member): ## """create astng for a living method descriptor object""" ## # FIXME get arguments ? ## func = build_function(member.__name__, doc=member.__doc__) ## func.object = member ## node.add_local_node(func) def _qt_member_build(self, node, obj): """recursive method which create a partial ast from real objects (only function, class, and method are handled) """ if self._done.has_key(obj): return node self._done[obj] = 1 # added __main__ to the list to fix problem with qt # (qt.QWidget.__module__ == '__main__') ! # and finally also added modutils since __module__ attribute on qt # classes seems to take the name of the importing module # FIXME: bug report on pyqt to remove this special handling... modname = self._module.__name__ modules = (modname, '__main__', 'logilab.common.modutils') modfile = getattr(self._module, '__file__', None) for name in dir(obj): try: member = getattr(obj, name) except AttributeError: # damned ExtensionClass.Base, I know you're there ! continue if ismethod(member): member = member.im_func if isfunction(member): # verify this is not an imported function if member.func_code.co_filename != modfile: continue self._function_member_build(node, member) elif ismethoddescriptor(member): self._methoddescriptor_member_build(node, member) elif isbuiltin(member): # verify this is not an imported member if not (member.__module__ is None or member.__module__ in modules): print 'skipping', member, member.__module__, modules continue self._builtin_member_build(node, member) elif isclass(member): # verify this is not an imported class if not member.__module__ in modules: continue # grrr: qt.QWidget.__name__ == 'qt.QWidget' ! member.__name__ = member.__name__.split('.')[-1] klass = self._class_member_build(node, member) # recursion self._qt_member_build(klass, member)
name
downloadbuilder.py
builder.py