astng.py

download

# Copyright (c) 2003 Sylvain Thenault (thenault@nerim.net)
# Copyright (c) 2003 Logilab
#
# This program is free software; you can redistribute it and/or modify it under
# the terms of the GNU General Public License as published by the Free Software
# Foundation; either version 2 of the License, or (at your option) any later
# version.
#
# This program is distributed in the hope that it will be useful, but WITHOUT
# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
# FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License along with
# this program; if not, write to the Free Software Foundation, Inc.,
# 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
""" Python Abstract Syntax Tree New Generation

The aim of this module is to provide a common base representation for projects
such as pychecker, pyreverse, pylint...

TODO:
_ partially parse missing statements (import, expression)
_ get locals for modules / classes ?
"""

__author__ = u"Sylvain Thenault"
__revision__ = "$Id: astng.py,v 1.12 2003/10/02 06:37:26 syt Exp $"

import parser
import token
import symbol
from types import ListType, TupleType
from inspect import getmembers, isfunction, ismethod, isclass
from os.path import splitext
from logilab.common.fileutils import norm_read

# for debugging purpose
#from pprint import pprint
#from logilab.pypasax.pypasax import cvrtr

from logilab.common.astng.astng_objects import *

def is_string(tuple):
""" given an expr_stmt tuple, return the string value if the expression is
a single string, else None
"""
if len(tuple) > 3:
return
if tuple[0] == token.STRING:
return tuple[1]
if type(tuple[1]) in (ListType, TupleType):
return is_string(tuple[1])

def get_line(tuple):
""" given an ast tuple return the line where this tuple begin
"""
if type(tuple[1]) in (ListType, TupleType):
return get_line(tuple[1])
else:
return tuple[-1]

class SkipChildren(Exception): pass

yield_stmt = hasattr(symbol, 'yield_stmt' ) and symbol.yield_stmt or None

CLASSES = {
symbol.file_input : Module,

symbol.classdef : ClassDef,
symbol.funcdef : FuncDef,

symbol.import_stmt : ImportStmt,
symbol.raise_stmt : RaiseStmt,
symbol.print_stmt : PrintStmt,
symbol.assert_stmt : AssertStmt,
symbol.global_stmt : GlobalStmt,
symbol.del_stmt : DelStmt,
symbol.expr_stmt : ExpressionStmt,
symbol.return_stmt : ReturnStmt,
symbol.try_stmt : TryStmt,
symbol.pass_stmt : PassStmt,
symbol.continue_stmt : ContinueStmt,
symbol.break_stmt : BreakStmt,
symbol.exec_stmt : ExecStmt,
symbol.while_stmt : WhileStmt,

symbol.except_clause: ExceptStmt,

symbol.if_stmt: IfStmt,
symbol.for_stmt: ForStmt,
yield_stmt: YieldStmt,
'elif' : ElifStmt,
'else' : ElseStmt,
'finally': FinallyStmt,
'string': StringStmt,
'assignment': AssignmentStmt,
}

class ASTNGBuildingException(Exception):
""" exception class when we are not able to build an astng representation
"""

def normalize_module_name(mod_name):
""" normalize a module name (i.e remove trailing __init__ if any)
"""
parts = mod_name.split('.')
if parts[-1] == '__init__':
return '.'.join(parts[:-1])
return mod_name

class ASTNGBuilder:
def __init__(self):
self.HOOKS = {
symbol.file_input : self.init_module,

symbol.classdef : self.init_object,
symbol.funcdef : self.init_object,
symbol.import_stmt : self.init_object,
symbol.exec_stmt : self.init_object,
symbol.assert_stmt : self.init_object,
symbol.raise_stmt : self.init_object,
symbol.print_stmt : self.init_object,
symbol.return_stmt : self.init_object,
symbol.global_stmt : self.init_object,
symbol.try_stmt : self.init_object,
symbol.pass_stmt: self.init_object,
symbol.continue_stmt: self.init_object,
symbol.break_stmt : self.init_object,
symbol.for_stmt: self.init_object,
symbol.except_clause: self.init_object,
yield_stmt: self.init_object,

symbol.if_stmt: self.init_flow_object,
symbol.while_stmt : self.init_flow_object,

symbol.expr_stmt : self.init_expression_object,

token.NAME : self.handle_name,

symbol.suite : self.indent,
token.DEDENT : self.dedent,
token.NEWLINE : self.inc_line,
}
self._extra_args = {}

def build_from_module(self, module, mod_name=None):
""" build an astng from a living module instance
"""
is_pure_python = 0
path = getattr(module, '__file__', None)
if path is not None:
path, ext = splitext(module.__file__)
if ext in ('.py', '.pyc', '.pyo'):
path = path + '.py'
is_pure_python = 1

if mod_name is None:
# guess the module's name
mod_name = module.__name__

mod_name = normalize_module_name(mod_name)
self._obj_count = 0
self._module = module
if is_pure_python:
# pure python module
# open file
try:
data = norm_read(path, '\n')
except IOError, e:
msg = 'Unable to load file %r (%s)' % (path, e)
raise ASTNGBuildingException(msg)
# build astng representation
#try:
astng = self._string_build(data)
#except Exception, e:
# msg = 'Unable to parse file %s (%s)' % (path, e)
# raise ASTNGBuildingException(msg)
else:
# this is a built-in module
# get a partial representation by introspection
astng = self._inspect_build(module)
astng.pure_python = is_pure_python
astng.file = path
return astng

def _inspect_build(self, module):
""" build astng from a living module (i.e. using inspect)
"""
return self._member_build(Module(module), module)

def _member_build(self, astng, object, done=None):
""" recursive method which create a partial ast from real objects
(only function, class, and method are handled)
"""
if done is None:
done = {}
if done.has_key(object):
return astng
done[object] = 1
for name, member in getmembers(object):
if ismethod(member):
member = member.im_func
if isfunction(member):
# verify this is not an imported function
code = member.func_code
modfile = getattr(self._module, '__file__', None)
if code.co_filename == modfile:
func = FuncDef(self._obj_count, code.co_firstlineno,
object=member)
astng.append(func)
self._obj_count += 1
elif isclass(member):
# verify this is not an imported classe
if member.__module__ == self._module.__name__:
classe = ClassDef(self._obj_count, 0, object=member)
astng.append(classe)
self._obj_count += 1
self._member_build(classe, member, done)
return astng

def _string_build(self, data):
""" build astng from a stream (i.e. from an ast)
"""
self.stack = []
self.last = None
self.need_dedent = 0
ast = parser.suite(data + '\n')
self._root = None
self._ast_build(ast.totuple(line_info=1))
return self._root

def _ast_build(self, tuple):
""" recurse on the ast to build an astng """
if tuple and type(tuple) in (TupleType, ListType) :
try:
self.HOOKS[tuple[0]](tuple)
except SkipChildren, e:
return
except KeyError:
pass
map(self._ast_build, tuple[1:])

# ast hooks ################################################################

def init_module(self, tuple):
obj = self.init_object(tuple)
self._root = obj
self.stack.append((obj, self._module))
return obj

def init_expression_object(self, tuple):
if is_string(tuple):
# pure string statement (usually docstring)
obj = self.init_object(('string', tuple))
elif len(tuple) > 2 and tuple[2][0] == token.EQUAL:
# assignment
obj = self.init_object(('assignment', tuple))
else:
obj = self.init_object(tuple)

def init_flow_object(self, tuple):
obj = self.init_object(tuple)
self._ast_build(tuple[4])
suite = tuple[5:]
for i in range(len(suite)):
tuple = suite[i]
if tuple[0] == token.NAME:
if tuple[1] == 'elif':
obj = self.init_object(('elif', suite[i+1]))
elif tuple[1] == 'else':
obj = self.init_object(('else', tuple))
elif tuple[0] == symbol.suite:
self._ast_build(tuple)
raise SkipChildren()

def init_object(self, tuple):
classe = CLASSES[tuple[0]]
if classe is Module:
obj = classe(self._module)
else:
self._obj_count += 1
if classe in (ClassDef, FuncDef):
obj = classe(self._obj_count, 0, name=tuple[2][1])
else:
obj = classe(self._obj_count, 0)
if self.stack:
self.stack[-1][0].append(obj)
if type(tuple[0]) == type(''):
obj.decode(tuple[1])
else:
obj.decode(tuple)
obj.line = get_line(tuple)
self.last = (obj, getattr(obj, 'object', None))
return obj

def handle_name(self, tuple):
name = tuple[1]
if name == 'else':
self.init_object(('else', tuple))
elif name == 'finally':
self.init_object(('finally', tuple))

def indent(self, tuple):
assert self.last
assert not isinstance(self.last[0], list)
self.stack.append(self.last)
self.last = None
if tuple[-1][0] != token.DEDENT:
self.need_dedent = 1

def dedent(self, tuple=None):
self.stack and self.stack.pop()
self.last = None

def inc_line(self, tuple):
if self.need_dedent:
self.need_dedent = 0
self.dedent()

download