2009-10-20 10:52:23 +00:00
# -*- coding: utf-8 -*-
2009-09-17 07:28:14 +00:00
# ast.py
# Copyright (C) Mako developers
#
# This module is part of Mako and is released under
# the MIT License: http://www.opensource.org/licenses/mit-license.php
""" Handles parsing of Python code.
Parsing to AST is done via _ast on Python > 2.5 , otherwise the compiler
module is used .
"""
from StringIO import StringIO
from mako import exceptions , util
# words that cannot be assigned to (notably smaller than the total keys in __builtins__)
reserved = util . Set ( [ ' True ' , ' False ' , ' None ' ] )
try :
import _ast
util . restore__ast ( _ast )
import _ast_util
except ImportError :
_ast = None
from compiler import parse as compiler_parse
from compiler import visitor
def parse ( code , mode = ' exec ' , * * exception_kwargs ) :
""" Parse an expression into AST """
try :
if _ast :
return _ast_util . parse ( code , ' <unknown> ' , mode )
else :
return compiler_parse ( code , mode )
except Exception , e :
raise exceptions . SyntaxException ( " ( %s ) %s ( %s ) " % ( e . __class__ . __name__ , str ( e ) , repr ( code [ 0 : 50 ] ) ) , * * exception_kwargs )
if _ast :
class FindIdentifiers ( _ast_util . NodeVisitor ) :
def __init__ ( self , listener , * * exception_kwargs ) :
self . in_function = False
self . in_assign_targets = False
self . local_ident_stack = { }
self . listener = listener
self . exception_kwargs = exception_kwargs
def _add_declared ( self , name ) :
if not self . in_function :
self . listener . declared_identifiers . add ( name )
def visit_ClassDef ( self , node ) :
self . _add_declared ( node . name )
def visit_Assign ( self , node ) :
# flip around the visiting of Assign so the expression gets evaluated first,
# in the case of a clause like "x=x+5" (x is undeclared)
self . visit ( node . value )
in_a = self . in_assign_targets
self . in_assign_targets = True
for n in node . targets :
self . visit ( n )
self . in_assign_targets = in_a
def visit_FunctionDef ( self , node ) :
self . _add_declared ( node . name )
# push function state onto stack. dont log any
# more identifiers as "declared" until outside of the function,
# but keep logging identifiers as "undeclared".
# track argument names in each function header so they arent counted as "undeclared"
saved = { }
inf = self . in_function
self . in_function = True
for arg in node . args . args :
if arg . id in self . local_ident_stack :
saved [ arg . id ] = True
else :
self . local_ident_stack [ arg . id ] = True
for n in node . body :
self . visit ( n )
self . in_function = inf
for arg in node . args . args :
if arg . id not in saved :
del self . local_ident_stack [ arg . id ]
def visit_For ( self , node ) :
# flip around visit
self . visit ( node . iter )
self . visit ( node . target )
for statement in node . body :
self . visit ( statement )
for statement in node . orelse :
self . visit ( statement )
def visit_Name ( self , node ) :
if isinstance ( node . ctx , _ast . Store ) :
self . _add_declared ( node . id )
if node . id not in reserved and node . id not in self . listener . declared_identifiers and node . id not in self . local_ident_stack :
self . listener . undeclared_identifiers . add ( node . id )
def visit_Import ( self , node ) :
for name in node . names :
if name . asname is not None :
self . _add_declared ( name . asname )
else :
self . _add_declared ( name . name . split ( ' . ' ) [ 0 ] )
def visit_ImportFrom ( self , node ) :
for name in node . names :
if name . asname is not None :
self . _add_declared ( name . asname )
else :
if name . name == ' * ' :
raise exceptions . CompileException ( " ' import * ' is not supported, since all identifier names must be explicitly declared. Please use the form ' from <modulename> import <name1>, <name2>, ... ' instead. " , * * self . exception_kwargs )
self . _add_declared ( name . name )
class FindTuple ( _ast_util . NodeVisitor ) :
def __init__ ( self , listener , code_factory , * * exception_kwargs ) :
self . listener = listener
self . exception_kwargs = exception_kwargs
self . code_factory = code_factory
def visit_Tuple ( self , node ) :
for n in node . elts :
p = self . code_factory ( n , * * self . exception_kwargs )
self . listener . codeargs . append ( p )
self . listener . args . append ( ExpressionGenerator ( n ) . value ( ) )
self . listener . declared_identifiers = self . listener . declared_identifiers . union ( p . declared_identifiers )
self . listener . undeclared_identifiers = self . listener . undeclared_identifiers . union ( p . undeclared_identifiers )
class ParseFunc ( _ast_util . NodeVisitor ) :
def __init__ ( self , listener , * * exception_kwargs ) :
self . listener = listener
self . exception_kwargs = exception_kwargs
def visit_FunctionDef ( self , node ) :
self . listener . funcname = node . name
argnames = [ arg . id for arg in node . args . args ]
if node . args . vararg :
argnames . append ( node . args . vararg )
if node . args . kwarg :
argnames . append ( node . args . kwarg )
self . listener . argnames = argnames
self . listener . defaults = node . args . defaults # ast
self . listener . varargs = node . args . vararg
self . listener . kwargs = node . args . kwarg
class ExpressionGenerator ( object ) :
def __init__ ( self , astnode ) :
self . generator = _ast_util . SourceGenerator ( ' ' * 4 )
self . generator . visit ( astnode )
def value ( self ) :
return ' ' . join ( self . generator . result )
else :
class FindIdentifiers ( object ) :
def __init__ ( self , listener , * * exception_kwargs ) :
self . in_function = False
self . local_ident_stack = { }
self . listener = listener
self . exception_kwargs = exception_kwargs
def _add_declared ( self , name ) :
if not self . in_function :
self . listener . declared_identifiers . add ( name )
def visitClass ( self , node , * args ) :
self . _add_declared ( node . name )
def visitAssName ( self , node , * args ) :
self . _add_declared ( node . name )
def visitAssign ( self , node , * args ) :
# flip around the visiting of Assign so the expression gets evaluated first,
# in the case of a clause like "x=x+5" (x is undeclared)
self . visit ( node . expr , * args )
for n in node . nodes :
self . visit ( n , * args )
def visitFunction ( self , node , * args ) :
self . _add_declared ( node . name )
# push function state onto stack. dont log any
# more identifiers as "declared" until outside of the function,
# but keep logging identifiers as "undeclared".
# track argument names in each function header so they arent counted as "undeclared"
saved = { }
inf = self . in_function
self . in_function = True
for arg in node . argnames :
if arg in self . local_ident_stack :
saved [ arg ] = True
else :
self . local_ident_stack [ arg ] = True
for n in node . getChildNodes ( ) :
self . visit ( n , * args )
self . in_function = inf
for arg in node . argnames :
if arg not in saved :
del self . local_ident_stack [ arg ]
def visitFor ( self , node , * args ) :
# flip around visit
self . visit ( node . list , * args )
self . visit ( node . assign , * args )
self . visit ( node . body , * args )
def visitName ( self , node , * args ) :
if node . name not in reserved and node . name not in self . listener . declared_identifiers and node . name not in self . local_ident_stack :
self . listener . undeclared_identifiers . add ( node . name )
def visitImport ( self , node , * args ) :
for ( mod , alias ) in node . names :
if alias is not None :
self . _add_declared ( alias )
else :
self . _add_declared ( mod . split ( ' . ' ) [ 0 ] )
def visitFrom ( self , node , * args ) :
for ( mod , alias ) in node . names :
if alias is not None :
self . _add_declared ( alias )
else :
if mod == ' * ' :
raise exceptions . CompileException ( " ' import * ' is not supported, since all identifier names must be explicitly declared. Please use the form ' from <modulename> import <name1>, <name2>, ... ' instead. " , * * self . exception_kwargs )
self . _add_declared ( mod )
def visit ( self , expr ) :
visitor . walk ( expr , self ) #, walker=walker())
class FindTuple ( object ) :
def __init__ ( self , listener , code_factory , * * exception_kwargs ) :
self . listener = listener
self . exception_kwargs = exception_kwargs
self . code_factory = code_factory
def visitTuple ( self , node , * args ) :
for n in node . nodes :
p = self . code_factory ( n , * * self . exception_kwargs )
self . listener . codeargs . append ( p )
self . listener . args . append ( ExpressionGenerator ( n ) . value ( ) )
self . listener . declared_identifiers = self . listener . declared_identifiers . union ( p . declared_identifiers )
self . listener . undeclared_identifiers = self . listener . undeclared_identifiers . union ( p . undeclared_identifiers )
def visit ( self , expr ) :
visitor . walk ( expr , self ) #, walker=walker())
class ParseFunc ( object ) :
def __init__ ( self , listener , * * exception_kwargs ) :
self . listener = listener
self . exception_kwargs = exception_kwargs
def visitFunction ( self , node , * args ) :
self . listener . funcname = node . name
self . listener . argnames = node . argnames
self . listener . defaults = node . defaults
self . listener . varargs = node . varargs
self . listener . kwargs = node . kwargs
def visit ( self , expr ) :
visitor . walk ( expr , self )
class ExpressionGenerator ( object ) :
""" given an AST node, generates an equivalent literal Python expression. """
def __init__ ( self , astnode ) :
self . buf = StringIO ( )
visitor . walk ( astnode , self ) #, walker=walker())
def value ( self ) :
return self . buf . getvalue ( )
def operator ( self , op , node , * args ) :
self . buf . write ( " ( " )
self . visit ( node . left , * args )
self . buf . write ( " %s " % op )
self . visit ( node . right , * args )
self . buf . write ( " ) " )
def booleanop ( self , op , node , * args ) :
self . visit ( node . nodes [ 0 ] )
for n in node . nodes [ 1 : ] :
self . buf . write ( " " + op + " " )
self . visit ( n , * args )
def visitConst ( self , node , * args ) :
self . buf . write ( repr ( node . value ) )
def visitAssName ( self , node , * args ) :
# TODO: figure out OP_ASSIGN, other OP_s
self . buf . write ( node . name )
def visitName ( self , node , * args ) :
self . buf . write ( node . name )
def visitMul ( self , node , * args ) :
self . operator ( " * " , node , * args )
def visitAnd ( self , node , * args ) :
self . booleanop ( " and " , node , * args )
def visitOr ( self , node , * args ) :
self . booleanop ( " or " , node , * args )
def visitBitand ( self , node , * args ) :
self . booleanop ( " & " , node , * args )
def visitBitor ( self , node , * args ) :
self . booleanop ( " | " , node , * args )
def visitBitxor ( self , node , * args ) :
self . booleanop ( " ^ " , node , * args )
def visitAdd ( self , node , * args ) :
self . operator ( " + " , node , * args )
def visitGetattr ( self , node , * args ) :
self . visit ( node . expr , * args )
self . buf . write ( " . %s " % node . attrname )
def visitSub ( self , node , * args ) :
self . operator ( " - " , node , * args )
def visitNot ( self , node , * args ) :
self . buf . write ( " not " )
self . visit ( node . expr )
def visitDiv ( self , node , * args ) :
self . operator ( " / " , node , * args )
def visitFloorDiv ( self , node , * args ) :
self . operator ( " // " , node , * args )
def visitSubscript ( self , node , * args ) :
self . visit ( node . expr )
self . buf . write ( " [ " )
[ self . visit ( x ) for x in node . subs ]
self . buf . write ( " ] " )
def visitUnarySub ( self , node , * args ) :
self . buf . write ( " - " )
self . visit ( node . expr )
def visitUnaryAdd ( self , node , * args ) :
self . buf . write ( " - " )
self . visit ( node . expr )
def visitSlice ( self , node , * args ) :
self . visit ( node . expr )
self . buf . write ( " [ " )
if node . lower is not None :
self . visit ( node . lower )
self . buf . write ( " : " )
if node . upper is not None :
self . visit ( node . upper )
self . buf . write ( " ] " )
def visitDict ( self , node ) :
self . buf . write ( " { " )
c = node . getChildren ( )
for i in range ( 0 , len ( c ) , 2 ) :
self . visit ( c [ i ] )
self . buf . write ( " : " )
self . visit ( c [ i + 1 ] )
if i < len ( c ) - 2 :
self . buf . write ( " , " )
self . buf . write ( " } " )
def visitTuple ( self , node ) :
self . buf . write ( " ( " )
c = node . getChildren ( )
for i in range ( 0 , len ( c ) ) :
self . visit ( c [ i ] )
if i < len ( c ) - 1 :
self . buf . write ( " , " )
self . buf . write ( " ) " )
def visitList ( self , node ) :
self . buf . write ( " [ " )
c = node . getChildren ( )
for i in range ( 0 , len ( c ) ) :
self . visit ( c [ i ] )
if i < len ( c ) - 1 :
self . buf . write ( " , " )
self . buf . write ( " ] " )
def visitListComp ( self , node ) :
self . buf . write ( " [ " )
self . visit ( node . expr )
self . buf . write ( " " )
for n in node . quals :
self . visit ( n )
self . buf . write ( " ] " )
def visitListCompFor ( self , node ) :
self . buf . write ( " for " )
self . visit ( node . assign )
self . buf . write ( " in " )
self . visit ( node . list )
for n in node . ifs :
self . visit ( n )
def visitListCompIf ( self , node ) :
self . buf . write ( " if " )
self . visit ( node . test )
def visitCompare ( self , node ) :
self . visit ( node . expr )
for tup in node . ops :
self . buf . write ( tup [ 0 ] )
self . visit ( tup [ 1 ] )
def visitCallFunc ( self , node , * args ) :
self . visit ( node . node )
self . buf . write ( " ( " )
if len ( node . args ) :
self . visit ( node . args [ 0 ] )
for a in node . args [ 1 : ] :
self . buf . write ( " , " )
self . visit ( a )
self . buf . write ( " ) " )
class walker ( visitor . ASTVisitor ) :
def dispatch ( self , node , * args ) :
print " Node: " , str ( node )
#print "dir:", dir(node)
return visitor . ASTVisitor . dispatch ( self , node , * args )