parse: pass filename, lineno into the ast

We will be needing this information to improve the tracebacks of python code
from the metadata, as well as to give the user information about where
variables were defined, so they know how it ended up the way it is.

(Bitbake rev: 9615c538b894f71a2d1a0ba6b3f260db91e75786)

Signed-off-by: Chris Larson <chris_larson@mentor.com>
Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>

bitbake/lib/bb/parse/ast.py

@@ -40,13 +40,14 @@ class StatementGroup(list):
             statement.eval(data)
 
 class AstNode(object):
-    pass
+    def __init__(self, filename, lineno):
+        self.filename = filename
+        self.lineno = lineno
 
 class IncludeNode(AstNode):
-    def __init__(self, what_file, fn, lineno, force):
+    def __init__(self, filename, lineno, what_file, force):
+        AstNode.__init__(self, filename, lineno)
         self.what_file = what_file
-        self.from_fn = fn
-        self.from_lineno = lineno
         self.force = force
 
     def eval(self, data):
@@ -54,16 +55,17 @@ class IncludeNode(AstNode):
         Include the file and evaluate the statements
         """
         s = bb.data.expand(self.what_file, data)
-        logger.debug(2, "CONF %s:%s: including %s", self.from_fn, self.from_lineno, s)
+        logger.debug(2, "CONF %s:%s: including %s", self.filename, self.lineno, s)
 
         # TODO: Cache those includes... maybe not here though
         if self.force:
-            bb.parse.ConfHandler.include(self.from_fn, s, data, "include required")
+            bb.parse.ConfHandler.include(self.filename, s, data, "include required")
         else:
-            bb.parse.ConfHandler.include(self.from_fn, s, data, False)
+            bb.parse.ConfHandler.include(self.filename, s, data, False)
 
 class ExportNode(AstNode):
-    def __init__(self, var):
+    def __init__(self, filename, lineno, var):
+        AstNode.__init__(self, filename, lineno)
         self.var = var
 
     def eval(self, data):
@@ -76,7 +78,8 @@ class DataNode(AstNode):
     this need to be re-evaluated... we might be able to do
     that faster with multiple classes.
     """
-    def __init__(self, groupd):
+    def __init__(self, filename, lineno, groupd):
+        AstNode.__init__(self, filename, lineno)
         self.groupd = groupd
 
     def getFunc(self, key, data):
@@ -119,19 +122,18 @@ class DataNode(AstNode):
         else:
             bb.data.setVar(key, val, data)
 
-class MethodNode:
+class MethodNode(AstNode):
-    def __init__(self, func_name, body, lineno, fn):
+    def __init__(self, filename, lineno, func_name, body):
+        AstNode.__init__(self, filename, lineno)
         self.func_name = func_name
         self.body = body
-        self.fn = fn
-        self.lineno = lineno
 
     def eval(self, data):
         if self.func_name == "__anonymous":
-            funcname = ("__anon_%s_%s" % (self.lineno, self.fn.translate(string.maketrans('/.+-', '____'))))
+            funcname = ("__anon_%s_%s" % (self.lineno, self.filename.translate(string.maketrans('/.+-', '____'))))
             if not funcname in bb.methodpool._parsed_fns:
                 text = "def %s(d):\n" % (funcname) + '\n'.join(self.body)
-                bb.methodpool.insert_method(funcname, text, self.fn)
+                bb.methodpool.insert_method(funcname, text, self.filename)
             anonfuncs = bb.data.getVar('__BBANONFUNCS', data) or []
             anonfuncs.append(funcname)
             bb.data.setVar('__BBANONFUNCS', anonfuncs, data)
@@ -140,25 +142,26 @@ class MethodNode:
             bb.data.setVar(self.func_name, '\n'.join(self.body), data)
 
 class PythonMethodNode(AstNode):
-    def __init__(self, funcname, root, body, fn):
+    def __init__(self, filename, lineno, function, define, body):
-        self.func_name = funcname
+        AstNode.__init__(self, filename, lineno)
-        self.root = root
+        self.function = function
+        self.define = define
         self.body = body
-        self.fn = fn
 
     def eval(self, data):
         # Note we will add root to parsedmethods after having parse
         # 'this' file. This means we will not parse methods from
         # bb classes twice
         text = '\n'.join(self.body)
-        if not bb.methodpool.parsed_module(self.root):
+        if not bb.methodpool.parsed_module(self.define):
-            bb.methodpool.insert_method(self.root, text, self.fn)
+            bb.methodpool.insert_method(self.define, text, self.filename)
-        bb.data.setVarFlag(self.func_name, "func", 1, data)
+        bb.data.setVarFlag(self.function, "func", 1, data)
-        bb.data.setVarFlag(self.func_name, "python", 1, data)
+        bb.data.setVarFlag(self.function, "python", 1, data)
-        bb.data.setVar(self.func_name, text, data)
+        bb.data.setVar(self.function, text, data)
 
 class MethodFlagsNode(AstNode):
-    def __init__(self, key, m):
+    def __init__(self, filename, lineno, key, m):
+        AstNode.__init__(self, filename, lineno)
         self.key = key
         self.m = m
 
@@ -178,7 +181,8 @@ class MethodFlagsNode(AstNode):
             bb.data.delVarFlag(self.key, "fakeroot", data)
 
 class ExportFuncsNode(AstNode):
-    def __init__(self, fns, classes):
+    def __init__(self, filename, lineno, fns, classes):
+        AstNode.__init__(self, filename, lineno)
         self.n = fns.split()
         self.classes = classes
 
@@ -217,7 +221,8 @@ class ExportFuncsNode(AstNode):
                 bb.data.setVarFlag(var, 'export_func', '1', data)
 
 class AddTaskNode(AstNode):
-    def __init__(self, func, before, after):
+    def __init__(self, filename, lineno, func, before, after):
+        AstNode.__init__(self, filename, lineno)
         self.func = func
         self.before = before
         self.after = after
@@ -248,7 +253,8 @@ class AddTaskNode(AstNode):
                     bb.data.setVarFlag(entry, "deps", [var] + existing, data)
 
 class BBHandlerNode(AstNode):
-    def __init__(self, fns):
+    def __init__(self, filename, lineno, fns):
+        AstNode.__init__(self, filename, lineno)
         self.hs = fns.split()
 
     def eval(self, data):
@@ -259,48 +265,49 @@ class BBHandlerNode(AstNode):
         bb.data.setVar('__BBHANDLERS', bbhands, data)
 
 class InheritNode(AstNode):
-    def __init__(self, classes):
+    def __init__(self, filename, lineno, classes):
+        AstNode.__init__(self, filename, lineno)
         self.classes = classes
 
     def eval(self, data):
         bb.parse.BBHandler.inherit(self.classes, data)
 
-def handleInclude(statements, m, fn, lineno, force):
+def handleInclude(statements, filename, lineno, m, force):
-    statements.append(IncludeNode(m.group(1), fn, lineno, force))
+    statements.append(IncludeNode(filename, lineno, m.group(1), force))
 
-def handleExport(statements, m):
+def handleExport(statements, filename, lineno, m):
-    statements.append(ExportNode(m.group(1)))
+    statements.append(ExportNode(filename, lineno, m.group(1)))
 
-def handleData(statements, groupd):
+def handleData(statements, filename, lineno, groupd):
-    statements.append(DataNode(groupd))
+    statements.append(DataNode(filename, lineno, groupd))
 
-def handleMethod(statements, func_name, lineno, fn, body):
+def handleMethod(statements, filename, lineno, func_name, body):
-    statements.append(MethodNode(func_name, body, lineno, fn))
+    statements.append(MethodNode(filename, lineno, func_name, body))
 
-def handlePythonMethod(statements, funcname, root, body, fn):
+def handlePythonMethod(statements, filename, lineno, funcname, root, body):
-    statements.append(PythonMethodNode(funcname, root, body, fn))
+    statements.append(PythonMethodNode(filename, lineno, funcname, root, body))
 
-def handleMethodFlags(statements, key, m):
+def handleMethodFlags(statements, filename, lineno, key, m):
-    statements.append(MethodFlagsNode(key, m))
+    statements.append(MethodFlagsNode(filename, lineno, key, m))
 
-def handleExportFuncs(statements, m, classes):
+def handleExportFuncs(statements, filename, lineno, m, classes):
-    statements.append(ExportFuncsNode(m.group(1), classes))
+    statements.append(ExportFuncsNode(filename, lineno, m.group(1), classes))
 
-def handleAddTask(statements, m):
+def handleAddTask(statements, filename, lineno, m):
     func = m.group("func")
     before = m.group("before")
     after = m.group("after")
     if func is None:
         return
 
-    statements.append(AddTaskNode(func, before, after))
+    statements.append(AddTaskNode(filename, lineno, func, before, after))
 
-def handleBBHandlers(statements, m):
+def handleBBHandlers(statements, filename, lineno, m):
-    statements.append(BBHandlerNode(m.group(1)))
+    statements.append(BBHandlerNode(filename, lineno, m.group(1)))
 
-def handleInherit(statements, m):
+def handleInherit(statements, filename, lineno, m):
     classes = m.group(1)
-    statements.append(InheritNode(classes.split()))
+    statements.append(InheritNode(filename, lineno, classes.split()))
 
 def finalize(fn, d, variant = None):
     for lazykey in bb.data.getVar("__lazy_assigned", d) or ():

bitbake/lib/bb/parse/parse_py/BBHandler.py

@@ -172,7 +172,7 @@ def feeder(lineno, s, fn, root, statements):
     if __infunc__:
         if s == '}':
             __body__.append('')
-            ast.handleMethod(statements, __infunc__, lineno, fn, __body__)
+            ast.handleMethod(statements, fn, lineno, __infunc__, __body__)
             __infunc__ = ""
             __body__ = []
         else:
@@ -185,7 +185,8 @@ def feeder(lineno, s, fn, root, statements):
             __body__.append(s)
             return
         else:
-            ast.handlePythonMethod(statements, __inpython__, root, __body__, fn)
+            ast.handlePythonMethod(statements, fn, lineno, __inpython__,
+                                   root, __body__)
             __body__ = []
             __inpython__ = False
 
@@ -206,7 +207,7 @@ def feeder(lineno, s, fn, root, statements):
     m = __func_start_regexp__.match(s)
     if m:
         __infunc__ = m.group("func") or "__anonymous"
-        ast.handleMethodFlags(statements, __infunc__, m)
+        ast.handleMethodFlags(statements, fn, lineno, __infunc__, m)
         return
 
     m = __def_regexp__.match(s)
@@ -218,22 +219,22 @@ def feeder(lineno, s, fn, root, statements):
 
     m = __export_func_regexp__.match(s)
     if m:
-        ast.handleExportFuncs(statements, m, classes)
+        ast.handleExportFuncs(statements, fn, lineno, m, classes)
         return
 
     m = __addtask_regexp__.match(s)
     if m:
-        ast.handleAddTask(statements, m)
+        ast.handleAddTask(statements, fn, lineno, m)
         return
 
     m = __addhandler_regexp__.match(s)
     if m:
-        ast.handleBBHandlers(statements, m)
+        ast.handleBBHandlers(statements, fn, lineno, m)
         return
 
     m = __inherit_regexp__.match(s)
     if m:
-        ast.handleInherit(statements, m)
+        ast.handleInherit(statements, fn, lineno, m)
         return
 
     return ConfHandler.feeder(lineno, s, fn, statements)

bitbake/lib/bb/parse/parse_py/ConfHandler.py

@@ -113,22 +113,22 @@ def feeder(lineno, s, fn, statements):
     m = __config_regexp__.match(s)
     if m:
         groupd = m.groupdict()
-        ast.handleData(statements, groupd)
+        ast.handleData(statements, fn, lineno, groupd)
         return
 
     m = __include_regexp__.match(s)
     if m:
-        ast.handleInclude(statements, m, fn, lineno, False)
+        ast.handleInclude(statements, fn, lineno, m, False)
         return
 
     m = __require_regexp__.match(s)
     if m:
-        ast.handleInclude(statements, m, fn, lineno, True)
+        ast.handleInclude(statements, fn, lineno, m, True)
         return
 
     m = __export_regexp__.match(s)
     if m:
-        ast.handleExport(statements, m)
+        ast.handleExport(statements, fn, lineno, m)
         return
 
     raise ParseError("%s:%d: unparsed line: '%s'" % (fn, lineno, s));