bitbake: Update with changes from bitbake 1.8 branch

git-svn-id: https://svn.o-hand.com/repos/poky/trunk@4006 311d38ba-8fff-0310-9ca6-ca027cbcb966

bitbake/ChangeLog

@@ -1,4 +1,5 @@
 Changes in BitBake 1.8.x:
+	- Fix -f (force) in conjunction with -b
 	- Fix exit code for build failures in --continue mode
 	- Fix git branch tags fetching
 	- Change parseConfigurationFile so it works on real data, not a copy
@@ -21,6 +22,10 @@ Changes in BitBake 1.8.x:
 	  the way bitbake schedules tasks
 	- Add BB_STAMP_POLICY variable/option ("perfile" or "full") controlling
 	  how extensively stamps are looked at for validity
+	- When handling build target failures make sure idepends are checked and
+	  failed where needed. Fixes --continue mode crashes.
+	- Fix problems with recrdeptask handling where some idepends weren't handled
+	  correctly.
 
 Changes in BitBake 1.8.10:
 	- Psyco is available only for x86 - do not use it on other architectures.

bitbake/lib/bb/cache.py

@@ -80,7 +80,7 @@ class Cache:
             if old_mtime > newest_mtime:
                 newest_mtime = old_mtime
 
-        if self.mtime(self.cachefile) >= newest_mtime:
+        if bb.parse.cached_mtime_noerror(self.cachefile) >= newest_mtime:
             try:
                 p = pickle.Unpickler(file(self.cachefile, "rb"))
                 self.depends_cache, version_data = p.load()
@@ -91,7 +91,7 @@ class Cache:
             except EOFError:
                 bb.msg.note(1, bb.msg.domain.Cache, "Truncated cache found, rebuilding...")
                 self.depends_cache = {}
-            except (ValueError, KeyError):
+            except:
                 bb.msg.note(1, bb.msg.domain.Cache, "Invalid cache found, rebuilding...")
                 self.depends_cache = {}
         else:
@@ -199,28 +199,31 @@ class Cache:
             self.remove(fn)
             return False
 
+        mtime = bb.parse.cached_mtime_noerror(fn) 
+
         # Check file still exists
-        if self.mtime(fn) == 0:
+        if mtime == 0:
             bb.msg.debug(2, bb.msg.domain.Cache, "Cache: %s not longer exists" % fn)
             self.remove(fn)
             return False
 
         # Check the file's timestamp
-        if bb.parse.cached_mtime(fn) > self.getVar("CACHETIMESTAMP", fn, True):
+        if mtime > self.getVar("CACHETIMESTAMP", fn, True):
             bb.msg.debug(2, bb.msg.domain.Cache, "Cache: %s changed" % fn)
             self.remove(fn)
             return False
 
         # Check dependencies are still valid
         depends = self.getVar("__depends", fn, True)
+        if depends:
             for f,old_mtime in depends:
+                fmtime = bb.parse.cached_mtime_noerror(f)
                 # Check if file still exists
-            if self.mtime(f) == 0:
+                if fmtime == 0:
                     self.remove(fn)
                     return False
 
-            new_mtime = bb.parse.cached_mtime(f)
+                if (fmtime > old_mtime):
-            if (new_mtime > old_mtime):
                     bb.msg.debug(2, bb.msg.domain.Cache, "Cache: %s's dependency %s changed" % (fn, f))
                     self.remove(fn)
                     return False
@@ -284,7 +287,6 @@ class Cache:
         pv       = self.getVar('PV', file_name, True)
         pr       = self.getVar('PR', file_name, True)
         dp       = int(self.getVar('DEFAULT_PREFERENCE', file_name, True) or "0")
-        provides  = Set([pn] + (self.getVar("PROVIDES", file_name, True) or "").split())
         depends   = bb.utils.explode_deps(self.getVar("DEPENDS", file_name, True) or "")
         packages  = (self.getVar('PACKAGES', file_name, True) or "").split()
         packages_dynamic = (self.getVar('PACKAGES_DYNAMIC', file_name, True) or "").split()
@@ -304,24 +306,31 @@ class Cache:
         cacheData.pkg_pepvpr[file_name] = (pe,pv,pr)
         cacheData.pkg_dp[file_name] = dp
 
+        provides = [pn]
+        for provide in (self.getVar("PROVIDES", file_name, True) or "").split():
+            if provide not in provides:
+                provides.append(provide)
+
         # Build forward and reverse provider hashes
         # Forward: virtual -> [filenames]
         # Reverse: PN -> [virtuals]
         if pn not in cacheData.pn_provides:
-            cacheData.pn_provides[pn] = Set()
+            cacheData.pn_provides[pn] = []
-        cacheData.pn_provides[pn] |= provides
 
-        cacheData.fn_provides[file_name] = Set()
+        cacheData.fn_provides[file_name] = provides
         for provide in provides:
             if provide not in cacheData.providers:
                 cacheData.providers[provide] = []
             cacheData.providers[provide].append(file_name)
-            cacheData.fn_provides[file_name].add(provide)
+            if not provide in cacheData.pn_provides[pn]:
+                cacheData.pn_provides[pn].append(provide)
 
-        cacheData.deps[file_name] = Set()
+        cacheData.deps[file_name] = []
         for dep in depends:
-            cacheData.all_depends.add(dep)
+            if not dep in cacheData.deps[file_name]:
-            cacheData.deps[file_name].add(dep)
+                cacheData.deps[file_name].append(dep)
+            if not dep in cacheData.all_depends:
+                cacheData.all_depends.append(dep)
 
         # Build reverse hash for PACKAGES, so runtime dependencies 
         # can be be resolved (RDEPENDS, RRECOMMENDS etc.)
@@ -343,28 +352,21 @@ class Cache:
 
         # Build hash of runtime depends and rececommends
 
-        def add_dep(deplist, deps):
-            for dep in deps:
-                if not dep in deplist:
-                    deplist[dep] = ""
-
         if not file_name in cacheData.rundeps:
             cacheData.rundeps[file_name] = {}
         if not file_name in cacheData.runrecs:
             cacheData.runrecs[file_name] = {}
 
-        rdepends = bb.utils.explode_deps(self.getVar('RDEPENDS', file_name, True) or "")
+        rdepends = self.getVar('RDEPENDS', file_name, True) or ""
-        rrecommends = bb.utils.explode_deps(self.getVar('RRECOMMENDS', file_name, True) or "")
+        rrecommends = self.getVar('RRECOMMENDS', file_name, True) or ""
         for package in packages + [pn]:
             if not package in cacheData.rundeps[file_name]:
-                cacheData.rundeps[file_name][package] = {}
+                cacheData.rundeps[file_name][package] = []
             if not package in cacheData.runrecs[file_name]:
-                cacheData.runrecs[file_name][package] = {}
+                cacheData.runrecs[file_name][package] = []
 
-            add_dep(cacheData.rundeps[file_name][package], rdepends)
+            cacheData.rundeps[file_name][package] = rdepends + " " + (self.getVar("RDEPENDS_%s" % package, file_name, True) or "")
-            add_dep(cacheData.runrecs[file_name][package], rrecommends)
+            cacheData.runrecs[file_name][package] = rrecommends + " " + (self.getVar("RRECOMMENDS_%s" % package, file_name, True) or "")
-            add_dep(cacheData.rundeps[file_name][package], bb.utils.explode_deps(self.getVar("RDEPENDS_%s" % package, file_name, True) or ""))
-            add_dep(cacheData.runrecs[file_name][package], bb.utils.explode_deps(self.getVar("RRECOMMENDS_%s" % package, file_name, True) or ""))
 
         # Collect files we may need for possible world-dep
         # calculations
@@ -385,7 +387,7 @@ class Cache:
         data.setVar('TMPDIR', data.getVar('TMPDIR', config, 1) or "", config)
         bbfile_loc = os.path.abspath(os.path.dirname(bbfile))
         oldpath = os.path.abspath(os.getcwd())
-        if self.mtime(bbfile_loc):
+        if bb.parse.cached_mtime_noerror(bbfile_loc):
             os.chdir(bbfile_loc)
         bb_data = data.init_db(config)
         try:
@@ -444,7 +446,7 @@ class CacheData:
         self.pkg_dp = {}
         self.pn_provides = {}
         self.fn_provides = {}
-        self.all_depends = Set()
+        self.all_depends = []
         self.deps = {}
         self.rundeps = {}
         self.runrecs = {}

bitbake/lib/bb/cooker.py

@@ -493,7 +493,7 @@ class BBCooker:
         # Remove stamp for target if force mode active
         if self.configuration.force:
             bb.msg.note(2, bb.msg.domain.RunQueue, "Remove stamp %s, %s" % (self.configuration.cmd, fn))
-            bb.build.del_stamp('do_%s' % self.configuration.cmd, bbfile_data)
+            bb.build.del_stamp('do_%s' % self.configuration.cmd, self.configuration.data)
 
         # Setup taskdata structure
         taskdata = bb.taskdata.TaskData(self.configuration.abort)
@@ -573,6 +573,7 @@ class BBCooker:
 
         bb.msg.debug(1, bb.msg.domain.Collection, "collecting .bb files")
         (filelist, masked) = self.collect_bbfiles()
+        bb.data.renameVar("__depends", "__base_depends", self.configuration.data)
         self.parse_bbfiles(filelist, masked, self.myProgressCallback)
         bb.msg.debug(1, bb.msg.domain.Collection, "parsing complete")
 

bitbake/lib/bb/runqueue.py

@@ -317,6 +317,7 @@ class RunQueue:
 
         depends = []
         runq_build = []
+        recursive_tdepends = {}
 
         taskData = self.taskData
 
@@ -382,14 +383,45 @@ class RunQueue:
                 # e.g. do_sometask[depends] = "targetname:do_someothertask"
                 # (makes sure sometask runs after targetname's someothertask)
                 idepends = taskData.tasks_idepends[task]
-                for idepend in idepends:
+                for (depid, idependtask) in idepends:
-                    depid = int(idepend.split(":")[0])
                     if depid in taskData.build_targets:
                         # Won't be in build_targets if ASSUME_PROVIDED
                         depdata = taskData.build_targets[depid][0]
                         if depdata is not None:
                             dep = taskData.fn_index[depdata]
-                            depends.append(taskData.gettask_id(dep, idepend.split(":")[1]))
+                            depends.append(taskData.gettask_id(dep, idependtask))
+
+                # Create a list of recursive dependent tasks (from tdepends) and cache
+                def get_recursive_tdepends(task):
+                    if not task:
+                        return []
+                    if task in recursive_tdepends:
+                        return recursive_tdepends[task]
+                    rectdepends = [task]
+                    nextdeps = [task]
+                    while len(nextdeps) != 0:
+                        newdeps = []
+                        for nextdep in nextdeps:
+                            for tdepend in taskData.tasks_tdepends[nextdep]:
+                                if tdepend not in rectdepends:
+                                    rectdepends.append(tdepend)
+                                    newdeps.append(tdepend)
+                        nextdeps = newdeps
+                    recursive_tdepends[task] = rectdepends
+                    return rectdepends
+
+                # Using the list of tdepends for this task create a list of 
+                # the recursive idepends we have
+                def get_recursive_idepends(task):
+                    if not task:
+                        return []
+                    rectdepends = get_recursive_tdepends(task)
+
+                    recidepends = []
+                    for tdepend in rectdepends:
+                        for idepend in taskData.tasks_idepends[tdepend]:
+                            recidepends.append(idepend)
+                    return recidepends
 
                 def add_recursive_build(depid, depfnid):
                     """
@@ -404,13 +436,11 @@ class RunQueue:
                         depdata = taskData.build_targets[depid][0]
                         if depdata is not None:
                             dep = taskData.fn_index[depdata]
-                            idepends = []
                             # Need to avoid creating new tasks here
                             taskid = taskData.gettask_id(dep, taskname, False)
                             if taskid is not None:
                                 depends.append(taskid)
                                 fnid = taskData.tasks_fnid[taskid]
-                                idepends = taskData.tasks_idepends[taskid]
                                 #print "Added %s (%s) due to %s" % (taskid, taskData.fn_index[fnid], taskData.fn_index[depfnid])
                             else:
                                 fnid = taskData.getfn_id(dep)
@@ -420,10 +450,9 @@ class RunQueue:
                             for nextdepid in taskData.rdepids[fnid]:
                                 if nextdepid not in rdep_seen:
                                     add_recursive_run(nextdepid, fnid)
-                            for idepend in idepends:
+                            for (idependid, idependtask) in get_recursive_idepends(taskid):
-                                nextdepid = int(idepend.split(":")[0])
+                                if idependid not in dep_seen:
-                                if nextdepid not in dep_seen:
+                                    add_recursive_build(idependid, fnid)
-                                    add_recursive_build(nextdepid, fnid)
 
                 def add_recursive_run(rdepid, depfnid):
                     """
@@ -438,13 +467,11 @@ class RunQueue:
                         depdata = taskData.run_targets[rdepid][0]
                         if depdata is not None:
                             dep = taskData.fn_index[depdata]
-                            idepends = []
                             # Need to avoid creating new tasks here
                             taskid = taskData.gettask_id(dep, taskname, False)
                             if taskid is not None:
                                 depends.append(taskid)
                                 fnid = taskData.tasks_fnid[taskid]
-                                idepends = taskData.tasks_idepends[taskid]
                                 #print "Added %s (%s) due to %s" % (taskid, taskData.fn_index[fnid], taskData.fn_index[depfnid])
                             else:
                                 fnid = taskData.getfn_id(dep)
@@ -454,10 +481,9 @@ class RunQueue:
                             for nextdepid in taskData.rdepids[fnid]:
                                 if nextdepid not in rdep_seen:
                                     add_recursive_run(nextdepid, fnid)
-                            for idepend in idepends:
+                            for (idependid, idependtask) in get_recursive_idepends(taskid):
-                                nextdepid = int(idepend.split(":")[0])
+                                if idependid not in dep_seen:
-                                if nextdepid not in dep_seen:
+                                    add_recursive_build(idependid, fnid)
-                                    add_recursive_build(nextdepid, fnid)
 
                 # Resolve recursive 'recrdeptask' dependencies 
                 #
@@ -472,9 +498,9 @@ class RunQueue:
                             add_recursive_build(depid, fnid)
                         for rdepid in taskData.rdepids[fnid]:
                             add_recursive_run(rdepid, fnid)
-                        for idepend in idepends:
+                        deptaskid = taskData.gettask_id(fn, taskname, False)
-                            depid = int(idepend.split(":")[0])
+                        for (idependid, idependtask) in get_recursive_idepends(deptaskid):
-                            add_recursive_build(depid, fnid)
+                            add_recursive_build(idependid, fnid)
 
                 # Rmove all self references
                 if task in depends:
@@ -659,6 +685,16 @@ class RunQueue:
             if len(self.runq_depends[task]) == 0:
                 buildable.append(task)
 
+        def check_buildable(self, task, buildable):
+             for revdep in self.runq_revdeps[task]:
+                alldeps = 1
+                for dep in self.runq_depends[revdep]:
+                    if dep in unchecked:
+                        alldeps = 0
+                if alldeps == 1:
+                    if revdep in unchecked:
+                        buildable.append(revdep)
+
         for task in range(len(self.runq_fnid)):
             if task not in unchecked:
                 continue
@@ -669,12 +705,14 @@ class RunQueue:
             if not os.access(stampfile, os.F_OK):
                 del unchecked[task]
                 notcurrent.append(task)
+                check_buildable(self, task, buildable)
                 continue
             # If its a 'nostamp' task, it's not current
             taskdep = self.dataCache.task_deps[fn]
             if 'nostamp' in taskdep and task in taskdep['nostamp']:
                 del unchecked[task]
                 notcurrent.append(task)
+                check_buildable(self, task, buildable)
                 continue
 
         while (len(buildable) > 0):
@@ -705,14 +743,7 @@ class RunQueue:
                     else:
                         notcurrent.append(task)
 
-                for revdep in self.runq_revdeps[task]:
+                check_buildable(self, task, nextbuildable)
-                    alldeps = 1
-                    for dep in self.runq_depends[revdep]:
-                        if dep in unchecked:
-                            alldeps = 0
-                    if alldeps == 1:
-                        if revdep in unchecked:
-                            nextbuildable.append(revdep)
 
             buildable = nextbuildable
 
@@ -729,6 +760,40 @@ class RunQueue:
             bb.fatal("check_stamps fatal internal error")
         return current
 
+    def check_stamp(self, task):
+
+        if self.stamppolicy == "perfile":
+            fulldeptree = False
+        else:
+            fulldeptree = True
+
+        fn = self.taskData.fn_index[self.runq_fnid[task]]
+        taskname = self.runq_task[task]
+        stampfile = "%s.%s" % (self.dataCache.stamp[fn], taskname)
+        # If the stamp is missing its not current
+        if not os.access(stampfile, os.F_OK):
+            return False
+        # If its a 'nostamp' task, it's not current
+        taskdep = self.dataCache.task_deps[fn]
+        if 'nostamp' in taskdep and task in taskdep['nostamp']:
+            return False
+
+        iscurrent = True
+        t1 =  os.stat(stampfile)[stat.ST_MTIME]
+        for dep in self.runq_depends[task]:
+            if iscurrent:
+                fn2 = self.taskData.fn_index[self.runq_fnid[dep]]
+                taskname2 = self.runq_task[dep]
+                stampfile2 = "%s.%s" % (self.dataCache.stamp[fn2], taskname2)
+                if fulldeptree or fn == fn2:
+                    try:
+                        t2 = os.stat(stampfile2)[stat.ST_MTIME]
+                        if t1 < t2:
+                            iscurrent = False
+                    except:
+                        iscurrent = False
+
+        return iscurrent
 
     def execute_runqueue(self):
         """
@@ -817,25 +882,18 @@ class RunQueue:
 
         event.fire(bb.event.StampUpdate(self.target_pairs, self.dataCache.stamp, self.cfgdata))
 
-        # Find out which tasks have current stamps which we can skip when the
-        # time comes
-        currentstamps = self.check_stamps()
-        self.stats.taskSkipped(len(currentstamps))
-        self.stats.taskCompleted(len(currentstamps))
-
         while True:
             task = self.sched.next()
             if task is not None:
                 fn = self.taskData.fn_index[self.runq_fnid[task]]
 
                 taskname = self.runq_task[task]
-                if task in currentstamps:
+                if self.check_stamp(task):
-                #if bb.build.stamp_is_current(taskname, self.dataCache, fn):
                     bb.msg.debug(2, bb.msg.domain.RunQueue, "Stamp current task %s (%s)" % (task, self.get_user_idstring(task)))
                     self.runq_running[task] = 1
                     self.task_complete(task)
-                    #self.stats.taskCompleted()
+                    self.stats.taskCompleted()
-                    #self.stats.taskSkipped()
+                    self.stats.taskSkipped()
                     continue
 
                 bb.msg.note(1, bb.msg.domain.RunQueue, "Running task %d of %d (ID: %s, %s)" % (self.stats.completed + self.active_builds + 1, len(self.runq_fnid), task, self.get_user_idstring(task)))

bitbake/lib/bb/taskdata.py

@@ -150,7 +150,7 @@ class TaskData:
                 ids = []
                 for dep in task_deps['depends'][task].split():
                     if dep:
-                        ids.append(str(self.getbuild_id(dep.split(":")[0])) + ":" + dep.split(":")[1])
+                        ids.append(((self.getbuild_id(dep.split(":")[0])), dep.split(":")[1]))
                 self.tasks_idepends[taskid].extend(ids)
 
         # Work out build dependencies
@@ -167,11 +167,11 @@ class TaskData:
             rdepends = dataCache.rundeps[fn]
             rrecs = dataCache.runrecs[fn]
             for package in rdepends:
-                for rdepend in rdepends[package]:
+                for rdepend in bb.utils.explode_deps(rdepends[package]):
                     bb.msg.debug(2, bb.msg.domain.TaskData, "Added runtime dependency %s for %s" % (rdepend, fn))
                     rdependids[self.getrun_id(rdepend)] = None
             for package in rrecs:
-                for rdepend in rrecs[package]:
+                for rdepend in bb.utils.explode_deps(rrecs[package]):
                     bb.msg.debug(2, bb.msg.domain.TaskData, "Added runtime recommendation %s for %s" % (rdepend, fn))
                     rdependids[self.getrun_id(rdepend)] = None
             self.rdepids[fnid] = rdependids.keys()
@@ -458,8 +458,6 @@ class TaskData:
         """
         if fnid in self.failed_fnids:
             return
-        if not missing_list:
-            missing_list = [fnid]
         bb.msg.debug(1, bb.msg.domain.Provider, "File '%s' is unbuildable, removing..." % self.fn_index[fnid])
         self.failed_fnids.append(fnid)
         for target in self.build_targets:
@@ -487,6 +485,12 @@ class TaskData:
         dependees = self.get_dependees(targetid)
         for fnid in dependees:
             self.fail_fnid(fnid, missing_list)
+        for taskid in range(len(self.tasks_idepends)):
+            idepends = self.tasks_idepends[taskid]
+            for (idependid, idependtask) in idepends:
+                if idependid == targetid:
+                    self.fail_fnid(self.tasks_fnid[taskid], missing_list)
+
         if self.abort and targetid in self.external_targets:
             bb.msg.error(bb.msg.domain.Provider, "Required build target '%s' has no buildable providers.\nMissing or unbuildable dependency chain was: %s" % (self.build_names_index[targetid], missing_list))
             raise bb.providers.NoProvider