bitbake: Pass task hash information to subprocesses

Pass task has informaiton to work processes, allowing full manipulation of
the hash data in the task context allowing checksums to be usable.

Signed-off-by: Richard Purdie <rpurdie@linux.intel.com>

bitbake/bin/bitbake-runtask

@@ -5,6 +5,12 @@ import sys
 import warnings
 sys.path.insert(0, os.path.join(os.path.dirname(os.path.dirname(sys.argv[0])), 'lib'))
 
+try:
+    import cPickle as pickle
+except ImportError:
+    import pickle
+    bb.msg.note(1, bb.msg.domain.Cache, "Importing cPickle failed. Falling back to a very slow implementation.")
+
 class BBConfiguration(object):
     """
     Manages build options and configurations for one run
@@ -62,8 +68,9 @@ bb.event.useStdout = False
 import bb.cooker
 
 cooker = bb.cooker.BBCooker(BBConfiguration(), None)
-buildfile = sys.argv[1]
-taskname = sys.argv[2]
+hashfile = sys.argv[1]
+buildfile = sys.argv[2]
+taskname = sys.argv[3]
 
 cooker.parseConfiguration()
 
@@ -84,8 +91,18 @@ cooker.bb_cache.handle_data(fn, cooker.status)
 if taskname.endswith("_setscene"):
     the_data.setVarFlag(taskname, "quieterrors", "1")
 
+p = pickle.Unpickler(file(hashfile, "rb"))
+hashdata = p.load()
+
+bb.parse.siggen.set_taskdata(hashdata["hashes"], hashdata["deps"])
+
+for h in hashdata["hashes"]:
+    bb.data.setVar("BBHASH_%s" % h, hashdata["hashes"][h], the_data)
+for h in hashdata["deps"]:
+    bb.data.setVar("BBHASHDEPS_%s" % h, hashdata["deps"][h], the_data)
+
 ret = 0
-if sys.argv[3] != "True":
+if sys.argv[4] != "True":
     ret = bb.build.exec_task(fn, taskname, the_data)
 sys.exit(ret)
 

bitbake/lib/bb/runqueue.py

@@ -30,6 +30,12 @@ import stat
 import fcntl
 import copy
 
+try:
+    import cPickle as pickle
+except ImportError:
+    import pickle
+    bb.msg.note(1, bb.msg.domain.Cache, "Importing cPickle failed. Falling back to a very slow implementation.")
+
 class RunQueueStats:
     """
     Holds statistics on the tasks handled by the associated runQueue
@@ -703,6 +709,21 @@ class RunQueueData:
                         procdep.append(self.taskData.fn_index[self.runq_fnid[dep]] + "." + self.runq_task[dep])
                     self.runq_hash[task] = bb.parse.siggen.get_taskhash(self.taskData.fn_index[self.runq_fnid[task]], self.runq_task[task], procdep, self.dataCache)
 
+        hashdata = {}
+        hashdata["hashes"] = {}
+        hashdata["deps"] = {}
+        for task in range(len(self.runq_fnid)):
+            hashdata["hashes"][self.taskData.fn_index[self.runq_fnid[task]] + "." + self.runq_task[task]] = self.runq_hash[task]
+            deps = []
+            for dep in self.runq_depends[task]:
+                deps.append(self.taskData.fn_index[self.runq_fnid[dep]] + "." + self.runq_task[dep])
+            hashdata["deps"][self.taskData.fn_index[self.runq_fnid[task]] + "." + self.runq_task[task]] = deps
+
+        # Write out the hashes into a file for use by the individual tasks
+        self.hashfile = bb.data.expand("${TMPDIR}/cache/hashdata.dat", self.cooker.configuration.data)
+        p = pickle.Pickler(file(self.hashfile, "wb"), -1)
+        p.dump(hashdata)
+
         return len(self.runq_fnid)
 
     def dump_data(self, taskQueue):
@@ -1047,7 +1068,7 @@ class RunQueueExecute:
             sys.stdout.flush()
             sys.stderr.flush()
 
-            proc = subprocess.Popen(["bitbake-runtask", fn, taskname, str(self.cooker.configuration.dry_run)], env=env, stdout=subprocess.PIPE, stdin=subprocess.PIPE)
+            proc = subprocess.Popen(["bitbake-runtask", self.rqdata.hashfile, fn, taskname, str(self.cooker.configuration.dry_run)], env=env, stdout=subprocess.PIPE, stdin=subprocess.PIPE)
             pipein = proc.stdout
             pipeout = proc.stdin
             pid = proc.pid

bitbake/lib/bb/siggen.py

@@ -107,6 +107,10 @@ class SignatureGeneratorBasic(SignatureGenerator):
         #d.setVar("BB_TASKHASH_task-%s" % task, taskhash[task])
         return h
 
+    def set_taskdata(self, hashes, deps):
+        self.runtaskdeps = deps
+        self.taskhash = hashes
+
     def dump_sigtask(self, fn, task, stampbase, runtime):
         k = fn + "." + task
         if runtime == "customfile":
@@ -128,7 +132,7 @@ class SignatureGeneratorBasic(SignatureGenerator):
             data['gendeps'][dep] = self.gendeps[fn][dep]
             data['varvals'][dep] = self.lookupcache[fn][dep]
 
-        if runtime and runtime != "customfile":
+        if runtime:
             data['runtaskdeps'] = self.runtaskdeps[k]
             data['runtaskhashes'] = {}
             for dep in data['runtaskdeps']: