bitbake: Pass task hash information to subprocesses
Pass task has informaiton to work processes, allowing full manipulation of the hash data in the task context allowing checksums to be usable. Signed-off-by: Richard Purdie <rpurdie@linux.intel.com>
This commit is contained in:
parent
15ceaaaaf7
commit
ec4d6b989a
|
@ -5,6 +5,12 @@ import sys
|
||||||
import warnings
|
import warnings
|
||||||
sys.path.insert(0, os.path.join(os.path.dirname(os.path.dirname(sys.argv[0])), 'lib'))
|
sys.path.insert(0, os.path.join(os.path.dirname(os.path.dirname(sys.argv[0])), 'lib'))
|
||||||
|
|
||||||
|
try:
|
||||||
|
import cPickle as pickle
|
||||||
|
except ImportError:
|
||||||
|
import pickle
|
||||||
|
bb.msg.note(1, bb.msg.domain.Cache, "Importing cPickle failed. Falling back to a very slow implementation.")
|
||||||
|
|
||||||
class BBConfiguration(object):
|
class BBConfiguration(object):
|
||||||
"""
|
"""
|
||||||
Manages build options and configurations for one run
|
Manages build options and configurations for one run
|
||||||
|
@ -62,8 +68,9 @@ bb.event.useStdout = False
|
||||||
import bb.cooker
|
import bb.cooker
|
||||||
|
|
||||||
cooker = bb.cooker.BBCooker(BBConfiguration(), None)
|
cooker = bb.cooker.BBCooker(BBConfiguration(), None)
|
||||||
buildfile = sys.argv[1]
|
hashfile = sys.argv[1]
|
||||||
taskname = sys.argv[2]
|
buildfile = sys.argv[2]
|
||||||
|
taskname = sys.argv[3]
|
||||||
|
|
||||||
cooker.parseConfiguration()
|
cooker.parseConfiguration()
|
||||||
|
|
||||||
|
@ -84,8 +91,18 @@ cooker.bb_cache.handle_data(fn, cooker.status)
|
||||||
if taskname.endswith("_setscene"):
|
if taskname.endswith("_setscene"):
|
||||||
the_data.setVarFlag(taskname, "quieterrors", "1")
|
the_data.setVarFlag(taskname, "quieterrors", "1")
|
||||||
|
|
||||||
|
p = pickle.Unpickler(file(hashfile, "rb"))
|
||||||
|
hashdata = p.load()
|
||||||
|
|
||||||
|
bb.parse.siggen.set_taskdata(hashdata["hashes"], hashdata["deps"])
|
||||||
|
|
||||||
|
for h in hashdata["hashes"]:
|
||||||
|
bb.data.setVar("BBHASH_%s" % h, hashdata["hashes"][h], the_data)
|
||||||
|
for h in hashdata["deps"]:
|
||||||
|
bb.data.setVar("BBHASHDEPS_%s" % h, hashdata["deps"][h], the_data)
|
||||||
|
|
||||||
ret = 0
|
ret = 0
|
||||||
if sys.argv[3] != "True":
|
if sys.argv[4] != "True":
|
||||||
ret = bb.build.exec_task(fn, taskname, the_data)
|
ret = bb.build.exec_task(fn, taskname, the_data)
|
||||||
sys.exit(ret)
|
sys.exit(ret)
|
||||||
|
|
||||||
|
|
|
@ -30,6 +30,12 @@ import stat
|
||||||
import fcntl
|
import fcntl
|
||||||
import copy
|
import copy
|
||||||
|
|
||||||
|
try:
|
||||||
|
import cPickle as pickle
|
||||||
|
except ImportError:
|
||||||
|
import pickle
|
||||||
|
bb.msg.note(1, bb.msg.domain.Cache, "Importing cPickle failed. Falling back to a very slow implementation.")
|
||||||
|
|
||||||
class RunQueueStats:
|
class RunQueueStats:
|
||||||
"""
|
"""
|
||||||
Holds statistics on the tasks handled by the associated runQueue
|
Holds statistics on the tasks handled by the associated runQueue
|
||||||
|
@ -703,6 +709,21 @@ class RunQueueData:
|
||||||
procdep.append(self.taskData.fn_index[self.runq_fnid[dep]] + "." + self.runq_task[dep])
|
procdep.append(self.taskData.fn_index[self.runq_fnid[dep]] + "." + self.runq_task[dep])
|
||||||
self.runq_hash[task] = bb.parse.siggen.get_taskhash(self.taskData.fn_index[self.runq_fnid[task]], self.runq_task[task], procdep, self.dataCache)
|
self.runq_hash[task] = bb.parse.siggen.get_taskhash(self.taskData.fn_index[self.runq_fnid[task]], self.runq_task[task], procdep, self.dataCache)
|
||||||
|
|
||||||
|
hashdata = {}
|
||||||
|
hashdata["hashes"] = {}
|
||||||
|
hashdata["deps"] = {}
|
||||||
|
for task in range(len(self.runq_fnid)):
|
||||||
|
hashdata["hashes"][self.taskData.fn_index[self.runq_fnid[task]] + "." + self.runq_task[task]] = self.runq_hash[task]
|
||||||
|
deps = []
|
||||||
|
for dep in self.runq_depends[task]:
|
||||||
|
deps.append(self.taskData.fn_index[self.runq_fnid[dep]] + "." + self.runq_task[dep])
|
||||||
|
hashdata["deps"][self.taskData.fn_index[self.runq_fnid[task]] + "." + self.runq_task[task]] = deps
|
||||||
|
|
||||||
|
# Write out the hashes into a file for use by the individual tasks
|
||||||
|
self.hashfile = bb.data.expand("${TMPDIR}/cache/hashdata.dat", self.cooker.configuration.data)
|
||||||
|
p = pickle.Pickler(file(self.hashfile, "wb"), -1)
|
||||||
|
p.dump(hashdata)
|
||||||
|
|
||||||
return len(self.runq_fnid)
|
return len(self.runq_fnid)
|
||||||
|
|
||||||
def dump_data(self, taskQueue):
|
def dump_data(self, taskQueue):
|
||||||
|
@ -1047,7 +1068,7 @@ class RunQueueExecute:
|
||||||
sys.stdout.flush()
|
sys.stdout.flush()
|
||||||
sys.stderr.flush()
|
sys.stderr.flush()
|
||||||
|
|
||||||
proc = subprocess.Popen(["bitbake-runtask", fn, taskname, str(self.cooker.configuration.dry_run)], env=env, stdout=subprocess.PIPE, stdin=subprocess.PIPE)
|
proc = subprocess.Popen(["bitbake-runtask", self.rqdata.hashfile, fn, taskname, str(self.cooker.configuration.dry_run)], env=env, stdout=subprocess.PIPE, stdin=subprocess.PIPE)
|
||||||
pipein = proc.stdout
|
pipein = proc.stdout
|
||||||
pipeout = proc.stdin
|
pipeout = proc.stdin
|
||||||
pid = proc.pid
|
pid = proc.pid
|
||||||
|
|
|
@ -107,6 +107,10 @@ class SignatureGeneratorBasic(SignatureGenerator):
|
||||||
#d.setVar("BB_TASKHASH_task-%s" % task, taskhash[task])
|
#d.setVar("BB_TASKHASH_task-%s" % task, taskhash[task])
|
||||||
return h
|
return h
|
||||||
|
|
||||||
|
def set_taskdata(self, hashes, deps):
|
||||||
|
self.runtaskdeps = deps
|
||||||
|
self.taskhash = hashes
|
||||||
|
|
||||||
def dump_sigtask(self, fn, task, stampbase, runtime):
|
def dump_sigtask(self, fn, task, stampbase, runtime):
|
||||||
k = fn + "." + task
|
k = fn + "." + task
|
||||||
if runtime == "customfile":
|
if runtime == "customfile":
|
||||||
|
@ -128,7 +132,7 @@ class SignatureGeneratorBasic(SignatureGenerator):
|
||||||
data['gendeps'][dep] = self.gendeps[fn][dep]
|
data['gendeps'][dep] = self.gendeps[fn][dep]
|
||||||
data['varvals'][dep] = self.lookupcache[fn][dep]
|
data['varvals'][dep] = self.lookupcache[fn][dep]
|
||||||
|
|
||||||
if runtime and runtime != "customfile":
|
if runtime:
|
||||||
data['runtaskdeps'] = self.runtaskdeps[k]
|
data['runtaskdeps'] = self.runtaskdeps[k]
|
||||||
data['runtaskhashes'] = {}
|
data['runtaskhashes'] = {}
|
||||||
for dep in data['runtaskdeps']:
|
for dep in data['runtaskdeps']:
|
||||||
|
|
Loading…
Reference in New Issue