sysmo-bts
/
generic-poky
9
0
Fork 0
Code Pull Requests Releases Activity

bitbake: taskdata/runqueue: Rewrite without use of ID indirection 

I'm not sure what possesed me when I wrote this code originally but its
indirection of everyting to use numeric IDs and position dependent lists
is horrific. Given the way python internals work, its completely and
utterly pointless from  performance perspective. It also makes the code
hard to understand and debug since any numeric ID has to be translated
into something human readable.

The hard part is that the IDs are infectous and spread from taskdata
into runqueue and even partly into cooker for the dependency graph
processing. The only real way to deal with this is to convert everything
to use a more sane data structure.

This patch:
* Uses "<fn>:<taskname>" as the ID for tasks rather than a number
* Changes to dict() based structures rather than position dependent lists
* Drops the build name, runtime name and filename ID indexes

On the most part there shouldn't be user visible changes. Sadly we did
leak datastructures to the setscene verify function which has to be
rewritten. To handle this, the variable name used to specifiy the version
changes from BB_SETSCENE_VERIFY_FUNCTION to BB_SETSCENE_VERIFY_FUNCTION2
allowing multiple versions of bitbake to work with suitably written
metadata. Anyone with custom schedulers may also need to change them.

I believe the benefits in code readability and easier debugging far
outweigh those issues though. It also means we have a saner codebase
to add multiconfig support on top of.

During development, I did have some of the original code coexisting with
the new data stores to allow comparision of the data and check it was
working correcty, particuarly for taskdata. I have also compared
task-depends.dot files before and after the change. There should be no
functionality changes in this patch, its purely a data structure change
and that is visible in the patch.

(Bitbake rev: 2c88afb60da54e58f555411a7bd7b006b0c29306)

Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
This commit is contained in:
Richard Purdie 2016-06-12 23:55:48 +01:00
parent 29b4fb1796
commit af04a52bf7
3 changed files with 635 additions and 799 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

72
bitbake/lib/bb/cooker.py
View File

@ -623,9 +623,7 @@ class BBCooker:
taskdata, runlist, pkgs_to_build = self.buildTaskData(pkgs_to_build, None, self.configuration.abort, allowincomplete=True)
targetid = taskdata.getbuild_id(pkgs_to_build[0])
fnid = taskdata.build_targets[targetid][0]
fn = taskdata.fn_index[fnid]
fn = taskdata.build_targets[pkgs_to_build[0]][0]
else:
envdata = self.data
@ -714,7 +712,7 @@ class BBCooker:
def buildDependTree(self, rq, taskdata):
seen_fnids = []
seen_fns = []
depend_tree = {}
depend_tree["depends"] = {}
depend_tree["tdepends"] = {}
@ -732,10 +730,9 @@ class BBCooker:
version = "%s:%s-%s" % self.recipecache.pkg_pepvpr[fn]
depend_tree['providermap'][name] = (pn, version)
for task in range(len(rq.rqdata.runq_fnid)):
taskname = rq.rqdata.runq_task[task]
fnid = rq.rqdata.runq_fnid[task]
fn = taskdata.fn_index[fnid]
for tid in rq.rqdata.runtaskentries:
taskname = bb.runqueue.taskname_from_tid(tid)
fn = bb.runqueue.fn_from_tid(tid)
pn = self.recipecache.pkg_fn[fn]
version = "%s:%s-%s" % self.recipecache.pkg_pepvpr[fn]
if pn not in depend_tree["pn"]:
@ -756,24 +753,24 @@ class BBCooker:
depend_tree["pn"][pn][ei] = vars(self.recipecache)[ei][fn]
for dep in rq.rqdata.runq_depends[task]:
depfn = taskdata.fn_index[rq.rqdata.runq_fnid[dep]]
for dep in rq.rqdata.runtaskentries[tid].depends:
depfn = bb.runqueue.fn_from_tid(dep)
deppn = self.recipecache.pkg_fn[depfn]
dotname = "%s.%s" % (pn, rq.rqdata.runq_task[task])
dotname = "%s.%s" % (pn, bb.runqueue.taskname_from_tid(dep))
if not dotname in depend_tree["tdepends"]:
depend_tree["tdepends"][dotname] = []
depend_tree["tdepends"][dotname].append("%s.%s" % (deppn, rq.rqdata.runq_task[dep]))
if fnid not in seen_fnids:
seen_fnids.append(fnid)
depend_tree["tdepends"][dotname].append("%s.%s" % (deppn, bb.runqueue.taskname_from_tid(dep)))
if fn not in seen_fns:
seen_fns.append(fn)
packages = []
depend_tree["depends"][pn] = []
for dep in taskdata.depids[fnid]:
depend_tree["depends"][pn].append(taskdata.build_names_index[dep])
for dep in taskdata.depids[fn]:
depend_tree["depends"][pn].append(dep)
depend_tree["rdepends-pn"][pn] = []
for rdep in taskdata.rdepids[fnid]:
depend_tree["rdepends-pn"][pn].append(taskdata.run_names_index[rdep])
for rdep in taskdata.rdepids[fn]:
depend_tree["rdepends-pn"][pn].append(rdep)
rdepends = self.recipecache.rundeps[fn]
for package in rdepends:
@ -805,12 +802,8 @@ class BBCooker:
Create a dependency tree of pkgs_to_build, returning the data.
"""
_, taskdata = self.prepareTreeData(pkgs_to_build, task)
tasks_fnid = []
if len(taskdata.tasks_name) != 0:
for task in range(len(taskdata.tasks_name)):
tasks_fnid.append(taskdata.tasks_fnid[task])
seen_fnids = []
seen_fns = []
depend_tree = {}
depend_tree["depends"] = {}
depend_tree["pn"] = {}
@ -825,9 +818,8 @@ class BBCooker:
cachefields = getattr(cache_class, 'cachefields', [])
extra_info = extra_info + cachefields
for task in range(len(tasks_fnid)):
fnid = tasks_fnid[task]
fn = taskdata.fn_index[fnid]
for tid in taskdata.taskentries:
fn = bb.runqueue.fn_from_tid(tid)
pn = self.recipecache.pkg_fn[fn]
if pn not in depend_tree["pn"]:
@ -843,33 +835,27 @@ class BBCooker:
for ei in extra_info:
depend_tree["pn"][pn][ei] = vars(self.recipecache)[ei][fn]
if fnid not in seen_fnids:
seen_fnids.append(fnid)
if fn not in seen_fns:
seen_fns.append(fn)
depend_tree["depends"][pn] = []
for dep in taskdata.depids[fnid]:
item = taskdata.build_names_index[dep]
for item in taskdata.depids[fn]:
pn_provider = ""
targetid = taskdata.getbuild_id(item)
if targetid in taskdata.build_targets and taskdata.build_targets[targetid]:
id = taskdata.build_targets[targetid][0]
fn_provider = taskdata.fn_index[id]
if dep in taskdata.build_targets and taskdata.build_targets[dep]:
fn_provider = taskdata.build_targets[dep][0]
pn_provider = self.recipecache.pkg_fn[fn_provider]
else:
pn_provider = item
depend_tree["depends"][pn].append(pn_provider)
depend_tree["rdepends-pn"][pn] = []
for rdep in taskdata.rdepids[fnid]:
item = taskdata.run_names_index[rdep]
for rdep in taskdata.rdepids[fn]:
pn_rprovider = ""
targetid = taskdata.getrun_id(item)
if targetid in taskdata.run_targets and taskdata.run_targets[targetid]:
id = taskdata.run_targets[targetid][0]
fn_rprovider = taskdata.fn_index[id]
if rdep in taskdata.run_targets and taskdata.run_targets[rdep]:
fn_rprovider = taskdata.run_targets[rdep][0]
pn_rprovider = self.recipecache.pkg_fn[fn_rprovider]
else:
pn_rprovider = item
pn_rprovider = rdep
depend_tree["rdepends-pn"][pn].append(pn_rprovider)
depend_tree["rdepends-pkg"].update(rdepends)
@ -1350,7 +1336,7 @@ class BBCooker:
return False
if not retval:
bb.event.fire(bb.event.BuildCompleted(len(rq.rqdata.runq_fnid), buildname, item, failures, interrupted), self.expanded_data)
bb.event.fire(bb.event.BuildCompleted(len(rq.rqdata.runtaskentries), buildname, item, failures, interrupted), self.expanded_data)
self.command.finishAsyncCommand(msg)
return False
if retval is True:
@ -1386,7 +1372,7 @@ class BBCooker:
return False
if not retval:
bb.event.fire(bb.event.BuildCompleted(len(rq.rqdata.runq_fnid), buildname, targets, failures, interrupted), self.data)
bb.event.fire(bb.event.BuildCompleted(len(rq.rqdata.runtaskentries), buildname, targets, failures, interrupted), self.data)
self.command.finishAsyncCommand(msg)
return False
if retval is True:

954
bitbake/lib/bb/runqueue.py
View File

File diff suppressed because it is too large Load Diff

408
bitbake/lib/bb/taskdata.py
View File

@ -37,27 +37,24 @@ def re_match_strings(target, strings):
return any(name == target or re.match(name, target)
for name in strings)
class TaskEntry:
def __init__(self):
self.tdepends = []
self.idepends = []
self.irdepends = []
class TaskData:
"""
BitBake Task Data implementation
"""
def __init__(self, abort = True, tryaltconfigs = False, skiplist = None, allowincomplete = False):
self.build_names_index = []
self.run_names_index = []
self.fn_index = []
self.build_targets = {}
self.run_targets = {}
self.external_targets = []
self.tasks_fnid = []
self.tasks_name = []
self.tasks_tdepends = []
self.tasks_idepends = []
self.tasks_irdepends = []
# Cache to speed up task ID lookups
self.tasks_lookup = {}
self.seenfns = []
self.taskentries = {}
self.depids = {}
self.rdepids = {}
@ -66,7 +63,7 @@ class TaskData:
self.failed_deps = []
self.failed_rdeps = []
self.failed_fnids = []
self.failed_fns = []
self.abort = abort
self.tryaltconfigs = tryaltconfigs
@ -74,88 +71,6 @@ class TaskData:
self.skiplist = skiplist
def getbuild_id(self, name):
"""
Return an ID number for the build target name.
If it doesn't exist, create one.
"""
if not name in self.build_names_index:
self.build_names_index.append(name)
return len(self.build_names_index) - 1
return self.build_names_index.index(name)
def getrun_id(self, name):
"""
Return an ID number for the run target name.
If it doesn't exist, create one.
"""
if not name in self.run_names_index:
self.run_names_index.append(name)
return len(self.run_names_index) - 1
return self.run_names_index.index(name)
def getfn_id(self, name):
"""
Return an ID number for the filename.
If it doesn't exist, create one.
"""
if not name in self.fn_index:
self.fn_index.append(name)
return len(self.fn_index) - 1
return self.fn_index.index(name)
def gettask_ids(self, fnid):
"""
Return an array of the ID numbers matching a given fnid.
"""
ids = []
if fnid in self.tasks_lookup:
for task in self.tasks_lookup[fnid]:
ids.append(self.tasks_lookup[fnid][task])
return ids
def gettask_id_fromfnid(self, fnid, task):
"""
Return an ID number for the task matching fnid and task.
"""
if fnid in self.tasks_lookup:
if task in self.tasks_lookup[fnid]:
return self.tasks_lookup[fnid][task]
return None
def gettask_id(self, fn, task, create = True):
"""
Return an ID number for the task matching fn and task.
If it doesn't exist, create one by default.
Optionally return None instead.
"""
fnid = self.getfn_id(fn)
if fnid in self.tasks_lookup:
if task in self.tasks_lookup[fnid]:
return self.tasks_lookup[fnid][task]
if not create:
return None
self.tasks_name.append(task)
self.tasks_fnid.append(fnid)
self.tasks_tdepends.append([])
self.tasks_idepends.append([])
self.tasks_irdepends.append([])
listid = len(self.tasks_name) - 1
if fnid not in self.tasks_lookup:
self.tasks_lookup[fnid] = {}
self.tasks_lookup[fnid][task] = listid
return listid
def add_tasks(self, fn, dataCache):
"""
Add tasks for a given fn to the database
@ -163,29 +78,31 @@ class TaskData:
task_deps = dataCache.task_deps[fn]
fnid = self.getfn_id(fn)
if fnid in self.failed_fnids:
if fn in self.failed_fns:
bb.msg.fatal("TaskData", "Trying to re-add a failed file? Something is broken...")
# Check if we've already seen this fn
if fnid in self.tasks_fnid:
if fn in self.seenfns:
return
self.seenfns.append(fn)
self.add_extra_deps(fn, dataCache)
for task in task_deps['tasks']:
tid = "%s:%s" % (fn, task)
self.taskentries[tid] = TaskEntry()
# Work out task dependencies
parentids = []
for dep in task_deps['parents'][task]:
if dep not in task_deps['tasks']:
bb.debug(2, "Not adding dependeny of %s on %s since %s does not exist" % (task, dep, dep))
continue
parentid = self.gettask_id(fn, dep)
parentid = "%s:%s" % (fn, dep)
parentids.append(parentid)
taskid = self.gettask_id(fn, task)
self.tasks_tdepends[taskid].extend(parentids)
self.taskentries[tid].tdepends.extend(parentids)
# Touch all intertask dependencies
if 'depends' in task_deps and task in task_deps['depends']:
@ -194,29 +111,30 @@ class TaskData:
if dep:
if ":" not in dep:
bb.msg.fatal("TaskData", "Error for %s, dependency %s does not contain ':' character\n. Task 'depends' should be specified in the form 'packagename:task'" % (fn, dep))
ids.append(((self.getbuild_id(dep.split(":")[0])), dep.split(":")[1]))
self.tasks_idepends[taskid].extend(ids)
ids.append(((dep.split(":")[0]), dep.split(":")[1]))
self.seen_build_target(dep.split(":")[0])
self.taskentries[tid].idepends.extend(ids)
if 'rdepends' in task_deps and task in task_deps['rdepends']:
ids = []
for dep in task_deps['rdepends'][task].split():
if dep:
if ":" not in dep:
bb.msg.fatal("TaskData", "Error for %s, dependency %s does not contain ':' character\n. Task 'rdepends' should be specified in the form 'packagename:task'" % (fn, dep))
ids.append(((self.getrun_id(dep.split(":")[0])), dep.split(":")[1]))
self.tasks_irdepends[taskid].extend(ids)
ids.append(((dep.split(":")[0]), dep.split(":")[1]))
self.seen_run_target(dep.split(":")[0])
self.taskentries[tid].irdepends.extend(ids)
# Work out build dependencies
if not fnid in self.depids:
dependids = {}
if not fn in self.depids:
dependids = set()
for depend in dataCache.deps[fn]:
dependids[self.getbuild_id(depend)] = None
self.depids[fnid] = dependids.keys()
dependids.add(depend)
self.depids[fn] = list(dependids)
logger.debug(2, "Added dependencies %s for %s", str(dataCache.deps[fn]), fn)
# Work out runtime dependencies
if not fnid in self.rdepids:
rdependids = {}
if not fn in self.rdepids:
rdependids = set()
rdepends = dataCache.rundeps[fn]
rrecs = dataCache.runrecs[fn]
rdependlist = []
@ -224,24 +142,26 @@ class TaskData:
for package in rdepends:
for rdepend in rdepends[package]:
rdependlist.append(rdepend)
rdependids[self.getrun_id(rdepend)] = None
rdependids.add(rdepend)
for package in rrecs:
for rdepend in rrecs[package]:
rreclist.append(rdepend)
rdependids[self.getrun_id(rdepend)] = None
rdependids.add(rdepend)
if rdependlist:
logger.debug(2, "Added runtime dependencies %s for %s", str(rdependlist), fn)
if rreclist:
logger.debug(2, "Added runtime recommendations %s for %s", str(rreclist), fn)
self.rdepids[fnid] = rdependids.keys()
self.rdepids[fn] = list(rdependids)
for dep in self.depids[fnid]:
for dep in self.depids[fn]:
self.seen_build_target(dep)
if dep in self.failed_deps:
self.fail_fnid(fnid)
self.fail_fn(fn)
return
for dep in self.rdepids[fnid]:
for dep in self.rdepids[fn]:
self.seen_run_target(dep)
if dep in self.failed_rdeps:
self.fail_fnid(fnid)
self.fail_fn(fn)
return
def add_extra_deps(self, fn, dataCache):
@ -263,9 +183,7 @@ class TaskData:
"""
Have we a build target matching this name?
"""
targetid = self.getbuild_id(target)
if targetid in self.build_targets:
if target in self.build_targets and self.build_targets[target]:
return True
return False
@ -273,50 +191,54 @@ class TaskData:
"""
Have we a runtime target matching this name?
"""
targetid = self.getrun_id(target)
if targetid in self.run_targets:
if target in self.run_targets and self.run_targets[target]:
return True
return False
def seen_build_target(self, name):
"""
Maintain a list of build targets
"""
if name not in self.build_targets:
self.build_targets[name] = []
def add_build_target(self, fn, item):
"""
Add a build target.
If already present, append the provider fn to the list
"""
targetid = self.getbuild_id(item)
fnid = self.getfn_id(fn)
if targetid in self.build_targets:
if fnid in self.build_targets[targetid]:
if item in self.build_targets:
if fn in self.build_targets[item]:
return
self.build_targets[targetid].append(fnid)
self.build_targets[item].append(fn)
return
self.build_targets[targetid] = [fnid]
self.build_targets[item] = [fn]
def seen_run_target(self, name):
"""
Maintain a list of runtime build targets
"""
if name not in self.run_targets:
self.run_targets[name] = []
def add_runtime_target(self, fn, item):
"""
Add a runtime target.
If already present, append the provider fn to the list
"""
targetid = self.getrun_id(item)
fnid = self.getfn_id(fn)
if targetid in self.run_targets:
if fnid in self.run_targets[targetid]:
if item in self.run_targets:
if fn in self.run_targets[item]:
return
self.run_targets[targetid].append(fnid)
self.run_targets[item].append(fn)
return
self.run_targets[targetid] = [fnid]
self.run_targets[item] = [fn]
def mark_external_target(self, item):
def mark_external_target(self, target):
"""
Mark a build target as being externally requested
"""
targetid = self.getbuild_id(item)
if targetid not in self.external_targets:
self.external_targets.append(targetid)
if target not in self.external_targets:
self.external_targets.append(target)
def get_unresolved_build_targets(self, dataCache):
"""
@ -324,12 +246,12 @@ class TaskData:
are unknown.
"""
unresolved = []
for target in self.build_names_index:
for target in self.build_targets:
if re_match_strings(target, dataCache.ignored_dependencies):
continue
if self.build_names_index.index(target) in self.failed_deps:
if target in self.failed_deps:
continue
if not self.have_build_target(target):
if not self.build_targets[target]:
unresolved.append(target)
return unresolved
@ -339,12 +261,12 @@ class TaskData:
are unknown.
"""
unresolved = []
for target in self.run_names_index:
for target in self.run_targets:
if re_match_strings(target, dataCache.ignored_dependencies):
continue
if self.run_names_index.index(target) in self.failed_rdeps:
if target in self.failed_rdeps:
continue
if not self.have_runtime_target(target):
if not self.run_targets[target]:
unresolved.append(target)
return unresolved
@ -352,50 +274,26 @@ class TaskData:
"""
Return a list of providers of item
"""
targetid = self.getbuild_id(item)
return self.build_targets[item]
return self.build_targets[targetid]
def get_dependees(self, itemid):
def get_dependees(self, item):
"""
Return a list of targets which depend on item
"""
dependees = []
for fnid in self.depids:
if itemid in self.depids[fnid]:
dependees.append(fnid)
for fn in self.depids:
if item in self.depids[fn]:
dependees.append(fn)
return dependees
def get_dependees_str(self, item):
"""
Return a list of targets which depend on item as a user readable string
"""
itemid = self.getbuild_id(item)
dependees = []
for fnid in self.depids:
if itemid in self.depids[fnid]:
dependees.append(self.fn_index[fnid])
return dependees
def get_rdependees(self, itemid):
def get_rdependees(self, item):
"""
Return a list of targets which depend on runtime item
"""
dependees = []
for fnid in self.rdepids:
if itemid in self.rdepids[fnid]:
dependees.append(fnid)
return dependees
def get_rdependees_str(self, item):
"""
Return a list of targets which depend on runtime item as a user readable string
"""
itemid = self.getrun_id(item)
dependees = []
for fnid in self.rdepids:
if itemid in self.rdepids[fnid]:
dependees.append(self.fn_index[fnid])
for fn in self.rdepids:
if item in self.rdepids[fn]:
dependees.append(fn)
return dependees
def get_reasons(self, item, runtime=False):
@ -431,7 +329,7 @@ class TaskData:
except bb.providers.NoProvider:
if self.abort:
raise
self.remove_buildtarget(self.getbuild_id(item))
self.remove_buildtarget(item)
self.mark_external_target(item)
@ -453,7 +351,7 @@ class TaskData:
new = dataCache.pkg_fn[fn] + " RPROVIDES " + item
if new not in close_matches:
close_matches.append(new)
bb.event.fire(bb.event.NoProvider(item, dependees=self.get_dependees_str(item), reasons=self.get_reasons(item), close_matches=close_matches), cfgData)
bb.event.fire(bb.event.NoProvider(item, dependees=self.get_dependees(item), reasons=self.get_reasons(item), close_matches=close_matches), cfgData)
raise bb.providers.NoProvider(item)
if self.have_build_target(item):
@ -462,10 +360,10 @@ class TaskData:
all_p = dataCache.providers[item]
eligible, foundUnique = bb.providers.filterProviders(all_p, item, cfgData, dataCache)
eligible = [p for p in eligible if not self.getfn_id(p) in self.failed_fnids]
eligible = [p for p in eligible if not p in self.failed_fns]
if not eligible:
bb.event.fire(bb.event.NoProvider(item, dependees=self.get_dependees_str(item), reasons=["No eligible PROVIDERs exist for '%s'" % item]), cfgData)
bb.event.fire(bb.event.NoProvider(item, dependees=self.get_dependees(item), reasons=["No eligible PROVIDERs exist for '%s'" % item]), cfgData)
raise bb.providers.NoProvider(item)
if len(eligible) > 1 and foundUnique == False:
@ -477,8 +375,7 @@ class TaskData:
self.consider_msgs_cache.append(item)
for fn in eligible:
fnid = self.getfn_id(fn)
if fnid in self.failed_fnids:
if fn in self.failed_fns:
continue
logger.debug(2, "adding %s to satisfy %s", fn, item)
self.add_build_target(fn, item)
@ -502,14 +399,14 @@ class TaskData:
all_p = bb.providers.getRuntimeProviders(dataCache, item)
if not all_p:
bb.event.fire(bb.event.NoProvider(item, runtime=True, dependees=self.get_rdependees_str(item), reasons=self.get_reasons(item, True)), cfgData)
bb.event.fire(bb.event.NoProvider(item, runtime=True, dependees=self.get_rdependees(item), reasons=self.get_reasons(item, True)), cfgData)
raise bb.providers.NoRProvider(item)
eligible, numberPreferred = bb.providers.filterProvidersRunTime(all_p, item, cfgData, dataCache)
eligible = [p for p in eligible if not self.getfn_id(p) in self.failed_fnids]
eligible = [p for p in eligible if not p in self.failed_fns]
if not eligible:
bb.event.fire(bb.event.NoProvider(item, runtime=True, dependees=self.get_rdependees_str(item), reasons=["No eligible RPROVIDERs exist for '%s'" % item]), cfgData)
bb.event.fire(bb.event.NoProvider(item, runtime=True, dependees=self.get_rdependees(item), reasons=["No eligible RPROVIDERs exist for '%s'" % item]), cfgData)
raise bb.providers.NoRProvider(item)
if len(eligible) > 1 and numberPreferred == 0:
@ -531,82 +428,80 @@ class TaskData:
# run through the list until we find one that we can build
for fn in eligible:
fnid = self.getfn_id(fn)
if fnid in self.failed_fnids:
if fn in self.failed_fns:
continue
logger.debug(2, "adding '%s' to satisfy runtime '%s'", fn, item)
self.add_runtime_target(fn, item)
self.add_tasks(fn, dataCache)
def fail_fnid(self, fnid, missing_list=None):
def fail_fn(self, fn, missing_list=None):
"""
Mark a file as failed (unbuildable)
Remove any references from build and runtime provider lists
missing_list, A list of missing requirements for this target
"""
if fnid in self.failed_fnids:
if fn in self.failed_fns:
return
if not missing_list:
missing_list = []
logger.debug(1, "File '%s' is unbuildable, removing...", self.fn_index[fnid])
self.failed_fnids.append(fnid)
logger.debug(1, "File '%s' is unbuildable, removing...", fn)
self.failed_fns.append(fn)
for target in self.build_targets:
if fnid in self.build_targets[target]:
self.build_targets[target].remove(fnid)
if fn in self.build_targets[target]:
self.build_targets[target].remove(fn)
if len(self.build_targets[target]) == 0:
self.remove_buildtarget(target, missing_list)
for target in self.run_targets:
if fnid in self.run_targets[target]:
self.run_targets[target].remove(fnid)
if fn in self.run_targets[target]:
self.run_targets[target].remove(fn)
if len(self.run_targets[target]) == 0:
self.remove_runtarget(target, missing_list)
def remove_buildtarget(self, targetid, missing_list=None):
def remove_buildtarget(self, target, missing_list=None):
"""
Mark a build target as failed (unbuildable)
Trigger removal of any files that have this as a dependency
"""
if not missing_list:
missing_list = [self.build_names_index[targetid]]
missing_list = [target]
else:
missing_list = [self.build_names_index[targetid]] + missing_list
logger.verbose("Target '%s' is unbuildable, removing...\nMissing or unbuildable dependency chain was: %s", self.build_names_index[targetid], missing_list)
self.failed_deps.append(targetid)
dependees = self.get_dependees(targetid)
for fnid in dependees:
self.fail_fnid(fnid, missing_list)
for taskid in range(len(self.tasks_idepends)):
idepends = self.tasks_idepends[taskid]
for (idependid, idependtask) in idepends:
if idependid == targetid:
self.fail_fnid(self.tasks_fnid[taskid], missing_list)
missing_list = [target] + missing_list
logger.verbose("Target '%s' is unbuildable, removing...\nMissing or unbuildable dependency chain was: %s", target, missing_list)
self.failed_deps.append(target)
dependees = self.get_dependees(target)
for fn in dependees:
self.fail_fn(fn, missing_list)
for tid in self.taskentries:
for (idepend, idependtask) in self.taskentries[tid].idepends:
if idepend == target:
fn = tid.rsplit(":",1)[0]
self.fail_fn(fn, missing_list)
if self.abort and targetid in self.external_targets:
target = self.build_names_index[targetid]
if self.abort and target in self.external_targets:
logger.error("Required build target '%s' has no buildable providers.\nMissing or unbuildable dependency chain was: %s", target, missing_list)
raise bb.providers.NoProvider(target)
def remove_runtarget(self, targetid, missing_list=None):
def remove_runtarget(self, target, missing_list=None):
"""
Mark a run target as failed (unbuildable)
Trigger removal of any files that have this as a dependency
"""
if not missing_list:
missing_list = [self.run_names_index[targetid]]
missing_list = [target]
else:
missing_list = [self.run_names_index[targetid]] + missing_list
missing_list = [target] + missing_list
logger.info("Runtime target '%s' is unbuildable, removing...\nMissing or unbuildable dependency chain was: %s", self.run_names_index[targetid], missing_list)
self.failed_rdeps.append(targetid)
dependees = self.get_rdependees(targetid)
for fnid in dependees:
self.fail_fnid(fnid, missing_list)
for taskid in range(len(self.tasks_irdepends)):
irdepends = self.tasks_irdepends[taskid]
for (idependid, idependtask) in irdepends:
if idependid == targetid:
self.fail_fnid(self.tasks_fnid[taskid], missing_list)
logger.info("Runtime target '%s' is unbuildable, removing...\nMissing or unbuildable dependency chain was: %s", target, missing_list)
self.failed_rdeps.append(target)
dependees = self.get_rdependees(target)
for fn in dependees:
self.fail_fn(fn, missing_list)
for tid in self.taskentries:
for (idepend, idependtask) in self.taskentries[tid].irdepends:
if idepend == target:
fn = tid.rsplit(":",1)[0]
self.fail_fn(fn, missing_list)
def add_unresolved(self, cfgData, dataCache):
"""
@ -620,17 +515,16 @@ class TaskData:
self.add_provider_internal(cfgData, dataCache, target)
added = added + 1
except bb.providers.NoProvider:
targetid = self.getbuild_id(target)
if self.abort and targetid in self.external_targets and not self.allowincomplete:
if self.abort and target in self.external_targets and not self.allowincomplete:
raise
if not self.allowincomplete:
self.remove_buildtarget(targetid)
self.remove_buildtarget(target)
for target in self.get_unresolved_run_targets(dataCache):
try:
self.add_rprovider(cfgData, dataCache, target)
added = added + 1
except (bb.providers.NoRProvider, bb.providers.MultipleRProvider):
self.remove_runtarget(self.getrun_id(target))
self.remove_runtarget(target)
logger.debug(1, "Resolved " + str(added) + " extra dependencies")
if added == 0:
break
@ -638,13 +532,13 @@ class TaskData:
def get_providermap(self, prefix=None):
provmap = {}
for name in self.build_names_index:
for name in self.build_targets:
if prefix and not name.startswith(prefix):
continue
if self.have_build_target(name):
provider = self.get_provider(name)
if provider:
provmap[name] = self.fn_index[provider[0]]
provmap[name] = provider[0]
return provmap
def dump_data(self):
@ -652,39 +546,37 @@ class TaskData:
Dump some debug information on the internal data structures
"""
logger.debug(3, "build_names:")
logger.debug(3, ", ".join(self.build_names_index))
logger.debug(3, ", ".join(self.build_targets))
logger.debug(3, "run_names:")
logger.debug(3, ", ".join(self.run_names_index))
logger.debug(3, ", ".join(self.run_targets))
logger.debug(3, "build_targets:")
for buildid in range(len(self.build_names_index)):
target = self.build_names_index[buildid]
for target in self.build_targets:
targets = "None"
if buildid in self.build_targets:
targets = self.build_targets[buildid]
logger.debug(3, " (%s)%s: %s", buildid, target, targets)
if target in self.build_targets:
targets = self.build_targets[target]
logger.debug(3, " %s: %s", target, targets)
logger.debug(3, "run_targets:")
for runid in range(len(self.run_names_index)):
target = self.run_names_index[runid]
for target in self.run_targets:
targets = "None"
if runid in self.run_targets:
targets = self.run_targets[runid]
logger.debug(3, " (%s)%s: %s", runid, target, targets)
if target in self.run_targets:
targets = self.run_targets[target]
logger.debug(3, " %s: %s", target, targets)
logger.debug(3, "tasks:")
for task in range(len(self.tasks_name)):
logger.debug(3, " (%s)%s - %s: %s",
task,
self.fn_index[self.tasks_fnid[task]],
self.tasks_name[task],
self.tasks_tdepends[task])
for tid in self.taskentries:
logger.debug(3, " %s: %s %s %s",
tid,
self.taskentries[tid].idepends,
self.taskentries[tid].irdepends,
self.taskentries[tid].tdepends)
logger.debug(3, "dependency ids (per fn):")
for fnid in self.depids:
logger.debug(3, " %s %s: %s", fnid, self.fn_index[fnid], self.depids[fnid])
for fn in self.depids:
logger.debug(3, " %s: %s", fn, self.depids[fn])
logger.debug(3, "runtime dependency ids (per fn):")
for fnid in self.rdepids:
logger.debug(3, " %s %s: %s", fnid, self.fn_index[fnid], self.rdepids[fnid])
for fn in self.rdepids:
logger.debug(3, " %s: %s", fn, self.rdepids[fn])