cache.py: Refactory Current Cache implementation
This patch is for refactorying current cache implementation, the main reason is for introducing extra cache fields requests for image creator as well as other users. The refactory parts include: Move cache data retrieve methods out of Cache Data Fields Definition. Since this retrieve methods will be shared for both CoreRecipeInfo as well as the new introduced extra RecipeInfo in the following patches. (Bitbake rev: f0f53506926a3f79181796dde177f11f0a396b75) Signed-off-by: Liping Ke <liping.ke@intel.com> Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
This commit is contained in:
Liping Ke
Richard Purdie
parent
911e8bb56a
commit
8df355140c
|
|
@ -30,7 +30,7 @@
|
||||||
|
|
||||||
import os
|
import os
|
||||||
import logging
|
import logging
|
||||||
from collections import defaultdict, namedtuple
|
from collections import defaultdict
|
||||||
import bb.data
|
import bb.data
|
||||||
import bb.utils
|
import bb.utils
|
||||||
|
|
||||||
|
|
@ -45,46 +45,11 @@ except ImportError:
|
||||||
|
|
||||||
__cache_version__ = "138"
|
__cache_version__ = "138"
|
||||||
|
|
||||||
recipe_fields = (
|
|
||||||
'pn',
|
|
||||||
'pv',
|
|
||||||
'pr',
|
|
||||||
'pe',
|
|
||||||
'defaultpref',
|
|
||||||
'depends',
|
|
||||||
'provides',
|
|
||||||
'task_deps',
|
|
||||||
'stamp',
|
|
||||||
'stamp_extrainfo',
|
|
||||||
'broken',
|
|
||||||
'not_world',
|
|
||||||
'skipped',
|
|
||||||
'timestamp',
|
|
||||||
'packages',
|
|
||||||
'packages_dynamic',
|
|
||||||
'rdepends',
|
|
||||||
'rdepends_pkg',
|
|
||||||
'rprovides',
|
|
||||||
'rprovides_pkg',
|
|
||||||
'rrecommends',
|
|
||||||
'rrecommends_pkg',
|
|
||||||
'nocache',
|
|
||||||
'variants',
|
|
||||||
'file_depends',
|
|
||||||
'tasks',
|
|
||||||
'basetaskhashes',
|
|
||||||
'hashfilename',
|
|
||||||
'inherits',
|
|
||||||
'summary',
|
|
||||||
'license',
|
|
||||||
'section',
|
|
||||||
'fakerootenv',
|
|
||||||
'fakerootdirs'
|
|
||||||
)
|
|
||||||
|
|
||||||
|
# RecipeInfoCommon defines common data retrieving methods
|
||||||
class RecipeInfo(namedtuple('RecipeInfo', recipe_fields)):
|
# from meta data for caches. CoreRecipeInfo as well as other
|
||||||
__slots__ = ()
|
# Extra RecipeInfo needs to inherit this class
|
||||||
|
class RecipeInfoCommon(object):
|
||||||
|
|
||||||
@classmethod
|
@classmethod
|
||||||
def listvar(cls, var, metadata):
|
def listvar(cls, var, metadata):
|
||||||
|
|
@ -117,69 +82,162 @@ class RecipeInfo(namedtuple('RecipeInfo', recipe_fields)):
|
||||||
def getvar(cls, var, metadata):
|
def getvar(cls, var, metadata):
|
||||||
return metadata.getVar(var, True) or ''
|
return metadata.getVar(var, True) or ''
|
||||||
|
|
||||||
@classmethod
|
|
||||||
def make_optional(cls, default=None, **kwargs):
|
class CoreRecipeInfo(RecipeInfoCommon):
|
||||||
"""Construct the namedtuple from the specified keyword arguments,
|
__slots__ = ()
|
||||||
with every value considered optional, using the default value if
|
|
||||||
it was not specified."""
|
def __init__(self, filename, metadata):
|
||||||
for field in cls._fields:
|
self.name = "base"
|
||||||
kwargs[field] = kwargs.get(field, default)
|
# please override this member with the correct data cache file
|
||||||
return cls(**kwargs)
|
# such as (bb_cache.dat, bb_extracache_hob.dat)
|
||||||
|
self.cachefile = "bb_cache.dat"
|
||||||
|
|
||||||
|
self.file_depends = metadata.getVar('__depends', False)
|
||||||
|
self.timestamp = bb.parse.cached_mtime(filename)
|
||||||
|
self.variants = self.listvar('__VARIANTS', metadata) + ['']
|
||||||
|
self.nocache = self.getvar('__BB_DONT_CACHE', metadata)
|
||||||
|
|
||||||
|
if self.getvar('__SKIPPED', metadata):
|
||||||
|
self.skipped = True
|
||||||
|
return
|
||||||
|
|
||||||
|
self.tasks = metadata.getVar('__BBTASKS', False)
|
||||||
|
|
||||||
|
self.pn = self.getvar('PN', metadata)
|
||||||
|
self.packages = self.listvar('PACKAGES', metadata)
|
||||||
|
if not self.pn in self.packages:
|
||||||
|
self.packages.append(self.pn)
|
||||||
|
|
||||||
|
self.basetaskhashes = self.taskvar('BB_BASEHASH', self.tasks, metadata)
|
||||||
|
self.hashfilename = self.getvar('BB_HASHFILENAME', metadata)
|
||||||
|
|
||||||
|
self.file_depends = metadata.getVar('__depends', False)
|
||||||
|
self.task_deps = metadata.getVar('_task_deps', False) or {'tasks': [], 'parents': {}}
|
||||||
|
|
||||||
|
self.skipped = False
|
||||||
|
self.pe = self.getvar('PE', metadata)
|
||||||
|
self.pv = self.getvar('PV', metadata)
|
||||||
|
self.pr = self.getvar('PR', metadata)
|
||||||
|
self.defaultpref = self.intvar('DEFAULT_PREFERENCE', metadata)
|
||||||
|
self.broken = self.getvar('BROKEN', metadata)
|
||||||
|
self.not_world = self.getvar('EXCLUDE_FROM_WORLD', metadata)
|
||||||
|
self.stamp = self.getvar('STAMP', metadata)
|
||||||
|
self.stamp_extrainfo = self.flaglist('stamp-extra-info', self.tasks, metadata)
|
||||||
|
self.packages_dynamic = self.listvar('PACKAGES_DYNAMIC', metadata)
|
||||||
|
self.depends = self.depvar('DEPENDS', metadata)
|
||||||
|
self.provides = self.depvar('PROVIDES', metadata)
|
||||||
|
self.rdepends = self.depvar('RDEPENDS', metadata)
|
||||||
|
self.rprovides = self.depvar('RPROVIDES', metadata)
|
||||||
|
self.rrecommends = self.depvar('RRECOMMENDS', metadata)
|
||||||
|
self.rprovides_pkg = self.pkgvar('RPROVIDES', self.packages, metadata)
|
||||||
|
self.rdepends_pkg = self.pkgvar('RDEPENDS', self.packages, metadata)
|
||||||
|
self.rrecommends_pkg = self.pkgvar('RRECOMMENDS', self.packages, metadata)
|
||||||
|
self.inherits = self.getvar('__inherit_cache', metadata)
|
||||||
|
self.summary = self.getvar('SUMMARY', metadata)
|
||||||
|
self.license = self.getvar('LICENSE', metadata)
|
||||||
|
self.section = self.getvar('SECTION', metadata)
|
||||||
|
self.fakerootenv = self.getvar('FAKEROOTENV', metadata)
|
||||||
|
self.fakerootdirs = self.getvar('FAKEROOTDIRS', metadata)
|
||||||
|
|
||||||
@classmethod
|
@classmethod
|
||||||
def from_metadata(cls, filename, metadata):
|
def init_cacheData(cls, cachedata):
|
||||||
if cls.getvar('__SKIPPED', metadata):
|
# CacheData in Core RecipeInfo Class
|
||||||
return cls.make_optional(skipped=True,
|
cachedata.task_deps = {}
|
||||||
file_depends=metadata.getVar('__depends', False),
|
cachedata.pkg_fn = {}
|
||||||
timestamp=bb.parse.cached_mtime(filename),
|
cachedata.pkg_pn = defaultdict(list)
|
||||||
variants=cls.listvar('__VARIANTS', metadata) + [''])
|
cachedata.pkg_pepvpr = {}
|
||||||
|
cachedata.pkg_dp = {}
|
||||||
|
|
||||||
tasks = metadata.getVar('__BBTASKS', False)
|
cachedata.stamp = {}
|
||||||
|
cachedata.stamp_extrainfo = {}
|
||||||
|
cachedata.fn_provides = {}
|
||||||
|
cachedata.pn_provides = defaultdict(list)
|
||||||
|
cachedata.all_depends = []
|
||||||
|
|
||||||
pn = cls.getvar('PN', metadata)
|
cachedata.deps = defaultdict(list)
|
||||||
packages = cls.listvar('PACKAGES', metadata)
|
cachedata.packages = defaultdict(list)
|
||||||
if not pn in packages:
|
cachedata.providers = defaultdict(list)
|
||||||
packages.append(pn)
|
cachedata.rproviders = defaultdict(list)
|
||||||
|
cachedata.packages_dynamic = defaultdict(list)
|
||||||
|
|
||||||
return RecipeInfo(
|
cachedata.rundeps = defaultdict(lambda: defaultdict(list))
|
||||||
tasks = tasks,
|
cachedata.runrecs = defaultdict(lambda: defaultdict(list))
|
||||||
basetaskhashes = cls.taskvar('BB_BASEHASH', tasks, metadata),
|
cachedata.possible_world = []
|
||||||
hashfilename = cls.getvar('BB_HASHFILENAME', metadata),
|
cachedata.universe_target = []
|
||||||
|
cachedata.hashfn = {}
|
||||||
|
|
||||||
file_depends = metadata.getVar('__depends', False),
|
cachedata.basetaskhash = {}
|
||||||
task_deps = metadata.getVar('_task_deps', False) or
|
cachedata.inherits = {}
|
||||||
{'tasks': [], 'parents': {}},
|
cachedata.summary = {}
|
||||||
variants = cls.listvar('__VARIANTS', metadata) + [''],
|
cachedata.license = {}
|
||||||
|
cachedata.section = {}
|
||||||
|
cachedata.fakerootenv = {}
|
||||||
|
cachedata.fakerootdirs = {}
|
||||||
|
|
||||||
|
def add_cacheData(self, cachedata, fn):
|
||||||
|
cachedata.task_deps[fn] = self.task_deps
|
||||||
|
cachedata.pkg_fn[fn] = self.pn
|
||||||
|
cachedata.pkg_pn[self].append(fn)
|
||||||
|
cachedata.pkg_pepvpr[fn] = (self.pe, self.pv, self.pr)
|
||||||
|
cachedata.pkg_dp[fn] = self.defaultpref
|
||||||
|
cachedata.stamp[fn] = self.stamp
|
||||||
|
cachedata.stamp_extrainfo[fn] = self.stamp_extrainfo
|
||||||
|
|
||||||
|
provides = [self.pn]
|
||||||
|
for provide in self.provides:
|
||||||
|
if provide not in provides:
|
||||||
|
provides.append(provide)
|
||||||
|
cachedata.fn_provides[fn] = provides
|
||||||
|
|
||||||
|
for provide in provides:
|
||||||
|
cachedata.providers[provide].append(fn)
|
||||||
|
if provide not in cachedata.pn_provides[self.pn]:
|
||||||
|
cachedata.pn_provides[self.pn].append(provide)
|
||||||
|
|
||||||
|
for dep in self.depends:
|
||||||
|
if dep not in cachedata.deps[fn]:
|
||||||
|
cachedata.deps[fn].append(dep)
|
||||||
|
if dep not in cachedata.all_depends:
|
||||||
|
cachedata.all_depends.append(dep)
|
||||||
|
|
||||||
|
rprovides = self.rprovides
|
||||||
|
for package in self.packages:
|
||||||
|
cachedata.packages[package].append(fn)
|
||||||
|
rprovides += self.rprovides_pkg[package]
|
||||||
|
|
||||||
|
for rprovide in rprovides:
|
||||||
|
cachedata.rproviders[rprovide].append(fn)
|
||||||
|
|
||||||
|
for package in self.packages_dynamic:
|
||||||
|
cachedata.packages_dynamic[package].append(fn)
|
||||||
|
|
||||||
|
# Build hash of runtime depends and rececommends
|
||||||
|
for package in self.packages + [self.pn]:
|
||||||
|
cachedata.rundeps[fn][package] = list(self.rdepends) + self.rdepends_pkg[package]
|
||||||
|
cachedata.runrecs[fn][package] = list(self.rrecommends) + self.rrecommends_pkg[package]
|
||||||
|
|
||||||
|
# Collect files we may need for possible world-dep
|
||||||
|
# calculations
|
||||||
|
if not self.broken and not self.not_world:
|
||||||
|
cachedata.possible_world.append(fn)
|
||||||
|
|
||||||
|
# create a collection of all targets for sanity checking
|
||||||
|
# tasks, such as upstream versions, license, and tools for
|
||||||
|
# task and image creation.
|
||||||
|
cachedata.universe_target.append(self.pn)
|
||||||
|
|
||||||
|
cachedata.hashfn[fn] = self.hashfilename
|
||||||
|
for task, taskhash in self.basetaskhashes.iteritems():
|
||||||
|
identifier = '%s.%s' % (fn, task)
|
||||||
|
cachedata.basetaskhash[identifier] = taskhash
|
||||||
|
|
||||||
|
cachedata.inherits[fn] = self.inherits
|
||||||
|
cachedata.summary[fn] = self.summary
|
||||||
|
cachedata.license[fn] = self.license
|
||||||
|
cachedata.section[fn] = self.section
|
||||||
|
cachedata.fakerootenv[fn] = self.fakerootenv
|
||||||
|
cachedata.fakerootdirs[fn] = self.fakerootdirs
|
||||||
|
|
||||||
skipped = False,
|
|
||||||
timestamp = bb.parse.cached_mtime(filename),
|
|
||||||
packages = cls.listvar('PACKAGES', metadata),
|
|
||||||
pn = pn,
|
|
||||||
pe = cls.getvar('PE', metadata),
|
|
||||||
pv = cls.getvar('PV', metadata),
|
|
||||||
pr = cls.getvar('PR', metadata),
|
|
||||||
nocache = cls.getvar('__BB_DONT_CACHE', metadata),
|
|
||||||
defaultpref = cls.intvar('DEFAULT_PREFERENCE', metadata),
|
|
||||||
broken = cls.getvar('BROKEN', metadata),
|
|
||||||
not_world = cls.getvar('EXCLUDE_FROM_WORLD', metadata),
|
|
||||||
stamp = cls.getvar('STAMP', metadata),
|
|
||||||
stamp_extrainfo = cls.flaglist('stamp-extra-info', tasks, metadata),
|
|
||||||
packages_dynamic = cls.listvar('PACKAGES_DYNAMIC', metadata),
|
|
||||||
depends = cls.depvar('DEPENDS', metadata),
|
|
||||||
provides = cls.depvar('PROVIDES', metadata),
|
|
||||||
rdepends = cls.depvar('RDEPENDS', metadata),
|
|
||||||
rprovides = cls.depvar('RPROVIDES', metadata),
|
|
||||||
rrecommends = cls.depvar('RRECOMMENDS', metadata),
|
|
||||||
rprovides_pkg = cls.pkgvar('RPROVIDES', packages, metadata),
|
|
||||||
rdepends_pkg = cls.pkgvar('RDEPENDS', packages, metadata),
|
|
||||||
rrecommends_pkg = cls.pkgvar('RRECOMMENDS', packages, metadata),
|
|
||||||
inherits = cls.getvar('__inherit_cache', metadata),
|
|
||||||
summary = cls.getvar('SUMMARY', metadata),
|
|
||||||
license = cls.getvar('LICENSE', metadata),
|
|
||||||
section = cls.getvar('SECTION', metadata),
|
|
||||||
fakerootenv = cls.getvar('FAKEROOTENV', metadata),
|
|
||||||
fakerootdirs = cls.getvar('FAKEROOTDIRS', metadata),
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
class Cache(object):
|
class Cache(object):
|
||||||
|
|
@ -314,7 +372,7 @@ class Cache(object):
|
||||||
depends |= (data.getVar("__depends", False) or set())
|
depends |= (data.getVar("__depends", False) or set())
|
||||||
if depends and not variant:
|
if depends and not variant:
|
||||||
data.setVar("__depends", depends)
|
data.setVar("__depends", depends)
|
||||||
info = RecipeInfo.from_metadata(filename, data)
|
info = CoreRecipeInfo(filename, data)
|
||||||
infos.append((virtualfn, info))
|
infos.append((virtualfn, info))
|
||||||
return infos
|
return infos
|
||||||
|
|
||||||
|
|
@ -500,7 +558,7 @@ class Cache(object):
|
||||||
"""
|
"""
|
||||||
|
|
||||||
realfn = self.virtualfn2realfn(file_name)[0]
|
realfn = self.virtualfn2realfn(file_name)[0]
|
||||||
info = RecipeInfo.from_metadata(realfn, data)
|
info = CoreRecipeInfo(realfn, data)
|
||||||
self.add_info(file_name, info, cacheData, parsed)
|
self.add_info(file_name, info, cacheData, parsed)
|
||||||
|
|
||||||
@staticmethod
|
@staticmethod
|
||||||
|
|
@ -566,38 +624,11 @@ class CacheData(object):
|
||||||
"""
|
"""
|
||||||
|
|
||||||
def __init__(self):
|
def __init__(self):
|
||||||
|
CoreRecipeInfo.init_cacheData(self)
|
||||||
# Direct cache variables
|
# Direct cache variables
|
||||||
self.providers = defaultdict(list)
|
|
||||||
self.rproviders = defaultdict(list)
|
|
||||||
self.packages = defaultdict(list)
|
|
||||||
self.packages_dynamic = defaultdict(list)
|
|
||||||
self.possible_world = []
|
|
||||||
self.universe_target = []
|
|
||||||
self.pkg_pn = defaultdict(list)
|
|
||||||
self.pkg_fn = {}
|
|
||||||
self.pkg_pepvpr = {}
|
|
||||||
self.pkg_dp = {}
|
|
||||||
self.pn_provides = defaultdict(list)
|
|
||||||
self.fn_provides = {}
|
|
||||||
self.all_depends = []
|
|
||||||
self.deps = defaultdict(list)
|
|
||||||
self.rundeps = defaultdict(lambda: defaultdict(list))
|
|
||||||
self.runrecs = defaultdict(lambda: defaultdict(list))
|
|
||||||
self.task_queues = {}
|
self.task_queues = {}
|
||||||
self.task_deps = {}
|
|
||||||
self.stamp = {}
|
|
||||||
self.stamp_extrainfo = {}
|
|
||||||
self.preferred = {}
|
self.preferred = {}
|
||||||
self.tasks = {}
|
self.tasks = {}
|
||||||
self.basetaskhash = {}
|
|
||||||
self.hashfn = {}
|
|
||||||
self.inherits = {}
|
|
||||||
self.summary = {}
|
|
||||||
self.license = {}
|
|
||||||
self.section = {}
|
|
||||||
self.fakerootenv = {}
|
|
||||||
self.fakerootdirs = {}
|
|
||||||
|
|
||||||
# Indirect Cache variables (set elsewhere)
|
# Indirect Cache variables (set elsewhere)
|
||||||
self.ignored_dependencies = []
|
self.ignored_dependencies = []
|
||||||
self.world_target = set()
|
self.world_target = set()
|
||||||
|
|
@ -605,65 +636,6 @@ class CacheData(object):
|
||||||
self.bbfile_config_priorities = []
|
self.bbfile_config_priorities = []
|
||||||
|
|
||||||
def add_from_recipeinfo(self, fn, info):
|
def add_from_recipeinfo(self, fn, info):
|
||||||
self.task_deps[fn] = info.task_deps
|
info.add_cacheData(self, fn)
|
||||||
self.pkg_fn[fn] = info.pn
|
|
||||||
self.pkg_pn[info.pn].append(fn)
|
|
||||||
self.pkg_pepvpr[fn] = (info.pe, info.pv, info.pr)
|
|
||||||
self.pkg_dp[fn] = info.defaultpref
|
|
||||||
self.stamp[fn] = info.stamp
|
|
||||||
self.stamp_extrainfo[fn] = info.stamp_extrainfo
|
|
||||||
|
|
||||||
provides = [info.pn]
|
|
||||||
for provide in info.provides:
|
|
||||||
if provide not in provides:
|
|
||||||
provides.append(provide)
|
|
||||||
self.fn_provides[fn] = provides
|
|
||||||
|
|
||||||
for provide in provides:
|
|
||||||
self.providers[provide].append(fn)
|
|
||||||
if provide not in self.pn_provides[info.pn]:
|
|
||||||
self.pn_provides[info.pn].append(provide)
|
|
||||||
|
|
||||||
for dep in info.depends:
|
|
||||||
if dep not in self.deps[fn]:
|
|
||||||
self.deps[fn].append(dep)
|
|
||||||
if dep not in self.all_depends:
|
|
||||||
self.all_depends.append(dep)
|
|
||||||
|
|
||||||
rprovides = info.rprovides
|
|
||||||
for package in info.packages:
|
|
||||||
self.packages[package].append(fn)
|
|
||||||
rprovides += info.rprovides_pkg[package]
|
|
||||||
|
|
||||||
for rprovide in rprovides:
|
|
||||||
self.rproviders[rprovide].append(fn)
|
|
||||||
|
|
||||||
for package in info.packages_dynamic:
|
|
||||||
self.packages_dynamic[package].append(fn)
|
|
||||||
|
|
||||||
# Build hash of runtime depends and rececommends
|
|
||||||
for package in info.packages + [info.pn]:
|
|
||||||
self.rundeps[fn][package] = list(info.rdepends) + info.rdepends_pkg[package]
|
|
||||||
self.runrecs[fn][package] = list(info.rrecommends) + info.rrecommends_pkg[package]
|
|
||||||
|
|
||||||
# Collect files we may need for possible world-dep
|
|
||||||
# calculations
|
|
||||||
if not info.broken and not info.not_world:
|
|
||||||
self.possible_world.append(fn)
|
|
||||||
|
|
||||||
# create a collection of all targets for sanity checking
|
|
||||||
# tasks, such as upstream versions, license, and tools for
|
|
||||||
# task and image creation.
|
|
||||||
self.universe_target.append(info.pn)
|
|
||||||
|
|
||||||
self.hashfn[fn] = info.hashfilename
|
|
||||||
for task, taskhash in info.basetaskhashes.iteritems():
|
|
||||||
identifier = '%s.%s' % (fn, task)
|
|
||||||
self.basetaskhash[identifier] = taskhash
|
|
||||||
|
|
||||||
self.inherits[fn] = info.inherits
|
|
||||||
self.summary[fn] = info.summary
|
|
||||||
self.license[fn] = info.license
|
|
||||||
self.section[fn] = info.section
|
|
||||||
self.fakerootenv[fn] = info.fakerootenv
|
|
||||||
self.fakerootdirs[fn] = info.fakerootdirs
|
|
||||||
|
|
|
||||||
Loading…
Reference in New Issue