cache: Implement multiple extra cache fields request support

This patch is to support extra cache. If user needs to request
extra cache fields besides CoreRecipeInfo fields, just add a new
XXXRecipeInfo class definition as Hob Does.
Currently supported Extra RecipeInfo name is an array. We can
support multiple extra cache fields at the same time besides
CoreRecipeInfo which is needed by all clients.

(Bitbake rev: 077657e50ad032c0fa876bf54e9802af2686e0fb)

Signed-off-by: Liping Ke <liping.ke@intel.com>
Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>

bitbake/lib/bb/cache.py

@@ -43,8 +43,10 @@ except ImportError:
     logger.info("Importing cPickle failed. "
                 "Falling back to a very slow implementation.")
 
-__cache_version__ = "138"
+__cache_version__ = "139"
 
+def getCacheFile(path, filename):
+    return os.path.join(path, filename)
 
 # RecipeInfoCommon defines common data retrieving methods
 # from meta data for caches. CoreRecipeInfo as well as other
@@ -86,12 +88,9 @@ class RecipeInfoCommon(object):
 class CoreRecipeInfo(RecipeInfoCommon):
     __slots__ = ()
 
-    def __init__(self, filename, metadata):
+    cachefile = "bb_cache.dat"   
-        self.name = "base"
-        # please override this member with the correct data cache file
-        # such as (bb_cache.dat, bb_extracache_hob.dat) 
-        self.cachefile = "bb_cache.dat"        
 
+    def __init__(self, filename, metadata):      
         self.file_depends = metadata.getVar('__depends', False)
         self.timestamp = bb.parse.cached_mtime(filename)
         self.variants = self.listvar('__VARIANTS', metadata) + ['']
@@ -265,7 +264,7 @@ class Cache(object):
             return
 
         self.has_cache = True
-        self.cachefile = os.path.join(self.cachedir, "bb_cache.dat")
+        self.cachefile = getCacheFile(self.cachedir, "bb_cache.dat")
 
         logger.debug(1, "Using cache in '%s'", self.cachedir)
         bb.utils.mkdirhier(self.cachedir)
@@ -279,12 +278,21 @@ class Cache(object):
         old_mtimes.append(newest_mtime)
         newest_mtime = max(old_mtimes)
 
-        if bb.parse.cached_mtime_noerror(self.cachefile) >= newest_mtime:
+        bNeedUpdate = True
+        if self.caches_array:
+            for cache_class in self.caches_array:
+                if type(cache_class) is type and issubclass(cache_class, RecipeInfoCommon):
+                    cachefile = getCacheFile(self.cachedir, cache_class.cachefile)
+                    bNeedUpdate = bNeedUpdate and (bb.parse.cached_mtime_noerror(cachefile) >= newest_mtime)
+                    cache_class.init_cacheData(self)
+        if bNeedUpdate:
             self.load_cachefile()
         elif os.path.isfile(self.cachefile):
             logger.info("Out of date cache found, rebuilding...")
 
     def load_cachefile(self):
+        # Firstly, using core cache file information for
+        # valid checking
         with open(self.cachefile, "rb") as cachefile:
             pickled = pickle.Unpickler(cachefile)
             try:
@@ -301,31 +309,52 @@ class Cache(object):
                 logger.info('Bitbake version mismatch, rebuilding...')
                 return
 
-            cachesize = os.fstat(cachefile.fileno()).st_size
-            bb.event.fire(bb.event.CacheLoadStarted(cachesize), self.data)
 
-            previous_percent = 0
+        cachesize = 0
-            while cachefile:
+        previous_progress = 0
-                try:
+        previous_percent = 0
-                    key = pickled.load()
-                    value = pickled.load()
-                except Exception:
-                    break
 
-                self.depends_cache[key] = value
+        # Calculate the correct cachesize of all those cache files
+        for cache_class in self.caches_array:
+            if type(cache_class) is type and issubclass(cache_class, RecipeInfoCommon):
+                cachefile = getCacheFile(self.cachedir, cache_class.cachefile)
+                with open(cachefile, "rb") as cachefile:
+                    cachesize += os.fstat(cachefile.fileno()).st_size
 
-                # only fire events on even percentage boundaries
+        bb.event.fire(bb.event.CacheLoadStarted(cachesize), self.data)
-                current_progress = cachefile.tell()
+        
-                current_percent = 100 * current_progress / cachesize
+        for cache_class in self.caches_array:
-                if current_percent > previous_percent:
+            if type(cache_class) is type and issubclass(cache_class, RecipeInfoCommon):
-                    previous_percent = current_percent
+                cachefile = getCacheFile(self.cachedir, cache_class.cachefile)
-                    bb.event.fire(bb.event.CacheLoadProgress(current_progress),
+                with open(cachefile, "rb") as cachefile:
-                                  self.data)
+                    pickled = pickle.Unpickler(cachefile)                    
+                    while cachefile:
+                        try:
+                            key = pickled.load()
+                            value = pickled.load()
+                        except Exception:
+                            break
+                        if self.depends_cache.has_key(key):
+                            self.depends_cache[key].append(value)
+                        else:
+                            self.depends_cache[key] = [value]
+                        # only fire events on even percentage boundaries
+                        current_progress = cachefile.tell() + previous_progress
+                        current_percent = 100 * current_progress / cachesize
+                        if current_percent > previous_percent:
+                            previous_percent = current_percent
+                            bb.event.fire(bb.event.CacheLoadProgress(current_progress),
+                                          self.data)
 
-            bb.event.fire(bb.event.CacheLoadCompleted(cachesize,
+                    previous_progress += current_progress
-                                                      len(self.depends_cache)),
-                          self.data)
 
+        # Note: depends cache number is corresponding to the parsing file numbers.
+        # The same file has several caches, still regarded as one item in the cache
+        bb.event.fire(bb.event.CacheLoadCompleted(cachesize,
+                                                  len(self.depends_cache)),
+                      self.data)
+
+    
     @staticmethod
     def virtualfn2realfn(virtualfn):
         """
@@ -376,8 +405,14 @@ class Cache(object):
             depends |= (data.getVar("__depends", False) or set())
             if depends and not variant:
                 data.setVar("__depends", depends)
-            info = CoreRecipeInfo(filename, data)
+
-            infos.append((virtualfn, info))
+            info_array = []
+            for cache_class in caches_array:
+                if type(cache_class) is type and issubclass(cache_class, RecipeInfoCommon):
+                    info = cache_class(filename, data)
+                    info_array.append(info)
+            infos.append((virtualfn, info_array))
+
         return infos
 
     def load(self, filename, appends, configdata):
@@ -391,8 +426,9 @@ class Cache(object):
         cached = self.cacheValid(filename)
         if cached:
             infos = []
-            info = self.depends_cache[filename]
+            # info_array item is a list of [CoreRecipeInfo, XXXRecipeInfo]
-            for variant in info.variants:
+            info_array = self.depends_cache[filename]
+            for variant in info_array[0].variants:
                 virtualfn = self.realfn2virtual(filename, variant)
                 infos.append((virtualfn, self.depends_cache[virtualfn]))
         else:
@@ -408,12 +444,12 @@ class Cache(object):
         skipped, virtuals = 0, 0
 
         cached, infos = self.load(fn, appends, cfgData)
-        for virtualfn, info in infos:
+        for virtualfn, info_array in infos:
-            if info.skipped:
+            if info_array[0].skipped:
                 logger.debug(1, "Skipping %s", virtualfn)
                 skipped += 1
             else:
-                self.add_info(virtualfn, info, cacheData, not cached)
+                self.add_info(virtualfn, info_array, cacheData, not cached)
                 virtuals += 1
 
         return cached, skipped, virtuals
@@ -457,15 +493,15 @@ class Cache(object):
             self.remove(fn)
             return False
 
-        info = self.depends_cache[fn]
+        info_array = self.depends_cache[fn]
         # Check the file's timestamp
-        if mtime != info.timestamp:
+        if mtime != info_array[0].timestamp:
             logger.debug(2, "Cache: %s changed", fn)
             self.remove(fn)
             return False
 
         # Check dependencies are still valid
-        depends = info.file_depends
+        depends = info_array[0].file_depends
         if depends:
             for f, old_mtime in depends:
                 fmtime = bb.parse.cached_mtime_noerror(f)
@@ -483,7 +519,7 @@ class Cache(object):
                     return False
 
         invalid = False
-        for cls in info.variants:
+        for cls in info_array[0].variants:
             virtualfn = self.realfn2virtual(fn, cls)
             self.clean.add(virtualfn)
             if virtualfn not in self.depends_cache:
@@ -530,13 +566,30 @@ class Cache(object):
             logger.debug(2, "Cache is clean, not saving.")
             return
 
-        with open(self.cachefile, "wb") as cachefile:
+        file_dict = {}
-            pickler = pickle.Pickler(cachefile, pickle.HIGHEST_PROTOCOL)
+        pickler_dict = {}
-            pickler.dump(__cache_version__)
+        for cache_class in self.caches_array:
-            pickler.dump(bb.__version__)
+            if type(cache_class) is type and issubclass(cache_class, RecipeInfoCommon):
-            for key, value in self.depends_cache.iteritems():
+                cache_class_name = cache_class.__name__
-                pickler.dump(key)
+                cachefile = getCacheFile(self.cachedir, cache_class.cachefile)
-                pickler.dump(value)
+                file_dict[cache_class_name] = open(cachefile, "wb")
+                pickler_dict[cache_class_name] =  pickle.Pickler(file_dict[cache_class_name], pickle.HIGHEST_PROTOCOL)
+                   
+        pickler_dict['CoreRecipeInfo'].dump(__cache_version__)
+        pickler_dict['CoreRecipeInfo'].dump(bb.__version__)
+
+        try:
+            for key, info_array in self.depends_cache.iteritems():
+                for info in info_array:
+                    if isinstance(info, RecipeInfoCommon):
+                        cache_class_name = info.__class__.__name__
+                        pickler_dict[cache_class_name].dump(key)
+                        pickler_dict[cache_class_name].dump(info)
+        finally:
+            for cache_class in self.caches_array:
+                if type(cache_class) is type and issubclass(cache_class, RecipeInfoCommon):
+                    cache_class_name = cache_class.__name__
+                    file_dict[cache_class_name].close()
 
         del self.depends_cache
 
@@ -544,17 +597,17 @@ class Cache(object):
     def mtime(cachefile):
         return bb.parse.cached_mtime_noerror(cachefile)
 
-    def add_info(self, filename, info, cacheData, parsed=None):
+    def add_info(self, filename, info_array, cacheData, parsed=None):
-        if not info.skipped:
+        if isinstance(info_array[0], CoreRecipeInfo) and (not info_array[0].skipped):
-            cacheData.add_from_recipeinfo(filename, info)
+            cacheData.add_from_recipeinfo(filename, info_array)
 
         if not self.has_cache:
             return
 
-        if (info.skipped or 'SRCREVINACTION' not in info.pv) and not info.nocache:
+        if (info_array[0].skipped or 'SRCREVINACTION' not in info_array[0].pv) and not info_array[0].nocache:
             if parsed:
                 self.cacheclean = False
-            self.depends_cache[filename] = info
+            self.depends_cache[filename] = info_array
 
     def add(self, file_name, data, cacheData, parsed=None):
         """
@@ -562,8 +615,12 @@ class Cache(object):
         """
 
         realfn = self.virtualfn2realfn(file_name)[0]
-        info = CoreRecipeInfo(realfn, data)
+
-        self.add_info(file_name, info, cacheData, parsed)
+        info_array = []
+        for cache_class in self.caches_array:
+            if type(cache_class) is type and issubclass(cache_class, RecipeInfoCommon):
+                info_array.append(cache_class(realfn, data))
+        self.add_info(file_name, info_array, cacheData, parsed)
 
     @staticmethod
     def load_bbfile(bbfile, appends, config):
@@ -629,7 +686,10 @@ class CacheData(object):
 
     def __init__(self, caches_array):
         self.caches_array = caches_array
-        CoreRecipeInfo.init_cacheData(self)
+        for cache_class in self.caches_array:
+            if type(cache_class) is type and issubclass(cache_class, RecipeInfoCommon):
+                cache_class.init_cacheData(self)        
+
         # Direct cache variables
         self.task_queues = {}
         self.preferred = {}
@@ -640,7 +700,8 @@ class CacheData(object):
         self.bbfile_priority = {}
         self.bbfile_config_priorities = []
 
-    def add_from_recipeinfo(self, fn, info):
+    def add_from_recipeinfo(self, fn, info_array):
-        info.add_cacheData(self, fn)
+        for info in info_array:
+            info.add_cacheData(self, fn)
 
         

bitbake/lib/bb/cooker.py

@@ -773,13 +773,13 @@ class BBCooker:
 
         fn = bb.cache.Cache.realfn2virtual(fn, cls)
         try:
-            maininfo = infos[fn]
+            info_array = infos[fn]
         except KeyError:
             bb.fatal("%s does not exist" % fn)
-        self.status.add_from_recipeinfo(fn, maininfo)
+        self.status.add_from_recipeinfo(fn, info_array)
 
         # Tweak some variables
-        item = maininfo.pn
+        item = info_array[0].pn
         self.status.ignored_dependencies = set()
         self.status.bbfile_priority[fn] = 1
 
@@ -1231,10 +1231,10 @@ class CookerParser(object):
         else:
             self.cached += 1
 
-        for virtualfn, info in result:
+        for virtualfn, info_array in result:
-            if info.skipped:
+            if info_array[0].skipped:
                 self.skipped += 1
-            self.bb_cache.add_info(virtualfn, info, self.cooker.status,
+            self.bb_cache.add_info(virtualfn, info_array, self.cooker.status,
                                         parsed=parsed)
         return True
 
@@ -1242,5 +1242,5 @@ class CookerParser(object):
         infos = self.bb_cache.parse(filename,
                                     self.cooker.get_file_appends(filename),
                                     self.cfgdata, self.cooker.caches_array)
-        for vfn, info in infos:
+        for vfn, info_array in infos:
-            self.cooker.status.add_from_recipeinfo(vfn, info)
+            self.cooker.status.add_from_recipeinfo(vfn, info_array)