sysmo-bts
/
generic-poky
9
0
Fork 0
Code Pull Requests Releases Activity

bitbake: cooker/cache/parse: Implement pyinofity based reconfigure 

Memory resident bitbake has one current flaw, changes in the base configuration
are not noticed by bitbake. The parsing cache is also refreshed on each invocation
of bitbake (although the mtime cache is not cleared so its pointless).

This change adds in pyinotify support and adds two different watchers, one
for the base configuration and one for the parsed recipes.

Changes in the latter will trigger a reparse (and an update of the mtime cache).
The former will trigger a complete reload of the configuration.

Note that this code will also correctly handle creation of new configuration files
since the __depends and __base_depends variables already track these for cache
correctness purposes.

We could be a little more clever about parsing cache invalidation, right now we just
invalidate the whole thing and recheck. For now, its better than what we have and doesn't
seem to perform that badly though.

For education and QA purposes I can document a workflow that illustrates this:

$ source oe-init-build-env-memres
$ time bitbake bash
[base configuration is loaded, recipes are parsed, bash builds]
$ time bitbake bash
[command returns quickly since all caches are valid]
$ touch ../meta/classes/gettext.bbclass
$ time bitbake bash
[reparse is triggered, time is longer than above]
$ echo 'FOO = "1"' >> conf/local.conf
$ time bitbake bash
[reparse is triggered, but with a base configuration reload too]

As far as changes go, I like this one a lot, it makes memory resident bitbake
truly usable and may be the tweak we need to make it the default.

The new pyinotify dependency is covered in the previous commit.

(Bitbake rev: 0557d03c170fba8d7efe82be1b9641d0eb229213)

Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
This commit is contained in:
Richard Purdie 2015-01-13 14:13:53 +00:00
parent 83f8a4002a
commit 6c3c3e11f6
3 changed files with 51 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
bitbake/lib/bb/cache.py
View File

@ -623,10 +623,13 @@ class Cache(object):
def mtime(cachefile):
return bb.parse.cached_mtime_noerror(cachefile)
def add_info(self, filename, info_array, cacheData, parsed=None):
def add_info(self, filename, info_array, cacheData, parsed=None, watcher=None):
if isinstance(info_array[0], CoreRecipeInfo) and (not info_array[0].skipped):
cacheData.add_from_recipeinfo(filename, info_array)
if watcher:
watcher(info_array[0].file_depends)
if not self.has_cache:
return

44
bitbake/lib/bb/cooker.py
View File

@ -39,6 +39,7 @@ from bb import utils, data, parse, event, cache, providers, taskdata, runqueue
import Queue
import signal
import prserv.serv
import pyinotify
logger = logging.getLogger("BitBake")
collectlog = logging.getLogger("BitBake.Collection")
@ -120,7 +121,18 @@ class BBCooker:
self.configuration = configuration
self.configwatcher = pyinotify.WatchManager()
self.confignotifier = pyinotify.Notifier(self.configwatcher, self.config_notifications)
self.watchmask = pyinotify.IN_CLOSE_WRITE | pyinotify.IN_CREATE | pyinotify.IN_DELETE | \
pyinotify.IN_DELETE_SELF | pyinotify.IN_MODIFY | pyinotify.IN_MOVE_SELF | \
pyinotify.IN_MOVED_FROM | pyinotify.IN_MOVED_TO
self.watcher = pyinotify.WatchManager()
self.notifier = pyinotify.Notifier(self.watcher, self.notifications)
self.initConfigurationData()
self.baseconfig_valid = True
self.parsecache_valid = False
# Take a lock so only one copy of bitbake can run against a given build
# directory at a time
@ -156,6 +168,18 @@ class BBCooker:
# Let SIGHUP exit as SIGTERM
signal.signal(signal.SIGHUP, self.sigterm_exception)
def config_notifications(self, event):
bb.parse.update_cache(event.path)
self.baseconfig_valid = False
def notifications(self, event):
bb.parse.update_cache(event.path)
self.parsecache_valid = False
def add_filewatch(self, deps):
for i in deps:
self.watcher.add_watch(i[0], self.watchmask, rec=True)
def sigterm_exception(self, signum, stackframe):
if signum == signal.SIGTERM:
bb.warn("Cooker recieved SIGTERM, shutting down...")
@ -1361,6 +1385,18 @@ class BBCooker:
raise bb.BBHandledException()
if self.state != state.parsing:
for n in [self.confignotifier, self.notifier]:
if n.check_events(timeout=0):
# read notified events and enqeue them
n.read_events()
n.process_events()
if not self.baseconfig_valid:
logger.debug(1, "Reloading base configuration data")
self.initConfigurationData()
self.baseconfig_valid = True
self.parsecache_valid = False
if self.state != state.parsing and not self.parsecache_valid:
self.parseConfiguration ()
if CookerFeatures.SEND_SANITYEVENTS in self.featureset:
bb.event.fire(bb.event.SanityCheck(False), self.data)
@ -1375,9 +1411,13 @@ class BBCooker:
(filelist, masked) = self.collection.collect_bbfiles(self.data, self.event_data)
self.data.renameVar("__depends", "__base_depends")
for i in self.data.getVar("__base_depends"):
self.wdd = self.configwatcher.add_watch(i[0], self.watchmask, rec=True)
self.parser = CookerParser(self, filelist, masked)
self.state = state.parsing
self.parsecache_valid = True
self.state = state.parsing
if not self.parser.parse_next():
collectlog.debug(1, "parsing complete")
@ -1940,7 +1980,7 @@ class CookerParser(object):
self.skipped += 1
self.cooker.skiplist[virtualfn] = SkippedPackage(info_array[0])
self.bb_cache.add_info(virtualfn, info_array, self.cooker.recipecache,
parsed=parsed)
parsed=parsed, watcher = self.cooker.add_filewatch)
return True
def reparse(self, filename):

5
bitbake/lib/bb/parse/__init__.py
View File

@ -73,6 +73,11 @@ def update_mtime(f):
__mtime_cache[f] = os.stat(f)[stat.ST_MTIME]
return __mtime_cache[f]
def update_cache(f):
if f in __mtime_cache:
logger.debug(1, "Updating mtime cache for %s" % f)
update_mtime(f)
def mark_dependency(d, f):
if f.startswith('./'):
f = "%s/%s" % (os.getcwd(), f[2:])