From 4a081b5a52e3d27da8d4b062f3fda292e8d8fb0a Mon Sep 17 00:00:00 2001 From: Richard Purdie Date: Thu, 9 May 2013 21:06:45 +0000 Subject: [PATCH] bitbake: lib: Clean up various file access syntax Python 3 is stricter about how files are accessed. Specficially: * Use open(), not file() * Use binary mode for binary files (when checksumming) * Use with statements to ensure files get closed * Add missing file close statements (Bitbake rev: 9f08b901375ba640f47596f1bcf43f98a931550f) Signed-off-by: Richard Purdie --- bitbake/lib/bb/build.py | 15 ++++++------ bitbake/lib/bb/cache.py | 25 ++++++++++++-------- bitbake/lib/bb/parse/parse_py/BBHandler.py | 1 + bitbake/lib/bb/parse/parse_py/ConfHandler.py | 2 ++ bitbake/lib/bb/process.py | 4 ++++ bitbake/lib/bb/utils.py | 14 +++++++---- 6 files changed, 39 insertions(+), 22 deletions(-) diff --git a/bitbake/lib/bb/build.py b/bitbake/lib/bb/build.py index d91ff53fcf..b5681b13e3 100644 --- a/bitbake/lib/bb/build.py +++ b/bitbake/lib/bb/build.py @@ -275,7 +275,8 @@ def exec_func_shell(func, d, runfile, cwd=None): bb.debug(2, "Executing shell function %s" % func) try: - bb.process.run(cmd, shell=False, stdin=NULL, log=logfile) + with open(os.devnull, 'r+') as stdin: + bb.process.run(cmd, shell=False, stdin=stdin, log=logfile) except bb.process.CmdError: logfn = d.getVar('BB_LOGFILE', True) raise FuncFailed(func, logfn) @@ -319,12 +320,11 @@ def _exec_task(fn, task, d, quieterr): # Document the order of the tasks... logorder = os.path.join(tempdir, 'log.task_order') try: - logorderfile = file(logorder, 'a') + with open(logorder, 'a') as logorderfile: + logorderfile.write('{0} ({1}): {2}\n'.format(task, os.getpid(), logbase)) except OSError: logger.exception("Opening log file '%s'", logorder) pass - logorderfile.write('{0} ({1}): {2}\n'.format(task, os.getpid(), logbase)) - logorderfile.close() # Setup the courtesy link to the logfn loglink = os.path.join(tempdir, 'log.{0}'.format(task)) @@ -348,10 +348,10 @@ def _exec_task(fn, task, d, quieterr): self.triggered = True # Handle logfiles - si = file('/dev/null', 'r') + si = open('/dev/null', 'r') try: bb.utils.mkdirhier(os.path.dirname(logfn)) - logfile = file(logfn, 'w') + logfile = open(logfn, 'w') except OSError: logger.exception("Opening log file '%s'", logfn) pass @@ -533,8 +533,7 @@ def make_stamp(task, d, file_name = None): # change on broken NFS filesystems if stamp: bb.utils.remove(stamp) - f = open(stamp, "w") - f.close() + open(stamp, "w").close() # If we're in task context, write out a signature file for each task # as it completes diff --git a/bitbake/lib/bb/cache.py b/bitbake/lib/bb/cache.py index 1c975b62e1..c92ba35641 100644 --- a/bitbake/lib/bb/cache.py +++ b/bitbake/lib/bb/cache.py @@ -738,8 +738,9 @@ class MultiProcessCache(object): logger.debug(1, "Using cache in '%s'", self.cachefile) try: - p = pickle.Unpickler(file(self.cachefile, "rb")) - data, version = p.load() + with open(self.cachefile, "rb") as f: + p = pickle.Unpickler(f) + data, version = p.load() except: return @@ -779,8 +780,9 @@ class MultiProcessCache(object): i = i + 1 continue - p = pickle.Pickler(file(self.cachefile + "-" + str(i), "wb"), -1) - p.dump([self.cachedata_extras, self.__class__.CACHE_VERSION]) + with open(self.cachefile + "-" + str(i), "wb") as f: + p = pickle.Pickler(f, -1) + p.dump([self.cachedata_extras, self.__class__.CACHE_VERSION]) bb.utils.unlockfile(lf) bb.utils.unlockfile(glf) @@ -798,8 +800,9 @@ class MultiProcessCache(object): glf = bb.utils.lockfile(self.cachefile + ".lock") try: - p = pickle.Unpickler(file(self.cachefile, "rb")) - data, version = p.load() + with open(self.cachefile, "rb") as f: + p = pickle.Unpickler(f) + data, version = p.load() except (IOError, EOFError): data, version = None, None @@ -809,8 +812,9 @@ class MultiProcessCache(object): for f in [y for y in os.listdir(os.path.dirname(self.cachefile)) if y.startswith(os.path.basename(self.cachefile) + '-')]: f = os.path.join(os.path.dirname(self.cachefile), f) try: - p = pickle.Unpickler(file(f, "rb")) - extradata, version = p.load() + with open(f, "rb") as fd: + p = pickle.Unpickler(fd) + extradata, version = p.load() except (IOError, EOFError): extradata, version = self.create_cachedata(), None @@ -822,8 +826,9 @@ class MultiProcessCache(object): self.compress_keys(data) - p = pickle.Pickler(file(self.cachefile, "wb"), -1) - p.dump([data, self.__class__.CACHE_VERSION]) + with open(self.cachefile, "wb") as f: + p = pickle.Pickler(f, -1) + p.dump([data, self.__class__.CACHE_VERSION]) bb.utils.unlockfile(glf) diff --git a/bitbake/lib/bb/parse/parse_py/BBHandler.py b/bitbake/lib/bb/parse/parse_py/BBHandler.py index 81fb8d3adf..87a1530cb5 100644 --- a/bitbake/lib/bb/parse/parse_py/BBHandler.py +++ b/bitbake/lib/bb/parse/parse_py/BBHandler.py @@ -104,6 +104,7 @@ def get_statements(filename, absolute_filename, base_name): if not s: break s = s.rstrip() feeder(lineno, s, filename, base_name, statements) + file.close() if __inpython__: # add a blank line to close out any python definition feeder(IN_PYTHON_EOF, "", filename, base_name, statements) diff --git a/bitbake/lib/bb/parse/parse_py/ConfHandler.py b/bitbake/lib/bb/parse/parse_py/ConfHandler.py index 3255c8beb8..7d4a5b14a7 100644 --- a/bitbake/lib/bb/parse/parse_py/ConfHandler.py +++ b/bitbake/lib/bb/parse/parse_py/ConfHandler.py @@ -145,6 +145,8 @@ def handle(fn, data, include): if oldfile: data.setVar('FILE', oldfile) + f.close() + for f in confFilters: f(fn, data) diff --git a/bitbake/lib/bb/process.py b/bitbake/lib/bb/process.py index 05b51725f1..1aeec788c1 100644 --- a/bitbake/lib/bb/process.py +++ b/bitbake/lib/bb/process.py @@ -102,6 +102,10 @@ def _logged_communicate(pipe, log, input): log.write(data) finally: log.flush() + if pipe.stdout is not None: + pipe.stdout.close() + if pipe.stderr is not None: + pipe.stderr.close() return ''.join(outdata), ''.join(errdata) def run(cmd, input=None, log=None, **options): diff --git a/bitbake/lib/bb/utils.py b/bitbake/lib/bb/utils.py index 462eb689b9..2e10fc24dd 100644 --- a/bitbake/lib/bb/utils.py +++ b/bitbake/lib/bb/utils.py @@ -414,6 +414,10 @@ def lockfile(name, shared=False, retry=True): return lf lf.close() except Exception: + try: + lf.close() + except Exception: + pass pass if not retry: return None @@ -443,8 +447,9 @@ def md5_file(filename): import md5 m = md5.new() - for line in open(filename): - m.update(line) + with open(filename, "rb") as f: + for line in f: + m.update(line) return m.hexdigest() def sha256_file(filename): @@ -460,8 +465,9 @@ def sha256_file(filename): return None s = hashlib.sha256() - for line in open(filename): - s.update(line) + with open(filename, "rb") as f: + for line in f: + s.update(line) return s.hexdigest() def preserved_envvars_exported():