recipetool: create: avoid decoding errors with Python 3

We're opening source files with the default encoding (utf-8) but we
can't necessarily be sure that they are UTF-8 clean - for example,
recipetool create ftp://mama.indstate.edu/linux/tree/tree-1.7.0.tgz
prior to this patch resulted in a UnicodeDecodeError. Use the
"surrogateescape" mode to avoid this.

Fixes [YOCTO #9822].

(From OE-Core rev: 50fcd9d1b9a20d49bc873467a82a071f2f2f8b5a)

Signed-off-by: Paul Eggleton <paul.eggleton@linux.intel.com>
Signed-off-by: Ross Burton <ross.burton@intel.com>
Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>

scripts/lib/recipetool/create.py

@@ -390,7 +390,7 @@ def create_recipe(args):
                 srcsubdir = dirlist[0]
                 srctree = os.path.join(srctree, srcsubdir)
             else:
-                with open(singleitem, 'r') as f:
+                with open(singleitem, 'r', errors='surrogateescape') as f:
                     if '<html' in f.read(100).lower():
                         logger.error('Fetching "%s" returned a single HTML page - check the URL is correct and functional' % fetchuri)
                         sys.exit(1)
@@ -840,7 +840,7 @@ def crunch_license(licfile):
     # https://github.com/FFmpeg/FFmpeg/blob/master/COPYING.LGPLv3
     crunched_md5sums['2ebfb3bb49b9a48a075cc1425e7f4129'] = 'LGPLv3'
     lictext = []
-    with open(licfile, 'r') as f:
+    with open(licfile, 'r', errors='surrogateescape') as f:
         for line in f:
             # Drop opening statements
             if copyright_re.match(line):
@@ -978,7 +978,7 @@ def convert_debian(debpath):
 
     values = {}
     depends = []
-    with open(os.path.join(debpath, 'control')) as f:
+    with open(os.path.join(debpath, 'control'), 'r', errors='surrogateescape') as f:
         indesc = False
         for line in f:
             if indesc:

scripts/lib/recipetool/create_buildsys.py

@@ -173,7 +173,7 @@ class CmakeRecipeHandler(RecipeHandler):
         def parse_cmake_file(fn, paths=None):
             searchpaths = (paths or []) + [os.path.dirname(fn)]
             logger.debug('Parsing file %s' % fn)
-            with open(fn, 'r') as f:
+            with open(fn, 'r', errors='surrogateescape') as f:
                 for line in f:
                     line = line.strip()
                     for handler in handlers:
@@ -354,7 +354,7 @@ class AutotoolsRecipeHandler(RecipeHandler):
             conffile = RecipeHandler.checkfiles(srctree, ['configure'])
             if conffile:
                 # Check if this is just a pre-generated autoconf configure script
-                with open(conffile[0], 'r') as f:
+                with open(conffile[0], 'r', errors='surrogateescape') as f:
                     for i in range(1, 10):
                         if 'Generated by GNU Autoconf' in f.readline():
                             autoconf = True
@@ -364,7 +364,7 @@ class AutotoolsRecipeHandler(RecipeHandler):
             # Last resort
             conffile = RecipeHandler.checkfiles(srctree, ['configure'])
             if conffile:
-                with open(conffile[0], 'r') as f:
+                with open(conffile[0], 'r', errors='surrogateescape') as f:
                     for line in f:
                         line = line.strip()
                         if line.startswith('VERSION=') or line.startswith('PACKAGE_VERSION='):
@@ -654,7 +654,7 @@ class AutotoolsRecipeHandler(RecipeHandler):
             nesting = 0
             in_keyword = ''
             partial = ''
-            with open(srcfile, 'r') as f:
+            with open(srcfile, 'r', errors='surrogateescape') as f:
                 for line in f:
                     if in_keyword:
                         partial += ' ' + line.strip()
@@ -780,7 +780,7 @@ class MakefileRecipeHandler(RecipeHandler):
             if installtarget:
                 func.append('# This is a guess; additional arguments may be required')
                 makeargs = ''
-                with open(makefile[0], 'r') as f:
+                with open(makefile[0], 'r', errors='surrogateescape') as f:
                     for i in range(1, 100):
                         if 'DESTDIR' in f.readline():
                             makeargs += " 'DESTDIR=${D}'"
@@ -809,7 +809,7 @@ class VersionFileRecipeHandler(RecipeHandler):
             version = None
             for fileitem in filelist:
                 linecount = 0
-                with open(fileitem, 'r') as f:
+                with open(fileitem, 'r', errors='surrogateescape') as f:
                     for line in f:
                         line = line.rstrip().strip('"\'')
                         linecount += 1
@@ -838,7 +838,7 @@ class SpecFileRecipeHandler(RecipeHandler):
         foundvalues = {}
         for fileitem in filelist:
             linecount = 0
-            with open(fileitem, 'r') as f:
+            with open(fileitem, 'r', errors='surrogateescape') as f:
                 for line in f:
                     for value, varname in valuemap.items():
                         if line.startswith(value + ':') and not varname in foundvalues:

scripts/lib/recipetool/create_kernel.py

@@ -59,7 +59,7 @@ class KernelRecipeHandler(RecipeHandler):
                     kpatchlevel = -1
                     ksublevel = -1
                     kextraversion = ''
-                    with open(makefile, 'r') as f:
+                    with open(makefile, 'r', errors='surrogateescape') as f:
                         for i, line in enumerate(f):
                             if i > 10:
                                 break

scripts/lib/recipetool/create_kmod.py

@@ -53,7 +53,7 @@ class KernelModuleRecipeHandler(RecipeHandler):
                             break
                 else:
                     continue
-                with open(cfile, 'r') as f:
+                with open(cfile, 'r', errors='surrogateescape') as f:
                     for line in f:
                         if module_inc_re.match(line.strip()):
                             is_module = True
@@ -73,7 +73,7 @@ class KernelModuleRecipeHandler(RecipeHandler):
             in_install = False
             in_compile = False
             install_target = None
-            with open(makefile, 'r') as f:
+            with open(makefile, 'r', errors='surrogateescape') as f:
                 for line in f:
                     if line.startswith('install:'):
                         if not install_lines:

scripts/lib/recipetool/create_npm.py

@@ -92,7 +92,7 @@ class NpmRecipeHandler(RecipeHandler):
             return False
 
         def read_package_json(fn):
-            with open(fn, 'r') as f:
+            with open(fn, 'r', errors='surrogateescape') as f:
                 return json.loads(f.read())
 
         files = RecipeHandler.checkfiles(srctree, ['package.json'])