FetchData: add SRC_URI checksum

This patch add the per-recipe SRC_URI checksum verification.

- SRC_URI format
The format of SRC_URI checksum follow OE definition:

1. SRC_URI has single src
SRC_URI = "http://some.domain/file.tar.gz"
SRC_URI[md5sum] = "xxxxxxxxxxxxxxx"
SRC_URI[sha256sum] = "xxxxxxxxxxxxxxxxxxxxxx"

2. SRC_URI has multiple src, every src need specify name
SRC_URI = "http://some.domain/file1.tar.gz;name=name1 \
           http://some.domain/file2.tar.gz;name=name2 "
SRC_URI[name1.md5sum] = "xxxxxxxxxxxxxxx"
SRC_URI[name1.sha256sum] = "xxxxxxxxxxxxxxxxxxxxxx"
SRC_URI[name2.md5sum] = "xxxxxxxxxxxxxxx"
SRC_URI[name2.sha256sum] = "xxxxxxxxxxxxxxxxxxxxxx"

- SRC_URI checking invocation:
the checksum checking is invoked in do_fetch phase,
so it can be invoked manually by

# bitbake -f -c fetch <recipe_name>

if recipes has no SRC_URI checksum item, bitbake will show warning:
"
WARNING: Missing SRC_URI checksum for xxxx.tar.gz, consider to add
SRC_URI[md5sum] = "5c69f16d452b0bb3d44bc3c10556c072"
SRC_URI[sha256sum] = "f4e0ada8d4d516bbb8600a3ee7d9046c9c79e38cd781df9ffc46d8f16acd1768"
"
thus recipe author can add it to recpie file after SRC_URI

- control variable BB_STRICT_CHECKSUM
when SRC_URI checksum is missing, this variable decide pass or not
if BB_STRICT_CHECKSUM = "1", bitbake should fatal in this case, otherwise bitbake just pass

Signed-off-by: Yu Ke <ke.yu@intel.com>

bitbake/lib/bb/fetch/__init__.py

@@ -231,6 +231,42 @@ def removefile(f):
     except:
         pass
 
+def verify_checksum(d, ud):
+    """
+    verify the MD5 and SHA256 checksum for downloaded src
+
+    return value:
+        - True: checksum matched
+        - False: checksum unmatched
+
+    if checksum is missing in recipes file, "BB_STRICT_CHECKSUM" decide the return value.
+    if BB_STRICT_CHECKSUM = "1" then return false as unmatched, otherwise return true as
+    matched
+    """
+
+    if not ud.type in ["http", "https", "ftp", "ftps"]:
+        return True
+
+    md5data = bb.utils.md5_file(ud.localpath)
+    sha256data = bb.utils.sha256_file(ud.localpath)
+
+    if (ud.md5_expected == None or ud.sha256_expected == None):
+        bb.warn("Missing SRC_URI checksum for %s, consider to add\n" \
+                "SRC_URI[%s] = \"%s\"\nSRC_URI[%s] = \"%s\"" \
+                % (ud.localpath, ud.md5_name, md5data, ud.sha256_name, sha256data))
+        if bb.data.getVar("BB_STRICT_CHECKSUM", d, True) == "1":
+            return False
+        else:
+            return True
+
+    if (ud.md5_expected != md5data or ud.sha256_expected != sha256data):
+        bb.error("The checksums for '%s' did not match." % ud.localpath)
+        bb.error("Expected MD5: '%s' and Got: '%s'" % (ud.md5_expected, md5data))
+        bb.error("Expected SHA256: '%s' and Got: '%s'" % (ud.sha256_expected, sha256data))
+        return False
+
+    return True
+
 def go(d, urls = None):
     """
     Fetch all urls
@@ -283,6 +319,9 @@ def go(d, urls = None):
         else:
             Fetch.write_md5sum(u, ud, d)
 
+        if not verify_checksum(d, ud):
+            raise FetchError("%s checksum mismatch." % u)
+
         bb.utils.unlockfile(lf)
 
 def checkstatus(d, urls = None):
@@ -502,6 +541,16 @@ class FetchData(object):
         if not self.pswd and "pswd" in self.parm:
             self.pswd = self.parm["pswd"]
         self.setup = False
+
+        if "name" in self.parm:
+            self.md5_name = "%s.md5sum" % self.parm["name"]
+            self.sha256_name = "%s.sha256sum" % self.parm["name"]
+        else:
+            self.md5_name = "md5sum"
+            self.sha256_name = "sha256sum"
+        self.md5_expected = bb.data.getVarFlag("SRC_URI", self.md5_name, d)
+        self.sha256_expected = bb.data.getVarFlag("SRC_URI", self.sha256_name, d)
+
         for m in methods:
             if m.supports(url, self, d):
                 self.method = m