sysmo-bts
/
generic-poky
9
0
Fork 0
Code Pull Requests Releases Activity

bitbake: fetch2: implement progress support 

Implement progress reporting support specifically for the fetchers. For
fetch tasks we don't necessarily know which fetcher will be used (we
might initially be fetching a git:// URI, but if we instead download a
mirror tarball we may fetch that over http using wget). These programs
also have different abilities as far as reporting progress goes (e.g.
wget gives us percentage complete and rate, git gives this some of the
time depending on what stage it's at). Additionally we filter out the
progress output before it makes it to the logs, in order to prevent the
logs filling up with junk.

At the moment this is only implemented for the wget and git fetchers
since they are the most commonly used (and svn doesn't seem to support
any kind of progress output, at least not without doing a relatively
expensive remote file listing first).

Line changes such as the ones you get in git's output as it progresses
don't make it to the log files, you only get the final state of the line
so the logs aren't filled with progress information that's useless after
the fact.

Part of the implementation for [YOCTO #5383].

(Bitbake rev: 4027649f422ee64b1c4e1ad8d48ac295050afbff)

Signed-off-by: Paul Eggleton <paul.eggleton@linux.intel.com>
Signed-off-by: Ross Burton <ross.burton@intel.com>
Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
This commit is contained in:
Paul Eggleton 2016-07-06 16:26:10 +12:00 committed by Richard Purdie
parent 843f7ae410
commit 45baa90c8c
4 changed files with 107 additions and 8 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
bitbake/lib/bb/fetch2/__init__.py
View File

@ -779,7 +779,7 @@ def localpath(url, d):
fetcher = bb.fetch2.Fetch([url], d)
return fetcher.localpath(url)
def runfetchcmd(cmd, d, quiet=False, cleanup=None):
def runfetchcmd(cmd, d, quiet=False, cleanup=None, log=None):
"""
Run cmd returning the command output
Raise an error if interrupted or cmd fails
@ -821,7 +821,7 @@ def runfetchcmd(cmd, d, quiet=False, cleanup=None):
error_message = ""
try:
(output, errors) = bb.process.run(cmd, shell=True, stderr=subprocess.PIPE)
(output, errors) = bb.process.run(cmd, log=log, shell=True, stderr=subprocess.PIPE)
success = True
except bb.process.NotFoundError as e:
error_message = "Fetch command %s" % (e.command)

52
bitbake/lib/bb/fetch2/git.py
View File

@ -71,11 +71,53 @@ import os
import re
import bb
import errno
import bb.progress
from bb import data
from bb.fetch2 import FetchMethod
from bb.fetch2 import runfetchcmd
from bb.fetch2 import logger
class GitProgressHandler(bb.progress.LineFilterProgressHandler):
"""Extract progress information from git output"""
def __init__(self, d):
self._buffer = ''
self._count = 0
super(GitProgressHandler, self).__init__(d)
# Send an initial progress event so the bar gets shown
self._fire_progress(-1)
def write(self, string):
self._buffer += string
stages = ['Counting objects', 'Compressing objects', 'Receiving objects', 'Resolving deltas']
stage_weights = [0.2, 0.05, 0.5, 0.25]
stagenum = 0
for i, stage in reversed(list(enumerate(stages))):
if stage in self._buffer:
stagenum = i
self._buffer = ''
break
self._status = stages[stagenum]
percs = re.findall(r'(\d+)%', string)
if percs:
progress = int(round((int(percs[-1]) * stage_weights[stagenum]) + (sum(stage_weights[:stagenum]) * 100)))
rates = re.findall(r'([\d.]+ [a-zA-Z]*/s+)', string)
if rates:
rate = rates[-1]
else:
rate = None
self.update(progress, rate)
else:
if stagenum == 0:
percs = re.findall(r': (\d+)', string)
if percs:
count = int(percs[-1])
if count > self._count:
self._count = count
self._fire_progress(-count)
super(GitProgressHandler, self).write(string)
class Git(FetchMethod):
"""Class to fetch a module or modules from git repositories"""
def init(self, d):
@ -196,10 +238,11 @@ class Git(FetchMethod):
# We do this since git will use a "-l" option automatically for local urls where possible
if repourl.startswith("file://"):
repourl = repourl[7:]
clone_cmd = "%s clone --bare --mirror %s %s" % (ud.basecmd, repourl, ud.clonedir)
clone_cmd = "LANG=C %s clone --bare --mirror %s %s --progress" % (ud.basecmd, repourl, ud.clonedir)
if ud.proto.lower() != 'file':
bb.fetch2.check_network_access(d, clone_cmd)
runfetchcmd(clone_cmd, d)
progresshandler = GitProgressHandler(d)
runfetchcmd(clone_cmd, d, log=progresshandler)
os.chdir(ud.clonedir)
# Update the checkout if needed
@ -214,10 +257,11 @@ class Git(FetchMethod):
logger.debug(1, "No Origin")
runfetchcmd("%s remote add --mirror=fetch origin %s" % (ud.basecmd, repourl), d)
fetch_cmd = "%s fetch -f --prune %s refs/*:refs/*" % (ud.basecmd, repourl)
fetch_cmd = "LANG=C %s fetch -f --prune --progress %s refs/*:refs/*" % (ud.basecmd, repourl)
if ud.proto.lower() != 'file':
bb.fetch2.check_network_access(d, fetch_cmd, ud.url)
runfetchcmd(fetch_cmd, d)
progresshandler = GitProgressHandler(d)
runfetchcmd(fetch_cmd, d, log=progresshandler)
runfetchcmd("%s prune-packed" % ud.basecmd, d)
runfetchcmd("%s pack-redundant --all | xargs -r rm" % ud.basecmd, d)
try:

28
bitbake/lib/bb/fetch2/wget.py
View File

@ -31,6 +31,7 @@ import subprocess
import os
import logging
import bb
import bb.progress
import urllib.request, urllib.parse, urllib.error
from bb import data
from bb.fetch2 import FetchMethod
@ -41,6 +42,27 @@ from bb.utils import export_proxies
from bs4 import BeautifulSoup
from bs4 import SoupStrainer
class WgetProgressHandler(bb.progress.LineFilterProgressHandler):
"""
Extract progress information from wget output.
Note: relies on --progress=dot (with -v or without -q/-nv) being
specified on the wget command line.
"""
def __init__(self, d):
super(WgetProgressHandler, self).__init__(d)
# Send an initial progress event so the bar gets shown
self._fire_progress(0)
def writeline(self, line):
percs = re.findall(r'(\d+)%\s+([\d.]+[A-Z])', line)
if percs:
progress = int(percs[-1][0])
rate = percs[-1][1] + '/s'
self.update(progress, rate)
return False
return True
class Wget(FetchMethod):
"""Class to fetch urls via 'wget'"""
def supports(self, ud, d):
@ -66,13 +88,15 @@ class Wget(FetchMethod):
if not ud.localfile:
ud.localfile = data.expand(urllib.parse.unquote(ud.host + ud.path).replace("/", "."), d)
self.basecmd = d.getVar("FETCHCMD_wget", True) or "/usr/bin/env wget -t 2 -T 30 -nv --passive-ftp --no-check-certificate"
self.basecmd = d.getVar("FETCHCMD_wget", True) or "/usr/bin/env wget -t 2 -T 30 --passive-ftp --no-check-certificate"
def _runwget(self, ud, d, command, quiet):
progresshandler = WgetProgressHandler(d)
logger.debug(2, "Fetching %s using command '%s'" % (ud.url, command))
bb.fetch2.check_network_access(d, command)
runfetchcmd(command, d, quiet)
runfetchcmd(command + ' --progress=dot -v', d, quiet, log=progresshandler)
def download(self, ud, d):
"""Fetch urls"""

31
bitbake/lib/bb/progress.py
View File

@ -58,6 +58,37 @@ class ProgressHandler(object):
self._lastevent = ts
self._progress = progress
class LineFilterProgressHandler(ProgressHandler):
"""
A ProgressHandler variant that provides the ability to filter out
the lines if they contain progress information. Additionally, it
filters out anything before the last line feed on a line. This can
be used to keep the logs clean of output that we've only enabled for
getting progress, assuming that that can be done on a per-line
basis.
"""
def __init__(self, d, outfile=None):
self._linebuffer = ''
super(LineFilterProgressHandler, self).__init__(d, outfile)
def write(self, string):
self._linebuffer += string
while True:
breakpos = self._linebuffer.find('\n') + 1
if breakpos == 0:
break
line = self._linebuffer[:breakpos]
self._linebuffer = self._linebuffer[breakpos:]
# Drop any line feeds and anything that precedes them
lbreakpos = line.rfind('\r') + 1
if lbreakpos:
line = line[lbreakpos:]
if self.writeline(line):
super(LineFilterProgressHandler, self).write(line)
def writeline(self, line):
return True
class BasicProgressHandler(ProgressHandler):
def __init__(self, d, regex=r'(\d+)%', outfile=None):
super(BasicProgressHandler, self).__init__(d, outfile)