2016-02-12 14:55:50 +00:00
|
|
|
#!/usr/bin/env python3
|
|
|
|
|
|
|
|
import sys, os, subprocess, re, shutil
|
|
|
|
|
|
|
|
whitelist = (
|
|
|
|
# type is supported by dash
|
|
|
|
'if type systemctl >/dev/null 2>/dev/null; then',
|
|
|
|
'if type systemd-tmpfiles >/dev/null 2>/dev/null; then',
|
2017-01-31 12:50:30 +00:00
|
|
|
'type update-rc.d >/dev/null 2>/dev/null; then',
|
2016-02-12 14:55:50 +00:00
|
|
|
'command -v',
|
|
|
|
# HOSTNAME is set locally
|
|
|
|
'buildhistory_single_commit "$CMDLINE" "$HOSTNAME"',
|
|
|
|
# False-positive, match is a grep not shell expression
|
|
|
|
'grep "^$groupname:[^:]*:[^:]*:\\([^,]*,\\)*$username\\(,[^,]*\\)*"',
|
|
|
|
# TODO verify dash's '. script args' behaviour
|
|
|
|
'. $target_sdk_dir/${oe_init_build_env_path} $target_sdk_dir >> $LOGFILE'
|
|
|
|
)
|
|
|
|
|
|
|
|
def is_whitelisted(s):
|
|
|
|
for w in whitelist:
|
|
|
|
if w in s:
|
|
|
|
return True
|
|
|
|
return False
|
|
|
|
|
2017-01-31 12:50:31 +00:00
|
|
|
SCRIPT_LINENO_RE = re.compile(r' line (\d+) ')
|
2017-01-31 12:50:33 +00:00
|
|
|
BASHISM_WARNING = re.compile(r'^(possible bashism in.*)$', re.MULTILINE)
|
2017-01-31 12:50:31 +00:00
|
|
|
|
|
|
|
def process(filename, function, lineno, script):
|
2016-02-12 14:55:50 +00:00
|
|
|
import tempfile
|
|
|
|
|
|
|
|
if not script.startswith("#!"):
|
|
|
|
script = "#! /bin/sh\n" + script
|
|
|
|
|
|
|
|
fn = tempfile.NamedTemporaryFile(mode="w+t")
|
|
|
|
fn.write(script)
|
|
|
|
fn.flush()
|
|
|
|
|
|
|
|
try:
|
|
|
|
subprocess.check_output(("checkbashisms.pl", fn.name), universal_newlines=True, stderr=subprocess.STDOUT)
|
|
|
|
# No bashisms, so just return
|
|
|
|
return
|
|
|
|
except subprocess.CalledProcessError as e:
|
|
|
|
# TODO check exit code is 1
|
|
|
|
|
|
|
|
# Replace the temporary filename with the function and split it
|
2017-01-31 12:50:33 +00:00
|
|
|
output = e.output.replace(fn.name, function)
|
|
|
|
if not output or not output.startswith('possible bashism'):
|
|
|
|
# Probably starts with or contains only warnings. Dump verbatim
|
|
|
|
# with one space indention. Can't do the splitting and whitelist
|
|
|
|
# checking below.
|
|
|
|
return '\n'.join([filename,
|
|
|
|
' Unexpected output from checkbashisms.pl'] +
|
|
|
|
[' ' + x for x in output.splitlines()])
|
2016-02-12 14:55:50 +00:00
|
|
|
|
2017-01-31 12:50:33 +00:00
|
|
|
# We know that the first line matches and that therefore the first
|
|
|
|
# list entry will be empty - skip it.
|
|
|
|
output = BASHISM_WARNING.split(output)[1:]
|
2017-01-31 12:50:31 +00:00
|
|
|
# Turn the output into a single string like this:
|
|
|
|
# /.../foobar.bb
|
|
|
|
# possible bashism in updatercd_postrm line 2 (type):
|
|
|
|
# if ${@use_updatercd(d)} && type update-rc.d >/dev/null 2>/dev/null; then
|
|
|
|
# ...
|
|
|
|
# ...
|
2016-02-12 14:55:50 +00:00
|
|
|
result = []
|
|
|
|
# Check the results against the whitelist
|
|
|
|
for message, source in zip(output[0::2], output[1::2]):
|
|
|
|
if not is_whitelisted(source):
|
2017-01-31 12:50:31 +00:00
|
|
|
if lineno is not None:
|
|
|
|
message = SCRIPT_LINENO_RE.sub(lambda m: ' line %d ' % (int(m.group(1)) + int(lineno) - 1),
|
|
|
|
message)
|
2017-01-31 12:50:33 +00:00
|
|
|
result.append(' ' + message.strip())
|
|
|
|
result.extend([' %s' % x for x in source.splitlines()])
|
2017-01-31 12:50:31 +00:00
|
|
|
if result:
|
|
|
|
result.insert(0, filename)
|
|
|
|
return '\n'.join(result)
|
|
|
|
else:
|
|
|
|
return None
|
2016-02-12 14:55:50 +00:00
|
|
|
|
|
|
|
def get_tinfoil():
|
|
|
|
scripts_path = os.path.dirname(os.path.realpath(__file__))
|
|
|
|
lib_path = scripts_path + '/lib'
|
|
|
|
sys.path = sys.path + [lib_path]
|
|
|
|
import scriptpath
|
|
|
|
scriptpath.add_bitbake_lib_path()
|
|
|
|
import bb.tinfoil
|
|
|
|
tinfoil = bb.tinfoil.Tinfoil()
|
|
|
|
tinfoil.prepare()
|
|
|
|
# tinfoil.logger.setLevel(logging.WARNING)
|
|
|
|
return tinfoil
|
|
|
|
|
|
|
|
if __name__=='__main__':
|
|
|
|
import shutil
|
|
|
|
if shutil.which("checkbashisms.pl") is None:
|
2017-01-31 12:50:27 +00:00
|
|
|
print("Cannot find checkbashisms.pl on $PATH, get it from https://anonscm.debian.org/cgit/collab-maint/devscripts.git/plain/scripts/checkbashisms.pl")
|
2016-02-12 14:55:50 +00:00
|
|
|
sys.exit(1)
|
|
|
|
|
verify-bashisms: fix problems with tinfoil2
tinfoil2 is based on a client/server architecture, which broke the
verify-bashisms script:
- The tinfoil instance and its data proxies can't be pickled, so
all interaction with the bitbake server has to run in the main
script process and only processing of the plain scripts can
be done with multiprocessing:
_pickle.PicklingError: Can't pickle <class 'bb.tinfoil.TinfoilCookerAdapter.TinfoilRecipeCacheAdapter'>: attribute lookup TinfoilRecipeCacheAdapter on bb.tinfoil failed
- The multiprocessing pool has to be created before initializing
tinfoil, otherwise the pool workers end up trying to communicate
with the bitbake server during shutdown:
ERROR: UI received SIGTERM
Process ForkPoolWorker-2:
Traceback (most recent call last):
File "/usr/lib/python3.4/multiprocessing/process.py", line 257, in _bootstrap
util._exit_function()
File "/usr/lib/python3.4/multiprocessing/util.py", line 286, in _exit_function
_run_finalizers(0)
...
File "/usr/lib/python3.4/multiprocessing/process.py", line 131, in is_alive
assert self._parent_pid == os.getpid(), 'can only test a child process'
AssertionError: can only test a child process
- func() needs to defined before creating the pool to avoid:
AttributeError: Can't get attribute 'func' on <module '__main__' from '/work/openembedded-core/scripts/verify-bashisms'>
(From OE-Core rev: aa439f11c7f414774843720d68ebe0a6d3375ea6)
Signed-off-by: Patrick Ohly <patrick.ohly@intel.com>
Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
2017-01-31 12:50:29 +00:00
|
|
|
# The order of defining the worker function,
|
|
|
|
# initializing the pool and connecting to the
|
|
|
|
# bitbake server is crucial, don't change it.
|
|
|
|
def func(item):
|
2017-01-31 12:50:31 +00:00
|
|
|
(filename, key, lineno), script = item
|
|
|
|
return process(filename, key, lineno, script)
|
verify-bashisms: fix problems with tinfoil2
tinfoil2 is based on a client/server architecture, which broke the
verify-bashisms script:
- The tinfoil instance and its data proxies can't be pickled, so
all interaction with the bitbake server has to run in the main
script process and only processing of the plain scripts can
be done with multiprocessing:
_pickle.PicklingError: Can't pickle <class 'bb.tinfoil.TinfoilCookerAdapter.TinfoilRecipeCacheAdapter'>: attribute lookup TinfoilRecipeCacheAdapter on bb.tinfoil failed
- The multiprocessing pool has to be created before initializing
tinfoil, otherwise the pool workers end up trying to communicate
with the bitbake server during shutdown:
ERROR: UI received SIGTERM
Process ForkPoolWorker-2:
Traceback (most recent call last):
File "/usr/lib/python3.4/multiprocessing/process.py", line 257, in _bootstrap
util._exit_function()
File "/usr/lib/python3.4/multiprocessing/util.py", line 286, in _exit_function
_run_finalizers(0)
...
File "/usr/lib/python3.4/multiprocessing/process.py", line 131, in is_alive
assert self._parent_pid == os.getpid(), 'can only test a child process'
AssertionError: can only test a child process
- func() needs to defined before creating the pool to avoid:
AttributeError: Can't get attribute 'func' on <module '__main__' from '/work/openembedded-core/scripts/verify-bashisms'>
(From OE-Core rev: aa439f11c7f414774843720d68ebe0a6d3375ea6)
Signed-off-by: Patrick Ohly <patrick.ohly@intel.com>
Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
2017-01-31 12:50:29 +00:00
|
|
|
|
|
|
|
import multiprocessing
|
|
|
|
pool = multiprocessing.Pool()
|
|
|
|
|
2016-02-12 14:55:50 +00:00
|
|
|
tinfoil = get_tinfoil()
|
|
|
|
|
|
|
|
# This is only the default configuration and should iterate over
|
|
|
|
# recipecaches to handle multiconfig environments
|
|
|
|
pkg_pn = tinfoil.cooker.recipecaches[""].pkg_pn
|
|
|
|
|
|
|
|
# TODO: use argparse and have --help
|
|
|
|
if len(sys.argv) > 1:
|
|
|
|
initial_pns = sys.argv[1:]
|
|
|
|
else:
|
|
|
|
initial_pns = sorted(pkg_pn)
|
|
|
|
|
2017-01-31 12:50:31 +00:00
|
|
|
pns = set()
|
|
|
|
scripts = {}
|
verify-bashisms: fix problems with tinfoil2
tinfoil2 is based on a client/server architecture, which broke the
verify-bashisms script:
- The tinfoil instance and its data proxies can't be pickled, so
all interaction with the bitbake server has to run in the main
script process and only processing of the plain scripts can
be done with multiprocessing:
_pickle.PicklingError: Can't pickle <class 'bb.tinfoil.TinfoilCookerAdapter.TinfoilRecipeCacheAdapter'>: attribute lookup TinfoilRecipeCacheAdapter on bb.tinfoil failed
- The multiprocessing pool has to be created before initializing
tinfoil, otherwise the pool workers end up trying to communicate
with the bitbake server during shutdown:
ERROR: UI received SIGTERM
Process ForkPoolWorker-2:
Traceback (most recent call last):
File "/usr/lib/python3.4/multiprocessing/process.py", line 257, in _bootstrap
util._exit_function()
File "/usr/lib/python3.4/multiprocessing/util.py", line 286, in _exit_function
_run_finalizers(0)
...
File "/usr/lib/python3.4/multiprocessing/process.py", line 131, in is_alive
assert self._parent_pid == os.getpid(), 'can only test a child process'
AssertionError: can only test a child process
- func() needs to defined before creating the pool to avoid:
AttributeError: Can't get attribute 'func' on <module '__main__' from '/work/openembedded-core/scripts/verify-bashisms'>
(From OE-Core rev: aa439f11c7f414774843720d68ebe0a6d3375ea6)
Signed-off-by: Patrick Ohly <patrick.ohly@intel.com>
Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
2017-01-31 12:50:29 +00:00
|
|
|
print("Generating scripts...")
|
2016-02-12 14:55:50 +00:00
|
|
|
for pn in initial_pns:
|
|
|
|
for fn in pkg_pn[pn]:
|
|
|
|
# There's no point checking multiple BBCLASSEXTENDed variants of the same recipe
|
2017-01-31 12:50:31 +00:00
|
|
|
# (at least in general - there is some risk that the variants contain different scripts)
|
2016-02-12 14:55:50 +00:00
|
|
|
realfn, _, _ = bb.cache.virtualfn2realfn(fn)
|
|
|
|
if realfn not in pns:
|
2017-01-31 12:50:31 +00:00
|
|
|
pns.add(realfn)
|
verify-bashisms: fix problems with tinfoil2
tinfoil2 is based on a client/server architecture, which broke the
verify-bashisms script:
- The tinfoil instance and its data proxies can't be pickled, so
all interaction with the bitbake server has to run in the main
script process and only processing of the plain scripts can
be done with multiprocessing:
_pickle.PicklingError: Can't pickle <class 'bb.tinfoil.TinfoilCookerAdapter.TinfoilRecipeCacheAdapter'>: attribute lookup TinfoilRecipeCacheAdapter on bb.tinfoil failed
- The multiprocessing pool has to be created before initializing
tinfoil, otherwise the pool workers end up trying to communicate
with the bitbake server during shutdown:
ERROR: UI received SIGTERM
Process ForkPoolWorker-2:
Traceback (most recent call last):
File "/usr/lib/python3.4/multiprocessing/process.py", line 257, in _bootstrap
util._exit_function()
File "/usr/lib/python3.4/multiprocessing/util.py", line 286, in _exit_function
_run_finalizers(0)
...
File "/usr/lib/python3.4/multiprocessing/process.py", line 131, in is_alive
assert self._parent_pid == os.getpid(), 'can only test a child process'
AssertionError: can only test a child process
- func() needs to defined before creating the pool to avoid:
AttributeError: Can't get attribute 'func' on <module '__main__' from '/work/openembedded-core/scripts/verify-bashisms'>
(From OE-Core rev: aa439f11c7f414774843720d68ebe0a6d3375ea6)
Signed-off-by: Patrick Ohly <patrick.ohly@intel.com>
Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
2017-01-31 12:50:29 +00:00
|
|
|
data = tinfoil.parse_recipe_file(realfn)
|
|
|
|
for key in data.keys():
|
|
|
|
if data.getVarFlag(key, "func") and not data.getVarFlag(key, "python"):
|
|
|
|
script = data.getVar(key, False)
|
|
|
|
if script:
|
2017-01-31 12:50:31 +00:00
|
|
|
filename = data.getVarFlag(key, "filename")
|
|
|
|
lineno = data.getVarFlag(key, "lineno")
|
|
|
|
# There's no point in checking a function multiple
|
|
|
|
# times just because different recipes include it.
|
|
|
|
# We identify unique scripts by file, name, and (just in case)
|
|
|
|
# line number.
|
|
|
|
attributes = (filename or realfn, key, lineno)
|
|
|
|
scripts.setdefault(attributes, script)
|
|
|
|
|
2016-02-12 14:55:50 +00:00
|
|
|
|
|
|
|
print("Scanning scripts...\n")
|
2017-01-31 12:50:31 +00:00
|
|
|
for result in pool.imap(func, scripts.items()):
|
|
|
|
if result:
|
|
|
|
print(result)
|
2017-01-31 12:50:28 +00:00
|
|
|
tinfoil.shutdown()
|