2016-02-12 14:55:50 +00:00
|
|
|
#!/usr/bin/env python3
|
|
|
|
|
|
|
|
import sys, os, subprocess, re, shutil
|
|
|
|
|
|
|
|
whitelist = (
|
|
|
|
# type is supported by dash
|
|
|
|
'if type systemctl >/dev/null 2>/dev/null; then',
|
|
|
|
'if type systemd-tmpfiles >/dev/null 2>/dev/null; then',
|
|
|
|
'if type update-rc.d >/dev/null 2>/dev/null; then',
|
|
|
|
'command -v',
|
|
|
|
# HOSTNAME is set locally
|
|
|
|
'buildhistory_single_commit "$CMDLINE" "$HOSTNAME"',
|
|
|
|
# False-positive, match is a grep not shell expression
|
|
|
|
'grep "^$groupname:[^:]*:[^:]*:\\([^,]*,\\)*$username\\(,[^,]*\\)*"',
|
|
|
|
# TODO verify dash's '. script args' behaviour
|
|
|
|
'. $target_sdk_dir/${oe_init_build_env_path} $target_sdk_dir >> $LOGFILE'
|
|
|
|
)
|
|
|
|
|
|
|
|
def is_whitelisted(s):
|
|
|
|
for w in whitelist:
|
|
|
|
if w in s:
|
|
|
|
return True
|
|
|
|
return False
|
|
|
|
|
|
|
|
def process(recipe, function, script):
|
|
|
|
import tempfile
|
|
|
|
|
|
|
|
if not script.startswith("#!"):
|
|
|
|
script = "#! /bin/sh\n" + script
|
|
|
|
|
|
|
|
fn = tempfile.NamedTemporaryFile(mode="w+t")
|
|
|
|
fn.write(script)
|
|
|
|
fn.flush()
|
|
|
|
|
|
|
|
try:
|
|
|
|
subprocess.check_output(("checkbashisms.pl", fn.name), universal_newlines=True, stderr=subprocess.STDOUT)
|
|
|
|
# No bashisms, so just return
|
|
|
|
return
|
|
|
|
except subprocess.CalledProcessError as e:
|
|
|
|
# TODO check exit code is 1
|
|
|
|
|
|
|
|
# Replace the temporary filename with the function and split it
|
|
|
|
output = e.output.replace(fn.name, function).splitlines()
|
2017-01-31 12:50:26 +00:00
|
|
|
if len(output) % 2 != 0:
|
2016-02-12 14:55:50 +00:00
|
|
|
print("Unexpected output from checkbashism: %s" % str(output))
|
|
|
|
return
|
|
|
|
|
|
|
|
# Turn the output into a list of (message, source) values
|
|
|
|
result = []
|
|
|
|
# Check the results against the whitelist
|
|
|
|
for message, source in zip(output[0::2], output[1::2]):
|
|
|
|
if not is_whitelisted(source):
|
|
|
|
result.append((message, source))
|
|
|
|
return result
|
|
|
|
|
|
|
|
def get_tinfoil():
|
|
|
|
scripts_path = os.path.dirname(os.path.realpath(__file__))
|
|
|
|
lib_path = scripts_path + '/lib'
|
|
|
|
sys.path = sys.path + [lib_path]
|
|
|
|
import scriptpath
|
|
|
|
scriptpath.add_bitbake_lib_path()
|
|
|
|
import bb.tinfoil
|
|
|
|
tinfoil = bb.tinfoil.Tinfoil()
|
|
|
|
tinfoil.prepare()
|
|
|
|
# tinfoil.logger.setLevel(logging.WARNING)
|
|
|
|
return tinfoil
|
|
|
|
|
|
|
|
if __name__=='__main__':
|
|
|
|
import shutil
|
|
|
|
if shutil.which("checkbashisms.pl") is None:
|
2017-01-31 12:50:27 +00:00
|
|
|
print("Cannot find checkbashisms.pl on $PATH, get it from https://anonscm.debian.org/cgit/collab-maint/devscripts.git/plain/scripts/checkbashisms.pl")
|
2016-02-12 14:55:50 +00:00
|
|
|
sys.exit(1)
|
|
|
|
|
verify-bashisms: fix problems with tinfoil2
tinfoil2 is based on a client/server architecture, which broke the
verify-bashisms script:
- The tinfoil instance and its data proxies can't be pickled, so
all interaction with the bitbake server has to run in the main
script process and only processing of the plain scripts can
be done with multiprocessing:
_pickle.PicklingError: Can't pickle <class 'bb.tinfoil.TinfoilCookerAdapter.TinfoilRecipeCacheAdapter'>: attribute lookup TinfoilRecipeCacheAdapter on bb.tinfoil failed
- The multiprocessing pool has to be created before initializing
tinfoil, otherwise the pool workers end up trying to communicate
with the bitbake server during shutdown:
ERROR: UI received SIGTERM
Process ForkPoolWorker-2:
Traceback (most recent call last):
File "/usr/lib/python3.4/multiprocessing/process.py", line 257, in _bootstrap
util._exit_function()
File "/usr/lib/python3.4/multiprocessing/util.py", line 286, in _exit_function
_run_finalizers(0)
...
File "/usr/lib/python3.4/multiprocessing/process.py", line 131, in is_alive
assert self._parent_pid == os.getpid(), 'can only test a child process'
AssertionError: can only test a child process
- func() needs to defined before creating the pool to avoid:
AttributeError: Can't get attribute 'func' on <module '__main__' from '/work/openembedded-core/scripts/verify-bashisms'>
(From OE-Core rev: aa439f11c7f414774843720d68ebe0a6d3375ea6)
Signed-off-by: Patrick Ohly <patrick.ohly@intel.com>
Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
2017-01-31 12:50:29 +00:00
|
|
|
# The order of defining the worker function,
|
|
|
|
# initializing the pool and connecting to the
|
|
|
|
# bitbake server is crucial, don't change it.
|
|
|
|
def func(item):
|
|
|
|
fn, scripts = item
|
|
|
|
result = []
|
|
|
|
for key, script in scripts:
|
|
|
|
r = process(fn, key, script)
|
|
|
|
if r: result.extend(r)
|
|
|
|
return fn, result
|
|
|
|
|
|
|
|
import multiprocessing
|
|
|
|
pool = multiprocessing.Pool()
|
|
|
|
|
2016-02-12 14:55:50 +00:00
|
|
|
tinfoil = get_tinfoil()
|
|
|
|
|
|
|
|
# This is only the default configuration and should iterate over
|
|
|
|
# recipecaches to handle multiconfig environments
|
|
|
|
pkg_pn = tinfoil.cooker.recipecaches[""].pkg_pn
|
|
|
|
|
|
|
|
# TODO: use argparse and have --help
|
|
|
|
if len(sys.argv) > 1:
|
|
|
|
initial_pns = sys.argv[1:]
|
|
|
|
else:
|
|
|
|
initial_pns = sorted(pkg_pn)
|
|
|
|
|
verify-bashisms: fix problems with tinfoil2
tinfoil2 is based on a client/server architecture, which broke the
verify-bashisms script:
- The tinfoil instance and its data proxies can't be pickled, so
all interaction with the bitbake server has to run in the main
script process and only processing of the plain scripts can
be done with multiprocessing:
_pickle.PicklingError: Can't pickle <class 'bb.tinfoil.TinfoilCookerAdapter.TinfoilRecipeCacheAdapter'>: attribute lookup TinfoilRecipeCacheAdapter on bb.tinfoil failed
- The multiprocessing pool has to be created before initializing
tinfoil, otherwise the pool workers end up trying to communicate
with the bitbake server during shutdown:
ERROR: UI received SIGTERM
Process ForkPoolWorker-2:
Traceback (most recent call last):
File "/usr/lib/python3.4/multiprocessing/process.py", line 257, in _bootstrap
util._exit_function()
File "/usr/lib/python3.4/multiprocessing/util.py", line 286, in _exit_function
_run_finalizers(0)
...
File "/usr/lib/python3.4/multiprocessing/process.py", line 131, in is_alive
assert self._parent_pid == os.getpid(), 'can only test a child process'
AssertionError: can only test a child process
- func() needs to defined before creating the pool to avoid:
AttributeError: Can't get attribute 'func' on <module '__main__' from '/work/openembedded-core/scripts/verify-bashisms'>
(From OE-Core rev: aa439f11c7f414774843720d68ebe0a6d3375ea6)
Signed-off-by: Patrick Ohly <patrick.ohly@intel.com>
Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
2017-01-31 12:50:29 +00:00
|
|
|
pns = {}
|
|
|
|
print("Generating scripts...")
|
2016-02-12 14:55:50 +00:00
|
|
|
for pn in initial_pns:
|
|
|
|
for fn in pkg_pn[pn]:
|
|
|
|
# There's no point checking multiple BBCLASSEXTENDed variants of the same recipe
|
|
|
|
realfn, _, _ = bb.cache.virtualfn2realfn(fn)
|
|
|
|
if realfn not in pns:
|
verify-bashisms: fix problems with tinfoil2
tinfoil2 is based on a client/server architecture, which broke the
verify-bashisms script:
- The tinfoil instance and its data proxies can't be pickled, so
all interaction with the bitbake server has to run in the main
script process and only processing of the plain scripts can
be done with multiprocessing:
_pickle.PicklingError: Can't pickle <class 'bb.tinfoil.TinfoilCookerAdapter.TinfoilRecipeCacheAdapter'>: attribute lookup TinfoilRecipeCacheAdapter on bb.tinfoil failed
- The multiprocessing pool has to be created before initializing
tinfoil, otherwise the pool workers end up trying to communicate
with the bitbake server during shutdown:
ERROR: UI received SIGTERM
Process ForkPoolWorker-2:
Traceback (most recent call last):
File "/usr/lib/python3.4/multiprocessing/process.py", line 257, in _bootstrap
util._exit_function()
File "/usr/lib/python3.4/multiprocessing/util.py", line 286, in _exit_function
_run_finalizers(0)
...
File "/usr/lib/python3.4/multiprocessing/process.py", line 131, in is_alive
assert self._parent_pid == os.getpid(), 'can only test a child process'
AssertionError: can only test a child process
- func() needs to defined before creating the pool to avoid:
AttributeError: Can't get attribute 'func' on <module '__main__' from '/work/openembedded-core/scripts/verify-bashisms'>
(From OE-Core rev: aa439f11c7f414774843720d68ebe0a6d3375ea6)
Signed-off-by: Patrick Ohly <patrick.ohly@intel.com>
Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
2017-01-31 12:50:29 +00:00
|
|
|
data = tinfoil.parse_recipe_file(realfn)
|
|
|
|
scripts = []
|
|
|
|
for key in data.keys():
|
|
|
|
if data.getVarFlag(key, "func") and not data.getVarFlag(key, "python"):
|
|
|
|
script = data.getVar(key, False)
|
|
|
|
if script:
|
|
|
|
scripts.append((key, script))
|
|
|
|
pns[realfn] = scripts
|
2016-02-12 14:55:50 +00:00
|
|
|
|
|
|
|
print("Scanning scripts...\n")
|
verify-bashisms: fix problems with tinfoil2
tinfoil2 is based on a client/server architecture, which broke the
verify-bashisms script:
- The tinfoil instance and its data proxies can't be pickled, so
all interaction with the bitbake server has to run in the main
script process and only processing of the plain scripts can
be done with multiprocessing:
_pickle.PicklingError: Can't pickle <class 'bb.tinfoil.TinfoilCookerAdapter.TinfoilRecipeCacheAdapter'>: attribute lookup TinfoilRecipeCacheAdapter on bb.tinfoil failed
- The multiprocessing pool has to be created before initializing
tinfoil, otherwise the pool workers end up trying to communicate
with the bitbake server during shutdown:
ERROR: UI received SIGTERM
Process ForkPoolWorker-2:
Traceback (most recent call last):
File "/usr/lib/python3.4/multiprocessing/process.py", line 257, in _bootstrap
util._exit_function()
File "/usr/lib/python3.4/multiprocessing/util.py", line 286, in _exit_function
_run_finalizers(0)
...
File "/usr/lib/python3.4/multiprocessing/process.py", line 131, in is_alive
assert self._parent_pid == os.getpid(), 'can only test a child process'
AssertionError: can only test a child process
- func() needs to defined before creating the pool to avoid:
AttributeError: Can't get attribute 'func' on <module '__main__' from '/work/openembedded-core/scripts/verify-bashisms'>
(From OE-Core rev: aa439f11c7f414774843720d68ebe0a6d3375ea6)
Signed-off-by: Patrick Ohly <patrick.ohly@intel.com>
Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
2017-01-31 12:50:29 +00:00
|
|
|
for pn, results in pool.imap(func, pns.items()):
|
2016-02-12 14:55:50 +00:00
|
|
|
if results:
|
|
|
|
print(pn)
|
|
|
|
for message,source in results:
|
|
|
|
print(" %s\n %s" % (message, source))
|
|
|
|
print()
|
2017-01-31 12:50:28 +00:00
|
|
|
tinfoil.shutdown()
|