2005-08-31 10:47:56 +00:00
# ex:ts=4:sw=4:sts=4:et
# -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*-
"""
BitBake Utility Functions
"""
2007-01-08 23:53:01 +00:00
# Copyright (C) 2004 Michael Lauer
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License version 2 as
# published by the Free Software Foundation.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License along
# with this program; if not, write to the Free Software Foundation, Inc.,
# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
2010-04-12 15:14:11 +00:00
import re , fcntl , os , string , stat , shutil , time
2010-04-08 17:22:29 +00:00
import sys
import errno
2010-06-10 17:35:31 +00:00
import logging
import bb
2010-04-08 17:22:29 +00:00
import bb . msg
2012-02-23 13:47:19 +00:00
import multiprocessing
2012-06-22 11:53:16 +00:00
import fcntl
2013-05-17 08:40:01 +00:00
import subprocess
import glob
2015-08-19 13:20:10 +00:00
import fnmatch
2013-05-17 08:40:01 +00:00
import traceback
import errno
2015-07-31 09:16:33 +00:00
import signal
2010-03-25 00:48:49 +00:00
from commands import getstatusoutput
2010-12-10 01:29:31 +00:00
from contextlib import contextmanager
2015-09-08 22:31:13 +00:00
from ctypes import cdll
2005-08-31 10:47:56 +00:00
2010-06-10 17:35:31 +00:00
logger = logging . getLogger ( " BitBake.Util " )
2013-05-23 09:49:57 +00:00
def clean_context ( ) :
return {
" os " : os ,
" bb " : bb ,
" time " : time ,
}
def get_context ( ) :
return _context
def set_context ( ctx ) :
_context = ctx
2010-03-31 03:06:07 +00:00
# Context used in better_exec, eval
2013-05-23 09:49:57 +00:00
_context = clean_context ( )
2010-03-31 03:06:07 +00:00
2015-02-10 18:13:23 +00:00
class VersionStringException ( Exception ) :
""" Exception raised when an invalid version specification is found """
2005-08-31 10:47:56 +00:00
def explode_version ( s ) :
r = [ ]
alpha_regexp = re . compile ( ' ^([a-zA-Z]+)(.*)$ ' )
numeric_regexp = re . compile ( ' ^( \ d+)(.*)$ ' )
while ( s != ' ' ) :
2010-03-24 03:33:19 +00:00
if s [ 0 ] in string . digits :
2005-08-31 10:47:56 +00:00
m = numeric_regexp . match ( s )
2012-04-12 09:28:01 +00:00
r . append ( ( 0 , int ( m . group ( 1 ) ) ) )
2005-08-31 10:47:56 +00:00
s = m . group ( 2 )
continue
2010-03-24 03:33:19 +00:00
if s [ 0 ] in string . letters :
2005-08-31 10:47:56 +00:00
m = alpha_regexp . match ( s )
2012-04-12 09:28:01 +00:00
r . append ( ( 1 , m . group ( 1 ) ) )
2005-08-31 10:47:56 +00:00
s = m . group ( 2 )
continue
2012-04-12 09:28:01 +00:00
if s [ 0 ] == ' ~ ' :
r . append ( ( - 1 , s [ 0 ] ) )
else :
r . append ( ( 2 , s [ 0 ] ) )
2005-08-31 10:47:56 +00:00
s = s [ 1 : ]
return r
2012-03-20 14:37:16 +00:00
def split_version ( s ) :
""" Split a version string into its constituent parts (PE, PV, PR) """
s = s . strip ( " <>= " )
e = 0
if s . count ( ' : ' ) :
e = int ( s . split ( " : " ) [ 0 ] )
s = s . split ( " : " ) [ 1 ]
r = " "
if s . count ( ' - ' ) :
r = s . rsplit ( " - " , 1 ) [ 1 ]
s = s . rsplit ( " - " , 1 ) [ 0 ]
v = s
return ( e , v , r )
2005-08-31 10:47:56 +00:00
def vercmp_part ( a , b ) :
va = explode_version ( a )
vb = explode_version ( b )
while True :
if va == [ ] :
2012-04-12 09:28:01 +00:00
( oa , ca ) = ( 0 , None )
2005-08-31 10:47:56 +00:00
else :
2012-04-12 09:28:01 +00:00
( oa , ca ) = va . pop ( 0 )
2005-08-31 10:47:56 +00:00
if vb == [ ] :
2012-04-12 09:28:01 +00:00
( ob , cb ) = ( 0 , None )
2005-08-31 10:47:56 +00:00
else :
2012-04-12 09:28:01 +00:00
( ob , cb ) = vb . pop ( 0 )
if ( oa , ca ) == ( 0 , None ) and ( ob , cb ) == ( 0 , None ) :
2005-08-31 10:47:56 +00:00
return 0
2012-04-12 09:28:01 +00:00
if oa < ob :
2010-01-20 18:46:02 +00:00
return - 1
2012-04-12 09:28:01 +00:00
elif oa > ob :
2010-01-20 18:46:02 +00:00
return 1
2012-04-12 09:28:01 +00:00
elif ca < cb :
2005-08-31 10:47:56 +00:00
return - 1
2012-04-12 09:28:01 +00:00
elif ca > cb :
return 1
2005-08-31 10:47:56 +00:00
def vercmp ( ta , tb ) :
2007-04-01 15:04:49 +00:00
( ea , va , ra ) = ta
( eb , vb , rb ) = tb
2005-08-31 10:47:56 +00:00
2010-11-26 21:11:06 +00:00
r = int ( ea or 0 ) - int ( eb or 0 )
2007-04-01 15:04:49 +00:00
if ( r == 0 ) :
r = vercmp_part ( va , vb )
2005-08-31 10:47:56 +00:00
if ( r == 0 ) :
r = vercmp_part ( ra , rb )
return r
2006-02-10 10:11:32 +00:00
2012-05-04 14:03:33 +00:00
def vercmp_string ( a , b ) :
ta = split_version ( a )
tb = split_version ( b )
return vercmp ( ta , tb )
2010-04-06 22:29:19 +00:00
2015-02-10 18:13:25 +00:00
def vercmp_string_op ( a , b , op ) :
"""
Compare two versions and check if the specified comparison operator matches the result of the comparison .
This function is fairly liberal about what operators it will accept since there are a variety of styles
depending on the context .
"""
res = vercmp_string ( a , b )
if op in ( ' = ' , ' == ' ) :
return res == 0
elif op == ' <= ' :
return res < = 0
elif op == ' >= ' :
return res > = 0
elif op in ( ' > ' , ' >> ' ) :
return res > 0
elif op in ( ' < ' , ' << ' ) :
return res < 0
elif op == ' != ' :
return res != 0
else :
raise VersionStringException ( ' Unsupported comparison operator " %s " ' % op )
2006-02-10 10:11:32 +00:00
def explode_deps ( s ) :
"""
Take an RDEPENDS style string of format :
" DEPEND1 (optional version) DEPEND2 (optional version) ... "
and return a list of dependencies .
Version information is ignored .
"""
r = [ ]
l = s . split ( )
flag = False
for i in l :
if i [ 0 ] == ' ( ' :
flag = True
2008-03-03 22:01:45 +00:00
#j = []
if not flag :
2006-03-20 17:45:11 +00:00
r . append ( i )
2008-03-03 22:01:45 +00:00
#else:
# j.append(i)
2006-03-20 17:45:11 +00:00
if flag and i . endswith ( ' ) ' ) :
2006-02-10 10:11:32 +00:00
flag = False
# Ignore version
#r[-1] += ' ' + ' '.join(j)
return r
2006-03-20 17:45:11 +00:00
2012-10-01 22:03:43 +00:00
def explode_dep_versions2 ( s ) :
2008-09-03 14:47:31 +00:00
"""
Take an RDEPENDS style string of format :
" DEPEND1 (optional version) DEPEND2 (optional version) ... "
2010-08-03 15:02:43 +00:00
and return a dictionary of dependencies and versions .
2008-09-03 14:47:31 +00:00
"""
r = { }
2010-08-17 08:28:33 +00:00
l = s . replace ( " , " , " " ) . split ( )
2008-09-03 14:47:31 +00:00
lastdep = None
2012-09-30 00:01:45 +00:00
lastcmp = " "
2008-09-03 14:47:31 +00:00
lastver = " "
2012-09-30 00:01:45 +00:00
incmp = False
2008-09-03 14:47:31 +00:00
inversion = False
for i in l :
if i [ 0 ] == ' ( ' :
2012-09-30 00:01:45 +00:00
incmp = True
i = i [ 1 : ] . strip ( )
if not i :
continue
if incmp :
incmp = False
2008-09-03 14:47:31 +00:00
inversion = True
2012-09-30 00:01:45 +00:00
# This list is based on behavior and supported comparisons from deb, opkg and rpm.
#
# Even though =<, <<, ==, !=, =>, and >> may not be supported,
# we list each possibly valid item.
# The build system is responsible for validation of what it supports.
if i . startswith ( ( ' <= ' , ' =< ' , ' << ' , ' == ' , ' != ' , ' >= ' , ' => ' , ' >> ' ) ) :
lastcmp = i [ 0 : 2 ]
i = i [ 2 : ]
elif i . startswith ( ( ' < ' , ' > ' , ' = ' ) ) :
lastcmp = i [ 0 : 1 ]
i = i [ 1 : ]
else :
# This is an unsupported case!
2015-02-10 18:13:23 +00:00
raise VersionStringException ( ' Invalid version specification in " ( %s " - invalid or missing operator ' % i )
2012-09-30 00:01:45 +00:00
lastcmp = ( i or " " )
i = " "
i . strip ( )
if not i :
continue
if inversion :
if i . endswith ( ' ) ' ) :
i = i [ : - 1 ] or " "
inversion = False
if lastver and i :
lastver + = " "
if i :
lastver + = i
2012-10-01 22:03:43 +00:00
if lastdep not in r :
r [ lastdep ] = [ ]
r [ lastdep ] . append ( lastcmp + " " + lastver )
2012-09-30 00:01:45 +00:00
continue
#if not inversion:
lastdep = i
lastver = " "
lastcmp = " "
if not ( i in r and r [ i ] ) :
2012-10-01 22:03:43 +00:00
r [ lastdep ] = [ ]
return r
2006-03-20 17:45:11 +00:00
2012-10-01 22:03:43 +00:00
def explode_dep_versions ( s ) :
r = explode_dep_versions2 ( s )
for d in r :
if not r [ d ] :
r [ d ] = None
continue
if len ( r [ d ] ) > 1 :
bb . warn ( " explode_dep_versions(): Item %s appeared in dependency string ' %s ' multiple times with different values. explode_dep_versions cannot cope with this. " % ( d , s ) )
r [ d ] = r [ d ] [ 0 ]
2008-09-03 14:47:31 +00:00
return r
2006-03-20 17:45:11 +00:00
2011-02-25 16:31:40 +00:00
def join_deps ( deps , commasep = True ) :
2010-06-30 08:47:36 +00:00
"""
Take the result from explode_dep_versions and generate a dependency string
"""
result = [ ]
for dep in deps :
if deps [ dep ] :
2012-10-01 22:03:43 +00:00
if isinstance ( deps [ dep ] , list ) :
for v in deps [ dep ] :
result . append ( dep + " ( " + v + " ) " )
else :
result . append ( dep + " ( " + deps [ dep ] + " ) " )
2010-06-30 08:47:36 +00:00
else :
result . append ( dep )
2011-02-25 16:31:40 +00:00
if commasep :
return " , " . join ( result )
else :
return " " . join ( result )
2010-06-30 08:47:36 +00:00
2006-03-20 17:45:11 +00:00
def _print_trace ( body , line ) :
"""
Print the Environment of a Text Body
"""
bitbake: utils: Improve better_exec traceback handling
The current bitbake tracebacks are hard to read/confusing and sometimes
incomplete. This patch attempts to do better by:
* Moving the note about the exact exception to the end to make things
read in sequence
* Merged the initial stack trace to become part of the code dump
* Added handling for "/xxxx" file paths since we can load these files
and include the data as part of the trace
* Dropped the ERROR: prefix to every line, allowing the error messages to
be spacially accosicated in the UIs
* Moved the "From:" line to the top of each code block and ensured its present
consistently
With the complexity now in this funciton, I've added try/except wrapping around
it to ensure we catch exceptions in the exception handler too.
Example before:
"""
ERROR: Error executing a python function in /media/build1/poky/meta/recipes-core/eglibc/eglibc-initial_2.17.bb:
TypeError: 'filter' object is not subscriptable
ERROR: The stack trace of python calls that resulted in this exception/failure was:
ERROR: File "do_populate_lic", line 13, in <module>
ERROR:
ERROR: File "do_populate_lic", line 6, in do_populate_lic
ERROR:
ERROR: File "license.bbclass", line 99, in find_license_files
ERROR:
ERROR: File "/media/build1/poky/meta/lib/oe/license.py", line 38, in visit_string
ERROR: if pos > 0 and license_pattern.match(elements[pos-1]):
ERROR:
ERROR: The code that was being executed was:
ERROR: 0009: destdir = os.path.join(d.getVar('LICSSTATEDIR', True), d.getVar('PN', True))
ERROR: 0010: copy_license_files(lic_files_paths, destdir)
ERROR: 0011:
ERROR: 0012:
ERROR: *** 0013:do_populate_lic(d)
ERROR: 0014:
ERROR: [From file: 'do_populate_lic', lineno: 13, function: <module>]
ERROR: 0002:def do_populate_lic(d):
ERROR: 0003: """
ERROR: 0004: Populate LICENSE_DIRECTORY with licenses.
ERROR: 0005: """
ERROR: *** 0006: lic_files_paths = find_license_files(d)
ERROR: 0007:
ERROR: 0008: # The base directory we wrangle licenses to
ERROR: 0009: destdir = os.path.join(d.getVar('LICSSTATEDIR', True), d.getVar('PN', True))
ERROR: 0010: copy_license_files(lic_files_paths, destdir)
ERROR: [From file: 'do_populate_lic', lineno: 6, function: do_populate_lic]
ERROR: 0095: lic_files_paths.append((os.path.basename(path), srclicfile))
ERROR: 0096:
ERROR: 0097: v = FindVisitor()
ERROR: 0098: try:
ERROR: *** 0099: v.visit_string(license_types)
ERROR: 0100: except oe.license.InvalidLicense as exc:
ERROR: 0101: bb.fatal('%s: %s' % (d.getVar('PF', True), exc))
ERROR: 0102: except SyntaxError:
ERROR: 0103: bb.warn("%s: Failed to parse it's LICENSE field." % (d.getVar('PF', True)))
ERROR: [From file: 'license.bbclass', lineno: 99, function: find_license_files]
ERROR: Function failed: do_populate_lic
ERROR: Logfile of failure stored in: /media/build1/poky/build/tmp/work/i586-poky-linux/eglibc-initial/2.17-r3/temp/log.do_populate_lic.17442
"""
Example after:
"""
ERROR: Error executing a python function in /media/build1/poky/meta/recipes-core/eglibc/eglibc-initial_2.17.bb:
The stack trace of python calls that resulted in this exception/failure was:
File: 'do_populate_lic', lineno: 13, function: <module>
0009: destdir = os.path.join(d.getVar('LICSSTATEDIR', True), d.getVar('PN', True))
0010: copy_license_files(lic_files_paths, destdir)
0011:
0012:
*** 0013:do_populate_lic(d)
0014:
File: 'do_populate_lic', lineno: 6, function: do_populate_lic
0002:def do_populate_lic(d):
0003: """
0004: Populate LICENSE_DIRECTORY with licenses.
0005: """
*** 0006: lic_files_paths = find_license_files(d)
0007:
0008: # The base directory we wrangle licenses to
0009: destdir = os.path.join(d.getVar('LICSSTATEDIR', True), d.getVar('PN', True))
0010: copy_license_files(lic_files_paths, destdir)
File: 'license.bbclass', lineno: 99, function: find_license_files
0095: lic_files_paths.append((os.path.basename(path), srclicfile))
0096:
0097: v = FindVisitor()
0098: try:
*** 0099: v.visit_string(license_types)
0100: except oe.license.InvalidLicense as exc:
0101: bb.fatal('%s: %s' % (d.getVar('PF', True), exc))
0102: except SyntaxError:
0103: bb.warn("%s: Failed to parse it's LICENSE field." % (d.getVar('PF', True)))
File: '/media/build1/poky/meta/lib/oe/license.py', lineno: 38, function: visit_string
0034: new_elements = []
0035: elements = filter(lambda x: x.strip(), license_operator.split(licensestr))
0036: for pos, element in enumerate(elements):
0037: if license_pattern.match(element):
*** 0038: if pos > 0 and license_pattern.match(elements[pos-1]):
0039: new_elements.append('&')
0040: element = '"' + element + '"'
0041: elif not license_operator.match(element):
0042: raise InvalidLicense(element)
Exception: TypeError: 'filter' object is not subscriptable
ERROR: Function failed: do_populate_lic
ERROR: Logfile of failure stored in: /media/build1/poky/build/tmp/work/i586-poky-linux/eglibc-initial/2.17-r3/temp/log.do_populate_lic.3275
ERROR: Task 9 (/media/build1/poky/meta/recipes-core/eglibc/eglibc-initial_2.17.bb, do_populate_lic) failed with exit code '1
"""
(Bitbake rev: c5de66b870406d9bd1161a9b7e2b04fe6eb065fe)
Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
2013-05-08 17:14:53 +00:00
error = [ ]
2006-03-20 17:45:11 +00:00
# print the environment of the method
2010-03-24 23:56:12 +00:00
min_line = max ( 1 , line - 4 )
2010-12-05 22:51:38 +00:00
max_line = min ( line + 4 , len ( body ) )
bitbake: utils: Improve better_exec traceback handling
The current bitbake tracebacks are hard to read/confusing and sometimes
incomplete. This patch attempts to do better by:
* Moving the note about the exact exception to the end to make things
read in sequence
* Merged the initial stack trace to become part of the code dump
* Added handling for "/xxxx" file paths since we can load these files
and include the data as part of the trace
* Dropped the ERROR: prefix to every line, allowing the error messages to
be spacially accosicated in the UIs
* Moved the "From:" line to the top of each code block and ensured its present
consistently
With the complexity now in this funciton, I've added try/except wrapping around
it to ensure we catch exceptions in the exception handler too.
Example before:
"""
ERROR: Error executing a python function in /media/build1/poky/meta/recipes-core/eglibc/eglibc-initial_2.17.bb:
TypeError: 'filter' object is not subscriptable
ERROR: The stack trace of python calls that resulted in this exception/failure was:
ERROR: File "do_populate_lic", line 13, in <module>
ERROR:
ERROR: File "do_populate_lic", line 6, in do_populate_lic
ERROR:
ERROR: File "license.bbclass", line 99, in find_license_files
ERROR:
ERROR: File "/media/build1/poky/meta/lib/oe/license.py", line 38, in visit_string
ERROR: if pos > 0 and license_pattern.match(elements[pos-1]):
ERROR:
ERROR: The code that was being executed was:
ERROR: 0009: destdir = os.path.join(d.getVar('LICSSTATEDIR', True), d.getVar('PN', True))
ERROR: 0010: copy_license_files(lic_files_paths, destdir)
ERROR: 0011:
ERROR: 0012:
ERROR: *** 0013:do_populate_lic(d)
ERROR: 0014:
ERROR: [From file: 'do_populate_lic', lineno: 13, function: <module>]
ERROR: 0002:def do_populate_lic(d):
ERROR: 0003: """
ERROR: 0004: Populate LICENSE_DIRECTORY with licenses.
ERROR: 0005: """
ERROR: *** 0006: lic_files_paths = find_license_files(d)
ERROR: 0007:
ERROR: 0008: # The base directory we wrangle licenses to
ERROR: 0009: destdir = os.path.join(d.getVar('LICSSTATEDIR', True), d.getVar('PN', True))
ERROR: 0010: copy_license_files(lic_files_paths, destdir)
ERROR: [From file: 'do_populate_lic', lineno: 6, function: do_populate_lic]
ERROR: 0095: lic_files_paths.append((os.path.basename(path), srclicfile))
ERROR: 0096:
ERROR: 0097: v = FindVisitor()
ERROR: 0098: try:
ERROR: *** 0099: v.visit_string(license_types)
ERROR: 0100: except oe.license.InvalidLicense as exc:
ERROR: 0101: bb.fatal('%s: %s' % (d.getVar('PF', True), exc))
ERROR: 0102: except SyntaxError:
ERROR: 0103: bb.warn("%s: Failed to parse it's LICENSE field." % (d.getVar('PF', True)))
ERROR: [From file: 'license.bbclass', lineno: 99, function: find_license_files]
ERROR: Function failed: do_populate_lic
ERROR: Logfile of failure stored in: /media/build1/poky/build/tmp/work/i586-poky-linux/eglibc-initial/2.17-r3/temp/log.do_populate_lic.17442
"""
Example after:
"""
ERROR: Error executing a python function in /media/build1/poky/meta/recipes-core/eglibc/eglibc-initial_2.17.bb:
The stack trace of python calls that resulted in this exception/failure was:
File: 'do_populate_lic', lineno: 13, function: <module>
0009: destdir = os.path.join(d.getVar('LICSSTATEDIR', True), d.getVar('PN', True))
0010: copy_license_files(lic_files_paths, destdir)
0011:
0012:
*** 0013:do_populate_lic(d)
0014:
File: 'do_populate_lic', lineno: 6, function: do_populate_lic
0002:def do_populate_lic(d):
0003: """
0004: Populate LICENSE_DIRECTORY with licenses.
0005: """
*** 0006: lic_files_paths = find_license_files(d)
0007:
0008: # The base directory we wrangle licenses to
0009: destdir = os.path.join(d.getVar('LICSSTATEDIR', True), d.getVar('PN', True))
0010: copy_license_files(lic_files_paths, destdir)
File: 'license.bbclass', lineno: 99, function: find_license_files
0095: lic_files_paths.append((os.path.basename(path), srclicfile))
0096:
0097: v = FindVisitor()
0098: try:
*** 0099: v.visit_string(license_types)
0100: except oe.license.InvalidLicense as exc:
0101: bb.fatal('%s: %s' % (d.getVar('PF', True), exc))
0102: except SyntaxError:
0103: bb.warn("%s: Failed to parse it's LICENSE field." % (d.getVar('PF', True)))
File: '/media/build1/poky/meta/lib/oe/license.py', lineno: 38, function: visit_string
0034: new_elements = []
0035: elements = filter(lambda x: x.strip(), license_operator.split(licensestr))
0036: for pos, element in enumerate(elements):
0037: if license_pattern.match(element):
*** 0038: if pos > 0 and license_pattern.match(elements[pos-1]):
0039: new_elements.append('&')
0040: element = '"' + element + '"'
0041: elif not license_operator.match(element):
0042: raise InvalidLicense(element)
Exception: TypeError: 'filter' object is not subscriptable
ERROR: Function failed: do_populate_lic
ERROR: Logfile of failure stored in: /media/build1/poky/build/tmp/work/i586-poky-linux/eglibc-initial/2.17-r3/temp/log.do_populate_lic.3275
ERROR: Task 9 (/media/build1/poky/meta/recipes-core/eglibc/eglibc-initial_2.17.bb, do_populate_lic) failed with exit code '1
"""
(Bitbake rev: c5de66b870406d9bd1161a9b7e2b04fe6eb065fe)
Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
2013-05-08 17:14:53 +00:00
for i in range ( min_line , max_line + 1 ) :
2010-12-05 22:51:38 +00:00
if line == i :
bitbake: utils: Improve better_exec traceback handling
The current bitbake tracebacks are hard to read/confusing and sometimes
incomplete. This patch attempts to do better by:
* Moving the note about the exact exception to the end to make things
read in sequence
* Merged the initial stack trace to become part of the code dump
* Added handling for "/xxxx" file paths since we can load these files
and include the data as part of the trace
* Dropped the ERROR: prefix to every line, allowing the error messages to
be spacially accosicated in the UIs
* Moved the "From:" line to the top of each code block and ensured its present
consistently
With the complexity now in this funciton, I've added try/except wrapping around
it to ensure we catch exceptions in the exception handler too.
Example before:
"""
ERROR: Error executing a python function in /media/build1/poky/meta/recipes-core/eglibc/eglibc-initial_2.17.bb:
TypeError: 'filter' object is not subscriptable
ERROR: The stack trace of python calls that resulted in this exception/failure was:
ERROR: File "do_populate_lic", line 13, in <module>
ERROR:
ERROR: File "do_populate_lic", line 6, in do_populate_lic
ERROR:
ERROR: File "license.bbclass", line 99, in find_license_files
ERROR:
ERROR: File "/media/build1/poky/meta/lib/oe/license.py", line 38, in visit_string
ERROR: if pos > 0 and license_pattern.match(elements[pos-1]):
ERROR:
ERROR: The code that was being executed was:
ERROR: 0009: destdir = os.path.join(d.getVar('LICSSTATEDIR', True), d.getVar('PN', True))
ERROR: 0010: copy_license_files(lic_files_paths, destdir)
ERROR: 0011:
ERROR: 0012:
ERROR: *** 0013:do_populate_lic(d)
ERROR: 0014:
ERROR: [From file: 'do_populate_lic', lineno: 13, function: <module>]
ERROR: 0002:def do_populate_lic(d):
ERROR: 0003: """
ERROR: 0004: Populate LICENSE_DIRECTORY with licenses.
ERROR: 0005: """
ERROR: *** 0006: lic_files_paths = find_license_files(d)
ERROR: 0007:
ERROR: 0008: # The base directory we wrangle licenses to
ERROR: 0009: destdir = os.path.join(d.getVar('LICSSTATEDIR', True), d.getVar('PN', True))
ERROR: 0010: copy_license_files(lic_files_paths, destdir)
ERROR: [From file: 'do_populate_lic', lineno: 6, function: do_populate_lic]
ERROR: 0095: lic_files_paths.append((os.path.basename(path), srclicfile))
ERROR: 0096:
ERROR: 0097: v = FindVisitor()
ERROR: 0098: try:
ERROR: *** 0099: v.visit_string(license_types)
ERROR: 0100: except oe.license.InvalidLicense as exc:
ERROR: 0101: bb.fatal('%s: %s' % (d.getVar('PF', True), exc))
ERROR: 0102: except SyntaxError:
ERROR: 0103: bb.warn("%s: Failed to parse it's LICENSE field." % (d.getVar('PF', True)))
ERROR: [From file: 'license.bbclass', lineno: 99, function: find_license_files]
ERROR: Function failed: do_populate_lic
ERROR: Logfile of failure stored in: /media/build1/poky/build/tmp/work/i586-poky-linux/eglibc-initial/2.17-r3/temp/log.do_populate_lic.17442
"""
Example after:
"""
ERROR: Error executing a python function in /media/build1/poky/meta/recipes-core/eglibc/eglibc-initial_2.17.bb:
The stack trace of python calls that resulted in this exception/failure was:
File: 'do_populate_lic', lineno: 13, function: <module>
0009: destdir = os.path.join(d.getVar('LICSSTATEDIR', True), d.getVar('PN', True))
0010: copy_license_files(lic_files_paths, destdir)
0011:
0012:
*** 0013:do_populate_lic(d)
0014:
File: 'do_populate_lic', lineno: 6, function: do_populate_lic
0002:def do_populate_lic(d):
0003: """
0004: Populate LICENSE_DIRECTORY with licenses.
0005: """
*** 0006: lic_files_paths = find_license_files(d)
0007:
0008: # The base directory we wrangle licenses to
0009: destdir = os.path.join(d.getVar('LICSSTATEDIR', True), d.getVar('PN', True))
0010: copy_license_files(lic_files_paths, destdir)
File: 'license.bbclass', lineno: 99, function: find_license_files
0095: lic_files_paths.append((os.path.basename(path), srclicfile))
0096:
0097: v = FindVisitor()
0098: try:
*** 0099: v.visit_string(license_types)
0100: except oe.license.InvalidLicense as exc:
0101: bb.fatal('%s: %s' % (d.getVar('PF', True), exc))
0102: except SyntaxError:
0103: bb.warn("%s: Failed to parse it's LICENSE field." % (d.getVar('PF', True)))
File: '/media/build1/poky/meta/lib/oe/license.py', lineno: 38, function: visit_string
0034: new_elements = []
0035: elements = filter(lambda x: x.strip(), license_operator.split(licensestr))
0036: for pos, element in enumerate(elements):
0037: if license_pattern.match(element):
*** 0038: if pos > 0 and license_pattern.match(elements[pos-1]):
0039: new_elements.append('&')
0040: element = '"' + element + '"'
0041: elif not license_operator.match(element):
0042: raise InvalidLicense(element)
Exception: TypeError: 'filter' object is not subscriptable
ERROR: Function failed: do_populate_lic
ERROR: Logfile of failure stored in: /media/build1/poky/build/tmp/work/i586-poky-linux/eglibc-initial/2.17-r3/temp/log.do_populate_lic.3275
ERROR: Task 9 (/media/build1/poky/meta/recipes-core/eglibc/eglibc-initial_2.17.bb, do_populate_lic) failed with exit code '1
"""
(Bitbake rev: c5de66b870406d9bd1161a9b7e2b04fe6eb065fe)
Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
2013-05-08 17:14:53 +00:00
error . append ( ' *** %.4d : %s ' % ( i , body [ i - 1 ] . rstrip ( ) ) )
2010-12-05 22:51:38 +00:00
else :
bitbake: utils: Improve better_exec traceback handling
The current bitbake tracebacks are hard to read/confusing and sometimes
incomplete. This patch attempts to do better by:
* Moving the note about the exact exception to the end to make things
read in sequence
* Merged the initial stack trace to become part of the code dump
* Added handling for "/xxxx" file paths since we can load these files
and include the data as part of the trace
* Dropped the ERROR: prefix to every line, allowing the error messages to
be spacially accosicated in the UIs
* Moved the "From:" line to the top of each code block and ensured its present
consistently
With the complexity now in this funciton, I've added try/except wrapping around
it to ensure we catch exceptions in the exception handler too.
Example before:
"""
ERROR: Error executing a python function in /media/build1/poky/meta/recipes-core/eglibc/eglibc-initial_2.17.bb:
TypeError: 'filter' object is not subscriptable
ERROR: The stack trace of python calls that resulted in this exception/failure was:
ERROR: File "do_populate_lic", line 13, in <module>
ERROR:
ERROR: File "do_populate_lic", line 6, in do_populate_lic
ERROR:
ERROR: File "license.bbclass", line 99, in find_license_files
ERROR:
ERROR: File "/media/build1/poky/meta/lib/oe/license.py", line 38, in visit_string
ERROR: if pos > 0 and license_pattern.match(elements[pos-1]):
ERROR:
ERROR: The code that was being executed was:
ERROR: 0009: destdir = os.path.join(d.getVar('LICSSTATEDIR', True), d.getVar('PN', True))
ERROR: 0010: copy_license_files(lic_files_paths, destdir)
ERROR: 0011:
ERROR: 0012:
ERROR: *** 0013:do_populate_lic(d)
ERROR: 0014:
ERROR: [From file: 'do_populate_lic', lineno: 13, function: <module>]
ERROR: 0002:def do_populate_lic(d):
ERROR: 0003: """
ERROR: 0004: Populate LICENSE_DIRECTORY with licenses.
ERROR: 0005: """
ERROR: *** 0006: lic_files_paths = find_license_files(d)
ERROR: 0007:
ERROR: 0008: # The base directory we wrangle licenses to
ERROR: 0009: destdir = os.path.join(d.getVar('LICSSTATEDIR', True), d.getVar('PN', True))
ERROR: 0010: copy_license_files(lic_files_paths, destdir)
ERROR: [From file: 'do_populate_lic', lineno: 6, function: do_populate_lic]
ERROR: 0095: lic_files_paths.append((os.path.basename(path), srclicfile))
ERROR: 0096:
ERROR: 0097: v = FindVisitor()
ERROR: 0098: try:
ERROR: *** 0099: v.visit_string(license_types)
ERROR: 0100: except oe.license.InvalidLicense as exc:
ERROR: 0101: bb.fatal('%s: %s' % (d.getVar('PF', True), exc))
ERROR: 0102: except SyntaxError:
ERROR: 0103: bb.warn("%s: Failed to parse it's LICENSE field." % (d.getVar('PF', True)))
ERROR: [From file: 'license.bbclass', lineno: 99, function: find_license_files]
ERROR: Function failed: do_populate_lic
ERROR: Logfile of failure stored in: /media/build1/poky/build/tmp/work/i586-poky-linux/eglibc-initial/2.17-r3/temp/log.do_populate_lic.17442
"""
Example after:
"""
ERROR: Error executing a python function in /media/build1/poky/meta/recipes-core/eglibc/eglibc-initial_2.17.bb:
The stack trace of python calls that resulted in this exception/failure was:
File: 'do_populate_lic', lineno: 13, function: <module>
0009: destdir = os.path.join(d.getVar('LICSSTATEDIR', True), d.getVar('PN', True))
0010: copy_license_files(lic_files_paths, destdir)
0011:
0012:
*** 0013:do_populate_lic(d)
0014:
File: 'do_populate_lic', lineno: 6, function: do_populate_lic
0002:def do_populate_lic(d):
0003: """
0004: Populate LICENSE_DIRECTORY with licenses.
0005: """
*** 0006: lic_files_paths = find_license_files(d)
0007:
0008: # The base directory we wrangle licenses to
0009: destdir = os.path.join(d.getVar('LICSSTATEDIR', True), d.getVar('PN', True))
0010: copy_license_files(lic_files_paths, destdir)
File: 'license.bbclass', lineno: 99, function: find_license_files
0095: lic_files_paths.append((os.path.basename(path), srclicfile))
0096:
0097: v = FindVisitor()
0098: try:
*** 0099: v.visit_string(license_types)
0100: except oe.license.InvalidLicense as exc:
0101: bb.fatal('%s: %s' % (d.getVar('PF', True), exc))
0102: except SyntaxError:
0103: bb.warn("%s: Failed to parse it's LICENSE field." % (d.getVar('PF', True)))
File: '/media/build1/poky/meta/lib/oe/license.py', lineno: 38, function: visit_string
0034: new_elements = []
0035: elements = filter(lambda x: x.strip(), license_operator.split(licensestr))
0036: for pos, element in enumerate(elements):
0037: if license_pattern.match(element):
*** 0038: if pos > 0 and license_pattern.match(elements[pos-1]):
0039: new_elements.append('&')
0040: element = '"' + element + '"'
0041: elif not license_operator.match(element):
0042: raise InvalidLicense(element)
Exception: TypeError: 'filter' object is not subscriptable
ERROR: Function failed: do_populate_lic
ERROR: Logfile of failure stored in: /media/build1/poky/build/tmp/work/i586-poky-linux/eglibc-initial/2.17-r3/temp/log.do_populate_lic.3275
ERROR: Task 9 (/media/build1/poky/meta/recipes-core/eglibc/eglibc-initial_2.17.bb, do_populate_lic) failed with exit code '1
"""
(Bitbake rev: c5de66b870406d9bd1161a9b7e2b04fe6eb065fe)
Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
2013-05-08 17:14:53 +00:00
error . append ( ' %.4d : %s ' % ( i , body [ i - 1 ] . rstrip ( ) ) )
return error
2006-03-20 17:45:11 +00:00
2010-03-31 03:06:07 +00:00
def better_compile ( text , file , realfile , mode = " exec " ) :
2006-03-20 17:45:11 +00:00
"""
A better compile method . This method
2014-08-20 12:23:27 +00:00
will print the offending lines .
2006-03-20 17:45:11 +00:00
"""
try :
2010-03-31 03:06:07 +00:00
return compile ( text , file , mode )
2010-04-12 00:03:55 +00:00
except Exception as e :
2013-05-08 17:16:00 +00:00
error = [ ]
2006-03-20 17:45:11 +00:00
# split the text into lines again
body = text . split ( ' \n ' )
2013-05-08 17:16:00 +00:00
error . append ( " Error in compiling python function in %s : \n " % realfile )
2010-08-19 22:26:46 +00:00
if e . lineno :
2013-05-08 17:16:00 +00:00
error . append ( " The code lines resulting in this error were: " )
error . extend ( _print_trace ( body , e . lineno ) )
2010-08-19 22:26:46 +00:00
else :
2013-05-08 17:16:00 +00:00
error . append ( " The function causing this error was: " )
2010-08-19 22:26:46 +00:00
for line in body :
2013-05-08 17:16:00 +00:00
error . append ( line )
error . append ( " %s : %s " % ( e . __class__ . __name__ , str ( e ) ) )
logger . error ( " \n " . join ( error ) )
2011-01-01 23:55:54 +00:00
2012-08-22 19:02:39 +00:00
e = bb . BBHandledException ( e )
raise e
2006-03-20 17:45:11 +00:00
bitbake: utils: Improve better_exec traceback handling
The current bitbake tracebacks are hard to read/confusing and sometimes
incomplete. This patch attempts to do better by:
* Moving the note about the exact exception to the end to make things
read in sequence
* Merged the initial stack trace to become part of the code dump
* Added handling for "/xxxx" file paths since we can load these files
and include the data as part of the trace
* Dropped the ERROR: prefix to every line, allowing the error messages to
be spacially accosicated in the UIs
* Moved the "From:" line to the top of each code block and ensured its present
consistently
With the complexity now in this funciton, I've added try/except wrapping around
it to ensure we catch exceptions in the exception handler too.
Example before:
"""
ERROR: Error executing a python function in /media/build1/poky/meta/recipes-core/eglibc/eglibc-initial_2.17.bb:
TypeError: 'filter' object is not subscriptable
ERROR: The stack trace of python calls that resulted in this exception/failure was:
ERROR: File "do_populate_lic", line 13, in <module>
ERROR:
ERROR: File "do_populate_lic", line 6, in do_populate_lic
ERROR:
ERROR: File "license.bbclass", line 99, in find_license_files
ERROR:
ERROR: File "/media/build1/poky/meta/lib/oe/license.py", line 38, in visit_string
ERROR: if pos > 0 and license_pattern.match(elements[pos-1]):
ERROR:
ERROR: The code that was being executed was:
ERROR: 0009: destdir = os.path.join(d.getVar('LICSSTATEDIR', True), d.getVar('PN', True))
ERROR: 0010: copy_license_files(lic_files_paths, destdir)
ERROR: 0011:
ERROR: 0012:
ERROR: *** 0013:do_populate_lic(d)
ERROR: 0014:
ERROR: [From file: 'do_populate_lic', lineno: 13, function: <module>]
ERROR: 0002:def do_populate_lic(d):
ERROR: 0003: """
ERROR: 0004: Populate LICENSE_DIRECTORY with licenses.
ERROR: 0005: """
ERROR: *** 0006: lic_files_paths = find_license_files(d)
ERROR: 0007:
ERROR: 0008: # The base directory we wrangle licenses to
ERROR: 0009: destdir = os.path.join(d.getVar('LICSSTATEDIR', True), d.getVar('PN', True))
ERROR: 0010: copy_license_files(lic_files_paths, destdir)
ERROR: [From file: 'do_populate_lic', lineno: 6, function: do_populate_lic]
ERROR: 0095: lic_files_paths.append((os.path.basename(path), srclicfile))
ERROR: 0096:
ERROR: 0097: v = FindVisitor()
ERROR: 0098: try:
ERROR: *** 0099: v.visit_string(license_types)
ERROR: 0100: except oe.license.InvalidLicense as exc:
ERROR: 0101: bb.fatal('%s: %s' % (d.getVar('PF', True), exc))
ERROR: 0102: except SyntaxError:
ERROR: 0103: bb.warn("%s: Failed to parse it's LICENSE field." % (d.getVar('PF', True)))
ERROR: [From file: 'license.bbclass', lineno: 99, function: find_license_files]
ERROR: Function failed: do_populate_lic
ERROR: Logfile of failure stored in: /media/build1/poky/build/tmp/work/i586-poky-linux/eglibc-initial/2.17-r3/temp/log.do_populate_lic.17442
"""
Example after:
"""
ERROR: Error executing a python function in /media/build1/poky/meta/recipes-core/eglibc/eglibc-initial_2.17.bb:
The stack trace of python calls that resulted in this exception/failure was:
File: 'do_populate_lic', lineno: 13, function: <module>
0009: destdir = os.path.join(d.getVar('LICSSTATEDIR', True), d.getVar('PN', True))
0010: copy_license_files(lic_files_paths, destdir)
0011:
0012:
*** 0013:do_populate_lic(d)
0014:
File: 'do_populate_lic', lineno: 6, function: do_populate_lic
0002:def do_populate_lic(d):
0003: """
0004: Populate LICENSE_DIRECTORY with licenses.
0005: """
*** 0006: lic_files_paths = find_license_files(d)
0007:
0008: # The base directory we wrangle licenses to
0009: destdir = os.path.join(d.getVar('LICSSTATEDIR', True), d.getVar('PN', True))
0010: copy_license_files(lic_files_paths, destdir)
File: 'license.bbclass', lineno: 99, function: find_license_files
0095: lic_files_paths.append((os.path.basename(path), srclicfile))
0096:
0097: v = FindVisitor()
0098: try:
*** 0099: v.visit_string(license_types)
0100: except oe.license.InvalidLicense as exc:
0101: bb.fatal('%s: %s' % (d.getVar('PF', True), exc))
0102: except SyntaxError:
0103: bb.warn("%s: Failed to parse it's LICENSE field." % (d.getVar('PF', True)))
File: '/media/build1/poky/meta/lib/oe/license.py', lineno: 38, function: visit_string
0034: new_elements = []
0035: elements = filter(lambda x: x.strip(), license_operator.split(licensestr))
0036: for pos, element in enumerate(elements):
0037: if license_pattern.match(element):
*** 0038: if pos > 0 and license_pattern.match(elements[pos-1]):
0039: new_elements.append('&')
0040: element = '"' + element + '"'
0041: elif not license_operator.match(element):
0042: raise InvalidLicense(element)
Exception: TypeError: 'filter' object is not subscriptable
ERROR: Function failed: do_populate_lic
ERROR: Logfile of failure stored in: /media/build1/poky/build/tmp/work/i586-poky-linux/eglibc-initial/2.17-r3/temp/log.do_populate_lic.3275
ERROR: Task 9 (/media/build1/poky/meta/recipes-core/eglibc/eglibc-initial_2.17.bb, do_populate_lic) failed with exit code '1
"""
(Bitbake rev: c5de66b870406d9bd1161a9b7e2b04fe6eb065fe)
Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
2013-05-08 17:14:53 +00:00
def _print_exception ( t , value , tb , realfile , text , context ) :
error = [ ]
2006-03-20 17:45:11 +00:00
try :
2010-12-22 19:06:54 +00:00
exception = traceback . format_exception_only ( t , value )
bitbake: utils: Improve better_exec traceback handling
The current bitbake tracebacks are hard to read/confusing and sometimes
incomplete. This patch attempts to do better by:
* Moving the note about the exact exception to the end to make things
read in sequence
* Merged the initial stack trace to become part of the code dump
* Added handling for "/xxxx" file paths since we can load these files
and include the data as part of the trace
* Dropped the ERROR: prefix to every line, allowing the error messages to
be spacially accosicated in the UIs
* Moved the "From:" line to the top of each code block and ensured its present
consistently
With the complexity now in this funciton, I've added try/except wrapping around
it to ensure we catch exceptions in the exception handler too.
Example before:
"""
ERROR: Error executing a python function in /media/build1/poky/meta/recipes-core/eglibc/eglibc-initial_2.17.bb:
TypeError: 'filter' object is not subscriptable
ERROR: The stack trace of python calls that resulted in this exception/failure was:
ERROR: File "do_populate_lic", line 13, in <module>
ERROR:
ERROR: File "do_populate_lic", line 6, in do_populate_lic
ERROR:
ERROR: File "license.bbclass", line 99, in find_license_files
ERROR:
ERROR: File "/media/build1/poky/meta/lib/oe/license.py", line 38, in visit_string
ERROR: if pos > 0 and license_pattern.match(elements[pos-1]):
ERROR:
ERROR: The code that was being executed was:
ERROR: 0009: destdir = os.path.join(d.getVar('LICSSTATEDIR', True), d.getVar('PN', True))
ERROR: 0010: copy_license_files(lic_files_paths, destdir)
ERROR: 0011:
ERROR: 0012:
ERROR: *** 0013:do_populate_lic(d)
ERROR: 0014:
ERROR: [From file: 'do_populate_lic', lineno: 13, function: <module>]
ERROR: 0002:def do_populate_lic(d):
ERROR: 0003: """
ERROR: 0004: Populate LICENSE_DIRECTORY with licenses.
ERROR: 0005: """
ERROR: *** 0006: lic_files_paths = find_license_files(d)
ERROR: 0007:
ERROR: 0008: # The base directory we wrangle licenses to
ERROR: 0009: destdir = os.path.join(d.getVar('LICSSTATEDIR', True), d.getVar('PN', True))
ERROR: 0010: copy_license_files(lic_files_paths, destdir)
ERROR: [From file: 'do_populate_lic', lineno: 6, function: do_populate_lic]
ERROR: 0095: lic_files_paths.append((os.path.basename(path), srclicfile))
ERROR: 0096:
ERROR: 0097: v = FindVisitor()
ERROR: 0098: try:
ERROR: *** 0099: v.visit_string(license_types)
ERROR: 0100: except oe.license.InvalidLicense as exc:
ERROR: 0101: bb.fatal('%s: %s' % (d.getVar('PF', True), exc))
ERROR: 0102: except SyntaxError:
ERROR: 0103: bb.warn("%s: Failed to parse it's LICENSE field." % (d.getVar('PF', True)))
ERROR: [From file: 'license.bbclass', lineno: 99, function: find_license_files]
ERROR: Function failed: do_populate_lic
ERROR: Logfile of failure stored in: /media/build1/poky/build/tmp/work/i586-poky-linux/eglibc-initial/2.17-r3/temp/log.do_populate_lic.17442
"""
Example after:
"""
ERROR: Error executing a python function in /media/build1/poky/meta/recipes-core/eglibc/eglibc-initial_2.17.bb:
The stack trace of python calls that resulted in this exception/failure was:
File: 'do_populate_lic', lineno: 13, function: <module>
0009: destdir = os.path.join(d.getVar('LICSSTATEDIR', True), d.getVar('PN', True))
0010: copy_license_files(lic_files_paths, destdir)
0011:
0012:
*** 0013:do_populate_lic(d)
0014:
File: 'do_populate_lic', lineno: 6, function: do_populate_lic
0002:def do_populate_lic(d):
0003: """
0004: Populate LICENSE_DIRECTORY with licenses.
0005: """
*** 0006: lic_files_paths = find_license_files(d)
0007:
0008: # The base directory we wrangle licenses to
0009: destdir = os.path.join(d.getVar('LICSSTATEDIR', True), d.getVar('PN', True))
0010: copy_license_files(lic_files_paths, destdir)
File: 'license.bbclass', lineno: 99, function: find_license_files
0095: lic_files_paths.append((os.path.basename(path), srclicfile))
0096:
0097: v = FindVisitor()
0098: try:
*** 0099: v.visit_string(license_types)
0100: except oe.license.InvalidLicense as exc:
0101: bb.fatal('%s: %s' % (d.getVar('PF', True), exc))
0102: except SyntaxError:
0103: bb.warn("%s: Failed to parse it's LICENSE field." % (d.getVar('PF', True)))
File: '/media/build1/poky/meta/lib/oe/license.py', lineno: 38, function: visit_string
0034: new_elements = []
0035: elements = filter(lambda x: x.strip(), license_operator.split(licensestr))
0036: for pos, element in enumerate(elements):
0037: if license_pattern.match(element):
*** 0038: if pos > 0 and license_pattern.match(elements[pos-1]):
0039: new_elements.append('&')
0040: element = '"' + element + '"'
0041: elif not license_operator.match(element):
0042: raise InvalidLicense(element)
Exception: TypeError: 'filter' object is not subscriptable
ERROR: Function failed: do_populate_lic
ERROR: Logfile of failure stored in: /media/build1/poky/build/tmp/work/i586-poky-linux/eglibc-initial/2.17-r3/temp/log.do_populate_lic.3275
ERROR: Task 9 (/media/build1/poky/meta/recipes-core/eglibc/eglibc-initial_2.17.bb, do_populate_lic) failed with exit code '1
"""
(Bitbake rev: c5de66b870406d9bd1161a9b7e2b04fe6eb065fe)
Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
2013-05-08 17:14:53 +00:00
error . append ( ' Error executing a python function in %s : \n ' % realfile )
2006-03-20 17:45:11 +00:00
2010-08-20 11:52:33 +00:00
# Strip 'us' from the stack (better_exec call)
tb = tb . tb_next
2006-03-20 17:45:11 +00:00
2010-12-05 22:52:44 +00:00
textarray = text . split ( ' \n ' )
bitbake: utils: Improve better_exec traceback handling
The current bitbake tracebacks are hard to read/confusing and sometimes
incomplete. This patch attempts to do better by:
* Moving the note about the exact exception to the end to make things
read in sequence
* Merged the initial stack trace to become part of the code dump
* Added handling for "/xxxx" file paths since we can load these files
and include the data as part of the trace
* Dropped the ERROR: prefix to every line, allowing the error messages to
be spacially accosicated in the UIs
* Moved the "From:" line to the top of each code block and ensured its present
consistently
With the complexity now in this funciton, I've added try/except wrapping around
it to ensure we catch exceptions in the exception handler too.
Example before:
"""
ERROR: Error executing a python function in /media/build1/poky/meta/recipes-core/eglibc/eglibc-initial_2.17.bb:
TypeError: 'filter' object is not subscriptable
ERROR: The stack trace of python calls that resulted in this exception/failure was:
ERROR: File "do_populate_lic", line 13, in <module>
ERROR:
ERROR: File "do_populate_lic", line 6, in do_populate_lic
ERROR:
ERROR: File "license.bbclass", line 99, in find_license_files
ERROR:
ERROR: File "/media/build1/poky/meta/lib/oe/license.py", line 38, in visit_string
ERROR: if pos > 0 and license_pattern.match(elements[pos-1]):
ERROR:
ERROR: The code that was being executed was:
ERROR: 0009: destdir = os.path.join(d.getVar('LICSSTATEDIR', True), d.getVar('PN', True))
ERROR: 0010: copy_license_files(lic_files_paths, destdir)
ERROR: 0011:
ERROR: 0012:
ERROR: *** 0013:do_populate_lic(d)
ERROR: 0014:
ERROR: [From file: 'do_populate_lic', lineno: 13, function: <module>]
ERROR: 0002:def do_populate_lic(d):
ERROR: 0003: """
ERROR: 0004: Populate LICENSE_DIRECTORY with licenses.
ERROR: 0005: """
ERROR: *** 0006: lic_files_paths = find_license_files(d)
ERROR: 0007:
ERROR: 0008: # The base directory we wrangle licenses to
ERROR: 0009: destdir = os.path.join(d.getVar('LICSSTATEDIR', True), d.getVar('PN', True))
ERROR: 0010: copy_license_files(lic_files_paths, destdir)
ERROR: [From file: 'do_populate_lic', lineno: 6, function: do_populate_lic]
ERROR: 0095: lic_files_paths.append((os.path.basename(path), srclicfile))
ERROR: 0096:
ERROR: 0097: v = FindVisitor()
ERROR: 0098: try:
ERROR: *** 0099: v.visit_string(license_types)
ERROR: 0100: except oe.license.InvalidLicense as exc:
ERROR: 0101: bb.fatal('%s: %s' % (d.getVar('PF', True), exc))
ERROR: 0102: except SyntaxError:
ERROR: 0103: bb.warn("%s: Failed to parse it's LICENSE field." % (d.getVar('PF', True)))
ERROR: [From file: 'license.bbclass', lineno: 99, function: find_license_files]
ERROR: Function failed: do_populate_lic
ERROR: Logfile of failure stored in: /media/build1/poky/build/tmp/work/i586-poky-linux/eglibc-initial/2.17-r3/temp/log.do_populate_lic.17442
"""
Example after:
"""
ERROR: Error executing a python function in /media/build1/poky/meta/recipes-core/eglibc/eglibc-initial_2.17.bb:
The stack trace of python calls that resulted in this exception/failure was:
File: 'do_populate_lic', lineno: 13, function: <module>
0009: destdir = os.path.join(d.getVar('LICSSTATEDIR', True), d.getVar('PN', True))
0010: copy_license_files(lic_files_paths, destdir)
0011:
0012:
*** 0013:do_populate_lic(d)
0014:
File: 'do_populate_lic', lineno: 6, function: do_populate_lic
0002:def do_populate_lic(d):
0003: """
0004: Populate LICENSE_DIRECTORY with licenses.
0005: """
*** 0006: lic_files_paths = find_license_files(d)
0007:
0008: # The base directory we wrangle licenses to
0009: destdir = os.path.join(d.getVar('LICSSTATEDIR', True), d.getVar('PN', True))
0010: copy_license_files(lic_files_paths, destdir)
File: 'license.bbclass', lineno: 99, function: find_license_files
0095: lic_files_paths.append((os.path.basename(path), srclicfile))
0096:
0097: v = FindVisitor()
0098: try:
*** 0099: v.visit_string(license_types)
0100: except oe.license.InvalidLicense as exc:
0101: bb.fatal('%s: %s' % (d.getVar('PF', True), exc))
0102: except SyntaxError:
0103: bb.warn("%s: Failed to parse it's LICENSE field." % (d.getVar('PF', True)))
File: '/media/build1/poky/meta/lib/oe/license.py', lineno: 38, function: visit_string
0034: new_elements = []
0035: elements = filter(lambda x: x.strip(), license_operator.split(licensestr))
0036: for pos, element in enumerate(elements):
0037: if license_pattern.match(element):
*** 0038: if pos > 0 and license_pattern.match(elements[pos-1]):
0039: new_elements.append('&')
0040: element = '"' + element + '"'
0041: elif not license_operator.match(element):
0042: raise InvalidLicense(element)
Exception: TypeError: 'filter' object is not subscriptable
ERROR: Function failed: do_populate_lic
ERROR: Logfile of failure stored in: /media/build1/poky/build/tmp/work/i586-poky-linux/eglibc-initial/2.17-r3/temp/log.do_populate_lic.3275
ERROR: Task 9 (/media/build1/poky/meta/recipes-core/eglibc/eglibc-initial_2.17.bb, do_populate_lic) failed with exit code '1
"""
(Bitbake rev: c5de66b870406d9bd1161a9b7e2b04fe6eb065fe)
Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
2013-05-08 17:14:53 +00:00
linefailed = tb . tb_lineno
2006-03-20 17:45:11 +00:00
bitbake: utils: Improve better_exec traceback handling
The current bitbake tracebacks are hard to read/confusing and sometimes
incomplete. This patch attempts to do better by:
* Moving the note about the exact exception to the end to make things
read in sequence
* Merged the initial stack trace to become part of the code dump
* Added handling for "/xxxx" file paths since we can load these files
and include the data as part of the trace
* Dropped the ERROR: prefix to every line, allowing the error messages to
be spacially accosicated in the UIs
* Moved the "From:" line to the top of each code block and ensured its present
consistently
With the complexity now in this funciton, I've added try/except wrapping around
it to ensure we catch exceptions in the exception handler too.
Example before:
"""
ERROR: Error executing a python function in /media/build1/poky/meta/recipes-core/eglibc/eglibc-initial_2.17.bb:
TypeError: 'filter' object is not subscriptable
ERROR: The stack trace of python calls that resulted in this exception/failure was:
ERROR: File "do_populate_lic", line 13, in <module>
ERROR:
ERROR: File "do_populate_lic", line 6, in do_populate_lic
ERROR:
ERROR: File "license.bbclass", line 99, in find_license_files
ERROR:
ERROR: File "/media/build1/poky/meta/lib/oe/license.py", line 38, in visit_string
ERROR: if pos > 0 and license_pattern.match(elements[pos-1]):
ERROR:
ERROR: The code that was being executed was:
ERROR: 0009: destdir = os.path.join(d.getVar('LICSSTATEDIR', True), d.getVar('PN', True))
ERROR: 0010: copy_license_files(lic_files_paths, destdir)
ERROR: 0011:
ERROR: 0012:
ERROR: *** 0013:do_populate_lic(d)
ERROR: 0014:
ERROR: [From file: 'do_populate_lic', lineno: 13, function: <module>]
ERROR: 0002:def do_populate_lic(d):
ERROR: 0003: """
ERROR: 0004: Populate LICENSE_DIRECTORY with licenses.
ERROR: 0005: """
ERROR: *** 0006: lic_files_paths = find_license_files(d)
ERROR: 0007:
ERROR: 0008: # The base directory we wrangle licenses to
ERROR: 0009: destdir = os.path.join(d.getVar('LICSSTATEDIR', True), d.getVar('PN', True))
ERROR: 0010: copy_license_files(lic_files_paths, destdir)
ERROR: [From file: 'do_populate_lic', lineno: 6, function: do_populate_lic]
ERROR: 0095: lic_files_paths.append((os.path.basename(path), srclicfile))
ERROR: 0096:
ERROR: 0097: v = FindVisitor()
ERROR: 0098: try:
ERROR: *** 0099: v.visit_string(license_types)
ERROR: 0100: except oe.license.InvalidLicense as exc:
ERROR: 0101: bb.fatal('%s: %s' % (d.getVar('PF', True), exc))
ERROR: 0102: except SyntaxError:
ERROR: 0103: bb.warn("%s: Failed to parse it's LICENSE field." % (d.getVar('PF', True)))
ERROR: [From file: 'license.bbclass', lineno: 99, function: find_license_files]
ERROR: Function failed: do_populate_lic
ERROR: Logfile of failure stored in: /media/build1/poky/build/tmp/work/i586-poky-linux/eglibc-initial/2.17-r3/temp/log.do_populate_lic.17442
"""
Example after:
"""
ERROR: Error executing a python function in /media/build1/poky/meta/recipes-core/eglibc/eglibc-initial_2.17.bb:
The stack trace of python calls that resulted in this exception/failure was:
File: 'do_populate_lic', lineno: 13, function: <module>
0009: destdir = os.path.join(d.getVar('LICSSTATEDIR', True), d.getVar('PN', True))
0010: copy_license_files(lic_files_paths, destdir)
0011:
0012:
*** 0013:do_populate_lic(d)
0014:
File: 'do_populate_lic', lineno: 6, function: do_populate_lic
0002:def do_populate_lic(d):
0003: """
0004: Populate LICENSE_DIRECTORY with licenses.
0005: """
*** 0006: lic_files_paths = find_license_files(d)
0007:
0008: # The base directory we wrangle licenses to
0009: destdir = os.path.join(d.getVar('LICSSTATEDIR', True), d.getVar('PN', True))
0010: copy_license_files(lic_files_paths, destdir)
File: 'license.bbclass', lineno: 99, function: find_license_files
0095: lic_files_paths.append((os.path.basename(path), srclicfile))
0096:
0097: v = FindVisitor()
0098: try:
*** 0099: v.visit_string(license_types)
0100: except oe.license.InvalidLicense as exc:
0101: bb.fatal('%s: %s' % (d.getVar('PF', True), exc))
0102: except SyntaxError:
0103: bb.warn("%s: Failed to parse it's LICENSE field." % (d.getVar('PF', True)))
File: '/media/build1/poky/meta/lib/oe/license.py', lineno: 38, function: visit_string
0034: new_elements = []
0035: elements = filter(lambda x: x.strip(), license_operator.split(licensestr))
0036: for pos, element in enumerate(elements):
0037: if license_pattern.match(element):
*** 0038: if pos > 0 and license_pattern.match(elements[pos-1]):
0039: new_elements.append('&')
0040: element = '"' + element + '"'
0041: elif not license_operator.match(element):
0042: raise InvalidLicense(element)
Exception: TypeError: 'filter' object is not subscriptable
ERROR: Function failed: do_populate_lic
ERROR: Logfile of failure stored in: /media/build1/poky/build/tmp/work/i586-poky-linux/eglibc-initial/2.17-r3/temp/log.do_populate_lic.3275
ERROR: Task 9 (/media/build1/poky/meta/recipes-core/eglibc/eglibc-initial_2.17.bb, do_populate_lic) failed with exit code '1
"""
(Bitbake rev: c5de66b870406d9bd1161a9b7e2b04fe6eb065fe)
Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
2013-05-08 17:14:53 +00:00
tbextract = traceback . extract_tb ( tb )
tbformat = traceback . format_list ( tbextract )
error . append ( " The stack trace of python calls that resulted in this exception/failure was: " )
error . append ( " File: ' %s ' , lineno: %s , function: %s " % ( tbextract [ 0 ] [ 0 ] , tbextract [ 0 ] [ 1 ] , tbextract [ 0 ] [ 2 ] ) )
error . extend ( _print_trace ( textarray , linefailed ) )
2010-12-05 22:52:44 +00:00
2011-01-01 23:55:54 +00:00
# See if this is a function we constructed and has calls back into other functions in
2010-12-05 22:52:44 +00:00
# "text". If so, try and improve the context of the error by diving down the trace
level = 0
nexttb = tb . tb_next
2012-08-22 19:01:12 +00:00
while nexttb is not None and ( level + 1 ) < len ( tbextract ) :
bitbake: utils: Improve better_exec traceback handling
The current bitbake tracebacks are hard to read/confusing and sometimes
incomplete. This patch attempts to do better by:
* Moving the note about the exact exception to the end to make things
read in sequence
* Merged the initial stack trace to become part of the code dump
* Added handling for "/xxxx" file paths since we can load these files
and include the data as part of the trace
* Dropped the ERROR: prefix to every line, allowing the error messages to
be spacially accosicated in the UIs
* Moved the "From:" line to the top of each code block and ensured its present
consistently
With the complexity now in this funciton, I've added try/except wrapping around
it to ensure we catch exceptions in the exception handler too.
Example before:
"""
ERROR: Error executing a python function in /media/build1/poky/meta/recipes-core/eglibc/eglibc-initial_2.17.bb:
TypeError: 'filter' object is not subscriptable
ERROR: The stack trace of python calls that resulted in this exception/failure was:
ERROR: File "do_populate_lic", line 13, in <module>
ERROR:
ERROR: File "do_populate_lic", line 6, in do_populate_lic
ERROR:
ERROR: File "license.bbclass", line 99, in find_license_files
ERROR:
ERROR: File "/media/build1/poky/meta/lib/oe/license.py", line 38, in visit_string
ERROR: if pos > 0 and license_pattern.match(elements[pos-1]):
ERROR:
ERROR: The code that was being executed was:
ERROR: 0009: destdir = os.path.join(d.getVar('LICSSTATEDIR', True), d.getVar('PN', True))
ERROR: 0010: copy_license_files(lic_files_paths, destdir)
ERROR: 0011:
ERROR: 0012:
ERROR: *** 0013:do_populate_lic(d)
ERROR: 0014:
ERROR: [From file: 'do_populate_lic', lineno: 13, function: <module>]
ERROR: 0002:def do_populate_lic(d):
ERROR: 0003: """
ERROR: 0004: Populate LICENSE_DIRECTORY with licenses.
ERROR: 0005: """
ERROR: *** 0006: lic_files_paths = find_license_files(d)
ERROR: 0007:
ERROR: 0008: # The base directory we wrangle licenses to
ERROR: 0009: destdir = os.path.join(d.getVar('LICSSTATEDIR', True), d.getVar('PN', True))
ERROR: 0010: copy_license_files(lic_files_paths, destdir)
ERROR: [From file: 'do_populate_lic', lineno: 6, function: do_populate_lic]
ERROR: 0095: lic_files_paths.append((os.path.basename(path), srclicfile))
ERROR: 0096:
ERROR: 0097: v = FindVisitor()
ERROR: 0098: try:
ERROR: *** 0099: v.visit_string(license_types)
ERROR: 0100: except oe.license.InvalidLicense as exc:
ERROR: 0101: bb.fatal('%s: %s' % (d.getVar('PF', True), exc))
ERROR: 0102: except SyntaxError:
ERROR: 0103: bb.warn("%s: Failed to parse it's LICENSE field." % (d.getVar('PF', True)))
ERROR: [From file: 'license.bbclass', lineno: 99, function: find_license_files]
ERROR: Function failed: do_populate_lic
ERROR: Logfile of failure stored in: /media/build1/poky/build/tmp/work/i586-poky-linux/eglibc-initial/2.17-r3/temp/log.do_populate_lic.17442
"""
Example after:
"""
ERROR: Error executing a python function in /media/build1/poky/meta/recipes-core/eglibc/eglibc-initial_2.17.bb:
The stack trace of python calls that resulted in this exception/failure was:
File: 'do_populate_lic', lineno: 13, function: <module>
0009: destdir = os.path.join(d.getVar('LICSSTATEDIR', True), d.getVar('PN', True))
0010: copy_license_files(lic_files_paths, destdir)
0011:
0012:
*** 0013:do_populate_lic(d)
0014:
File: 'do_populate_lic', lineno: 6, function: do_populate_lic
0002:def do_populate_lic(d):
0003: """
0004: Populate LICENSE_DIRECTORY with licenses.
0005: """
*** 0006: lic_files_paths = find_license_files(d)
0007:
0008: # The base directory we wrangle licenses to
0009: destdir = os.path.join(d.getVar('LICSSTATEDIR', True), d.getVar('PN', True))
0010: copy_license_files(lic_files_paths, destdir)
File: 'license.bbclass', lineno: 99, function: find_license_files
0095: lic_files_paths.append((os.path.basename(path), srclicfile))
0096:
0097: v = FindVisitor()
0098: try:
*** 0099: v.visit_string(license_types)
0100: except oe.license.InvalidLicense as exc:
0101: bb.fatal('%s: %s' % (d.getVar('PF', True), exc))
0102: except SyntaxError:
0103: bb.warn("%s: Failed to parse it's LICENSE field." % (d.getVar('PF', True)))
File: '/media/build1/poky/meta/lib/oe/license.py', lineno: 38, function: visit_string
0034: new_elements = []
0035: elements = filter(lambda x: x.strip(), license_operator.split(licensestr))
0036: for pos, element in enumerate(elements):
0037: if license_pattern.match(element):
*** 0038: if pos > 0 and license_pattern.match(elements[pos-1]):
0039: new_elements.append('&')
0040: element = '"' + element + '"'
0041: elif not license_operator.match(element):
0042: raise InvalidLicense(element)
Exception: TypeError: 'filter' object is not subscriptable
ERROR: Function failed: do_populate_lic
ERROR: Logfile of failure stored in: /media/build1/poky/build/tmp/work/i586-poky-linux/eglibc-initial/2.17-r3/temp/log.do_populate_lic.3275
ERROR: Task 9 (/media/build1/poky/meta/recipes-core/eglibc/eglibc-initial_2.17.bb, do_populate_lic) failed with exit code '1
"""
(Bitbake rev: c5de66b870406d9bd1161a9b7e2b04fe6eb065fe)
Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
2013-05-08 17:14:53 +00:00
error . append ( " File: ' %s ' , lineno: %s , function: %s " % ( tbextract [ level + 1 ] [ 0 ] , tbextract [ level + 1 ] [ 1 ] , tbextract [ level + 1 ] [ 2 ] ) )
2010-12-05 22:52:44 +00:00
if tbextract [ level ] [ 0 ] == tbextract [ level + 1 ] [ 0 ] and tbextract [ level + 1 ] [ 2 ] == tbextract [ level ] [ 0 ] :
bitbake: utils: Improve better_exec traceback handling
The current bitbake tracebacks are hard to read/confusing and sometimes
incomplete. This patch attempts to do better by:
* Moving the note about the exact exception to the end to make things
read in sequence
* Merged the initial stack trace to become part of the code dump
* Added handling for "/xxxx" file paths since we can load these files
and include the data as part of the trace
* Dropped the ERROR: prefix to every line, allowing the error messages to
be spacially accosicated in the UIs
* Moved the "From:" line to the top of each code block and ensured its present
consistently
With the complexity now in this funciton, I've added try/except wrapping around
it to ensure we catch exceptions in the exception handler too.
Example before:
"""
ERROR: Error executing a python function in /media/build1/poky/meta/recipes-core/eglibc/eglibc-initial_2.17.bb:
TypeError: 'filter' object is not subscriptable
ERROR: The stack trace of python calls that resulted in this exception/failure was:
ERROR: File "do_populate_lic", line 13, in <module>
ERROR:
ERROR: File "do_populate_lic", line 6, in do_populate_lic
ERROR:
ERROR: File "license.bbclass", line 99, in find_license_files
ERROR:
ERROR: File "/media/build1/poky/meta/lib/oe/license.py", line 38, in visit_string
ERROR: if pos > 0 and license_pattern.match(elements[pos-1]):
ERROR:
ERROR: The code that was being executed was:
ERROR: 0009: destdir = os.path.join(d.getVar('LICSSTATEDIR', True), d.getVar('PN', True))
ERROR: 0010: copy_license_files(lic_files_paths, destdir)
ERROR: 0011:
ERROR: 0012:
ERROR: *** 0013:do_populate_lic(d)
ERROR: 0014:
ERROR: [From file: 'do_populate_lic', lineno: 13, function: <module>]
ERROR: 0002:def do_populate_lic(d):
ERROR: 0003: """
ERROR: 0004: Populate LICENSE_DIRECTORY with licenses.
ERROR: 0005: """
ERROR: *** 0006: lic_files_paths = find_license_files(d)
ERROR: 0007:
ERROR: 0008: # The base directory we wrangle licenses to
ERROR: 0009: destdir = os.path.join(d.getVar('LICSSTATEDIR', True), d.getVar('PN', True))
ERROR: 0010: copy_license_files(lic_files_paths, destdir)
ERROR: [From file: 'do_populate_lic', lineno: 6, function: do_populate_lic]
ERROR: 0095: lic_files_paths.append((os.path.basename(path), srclicfile))
ERROR: 0096:
ERROR: 0097: v = FindVisitor()
ERROR: 0098: try:
ERROR: *** 0099: v.visit_string(license_types)
ERROR: 0100: except oe.license.InvalidLicense as exc:
ERROR: 0101: bb.fatal('%s: %s' % (d.getVar('PF', True), exc))
ERROR: 0102: except SyntaxError:
ERROR: 0103: bb.warn("%s: Failed to parse it's LICENSE field." % (d.getVar('PF', True)))
ERROR: [From file: 'license.bbclass', lineno: 99, function: find_license_files]
ERROR: Function failed: do_populate_lic
ERROR: Logfile of failure stored in: /media/build1/poky/build/tmp/work/i586-poky-linux/eglibc-initial/2.17-r3/temp/log.do_populate_lic.17442
"""
Example after:
"""
ERROR: Error executing a python function in /media/build1/poky/meta/recipes-core/eglibc/eglibc-initial_2.17.bb:
The stack trace of python calls that resulted in this exception/failure was:
File: 'do_populate_lic', lineno: 13, function: <module>
0009: destdir = os.path.join(d.getVar('LICSSTATEDIR', True), d.getVar('PN', True))
0010: copy_license_files(lic_files_paths, destdir)
0011:
0012:
*** 0013:do_populate_lic(d)
0014:
File: 'do_populate_lic', lineno: 6, function: do_populate_lic
0002:def do_populate_lic(d):
0003: """
0004: Populate LICENSE_DIRECTORY with licenses.
0005: """
*** 0006: lic_files_paths = find_license_files(d)
0007:
0008: # The base directory we wrangle licenses to
0009: destdir = os.path.join(d.getVar('LICSSTATEDIR', True), d.getVar('PN', True))
0010: copy_license_files(lic_files_paths, destdir)
File: 'license.bbclass', lineno: 99, function: find_license_files
0095: lic_files_paths.append((os.path.basename(path), srclicfile))
0096:
0097: v = FindVisitor()
0098: try:
*** 0099: v.visit_string(license_types)
0100: except oe.license.InvalidLicense as exc:
0101: bb.fatal('%s: %s' % (d.getVar('PF', True), exc))
0102: except SyntaxError:
0103: bb.warn("%s: Failed to parse it's LICENSE field." % (d.getVar('PF', True)))
File: '/media/build1/poky/meta/lib/oe/license.py', lineno: 38, function: visit_string
0034: new_elements = []
0035: elements = filter(lambda x: x.strip(), license_operator.split(licensestr))
0036: for pos, element in enumerate(elements):
0037: if license_pattern.match(element):
*** 0038: if pos > 0 and license_pattern.match(elements[pos-1]):
0039: new_elements.append('&')
0040: element = '"' + element + '"'
0041: elif not license_operator.match(element):
0042: raise InvalidLicense(element)
Exception: TypeError: 'filter' object is not subscriptable
ERROR: Function failed: do_populate_lic
ERROR: Logfile of failure stored in: /media/build1/poky/build/tmp/work/i586-poky-linux/eglibc-initial/2.17-r3/temp/log.do_populate_lic.3275
ERROR: Task 9 (/media/build1/poky/meta/recipes-core/eglibc/eglibc-initial_2.17.bb, do_populate_lic) failed with exit code '1
"""
(Bitbake rev: c5de66b870406d9bd1161a9b7e2b04fe6eb065fe)
Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
2013-05-08 17:14:53 +00:00
# The code was possibly in the string we compiled ourselves
error . extend ( _print_trace ( textarray , tbextract [ level + 1 ] [ 1 ] ) )
elif tbextract [ level + 1 ] [ 0 ] . startswith ( " / " ) :
# The code looks like it might be in a file, try and load it
try :
with open ( tbextract [ level + 1 ] [ 0 ] , " r " ) as f :
text = f . readlines ( )
error . extend ( _print_trace ( text , tbextract [ level + 1 ] [ 1 ] ) )
except :
error . append ( tbformat [ level + 1 ] )
2012-08-22 19:01:12 +00:00
elif " d " in context and tbextract [ level + 1 ] [ 2 ] :
bitbake: utils: Improve better_exec traceback handling
The current bitbake tracebacks are hard to read/confusing and sometimes
incomplete. This patch attempts to do better by:
* Moving the note about the exact exception to the end to make things
read in sequence
* Merged the initial stack trace to become part of the code dump
* Added handling for "/xxxx" file paths since we can load these files
and include the data as part of the trace
* Dropped the ERROR: prefix to every line, allowing the error messages to
be spacially accosicated in the UIs
* Moved the "From:" line to the top of each code block and ensured its present
consistently
With the complexity now in this funciton, I've added try/except wrapping around
it to ensure we catch exceptions in the exception handler too.
Example before:
"""
ERROR: Error executing a python function in /media/build1/poky/meta/recipes-core/eglibc/eglibc-initial_2.17.bb:
TypeError: 'filter' object is not subscriptable
ERROR: The stack trace of python calls that resulted in this exception/failure was:
ERROR: File "do_populate_lic", line 13, in <module>
ERROR:
ERROR: File "do_populate_lic", line 6, in do_populate_lic
ERROR:
ERROR: File "license.bbclass", line 99, in find_license_files
ERROR:
ERROR: File "/media/build1/poky/meta/lib/oe/license.py", line 38, in visit_string
ERROR: if pos > 0 and license_pattern.match(elements[pos-1]):
ERROR:
ERROR: The code that was being executed was:
ERROR: 0009: destdir = os.path.join(d.getVar('LICSSTATEDIR', True), d.getVar('PN', True))
ERROR: 0010: copy_license_files(lic_files_paths, destdir)
ERROR: 0011:
ERROR: 0012:
ERROR: *** 0013:do_populate_lic(d)
ERROR: 0014:
ERROR: [From file: 'do_populate_lic', lineno: 13, function: <module>]
ERROR: 0002:def do_populate_lic(d):
ERROR: 0003: """
ERROR: 0004: Populate LICENSE_DIRECTORY with licenses.
ERROR: 0005: """
ERROR: *** 0006: lic_files_paths = find_license_files(d)
ERROR: 0007:
ERROR: 0008: # The base directory we wrangle licenses to
ERROR: 0009: destdir = os.path.join(d.getVar('LICSSTATEDIR', True), d.getVar('PN', True))
ERROR: 0010: copy_license_files(lic_files_paths, destdir)
ERROR: [From file: 'do_populate_lic', lineno: 6, function: do_populate_lic]
ERROR: 0095: lic_files_paths.append((os.path.basename(path), srclicfile))
ERROR: 0096:
ERROR: 0097: v = FindVisitor()
ERROR: 0098: try:
ERROR: *** 0099: v.visit_string(license_types)
ERROR: 0100: except oe.license.InvalidLicense as exc:
ERROR: 0101: bb.fatal('%s: %s' % (d.getVar('PF', True), exc))
ERROR: 0102: except SyntaxError:
ERROR: 0103: bb.warn("%s: Failed to parse it's LICENSE field." % (d.getVar('PF', True)))
ERROR: [From file: 'license.bbclass', lineno: 99, function: find_license_files]
ERROR: Function failed: do_populate_lic
ERROR: Logfile of failure stored in: /media/build1/poky/build/tmp/work/i586-poky-linux/eglibc-initial/2.17-r3/temp/log.do_populate_lic.17442
"""
Example after:
"""
ERROR: Error executing a python function in /media/build1/poky/meta/recipes-core/eglibc/eglibc-initial_2.17.bb:
The stack trace of python calls that resulted in this exception/failure was:
File: 'do_populate_lic', lineno: 13, function: <module>
0009: destdir = os.path.join(d.getVar('LICSSTATEDIR', True), d.getVar('PN', True))
0010: copy_license_files(lic_files_paths, destdir)
0011:
0012:
*** 0013:do_populate_lic(d)
0014:
File: 'do_populate_lic', lineno: 6, function: do_populate_lic
0002:def do_populate_lic(d):
0003: """
0004: Populate LICENSE_DIRECTORY with licenses.
0005: """
*** 0006: lic_files_paths = find_license_files(d)
0007:
0008: # The base directory we wrangle licenses to
0009: destdir = os.path.join(d.getVar('LICSSTATEDIR', True), d.getVar('PN', True))
0010: copy_license_files(lic_files_paths, destdir)
File: 'license.bbclass', lineno: 99, function: find_license_files
0095: lic_files_paths.append((os.path.basename(path), srclicfile))
0096:
0097: v = FindVisitor()
0098: try:
*** 0099: v.visit_string(license_types)
0100: except oe.license.InvalidLicense as exc:
0101: bb.fatal('%s: %s' % (d.getVar('PF', True), exc))
0102: except SyntaxError:
0103: bb.warn("%s: Failed to parse it's LICENSE field." % (d.getVar('PF', True)))
File: '/media/build1/poky/meta/lib/oe/license.py', lineno: 38, function: visit_string
0034: new_elements = []
0035: elements = filter(lambda x: x.strip(), license_operator.split(licensestr))
0036: for pos, element in enumerate(elements):
0037: if license_pattern.match(element):
*** 0038: if pos > 0 and license_pattern.match(elements[pos-1]):
0039: new_elements.append('&')
0040: element = '"' + element + '"'
0041: elif not license_operator.match(element):
0042: raise InvalidLicense(element)
Exception: TypeError: 'filter' object is not subscriptable
ERROR: Function failed: do_populate_lic
ERROR: Logfile of failure stored in: /media/build1/poky/build/tmp/work/i586-poky-linux/eglibc-initial/2.17-r3/temp/log.do_populate_lic.3275
ERROR: Task 9 (/media/build1/poky/meta/recipes-core/eglibc/eglibc-initial_2.17.bb, do_populate_lic) failed with exit code '1
"""
(Bitbake rev: c5de66b870406d9bd1161a9b7e2b04fe6eb065fe)
Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
2013-05-08 17:14:53 +00:00
# Try and find the code in the datastore based on the functionname
2012-08-22 19:01:12 +00:00
d = context [ " d " ]
functionname = tbextract [ level + 1 ] [ 2 ]
text = d . getVar ( functionname , True )
if text :
bitbake: utils: Improve better_exec traceback handling
The current bitbake tracebacks are hard to read/confusing and sometimes
incomplete. This patch attempts to do better by:
* Moving the note about the exact exception to the end to make things
read in sequence
* Merged the initial stack trace to become part of the code dump
* Added handling for "/xxxx" file paths since we can load these files
and include the data as part of the trace
* Dropped the ERROR: prefix to every line, allowing the error messages to
be spacially accosicated in the UIs
* Moved the "From:" line to the top of each code block and ensured its present
consistently
With the complexity now in this funciton, I've added try/except wrapping around
it to ensure we catch exceptions in the exception handler too.
Example before:
"""
ERROR: Error executing a python function in /media/build1/poky/meta/recipes-core/eglibc/eglibc-initial_2.17.bb:
TypeError: 'filter' object is not subscriptable
ERROR: The stack trace of python calls that resulted in this exception/failure was:
ERROR: File "do_populate_lic", line 13, in <module>
ERROR:
ERROR: File "do_populate_lic", line 6, in do_populate_lic
ERROR:
ERROR: File "license.bbclass", line 99, in find_license_files
ERROR:
ERROR: File "/media/build1/poky/meta/lib/oe/license.py", line 38, in visit_string
ERROR: if pos > 0 and license_pattern.match(elements[pos-1]):
ERROR:
ERROR: The code that was being executed was:
ERROR: 0009: destdir = os.path.join(d.getVar('LICSSTATEDIR', True), d.getVar('PN', True))
ERROR: 0010: copy_license_files(lic_files_paths, destdir)
ERROR: 0011:
ERROR: 0012:
ERROR: *** 0013:do_populate_lic(d)
ERROR: 0014:
ERROR: [From file: 'do_populate_lic', lineno: 13, function: <module>]
ERROR: 0002:def do_populate_lic(d):
ERROR: 0003: """
ERROR: 0004: Populate LICENSE_DIRECTORY with licenses.
ERROR: 0005: """
ERROR: *** 0006: lic_files_paths = find_license_files(d)
ERROR: 0007:
ERROR: 0008: # The base directory we wrangle licenses to
ERROR: 0009: destdir = os.path.join(d.getVar('LICSSTATEDIR', True), d.getVar('PN', True))
ERROR: 0010: copy_license_files(lic_files_paths, destdir)
ERROR: [From file: 'do_populate_lic', lineno: 6, function: do_populate_lic]
ERROR: 0095: lic_files_paths.append((os.path.basename(path), srclicfile))
ERROR: 0096:
ERROR: 0097: v = FindVisitor()
ERROR: 0098: try:
ERROR: *** 0099: v.visit_string(license_types)
ERROR: 0100: except oe.license.InvalidLicense as exc:
ERROR: 0101: bb.fatal('%s: %s' % (d.getVar('PF', True), exc))
ERROR: 0102: except SyntaxError:
ERROR: 0103: bb.warn("%s: Failed to parse it's LICENSE field." % (d.getVar('PF', True)))
ERROR: [From file: 'license.bbclass', lineno: 99, function: find_license_files]
ERROR: Function failed: do_populate_lic
ERROR: Logfile of failure stored in: /media/build1/poky/build/tmp/work/i586-poky-linux/eglibc-initial/2.17-r3/temp/log.do_populate_lic.17442
"""
Example after:
"""
ERROR: Error executing a python function in /media/build1/poky/meta/recipes-core/eglibc/eglibc-initial_2.17.bb:
The stack trace of python calls that resulted in this exception/failure was:
File: 'do_populate_lic', lineno: 13, function: <module>
0009: destdir = os.path.join(d.getVar('LICSSTATEDIR', True), d.getVar('PN', True))
0010: copy_license_files(lic_files_paths, destdir)
0011:
0012:
*** 0013:do_populate_lic(d)
0014:
File: 'do_populate_lic', lineno: 6, function: do_populate_lic
0002:def do_populate_lic(d):
0003: """
0004: Populate LICENSE_DIRECTORY with licenses.
0005: """
*** 0006: lic_files_paths = find_license_files(d)
0007:
0008: # The base directory we wrangle licenses to
0009: destdir = os.path.join(d.getVar('LICSSTATEDIR', True), d.getVar('PN', True))
0010: copy_license_files(lic_files_paths, destdir)
File: 'license.bbclass', lineno: 99, function: find_license_files
0095: lic_files_paths.append((os.path.basename(path), srclicfile))
0096:
0097: v = FindVisitor()
0098: try:
*** 0099: v.visit_string(license_types)
0100: except oe.license.InvalidLicense as exc:
0101: bb.fatal('%s: %s' % (d.getVar('PF', True), exc))
0102: except SyntaxError:
0103: bb.warn("%s: Failed to parse it's LICENSE field." % (d.getVar('PF', True)))
File: '/media/build1/poky/meta/lib/oe/license.py', lineno: 38, function: visit_string
0034: new_elements = []
0035: elements = filter(lambda x: x.strip(), license_operator.split(licensestr))
0036: for pos, element in enumerate(elements):
0037: if license_pattern.match(element):
*** 0038: if pos > 0 and license_pattern.match(elements[pos-1]):
0039: new_elements.append('&')
0040: element = '"' + element + '"'
0041: elif not license_operator.match(element):
0042: raise InvalidLicense(element)
Exception: TypeError: 'filter' object is not subscriptable
ERROR: Function failed: do_populate_lic
ERROR: Logfile of failure stored in: /media/build1/poky/build/tmp/work/i586-poky-linux/eglibc-initial/2.17-r3/temp/log.do_populate_lic.3275
ERROR: Task 9 (/media/build1/poky/meta/recipes-core/eglibc/eglibc-initial_2.17.bb, do_populate_lic) failed with exit code '1
"""
(Bitbake rev: c5de66b870406d9bd1161a9b7e2b04fe6eb065fe)
Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
2013-05-08 17:14:53 +00:00
error . extend ( _print_trace ( text . split ( ' \n ' ) , tbextract [ level + 1 ] [ 1 ] ) )
2012-08-22 19:01:12 +00:00
else :
bitbake: utils: Improve better_exec traceback handling
The current bitbake tracebacks are hard to read/confusing and sometimes
incomplete. This patch attempts to do better by:
* Moving the note about the exact exception to the end to make things
read in sequence
* Merged the initial stack trace to become part of the code dump
* Added handling for "/xxxx" file paths since we can load these files
and include the data as part of the trace
* Dropped the ERROR: prefix to every line, allowing the error messages to
be spacially accosicated in the UIs
* Moved the "From:" line to the top of each code block and ensured its present
consistently
With the complexity now in this funciton, I've added try/except wrapping around
it to ensure we catch exceptions in the exception handler too.
Example before:
"""
ERROR: Error executing a python function in /media/build1/poky/meta/recipes-core/eglibc/eglibc-initial_2.17.bb:
TypeError: 'filter' object is not subscriptable
ERROR: The stack trace of python calls that resulted in this exception/failure was:
ERROR: File "do_populate_lic", line 13, in <module>
ERROR:
ERROR: File "do_populate_lic", line 6, in do_populate_lic
ERROR:
ERROR: File "license.bbclass", line 99, in find_license_files
ERROR:
ERROR: File "/media/build1/poky/meta/lib/oe/license.py", line 38, in visit_string
ERROR: if pos > 0 and license_pattern.match(elements[pos-1]):
ERROR:
ERROR: The code that was being executed was:
ERROR: 0009: destdir = os.path.join(d.getVar('LICSSTATEDIR', True), d.getVar('PN', True))
ERROR: 0010: copy_license_files(lic_files_paths, destdir)
ERROR: 0011:
ERROR: 0012:
ERROR: *** 0013:do_populate_lic(d)
ERROR: 0014:
ERROR: [From file: 'do_populate_lic', lineno: 13, function: <module>]
ERROR: 0002:def do_populate_lic(d):
ERROR: 0003: """
ERROR: 0004: Populate LICENSE_DIRECTORY with licenses.
ERROR: 0005: """
ERROR: *** 0006: lic_files_paths = find_license_files(d)
ERROR: 0007:
ERROR: 0008: # The base directory we wrangle licenses to
ERROR: 0009: destdir = os.path.join(d.getVar('LICSSTATEDIR', True), d.getVar('PN', True))
ERROR: 0010: copy_license_files(lic_files_paths, destdir)
ERROR: [From file: 'do_populate_lic', lineno: 6, function: do_populate_lic]
ERROR: 0095: lic_files_paths.append((os.path.basename(path), srclicfile))
ERROR: 0096:
ERROR: 0097: v = FindVisitor()
ERROR: 0098: try:
ERROR: *** 0099: v.visit_string(license_types)
ERROR: 0100: except oe.license.InvalidLicense as exc:
ERROR: 0101: bb.fatal('%s: %s' % (d.getVar('PF', True), exc))
ERROR: 0102: except SyntaxError:
ERROR: 0103: bb.warn("%s: Failed to parse it's LICENSE field." % (d.getVar('PF', True)))
ERROR: [From file: 'license.bbclass', lineno: 99, function: find_license_files]
ERROR: Function failed: do_populate_lic
ERROR: Logfile of failure stored in: /media/build1/poky/build/tmp/work/i586-poky-linux/eglibc-initial/2.17-r3/temp/log.do_populate_lic.17442
"""
Example after:
"""
ERROR: Error executing a python function in /media/build1/poky/meta/recipes-core/eglibc/eglibc-initial_2.17.bb:
The stack trace of python calls that resulted in this exception/failure was:
File: 'do_populate_lic', lineno: 13, function: <module>
0009: destdir = os.path.join(d.getVar('LICSSTATEDIR', True), d.getVar('PN', True))
0010: copy_license_files(lic_files_paths, destdir)
0011:
0012:
*** 0013:do_populate_lic(d)
0014:
File: 'do_populate_lic', lineno: 6, function: do_populate_lic
0002:def do_populate_lic(d):
0003: """
0004: Populate LICENSE_DIRECTORY with licenses.
0005: """
*** 0006: lic_files_paths = find_license_files(d)
0007:
0008: # The base directory we wrangle licenses to
0009: destdir = os.path.join(d.getVar('LICSSTATEDIR', True), d.getVar('PN', True))
0010: copy_license_files(lic_files_paths, destdir)
File: 'license.bbclass', lineno: 99, function: find_license_files
0095: lic_files_paths.append((os.path.basename(path), srclicfile))
0096:
0097: v = FindVisitor()
0098: try:
*** 0099: v.visit_string(license_types)
0100: except oe.license.InvalidLicense as exc:
0101: bb.fatal('%s: %s' % (d.getVar('PF', True), exc))
0102: except SyntaxError:
0103: bb.warn("%s: Failed to parse it's LICENSE field." % (d.getVar('PF', True)))
File: '/media/build1/poky/meta/lib/oe/license.py', lineno: 38, function: visit_string
0034: new_elements = []
0035: elements = filter(lambda x: x.strip(), license_operator.split(licensestr))
0036: for pos, element in enumerate(elements):
0037: if license_pattern.match(element):
*** 0038: if pos > 0 and license_pattern.match(elements[pos-1]):
0039: new_elements.append('&')
0040: element = '"' + element + '"'
0041: elif not license_operator.match(element):
0042: raise InvalidLicense(element)
Exception: TypeError: 'filter' object is not subscriptable
ERROR: Function failed: do_populate_lic
ERROR: Logfile of failure stored in: /media/build1/poky/build/tmp/work/i586-poky-linux/eglibc-initial/2.17-r3/temp/log.do_populate_lic.3275
ERROR: Task 9 (/media/build1/poky/meta/recipes-core/eglibc/eglibc-initial_2.17.bb, do_populate_lic) failed with exit code '1
"""
(Bitbake rev: c5de66b870406d9bd1161a9b7e2b04fe6eb065fe)
Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
2013-05-08 17:14:53 +00:00
error . append ( tbformat [ level + 1 ] )
2010-12-05 22:52:44 +00:00
else :
bitbake: utils: Improve better_exec traceback handling
The current bitbake tracebacks are hard to read/confusing and sometimes
incomplete. This patch attempts to do better by:
* Moving the note about the exact exception to the end to make things
read in sequence
* Merged the initial stack trace to become part of the code dump
* Added handling for "/xxxx" file paths since we can load these files
and include the data as part of the trace
* Dropped the ERROR: prefix to every line, allowing the error messages to
be spacially accosicated in the UIs
* Moved the "From:" line to the top of each code block and ensured its present
consistently
With the complexity now in this funciton, I've added try/except wrapping around
it to ensure we catch exceptions in the exception handler too.
Example before:
"""
ERROR: Error executing a python function in /media/build1/poky/meta/recipes-core/eglibc/eglibc-initial_2.17.bb:
TypeError: 'filter' object is not subscriptable
ERROR: The stack trace of python calls that resulted in this exception/failure was:
ERROR: File "do_populate_lic", line 13, in <module>
ERROR:
ERROR: File "do_populate_lic", line 6, in do_populate_lic
ERROR:
ERROR: File "license.bbclass", line 99, in find_license_files
ERROR:
ERROR: File "/media/build1/poky/meta/lib/oe/license.py", line 38, in visit_string
ERROR: if pos > 0 and license_pattern.match(elements[pos-1]):
ERROR:
ERROR: The code that was being executed was:
ERROR: 0009: destdir = os.path.join(d.getVar('LICSSTATEDIR', True), d.getVar('PN', True))
ERROR: 0010: copy_license_files(lic_files_paths, destdir)
ERROR: 0011:
ERROR: 0012:
ERROR: *** 0013:do_populate_lic(d)
ERROR: 0014:
ERROR: [From file: 'do_populate_lic', lineno: 13, function: <module>]
ERROR: 0002:def do_populate_lic(d):
ERROR: 0003: """
ERROR: 0004: Populate LICENSE_DIRECTORY with licenses.
ERROR: 0005: """
ERROR: *** 0006: lic_files_paths = find_license_files(d)
ERROR: 0007:
ERROR: 0008: # The base directory we wrangle licenses to
ERROR: 0009: destdir = os.path.join(d.getVar('LICSSTATEDIR', True), d.getVar('PN', True))
ERROR: 0010: copy_license_files(lic_files_paths, destdir)
ERROR: [From file: 'do_populate_lic', lineno: 6, function: do_populate_lic]
ERROR: 0095: lic_files_paths.append((os.path.basename(path), srclicfile))
ERROR: 0096:
ERROR: 0097: v = FindVisitor()
ERROR: 0098: try:
ERROR: *** 0099: v.visit_string(license_types)
ERROR: 0100: except oe.license.InvalidLicense as exc:
ERROR: 0101: bb.fatal('%s: %s' % (d.getVar('PF', True), exc))
ERROR: 0102: except SyntaxError:
ERROR: 0103: bb.warn("%s: Failed to parse it's LICENSE field." % (d.getVar('PF', True)))
ERROR: [From file: 'license.bbclass', lineno: 99, function: find_license_files]
ERROR: Function failed: do_populate_lic
ERROR: Logfile of failure stored in: /media/build1/poky/build/tmp/work/i586-poky-linux/eglibc-initial/2.17-r3/temp/log.do_populate_lic.17442
"""
Example after:
"""
ERROR: Error executing a python function in /media/build1/poky/meta/recipes-core/eglibc/eglibc-initial_2.17.bb:
The stack trace of python calls that resulted in this exception/failure was:
File: 'do_populate_lic', lineno: 13, function: <module>
0009: destdir = os.path.join(d.getVar('LICSSTATEDIR', True), d.getVar('PN', True))
0010: copy_license_files(lic_files_paths, destdir)
0011:
0012:
*** 0013:do_populate_lic(d)
0014:
File: 'do_populate_lic', lineno: 6, function: do_populate_lic
0002:def do_populate_lic(d):
0003: """
0004: Populate LICENSE_DIRECTORY with licenses.
0005: """
*** 0006: lic_files_paths = find_license_files(d)
0007:
0008: # The base directory we wrangle licenses to
0009: destdir = os.path.join(d.getVar('LICSSTATEDIR', True), d.getVar('PN', True))
0010: copy_license_files(lic_files_paths, destdir)
File: 'license.bbclass', lineno: 99, function: find_license_files
0095: lic_files_paths.append((os.path.basename(path), srclicfile))
0096:
0097: v = FindVisitor()
0098: try:
*** 0099: v.visit_string(license_types)
0100: except oe.license.InvalidLicense as exc:
0101: bb.fatal('%s: %s' % (d.getVar('PF', True), exc))
0102: except SyntaxError:
0103: bb.warn("%s: Failed to parse it's LICENSE field." % (d.getVar('PF', True)))
File: '/media/build1/poky/meta/lib/oe/license.py', lineno: 38, function: visit_string
0034: new_elements = []
0035: elements = filter(lambda x: x.strip(), license_operator.split(licensestr))
0036: for pos, element in enumerate(elements):
0037: if license_pattern.match(element):
*** 0038: if pos > 0 and license_pattern.match(elements[pos-1]):
0039: new_elements.append('&')
0040: element = '"' + element + '"'
0041: elif not license_operator.match(element):
0042: raise InvalidLicense(element)
Exception: TypeError: 'filter' object is not subscriptable
ERROR: Function failed: do_populate_lic
ERROR: Logfile of failure stored in: /media/build1/poky/build/tmp/work/i586-poky-linux/eglibc-initial/2.17-r3/temp/log.do_populate_lic.3275
ERROR: Task 9 (/media/build1/poky/meta/recipes-core/eglibc/eglibc-initial_2.17.bb, do_populate_lic) failed with exit code '1
"""
(Bitbake rev: c5de66b870406d9bd1161a9b7e2b04fe6eb065fe)
Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
2013-05-08 17:14:53 +00:00
error . append ( tbformat [ level + 1 ] )
2010-12-05 22:52:44 +00:00
nexttb = tb . tb_next
level = level + 1
2010-03-24 23:56:12 +00:00
bitbake: utils: Improve better_exec traceback handling
The current bitbake tracebacks are hard to read/confusing and sometimes
incomplete. This patch attempts to do better by:
* Moving the note about the exact exception to the end to make things
read in sequence
* Merged the initial stack trace to become part of the code dump
* Added handling for "/xxxx" file paths since we can load these files
and include the data as part of the trace
* Dropped the ERROR: prefix to every line, allowing the error messages to
be spacially accosicated in the UIs
* Moved the "From:" line to the top of each code block and ensured its present
consistently
With the complexity now in this funciton, I've added try/except wrapping around
it to ensure we catch exceptions in the exception handler too.
Example before:
"""
ERROR: Error executing a python function in /media/build1/poky/meta/recipes-core/eglibc/eglibc-initial_2.17.bb:
TypeError: 'filter' object is not subscriptable
ERROR: The stack trace of python calls that resulted in this exception/failure was:
ERROR: File "do_populate_lic", line 13, in <module>
ERROR:
ERROR: File "do_populate_lic", line 6, in do_populate_lic
ERROR:
ERROR: File "license.bbclass", line 99, in find_license_files
ERROR:
ERROR: File "/media/build1/poky/meta/lib/oe/license.py", line 38, in visit_string
ERROR: if pos > 0 and license_pattern.match(elements[pos-1]):
ERROR:
ERROR: The code that was being executed was:
ERROR: 0009: destdir = os.path.join(d.getVar('LICSSTATEDIR', True), d.getVar('PN', True))
ERROR: 0010: copy_license_files(lic_files_paths, destdir)
ERROR: 0011:
ERROR: 0012:
ERROR: *** 0013:do_populate_lic(d)
ERROR: 0014:
ERROR: [From file: 'do_populate_lic', lineno: 13, function: <module>]
ERROR: 0002:def do_populate_lic(d):
ERROR: 0003: """
ERROR: 0004: Populate LICENSE_DIRECTORY with licenses.
ERROR: 0005: """
ERROR: *** 0006: lic_files_paths = find_license_files(d)
ERROR: 0007:
ERROR: 0008: # The base directory we wrangle licenses to
ERROR: 0009: destdir = os.path.join(d.getVar('LICSSTATEDIR', True), d.getVar('PN', True))
ERROR: 0010: copy_license_files(lic_files_paths, destdir)
ERROR: [From file: 'do_populate_lic', lineno: 6, function: do_populate_lic]
ERROR: 0095: lic_files_paths.append((os.path.basename(path), srclicfile))
ERROR: 0096:
ERROR: 0097: v = FindVisitor()
ERROR: 0098: try:
ERROR: *** 0099: v.visit_string(license_types)
ERROR: 0100: except oe.license.InvalidLicense as exc:
ERROR: 0101: bb.fatal('%s: %s' % (d.getVar('PF', True), exc))
ERROR: 0102: except SyntaxError:
ERROR: 0103: bb.warn("%s: Failed to parse it's LICENSE field." % (d.getVar('PF', True)))
ERROR: [From file: 'license.bbclass', lineno: 99, function: find_license_files]
ERROR: Function failed: do_populate_lic
ERROR: Logfile of failure stored in: /media/build1/poky/build/tmp/work/i586-poky-linux/eglibc-initial/2.17-r3/temp/log.do_populate_lic.17442
"""
Example after:
"""
ERROR: Error executing a python function in /media/build1/poky/meta/recipes-core/eglibc/eglibc-initial_2.17.bb:
The stack trace of python calls that resulted in this exception/failure was:
File: 'do_populate_lic', lineno: 13, function: <module>
0009: destdir = os.path.join(d.getVar('LICSSTATEDIR', True), d.getVar('PN', True))
0010: copy_license_files(lic_files_paths, destdir)
0011:
0012:
*** 0013:do_populate_lic(d)
0014:
File: 'do_populate_lic', lineno: 6, function: do_populate_lic
0002:def do_populate_lic(d):
0003: """
0004: Populate LICENSE_DIRECTORY with licenses.
0005: """
*** 0006: lic_files_paths = find_license_files(d)
0007:
0008: # The base directory we wrangle licenses to
0009: destdir = os.path.join(d.getVar('LICSSTATEDIR', True), d.getVar('PN', True))
0010: copy_license_files(lic_files_paths, destdir)
File: 'license.bbclass', lineno: 99, function: find_license_files
0095: lic_files_paths.append((os.path.basename(path), srclicfile))
0096:
0097: v = FindVisitor()
0098: try:
*** 0099: v.visit_string(license_types)
0100: except oe.license.InvalidLicense as exc:
0101: bb.fatal('%s: %s' % (d.getVar('PF', True), exc))
0102: except SyntaxError:
0103: bb.warn("%s: Failed to parse it's LICENSE field." % (d.getVar('PF', True)))
File: '/media/build1/poky/meta/lib/oe/license.py', lineno: 38, function: visit_string
0034: new_elements = []
0035: elements = filter(lambda x: x.strip(), license_operator.split(licensestr))
0036: for pos, element in enumerate(elements):
0037: if license_pattern.match(element):
*** 0038: if pos > 0 and license_pattern.match(elements[pos-1]):
0039: new_elements.append('&')
0040: element = '"' + element + '"'
0041: elif not license_operator.match(element):
0042: raise InvalidLicense(element)
Exception: TypeError: 'filter' object is not subscriptable
ERROR: Function failed: do_populate_lic
ERROR: Logfile of failure stored in: /media/build1/poky/build/tmp/work/i586-poky-linux/eglibc-initial/2.17-r3/temp/log.do_populate_lic.3275
ERROR: Task 9 (/media/build1/poky/meta/recipes-core/eglibc/eglibc-initial_2.17.bb, do_populate_lic) failed with exit code '1
"""
(Bitbake rev: c5de66b870406d9bd1161a9b7e2b04fe6eb065fe)
Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
2013-05-08 17:14:53 +00:00
error . append ( " Exception: %s " % ' ' . join ( exception ) )
finally :
logger . error ( " \n " . join ( error ) )
def better_exec ( code , context , text = None , realfile = " <code> " ) :
"""
Similiar to better_compile , better_exec will
print the lines that are responsible for the
error .
"""
import bb . parse
if not text :
text = code
if not hasattr ( code , " co_filename " ) :
code = better_compile ( code , realfile , realfile )
try :
2013-05-23 09:49:57 +00:00
exec ( code , get_context ( ) , context )
2014-05-30 14:56:11 +00:00
except ( bb . BBHandledException , bb . parse . SkipRecipe , bb . build . FuncFailed , bb . data_smart . ExpansionError ) :
# Error already shown so passthrough, no need for traceback
2014-05-29 17:17:16 +00:00
raise
bitbake: utils: Improve better_exec traceback handling
The current bitbake tracebacks are hard to read/confusing and sometimes
incomplete. This patch attempts to do better by:
* Moving the note about the exact exception to the end to make things
read in sequence
* Merged the initial stack trace to become part of the code dump
* Added handling for "/xxxx" file paths since we can load these files
and include the data as part of the trace
* Dropped the ERROR: prefix to every line, allowing the error messages to
be spacially accosicated in the UIs
* Moved the "From:" line to the top of each code block and ensured its present
consistently
With the complexity now in this funciton, I've added try/except wrapping around
it to ensure we catch exceptions in the exception handler too.
Example before:
"""
ERROR: Error executing a python function in /media/build1/poky/meta/recipes-core/eglibc/eglibc-initial_2.17.bb:
TypeError: 'filter' object is not subscriptable
ERROR: The stack trace of python calls that resulted in this exception/failure was:
ERROR: File "do_populate_lic", line 13, in <module>
ERROR:
ERROR: File "do_populate_lic", line 6, in do_populate_lic
ERROR:
ERROR: File "license.bbclass", line 99, in find_license_files
ERROR:
ERROR: File "/media/build1/poky/meta/lib/oe/license.py", line 38, in visit_string
ERROR: if pos > 0 and license_pattern.match(elements[pos-1]):
ERROR:
ERROR: The code that was being executed was:
ERROR: 0009: destdir = os.path.join(d.getVar('LICSSTATEDIR', True), d.getVar('PN', True))
ERROR: 0010: copy_license_files(lic_files_paths, destdir)
ERROR: 0011:
ERROR: 0012:
ERROR: *** 0013:do_populate_lic(d)
ERROR: 0014:
ERROR: [From file: 'do_populate_lic', lineno: 13, function: <module>]
ERROR: 0002:def do_populate_lic(d):
ERROR: 0003: """
ERROR: 0004: Populate LICENSE_DIRECTORY with licenses.
ERROR: 0005: """
ERROR: *** 0006: lic_files_paths = find_license_files(d)
ERROR: 0007:
ERROR: 0008: # The base directory we wrangle licenses to
ERROR: 0009: destdir = os.path.join(d.getVar('LICSSTATEDIR', True), d.getVar('PN', True))
ERROR: 0010: copy_license_files(lic_files_paths, destdir)
ERROR: [From file: 'do_populate_lic', lineno: 6, function: do_populate_lic]
ERROR: 0095: lic_files_paths.append((os.path.basename(path), srclicfile))
ERROR: 0096:
ERROR: 0097: v = FindVisitor()
ERROR: 0098: try:
ERROR: *** 0099: v.visit_string(license_types)
ERROR: 0100: except oe.license.InvalidLicense as exc:
ERROR: 0101: bb.fatal('%s: %s' % (d.getVar('PF', True), exc))
ERROR: 0102: except SyntaxError:
ERROR: 0103: bb.warn("%s: Failed to parse it's LICENSE field." % (d.getVar('PF', True)))
ERROR: [From file: 'license.bbclass', lineno: 99, function: find_license_files]
ERROR: Function failed: do_populate_lic
ERROR: Logfile of failure stored in: /media/build1/poky/build/tmp/work/i586-poky-linux/eglibc-initial/2.17-r3/temp/log.do_populate_lic.17442
"""
Example after:
"""
ERROR: Error executing a python function in /media/build1/poky/meta/recipes-core/eglibc/eglibc-initial_2.17.bb:
The stack trace of python calls that resulted in this exception/failure was:
File: 'do_populate_lic', lineno: 13, function: <module>
0009: destdir = os.path.join(d.getVar('LICSSTATEDIR', True), d.getVar('PN', True))
0010: copy_license_files(lic_files_paths, destdir)
0011:
0012:
*** 0013:do_populate_lic(d)
0014:
File: 'do_populate_lic', lineno: 6, function: do_populate_lic
0002:def do_populate_lic(d):
0003: """
0004: Populate LICENSE_DIRECTORY with licenses.
0005: """
*** 0006: lic_files_paths = find_license_files(d)
0007:
0008: # The base directory we wrangle licenses to
0009: destdir = os.path.join(d.getVar('LICSSTATEDIR', True), d.getVar('PN', True))
0010: copy_license_files(lic_files_paths, destdir)
File: 'license.bbclass', lineno: 99, function: find_license_files
0095: lic_files_paths.append((os.path.basename(path), srclicfile))
0096:
0097: v = FindVisitor()
0098: try:
*** 0099: v.visit_string(license_types)
0100: except oe.license.InvalidLicense as exc:
0101: bb.fatal('%s: %s' % (d.getVar('PF', True), exc))
0102: except SyntaxError:
0103: bb.warn("%s: Failed to parse it's LICENSE field." % (d.getVar('PF', True)))
File: '/media/build1/poky/meta/lib/oe/license.py', lineno: 38, function: visit_string
0034: new_elements = []
0035: elements = filter(lambda x: x.strip(), license_operator.split(licensestr))
0036: for pos, element in enumerate(elements):
0037: if license_pattern.match(element):
*** 0038: if pos > 0 and license_pattern.match(elements[pos-1]):
0039: new_elements.append('&')
0040: element = '"' + element + '"'
0041: elif not license_operator.match(element):
0042: raise InvalidLicense(element)
Exception: TypeError: 'filter' object is not subscriptable
ERROR: Function failed: do_populate_lic
ERROR: Logfile of failure stored in: /media/build1/poky/build/tmp/work/i586-poky-linux/eglibc-initial/2.17-r3/temp/log.do_populate_lic.3275
ERROR: Task 9 (/media/build1/poky/meta/recipes-core/eglibc/eglibc-initial_2.17.bb, do_populate_lic) failed with exit code '1
"""
(Bitbake rev: c5de66b870406d9bd1161a9b7e2b04fe6eb065fe)
Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
2013-05-08 17:14:53 +00:00
except Exception as e :
( t , value , tb ) = sys . exc_info ( )
try :
_print_exception ( t , value , tb , realfile , text , context )
except Exception as e :
logger . error ( " Exception handler error: %s " % str ( e ) )
2012-08-22 19:02:39 +00:00
e = bb . BBHandledException ( e )
raise e
2006-11-16 15:02:15 +00:00
2010-03-31 15:52:41 +00:00
def simple_exec ( code , context ) :
2013-05-23 09:49:57 +00:00
exec ( code , get_context ( ) , context )
2010-03-31 15:52:41 +00:00
2010-03-31 03:06:07 +00:00
def better_eval ( source , locals ) :
2013-05-23 09:49:57 +00:00
return eval ( source , get_context ( ) , locals )
2010-03-31 03:06:07 +00:00
2010-12-10 01:29:31 +00:00
@contextmanager
def fileslocked ( files ) :
2010-12-10 02:50:23 +00:00
""" Context manager for locking and unlocking file locks. """
2010-12-10 01:29:31 +00:00
locks = [ ]
if files :
for lockfile in files :
2010-12-10 21:27:28 +00:00
locks . append ( bb . utils . lockfile ( lockfile ) )
2010-12-10 01:29:31 +00:00
yield
for lock in locks :
bb . utils . unlockfile ( lock )
2015-07-31 09:16:33 +00:00
@contextmanager
def timeout ( seconds ) :
def timeout_handler ( signum , frame ) :
pass
original_handler = signal . signal ( signal . SIGALRM , timeout_handler )
try :
signal . alarm ( seconds )
yield
finally :
signal . alarm ( 0 )
signal . signal ( signal . SIGALRM , original_handler )
def lockfile ( name , shared = False , retry = True , block = False ) :
2007-11-24 14:44:36 +00:00
"""
2015-07-31 09:16:33 +00:00
Use the specified file as a lock file , return when the lock has
been acquired . Returns a variable to pass to unlockfile ( ) .
Parameters :
retry : True to re - try locking if it fails , False otherwise
block : True to block until the lock succeeds , False otherwise
The retry and block parameters are kind of equivalent unless you
consider the possibility of sending a signal to the process to break
out - at which point you want block = True rather than retry = True .
2007-11-24 14:44:36 +00:00
"""
2011-03-08 16:33:40 +00:00
dirname = os . path . dirname ( name )
mkdirhier ( dirname )
2008-10-17 11:04:26 +00:00
2011-03-08 16:33:40 +00:00
if not os . access ( dirname , os . W_OK ) :
logger . error ( " Unable to acquire lock ' %s ' , directory is not writable " ,
2011-03-08 16:39:40 +00:00
name )
2010-12-29 07:28:48 +00:00
sys . exit ( 1 )
2011-01-19 11:01:54 +00:00
op = fcntl . LOCK_EX
if shared :
op = fcntl . LOCK_SH
2015-07-31 09:16:33 +00:00
if not retry and not block :
2011-05-27 15:13:27 +00:00
op = op | fcntl . LOCK_NB
2011-01-19 11:01:54 +00:00
2007-11-24 14:44:36 +00:00
while True :
# If we leave the lockfiles lying around there is no problem
# but we should clean up after ourselves. This gives potential
2010-03-24 23:56:12 +00:00
# for races though. To work around this, when we acquire the lock
# we check the file we locked was still the lock file on disk.
# by comparing inode numbers. If they don't match or the lockfile
2007-11-24 14:44:36 +00:00
# no longer exists, we start again.
2010-03-24 23:56:12 +00:00
# This implementation is unfair since the last person to request the
2007-11-24 14:44:36 +00:00
# lock is the most likely to win it.
2008-10-06 08:09:11 +00:00
try :
2010-12-10 02:50:23 +00:00
lf = open ( name , ' a+ ' )
fileno = lf . fileno ( )
2011-01-19 11:01:54 +00:00
fcntl . flock ( fileno , op )
2010-12-10 02:50:23 +00:00
statinfo = os . fstat ( fileno )
2008-10-06 08:09:11 +00:00
if os . path . exists ( lf . name ) :
2010-03-24 23:56:12 +00:00
statinfo2 = os . stat ( lf . name )
if statinfo . st_ino == statinfo2 . st_ino :
return lf
2010-12-10 02:50:23 +00:00
lf . close ( )
except Exception :
2013-05-09 21:06:45 +00:00
try :
lf . close ( )
except Exception :
pass
2011-11-08 17:49:32 +00:00
pass
2011-05-27 15:13:27 +00:00
if not retry :
return None
2007-11-24 14:44:36 +00:00
def unlockfile ( lf ) :
"""
2010-03-24 23:56:12 +00:00
Unlock a file locked using lockfile ( )
2007-11-24 14:44:36 +00:00
"""
2011-01-19 13:30:14 +00:00
try :
2011-03-08 16:33:40 +00:00
# If we had a shared lock, we need to promote to exclusive before
2011-01-19 13:30:14 +00:00
# removing the lockfile. Attempt this, ignore failures.
fcntl . flock ( lf . fileno ( ) , fcntl . LOCK_EX | fcntl . LOCK_NB )
os . unlink ( lf . name )
2011-01-26 12:20:14 +00:00
except ( IOError , OSError ) :
2011-01-19 13:30:14 +00:00
pass
2007-11-24 14:44:36 +00:00
fcntl . flock ( lf . fileno ( ) , fcntl . LOCK_UN )
2010-12-10 02:50:23 +00:00
lf . close ( )
2007-11-24 14:44:36 +00:00
2008-05-01 10:59:00 +00:00
def md5_file ( filename ) :
"""
Return the hex string representation of the MD5 checksum of filename .
"""
try :
import hashlib
m = hashlib . md5 ( )
except ImportError :
import md5
m = md5 . new ( )
2010-03-24 23:56:12 +00:00
2013-05-09 21:06:45 +00:00
with open ( filename , " rb " ) as f :
for line in f :
m . update ( line )
2008-05-01 10:59:00 +00:00
return m . hexdigest ( )
def sha256_file ( filename ) :
"""
Return the hex string representation of the 256 - bit SHA checksum of
filename . On Python 2.4 this will return None , so callers will need to
handle that by either skipping SHA checks , or running a standalone sha256sum
binary .
"""
try :
import hashlib
except ImportError :
return None
s = hashlib . sha256 ( )
2013-05-09 21:06:45 +00:00
with open ( filename , " rb " ) as f :
for line in f :
s . update ( line )
2008-05-01 10:59:00 +00:00
return s . hexdigest ( )
2008-08-18 07:56:04 +00:00
2011-01-10 12:03:40 +00:00
def preserved_envvars_exported ( ) :
""" Variables which are taken from the environment and placed in and exported
from the metadata """
2008-09-30 20:57:18 +00:00
return [
2010-08-31 13:49:43 +00:00
' BB_TASKHASH ' ,
2010-11-28 17:39:09 +00:00
' HOME ' ,
' LOGNAME ' ,
' PATH ' ,
' PWD ' ,
' SHELL ' ,
' TERM ' ,
' USER ' ,
]
2011-01-10 12:03:40 +00:00
def preserved_envvars ( ) :
""" Variables which are taken from the environment and placed in the metadata """
2010-11-28 17:39:09 +00:00
v = [
' BBPATH ' ,
' BB_PRESERVE_ENV ' ,
' BB_ENV_WHITELIST ' ,
' BB_ENV_EXTRAWHITE ' ,
]
2013-02-05 16:02:03 +00:00
return v + preserved_envvars_exported ( )
2010-11-28 17:39:09 +00:00
2008-09-30 20:57:18 +00:00
def filter_environment ( good_vars ) :
"""
Create a pristine environment for bitbake . This will remove variables that
are not known and may influence the build in a negative way .
"""
2013-02-11 11:00:45 +00:00
removed_vars = { }
2008-09-30 20:57:18 +00:00
for key in os . environ . keys ( ) :
if key in good_vars :
continue
2010-03-24 23:56:12 +00:00
2013-02-11 11:00:45 +00:00
removed_vars [ key ] = os . environ [ key ]
2008-09-30 20:57:18 +00:00
os . unsetenv ( key )
del os . environ [ key ]
2014-10-26 22:43:32 +00:00
if removed_vars :
2013-02-11 11:00:45 +00:00
logger . debug ( 1 , " Removed the following variables from the environment: %s " , " , " . join ( removed_vars . keys ( ) ) )
2008-09-30 20:57:18 +00:00
return removed_vars
2011-08-13 00:58:11 +00:00
def approved_variables ( ) :
"""
Determine and return the list of whitelisted variables which are approved
2014-08-20 12:23:27 +00:00
to remain in the environment .
2011-08-13 00:58:11 +00:00
"""
2013-09-11 10:01:47 +00:00
if ' BB_PRESERVE_ENV ' in os . environ :
return os . environ . keys ( )
2011-08-13 00:58:11 +00:00
approved = [ ]
if ' BB_ENV_WHITELIST ' in os . environ :
approved = os . environ [ ' BB_ENV_WHITELIST ' ] . split ( )
2013-03-16 08:07:41 +00:00
approved . extend ( [ ' BB_ENV_WHITELIST ' ] )
2011-08-13 00:58:11 +00:00
else :
approved = preserved_envvars ( )
if ' BB_ENV_EXTRAWHITE ' in os . environ :
approved . extend ( os . environ [ ' BB_ENV_EXTRAWHITE ' ] . split ( ) )
2013-03-16 08:07:41 +00:00
if ' BB_ENV_EXTRAWHITE ' not in approved :
approved . extend ( [ ' BB_ENV_EXTRAWHITE ' ] )
2011-08-13 00:58:11 +00:00
return approved
2008-10-17 09:46:35 +00:00
def clean_environment ( ) :
"""
Clean up any spurious environment variables . This will remove any
2011-08-13 00:58:11 +00:00
variables the user hasn ' t chosen to preserve.
2008-10-17 09:46:35 +00:00
"""
if ' BB_PRESERVE_ENV ' not in os . environ :
2011-08-13 00:58:11 +00:00
good_vars = approved_variables ( )
2013-02-11 11:00:45 +00:00
return filter_environment ( good_vars )
return { }
2008-10-17 09:46:35 +00:00
def empty_environment ( ) :
"""
Remove all variables from the environment .
"""
for s in os . environ . keys ( ) :
os . unsetenv ( s )
del os . environ [ s ]
2009-07-02 13:33:53 +00:00
def build_environment ( d ) :
"""
Build an environment from all exported variables .
"""
2010-04-08 17:22:29 +00:00
import bb . data
2009-07-02 13:33:53 +00:00
for var in bb . data . keys ( d ) :
2011-11-25 14:57:53 +00:00
export = d . getVarFlag ( var , " export " )
2009-07-02 13:33:53 +00:00
if export :
2011-11-25 14:57:53 +00:00
os . environ [ var ] = d . getVar ( var , True ) or " "
2009-07-02 13:33:53 +00:00
2015-04-17 14:26:59 +00:00
def _check_unsafe_delete_path ( path ) :
"""
Basic safeguard against recursively deleting something we shouldn ' t. If it returns True,
the caller should raise an exception with an appropriate message .
NOTE : This is NOT meant to be a security mechanism - just a guard against silly mistakes
with potentially disastrous results .
"""
extra = ' '
# HOME might not be /home/something, so in case we can get it, check against it
homedir = os . environ . get ( ' HOME ' , ' ' )
if homedir :
extra = ' | %s ' % homedir
if re . match ( ' (/|//|/home|/home/[^/]* %s )$ ' % extra , os . path . abspath ( path ) ) :
return True
return False
2010-12-10 01:14:48 +00:00
def remove ( path , recurse = False ) :
""" Equivalent to rm -f or rm -rf """
2011-01-10 22:05:19 +00:00
if not path :
return
2013-02-03 16:51:34 +00:00
if recurse :
2015-04-17 14:26:59 +00:00
for name in glob . glob ( path ) :
if _check_unsafe_delete_path ( path ) :
raise Exception ( ' bb.utils.remove: called with dangerous path " %s " and recurse=True, refusing to delete! ' % path )
2013-02-03 16:51:34 +00:00
# shutil.rmtree(name) would be ideal but its too slow
2013-02-19 22:31:52 +00:00
subprocess . call ( [ ' rm ' , ' -rf ' ] + glob . glob ( path ) )
2013-02-03 16:51:34 +00:00
return
2011-02-08 17:24:12 +00:00
for name in glob . glob ( path ) :
try :
os . unlink ( name )
except OSError as exc :
2013-02-03 16:51:34 +00:00
if exc . errno != errno . ENOENT :
2011-02-08 17:24:12 +00:00
raise
2010-12-10 01:14:48 +00:00
2008-08-18 07:56:04 +00:00
def prunedir ( topdir ) :
# Delete everything reachable from the directory named in 'topdir'.
# CAUTION: This is dangerous!
2015-04-17 14:26:59 +00:00
if _check_unsafe_delete_path ( topdir ) :
raise Exception ( ' bb.utils.prunedir: called with dangerous path " %s " , refusing to delete! ' % topdir )
2010-03-24 23:56:12 +00:00
for root , dirs , files in os . walk ( topdir , topdown = False ) :
2008-08-18 07:56:04 +00:00
for name in files :
os . remove ( os . path . join ( root , name ) )
for name in dirs :
2008-10-20 14:19:36 +00:00
if os . path . islink ( os . path . join ( root , name ) ) :
os . remove ( os . path . join ( root , name ) )
else :
os . rmdir ( os . path . join ( root , name ) )
2008-08-18 07:56:04 +00:00
os . rmdir ( topdir )
2008-09-30 20:46:17 +00:00
2008-12-31 22:58:57 +00:00
#
# Could also use return re.compile("(%s)" % "|".join(map(re.escape, suffixes))).sub(lambda mo: "", var)
# but thats possibly insane and suffixes is probably going to be small
#
def prune_suffix ( var , suffixes , d ) :
2010-03-24 23:56:12 +00:00
# See if var ends with any of the suffixes listed and
2008-12-31 22:58:57 +00:00
# remove it if found
for suffix in suffixes :
if var . endswith ( suffix ) :
return var . replace ( suffix , " " )
return var
2009-07-19 17:05:52 +00:00
2010-12-17 11:29:49 +00:00
def mkdirhier ( directory ) :
2009-07-19 17:05:52 +00:00
""" Create a directory like ' mkdir -p ' , but does not complain if
directory already exists like os . makedirs
"""
try :
2010-12-17 11:29:49 +00:00
os . makedirs ( directory )
2010-04-12 00:03:55 +00:00
except OSError as e :
2010-04-08 17:22:29 +00:00
if e . errno != errno . EEXIST :
raise e
2009-07-19 17:05:52 +00:00
2010-03-24 23:56:12 +00:00
def movefile ( src , dest , newmtime = None , sstat = None ) :
2009-07-19 17:05:52 +00:00
""" Moves a file from src to dest, preserving all permissions and
attributes ; mtime will be preserved even when moving across
filesystems . Returns true on success and false on failure . Move is
atomic .
"""
2010-03-24 23:56:12 +00:00
#print "movefile(" + src + "," + dest + "," + str(newmtime) + "," + str(sstat) + ")"
2009-07-19 17:05:52 +00:00
try :
if not sstat :
2010-03-24 23:56:12 +00:00
sstat = os . lstat ( src )
2010-04-12 00:03:55 +00:00
except Exception as e :
2010-04-10 02:46:14 +00:00
print ( " movefile: Stating source file failed... " , e )
2009-07-19 17:05:52 +00:00
return None
2010-03-24 23:56:12 +00:00
destexists = 1
2009-07-19 17:05:52 +00:00
try :
2010-03-24 23:56:12 +00:00
dstat = os . lstat ( dest )
2009-07-19 17:05:52 +00:00
except :
2010-03-24 23:56:12 +00:00
dstat = os . lstat ( os . path . dirname ( dest ) )
destexists = 0
2009-07-19 17:05:52 +00:00
if destexists :
if stat . S_ISLNK ( dstat [ stat . ST_MODE ] ) :
try :
os . unlink ( dest )
2010-03-24 23:56:12 +00:00
destexists = 0
2010-04-12 00:03:55 +00:00
except Exception as e :
2009-07-19 17:05:52 +00:00
pass
if stat . S_ISLNK ( sstat [ stat . ST_MODE ] ) :
try :
2010-03-24 23:56:12 +00:00
target = os . readlink ( src )
2009-07-19 17:05:52 +00:00
if destexists and not stat . S_ISDIR ( dstat [ stat . ST_MODE ] ) :
os . unlink ( dest )
2010-03-24 23:56:12 +00:00
os . symlink ( target , dest )
2009-07-19 17:05:52 +00:00
#os.lchown(dest,sstat[stat.ST_UID],sstat[stat.ST_GID])
os . unlink ( src )
return os . lstat ( dest )
2010-04-12 00:03:55 +00:00
except Exception as e :
2010-04-10 02:46:14 +00:00
print ( " movefile: failed to properly create symlink: " , dest , " -> " , target , e )
2009-07-19 17:05:52 +00:00
return None
2010-03-24 23:56:12 +00:00
renamefailed = 1
if sstat [ stat . ST_DEV ] == dstat [ stat . ST_DEV ] :
2009-07-19 17:05:52 +00:00
try :
2015-09-03 07:10:45 +00:00
# os.rename needs to know the dest path ending with file name
# so append the file name to a path only if it's a dir specified
srcfname = os . path . basename ( src )
destpath = os . path . join ( dest , srcfname ) if os . path . isdir ( dest ) \
else dest
os . rename ( src , destpath )
2010-03-24 23:56:12 +00:00
renamefailed = 0
2010-04-12 00:03:55 +00:00
except Exception as e :
2010-03-24 23:56:12 +00:00
if e [ 0 ] != errno . EXDEV :
2009-07-19 17:05:52 +00:00
# Some random error.
2010-04-10 02:46:14 +00:00
print ( " movefile: Failed to move " , src , " to " , dest , e )
2009-07-19 17:05:52 +00:00
return None
# Invalid cross-device-link 'bind' mounted or actually Cross-Device
if renamefailed :
2010-03-24 23:56:12 +00:00
didcopy = 0
2009-07-19 17:05:52 +00:00
if stat . S_ISREG ( sstat [ stat . ST_MODE ] ) :
try : # For safety copy then move it over.
2010-03-24 23:56:12 +00:00
shutil . copyfile ( src , dest + " #new " )
os . rename ( dest + " #new " , dest )
didcopy = 1
2010-04-12 00:03:55 +00:00
except Exception as e :
2010-04-10 02:46:14 +00:00
print ( ' movefile: copy ' , src , ' -> ' , dest , ' failed. ' , e )
2009-07-19 17:05:52 +00:00
return None
else :
#we don't yet handle special, so we need to fall back to /bin/mv
2010-03-24 23:56:12 +00:00
a = getstatusoutput ( " /bin/mv -f " + " ' " + src + " ' ' " + dest + " ' " )
if a [ 0 ] != 0 :
2010-04-10 02:46:14 +00:00
print ( " movefile: Failed to move special file: " + src + " ' to ' " + dest + " ' " , a )
2009-07-19 17:05:52 +00:00
return None # failure
try :
if didcopy :
2010-03-24 23:56:12 +00:00
os . lchown ( dest , sstat [ stat . ST_UID ] , sstat [ stat . ST_GID ] )
2009-07-19 17:05:52 +00:00
os . chmod ( dest , stat . S_IMODE ( sstat [ stat . ST_MODE ] ) ) # Sticky is reset on chown
os . unlink ( src )
2010-04-12 00:03:55 +00:00
except Exception as e :
2010-04-10 02:46:14 +00:00
print ( " movefile: Failed to chown/chmod/unlink " , dest , e )
2009-07-19 17:05:52 +00:00
return None
if newmtime :
2010-03-24 23:56:12 +00:00
os . utime ( dest , ( newmtime , newmtime ) )
2009-07-19 17:05:52 +00:00
else :
os . utime ( dest , ( sstat [ stat . ST_ATIME ] , sstat [ stat . ST_MTIME ] ) )
2010-03-24 23:56:12 +00:00
newmtime = sstat [ stat . ST_MTIME ]
2009-07-19 17:05:52 +00:00
return newmtime
2010-03-24 23:56:12 +00:00
def copyfile ( src , dest , newmtime = None , sstat = None ) :
2009-07-19 17:05:52 +00:00
"""
Copies a file from src to dest , preserving all permissions and
attributes ; mtime will be preserved even when moving across
filesystems . Returns true on success and false on failure .
"""
2010-03-24 23:56:12 +00:00
#print "copyfile(" + src + "," + dest + "," + str(newmtime) + "," + str(sstat) + ")"
2009-07-19 17:05:52 +00:00
try :
if not sstat :
2010-03-24 23:56:12 +00:00
sstat = os . lstat ( src )
2010-04-12 00:03:55 +00:00
except Exception as e :
2013-10-02 16:47:28 +00:00
logger . warn ( " copyfile: stat of %s failed ( %s ) " % ( src , e ) )
2009-07-19 17:05:52 +00:00
return False
2010-03-24 23:56:12 +00:00
destexists = 1
2009-07-19 17:05:52 +00:00
try :
2010-03-24 23:56:12 +00:00
dstat = os . lstat ( dest )
2009-07-19 17:05:52 +00:00
except :
2010-03-24 23:56:12 +00:00
dstat = os . lstat ( os . path . dirname ( dest ) )
destexists = 0
2009-07-19 17:05:52 +00:00
if destexists :
if stat . S_ISLNK ( dstat [ stat . ST_MODE ] ) :
try :
os . unlink ( dest )
2010-03-24 23:56:12 +00:00
destexists = 0
2010-04-12 00:03:55 +00:00
except Exception as e :
2009-07-19 17:05:52 +00:00
pass
if stat . S_ISLNK ( sstat [ stat . ST_MODE ] ) :
try :
2010-03-24 23:56:12 +00:00
target = os . readlink ( src )
2009-07-19 17:05:52 +00:00
if destexists and not stat . S_ISDIR ( dstat [ stat . ST_MODE ] ) :
os . unlink ( dest )
2010-03-24 23:56:12 +00:00
os . symlink ( target , dest )
2009-07-19 17:05:52 +00:00
#os.lchown(dest,sstat[stat.ST_UID],sstat[stat.ST_GID])
return os . lstat ( dest )
2010-04-12 00:03:55 +00:00
except Exception as e :
2013-10-02 16:47:28 +00:00
logger . warn ( " copyfile: failed to create symlink %s to %s ( %s ) " % ( dest , target , e ) )
2009-07-19 17:05:52 +00:00
return False
if stat . S_ISREG ( sstat [ stat . ST_MODE ] ) :
2011-02-25 17:22:00 +00:00
try :
srcchown = False
if not os . access ( src , os . R_OK ) :
# Make sure we can read it
srcchown = True
os . chmod ( src , sstat [ stat . ST_MODE ] | stat . S_IRUSR )
# For safety copy then move it over.
2010-03-24 23:56:12 +00:00
shutil . copyfile ( src , dest + " #new " )
os . rename ( dest + " #new " , dest )
2010-04-12 00:03:55 +00:00
except Exception as e :
2013-10-02 16:47:28 +00:00
logger . warn ( " copyfile: copy %s to %s failed ( %s ) " % ( src , dest , e ) )
2010-03-24 23:56:12 +00:00
return False
2010-08-23 12:01:05 +00:00
finally :
2011-02-25 17:22:00 +00:00
if srcchown :
os . chmod ( src , sstat [ stat . ST_MODE ] )
os . utime ( src , ( sstat [ stat . ST_ATIME ] , sstat [ stat . ST_MTIME ] ) )
2010-08-23 12:01:05 +00:00
2009-07-19 17:05:52 +00:00
else :
2010-03-24 23:56:12 +00:00
#we don't yet handle special, so we need to fall back to /bin/mv
a = getstatusoutput ( " /bin/cp -f " + " ' " + src + " ' ' " + dest + " ' " )
if a [ 0 ] != 0 :
2013-10-02 16:47:28 +00:00
logger . warn ( " copyfile: failed to copy special file %s to %s ( %s ) " % ( src , dest , a ) )
2010-03-24 23:56:12 +00:00
return False # failure
2009-07-19 17:05:52 +00:00
try :
2010-03-24 23:56:12 +00:00
os . lchown ( dest , sstat [ stat . ST_UID ] , sstat [ stat . ST_GID ] )
2009-07-19 17:05:52 +00:00
os . chmod ( dest , stat . S_IMODE ( sstat [ stat . ST_MODE ] ) ) # Sticky is reset on chown
2010-04-12 00:03:55 +00:00
except Exception as e :
2013-10-02 16:47:28 +00:00
logger . warn ( " copyfile: failed to chown/chmod %s ( %s ) " % ( dest , e ) )
2009-07-19 17:05:52 +00:00
return False
if newmtime :
2010-03-24 23:56:12 +00:00
os . utime ( dest , ( newmtime , newmtime ) )
2009-07-19 17:05:52 +00:00
else :
os . utime ( dest , ( sstat [ stat . ST_ATIME ] , sstat [ stat . ST_MTIME ] ) )
2010-03-24 23:56:12 +00:00
newmtime = sstat [ stat . ST_MTIME ]
2009-07-19 17:05:52 +00:00
return newmtime
2013-11-29 23:15:56 +00:00
def which ( path , item , direction = 0 , history = False ) :
2009-07-19 17:05:52 +00:00
"""
Locate a file in a PATH
"""
2013-11-29 23:15:56 +00:00
hist = [ ]
2009-07-19 17:05:52 +00:00
paths = ( path or " " ) . split ( ' : ' )
if direction != 0 :
paths . reverse ( )
2010-03-24 17:31:06 +00:00
for p in paths :
2009-07-19 17:05:52 +00:00
next = os . path . join ( p , item )
2013-11-29 23:15:56 +00:00
hist . append ( next )
2009-07-19 17:05:52 +00:00
if os . path . exists ( next ) :
2012-05-24 12:57:16 +00:00
if not os . path . isabs ( next ) :
next = os . path . abspath ( next )
2013-11-29 23:15:56 +00:00
if history :
return next , hist
2009-07-19 17:05:52 +00:00
return next
2013-11-29 23:15:56 +00:00
if history :
return " " , hist
2009-07-19 17:05:52 +00:00
return " "
2010-06-29 12:32:04 +00:00
2011-02-11 17:43:54 +00:00
def to_boolean ( string , default = None ) :
if not string :
return default
normalized = string . lower ( )
if normalized in ( " y " , " yes " , " 1 " , " true " ) :
return True
elif normalized in ( " n " , " no " , " 0 " , " false " ) :
return False
else :
raise ValueError ( " Invalid value for to_boolean: %s " % string )
2011-07-11 13:38:50 +00:00
def contains ( variable , checkvalues , truevalue , falsevalue , d ) :
val = d . getVar ( variable , True )
if not val :
return falsevalue
2011-07-25 18:02:48 +00:00
val = set ( val . split ( ) )
if isinstance ( checkvalues , basestring ) :
checkvalues = set ( checkvalues . split ( ) )
else :
checkvalues = set ( checkvalues )
if checkvalues . issubset ( val ) :
2011-07-11 13:38:50 +00:00
return truevalue
return falsevalue
2012-02-23 13:47:19 +00:00
2014-04-25 21:21:24 +00:00
def contains_any ( variable , checkvalues , truevalue , falsevalue , d ) :
val = d . getVar ( variable , True )
if not val :
return falsevalue
val = set ( val . split ( ) )
if isinstance ( checkvalues , basestring ) :
checkvalues = set ( checkvalues . split ( ) )
else :
checkvalues = set ( checkvalues )
2014-06-11 10:04:44 +00:00
if checkvalues & val :
2014-04-25 21:21:24 +00:00
return truevalue
return falsevalue
2012-02-23 13:47:19 +00:00
def cpu_count ( ) :
return multiprocessing . cpu_count ( )
2012-06-22 11:53:16 +00:00
def nonblockingfd ( fd ) :
fcntl . fcntl ( fd , fcntl . F_SETFL , fcntl . fcntl ( fd , fcntl . F_GETFL ) | os . O_NONBLOCK )
2015-05-27 16:31:10 +00:00
def process_profilelog ( fn , pout = None ) :
# Either call with a list of filenames and set pout or a filename and optionally pout.
if not pout :
pout = fn + ' .processed '
pout = open ( pout , ' w ' )
2013-01-28 14:40:04 +00:00
import pstats
2015-05-27 16:31:10 +00:00
if isinstance ( fn , list ) :
p = pstats . Stats ( * fn , stream = pout )
else :
p = pstats . Stats ( fn , stream = pout )
2013-01-28 14:40:04 +00:00
p . sort_stats ( ' time ' )
p . print_stats ( )
p . print_callers ( )
p . sort_stats ( ' cumulative ' )
p . print_stats ( )
pout . flush ( )
pout . close ( )
2013-02-06 00:28:08 +00:00
#
2013-06-14 16:22:51 +00:00
# Was present to work around multiprocessing pool bugs in python < 2.7.3
2013-02-06 00:28:08 +00:00
#
def multiprocessingpool ( * args , * * kwargs ) :
2014-08-21 20:47:25 +00:00
import multiprocessing . pool
#import multiprocessing.util
#multiprocessing.util.log_to_stderr(10)
# Deal with a multiprocessing bug where signals to the processes would be delayed until the work
# completes. Putting in a timeout means the signals (like SIGINT/SIGTERM) get processed.
def wrapper ( func ) :
def wrap ( self , timeout = None ) :
return func ( self , timeout = timeout if timeout is not None else 1e100 )
return wrap
multiprocessing . pool . IMapIterator . next = wrapper ( multiprocessing . pool . IMapIterator . next )
2013-06-14 21:00:29 +00:00
return multiprocessing . Pool ( * args , * * kwargs )
2013-01-28 14:40:04 +00:00
2014-12-08 10:50:22 +00:00
def exec_flat_python_func ( func , * args , * * kwargs ) :
""" Execute a flat python function (defined with def funcname(args):...) """
# Prepare a small piece of python code which calls the requested function
# To do this we need to prepare two things - a set of variables we can use to pass
# the values of arguments into the calling function, and the list of arguments for
# the function being called
context = { }
funcargs = [ ]
# Handle unnamed arguments
aidx = 1
for arg in args :
argname = ' arg_ %s ' % aidx
context [ argname ] = arg
funcargs . append ( argname )
aidx + = 1
# Handle keyword arguments
context . update ( kwargs )
funcargs . extend ( [ ' %s = %s ' % ( arg , arg ) for arg in kwargs . iterkeys ( ) ] )
code = ' retval = %s ( %s ) ' % ( func , ' , ' . join ( funcargs ) )
comp = bb . utils . better_compile ( code , ' <string> ' , ' <string> ' )
bb . utils . better_exec ( comp , context , code , ' <string> ' )
return context [ ' retval ' ]
2014-12-19 10:20:30 +00:00
2015-05-18 15:08:36 +00:00
def edit_metadata ( meta_lines , variables , varfunc , match_overrides = False ) :
""" Edit lines from a recipe or config file and modify one or more
specified variable values set in the file using a specified callback
function . Lines are expected to have trailing newlines .
Parameters :
meta_lines : lines from the file ; can be a list or an iterable
( e . g . file pointer )
variables : a list of variable names to look for . Functions
may also be specified , but must be specified with ' () ' at
the end of the name . Note that the function doesn ' t have
any intrinsic understanding of _append , _prepend , _remove ,
or overrides , so these are considered as part of the name .
These values go into a regular expression , so regular
expression syntax is allowed .
varfunc : callback function called for every variable matching
one of the entries in the variables parameter . The function
should take four arguments :
varname : name of variable matched
origvalue : current value in file
op : the operator ( e . g . ' += ' )
newlines : list of lines up to this point . You can use
this to prepend lines before this variable setting
if you wish .
and should return a three - element tuple :
newvalue : new value to substitute in , or None to drop
the variable setting entirely . ( If the removal
results in two consecutive blank lines , one of the
blank lines will also be dropped ) .
newop : the operator to use - if you specify None here ,
the original operation will be used .
indent : number of spaces to indent multi - line entries ,
or - 1 to indent up to the level of the assignment
and opening quote , or a string to use as the indent .
minbreak : True to allow the first element of a
multi - line value to continue on the same line as
the assignment , False to indent before the first
element .
match_overrides : True to match items with _overrides on the end ,
False otherwise
Returns a tuple :
updated :
True if changes were made , False otherwise .
newlines :
Lines after processing
2014-12-19 10:20:30 +00:00
"""
2015-05-18 15:08:36 +00:00
2014-12-19 10:20:30 +00:00
var_res = { }
2015-05-18 15:08:36 +00:00
if match_overrides :
override_re = ' (_[a-zA-Z0-9-_$() {} ]+)? '
else :
override_re = ' '
2014-12-19 10:20:30 +00:00
for var in variables :
2015-05-18 15:08:36 +00:00
if var . endswith ( ' () ' ) :
var_res [ var ] = re . compile ( ' ^( %s %s )[ \\ t]* \ ([ \\ t]* \ )[ \\ t]* { ' % ( var [ : - 2 ] . rstrip ( ) , override_re ) )
else :
var_res [ var ] = re . compile ( ' ^( %s %s )[ \\ t]*[?+:.]*=[+.]*[ \\ t]*([ " \' ]) ' % ( var , override_re ) )
2014-12-19 10:20:30 +00:00
updated = False
varset_start = ' '
2015-05-18 15:08:35 +00:00
varlines = [ ]
2014-12-19 10:20:30 +00:00
newlines = [ ]
in_var = None
full_value = ' '
2015-05-18 15:08:36 +00:00
var_end = ' '
2014-12-19 10:20:30 +00:00
def handle_var_end ( ) :
2015-05-18 15:08:36 +00:00
prerun_newlines = newlines [ : ]
op = varset_start [ len ( in_var ) : ] . strip ( )
( newvalue , newop , indent , minbreak ) = varfunc ( in_var , full_value , op , newlines )
changed = ( prerun_newlines != newlines )
if newvalue is None :
# Drop the value
return True
elif newvalue != full_value or ( newop not in [ None , op ] ) :
if newop not in [ None , op ] :
# Callback changed the operator
varset_new = " %s %s " % ( in_var , newop )
else :
varset_new = varset_start
if isinstance ( indent , ( int , long ) ) :
if indent == - 1 :
indentspc = ' ' * ( len ( varset_new ) + 2 )
else :
indentspc = ' ' * indent
else :
indentspc = indent
if in_var . endswith ( ' () ' ) :
# A function definition
if isinstance ( newvalue , list ) :
newlines . append ( ' %s { \n %s %s \n } \n ' % ( varset_new , indentspc , ( ' \n %s ' % indentspc ) . join ( newvalue ) ) )
else :
if not newvalue . startswith ( ' \n ' ) :
newvalue = ' \n ' + newvalue
if not newvalue . endswith ( ' \n ' ) :
newvalue = newvalue + ' \n '
newlines . append ( ' %s { %s } \n ' % ( varset_new , newvalue ) )
else :
# Normal variable
if isinstance ( newvalue , list ) :
if not newvalue :
# Empty list -> empty string
newlines . append ( ' %s " " \n ' % varset_new )
elif minbreak :
# First item on first line
if len ( newvalue ) == 1 :
newlines . append ( ' %s " %s " \n ' % ( varset_new , newvalue [ 0 ] ) )
else :
newlines . append ( ' %s " %s \\ \n ' % ( varset_new , newvalue [ 0 ] ) )
for item in newvalue [ 1 : ] :
newlines . append ( ' %s %s \\ \n ' % ( indentspc , item ) )
newlines . append ( ' %s " \n ' % indentspc )
2014-12-19 10:20:30 +00:00
else :
2015-05-18 15:08:36 +00:00
# No item on first line
newlines . append ( ' %s " \\ \n ' % varset_new )
for item in newvalue :
newlines . append ( ' %s %s \\ \n ' % ( indentspc , item ) )
2014-12-19 10:20:30 +00:00
newlines . append ( ' %s " \n ' % indentspc )
else :
2015-05-18 15:08:36 +00:00
newlines . append ( ' %s " %s " \n ' % ( varset_new , newvalue ) )
2014-12-19 10:20:30 +00:00
return True
2015-05-18 15:08:35 +00:00
else :
# Put the old lines back where they were
newlines . extend ( varlines )
2015-05-18 15:08:36 +00:00
# If newlines was touched by the function, we'll need to return True
return changed
2014-12-19 10:20:30 +00:00
2015-05-18 15:08:36 +00:00
checkspc = False
for line in meta_lines :
if in_var :
value = line . rstrip ( )
varlines . append ( line )
if in_var . endswith ( ' () ' ) :
full_value + = ' \n ' + value
2014-12-19 10:20:30 +00:00
else :
2015-05-18 15:08:36 +00:00
full_value + = value [ : - 1 ]
if value . endswith ( var_end ) :
if in_var . endswith ( ' () ' ) :
if full_value . count ( ' { ' ) - full_value . count ( ' } ' ) > = 0 :
continue
full_value = full_value [ : - 1 ]
if handle_var_end ( ) :
updated = True
checkspc = True
in_var = None
else :
skip = False
for ( varname , var_re ) in var_res . iteritems ( ) :
res = var_re . match ( line )
if res :
isfunc = varname . endswith ( ' () ' )
if isfunc :
splitvalue = line . split ( ' { ' , 1 )
var_end = ' } '
else :
var_end = res . groups ( ) [ - 1 ]
splitvalue = line . split ( var_end , 1 )
varset_start = splitvalue [ 0 ] . rstrip ( )
value = splitvalue [ 1 ] . rstrip ( )
if not isfunc and value . endswith ( ' \\ ' ) :
value = value [ : - 1 ]
full_value = value
varlines = [ line ]
in_var = res . group ( 1 )
if isfunc :
in_var + = ' () '
if value . endswith ( var_end ) :
full_value = full_value [ : - 1 ]
if handle_var_end ( ) :
updated = True
checkspc = True
in_var = None
skip = True
break
if not skip :
if checkspc :
checkspc = False
if newlines [ - 1 ] == ' \n ' and line == ' \n ' :
# Squash blank line if there are two consecutive blanks after a removal
continue
newlines . append ( line )
return ( updated , newlines )
def edit_metadata_file ( meta_file , variables , varfunc ) :
""" Edit a recipe or config file and modify one or more specified
variable values set in the file using a specified callback function .
The file is only written to if the value ( s ) actually change .
This is basically the file version of edit_metadata ( ) , see that
function ' s description for parameter/usage information.
Returns True if the file was written to , False otherwise .
"""
with open ( meta_file , ' r ' ) as f :
( updated , newlines ) = edit_metadata ( f , variables , varfunc )
2014-12-19 10:20:30 +00:00
if updated :
with open ( meta_file , ' w ' ) as f :
f . writelines ( newlines )
2015-05-18 15:08:36 +00:00
return updated
2014-12-19 10:20:30 +00:00
def edit_bblayers_conf ( bblayers_conf , add , remove ) :
""" Edit bblayers.conf, adding and/or removing layers """
import fnmatch
def remove_trailing_sep ( pth ) :
if pth and pth [ - 1 ] == os . sep :
pth = pth [ : - 1 ]
return pth
def layerlist_param ( value ) :
if not value :
return [ ]
elif isinstance ( value , list ) :
return [ remove_trailing_sep ( x ) for x in value ]
else :
return [ remove_trailing_sep ( value ) ]
notadded = [ ]
notremoved = [ ]
addlayers = layerlist_param ( add )
removelayers = layerlist_param ( remove )
2014-12-29 11:14:26 +00:00
# Need to use a list here because we can't set non-local variables from a callback in python 2.x
bblayercalls = [ ]
2015-05-18 15:08:36 +00:00
def handle_bblayers ( varname , origvalue , op , newlines ) :
2014-12-29 11:14:26 +00:00
bblayercalls . append ( varname )
2014-12-19 10:20:30 +00:00
updated = False
bblayers = [ remove_trailing_sep ( x ) for x in origvalue . split ( ) ]
if removelayers :
for removelayer in removelayers :
matched = False
for layer in bblayers :
if fnmatch . fnmatch ( layer , removelayer ) :
updated = True
matched = True
bblayers . remove ( layer )
break
if not matched :
notremoved . append ( removelayer )
if addlayers :
for addlayer in addlayers :
if addlayer not in bblayers :
updated = True
bblayers . append ( addlayer )
else :
notadded . append ( addlayer )
2015-09-24 13:02:23 +00:00
del addlayers [ : ]
2014-12-19 10:20:30 +00:00
if updated :
2015-05-18 15:08:36 +00:00
return ( bblayers , None , 2 , False )
2014-12-19 10:20:30 +00:00
else :
2015-05-18 15:08:36 +00:00
return ( origvalue , None , 2 , False )
2014-12-19 10:20:30 +00:00
edit_metadata_file ( bblayers_conf , [ ' BBLAYERS ' ] , handle_bblayers )
2014-12-29 11:14:26 +00:00
if not bblayercalls :
raise Exception ( ' Unable to find BBLAYERS in %s ' % bblayers_conf )
2014-12-19 10:20:30 +00:00
return ( notadded , notremoved )
2015-05-18 15:08:34 +00:00
def get_file_layer ( filename , d ) :
""" Determine the collection (as defined by a layer ' s layer.conf file) containing the specified file """
collections = ( d . getVar ( ' BBFILE_COLLECTIONS ' , True ) or ' ' ) . split ( )
collection_res = { }
for collection in collections :
collection_res [ collection ] = d . getVar ( ' BBFILE_PATTERN_ %s ' % collection , True ) or ' '
2015-08-19 13:20:10 +00:00
def path_to_layer ( path ) :
# Use longest path so we handle nested layers
matchlen = 0
match = None
for collection , regex in collection_res . iteritems ( ) :
if len ( regex ) > matchlen and re . match ( regex , path ) :
matchlen = len ( regex )
match = collection
return match
result = None
bbfiles = ( d . getVar ( ' BBFILES ' , True ) or ' ' ) . split ( )
bbfilesmatch = False
for bbfilesentry in bbfiles :
if fnmatch . fnmatch ( filename , bbfilesentry ) :
bbfilesmatch = True
result = path_to_layer ( bbfilesentry )
if not bbfilesmatch :
# Probably a bbclass
result = path_to_layer ( filename )
return result
2015-09-08 22:31:13 +00:00
# Constant taken from http://linux.die.net/include/linux/prctl.h
PR_SET_PDEATHSIG = 1
class PrCtlError ( Exception ) :
pass
def signal_on_parent_exit ( signame ) :
"""
Trigger signame to be sent when the parent process dies
"""
signum = getattr ( signal , signame )
# http://linux.die.net/man/2/prctl
result = cdll [ ' libc.so.6 ' ] . prctl ( PR_SET_PDEATHSIG , signum )
if result != 0 :
raise PrCtlError ( ' prctl failed with error code %s ' % result )