2005-08-31 10:47:56 +00:00
# ex:ts=4:sw=4:sts=4:et
# -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*-
"""
BitBake Utility Functions
"""
2007-01-08 23:53:01 +00:00
# Copyright (C) 2004 Michael Lauer
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License version 2 as
# published by the Free Software Foundation.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License along
# with this program; if not, write to the Free Software Foundation, Inc.,
# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
2010-04-12 15:14:11 +00:00
import re , fcntl , os , string , stat , shutil , time
2010-04-08 17:22:29 +00:00
import sys
import errno
2010-06-10 17:35:31 +00:00
import logging
import bb
2010-04-08 17:22:29 +00:00
import bb . msg
2012-02-23 13:47:19 +00:00
import multiprocessing
2012-06-22 11:53:16 +00:00
import fcntl
2013-05-17 08:40:01 +00:00
import subprocess
import glob
import traceback
import errno
2010-03-25 00:48:49 +00:00
from commands import getstatusoutput
2010-12-10 01:29:31 +00:00
from contextlib import contextmanager
2005-08-31 10:47:56 +00:00
2010-06-10 17:35:31 +00:00
logger = logging . getLogger ( " BitBake.Util " )
2013-05-23 09:49:57 +00:00
def clean_context ( ) :
return {
" os " : os ,
" bb " : bb ,
" time " : time ,
}
def get_context ( ) :
return _context
def set_context ( ctx ) :
_context = ctx
2010-03-31 03:06:07 +00:00
# Context used in better_exec, eval
2013-05-23 09:49:57 +00:00
_context = clean_context ( )
2010-03-31 03:06:07 +00:00
2005-08-31 10:47:56 +00:00
def explode_version ( s ) :
r = [ ]
alpha_regexp = re . compile ( ' ^([a-zA-Z]+)(.*)$ ' )
numeric_regexp = re . compile ( ' ^( \ d+)(.*)$ ' )
while ( s != ' ' ) :
2010-03-24 03:33:19 +00:00
if s [ 0 ] in string . digits :
2005-08-31 10:47:56 +00:00
m = numeric_regexp . match ( s )
2012-04-12 09:28:01 +00:00
r . append ( ( 0 , int ( m . group ( 1 ) ) ) )
2005-08-31 10:47:56 +00:00
s = m . group ( 2 )
continue
2010-03-24 03:33:19 +00:00
if s [ 0 ] in string . letters :
2005-08-31 10:47:56 +00:00
m = alpha_regexp . match ( s )
2012-04-12 09:28:01 +00:00
r . append ( ( 1 , m . group ( 1 ) ) )
2005-08-31 10:47:56 +00:00
s = m . group ( 2 )
continue
2012-04-12 09:28:01 +00:00
if s [ 0 ] == ' ~ ' :
r . append ( ( - 1 , s [ 0 ] ) )
else :
r . append ( ( 2 , s [ 0 ] ) )
2005-08-31 10:47:56 +00:00
s = s [ 1 : ]
return r
2012-03-20 14:37:16 +00:00
def split_version ( s ) :
""" Split a version string into its constituent parts (PE, PV, PR) """
s = s . strip ( " <>= " )
e = 0
if s . count ( ' : ' ) :
e = int ( s . split ( " : " ) [ 0 ] )
s = s . split ( " : " ) [ 1 ]
r = " "
if s . count ( ' - ' ) :
r = s . rsplit ( " - " , 1 ) [ 1 ]
s = s . rsplit ( " - " , 1 ) [ 0 ]
v = s
return ( e , v , r )
2005-08-31 10:47:56 +00:00
def vercmp_part ( a , b ) :
va = explode_version ( a )
vb = explode_version ( b )
while True :
if va == [ ] :
2012-04-12 09:28:01 +00:00
( oa , ca ) = ( 0 , None )
2005-08-31 10:47:56 +00:00
else :
2012-04-12 09:28:01 +00:00
( oa , ca ) = va . pop ( 0 )
2005-08-31 10:47:56 +00:00
if vb == [ ] :
2012-04-12 09:28:01 +00:00
( ob , cb ) = ( 0 , None )
2005-08-31 10:47:56 +00:00
else :
2012-04-12 09:28:01 +00:00
( ob , cb ) = vb . pop ( 0 )
if ( oa , ca ) == ( 0 , None ) and ( ob , cb ) == ( 0 , None ) :
2005-08-31 10:47:56 +00:00
return 0
2012-04-12 09:28:01 +00:00
if oa < ob :
2010-01-20 18:46:02 +00:00
return - 1
2012-04-12 09:28:01 +00:00
elif oa > ob :
2010-01-20 18:46:02 +00:00
return 1
2012-04-12 09:28:01 +00:00
elif ca < cb :
2005-08-31 10:47:56 +00:00
return - 1
2012-04-12 09:28:01 +00:00
elif ca > cb :
return 1
2005-08-31 10:47:56 +00:00
def vercmp ( ta , tb ) :
2007-04-01 15:04:49 +00:00
( ea , va , ra ) = ta
( eb , vb , rb ) = tb
2005-08-31 10:47:56 +00:00
2010-11-26 21:11:06 +00:00
r = int ( ea or 0 ) - int ( eb or 0 )
2007-04-01 15:04:49 +00:00
if ( r == 0 ) :
r = vercmp_part ( va , vb )
2005-08-31 10:47:56 +00:00
if ( r == 0 ) :
r = vercmp_part ( ra , rb )
return r
2006-02-10 10:11:32 +00:00
2012-05-04 14:03:33 +00:00
def vercmp_string ( a , b ) :
ta = split_version ( a )
tb = split_version ( b )
return vercmp ( ta , tb )
2010-04-06 22:29:19 +00:00
2006-02-10 10:11:32 +00:00
def explode_deps ( s ) :
"""
Take an RDEPENDS style string of format :
" DEPEND1 (optional version) DEPEND2 (optional version) ... "
and return a list of dependencies .
Version information is ignored .
"""
r = [ ]
l = s . split ( )
flag = False
for i in l :
if i [ 0 ] == ' ( ' :
flag = True
2008-03-03 22:01:45 +00:00
#j = []
if not flag :
2006-03-20 17:45:11 +00:00
r . append ( i )
2008-03-03 22:01:45 +00:00
#else:
# j.append(i)
2006-03-20 17:45:11 +00:00
if flag and i . endswith ( ' ) ' ) :
2006-02-10 10:11:32 +00:00
flag = False
# Ignore version
#r[-1] += ' ' + ' '.join(j)
return r
2006-03-20 17:45:11 +00:00
2012-10-01 22:03:43 +00:00
def explode_dep_versions2 ( s ) :
2008-09-03 14:47:31 +00:00
"""
Take an RDEPENDS style string of format :
" DEPEND1 (optional version) DEPEND2 (optional version) ... "
2010-08-03 15:02:43 +00:00
and return a dictionary of dependencies and versions .
2008-09-03 14:47:31 +00:00
"""
r = { }
2010-08-17 08:28:33 +00:00
l = s . replace ( " , " , " " ) . split ( )
2008-09-03 14:47:31 +00:00
lastdep = None
2012-09-30 00:01:45 +00:00
lastcmp = " "
2008-09-03 14:47:31 +00:00
lastver = " "
2012-09-30 00:01:45 +00:00
incmp = False
2008-09-03 14:47:31 +00:00
inversion = False
for i in l :
if i [ 0 ] == ' ( ' :
2012-09-30 00:01:45 +00:00
incmp = True
i = i [ 1 : ] . strip ( )
if not i :
continue
if incmp :
incmp = False
2008-09-03 14:47:31 +00:00
inversion = True
2012-09-30 00:01:45 +00:00
# This list is based on behavior and supported comparisons from deb, opkg and rpm.
#
# Even though =<, <<, ==, !=, =>, and >> may not be supported,
# we list each possibly valid item.
# The build system is responsible for validation of what it supports.
if i . startswith ( ( ' <= ' , ' =< ' , ' << ' , ' == ' , ' != ' , ' >= ' , ' => ' , ' >> ' ) ) :
lastcmp = i [ 0 : 2 ]
i = i [ 2 : ]
elif i . startswith ( ( ' < ' , ' > ' , ' = ' ) ) :
lastcmp = i [ 0 : 1 ]
i = i [ 1 : ]
else :
# This is an unsupported case!
lastcmp = ( i or " " )
i = " "
i . strip ( )
if not i :
continue
if inversion :
if i . endswith ( ' ) ' ) :
i = i [ : - 1 ] or " "
inversion = False
if lastver and i :
lastver + = " "
if i :
lastver + = i
2012-10-01 22:03:43 +00:00
if lastdep not in r :
r [ lastdep ] = [ ]
r [ lastdep ] . append ( lastcmp + " " + lastver )
2012-09-30 00:01:45 +00:00
continue
#if not inversion:
lastdep = i
lastver = " "
lastcmp = " "
if not ( i in r and r [ i ] ) :
2012-10-01 22:03:43 +00:00
r [ lastdep ] = [ ]
return r
2006-03-20 17:45:11 +00:00
2012-10-01 22:03:43 +00:00
def explode_dep_versions ( s ) :
r = explode_dep_versions2 ( s )
for d in r :
if not r [ d ] :
r [ d ] = None
continue
if len ( r [ d ] ) > 1 :
bb . warn ( " explode_dep_versions(): Item %s appeared in dependency string ' %s ' multiple times with different values. explode_dep_versions cannot cope with this. " % ( d , s ) )
r [ d ] = r [ d ] [ 0 ]
2008-09-03 14:47:31 +00:00
return r
2006-03-20 17:45:11 +00:00
2011-02-25 16:31:40 +00:00
def join_deps ( deps , commasep = True ) :
2010-06-30 08:47:36 +00:00
"""
Take the result from explode_dep_versions and generate a dependency string
"""
result = [ ]
for dep in deps :
if deps [ dep ] :
2012-10-01 22:03:43 +00:00
if isinstance ( deps [ dep ] , list ) :
for v in deps [ dep ] :
result . append ( dep + " ( " + v + " ) " )
else :
result . append ( dep + " ( " + deps [ dep ] + " ) " )
2010-06-30 08:47:36 +00:00
else :
result . append ( dep )
2011-02-25 16:31:40 +00:00
if commasep :
return " , " . join ( result )
else :
return " " . join ( result )
2010-06-30 08:47:36 +00:00
2006-03-20 17:45:11 +00:00
def _print_trace ( body , line ) :
"""
Print the Environment of a Text Body
"""
bitbake: utils: Improve better_exec traceback handling
The current bitbake tracebacks are hard to read/confusing and sometimes
incomplete. This patch attempts to do better by:
* Moving the note about the exact exception to the end to make things
read in sequence
* Merged the initial stack trace to become part of the code dump
* Added handling for "/xxxx" file paths since we can load these files
and include the data as part of the trace
* Dropped the ERROR: prefix to every line, allowing the error messages to
be spacially accosicated in the UIs
* Moved the "From:" line to the top of each code block and ensured its present
consistently
With the complexity now in this funciton, I've added try/except wrapping around
it to ensure we catch exceptions in the exception handler too.
Example before:
"""
ERROR: Error executing a python function in /media/build1/poky/meta/recipes-core/eglibc/eglibc-initial_2.17.bb:
TypeError: 'filter' object is not subscriptable
ERROR: The stack trace of python calls that resulted in this exception/failure was:
ERROR: File "do_populate_lic", line 13, in <module>
ERROR:
ERROR: File "do_populate_lic", line 6, in do_populate_lic
ERROR:
ERROR: File "license.bbclass", line 99, in find_license_files
ERROR:
ERROR: File "/media/build1/poky/meta/lib/oe/license.py", line 38, in visit_string
ERROR: if pos > 0 and license_pattern.match(elements[pos-1]):
ERROR:
ERROR: The code that was being executed was:
ERROR: 0009: destdir = os.path.join(d.getVar('LICSSTATEDIR', True), d.getVar('PN', True))
ERROR: 0010: copy_license_files(lic_files_paths, destdir)
ERROR: 0011:
ERROR: 0012:
ERROR: *** 0013:do_populate_lic(d)
ERROR: 0014:
ERROR: [From file: 'do_populate_lic', lineno: 13, function: <module>]
ERROR: 0002:def do_populate_lic(d):
ERROR: 0003: """
ERROR: 0004: Populate LICENSE_DIRECTORY with licenses.
ERROR: 0005: """
ERROR: *** 0006: lic_files_paths = find_license_files(d)
ERROR: 0007:
ERROR: 0008: # The base directory we wrangle licenses to
ERROR: 0009: destdir = os.path.join(d.getVar('LICSSTATEDIR', True), d.getVar('PN', True))
ERROR: 0010: copy_license_files(lic_files_paths, destdir)
ERROR: [From file: 'do_populate_lic', lineno: 6, function: do_populate_lic]
ERROR: 0095: lic_files_paths.append((os.path.basename(path), srclicfile))
ERROR: 0096:
ERROR: 0097: v = FindVisitor()
ERROR: 0098: try:
ERROR: *** 0099: v.visit_string(license_types)
ERROR: 0100: except oe.license.InvalidLicense as exc:
ERROR: 0101: bb.fatal('%s: %s' % (d.getVar('PF', True), exc))
ERROR: 0102: except SyntaxError:
ERROR: 0103: bb.warn("%s: Failed to parse it's LICENSE field." % (d.getVar('PF', True)))
ERROR: [From file: 'license.bbclass', lineno: 99, function: find_license_files]
ERROR: Function failed: do_populate_lic
ERROR: Logfile of failure stored in: /media/build1/poky/build/tmp/work/i586-poky-linux/eglibc-initial/2.17-r3/temp/log.do_populate_lic.17442
"""
Example after:
"""
ERROR: Error executing a python function in /media/build1/poky/meta/recipes-core/eglibc/eglibc-initial_2.17.bb:
The stack trace of python calls that resulted in this exception/failure was:
File: 'do_populate_lic', lineno: 13, function: <module>
0009: destdir = os.path.join(d.getVar('LICSSTATEDIR', True), d.getVar('PN', True))
0010: copy_license_files(lic_files_paths, destdir)
0011:
0012:
*** 0013:do_populate_lic(d)
0014:
File: 'do_populate_lic', lineno: 6, function: do_populate_lic
0002:def do_populate_lic(d):
0003: """
0004: Populate LICENSE_DIRECTORY with licenses.
0005: """
*** 0006: lic_files_paths = find_license_files(d)
0007:
0008: # The base directory we wrangle licenses to
0009: destdir = os.path.join(d.getVar('LICSSTATEDIR', True), d.getVar('PN', True))
0010: copy_license_files(lic_files_paths, destdir)
File: 'license.bbclass', lineno: 99, function: find_license_files
0095: lic_files_paths.append((os.path.basename(path), srclicfile))
0096:
0097: v = FindVisitor()
0098: try:
*** 0099: v.visit_string(license_types)
0100: except oe.license.InvalidLicense as exc:
0101: bb.fatal('%s: %s' % (d.getVar('PF', True), exc))
0102: except SyntaxError:
0103: bb.warn("%s: Failed to parse it's LICENSE field." % (d.getVar('PF', True)))
File: '/media/build1/poky/meta/lib/oe/license.py', lineno: 38, function: visit_string
0034: new_elements = []
0035: elements = filter(lambda x: x.strip(), license_operator.split(licensestr))
0036: for pos, element in enumerate(elements):
0037: if license_pattern.match(element):
*** 0038: if pos > 0 and license_pattern.match(elements[pos-1]):
0039: new_elements.append('&')
0040: element = '"' + element + '"'
0041: elif not license_operator.match(element):
0042: raise InvalidLicense(element)
Exception: TypeError: 'filter' object is not subscriptable
ERROR: Function failed: do_populate_lic
ERROR: Logfile of failure stored in: /media/build1/poky/build/tmp/work/i586-poky-linux/eglibc-initial/2.17-r3/temp/log.do_populate_lic.3275
ERROR: Task 9 (/media/build1/poky/meta/recipes-core/eglibc/eglibc-initial_2.17.bb, do_populate_lic) failed with exit code '1
"""
(Bitbake rev: c5de66b870406d9bd1161a9b7e2b04fe6eb065fe)
Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
2013-05-08 17:14:53 +00:00
error = [ ]
2006-03-20 17:45:11 +00:00
# print the environment of the method
2010-03-24 23:56:12 +00:00
min_line = max ( 1 , line - 4 )
2010-12-05 22:51:38 +00:00
max_line = min ( line + 4 , len ( body ) )
bitbake: utils: Improve better_exec traceback handling
The current bitbake tracebacks are hard to read/confusing and sometimes
incomplete. This patch attempts to do better by:
* Moving the note about the exact exception to the end to make things
read in sequence
* Merged the initial stack trace to become part of the code dump
* Added handling for "/xxxx" file paths since we can load these files
and include the data as part of the trace
* Dropped the ERROR: prefix to every line, allowing the error messages to
be spacially accosicated in the UIs
* Moved the "From:" line to the top of each code block and ensured its present
consistently
With the complexity now in this funciton, I've added try/except wrapping around
it to ensure we catch exceptions in the exception handler too.
Example before:
"""
ERROR: Error executing a python function in /media/build1/poky/meta/recipes-core/eglibc/eglibc-initial_2.17.bb:
TypeError: 'filter' object is not subscriptable
ERROR: The stack trace of python calls that resulted in this exception/failure was:
ERROR: File "do_populate_lic", line 13, in <module>
ERROR:
ERROR: File "do_populate_lic", line 6, in do_populate_lic
ERROR:
ERROR: File "license.bbclass", line 99, in find_license_files
ERROR:
ERROR: File "/media/build1/poky/meta/lib/oe/license.py", line 38, in visit_string
ERROR: if pos > 0 and license_pattern.match(elements[pos-1]):
ERROR:
ERROR: The code that was being executed was:
ERROR: 0009: destdir = os.path.join(d.getVar('LICSSTATEDIR', True), d.getVar('PN', True))
ERROR: 0010: copy_license_files(lic_files_paths, destdir)
ERROR: 0011:
ERROR: 0012:
ERROR: *** 0013:do_populate_lic(d)
ERROR: 0014:
ERROR: [From file: 'do_populate_lic', lineno: 13, function: <module>]
ERROR: 0002:def do_populate_lic(d):
ERROR: 0003: """
ERROR: 0004: Populate LICENSE_DIRECTORY with licenses.
ERROR: 0005: """
ERROR: *** 0006: lic_files_paths = find_license_files(d)
ERROR: 0007:
ERROR: 0008: # The base directory we wrangle licenses to
ERROR: 0009: destdir = os.path.join(d.getVar('LICSSTATEDIR', True), d.getVar('PN', True))
ERROR: 0010: copy_license_files(lic_files_paths, destdir)
ERROR: [From file: 'do_populate_lic', lineno: 6, function: do_populate_lic]
ERROR: 0095: lic_files_paths.append((os.path.basename(path), srclicfile))
ERROR: 0096:
ERROR: 0097: v = FindVisitor()
ERROR: 0098: try:
ERROR: *** 0099: v.visit_string(license_types)
ERROR: 0100: except oe.license.InvalidLicense as exc:
ERROR: 0101: bb.fatal('%s: %s' % (d.getVar('PF', True), exc))
ERROR: 0102: except SyntaxError:
ERROR: 0103: bb.warn("%s: Failed to parse it's LICENSE field." % (d.getVar('PF', True)))
ERROR: [From file: 'license.bbclass', lineno: 99, function: find_license_files]
ERROR: Function failed: do_populate_lic
ERROR: Logfile of failure stored in: /media/build1/poky/build/tmp/work/i586-poky-linux/eglibc-initial/2.17-r3/temp/log.do_populate_lic.17442
"""
Example after:
"""
ERROR: Error executing a python function in /media/build1/poky/meta/recipes-core/eglibc/eglibc-initial_2.17.bb:
The stack trace of python calls that resulted in this exception/failure was:
File: 'do_populate_lic', lineno: 13, function: <module>
0009: destdir = os.path.join(d.getVar('LICSSTATEDIR', True), d.getVar('PN', True))
0010: copy_license_files(lic_files_paths, destdir)
0011:
0012:
*** 0013:do_populate_lic(d)
0014:
File: 'do_populate_lic', lineno: 6, function: do_populate_lic
0002:def do_populate_lic(d):
0003: """
0004: Populate LICENSE_DIRECTORY with licenses.
0005: """
*** 0006: lic_files_paths = find_license_files(d)
0007:
0008: # The base directory we wrangle licenses to
0009: destdir = os.path.join(d.getVar('LICSSTATEDIR', True), d.getVar('PN', True))
0010: copy_license_files(lic_files_paths, destdir)
File: 'license.bbclass', lineno: 99, function: find_license_files
0095: lic_files_paths.append((os.path.basename(path), srclicfile))
0096:
0097: v = FindVisitor()
0098: try:
*** 0099: v.visit_string(license_types)
0100: except oe.license.InvalidLicense as exc:
0101: bb.fatal('%s: %s' % (d.getVar('PF', True), exc))
0102: except SyntaxError:
0103: bb.warn("%s: Failed to parse it's LICENSE field." % (d.getVar('PF', True)))
File: '/media/build1/poky/meta/lib/oe/license.py', lineno: 38, function: visit_string
0034: new_elements = []
0035: elements = filter(lambda x: x.strip(), license_operator.split(licensestr))
0036: for pos, element in enumerate(elements):
0037: if license_pattern.match(element):
*** 0038: if pos > 0 and license_pattern.match(elements[pos-1]):
0039: new_elements.append('&')
0040: element = '"' + element + '"'
0041: elif not license_operator.match(element):
0042: raise InvalidLicense(element)
Exception: TypeError: 'filter' object is not subscriptable
ERROR: Function failed: do_populate_lic
ERROR: Logfile of failure stored in: /media/build1/poky/build/tmp/work/i586-poky-linux/eglibc-initial/2.17-r3/temp/log.do_populate_lic.3275
ERROR: Task 9 (/media/build1/poky/meta/recipes-core/eglibc/eglibc-initial_2.17.bb, do_populate_lic) failed with exit code '1
"""
(Bitbake rev: c5de66b870406d9bd1161a9b7e2b04fe6eb065fe)
Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
2013-05-08 17:14:53 +00:00
for i in range ( min_line , max_line + 1 ) :
2010-12-05 22:51:38 +00:00
if line == i :
bitbake: utils: Improve better_exec traceback handling
The current bitbake tracebacks are hard to read/confusing and sometimes
incomplete. This patch attempts to do better by:
* Moving the note about the exact exception to the end to make things
read in sequence
* Merged the initial stack trace to become part of the code dump
* Added handling for "/xxxx" file paths since we can load these files
and include the data as part of the trace
* Dropped the ERROR: prefix to every line, allowing the error messages to
be spacially accosicated in the UIs
* Moved the "From:" line to the top of each code block and ensured its present
consistently
With the complexity now in this funciton, I've added try/except wrapping around
it to ensure we catch exceptions in the exception handler too.
Example before:
"""
ERROR: Error executing a python function in /media/build1/poky/meta/recipes-core/eglibc/eglibc-initial_2.17.bb:
TypeError: 'filter' object is not subscriptable
ERROR: The stack trace of python calls that resulted in this exception/failure was:
ERROR: File "do_populate_lic", line 13, in <module>
ERROR:
ERROR: File "do_populate_lic", line 6, in do_populate_lic
ERROR:
ERROR: File "license.bbclass", line 99, in find_license_files
ERROR:
ERROR: File "/media/build1/poky/meta/lib/oe/license.py", line 38, in visit_string
ERROR: if pos > 0 and license_pattern.match(elements[pos-1]):
ERROR:
ERROR: The code that was being executed was:
ERROR: 0009: destdir = os.path.join(d.getVar('LICSSTATEDIR', True), d.getVar('PN', True))
ERROR: 0010: copy_license_files(lic_files_paths, destdir)
ERROR: 0011:
ERROR: 0012:
ERROR: *** 0013:do_populate_lic(d)
ERROR: 0014:
ERROR: [From file: 'do_populate_lic', lineno: 13, function: <module>]
ERROR: 0002:def do_populate_lic(d):
ERROR: 0003: """
ERROR: 0004: Populate LICENSE_DIRECTORY with licenses.
ERROR: 0005: """
ERROR: *** 0006: lic_files_paths = find_license_files(d)
ERROR: 0007:
ERROR: 0008: # The base directory we wrangle licenses to
ERROR: 0009: destdir = os.path.join(d.getVar('LICSSTATEDIR', True), d.getVar('PN', True))
ERROR: 0010: copy_license_files(lic_files_paths, destdir)
ERROR: [From file: 'do_populate_lic', lineno: 6, function: do_populate_lic]
ERROR: 0095: lic_files_paths.append((os.path.basename(path), srclicfile))
ERROR: 0096:
ERROR: 0097: v = FindVisitor()
ERROR: 0098: try:
ERROR: *** 0099: v.visit_string(license_types)
ERROR: 0100: except oe.license.InvalidLicense as exc:
ERROR: 0101: bb.fatal('%s: %s' % (d.getVar('PF', True), exc))
ERROR: 0102: except SyntaxError:
ERROR: 0103: bb.warn("%s: Failed to parse it's LICENSE field." % (d.getVar('PF', True)))
ERROR: [From file: 'license.bbclass', lineno: 99, function: find_license_files]
ERROR: Function failed: do_populate_lic
ERROR: Logfile of failure stored in: /media/build1/poky/build/tmp/work/i586-poky-linux/eglibc-initial/2.17-r3/temp/log.do_populate_lic.17442
"""
Example after:
"""
ERROR: Error executing a python function in /media/build1/poky/meta/recipes-core/eglibc/eglibc-initial_2.17.bb:
The stack trace of python calls that resulted in this exception/failure was:
File: 'do_populate_lic', lineno: 13, function: <module>
0009: destdir = os.path.join(d.getVar('LICSSTATEDIR', True), d.getVar('PN', True))
0010: copy_license_files(lic_files_paths, destdir)
0011:
0012:
*** 0013:do_populate_lic(d)
0014:
File: 'do_populate_lic', lineno: 6, function: do_populate_lic
0002:def do_populate_lic(d):
0003: """
0004: Populate LICENSE_DIRECTORY with licenses.
0005: """
*** 0006: lic_files_paths = find_license_files(d)
0007:
0008: # The base directory we wrangle licenses to
0009: destdir = os.path.join(d.getVar('LICSSTATEDIR', True), d.getVar('PN', True))
0010: copy_license_files(lic_files_paths, destdir)
File: 'license.bbclass', lineno: 99, function: find_license_files
0095: lic_files_paths.append((os.path.basename(path), srclicfile))
0096:
0097: v = FindVisitor()
0098: try:
*** 0099: v.visit_string(license_types)
0100: except oe.license.InvalidLicense as exc:
0101: bb.fatal('%s: %s' % (d.getVar('PF', True), exc))
0102: except SyntaxError:
0103: bb.warn("%s: Failed to parse it's LICENSE field." % (d.getVar('PF', True)))
File: '/media/build1/poky/meta/lib/oe/license.py', lineno: 38, function: visit_string
0034: new_elements = []
0035: elements = filter(lambda x: x.strip(), license_operator.split(licensestr))
0036: for pos, element in enumerate(elements):
0037: if license_pattern.match(element):
*** 0038: if pos > 0 and license_pattern.match(elements[pos-1]):
0039: new_elements.append('&')
0040: element = '"' + element + '"'
0041: elif not license_operator.match(element):
0042: raise InvalidLicense(element)
Exception: TypeError: 'filter' object is not subscriptable
ERROR: Function failed: do_populate_lic
ERROR: Logfile of failure stored in: /media/build1/poky/build/tmp/work/i586-poky-linux/eglibc-initial/2.17-r3/temp/log.do_populate_lic.3275
ERROR: Task 9 (/media/build1/poky/meta/recipes-core/eglibc/eglibc-initial_2.17.bb, do_populate_lic) failed with exit code '1
"""
(Bitbake rev: c5de66b870406d9bd1161a9b7e2b04fe6eb065fe)
Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
2013-05-08 17:14:53 +00:00
error . append ( ' *** %.4d : %s ' % ( i , body [ i - 1 ] . rstrip ( ) ) )
2010-12-05 22:51:38 +00:00
else :
bitbake: utils: Improve better_exec traceback handling
The current bitbake tracebacks are hard to read/confusing and sometimes
incomplete. This patch attempts to do better by:
* Moving the note about the exact exception to the end to make things
read in sequence
* Merged the initial stack trace to become part of the code dump
* Added handling for "/xxxx" file paths since we can load these files
and include the data as part of the trace
* Dropped the ERROR: prefix to every line, allowing the error messages to
be spacially accosicated in the UIs
* Moved the "From:" line to the top of each code block and ensured its present
consistently
With the complexity now in this funciton, I've added try/except wrapping around
it to ensure we catch exceptions in the exception handler too.
Example before:
"""
ERROR: Error executing a python function in /media/build1/poky/meta/recipes-core/eglibc/eglibc-initial_2.17.bb:
TypeError: 'filter' object is not subscriptable
ERROR: The stack trace of python calls that resulted in this exception/failure was:
ERROR: File "do_populate_lic", line 13, in <module>
ERROR:
ERROR: File "do_populate_lic", line 6, in do_populate_lic
ERROR:
ERROR: File "license.bbclass", line 99, in find_license_files
ERROR:
ERROR: File "/media/build1/poky/meta/lib/oe/license.py", line 38, in visit_string
ERROR: if pos > 0 and license_pattern.match(elements[pos-1]):
ERROR:
ERROR: The code that was being executed was:
ERROR: 0009: destdir = os.path.join(d.getVar('LICSSTATEDIR', True), d.getVar('PN', True))
ERROR: 0010: copy_license_files(lic_files_paths, destdir)
ERROR: 0011:
ERROR: 0012:
ERROR: *** 0013:do_populate_lic(d)
ERROR: 0014:
ERROR: [From file: 'do_populate_lic', lineno: 13, function: <module>]
ERROR: 0002:def do_populate_lic(d):
ERROR: 0003: """
ERROR: 0004: Populate LICENSE_DIRECTORY with licenses.
ERROR: 0005: """
ERROR: *** 0006: lic_files_paths = find_license_files(d)
ERROR: 0007:
ERROR: 0008: # The base directory we wrangle licenses to
ERROR: 0009: destdir = os.path.join(d.getVar('LICSSTATEDIR', True), d.getVar('PN', True))
ERROR: 0010: copy_license_files(lic_files_paths, destdir)
ERROR: [From file: 'do_populate_lic', lineno: 6, function: do_populate_lic]
ERROR: 0095: lic_files_paths.append((os.path.basename(path), srclicfile))
ERROR: 0096:
ERROR: 0097: v = FindVisitor()
ERROR: 0098: try:
ERROR: *** 0099: v.visit_string(license_types)
ERROR: 0100: except oe.license.InvalidLicense as exc:
ERROR: 0101: bb.fatal('%s: %s' % (d.getVar('PF', True), exc))
ERROR: 0102: except SyntaxError:
ERROR: 0103: bb.warn("%s: Failed to parse it's LICENSE field." % (d.getVar('PF', True)))
ERROR: [From file: 'license.bbclass', lineno: 99, function: find_license_files]
ERROR: Function failed: do_populate_lic
ERROR: Logfile of failure stored in: /media/build1/poky/build/tmp/work/i586-poky-linux/eglibc-initial/2.17-r3/temp/log.do_populate_lic.17442
"""
Example after:
"""
ERROR: Error executing a python function in /media/build1/poky/meta/recipes-core/eglibc/eglibc-initial_2.17.bb:
The stack trace of python calls that resulted in this exception/failure was:
File: 'do_populate_lic', lineno: 13, function: <module>
0009: destdir = os.path.join(d.getVar('LICSSTATEDIR', True), d.getVar('PN', True))
0010: copy_license_files(lic_files_paths, destdir)
0011:
0012:
*** 0013:do_populate_lic(d)
0014:
File: 'do_populate_lic', lineno: 6, function: do_populate_lic
0002:def do_populate_lic(d):
0003: """
0004: Populate LICENSE_DIRECTORY with licenses.
0005: """
*** 0006: lic_files_paths = find_license_files(d)
0007:
0008: # The base directory we wrangle licenses to
0009: destdir = os.path.join(d.getVar('LICSSTATEDIR', True), d.getVar('PN', True))
0010: copy_license_files(lic_files_paths, destdir)
File: 'license.bbclass', lineno: 99, function: find_license_files
0095: lic_files_paths.append((os.path.basename(path), srclicfile))
0096:
0097: v = FindVisitor()
0098: try:
*** 0099: v.visit_string(license_types)
0100: except oe.license.InvalidLicense as exc:
0101: bb.fatal('%s: %s' % (d.getVar('PF', True), exc))
0102: except SyntaxError:
0103: bb.warn("%s: Failed to parse it's LICENSE field." % (d.getVar('PF', True)))
File: '/media/build1/poky/meta/lib/oe/license.py', lineno: 38, function: visit_string
0034: new_elements = []
0035: elements = filter(lambda x: x.strip(), license_operator.split(licensestr))
0036: for pos, element in enumerate(elements):
0037: if license_pattern.match(element):
*** 0038: if pos > 0 and license_pattern.match(elements[pos-1]):
0039: new_elements.append('&')
0040: element = '"' + element + '"'
0041: elif not license_operator.match(element):
0042: raise InvalidLicense(element)
Exception: TypeError: 'filter' object is not subscriptable
ERROR: Function failed: do_populate_lic
ERROR: Logfile of failure stored in: /media/build1/poky/build/tmp/work/i586-poky-linux/eglibc-initial/2.17-r3/temp/log.do_populate_lic.3275
ERROR: Task 9 (/media/build1/poky/meta/recipes-core/eglibc/eglibc-initial_2.17.bb, do_populate_lic) failed with exit code '1
"""
(Bitbake rev: c5de66b870406d9bd1161a9b7e2b04fe6eb065fe)
Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
2013-05-08 17:14:53 +00:00
error . append ( ' %.4d : %s ' % ( i , body [ i - 1 ] . rstrip ( ) ) )
return error
2006-03-20 17:45:11 +00:00
2010-03-31 03:06:07 +00:00
def better_compile ( text , file , realfile , mode = " exec " ) :
2006-03-20 17:45:11 +00:00
"""
A better compile method . This method
will print the offending lines .
"""
try :
2010-03-31 03:06:07 +00:00
return compile ( text , file , mode )
2010-04-12 00:03:55 +00:00
except Exception as e :
2013-05-08 17:16:00 +00:00
error = [ ]
2006-03-20 17:45:11 +00:00
# split the text into lines again
body = text . split ( ' \n ' )
2013-05-08 17:16:00 +00:00
error . append ( " Error in compiling python function in %s : \n " % realfile )
2010-08-19 22:26:46 +00:00
if e . lineno :
2013-05-08 17:16:00 +00:00
error . append ( " The code lines resulting in this error were: " )
error . extend ( _print_trace ( body , e . lineno ) )
2010-08-19 22:26:46 +00:00
else :
2013-05-08 17:16:00 +00:00
error . append ( " The function causing this error was: " )
2010-08-19 22:26:46 +00:00
for line in body :
2013-05-08 17:16:00 +00:00
error . append ( line )
error . append ( " %s : %s " % ( e . __class__ . __name__ , str ( e ) ) )
logger . error ( " \n " . join ( error ) )
2011-01-01 23:55:54 +00:00
2012-08-22 19:02:39 +00:00
e = bb . BBHandledException ( e )
raise e
2006-03-20 17:45:11 +00:00
bitbake: utils: Improve better_exec traceback handling
The current bitbake tracebacks are hard to read/confusing and sometimes
incomplete. This patch attempts to do better by:
* Moving the note about the exact exception to the end to make things
read in sequence
* Merged the initial stack trace to become part of the code dump
* Added handling for "/xxxx" file paths since we can load these files
and include the data as part of the trace
* Dropped the ERROR: prefix to every line, allowing the error messages to
be spacially accosicated in the UIs
* Moved the "From:" line to the top of each code block and ensured its present
consistently
With the complexity now in this funciton, I've added try/except wrapping around
it to ensure we catch exceptions in the exception handler too.
Example before:
"""
ERROR: Error executing a python function in /media/build1/poky/meta/recipes-core/eglibc/eglibc-initial_2.17.bb:
TypeError: 'filter' object is not subscriptable
ERROR: The stack trace of python calls that resulted in this exception/failure was:
ERROR: File "do_populate_lic", line 13, in <module>
ERROR:
ERROR: File "do_populate_lic", line 6, in do_populate_lic
ERROR:
ERROR: File "license.bbclass", line 99, in find_license_files
ERROR:
ERROR: File "/media/build1/poky/meta/lib/oe/license.py", line 38, in visit_string
ERROR: if pos > 0 and license_pattern.match(elements[pos-1]):
ERROR:
ERROR: The code that was being executed was:
ERROR: 0009: destdir = os.path.join(d.getVar('LICSSTATEDIR', True), d.getVar('PN', True))
ERROR: 0010: copy_license_files(lic_files_paths, destdir)
ERROR: 0011:
ERROR: 0012:
ERROR: *** 0013:do_populate_lic(d)
ERROR: 0014:
ERROR: [From file: 'do_populate_lic', lineno: 13, function: <module>]
ERROR: 0002:def do_populate_lic(d):
ERROR: 0003: """
ERROR: 0004: Populate LICENSE_DIRECTORY with licenses.
ERROR: 0005: """
ERROR: *** 0006: lic_files_paths = find_license_files(d)
ERROR: 0007:
ERROR: 0008: # The base directory we wrangle licenses to
ERROR: 0009: destdir = os.path.join(d.getVar('LICSSTATEDIR', True), d.getVar('PN', True))
ERROR: 0010: copy_license_files(lic_files_paths, destdir)
ERROR: [From file: 'do_populate_lic', lineno: 6, function: do_populate_lic]
ERROR: 0095: lic_files_paths.append((os.path.basename(path), srclicfile))
ERROR: 0096:
ERROR: 0097: v = FindVisitor()
ERROR: 0098: try:
ERROR: *** 0099: v.visit_string(license_types)
ERROR: 0100: except oe.license.InvalidLicense as exc:
ERROR: 0101: bb.fatal('%s: %s' % (d.getVar('PF', True), exc))
ERROR: 0102: except SyntaxError:
ERROR: 0103: bb.warn("%s: Failed to parse it's LICENSE field." % (d.getVar('PF', True)))
ERROR: [From file: 'license.bbclass', lineno: 99, function: find_license_files]
ERROR: Function failed: do_populate_lic
ERROR: Logfile of failure stored in: /media/build1/poky/build/tmp/work/i586-poky-linux/eglibc-initial/2.17-r3/temp/log.do_populate_lic.17442
"""
Example after:
"""
ERROR: Error executing a python function in /media/build1/poky/meta/recipes-core/eglibc/eglibc-initial_2.17.bb:
The stack trace of python calls that resulted in this exception/failure was:
File: 'do_populate_lic', lineno: 13, function: <module>
0009: destdir = os.path.join(d.getVar('LICSSTATEDIR', True), d.getVar('PN', True))
0010: copy_license_files(lic_files_paths, destdir)
0011:
0012:
*** 0013:do_populate_lic(d)
0014:
File: 'do_populate_lic', lineno: 6, function: do_populate_lic
0002:def do_populate_lic(d):
0003: """
0004: Populate LICENSE_DIRECTORY with licenses.
0005: """
*** 0006: lic_files_paths = find_license_files(d)
0007:
0008: # The base directory we wrangle licenses to
0009: destdir = os.path.join(d.getVar('LICSSTATEDIR', True), d.getVar('PN', True))
0010: copy_license_files(lic_files_paths, destdir)
File: 'license.bbclass', lineno: 99, function: find_license_files
0095: lic_files_paths.append((os.path.basename(path), srclicfile))
0096:
0097: v = FindVisitor()
0098: try:
*** 0099: v.visit_string(license_types)
0100: except oe.license.InvalidLicense as exc:
0101: bb.fatal('%s: %s' % (d.getVar('PF', True), exc))
0102: except SyntaxError:
0103: bb.warn("%s: Failed to parse it's LICENSE field." % (d.getVar('PF', True)))
File: '/media/build1/poky/meta/lib/oe/license.py', lineno: 38, function: visit_string
0034: new_elements = []
0035: elements = filter(lambda x: x.strip(), license_operator.split(licensestr))
0036: for pos, element in enumerate(elements):
0037: if license_pattern.match(element):
*** 0038: if pos > 0 and license_pattern.match(elements[pos-1]):
0039: new_elements.append('&')
0040: element = '"' + element + '"'
0041: elif not license_operator.match(element):
0042: raise InvalidLicense(element)
Exception: TypeError: 'filter' object is not subscriptable
ERROR: Function failed: do_populate_lic
ERROR: Logfile of failure stored in: /media/build1/poky/build/tmp/work/i586-poky-linux/eglibc-initial/2.17-r3/temp/log.do_populate_lic.3275
ERROR: Task 9 (/media/build1/poky/meta/recipes-core/eglibc/eglibc-initial_2.17.bb, do_populate_lic) failed with exit code '1
"""
(Bitbake rev: c5de66b870406d9bd1161a9b7e2b04fe6eb065fe)
Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
2013-05-08 17:14:53 +00:00
def _print_exception ( t , value , tb , realfile , text , context ) :
error = [ ]
2006-03-20 17:45:11 +00:00
try :
2010-12-22 19:06:54 +00:00
exception = traceback . format_exception_only ( t , value )
bitbake: utils: Improve better_exec traceback handling
The current bitbake tracebacks are hard to read/confusing and sometimes
incomplete. This patch attempts to do better by:
* Moving the note about the exact exception to the end to make things
read in sequence
* Merged the initial stack trace to become part of the code dump
* Added handling for "/xxxx" file paths since we can load these files
and include the data as part of the trace
* Dropped the ERROR: prefix to every line, allowing the error messages to
be spacially accosicated in the UIs
* Moved the "From:" line to the top of each code block and ensured its present
consistently
With the complexity now in this funciton, I've added try/except wrapping around
it to ensure we catch exceptions in the exception handler too.
Example before:
"""
ERROR: Error executing a python function in /media/build1/poky/meta/recipes-core/eglibc/eglibc-initial_2.17.bb:
TypeError: 'filter' object is not subscriptable
ERROR: The stack trace of python calls that resulted in this exception/failure was:
ERROR: File "do_populate_lic", line 13, in <module>
ERROR:
ERROR: File "do_populate_lic", line 6, in do_populate_lic
ERROR:
ERROR: File "license.bbclass", line 99, in find_license_files
ERROR:
ERROR: File "/media/build1/poky/meta/lib/oe/license.py", line 38, in visit_string
ERROR: if pos > 0 and license_pattern.match(elements[pos-1]):
ERROR:
ERROR: The code that was being executed was:
ERROR: 0009: destdir = os.path.join(d.getVar('LICSSTATEDIR', True), d.getVar('PN', True))
ERROR: 0010: copy_license_files(lic_files_paths, destdir)
ERROR: 0011:
ERROR: 0012:
ERROR: *** 0013:do_populate_lic(d)
ERROR: 0014:
ERROR: [From file: 'do_populate_lic', lineno: 13, function: <module>]
ERROR: 0002:def do_populate_lic(d):
ERROR: 0003: """
ERROR: 0004: Populate LICENSE_DIRECTORY with licenses.
ERROR: 0005: """
ERROR: *** 0006: lic_files_paths = find_license_files(d)
ERROR: 0007:
ERROR: 0008: # The base directory we wrangle licenses to
ERROR: 0009: destdir = os.path.join(d.getVar('LICSSTATEDIR', True), d.getVar('PN', True))
ERROR: 0010: copy_license_files(lic_files_paths, destdir)
ERROR: [From file: 'do_populate_lic', lineno: 6, function: do_populate_lic]
ERROR: 0095: lic_files_paths.append((os.path.basename(path), srclicfile))
ERROR: 0096:
ERROR: 0097: v = FindVisitor()
ERROR: 0098: try:
ERROR: *** 0099: v.visit_string(license_types)
ERROR: 0100: except oe.license.InvalidLicense as exc:
ERROR: 0101: bb.fatal('%s: %s' % (d.getVar('PF', True), exc))
ERROR: 0102: except SyntaxError:
ERROR: 0103: bb.warn("%s: Failed to parse it's LICENSE field." % (d.getVar('PF', True)))
ERROR: [From file: 'license.bbclass', lineno: 99, function: find_license_files]
ERROR: Function failed: do_populate_lic
ERROR: Logfile of failure stored in: /media/build1/poky/build/tmp/work/i586-poky-linux/eglibc-initial/2.17-r3/temp/log.do_populate_lic.17442
"""
Example after:
"""
ERROR: Error executing a python function in /media/build1/poky/meta/recipes-core/eglibc/eglibc-initial_2.17.bb:
The stack trace of python calls that resulted in this exception/failure was:
File: 'do_populate_lic', lineno: 13, function: <module>
0009: destdir = os.path.join(d.getVar('LICSSTATEDIR', True), d.getVar('PN', True))
0010: copy_license_files(lic_files_paths, destdir)
0011:
0012:
*** 0013:do_populate_lic(d)
0014:
File: 'do_populate_lic', lineno: 6, function: do_populate_lic
0002:def do_populate_lic(d):
0003: """
0004: Populate LICENSE_DIRECTORY with licenses.
0005: """
*** 0006: lic_files_paths = find_license_files(d)
0007:
0008: # The base directory we wrangle licenses to
0009: destdir = os.path.join(d.getVar('LICSSTATEDIR', True), d.getVar('PN', True))
0010: copy_license_files(lic_files_paths, destdir)
File: 'license.bbclass', lineno: 99, function: find_license_files
0095: lic_files_paths.append((os.path.basename(path), srclicfile))
0096:
0097: v = FindVisitor()
0098: try:
*** 0099: v.visit_string(license_types)
0100: except oe.license.InvalidLicense as exc:
0101: bb.fatal('%s: %s' % (d.getVar('PF', True), exc))
0102: except SyntaxError:
0103: bb.warn("%s: Failed to parse it's LICENSE field." % (d.getVar('PF', True)))
File: '/media/build1/poky/meta/lib/oe/license.py', lineno: 38, function: visit_string
0034: new_elements = []
0035: elements = filter(lambda x: x.strip(), license_operator.split(licensestr))
0036: for pos, element in enumerate(elements):
0037: if license_pattern.match(element):
*** 0038: if pos > 0 and license_pattern.match(elements[pos-1]):
0039: new_elements.append('&')
0040: element = '"' + element + '"'
0041: elif not license_operator.match(element):
0042: raise InvalidLicense(element)
Exception: TypeError: 'filter' object is not subscriptable
ERROR: Function failed: do_populate_lic
ERROR: Logfile of failure stored in: /media/build1/poky/build/tmp/work/i586-poky-linux/eglibc-initial/2.17-r3/temp/log.do_populate_lic.3275
ERROR: Task 9 (/media/build1/poky/meta/recipes-core/eglibc/eglibc-initial_2.17.bb, do_populate_lic) failed with exit code '1
"""
(Bitbake rev: c5de66b870406d9bd1161a9b7e2b04fe6eb065fe)
Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
2013-05-08 17:14:53 +00:00
error . append ( ' Error executing a python function in %s : \n ' % realfile )
2006-03-20 17:45:11 +00:00
2010-08-20 11:52:33 +00:00
# Strip 'us' from the stack (better_exec call)
tb = tb . tb_next
2006-03-20 17:45:11 +00:00
2010-12-05 22:52:44 +00:00
textarray = text . split ( ' \n ' )
bitbake: utils: Improve better_exec traceback handling
The current bitbake tracebacks are hard to read/confusing and sometimes
incomplete. This patch attempts to do better by:
* Moving the note about the exact exception to the end to make things
read in sequence
* Merged the initial stack trace to become part of the code dump
* Added handling for "/xxxx" file paths since we can load these files
and include the data as part of the trace
* Dropped the ERROR: prefix to every line, allowing the error messages to
be spacially accosicated in the UIs
* Moved the "From:" line to the top of each code block and ensured its present
consistently
With the complexity now in this funciton, I've added try/except wrapping around
it to ensure we catch exceptions in the exception handler too.
Example before:
"""
ERROR: Error executing a python function in /media/build1/poky/meta/recipes-core/eglibc/eglibc-initial_2.17.bb:
TypeError: 'filter' object is not subscriptable
ERROR: The stack trace of python calls that resulted in this exception/failure was:
ERROR: File "do_populate_lic", line 13, in <module>
ERROR:
ERROR: File "do_populate_lic", line 6, in do_populate_lic
ERROR:
ERROR: File "license.bbclass", line 99, in find_license_files
ERROR:
ERROR: File "/media/build1/poky/meta/lib/oe/license.py", line 38, in visit_string
ERROR: if pos > 0 and license_pattern.match(elements[pos-1]):
ERROR:
ERROR: The code that was being executed was:
ERROR: 0009: destdir = os.path.join(d.getVar('LICSSTATEDIR', True), d.getVar('PN', True))
ERROR: 0010: copy_license_files(lic_files_paths, destdir)
ERROR: 0011:
ERROR: 0012:
ERROR: *** 0013:do_populate_lic(d)
ERROR: 0014:
ERROR: [From file: 'do_populate_lic', lineno: 13, function: <module>]
ERROR: 0002:def do_populate_lic(d):
ERROR: 0003: """
ERROR: 0004: Populate LICENSE_DIRECTORY with licenses.
ERROR: 0005: """
ERROR: *** 0006: lic_files_paths = find_license_files(d)
ERROR: 0007:
ERROR: 0008: # The base directory we wrangle licenses to
ERROR: 0009: destdir = os.path.join(d.getVar('LICSSTATEDIR', True), d.getVar('PN', True))
ERROR: 0010: copy_license_files(lic_files_paths, destdir)
ERROR: [From file: 'do_populate_lic', lineno: 6, function: do_populate_lic]
ERROR: 0095: lic_files_paths.append((os.path.basename(path), srclicfile))
ERROR: 0096:
ERROR: 0097: v = FindVisitor()
ERROR: 0098: try:
ERROR: *** 0099: v.visit_string(license_types)
ERROR: 0100: except oe.license.InvalidLicense as exc:
ERROR: 0101: bb.fatal('%s: %s' % (d.getVar('PF', True), exc))
ERROR: 0102: except SyntaxError:
ERROR: 0103: bb.warn("%s: Failed to parse it's LICENSE field." % (d.getVar('PF', True)))
ERROR: [From file: 'license.bbclass', lineno: 99, function: find_license_files]
ERROR: Function failed: do_populate_lic
ERROR: Logfile of failure stored in: /media/build1/poky/build/tmp/work/i586-poky-linux/eglibc-initial/2.17-r3/temp/log.do_populate_lic.17442
"""
Example after:
"""
ERROR: Error executing a python function in /media/build1/poky/meta/recipes-core/eglibc/eglibc-initial_2.17.bb:
The stack trace of python calls that resulted in this exception/failure was:
File: 'do_populate_lic', lineno: 13, function: <module>
0009: destdir = os.path.join(d.getVar('LICSSTATEDIR', True), d.getVar('PN', True))
0010: copy_license_files(lic_files_paths, destdir)
0011:
0012:
*** 0013:do_populate_lic(d)
0014:
File: 'do_populate_lic', lineno: 6, function: do_populate_lic
0002:def do_populate_lic(d):
0003: """
0004: Populate LICENSE_DIRECTORY with licenses.
0005: """
*** 0006: lic_files_paths = find_license_files(d)
0007:
0008: # The base directory we wrangle licenses to
0009: destdir = os.path.join(d.getVar('LICSSTATEDIR', True), d.getVar('PN', True))
0010: copy_license_files(lic_files_paths, destdir)
File: 'license.bbclass', lineno: 99, function: find_license_files
0095: lic_files_paths.append((os.path.basename(path), srclicfile))
0096:
0097: v = FindVisitor()
0098: try:
*** 0099: v.visit_string(license_types)
0100: except oe.license.InvalidLicense as exc:
0101: bb.fatal('%s: %s' % (d.getVar('PF', True), exc))
0102: except SyntaxError:
0103: bb.warn("%s: Failed to parse it's LICENSE field." % (d.getVar('PF', True)))
File: '/media/build1/poky/meta/lib/oe/license.py', lineno: 38, function: visit_string
0034: new_elements = []
0035: elements = filter(lambda x: x.strip(), license_operator.split(licensestr))
0036: for pos, element in enumerate(elements):
0037: if license_pattern.match(element):
*** 0038: if pos > 0 and license_pattern.match(elements[pos-1]):
0039: new_elements.append('&')
0040: element = '"' + element + '"'
0041: elif not license_operator.match(element):
0042: raise InvalidLicense(element)
Exception: TypeError: 'filter' object is not subscriptable
ERROR: Function failed: do_populate_lic
ERROR: Logfile of failure stored in: /media/build1/poky/build/tmp/work/i586-poky-linux/eglibc-initial/2.17-r3/temp/log.do_populate_lic.3275
ERROR: Task 9 (/media/build1/poky/meta/recipes-core/eglibc/eglibc-initial_2.17.bb, do_populate_lic) failed with exit code '1
"""
(Bitbake rev: c5de66b870406d9bd1161a9b7e2b04fe6eb065fe)
Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
2013-05-08 17:14:53 +00:00
linefailed = tb . tb_lineno
2006-03-20 17:45:11 +00:00
bitbake: utils: Improve better_exec traceback handling
The current bitbake tracebacks are hard to read/confusing and sometimes
incomplete. This patch attempts to do better by:
* Moving the note about the exact exception to the end to make things
read in sequence
* Merged the initial stack trace to become part of the code dump
* Added handling for "/xxxx" file paths since we can load these files
and include the data as part of the trace
* Dropped the ERROR: prefix to every line, allowing the error messages to
be spacially accosicated in the UIs
* Moved the "From:" line to the top of each code block and ensured its present
consistently
With the complexity now in this funciton, I've added try/except wrapping around
it to ensure we catch exceptions in the exception handler too.
Example before:
"""
ERROR: Error executing a python function in /media/build1/poky/meta/recipes-core/eglibc/eglibc-initial_2.17.bb:
TypeError: 'filter' object is not subscriptable
ERROR: The stack trace of python calls that resulted in this exception/failure was:
ERROR: File "do_populate_lic", line 13, in <module>
ERROR:
ERROR: File "do_populate_lic", line 6, in do_populate_lic
ERROR:
ERROR: File "license.bbclass", line 99, in find_license_files
ERROR:
ERROR: File "/media/build1/poky/meta/lib/oe/license.py", line 38, in visit_string
ERROR: if pos > 0 and license_pattern.match(elements[pos-1]):
ERROR:
ERROR: The code that was being executed was:
ERROR: 0009: destdir = os.path.join(d.getVar('LICSSTATEDIR', True), d.getVar('PN', True))
ERROR: 0010: copy_license_files(lic_files_paths, destdir)
ERROR: 0011:
ERROR: 0012:
ERROR: *** 0013:do_populate_lic(d)
ERROR: 0014:
ERROR: [From file: 'do_populate_lic', lineno: 13, function: <module>]
ERROR: 0002:def do_populate_lic(d):
ERROR: 0003: """
ERROR: 0004: Populate LICENSE_DIRECTORY with licenses.
ERROR: 0005: """
ERROR: *** 0006: lic_files_paths = find_license_files(d)
ERROR: 0007:
ERROR: 0008: # The base directory we wrangle licenses to
ERROR: 0009: destdir = os.path.join(d.getVar('LICSSTATEDIR', True), d.getVar('PN', True))
ERROR: 0010: copy_license_files(lic_files_paths, destdir)
ERROR: [From file: 'do_populate_lic', lineno: 6, function: do_populate_lic]
ERROR: 0095: lic_files_paths.append((os.path.basename(path), srclicfile))
ERROR: 0096:
ERROR: 0097: v = FindVisitor()
ERROR: 0098: try:
ERROR: *** 0099: v.visit_string(license_types)
ERROR: 0100: except oe.license.InvalidLicense as exc:
ERROR: 0101: bb.fatal('%s: %s' % (d.getVar('PF', True), exc))
ERROR: 0102: except SyntaxError:
ERROR: 0103: bb.warn("%s: Failed to parse it's LICENSE field." % (d.getVar('PF', True)))
ERROR: [From file: 'license.bbclass', lineno: 99, function: find_license_files]
ERROR: Function failed: do_populate_lic
ERROR: Logfile of failure stored in: /media/build1/poky/build/tmp/work/i586-poky-linux/eglibc-initial/2.17-r3/temp/log.do_populate_lic.17442
"""
Example after:
"""
ERROR: Error executing a python function in /media/build1/poky/meta/recipes-core/eglibc/eglibc-initial_2.17.bb:
The stack trace of python calls that resulted in this exception/failure was:
File: 'do_populate_lic', lineno: 13, function: <module>
0009: destdir = os.path.join(d.getVar('LICSSTATEDIR', True), d.getVar('PN', True))
0010: copy_license_files(lic_files_paths, destdir)
0011:
0012:
*** 0013:do_populate_lic(d)
0014:
File: 'do_populate_lic', lineno: 6, function: do_populate_lic
0002:def do_populate_lic(d):
0003: """
0004: Populate LICENSE_DIRECTORY with licenses.
0005: """
*** 0006: lic_files_paths = find_license_files(d)
0007:
0008: # The base directory we wrangle licenses to
0009: destdir = os.path.join(d.getVar('LICSSTATEDIR', True), d.getVar('PN', True))
0010: copy_license_files(lic_files_paths, destdir)
File: 'license.bbclass', lineno: 99, function: find_license_files
0095: lic_files_paths.append((os.path.basename(path), srclicfile))
0096:
0097: v = FindVisitor()
0098: try:
*** 0099: v.visit_string(license_types)
0100: except oe.license.InvalidLicense as exc:
0101: bb.fatal('%s: %s' % (d.getVar('PF', True), exc))
0102: except SyntaxError:
0103: bb.warn("%s: Failed to parse it's LICENSE field." % (d.getVar('PF', True)))
File: '/media/build1/poky/meta/lib/oe/license.py', lineno: 38, function: visit_string
0034: new_elements = []
0035: elements = filter(lambda x: x.strip(), license_operator.split(licensestr))
0036: for pos, element in enumerate(elements):
0037: if license_pattern.match(element):
*** 0038: if pos > 0 and license_pattern.match(elements[pos-1]):
0039: new_elements.append('&')
0040: element = '"' + element + '"'
0041: elif not license_operator.match(element):
0042: raise InvalidLicense(element)
Exception: TypeError: 'filter' object is not subscriptable
ERROR: Function failed: do_populate_lic
ERROR: Logfile of failure stored in: /media/build1/poky/build/tmp/work/i586-poky-linux/eglibc-initial/2.17-r3/temp/log.do_populate_lic.3275
ERROR: Task 9 (/media/build1/poky/meta/recipes-core/eglibc/eglibc-initial_2.17.bb, do_populate_lic) failed with exit code '1
"""
(Bitbake rev: c5de66b870406d9bd1161a9b7e2b04fe6eb065fe)
Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
2013-05-08 17:14:53 +00:00
tbextract = traceback . extract_tb ( tb )
tbformat = traceback . format_list ( tbextract )
error . append ( " The stack trace of python calls that resulted in this exception/failure was: " )
error . append ( " File: ' %s ' , lineno: %s , function: %s " % ( tbextract [ 0 ] [ 0 ] , tbextract [ 0 ] [ 1 ] , tbextract [ 0 ] [ 2 ] ) )
error . extend ( _print_trace ( textarray , linefailed ) )
2010-12-05 22:52:44 +00:00
2011-01-01 23:55:54 +00:00
# See if this is a function we constructed and has calls back into other functions in
2010-12-05 22:52:44 +00:00
# "text". If so, try and improve the context of the error by diving down the trace
level = 0
nexttb = tb . tb_next
2012-08-22 19:01:12 +00:00
while nexttb is not None and ( level + 1 ) < len ( tbextract ) :
bitbake: utils: Improve better_exec traceback handling
The current bitbake tracebacks are hard to read/confusing and sometimes
incomplete. This patch attempts to do better by:
* Moving the note about the exact exception to the end to make things
read in sequence
* Merged the initial stack trace to become part of the code dump
* Added handling for "/xxxx" file paths since we can load these files
and include the data as part of the trace
* Dropped the ERROR: prefix to every line, allowing the error messages to
be spacially accosicated in the UIs
* Moved the "From:" line to the top of each code block and ensured its present
consistently
With the complexity now in this funciton, I've added try/except wrapping around
it to ensure we catch exceptions in the exception handler too.
Example before:
"""
ERROR: Error executing a python function in /media/build1/poky/meta/recipes-core/eglibc/eglibc-initial_2.17.bb:
TypeError: 'filter' object is not subscriptable
ERROR: The stack trace of python calls that resulted in this exception/failure was:
ERROR: File "do_populate_lic", line 13, in <module>
ERROR:
ERROR: File "do_populate_lic", line 6, in do_populate_lic
ERROR:
ERROR: File "license.bbclass", line 99, in find_license_files
ERROR:
ERROR: File "/media/build1/poky/meta/lib/oe/license.py", line 38, in visit_string
ERROR: if pos > 0 and license_pattern.match(elements[pos-1]):
ERROR:
ERROR: The code that was being executed was:
ERROR: 0009: destdir = os.path.join(d.getVar('LICSSTATEDIR', True), d.getVar('PN', True))
ERROR: 0010: copy_license_files(lic_files_paths, destdir)
ERROR: 0011:
ERROR: 0012:
ERROR: *** 0013:do_populate_lic(d)
ERROR: 0014:
ERROR: [From file: 'do_populate_lic', lineno: 13, function: <module>]
ERROR: 0002:def do_populate_lic(d):
ERROR: 0003: """
ERROR: 0004: Populate LICENSE_DIRECTORY with licenses.
ERROR: 0005: """
ERROR: *** 0006: lic_files_paths = find_license_files(d)
ERROR: 0007:
ERROR: 0008: # The base directory we wrangle licenses to
ERROR: 0009: destdir = os.path.join(d.getVar('LICSSTATEDIR', True), d.getVar('PN', True))
ERROR: 0010: copy_license_files(lic_files_paths, destdir)
ERROR: [From file: 'do_populate_lic', lineno: 6, function: do_populate_lic]
ERROR: 0095: lic_files_paths.append((os.path.basename(path), srclicfile))
ERROR: 0096:
ERROR: 0097: v = FindVisitor()
ERROR: 0098: try:
ERROR: *** 0099: v.visit_string(license_types)
ERROR: 0100: except oe.license.InvalidLicense as exc:
ERROR: 0101: bb.fatal('%s: %s' % (d.getVar('PF', True), exc))
ERROR: 0102: except SyntaxError:
ERROR: 0103: bb.warn("%s: Failed to parse it's LICENSE field." % (d.getVar('PF', True)))
ERROR: [From file: 'license.bbclass', lineno: 99, function: find_license_files]
ERROR: Function failed: do_populate_lic
ERROR: Logfile of failure stored in: /media/build1/poky/build/tmp/work/i586-poky-linux/eglibc-initial/2.17-r3/temp/log.do_populate_lic.17442
"""
Example after:
"""
ERROR: Error executing a python function in /media/build1/poky/meta/recipes-core/eglibc/eglibc-initial_2.17.bb:
The stack trace of python calls that resulted in this exception/failure was:
File: 'do_populate_lic', lineno: 13, function: <module>
0009: destdir = os.path.join(d.getVar('LICSSTATEDIR', True), d.getVar('PN', True))
0010: copy_license_files(lic_files_paths, destdir)
0011:
0012:
*** 0013:do_populate_lic(d)
0014:
File: 'do_populate_lic', lineno: 6, function: do_populate_lic
0002:def do_populate_lic(d):
0003: """
0004: Populate LICENSE_DIRECTORY with licenses.
0005: """
*** 0006: lic_files_paths = find_license_files(d)
0007:
0008: # The base directory we wrangle licenses to
0009: destdir = os.path.join(d.getVar('LICSSTATEDIR', True), d.getVar('PN', True))
0010: copy_license_files(lic_files_paths, destdir)
File: 'license.bbclass', lineno: 99, function: find_license_files
0095: lic_files_paths.append((os.path.basename(path), srclicfile))
0096:
0097: v = FindVisitor()
0098: try:
*** 0099: v.visit_string(license_types)
0100: except oe.license.InvalidLicense as exc:
0101: bb.fatal('%s: %s' % (d.getVar('PF', True), exc))
0102: except SyntaxError:
0103: bb.warn("%s: Failed to parse it's LICENSE field." % (d.getVar('PF', True)))
File: '/media/build1/poky/meta/lib/oe/license.py', lineno: 38, function: visit_string
0034: new_elements = []
0035: elements = filter(lambda x: x.strip(), license_operator.split(licensestr))
0036: for pos, element in enumerate(elements):
0037: if license_pattern.match(element):
*** 0038: if pos > 0 and license_pattern.match(elements[pos-1]):
0039: new_elements.append('&')
0040: element = '"' + element + '"'
0041: elif not license_operator.match(element):
0042: raise InvalidLicense(element)
Exception: TypeError: 'filter' object is not subscriptable
ERROR: Function failed: do_populate_lic
ERROR: Logfile of failure stored in: /media/build1/poky/build/tmp/work/i586-poky-linux/eglibc-initial/2.17-r3/temp/log.do_populate_lic.3275
ERROR: Task 9 (/media/build1/poky/meta/recipes-core/eglibc/eglibc-initial_2.17.bb, do_populate_lic) failed with exit code '1
"""
(Bitbake rev: c5de66b870406d9bd1161a9b7e2b04fe6eb065fe)
Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
2013-05-08 17:14:53 +00:00
error . append ( " File: ' %s ' , lineno: %s , function: %s " % ( tbextract [ level + 1 ] [ 0 ] , tbextract [ level + 1 ] [ 1 ] , tbextract [ level + 1 ] [ 2 ] ) )
2010-12-05 22:52:44 +00:00
if tbextract [ level ] [ 0 ] == tbextract [ level + 1 ] [ 0 ] and tbextract [ level + 1 ] [ 2 ] == tbextract [ level ] [ 0 ] :
bitbake: utils: Improve better_exec traceback handling
The current bitbake tracebacks are hard to read/confusing and sometimes
incomplete. This patch attempts to do better by:
* Moving the note about the exact exception to the end to make things
read in sequence
* Merged the initial stack trace to become part of the code dump
* Added handling for "/xxxx" file paths since we can load these files
and include the data as part of the trace
* Dropped the ERROR: prefix to every line, allowing the error messages to
be spacially accosicated in the UIs
* Moved the "From:" line to the top of each code block and ensured its present
consistently
With the complexity now in this funciton, I've added try/except wrapping around
it to ensure we catch exceptions in the exception handler too.
Example before:
"""
ERROR: Error executing a python function in /media/build1/poky/meta/recipes-core/eglibc/eglibc-initial_2.17.bb:
TypeError: 'filter' object is not subscriptable
ERROR: The stack trace of python calls that resulted in this exception/failure was:
ERROR: File "do_populate_lic", line 13, in <module>
ERROR:
ERROR: File "do_populate_lic", line 6, in do_populate_lic
ERROR:
ERROR: File "license.bbclass", line 99, in find_license_files
ERROR:
ERROR: File "/media/build1/poky/meta/lib/oe/license.py", line 38, in visit_string
ERROR: if pos > 0 and license_pattern.match(elements[pos-1]):
ERROR:
ERROR: The code that was being executed was:
ERROR: 0009: destdir = os.path.join(d.getVar('LICSSTATEDIR', True), d.getVar('PN', True))
ERROR: 0010: copy_license_files(lic_files_paths, destdir)
ERROR: 0011:
ERROR: 0012:
ERROR: *** 0013:do_populate_lic(d)
ERROR: 0014:
ERROR: [From file: 'do_populate_lic', lineno: 13, function: <module>]
ERROR: 0002:def do_populate_lic(d):
ERROR: 0003: """
ERROR: 0004: Populate LICENSE_DIRECTORY with licenses.
ERROR: 0005: """
ERROR: *** 0006: lic_files_paths = find_license_files(d)
ERROR: 0007:
ERROR: 0008: # The base directory we wrangle licenses to
ERROR: 0009: destdir = os.path.join(d.getVar('LICSSTATEDIR', True), d.getVar('PN', True))
ERROR: 0010: copy_license_files(lic_files_paths, destdir)
ERROR: [From file: 'do_populate_lic', lineno: 6, function: do_populate_lic]
ERROR: 0095: lic_files_paths.append((os.path.basename(path), srclicfile))
ERROR: 0096:
ERROR: 0097: v = FindVisitor()
ERROR: 0098: try:
ERROR: *** 0099: v.visit_string(license_types)
ERROR: 0100: except oe.license.InvalidLicense as exc:
ERROR: 0101: bb.fatal('%s: %s' % (d.getVar('PF', True), exc))
ERROR: 0102: except SyntaxError:
ERROR: 0103: bb.warn("%s: Failed to parse it's LICENSE field." % (d.getVar('PF', True)))
ERROR: [From file: 'license.bbclass', lineno: 99, function: find_license_files]
ERROR: Function failed: do_populate_lic
ERROR: Logfile of failure stored in: /media/build1/poky/build/tmp/work/i586-poky-linux/eglibc-initial/2.17-r3/temp/log.do_populate_lic.17442
"""
Example after:
"""
ERROR: Error executing a python function in /media/build1/poky/meta/recipes-core/eglibc/eglibc-initial_2.17.bb:
The stack trace of python calls that resulted in this exception/failure was:
File: 'do_populate_lic', lineno: 13, function: <module>
0009: destdir = os.path.join(d.getVar('LICSSTATEDIR', True), d.getVar('PN', True))
0010: copy_license_files(lic_files_paths, destdir)
0011:
0012:
*** 0013:do_populate_lic(d)
0014:
File: 'do_populate_lic', lineno: 6, function: do_populate_lic
0002:def do_populate_lic(d):
0003: """
0004: Populate LICENSE_DIRECTORY with licenses.
0005: """
*** 0006: lic_files_paths = find_license_files(d)
0007:
0008: # The base directory we wrangle licenses to
0009: destdir = os.path.join(d.getVar('LICSSTATEDIR', True), d.getVar('PN', True))
0010: copy_license_files(lic_files_paths, destdir)
File: 'license.bbclass', lineno: 99, function: find_license_files
0095: lic_files_paths.append((os.path.basename(path), srclicfile))
0096:
0097: v = FindVisitor()
0098: try:
*** 0099: v.visit_string(license_types)
0100: except oe.license.InvalidLicense as exc:
0101: bb.fatal('%s: %s' % (d.getVar('PF', True), exc))
0102: except SyntaxError:
0103: bb.warn("%s: Failed to parse it's LICENSE field." % (d.getVar('PF', True)))
File: '/media/build1/poky/meta/lib/oe/license.py', lineno: 38, function: visit_string
0034: new_elements = []
0035: elements = filter(lambda x: x.strip(), license_operator.split(licensestr))
0036: for pos, element in enumerate(elements):
0037: if license_pattern.match(element):
*** 0038: if pos > 0 and license_pattern.match(elements[pos-1]):
0039: new_elements.append('&')
0040: element = '"' + element + '"'
0041: elif not license_operator.match(element):
0042: raise InvalidLicense(element)
Exception: TypeError: 'filter' object is not subscriptable
ERROR: Function failed: do_populate_lic
ERROR: Logfile of failure stored in: /media/build1/poky/build/tmp/work/i586-poky-linux/eglibc-initial/2.17-r3/temp/log.do_populate_lic.3275
ERROR: Task 9 (/media/build1/poky/meta/recipes-core/eglibc/eglibc-initial_2.17.bb, do_populate_lic) failed with exit code '1
"""
(Bitbake rev: c5de66b870406d9bd1161a9b7e2b04fe6eb065fe)
Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
2013-05-08 17:14:53 +00:00
# The code was possibly in the string we compiled ourselves
error . extend ( _print_trace ( textarray , tbextract [ level + 1 ] [ 1 ] ) )
elif tbextract [ level + 1 ] [ 0 ] . startswith ( " / " ) :
# The code looks like it might be in a file, try and load it
try :
with open ( tbextract [ level + 1 ] [ 0 ] , " r " ) as f :
text = f . readlines ( )
error . extend ( _print_trace ( text , tbextract [ level + 1 ] [ 1 ] ) )
except :
error . append ( tbformat [ level + 1 ] )
2012-08-22 19:01:12 +00:00
elif " d " in context and tbextract [ level + 1 ] [ 2 ] :
bitbake: utils: Improve better_exec traceback handling
The current bitbake tracebacks are hard to read/confusing and sometimes
incomplete. This patch attempts to do better by:
* Moving the note about the exact exception to the end to make things
read in sequence
* Merged the initial stack trace to become part of the code dump
* Added handling for "/xxxx" file paths since we can load these files
and include the data as part of the trace
* Dropped the ERROR: prefix to every line, allowing the error messages to
be spacially accosicated in the UIs
* Moved the "From:" line to the top of each code block and ensured its present
consistently
With the complexity now in this funciton, I've added try/except wrapping around
it to ensure we catch exceptions in the exception handler too.
Example before:
"""
ERROR: Error executing a python function in /media/build1/poky/meta/recipes-core/eglibc/eglibc-initial_2.17.bb:
TypeError: 'filter' object is not subscriptable
ERROR: The stack trace of python calls that resulted in this exception/failure was:
ERROR: File "do_populate_lic", line 13, in <module>
ERROR:
ERROR: File "do_populate_lic", line 6, in do_populate_lic
ERROR:
ERROR: File "license.bbclass", line 99, in find_license_files
ERROR:
ERROR: File "/media/build1/poky/meta/lib/oe/license.py", line 38, in visit_string
ERROR: if pos > 0 and license_pattern.match(elements[pos-1]):
ERROR:
ERROR: The code that was being executed was:
ERROR: 0009: destdir = os.path.join(d.getVar('LICSSTATEDIR', True), d.getVar('PN', True))
ERROR: 0010: copy_license_files(lic_files_paths, destdir)
ERROR: 0011:
ERROR: 0012:
ERROR: *** 0013:do_populate_lic(d)
ERROR: 0014:
ERROR: [From file: 'do_populate_lic', lineno: 13, function: <module>]
ERROR: 0002:def do_populate_lic(d):
ERROR: 0003: """
ERROR: 0004: Populate LICENSE_DIRECTORY with licenses.
ERROR: 0005: """
ERROR: *** 0006: lic_files_paths = find_license_files(d)
ERROR: 0007:
ERROR: 0008: # The base directory we wrangle licenses to
ERROR: 0009: destdir = os.path.join(d.getVar('LICSSTATEDIR', True), d.getVar('PN', True))
ERROR: 0010: copy_license_files(lic_files_paths, destdir)
ERROR: [From file: 'do_populate_lic', lineno: 6, function: do_populate_lic]
ERROR: 0095: lic_files_paths.append((os.path.basename(path), srclicfile))
ERROR: 0096:
ERROR: 0097: v = FindVisitor()
ERROR: 0098: try:
ERROR: *** 0099: v.visit_string(license_types)
ERROR: 0100: except oe.license.InvalidLicense as exc:
ERROR: 0101: bb.fatal('%s: %s' % (d.getVar('PF', True), exc))
ERROR: 0102: except SyntaxError:
ERROR: 0103: bb.warn("%s: Failed to parse it's LICENSE field." % (d.getVar('PF', True)))
ERROR: [From file: 'license.bbclass', lineno: 99, function: find_license_files]
ERROR: Function failed: do_populate_lic
ERROR: Logfile of failure stored in: /media/build1/poky/build/tmp/work/i586-poky-linux/eglibc-initial/2.17-r3/temp/log.do_populate_lic.17442
"""
Example after:
"""
ERROR: Error executing a python function in /media/build1/poky/meta/recipes-core/eglibc/eglibc-initial_2.17.bb:
The stack trace of python calls that resulted in this exception/failure was:
File: 'do_populate_lic', lineno: 13, function: <module>
0009: destdir = os.path.join(d.getVar('LICSSTATEDIR', True), d.getVar('PN', True))
0010: copy_license_files(lic_files_paths, destdir)
0011:
0012:
*** 0013:do_populate_lic(d)
0014:
File: 'do_populate_lic', lineno: 6, function: do_populate_lic
0002:def do_populate_lic(d):
0003: """
0004: Populate LICENSE_DIRECTORY with licenses.
0005: """
*** 0006: lic_files_paths = find_license_files(d)
0007:
0008: # The base directory we wrangle licenses to
0009: destdir = os.path.join(d.getVar('LICSSTATEDIR', True), d.getVar('PN', True))
0010: copy_license_files(lic_files_paths, destdir)
File: 'license.bbclass', lineno: 99, function: find_license_files
0095: lic_files_paths.append((os.path.basename(path), srclicfile))
0096:
0097: v = FindVisitor()
0098: try:
*** 0099: v.visit_string(license_types)
0100: except oe.license.InvalidLicense as exc:
0101: bb.fatal('%s: %s' % (d.getVar('PF', True), exc))
0102: except SyntaxError:
0103: bb.warn("%s: Failed to parse it's LICENSE field." % (d.getVar('PF', True)))
File: '/media/build1/poky/meta/lib/oe/license.py', lineno: 38, function: visit_string
0034: new_elements = []
0035: elements = filter(lambda x: x.strip(), license_operator.split(licensestr))
0036: for pos, element in enumerate(elements):
0037: if license_pattern.match(element):
*** 0038: if pos > 0 and license_pattern.match(elements[pos-1]):
0039: new_elements.append('&')
0040: element = '"' + element + '"'
0041: elif not license_operator.match(element):
0042: raise InvalidLicense(element)
Exception: TypeError: 'filter' object is not subscriptable
ERROR: Function failed: do_populate_lic
ERROR: Logfile of failure stored in: /media/build1/poky/build/tmp/work/i586-poky-linux/eglibc-initial/2.17-r3/temp/log.do_populate_lic.3275
ERROR: Task 9 (/media/build1/poky/meta/recipes-core/eglibc/eglibc-initial_2.17.bb, do_populate_lic) failed with exit code '1
"""
(Bitbake rev: c5de66b870406d9bd1161a9b7e2b04fe6eb065fe)
Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
2013-05-08 17:14:53 +00:00
# Try and find the code in the datastore based on the functionname
2012-08-22 19:01:12 +00:00
d = context [ " d " ]
functionname = tbextract [ level + 1 ] [ 2 ]
text = d . getVar ( functionname , True )
if text :
bitbake: utils: Improve better_exec traceback handling
The current bitbake tracebacks are hard to read/confusing and sometimes
incomplete. This patch attempts to do better by:
* Moving the note about the exact exception to the end to make things
read in sequence
* Merged the initial stack trace to become part of the code dump
* Added handling for "/xxxx" file paths since we can load these files
and include the data as part of the trace
* Dropped the ERROR: prefix to every line, allowing the error messages to
be spacially accosicated in the UIs
* Moved the "From:" line to the top of each code block and ensured its present
consistently
With the complexity now in this funciton, I've added try/except wrapping around
it to ensure we catch exceptions in the exception handler too.
Example before:
"""
ERROR: Error executing a python function in /media/build1/poky/meta/recipes-core/eglibc/eglibc-initial_2.17.bb:
TypeError: 'filter' object is not subscriptable
ERROR: The stack trace of python calls that resulted in this exception/failure was:
ERROR: File "do_populate_lic", line 13, in <module>
ERROR:
ERROR: File "do_populate_lic", line 6, in do_populate_lic
ERROR:
ERROR: File "license.bbclass", line 99, in find_license_files
ERROR:
ERROR: File "/media/build1/poky/meta/lib/oe/license.py", line 38, in visit_string
ERROR: if pos > 0 and license_pattern.match(elements[pos-1]):
ERROR:
ERROR: The code that was being executed was:
ERROR: 0009: destdir = os.path.join(d.getVar('LICSSTATEDIR', True), d.getVar('PN', True))
ERROR: 0010: copy_license_files(lic_files_paths, destdir)
ERROR: 0011:
ERROR: 0012:
ERROR: *** 0013:do_populate_lic(d)
ERROR: 0014:
ERROR: [From file: 'do_populate_lic', lineno: 13, function: <module>]
ERROR: 0002:def do_populate_lic(d):
ERROR: 0003: """
ERROR: 0004: Populate LICENSE_DIRECTORY with licenses.
ERROR: 0005: """
ERROR: *** 0006: lic_files_paths = find_license_files(d)
ERROR: 0007:
ERROR: 0008: # The base directory we wrangle licenses to
ERROR: 0009: destdir = os.path.join(d.getVar('LICSSTATEDIR', True), d.getVar('PN', True))
ERROR: 0010: copy_license_files(lic_files_paths, destdir)
ERROR: [From file: 'do_populate_lic', lineno: 6, function: do_populate_lic]
ERROR: 0095: lic_files_paths.append((os.path.basename(path), srclicfile))
ERROR: 0096:
ERROR: 0097: v = FindVisitor()
ERROR: 0098: try:
ERROR: *** 0099: v.visit_string(license_types)
ERROR: 0100: except oe.license.InvalidLicense as exc:
ERROR: 0101: bb.fatal('%s: %s' % (d.getVar('PF', True), exc))
ERROR: 0102: except SyntaxError:
ERROR: 0103: bb.warn("%s: Failed to parse it's LICENSE field." % (d.getVar('PF', True)))
ERROR: [From file: 'license.bbclass', lineno: 99, function: find_license_files]
ERROR: Function failed: do_populate_lic
ERROR: Logfile of failure stored in: /media/build1/poky/build/tmp/work/i586-poky-linux/eglibc-initial/2.17-r3/temp/log.do_populate_lic.17442
"""
Example after:
"""
ERROR: Error executing a python function in /media/build1/poky/meta/recipes-core/eglibc/eglibc-initial_2.17.bb:
The stack trace of python calls that resulted in this exception/failure was:
File: 'do_populate_lic', lineno: 13, function: <module>
0009: destdir = os.path.join(d.getVar('LICSSTATEDIR', True), d.getVar('PN', True))
0010: copy_license_files(lic_files_paths, destdir)
0011:
0012:
*** 0013:do_populate_lic(d)
0014:
File: 'do_populate_lic', lineno: 6, function: do_populate_lic
0002:def do_populate_lic(d):
0003: """
0004: Populate LICENSE_DIRECTORY with licenses.
0005: """
*** 0006: lic_files_paths = find_license_files(d)
0007:
0008: # The base directory we wrangle licenses to
0009: destdir = os.path.join(d.getVar('LICSSTATEDIR', True), d.getVar('PN', True))
0010: copy_license_files(lic_files_paths, destdir)
File: 'license.bbclass', lineno: 99, function: find_license_files
0095: lic_files_paths.append((os.path.basename(path), srclicfile))
0096:
0097: v = FindVisitor()
0098: try:
*** 0099: v.visit_string(license_types)
0100: except oe.license.InvalidLicense as exc:
0101: bb.fatal('%s: %s' % (d.getVar('PF', True), exc))
0102: except SyntaxError:
0103: bb.warn("%s: Failed to parse it's LICENSE field." % (d.getVar('PF', True)))
File: '/media/build1/poky/meta/lib/oe/license.py', lineno: 38, function: visit_string
0034: new_elements = []
0035: elements = filter(lambda x: x.strip(), license_operator.split(licensestr))
0036: for pos, element in enumerate(elements):
0037: if license_pattern.match(element):
*** 0038: if pos > 0 and license_pattern.match(elements[pos-1]):
0039: new_elements.append('&')
0040: element = '"' + element + '"'
0041: elif not license_operator.match(element):
0042: raise InvalidLicense(element)
Exception: TypeError: 'filter' object is not subscriptable
ERROR: Function failed: do_populate_lic
ERROR: Logfile of failure stored in: /media/build1/poky/build/tmp/work/i586-poky-linux/eglibc-initial/2.17-r3/temp/log.do_populate_lic.3275
ERROR: Task 9 (/media/build1/poky/meta/recipes-core/eglibc/eglibc-initial_2.17.bb, do_populate_lic) failed with exit code '1
"""
(Bitbake rev: c5de66b870406d9bd1161a9b7e2b04fe6eb065fe)
Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
2013-05-08 17:14:53 +00:00
error . extend ( _print_trace ( text . split ( ' \n ' ) , tbextract [ level + 1 ] [ 1 ] ) )
2012-08-22 19:01:12 +00:00
else :
bitbake: utils: Improve better_exec traceback handling
The current bitbake tracebacks are hard to read/confusing and sometimes
incomplete. This patch attempts to do better by:
* Moving the note about the exact exception to the end to make things
read in sequence
* Merged the initial stack trace to become part of the code dump
* Added handling for "/xxxx" file paths since we can load these files
and include the data as part of the trace
* Dropped the ERROR: prefix to every line, allowing the error messages to
be spacially accosicated in the UIs
* Moved the "From:" line to the top of each code block and ensured its present
consistently
With the complexity now in this funciton, I've added try/except wrapping around
it to ensure we catch exceptions in the exception handler too.
Example before:
"""
ERROR: Error executing a python function in /media/build1/poky/meta/recipes-core/eglibc/eglibc-initial_2.17.bb:
TypeError: 'filter' object is not subscriptable
ERROR: The stack trace of python calls that resulted in this exception/failure was:
ERROR: File "do_populate_lic", line 13, in <module>
ERROR:
ERROR: File "do_populate_lic", line 6, in do_populate_lic
ERROR:
ERROR: File "license.bbclass", line 99, in find_license_files
ERROR:
ERROR: File "/media/build1/poky/meta/lib/oe/license.py", line 38, in visit_string
ERROR: if pos > 0 and license_pattern.match(elements[pos-1]):
ERROR:
ERROR: The code that was being executed was:
ERROR: 0009: destdir = os.path.join(d.getVar('LICSSTATEDIR', True), d.getVar('PN', True))
ERROR: 0010: copy_license_files(lic_files_paths, destdir)
ERROR: 0011:
ERROR: 0012:
ERROR: *** 0013:do_populate_lic(d)
ERROR: 0014:
ERROR: [From file: 'do_populate_lic', lineno: 13, function: <module>]
ERROR: 0002:def do_populate_lic(d):
ERROR: 0003: """
ERROR: 0004: Populate LICENSE_DIRECTORY with licenses.
ERROR: 0005: """
ERROR: *** 0006: lic_files_paths = find_license_files(d)
ERROR: 0007:
ERROR: 0008: # The base directory we wrangle licenses to
ERROR: 0009: destdir = os.path.join(d.getVar('LICSSTATEDIR', True), d.getVar('PN', True))
ERROR: 0010: copy_license_files(lic_files_paths, destdir)
ERROR: [From file: 'do_populate_lic', lineno: 6, function: do_populate_lic]
ERROR: 0095: lic_files_paths.append((os.path.basename(path), srclicfile))
ERROR: 0096:
ERROR: 0097: v = FindVisitor()
ERROR: 0098: try:
ERROR: *** 0099: v.visit_string(license_types)
ERROR: 0100: except oe.license.InvalidLicense as exc:
ERROR: 0101: bb.fatal('%s: %s' % (d.getVar('PF', True), exc))
ERROR: 0102: except SyntaxError:
ERROR: 0103: bb.warn("%s: Failed to parse it's LICENSE field." % (d.getVar('PF', True)))
ERROR: [From file: 'license.bbclass', lineno: 99, function: find_license_files]
ERROR: Function failed: do_populate_lic
ERROR: Logfile of failure stored in: /media/build1/poky/build/tmp/work/i586-poky-linux/eglibc-initial/2.17-r3/temp/log.do_populate_lic.17442
"""
Example after:
"""
ERROR: Error executing a python function in /media/build1/poky/meta/recipes-core/eglibc/eglibc-initial_2.17.bb:
The stack trace of python calls that resulted in this exception/failure was:
File: 'do_populate_lic', lineno: 13, function: <module>
0009: destdir = os.path.join(d.getVar('LICSSTATEDIR', True), d.getVar('PN', True))
0010: copy_license_files(lic_files_paths, destdir)
0011:
0012:
*** 0013:do_populate_lic(d)
0014:
File: 'do_populate_lic', lineno: 6, function: do_populate_lic
0002:def do_populate_lic(d):
0003: """
0004: Populate LICENSE_DIRECTORY with licenses.
0005: """
*** 0006: lic_files_paths = find_license_files(d)
0007:
0008: # The base directory we wrangle licenses to
0009: destdir = os.path.join(d.getVar('LICSSTATEDIR', True), d.getVar('PN', True))
0010: copy_license_files(lic_files_paths, destdir)
File: 'license.bbclass', lineno: 99, function: find_license_files
0095: lic_files_paths.append((os.path.basename(path), srclicfile))
0096:
0097: v = FindVisitor()
0098: try:
*** 0099: v.visit_string(license_types)
0100: except oe.license.InvalidLicense as exc:
0101: bb.fatal('%s: %s' % (d.getVar('PF', True), exc))
0102: except SyntaxError:
0103: bb.warn("%s: Failed to parse it's LICENSE field." % (d.getVar('PF', True)))
File: '/media/build1/poky/meta/lib/oe/license.py', lineno: 38, function: visit_string
0034: new_elements = []
0035: elements = filter(lambda x: x.strip(), license_operator.split(licensestr))
0036: for pos, element in enumerate(elements):
0037: if license_pattern.match(element):
*** 0038: if pos > 0 and license_pattern.match(elements[pos-1]):
0039: new_elements.append('&')
0040: element = '"' + element + '"'
0041: elif not license_operator.match(element):
0042: raise InvalidLicense(element)
Exception: TypeError: 'filter' object is not subscriptable
ERROR: Function failed: do_populate_lic
ERROR: Logfile of failure stored in: /media/build1/poky/build/tmp/work/i586-poky-linux/eglibc-initial/2.17-r3/temp/log.do_populate_lic.3275
ERROR: Task 9 (/media/build1/poky/meta/recipes-core/eglibc/eglibc-initial_2.17.bb, do_populate_lic) failed with exit code '1
"""
(Bitbake rev: c5de66b870406d9bd1161a9b7e2b04fe6eb065fe)
Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
2013-05-08 17:14:53 +00:00
error . append ( tbformat [ level + 1 ] )
2010-12-05 22:52:44 +00:00
else :
bitbake: utils: Improve better_exec traceback handling
The current bitbake tracebacks are hard to read/confusing and sometimes
incomplete. This patch attempts to do better by:
* Moving the note about the exact exception to the end to make things
read in sequence
* Merged the initial stack trace to become part of the code dump
* Added handling for "/xxxx" file paths since we can load these files
and include the data as part of the trace
* Dropped the ERROR: prefix to every line, allowing the error messages to
be spacially accosicated in the UIs
* Moved the "From:" line to the top of each code block and ensured its present
consistently
With the complexity now in this funciton, I've added try/except wrapping around
it to ensure we catch exceptions in the exception handler too.
Example before:
"""
ERROR: Error executing a python function in /media/build1/poky/meta/recipes-core/eglibc/eglibc-initial_2.17.bb:
TypeError: 'filter' object is not subscriptable
ERROR: The stack trace of python calls that resulted in this exception/failure was:
ERROR: File "do_populate_lic", line 13, in <module>
ERROR:
ERROR: File "do_populate_lic", line 6, in do_populate_lic
ERROR:
ERROR: File "license.bbclass", line 99, in find_license_files
ERROR:
ERROR: File "/media/build1/poky/meta/lib/oe/license.py", line 38, in visit_string
ERROR: if pos > 0 and license_pattern.match(elements[pos-1]):
ERROR:
ERROR: The code that was being executed was:
ERROR: 0009: destdir = os.path.join(d.getVar('LICSSTATEDIR', True), d.getVar('PN', True))
ERROR: 0010: copy_license_files(lic_files_paths, destdir)
ERROR: 0011:
ERROR: 0012:
ERROR: *** 0013:do_populate_lic(d)
ERROR: 0014:
ERROR: [From file: 'do_populate_lic', lineno: 13, function: <module>]
ERROR: 0002:def do_populate_lic(d):
ERROR: 0003: """
ERROR: 0004: Populate LICENSE_DIRECTORY with licenses.
ERROR: 0005: """
ERROR: *** 0006: lic_files_paths = find_license_files(d)
ERROR: 0007:
ERROR: 0008: # The base directory we wrangle licenses to
ERROR: 0009: destdir = os.path.join(d.getVar('LICSSTATEDIR', True), d.getVar('PN', True))
ERROR: 0010: copy_license_files(lic_files_paths, destdir)
ERROR: [From file: 'do_populate_lic', lineno: 6, function: do_populate_lic]
ERROR: 0095: lic_files_paths.append((os.path.basename(path), srclicfile))
ERROR: 0096:
ERROR: 0097: v = FindVisitor()
ERROR: 0098: try:
ERROR: *** 0099: v.visit_string(license_types)
ERROR: 0100: except oe.license.InvalidLicense as exc:
ERROR: 0101: bb.fatal('%s: %s' % (d.getVar('PF', True), exc))
ERROR: 0102: except SyntaxError:
ERROR: 0103: bb.warn("%s: Failed to parse it's LICENSE field." % (d.getVar('PF', True)))
ERROR: [From file: 'license.bbclass', lineno: 99, function: find_license_files]
ERROR: Function failed: do_populate_lic
ERROR: Logfile of failure stored in: /media/build1/poky/build/tmp/work/i586-poky-linux/eglibc-initial/2.17-r3/temp/log.do_populate_lic.17442
"""
Example after:
"""
ERROR: Error executing a python function in /media/build1/poky/meta/recipes-core/eglibc/eglibc-initial_2.17.bb:
The stack trace of python calls that resulted in this exception/failure was:
File: 'do_populate_lic', lineno: 13, function: <module>
0009: destdir = os.path.join(d.getVar('LICSSTATEDIR', True), d.getVar('PN', True))
0010: copy_license_files(lic_files_paths, destdir)
0011:
0012:
*** 0013:do_populate_lic(d)
0014:
File: 'do_populate_lic', lineno: 6, function: do_populate_lic
0002:def do_populate_lic(d):
0003: """
0004: Populate LICENSE_DIRECTORY with licenses.
0005: """
*** 0006: lic_files_paths = find_license_files(d)
0007:
0008: # The base directory we wrangle licenses to
0009: destdir = os.path.join(d.getVar('LICSSTATEDIR', True), d.getVar('PN', True))
0010: copy_license_files(lic_files_paths, destdir)
File: 'license.bbclass', lineno: 99, function: find_license_files
0095: lic_files_paths.append((os.path.basename(path), srclicfile))
0096:
0097: v = FindVisitor()
0098: try:
*** 0099: v.visit_string(license_types)
0100: except oe.license.InvalidLicense as exc:
0101: bb.fatal('%s: %s' % (d.getVar('PF', True), exc))
0102: except SyntaxError:
0103: bb.warn("%s: Failed to parse it's LICENSE field." % (d.getVar('PF', True)))
File: '/media/build1/poky/meta/lib/oe/license.py', lineno: 38, function: visit_string
0034: new_elements = []
0035: elements = filter(lambda x: x.strip(), license_operator.split(licensestr))
0036: for pos, element in enumerate(elements):
0037: if license_pattern.match(element):
*** 0038: if pos > 0 and license_pattern.match(elements[pos-1]):
0039: new_elements.append('&')
0040: element = '"' + element + '"'
0041: elif not license_operator.match(element):
0042: raise InvalidLicense(element)
Exception: TypeError: 'filter' object is not subscriptable
ERROR: Function failed: do_populate_lic
ERROR: Logfile of failure stored in: /media/build1/poky/build/tmp/work/i586-poky-linux/eglibc-initial/2.17-r3/temp/log.do_populate_lic.3275
ERROR: Task 9 (/media/build1/poky/meta/recipes-core/eglibc/eglibc-initial_2.17.bb, do_populate_lic) failed with exit code '1
"""
(Bitbake rev: c5de66b870406d9bd1161a9b7e2b04fe6eb065fe)
Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
2013-05-08 17:14:53 +00:00
error . append ( tbformat [ level + 1 ] )
2010-12-05 22:52:44 +00:00
nexttb = tb . tb_next
level = level + 1
2010-03-24 23:56:12 +00:00
bitbake: utils: Improve better_exec traceback handling
The current bitbake tracebacks are hard to read/confusing and sometimes
incomplete. This patch attempts to do better by:
* Moving the note about the exact exception to the end to make things
read in sequence
* Merged the initial stack trace to become part of the code dump
* Added handling for "/xxxx" file paths since we can load these files
and include the data as part of the trace
* Dropped the ERROR: prefix to every line, allowing the error messages to
be spacially accosicated in the UIs
* Moved the "From:" line to the top of each code block and ensured its present
consistently
With the complexity now in this funciton, I've added try/except wrapping around
it to ensure we catch exceptions in the exception handler too.
Example before:
"""
ERROR: Error executing a python function in /media/build1/poky/meta/recipes-core/eglibc/eglibc-initial_2.17.bb:
TypeError: 'filter' object is not subscriptable
ERROR: The stack trace of python calls that resulted in this exception/failure was:
ERROR: File "do_populate_lic", line 13, in <module>
ERROR:
ERROR: File "do_populate_lic", line 6, in do_populate_lic
ERROR:
ERROR: File "license.bbclass", line 99, in find_license_files
ERROR:
ERROR: File "/media/build1/poky/meta/lib/oe/license.py", line 38, in visit_string
ERROR: if pos > 0 and license_pattern.match(elements[pos-1]):
ERROR:
ERROR: The code that was being executed was:
ERROR: 0009: destdir = os.path.join(d.getVar('LICSSTATEDIR', True), d.getVar('PN', True))
ERROR: 0010: copy_license_files(lic_files_paths, destdir)
ERROR: 0011:
ERROR: 0012:
ERROR: *** 0013:do_populate_lic(d)
ERROR: 0014:
ERROR: [From file: 'do_populate_lic', lineno: 13, function: <module>]
ERROR: 0002:def do_populate_lic(d):
ERROR: 0003: """
ERROR: 0004: Populate LICENSE_DIRECTORY with licenses.
ERROR: 0005: """
ERROR: *** 0006: lic_files_paths = find_license_files(d)
ERROR: 0007:
ERROR: 0008: # The base directory we wrangle licenses to
ERROR: 0009: destdir = os.path.join(d.getVar('LICSSTATEDIR', True), d.getVar('PN', True))
ERROR: 0010: copy_license_files(lic_files_paths, destdir)
ERROR: [From file: 'do_populate_lic', lineno: 6, function: do_populate_lic]
ERROR: 0095: lic_files_paths.append((os.path.basename(path), srclicfile))
ERROR: 0096:
ERROR: 0097: v = FindVisitor()
ERROR: 0098: try:
ERROR: *** 0099: v.visit_string(license_types)
ERROR: 0100: except oe.license.InvalidLicense as exc:
ERROR: 0101: bb.fatal('%s: %s' % (d.getVar('PF', True), exc))
ERROR: 0102: except SyntaxError:
ERROR: 0103: bb.warn("%s: Failed to parse it's LICENSE field." % (d.getVar('PF', True)))
ERROR: [From file: 'license.bbclass', lineno: 99, function: find_license_files]
ERROR: Function failed: do_populate_lic
ERROR: Logfile of failure stored in: /media/build1/poky/build/tmp/work/i586-poky-linux/eglibc-initial/2.17-r3/temp/log.do_populate_lic.17442
"""
Example after:
"""
ERROR: Error executing a python function in /media/build1/poky/meta/recipes-core/eglibc/eglibc-initial_2.17.bb:
The stack trace of python calls that resulted in this exception/failure was:
File: 'do_populate_lic', lineno: 13, function: <module>
0009: destdir = os.path.join(d.getVar('LICSSTATEDIR', True), d.getVar('PN', True))
0010: copy_license_files(lic_files_paths, destdir)
0011:
0012:
*** 0013:do_populate_lic(d)
0014:
File: 'do_populate_lic', lineno: 6, function: do_populate_lic
0002:def do_populate_lic(d):
0003: """
0004: Populate LICENSE_DIRECTORY with licenses.
0005: """
*** 0006: lic_files_paths = find_license_files(d)
0007:
0008: # The base directory we wrangle licenses to
0009: destdir = os.path.join(d.getVar('LICSSTATEDIR', True), d.getVar('PN', True))
0010: copy_license_files(lic_files_paths, destdir)
File: 'license.bbclass', lineno: 99, function: find_license_files
0095: lic_files_paths.append((os.path.basename(path), srclicfile))
0096:
0097: v = FindVisitor()
0098: try:
*** 0099: v.visit_string(license_types)
0100: except oe.license.InvalidLicense as exc:
0101: bb.fatal('%s: %s' % (d.getVar('PF', True), exc))
0102: except SyntaxError:
0103: bb.warn("%s: Failed to parse it's LICENSE field." % (d.getVar('PF', True)))
File: '/media/build1/poky/meta/lib/oe/license.py', lineno: 38, function: visit_string
0034: new_elements = []
0035: elements = filter(lambda x: x.strip(), license_operator.split(licensestr))
0036: for pos, element in enumerate(elements):
0037: if license_pattern.match(element):
*** 0038: if pos > 0 and license_pattern.match(elements[pos-1]):
0039: new_elements.append('&')
0040: element = '"' + element + '"'
0041: elif not license_operator.match(element):
0042: raise InvalidLicense(element)
Exception: TypeError: 'filter' object is not subscriptable
ERROR: Function failed: do_populate_lic
ERROR: Logfile of failure stored in: /media/build1/poky/build/tmp/work/i586-poky-linux/eglibc-initial/2.17-r3/temp/log.do_populate_lic.3275
ERROR: Task 9 (/media/build1/poky/meta/recipes-core/eglibc/eglibc-initial_2.17.bb, do_populate_lic) failed with exit code '1
"""
(Bitbake rev: c5de66b870406d9bd1161a9b7e2b04fe6eb065fe)
Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
2013-05-08 17:14:53 +00:00
error . append ( " Exception: %s " % ' ' . join ( exception ) )
finally :
logger . error ( " \n " . join ( error ) )
def better_exec ( code , context , text = None , realfile = " <code> " ) :
"""
Similiar to better_compile , better_exec will
print the lines that are responsible for the
error .
"""
import bb . parse
if not text :
text = code
if not hasattr ( code , " co_filename " ) :
code = better_compile ( code , realfile , realfile )
try :
2013-05-23 09:49:57 +00:00
exec ( code , get_context ( ) , context )
bitbake: utils: Improve better_exec traceback handling
The current bitbake tracebacks are hard to read/confusing and sometimes
incomplete. This patch attempts to do better by:
* Moving the note about the exact exception to the end to make things
read in sequence
* Merged the initial stack trace to become part of the code dump
* Added handling for "/xxxx" file paths since we can load these files
and include the data as part of the trace
* Dropped the ERROR: prefix to every line, allowing the error messages to
be spacially accosicated in the UIs
* Moved the "From:" line to the top of each code block and ensured its present
consistently
With the complexity now in this funciton, I've added try/except wrapping around
it to ensure we catch exceptions in the exception handler too.
Example before:
"""
ERROR: Error executing a python function in /media/build1/poky/meta/recipes-core/eglibc/eglibc-initial_2.17.bb:
TypeError: 'filter' object is not subscriptable
ERROR: The stack trace of python calls that resulted in this exception/failure was:
ERROR: File "do_populate_lic", line 13, in <module>
ERROR:
ERROR: File "do_populate_lic", line 6, in do_populate_lic
ERROR:
ERROR: File "license.bbclass", line 99, in find_license_files
ERROR:
ERROR: File "/media/build1/poky/meta/lib/oe/license.py", line 38, in visit_string
ERROR: if pos > 0 and license_pattern.match(elements[pos-1]):
ERROR:
ERROR: The code that was being executed was:
ERROR: 0009: destdir = os.path.join(d.getVar('LICSSTATEDIR', True), d.getVar('PN', True))
ERROR: 0010: copy_license_files(lic_files_paths, destdir)
ERROR: 0011:
ERROR: 0012:
ERROR: *** 0013:do_populate_lic(d)
ERROR: 0014:
ERROR: [From file: 'do_populate_lic', lineno: 13, function: <module>]
ERROR: 0002:def do_populate_lic(d):
ERROR: 0003: """
ERROR: 0004: Populate LICENSE_DIRECTORY with licenses.
ERROR: 0005: """
ERROR: *** 0006: lic_files_paths = find_license_files(d)
ERROR: 0007:
ERROR: 0008: # The base directory we wrangle licenses to
ERROR: 0009: destdir = os.path.join(d.getVar('LICSSTATEDIR', True), d.getVar('PN', True))
ERROR: 0010: copy_license_files(lic_files_paths, destdir)
ERROR: [From file: 'do_populate_lic', lineno: 6, function: do_populate_lic]
ERROR: 0095: lic_files_paths.append((os.path.basename(path), srclicfile))
ERROR: 0096:
ERROR: 0097: v = FindVisitor()
ERROR: 0098: try:
ERROR: *** 0099: v.visit_string(license_types)
ERROR: 0100: except oe.license.InvalidLicense as exc:
ERROR: 0101: bb.fatal('%s: %s' % (d.getVar('PF', True), exc))
ERROR: 0102: except SyntaxError:
ERROR: 0103: bb.warn("%s: Failed to parse it's LICENSE field." % (d.getVar('PF', True)))
ERROR: [From file: 'license.bbclass', lineno: 99, function: find_license_files]
ERROR: Function failed: do_populate_lic
ERROR: Logfile of failure stored in: /media/build1/poky/build/tmp/work/i586-poky-linux/eglibc-initial/2.17-r3/temp/log.do_populate_lic.17442
"""
Example after:
"""
ERROR: Error executing a python function in /media/build1/poky/meta/recipes-core/eglibc/eglibc-initial_2.17.bb:
The stack trace of python calls that resulted in this exception/failure was:
File: 'do_populate_lic', lineno: 13, function: <module>
0009: destdir = os.path.join(d.getVar('LICSSTATEDIR', True), d.getVar('PN', True))
0010: copy_license_files(lic_files_paths, destdir)
0011:
0012:
*** 0013:do_populate_lic(d)
0014:
File: 'do_populate_lic', lineno: 6, function: do_populate_lic
0002:def do_populate_lic(d):
0003: """
0004: Populate LICENSE_DIRECTORY with licenses.
0005: """
*** 0006: lic_files_paths = find_license_files(d)
0007:
0008: # The base directory we wrangle licenses to
0009: destdir = os.path.join(d.getVar('LICSSTATEDIR', True), d.getVar('PN', True))
0010: copy_license_files(lic_files_paths, destdir)
File: 'license.bbclass', lineno: 99, function: find_license_files
0095: lic_files_paths.append((os.path.basename(path), srclicfile))
0096:
0097: v = FindVisitor()
0098: try:
*** 0099: v.visit_string(license_types)
0100: except oe.license.InvalidLicense as exc:
0101: bb.fatal('%s: %s' % (d.getVar('PF', True), exc))
0102: except SyntaxError:
0103: bb.warn("%s: Failed to parse it's LICENSE field." % (d.getVar('PF', True)))
File: '/media/build1/poky/meta/lib/oe/license.py', lineno: 38, function: visit_string
0034: new_elements = []
0035: elements = filter(lambda x: x.strip(), license_operator.split(licensestr))
0036: for pos, element in enumerate(elements):
0037: if license_pattern.match(element):
*** 0038: if pos > 0 and license_pattern.match(elements[pos-1]):
0039: new_elements.append('&')
0040: element = '"' + element + '"'
0041: elif not license_operator.match(element):
0042: raise InvalidLicense(element)
Exception: TypeError: 'filter' object is not subscriptable
ERROR: Function failed: do_populate_lic
ERROR: Logfile of failure stored in: /media/build1/poky/build/tmp/work/i586-poky-linux/eglibc-initial/2.17-r3/temp/log.do_populate_lic.3275
ERROR: Task 9 (/media/build1/poky/meta/recipes-core/eglibc/eglibc-initial_2.17.bb, do_populate_lic) failed with exit code '1
"""
(Bitbake rev: c5de66b870406d9bd1161a9b7e2b04fe6eb065fe)
Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
2013-05-08 17:14:53 +00:00
except Exception as e :
( t , value , tb ) = sys . exc_info ( )
if t in [ bb . parse . SkipPackage , bb . build . FuncFailed ] :
raise
try :
_print_exception ( t , value , tb , realfile , text , context )
except Exception as e :
logger . error ( " Exception handler error: %s " % str ( e ) )
2012-08-22 19:02:39 +00:00
e = bb . BBHandledException ( e )
raise e
2006-11-16 15:02:15 +00:00
2010-03-31 15:52:41 +00:00
def simple_exec ( code , context ) :
2013-05-23 09:49:57 +00:00
exec ( code , get_context ( ) , context )
2010-03-31 15:52:41 +00:00
2010-03-31 03:06:07 +00:00
def better_eval ( source , locals ) :
2013-05-23 09:49:57 +00:00
return eval ( source , get_context ( ) , locals )
2010-03-31 03:06:07 +00:00
2010-12-10 01:29:31 +00:00
@contextmanager
def fileslocked ( files ) :
2010-12-10 02:50:23 +00:00
""" Context manager for locking and unlocking file locks. """
2010-12-10 01:29:31 +00:00
locks = [ ]
if files :
for lockfile in files :
2010-12-10 21:27:28 +00:00
locks . append ( bb . utils . lockfile ( lockfile ) )
2010-12-10 01:29:31 +00:00
yield
for lock in locks :
bb . utils . unlockfile ( lock )
2011-05-27 15:13:27 +00:00
def lockfile ( name , shared = False , retry = True ) :
2007-11-24 14:44:36 +00:00
"""
2007-11-25 13:54:43 +00:00
Use the file fn as a lock file , return when the lock has been acquired .
2007-11-24 14:44:36 +00:00
Returns a variable to pass to unlockfile ( ) .
"""
2011-03-08 16:33:40 +00:00
dirname = os . path . dirname ( name )
mkdirhier ( dirname )
2008-10-17 11:04:26 +00:00
2011-03-08 16:33:40 +00:00
if not os . access ( dirname , os . W_OK ) :
logger . error ( " Unable to acquire lock ' %s ' , directory is not writable " ,
2011-03-08 16:39:40 +00:00
name )
2010-12-29 07:28:48 +00:00
sys . exit ( 1 )
2011-01-19 11:01:54 +00:00
op = fcntl . LOCK_EX
if shared :
op = fcntl . LOCK_SH
2011-05-27 15:13:27 +00:00
if not retry :
op = op | fcntl . LOCK_NB
2011-01-19 11:01:54 +00:00
2007-11-24 14:44:36 +00:00
while True :
# If we leave the lockfiles lying around there is no problem
# but we should clean up after ourselves. This gives potential
2010-03-24 23:56:12 +00:00
# for races though. To work around this, when we acquire the lock
# we check the file we locked was still the lock file on disk.
# by comparing inode numbers. If they don't match or the lockfile
2007-11-24 14:44:36 +00:00
# no longer exists, we start again.
2010-03-24 23:56:12 +00:00
# This implementation is unfair since the last person to request the
2007-11-24 14:44:36 +00:00
# lock is the most likely to win it.
2008-10-06 08:09:11 +00:00
try :
2010-12-10 02:50:23 +00:00
lf = open ( name , ' a+ ' )
fileno = lf . fileno ( )
2011-01-19 11:01:54 +00:00
fcntl . flock ( fileno , op )
2010-12-10 02:50:23 +00:00
statinfo = os . fstat ( fileno )
2008-10-06 08:09:11 +00:00
if os . path . exists ( lf . name ) :
2010-03-24 23:56:12 +00:00
statinfo2 = os . stat ( lf . name )
if statinfo . st_ino == statinfo2 . st_ino :
return lf
2010-12-10 02:50:23 +00:00
lf . close ( )
except Exception :
2013-05-09 21:06:45 +00:00
try :
lf . close ( )
except Exception :
pass
2011-11-08 17:49:32 +00:00
pass
2011-05-27 15:13:27 +00:00
if not retry :
return None
2007-11-24 14:44:36 +00:00
def unlockfile ( lf ) :
"""
2010-03-24 23:56:12 +00:00
Unlock a file locked using lockfile ( )
2007-11-24 14:44:36 +00:00
"""
2011-01-19 13:30:14 +00:00
try :
2011-03-08 16:33:40 +00:00
# If we had a shared lock, we need to promote to exclusive before
2011-01-19 13:30:14 +00:00
# removing the lockfile. Attempt this, ignore failures.
fcntl . flock ( lf . fileno ( ) , fcntl . LOCK_EX | fcntl . LOCK_NB )
os . unlink ( lf . name )
2011-01-26 12:20:14 +00:00
except ( IOError , OSError ) :
2011-01-19 13:30:14 +00:00
pass
2007-11-24 14:44:36 +00:00
fcntl . flock ( lf . fileno ( ) , fcntl . LOCK_UN )
2010-12-10 02:50:23 +00:00
lf . close ( )
2007-11-24 14:44:36 +00:00
2008-05-01 10:59:00 +00:00
def md5_file ( filename ) :
"""
Return the hex string representation of the MD5 checksum of filename .
"""
try :
import hashlib
m = hashlib . md5 ( )
except ImportError :
import md5
m = md5 . new ( )
2010-03-24 23:56:12 +00:00
2013-05-09 21:06:45 +00:00
with open ( filename , " rb " ) as f :
for line in f :
m . update ( line )
2008-05-01 10:59:00 +00:00
return m . hexdigest ( )
def sha256_file ( filename ) :
"""
Return the hex string representation of the 256 - bit SHA checksum of
filename . On Python 2.4 this will return None , so callers will need to
handle that by either skipping SHA checks , or running a standalone sha256sum
binary .
"""
try :
import hashlib
except ImportError :
return None
s = hashlib . sha256 ( )
2013-05-09 21:06:45 +00:00
with open ( filename , " rb " ) as f :
for line in f :
s . update ( line )
2008-05-01 10:59:00 +00:00
return s . hexdigest ( )
2008-08-18 07:56:04 +00:00
2011-01-10 12:03:40 +00:00
def preserved_envvars_exported ( ) :
""" Variables which are taken from the environment and placed in and exported
from the metadata """
2008-09-30 20:57:18 +00:00
return [
2010-08-31 13:49:43 +00:00
' BB_TASKHASH ' ,
2010-11-28 17:39:09 +00:00
' HOME ' ,
' LOGNAME ' ,
' PATH ' ,
' PWD ' ,
' SHELL ' ,
' TERM ' ,
' USER ' ,
]
2011-01-10 12:03:40 +00:00
def preserved_envvars ( ) :
""" Variables which are taken from the environment and placed in the metadata """
2010-11-28 17:39:09 +00:00
v = [
' BBPATH ' ,
' BB_PRESERVE_ENV ' ,
' BB_ENV_WHITELIST ' ,
' BB_ENV_EXTRAWHITE ' ,
]
2013-02-05 16:02:03 +00:00
return v + preserved_envvars_exported ( )
2010-11-28 17:39:09 +00:00
2008-09-30 20:57:18 +00:00
def filter_environment ( good_vars ) :
"""
Create a pristine environment for bitbake . This will remove variables that
are not known and may influence the build in a negative way .
"""
2013-02-11 11:00:45 +00:00
removed_vars = { }
2008-09-30 20:57:18 +00:00
for key in os . environ . keys ( ) :
if key in good_vars :
continue
2010-03-24 23:56:12 +00:00
2013-02-11 11:00:45 +00:00
removed_vars [ key ] = os . environ [ key ]
2008-09-30 20:57:18 +00:00
os . unsetenv ( key )
del os . environ [ key ]
if len ( removed_vars ) :
2013-02-11 11:00:45 +00:00
logger . debug ( 1 , " Removed the following variables from the environment: %s " , " , " . join ( removed_vars . keys ( ) ) )
2008-09-30 20:57:18 +00:00
return removed_vars
2011-08-13 00:58:11 +00:00
def approved_variables ( ) :
"""
Determine and return the list of whitelisted variables which are approved
to remain in the envrionment .
"""
2013-09-11 10:01:47 +00:00
if ' BB_PRESERVE_ENV ' in os . environ :
return os . environ . keys ( )
2011-08-13 00:58:11 +00:00
approved = [ ]
if ' BB_ENV_WHITELIST ' in os . environ :
approved = os . environ [ ' BB_ENV_WHITELIST ' ] . split ( )
2013-03-16 08:07:41 +00:00
approved . extend ( [ ' BB_ENV_WHITELIST ' ] )
2011-08-13 00:58:11 +00:00
else :
approved = preserved_envvars ( )
if ' BB_ENV_EXTRAWHITE ' in os . environ :
approved . extend ( os . environ [ ' BB_ENV_EXTRAWHITE ' ] . split ( ) )
2013-03-16 08:07:41 +00:00
if ' BB_ENV_EXTRAWHITE ' not in approved :
approved . extend ( [ ' BB_ENV_EXTRAWHITE ' ] )
2011-08-13 00:58:11 +00:00
return approved
2008-10-17 09:46:35 +00:00
def clean_environment ( ) :
"""
Clean up any spurious environment variables . This will remove any
2011-08-13 00:58:11 +00:00
variables the user hasn ' t chosen to preserve.
2008-10-17 09:46:35 +00:00
"""
if ' BB_PRESERVE_ENV ' not in os . environ :
2011-08-13 00:58:11 +00:00
good_vars = approved_variables ( )
2013-02-11 11:00:45 +00:00
return filter_environment ( good_vars )
return { }
2008-10-17 09:46:35 +00:00
def empty_environment ( ) :
"""
Remove all variables from the environment .
"""
for s in os . environ . keys ( ) :
os . unsetenv ( s )
del os . environ [ s ]
2009-07-02 13:33:53 +00:00
def build_environment ( d ) :
"""
Build an environment from all exported variables .
"""
2010-04-08 17:22:29 +00:00
import bb . data
2009-07-02 13:33:53 +00:00
for var in bb . data . keys ( d ) :
2011-11-25 14:57:53 +00:00
export = d . getVarFlag ( var , " export " )
2009-07-02 13:33:53 +00:00
if export :
2011-11-25 14:57:53 +00:00
os . environ [ var ] = d . getVar ( var , True ) or " "
2009-07-02 13:33:53 +00:00
2010-12-10 01:14:48 +00:00
def remove ( path , recurse = False ) :
""" Equivalent to rm -f or rm -rf """
2011-01-10 22:05:19 +00:00
if not path :
return
2013-02-03 16:51:34 +00:00
if recurse :
# shutil.rmtree(name) would be ideal but its too slow
2013-02-19 22:31:52 +00:00
subprocess . call ( [ ' rm ' , ' -rf ' ] + glob . glob ( path ) )
2013-02-03 16:51:34 +00:00
return
2011-02-08 17:24:12 +00:00
for name in glob . glob ( path ) :
try :
os . unlink ( name )
except OSError as exc :
2013-02-03 16:51:34 +00:00
if exc . errno != errno . ENOENT :
2011-02-08 17:24:12 +00:00
raise
2010-12-10 01:14:48 +00:00
2008-08-18 07:56:04 +00:00
def prunedir ( topdir ) :
# Delete everything reachable from the directory named in 'topdir'.
# CAUTION: This is dangerous!
2010-03-24 23:56:12 +00:00
for root , dirs , files in os . walk ( topdir , topdown = False ) :
2008-08-18 07:56:04 +00:00
for name in files :
os . remove ( os . path . join ( root , name ) )
for name in dirs :
2008-10-20 14:19:36 +00:00
if os . path . islink ( os . path . join ( root , name ) ) :
os . remove ( os . path . join ( root , name ) )
else :
os . rmdir ( os . path . join ( root , name ) )
2008-08-18 07:56:04 +00:00
os . rmdir ( topdir )
2008-09-30 20:46:17 +00:00
2008-12-31 22:58:57 +00:00
#
# Could also use return re.compile("(%s)" % "|".join(map(re.escape, suffixes))).sub(lambda mo: "", var)
# but thats possibly insane and suffixes is probably going to be small
#
def prune_suffix ( var , suffixes , d ) :
2010-03-24 23:56:12 +00:00
# See if var ends with any of the suffixes listed and
2008-12-31 22:58:57 +00:00
# remove it if found
for suffix in suffixes :
if var . endswith ( suffix ) :
return var . replace ( suffix , " " )
return var
2009-07-19 17:05:52 +00:00
2010-12-17 11:29:49 +00:00
def mkdirhier ( directory ) :
2009-07-19 17:05:52 +00:00
""" Create a directory like ' mkdir -p ' , but does not complain if
directory already exists like os . makedirs
"""
try :
2010-12-17 11:29:49 +00:00
os . makedirs ( directory )
2010-04-12 00:03:55 +00:00
except OSError as e :
2010-04-08 17:22:29 +00:00
if e . errno != errno . EEXIST :
raise e
2009-07-19 17:05:52 +00:00
2010-03-24 23:56:12 +00:00
def movefile ( src , dest , newmtime = None , sstat = None ) :
2009-07-19 17:05:52 +00:00
""" Moves a file from src to dest, preserving all permissions and
attributes ; mtime will be preserved even when moving across
filesystems . Returns true on success and false on failure . Move is
atomic .
"""
2010-03-24 23:56:12 +00:00
#print "movefile(" + src + "," + dest + "," + str(newmtime) + "," + str(sstat) + ")"
2009-07-19 17:05:52 +00:00
try :
if not sstat :
2010-03-24 23:56:12 +00:00
sstat = os . lstat ( src )
2010-04-12 00:03:55 +00:00
except Exception as e :
2010-04-10 02:46:14 +00:00
print ( " movefile: Stating source file failed... " , e )
2009-07-19 17:05:52 +00:00
return None
2010-03-24 23:56:12 +00:00
destexists = 1
2009-07-19 17:05:52 +00:00
try :
2010-03-24 23:56:12 +00:00
dstat = os . lstat ( dest )
2009-07-19 17:05:52 +00:00
except :
2010-03-24 23:56:12 +00:00
dstat = os . lstat ( os . path . dirname ( dest ) )
destexists = 0
2009-07-19 17:05:52 +00:00
if destexists :
if stat . S_ISLNK ( dstat [ stat . ST_MODE ] ) :
try :
os . unlink ( dest )
2010-03-24 23:56:12 +00:00
destexists = 0
2010-04-12 00:03:55 +00:00
except Exception as e :
2009-07-19 17:05:52 +00:00
pass
if stat . S_ISLNK ( sstat [ stat . ST_MODE ] ) :
try :
2010-03-24 23:56:12 +00:00
target = os . readlink ( src )
2009-07-19 17:05:52 +00:00
if destexists and not stat . S_ISDIR ( dstat [ stat . ST_MODE ] ) :
os . unlink ( dest )
2010-03-24 23:56:12 +00:00
os . symlink ( target , dest )
2009-07-19 17:05:52 +00:00
#os.lchown(dest,sstat[stat.ST_UID],sstat[stat.ST_GID])
os . unlink ( src )
return os . lstat ( dest )
2010-04-12 00:03:55 +00:00
except Exception as e :
2010-04-10 02:46:14 +00:00
print ( " movefile: failed to properly create symlink: " , dest , " -> " , target , e )
2009-07-19 17:05:52 +00:00
return None
2010-03-24 23:56:12 +00:00
renamefailed = 1
if sstat [ stat . ST_DEV ] == dstat [ stat . ST_DEV ] :
2009-07-19 17:05:52 +00:00
try :
2010-04-09 18:47:08 +00:00
os . rename ( src , dest )
2010-03-24 23:56:12 +00:00
renamefailed = 0
2010-04-12 00:03:55 +00:00
except Exception as e :
2010-03-24 23:56:12 +00:00
if e [ 0 ] != errno . EXDEV :
2009-07-19 17:05:52 +00:00
# Some random error.
2010-04-10 02:46:14 +00:00
print ( " movefile: Failed to move " , src , " to " , dest , e )
2009-07-19 17:05:52 +00:00
return None
# Invalid cross-device-link 'bind' mounted or actually Cross-Device
if renamefailed :
2010-03-24 23:56:12 +00:00
didcopy = 0
2009-07-19 17:05:52 +00:00
if stat . S_ISREG ( sstat [ stat . ST_MODE ] ) :
try : # For safety copy then move it over.
2010-03-24 23:56:12 +00:00
shutil . copyfile ( src , dest + " #new " )
os . rename ( dest + " #new " , dest )
didcopy = 1
2010-04-12 00:03:55 +00:00
except Exception as e :
2010-04-10 02:46:14 +00:00
print ( ' movefile: copy ' , src , ' -> ' , dest , ' failed. ' , e )
2009-07-19 17:05:52 +00:00
return None
else :
#we don't yet handle special, so we need to fall back to /bin/mv
2010-03-24 23:56:12 +00:00
a = getstatusoutput ( " /bin/mv -f " + " ' " + src + " ' ' " + dest + " ' " )
if a [ 0 ] != 0 :
2010-04-10 02:46:14 +00:00
print ( " movefile: Failed to move special file: " + src + " ' to ' " + dest + " ' " , a )
2009-07-19 17:05:52 +00:00
return None # failure
try :
if didcopy :
2010-03-24 23:56:12 +00:00
os . lchown ( dest , sstat [ stat . ST_UID ] , sstat [ stat . ST_GID ] )
2009-07-19 17:05:52 +00:00
os . chmod ( dest , stat . S_IMODE ( sstat [ stat . ST_MODE ] ) ) # Sticky is reset on chown
os . unlink ( src )
2010-04-12 00:03:55 +00:00
except Exception as e :
2010-04-10 02:46:14 +00:00
print ( " movefile: Failed to chown/chmod/unlink " , dest , e )
2009-07-19 17:05:52 +00:00
return None
if newmtime :
2010-03-24 23:56:12 +00:00
os . utime ( dest , ( newmtime , newmtime ) )
2009-07-19 17:05:52 +00:00
else :
os . utime ( dest , ( sstat [ stat . ST_ATIME ] , sstat [ stat . ST_MTIME ] ) )
2010-03-24 23:56:12 +00:00
newmtime = sstat [ stat . ST_MTIME ]
2009-07-19 17:05:52 +00:00
return newmtime
2010-03-24 23:56:12 +00:00
def copyfile ( src , dest , newmtime = None , sstat = None ) :
2009-07-19 17:05:52 +00:00
"""
Copies a file from src to dest , preserving all permissions and
attributes ; mtime will be preserved even when moving across
filesystems . Returns true on success and false on failure .
"""
2010-03-24 23:56:12 +00:00
#print "copyfile(" + src + "," + dest + "," + str(newmtime) + "," + str(sstat) + ")"
2009-07-19 17:05:52 +00:00
try :
if not sstat :
2010-03-24 23:56:12 +00:00
sstat = os . lstat ( src )
2010-04-12 00:03:55 +00:00
except Exception as e :
2013-10-02 16:47:28 +00:00
logger . warn ( " copyfile: stat of %s failed ( %s ) " % ( src , e ) )
2009-07-19 17:05:52 +00:00
return False
2010-03-24 23:56:12 +00:00
destexists = 1
2009-07-19 17:05:52 +00:00
try :
2010-03-24 23:56:12 +00:00
dstat = os . lstat ( dest )
2009-07-19 17:05:52 +00:00
except :
2010-03-24 23:56:12 +00:00
dstat = os . lstat ( os . path . dirname ( dest ) )
destexists = 0
2009-07-19 17:05:52 +00:00
if destexists :
if stat . S_ISLNK ( dstat [ stat . ST_MODE ] ) :
try :
os . unlink ( dest )
2010-03-24 23:56:12 +00:00
destexists = 0
2010-04-12 00:03:55 +00:00
except Exception as e :
2009-07-19 17:05:52 +00:00
pass
if stat . S_ISLNK ( sstat [ stat . ST_MODE ] ) :
try :
2010-03-24 23:56:12 +00:00
target = os . readlink ( src )
2009-07-19 17:05:52 +00:00
if destexists and not stat . S_ISDIR ( dstat [ stat . ST_MODE ] ) :
os . unlink ( dest )
2010-03-24 23:56:12 +00:00
os . symlink ( target , dest )
2009-07-19 17:05:52 +00:00
#os.lchown(dest,sstat[stat.ST_UID],sstat[stat.ST_GID])
return os . lstat ( dest )
2010-04-12 00:03:55 +00:00
except Exception as e :
2013-10-02 16:47:28 +00:00
logger . warn ( " copyfile: failed to create symlink %s to %s ( %s ) " % ( dest , target , e ) )
2009-07-19 17:05:52 +00:00
return False
if stat . S_ISREG ( sstat [ stat . ST_MODE ] ) :
2011-02-25 17:22:00 +00:00
try :
srcchown = False
if not os . access ( src , os . R_OK ) :
# Make sure we can read it
srcchown = True
os . chmod ( src , sstat [ stat . ST_MODE ] | stat . S_IRUSR )
# For safety copy then move it over.
2010-03-24 23:56:12 +00:00
shutil . copyfile ( src , dest + " #new " )
os . rename ( dest + " #new " , dest )
2010-04-12 00:03:55 +00:00
except Exception as e :
2013-10-02 16:47:28 +00:00
logger . warn ( " copyfile: copy %s to %s failed ( %s ) " % ( src , dest , e ) )
2010-03-24 23:56:12 +00:00
return False
2010-08-23 12:01:05 +00:00
finally :
2011-02-25 17:22:00 +00:00
if srcchown :
os . chmod ( src , sstat [ stat . ST_MODE ] )
os . utime ( src , ( sstat [ stat . ST_ATIME ] , sstat [ stat . ST_MTIME ] ) )
2010-08-23 12:01:05 +00:00
2009-07-19 17:05:52 +00:00
else :
2010-03-24 23:56:12 +00:00
#we don't yet handle special, so we need to fall back to /bin/mv
a = getstatusoutput ( " /bin/cp -f " + " ' " + src + " ' ' " + dest + " ' " )
if a [ 0 ] != 0 :
2013-10-02 16:47:28 +00:00
logger . warn ( " copyfile: failed to copy special file %s to %s ( %s ) " % ( src , dest , a ) )
2010-03-24 23:56:12 +00:00
return False # failure
2009-07-19 17:05:52 +00:00
try :
2010-03-24 23:56:12 +00:00
os . lchown ( dest , sstat [ stat . ST_UID ] , sstat [ stat . ST_GID ] )
2009-07-19 17:05:52 +00:00
os . chmod ( dest , stat . S_IMODE ( sstat [ stat . ST_MODE ] ) ) # Sticky is reset on chown
2010-04-12 00:03:55 +00:00
except Exception as e :
2013-10-02 16:47:28 +00:00
logger . warn ( " copyfile: failed to chown/chmod %s ( %s ) " % ( dest , e ) )
2009-07-19 17:05:52 +00:00
return False
if newmtime :
2010-03-24 23:56:12 +00:00
os . utime ( dest , ( newmtime , newmtime ) )
2009-07-19 17:05:52 +00:00
else :
os . utime ( dest , ( sstat [ stat . ST_ATIME ] , sstat [ stat . ST_MTIME ] ) )
2010-03-24 23:56:12 +00:00
newmtime = sstat [ stat . ST_MTIME ]
2009-07-19 17:05:52 +00:00
return newmtime
2013-11-29 23:15:56 +00:00
def which ( path , item , direction = 0 , history = False ) :
2009-07-19 17:05:52 +00:00
"""
Locate a file in a PATH
"""
2013-11-29 23:15:56 +00:00
hist = [ ]
2009-07-19 17:05:52 +00:00
paths = ( path or " " ) . split ( ' : ' )
if direction != 0 :
paths . reverse ( )
2010-03-24 17:31:06 +00:00
for p in paths :
2009-07-19 17:05:52 +00:00
next = os . path . join ( p , item )
2013-11-29 23:15:56 +00:00
hist . append ( next )
2009-07-19 17:05:52 +00:00
if os . path . exists ( next ) :
2012-05-24 12:57:16 +00:00
if not os . path . isabs ( next ) :
next = os . path . abspath ( next )
2013-11-29 23:15:56 +00:00
if history :
return next , hist
2009-07-19 17:05:52 +00:00
return next
2013-11-29 23:15:56 +00:00
if history :
return " " , hist
2009-07-19 17:05:52 +00:00
return " "
2010-06-29 12:32:04 +00:00
2011-02-11 17:43:54 +00:00
def to_boolean ( string , default = None ) :
if not string :
return default
normalized = string . lower ( )
if normalized in ( " y " , " yes " , " 1 " , " true " ) :
return True
elif normalized in ( " n " , " no " , " 0 " , " false " ) :
return False
else :
raise ValueError ( " Invalid value for to_boolean: %s " % string )
2011-07-11 13:38:50 +00:00
def contains ( variable , checkvalues , truevalue , falsevalue , d ) :
val = d . getVar ( variable , True )
if not val :
return falsevalue
2011-07-25 18:02:48 +00:00
val = set ( val . split ( ) )
if isinstance ( checkvalues , basestring ) :
checkvalues = set ( checkvalues . split ( ) )
else :
checkvalues = set ( checkvalues )
if checkvalues . issubset ( val ) :
2011-07-11 13:38:50 +00:00
return truevalue
return falsevalue
2012-02-23 13:47:19 +00:00
def cpu_count ( ) :
return multiprocessing . cpu_count ( )
2012-06-22 11:53:16 +00:00
def nonblockingfd ( fd ) :
fcntl . fcntl ( fd , fcntl . F_SETFL , fcntl . fcntl ( fd , fcntl . F_GETFL ) | os . O_NONBLOCK )
2013-01-28 14:40:04 +00:00
def process_profilelog ( fn ) :
# Redirect stdout to capture profile information
pout = open ( fn + ' .processed ' , ' w ' )
so = sys . stdout . fileno ( )
orig_so = os . dup ( sys . stdout . fileno ( ) )
os . dup2 ( pout . fileno ( ) , so )
import pstats
p = pstats . Stats ( fn )
p . sort_stats ( ' time ' )
p . print_stats ( )
p . print_callers ( )
p . sort_stats ( ' cumulative ' )
p . print_stats ( )
os . dup2 ( orig_so , so )
pout . flush ( )
pout . close ( )
2013-02-06 00:28:08 +00:00
#
2013-06-14 16:22:51 +00:00
# Was present to work around multiprocessing pool bugs in python < 2.7.3
2013-02-06 00:28:08 +00:00
#
def multiprocessingpool ( * args , * * kwargs ) :
2013-06-14 21:00:29 +00:00
return multiprocessing . Pool ( * args , * * kwargs )
2013-01-28 14:40:04 +00:00