2011-01-10 14:23:36 +00:00
# ex:ts=4:sw=4:sts=4:et
# -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*-
"""
BitBake ' Fetch ' implementations
Classes for obtaining upstream sources for the
BitBake build tools .
"""
# Copyright (C) 2003, 2004 Chris Larson
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License version 2 as
# published by the Free Software Foundation.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License along
# with this program; if not, write to the Free Software Foundation, Inc.,
# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
#
# Based on functions from the base bb module, Copyright 2003 Holger Schurig
2014-11-05 18:10:29 +00:00
import re
import tempfile
import subprocess
2011-01-10 14:23:36 +00:00
import os
import logging
import bb
2016-05-12 07:30:35 +00:00
import urllib . request , urllib . parse , urllib . error
2011-01-10 14:23:36 +00:00
from bb import data
2011-02-07 23:42:45 +00:00
from bb . fetch2 import FetchMethod
from bb . fetch2 import FetchError
from bb . fetch2 import logger
from bb . fetch2 import runfetchcmd
2016-01-28 22:27:52 +00:00
from bb . utils import export_proxies
2014-11-05 18:10:29 +00:00
from bs4 import BeautifulSoup
2015-12-04 11:00:20 +00:00
from bs4 import SoupStrainer
2011-01-10 14:23:36 +00:00
2011-02-04 10:49:27 +00:00
class Wget ( FetchMethod ) :
2011-01-10 14:23:36 +00:00
""" Class to fetch urls via ' wget ' """
2013-11-19 14:32:08 +00:00
def supports ( self , ud , d ) :
2011-01-10 14:23:36 +00:00
"""
Check to see if a given url can be fetched with wget .
"""
return ud . type in [ ' http ' , ' https ' , ' ftp ' ]
2012-05-15 19:49:36 +00:00
def recommends_checksum ( self , urldata ) :
return True
2011-02-03 21:17:26 +00:00
def urldata_init ( self , ud , d ) :
2012-10-03 12:27:57 +00:00
if ' protocol ' in ud . parm :
if ud . parm [ ' protocol ' ] == ' git ' :
raise bb . fetch2 . ParameterError ( " Invalid protocol - if you wish to fetch from a git repository using http, you need to instead use the git:// prefix with protocol=http " , ud . url )
2011-01-10 14:23:36 +00:00
2012-07-19 00:39:18 +00:00
if ' downloadfilename ' in ud . parm :
ud . basename = ud . parm [ ' downloadfilename ' ]
else :
ud . basename = os . path . basename ( ud . path )
2016-05-12 07:30:35 +00:00
ud . localfile = data . expand ( urllib . parse . unquote ( ud . basename ) , d )
2016-02-26 17:48:55 +00:00
if not ud . localfile :
2016-05-12 07:30:35 +00:00
ud . localfile = data . expand ( urllib . parse . unquote ( ud . host + ud . path ) . replace ( " / " , " . " ) , d )
2011-01-10 14:23:36 +00:00
2014-02-28 17:25:21 +00:00
self . basecmd = d . getVar ( " FETCHCMD_wget " , True ) or " /usr/bin/env wget -t 2 -T 30 -nv --passive-ftp --no-check-certificate "
2014-02-28 17:26:20 +00:00
def _runwget ( self , ud , d , command , quiet ) :
logger . debug ( 2 , " Fetching %s using command ' %s ' " % ( ud . url , command ) )
bb . fetch2 . check_network_access ( d , command )
runfetchcmd ( command , d , quiet )
def download ( self , ud , d ) :
2011-01-10 14:23:36 +00:00
""" Fetch urls """
2014-02-28 17:25:21 +00:00
fetchcmd = self . basecmd
2012-07-09 16:44:08 +00:00
2014-02-28 17:26:20 +00:00
if ' downloadfilename ' in ud . parm :
2013-02-19 21:52:38 +00:00
dldir = d . getVar ( " DL_DIR " , True )
bb . utils . mkdirhier ( os . path . dirname ( dldir + os . sep + ud . localfile ) )
2014-02-28 17:25:21 +00:00
fetchcmd + = " -O " + dldir + os . sep + ud . localfile
2012-07-19 00:39:18 +00:00
2014-02-28 17:25:21 +00:00
uri = ud . url . split ( " ; " ) [ 0 ]
2014-02-28 17:26:20 +00:00
if os . path . exists ( ud . localpath ) :
2012-05-04 17:26:40 +00:00
# file exists, but we didnt complete it.. trying again..
2014-03-09 18:11:30 +00:00
fetchcmd + = d . expand ( " -c -P $ {DL_DIR} ' %s ' " % uri )
2012-05-04 17:26:40 +00:00
else :
2014-03-09 18:11:30 +00:00
fetchcmd + = d . expand ( " -P $ {DL_DIR} ' %s ' " % uri )
2011-01-10 14:23:36 +00:00
2014-02-28 17:26:20 +00:00
self . _runwget ( ud , d , fetchcmd , False )
2011-01-10 14:23:36 +00:00
2012-05-04 17:26:40 +00:00
# Sanity check since wget can pretend it succeed when it didn't
# Also, this used to happen if sourceforge sent us to the mirror page
2014-02-28 17:26:20 +00:00
if not os . path . exists ( ud . localpath ) :
2012-05-04 17:26:40 +00:00
raise FetchError ( " The fetch command returned success for url %s but %s doesn ' t exist?! " % ( uri , ud . localpath ) , uri )
2012-05-04 17:27:31 +00:00
2014-02-28 17:26:20 +00:00
if os . path . getsize ( ud . localpath ) == 0 :
2014-01-21 09:58:21 +00:00
os . remove ( ud . localpath )
raise FetchError ( " The fetch of %s resulted in a zero size file?! Deleting and failing since this isn ' t right. " % ( uri ) , uri )
2011-02-04 10:26:21 +00:00
return True
2011-01-10 14:23:36 +00:00
2015-06-30 14:39:11 +00:00
def checkstatus ( self , fetch , ud , d ) :
2016-05-12 07:30:35 +00:00
import urllib . request , urllib . error , urllib . parse , socket , http . client
from urllib . response import addinfourl
2015-07-08 23:34:17 +00:00
from bb . fetch2 import FetchConnectionCache
2016-05-12 07:30:35 +00:00
class HTTPConnectionCache ( http . client . HTTPConnection ) :
2015-07-08 23:34:17 +00:00
if fetch . connection_cache :
def connect ( self ) :
""" Connect to the host and port specified in __init__. """
sock = fetch . connection_cache . get_connection ( self . host , self . port )
if sock :
self . sock = sock
else :
self . sock = socket . create_connection ( ( self . host , self . port ) ,
self . timeout , self . source_address )
fetch . connection_cache . add_connection ( self . host , self . port , self . sock )
if self . _tunnel_host :
self . _tunnel ( )
2016-05-12 07:30:35 +00:00
class CacheHTTPHandler ( urllib . request . HTTPHandler ) :
2015-07-08 23:34:17 +00:00
def http_open ( self , req ) :
return self . do_open ( HTTPConnectionCache , req )
def do_open ( self , http_class , req ) :
""" Return an addinfourl object for the request, using http_class.
http_class must implement the HTTPConnection API from httplib .
The addinfourl return value is a file - like object . It also
has methods and attributes including :
- info ( ) : return a mimetools . Message object for the headers
- geturl ( ) : return the original request URL
- code : HTTP status code
"""
2016-05-12 07:30:35 +00:00
host = req . host
2015-07-08 23:34:17 +00:00
if not host :
raise urlllib2 . URLError ( ' no host given ' )
h = http_class ( host , timeout = req . timeout ) # will parse host:port
h . set_debuglevel ( self . _debuglevel )
headers = dict ( req . unredirected_hdrs )
2016-05-12 07:30:35 +00:00
headers . update ( dict ( ( k , v ) for k , v in list ( req . headers . items ( ) )
2015-07-08 23:34:17 +00:00
if k not in headers ) )
# We want to make an HTTP/1.1 request, but the addinfourl
# class isn't prepared to deal with a persistent connection.
# It will try to read all remaining data from the socket,
# which will block while the server waits for the next request.
# So make sure the connection gets closed after the (only)
# request.
# Don't close connection when connection_cache is enabled,
if fetch . connection_cache is None :
headers [ " Connection " ] = " close "
else :
headers [ " Connection " ] = " Keep-Alive " # Works for HTTP/1.0
headers = dict (
2016-05-12 07:30:35 +00:00
( name . title ( ) , val ) for name , val in list ( headers . items ( ) ) )
2015-07-08 23:34:17 +00:00
if req . _tunnel_host :
tunnel_headers = { }
proxy_auth_hdr = " Proxy-Authorization "
if proxy_auth_hdr in headers :
tunnel_headers [ proxy_auth_hdr ] = headers [ proxy_auth_hdr ]
# Proxy-Authorization should not be sent to origin
# server.
del headers [ proxy_auth_hdr ]
h . set_tunnel ( req . _tunnel_host , headers = tunnel_headers )
try :
2016-05-12 07:30:35 +00:00
h . request ( req . get_method ( ) , req . selector , req . data , headers )
except socket . error as err : # XXX what error?
2015-07-08 23:34:17 +00:00
# Don't close connection when cache is enabled.
if fetch . connection_cache is None :
h . close ( )
2016-05-12 07:30:35 +00:00
raise urllib . error . URLError ( err )
2015-07-08 23:34:17 +00:00
else :
try :
r = h . getresponse ( buffering = True )
except TypeError : # buffering kw not supported
r = h . getresponse ( )
# Pick apart the HTTPResponse object to get the addinfourl
# object initialized properly.
# Wrap the HTTPResponse object in socket's file object adapter
# for Windows. That adapter calls recv(), so delegate recv()
# to read(). This weird wrapping allows the returned object to
# have readline() and readlines() methods.
# XXX It might be better to extract the read buffering code
# out of socket._fileobject() and into a base class.
r . recv = r . read
# no data, just have to read
r . read ( )
class fp_dummy ( object ) :
def read ( self ) :
return " "
def readline ( self ) :
return " "
def close ( self ) :
pass
resp = addinfourl ( fp_dummy ( ) , r . msg , req . get_full_url ( ) )
resp . code = r . status
resp . msg = r . reason
# Close connection when server request it.
if fetch . connection_cache is not None :
if ' Connection ' in r . msg and r . msg [ ' Connection ' ] == ' close ' :
fetch . connection_cache . remove_connection ( h . host , h . port )
return resp
2016-05-12 07:30:35 +00:00
class HTTPMethodFallback ( urllib . request . BaseHandler ) :
2016-01-20 13:10:25 +00:00
"""
Fallback to GET if HEAD is not allowed ( 405 HTTP error )
"""
def http_error_405 ( self , req , fp , code , msg , headers ) :
fp . read ( )
fp . close ( )
2016-05-12 07:30:35 +00:00
newheaders = dict ( ( k , v ) for k , v in list ( req . headers . items ( ) )
2016-01-20 13:10:25 +00:00
if k . lower ( ) not in ( " content-length " , " content-type " ) )
2016-05-12 07:30:35 +00:00
return self . parent . open ( urllib . request . Request ( req . get_full_url ( ) ,
2016-01-20 13:10:25 +00:00
headers = newheaders ,
2016-05-12 07:30:35 +00:00
origin_req_host = req . origin_req_host ,
2016-01-20 13:10:25 +00:00
unverifiable = True ) )
"""
Some servers ( e . g . GitHub archives , hosted on Amazon S3 ) return 403
Forbidden when they actually mean 405 Method Not Allowed .
"""
http_error_403 = http_error_405
"""
Some servers ( e . g . FusionForge ) returns 406 Not Acceptable when they
actually mean 405 Method Not Allowed .
"""
http_error_406 = http_error_405
2016-05-12 07:30:35 +00:00
class FixedHTTPRedirectHandler ( urllib . request . HTTPRedirectHandler ) :
2016-01-20 13:10:25 +00:00
"""
urllib2 . HTTPRedirectHandler resets the method to GET on redirect ,
when we want to follow redirects using the original method .
"""
def redirect_request ( self , req , fp , code , msg , headers , newurl ) :
2016-05-12 07:30:35 +00:00
newreq = urllib . request . HTTPRedirectHandler . redirect_request ( self , req , fp , code , msg , headers , newurl )
2016-01-20 13:10:25 +00:00
newreq . get_method = lambda : req . get_method ( )
return newreq
2015-07-08 23:34:19 +00:00
exported_proxies = export_proxies ( d )
2015-07-08 23:34:21 +00:00
2016-01-20 13:10:25 +00:00
handlers = [ FixedHTTPRedirectHandler , HTTPMethodFallback ]
if export_proxies :
2016-05-12 07:30:35 +00:00
handlers . append ( urllib . request . ProxyHandler ( ) )
2016-01-20 13:10:25 +00:00
handlers . append ( CacheHTTPHandler ( ) )
2015-07-08 23:34:21 +00:00
# XXX: Since Python 2.7.9 ssl cert validation is enabled by default
# see PEP-0476, this causes verification errors on some https servers
# so disable by default.
import ssl
if hasattr ( ssl , ' _create_unverified_context ' ) :
2016-05-12 07:30:35 +00:00
handlers . append ( urllib . request . HTTPSHandler ( context = ssl . _create_unverified_context ( ) ) )
opener = urllib . request . build_opener ( * handlers )
2015-07-08 23:34:21 +00:00
2015-07-08 23:34:17 +00:00
try :
2016-01-20 13:10:25 +00:00
uri = ud . url . split ( " ; " ) [ 0 ]
2016-05-12 07:30:35 +00:00
r = urllib . request . Request ( uri )
2016-01-20 13:10:25 +00:00
r . get_method = lambda : " HEAD "
opener . open ( r )
2016-05-12 07:30:35 +00:00
except urllib . error . URLError as e :
2016-01-20 13:10:25 +00:00
# debug for now to avoid spamming the logs in e.g. remote sstate searches
logger . debug ( 2 , " checkstatus() urlopen failed: %s " % e )
2015-07-08 23:34:17 +00:00
return False
2014-02-28 17:26:20 +00:00
return True
2014-11-05 18:10:29 +00:00
def _parse_path ( self , regex , s ) :
"""
Find and group name , version and archive type in the given string s
"""
2015-02-13 21:58:08 +00:00
2014-11-05 18:10:29 +00:00
m = regex . search ( s )
if m :
2015-02-13 21:58:08 +00:00
pname = ' '
pver = ' '
ptype = ' '
mdict = m . groupdict ( )
if ' name ' in mdict . keys ( ) :
pname = mdict [ ' name ' ]
if ' pver ' in mdict . keys ( ) :
pver = mdict [ ' pver ' ]
if ' type ' in mdict . keys ( ) :
ptype = mdict [ ' type ' ]
bb . debug ( 3 , " _parse_path: %s , %s , %s " % ( pname , pver , ptype ) )
return ( pname , pver , ptype )
2014-11-05 18:10:29 +00:00
return None
def _modelate_version ( self , version ) :
if version [ 0 ] in [ ' . ' , ' - ' ] :
if version [ 1 ] . isdigit ( ) :
version = version [ 1 ] + version [ 0 ] + version [ 2 : len ( version ) ]
else :
version = version [ 1 : len ( version ) ]
2015-02-13 21:58:13 +00:00
version = re . sub ( ' - ' , ' . ' , version )
2014-11-05 18:10:29 +00:00
version = re . sub ( ' _ ' , ' . ' , version )
2015-02-13 21:58:12 +00:00
version = re . sub ( ' (rc)+ ' , ' .1000. ' , version )
version = re . sub ( ' (beta)+ ' , ' .100. ' , version )
version = re . sub ( ' (alpha)+ ' , ' .10. ' , version )
2014-11-05 18:10:29 +00:00
if version [ 0 ] == ' v ' :
version = version [ 1 : len ( version ) ]
return version
def _vercmp ( self , old , new ) :
"""
Check whether ' new ' is newer than ' old ' version . We use existing vercmp ( ) for the
purpose . PE is cleared in comparison as it ' s not for build, and PR is cleared too
for simplicity as it ' s somehow difficult to get from various upstream format
"""
( oldpn , oldpv , oldsuffix ) = old
( newpn , newpv , newsuffix ) = new
"""
Check for a new suffix type that we have never heard of before
"""
if ( newsuffix ) :
2014-11-28 01:12:02 +00:00
m = self . suffix_regex_comp . search ( newsuffix )
2014-11-05 18:10:29 +00:00
if not m :
bb . warn ( " %s has a possible unknown suffix: %s " % ( newpn , newsuffix ) )
return False
"""
Not our package so ignore it
"""
if oldpn != newpn :
return False
oldpv = self . _modelate_version ( oldpv )
newpv = self . _modelate_version ( newpv )
2015-02-13 21:58:10 +00:00
return bb . utils . vercmp ( ( " 0 " , oldpv , " " ) , ( " 0 " , newpv , " " ) )
2014-11-05 18:10:29 +00:00
def _fetch_index ( self , uri , ud , d ) :
"""
Run fetch checkstatus to get directory information
"""
2014-12-23 12:32:36 +00:00
f = tempfile . NamedTemporaryFile ( )
2014-11-05 18:10:29 +00:00
agent = " Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.9.2.12) Gecko/20101027 Ubuntu/9.10 (karmic) Firefox/3.6.12 "
fetchcmd = self . basecmd
fetchcmd + = " -O " + f . name + " --user-agent= ' " + agent + " ' ' " + uri + " ' "
try :
self . _runwget ( ud , d , fetchcmd , True )
fetchresult = f . read ( )
except bb . fetch2 . BBFetchException :
fetchresult = " "
f . close ( )
return fetchresult
2014-12-18 23:58:57 +00:00
def _check_latest_version ( self , url , package , package_regex , current_version , ud , d ) :
2014-11-05 18:10:29 +00:00
"""
Return the latest version of a package inside a given directory path
2015-02-13 21:58:09 +00:00
If error or no version , return " "
2014-11-05 18:10:29 +00:00
"""
valid = 0
2015-02-13 21:58:09 +00:00
version = [ ' ' , ' ' , ' ' ]
2014-11-05 18:10:29 +00:00
bb . debug ( 3 , " VersionURL: %s " % ( url ) )
2015-12-04 11:00:20 +00:00
soup = BeautifulSoup ( self . _fetch_index ( url , ud , d ) , " html.parser " , parse_only = SoupStrainer ( " a " ) )
2014-11-05 18:10:29 +00:00
if not soup :
2014-11-28 01:12:05 +00:00
bb . debug ( 3 , " *** %s NO SOUP " % ( url ) )
2015-02-13 21:58:09 +00:00
return " "
2014-11-05 18:10:29 +00:00
for line in soup . find_all ( ' a ' , href = True ) :
2015-02-13 21:58:09 +00:00
bb . debug ( 3 , " line[ ' href ' ] = ' %s ' " % ( line [ ' href ' ] ) )
2015-02-13 21:58:11 +00:00
bb . debug ( 3 , " line = ' %s ' " % ( str ( line ) ) )
2015-02-13 21:58:09 +00:00
2015-02-13 21:58:08 +00:00
newver = self . _parse_path ( package_regex , line [ ' href ' ] )
if not newver :
newver = self . _parse_path ( package_regex , str ( line ) )
2014-11-28 01:12:05 +00:00
if newver :
bb . debug ( 3 , " Upstream version found: %s " % newver [ 1 ] )
if valid == 0 :
version = newver
valid = 1
2015-02-13 21:58:10 +00:00
elif self . _vercmp ( version , newver ) < 0 :
2014-11-28 01:12:05 +00:00
version = newver
2014-11-05 18:10:29 +00:00
2015-02-13 21:58:11 +00:00
pupver = re . sub ( ' _ ' , ' . ' , version [ 1 ] )
2014-11-28 01:12:05 +00:00
bb . debug ( 3 , " *** %s -> UpstreamVersion = %s (CurrentVersion = %s ) " %
2015-02-13 21:58:11 +00:00
( package , pupver or " N/A " , current_version [ 1 ] ) )
2014-11-05 18:10:29 +00:00
2015-02-13 21:58:09 +00:00
if valid :
2015-02-13 21:58:11 +00:00
return pupver
2014-11-05 18:10:29 +00:00
2015-02-13 21:58:09 +00:00
return " "
2014-11-28 01:12:03 +00:00
2015-02-13 21:58:10 +00:00
def _check_latest_version_by_dir ( self , dirver , package , package_regex ,
current_version , ud , d ) :
"""
Scan every directory in order to get upstream version .
"""
version_dir = [ ' ' , ' ' , ' ' ]
version = [ ' ' , ' ' , ' ' ]
2016-03-24 18:08:18 +00:00
dirver_regex = re . compile ( " (?P<pfx> \ D*)(?P<ver>( \ d+[ \ . \ -_])+( \ d+)) " )
2015-02-13 21:58:10 +00:00
s = dirver_regex . search ( dirver )
if s :
2016-03-24 18:08:18 +00:00
version_dir [ 1 ] = s . group ( ' ver ' )
2015-02-13 21:58:10 +00:00
else :
version_dir [ 1 ] = dirver
dirs_uri = bb . fetch . encodeurl ( [ ud . type , ud . host ,
ud . path . split ( dirver ) [ 0 ] , ud . user , ud . pswd , { } ] )
bb . debug ( 3 , " DirURL: %s , %s " % ( dirs_uri , package ) )
2015-12-04 11:00:20 +00:00
soup = BeautifulSoup ( self . _fetch_index ( dirs_uri , ud , d ) , " html.parser " , parse_only = SoupStrainer ( " a " ) )
2015-02-13 21:58:10 +00:00
if not soup :
return version [ 1 ]
for line in soup . find_all ( ' a ' , href = True ) :
s = dirver_regex . search ( line [ ' href ' ] . strip ( " / " ) )
if s :
2016-03-24 18:08:19 +00:00
sver = s . group ( ' ver ' )
# When prefix is part of the version directory it need to
# ensure that only version directory is used so remove previous
# directories if exists.
#
# Example: pfx = '/dir1/dir2/v' and version = '2.5' the expected
# result is v2.5.
spfx = s . group ( ' pfx ' ) . split ( ' / ' ) [ - 1 ]
version_dir_new = [ ' ' , sver , ' ' ]
2015-02-13 21:58:10 +00:00
if self . _vercmp ( version_dir , version_dir_new ) < = 0 :
2016-03-24 18:08:19 +00:00
dirver_new = spfx + sver
2015-02-13 21:58:10 +00:00
path = ud . path . replace ( dirver , dirver_new , True ) \
. split ( package ) [ 0 ]
uri = bb . fetch . encodeurl ( [ ud . type , ud . host , path ,
ud . user , ud . pswd , { } ] )
pupver = self . _check_latest_version ( uri ,
package , package_regex , current_version , ud , d )
if pupver :
version [ 1 ] = pupver
version_dir = version_dir_new
return version [ 1 ]
2015-02-13 21:58:09 +00:00
def _init_regexes ( self , package , ud , d ) :
2014-11-05 18:10:29 +00:00
"""
2014-11-28 01:12:01 +00:00
Match as many patterns as possible such as :
2014-11-05 18:10:29 +00:00
gnome - common - 2.20 .0 . tar . gz ( most common format )
gtk + - 2.90 .1 . tar . gz
xf86 - input - synaptics - 12.6 .9 . tar . gz
dri2proto - 2.3 . tar . gz
blktool_4 . orig . tar . gz
libid3tag - 0.15 .1 b . tar . gz
unzip552 . tar . gz
icu4c - 3_6 - src . tgz
genext2fs_1 .3 . orig . tar . gz
gst - fluendo - mp3
"""
# match most patterns which uses "-" as separator to version digits
2015-02-13 21:58:13 +00:00
pn_prefix1 = " [a-zA-Z][a-zA-Z0-9]*([-_][a-zA-Z] \ w+)* \ +?[-_] "
2014-11-05 18:10:29 +00:00
# a loose pattern such as for unzip552.tar.gz
pn_prefix2 = " [a-zA-Z]+ "
# a loose pattern such as for 80325-quicky-0.4.tar.gz
2015-02-13 21:58:13 +00:00
pn_prefix3 = " [0-9]+[-]?[a-zA-Z]+ "
2014-11-05 18:10:29 +00:00
# Save the Package Name (pn) Regex for use later
2014-11-28 01:12:02 +00:00
pn_regex = " ( %s | %s | %s ) " % ( pn_prefix1 , pn_prefix2 , pn_prefix3 )
2014-11-05 18:10:29 +00:00
# match version
2015-07-07 14:36:43 +00:00
pver_regex = " (([A-Z]* \ d+[a-zA-Z]*[ \ . \ -_]*)+) "
2014-11-28 01:12:02 +00:00
# match arch
2015-02-13 21:58:13 +00:00
parch_regex = " -source|_all_ "
2014-11-05 18:10:29 +00:00
# src.rpm extension was added only for rpm package. Can be removed if the rpm
# packaged will always be considered as having to be manually upgraded
2014-11-28 01:12:02 +00:00
psuffix_regex = " (tar \ .gz|tgz|tar \ .bz2|zip|xz|rpm|bz2|orig \ .tar \ .gz|tar \ .xz|src \ .tar \ .gz|src \ .tgz|svnr \ d+ \ .tar \ .bz2|stable \ .tar \ .gz|src \ .rpm) "
2014-11-05 18:10:29 +00:00
# match name, version and archive type of a package
2015-02-13 21:58:13 +00:00
package_regex_comp = re . compile ( " (?P<name> %s ? \ .?v?)(?P<pver> %s )(?P<arch> %s )?[ \ .-](?P<type> %s $) "
2014-11-28 01:12:02 +00:00
% ( pn_regex , pver_regex , parch_regex , psuffix_regex ) )
self . suffix_regex_comp = re . compile ( psuffix_regex )
2015-02-13 21:58:09 +00:00
# compile regex, can be specific by package or generic regex
2015-12-04 14:59:26 +00:00
pn_regex = d . getVar ( ' UPSTREAM_CHECK_REGEX ' , True )
2015-02-13 21:58:09 +00:00
if pn_regex :
package_custom_regex_comp = re . compile ( pn_regex )
else :
version = self . _parse_path ( package_regex_comp , package )
if version :
package_custom_regex_comp = re . compile (
2015-02-13 21:58:13 +00:00
" (?P<name> %s )(?P<pver> %s )(?P<arch> %s )?[ \ .-](?P<type> %s ) " %
2015-02-13 21:58:09 +00:00
( re . escape ( version [ 0 ] ) , pver_regex , parch_regex , psuffix_regex ) )
else :
2015-02-13 21:58:11 +00:00
package_custom_regex_comp = None
2014-11-28 01:12:04 +00:00
2014-12-18 23:58:57 +00:00
return package_custom_regex_comp
2014-11-28 01:12:01 +00:00
def latest_versionstring ( self , ud , d ) :
"""
Manipulate the URL and try to obtain the latest package version
sanity check to ensure same name and type .
"""
2014-11-28 01:12:03 +00:00
package = ud . path . split ( " / " ) [ - 1 ]
2015-02-13 21:58:11 +00:00
current_version = [ ' ' , d . getVar ( ' PV ' , True ) , ' ' ]
2014-11-28 01:12:05 +00:00
""" possible to have no version in pkg name, such as spectrum-fw """
if not re . search ( " \ d+ " , package ) :
2015-02-13 21:58:11 +00:00
current_version [ 1 ] = re . sub ( ' _ ' , ' . ' , current_version [ 1 ] )
current_version [ 1 ] = re . sub ( ' - ' , ' . ' , current_version [ 1 ] )
2015-07-15 00:30:58 +00:00
return ( current_version [ 1 ] , ' ' )
2014-11-28 01:12:01 +00:00
2015-02-13 21:58:11 +00:00
package_regex = self . _init_regexes ( package , ud , d )
if package_regex is None :
bb . warn ( " latest_versionstring: package %s don ' t match pattern " % ( package ) )
2015-07-15 00:30:58 +00:00
return ( ' ' , ' ' )
2015-02-13 21:58:11 +00:00
bb . debug ( 3 , " latest_versionstring, regex: %s " % ( package_regex . pattern ) )
uri = " "
2015-12-04 14:59:26 +00:00
regex_uri = d . getVar ( " UPSTREAM_CHECK_URI " , True )
2014-11-28 01:12:03 +00:00
if not regex_uri :
2015-02-13 21:58:11 +00:00
path = ud . path . split ( package ) [ 0 ]
# search for version matches on folders inside the path, like:
# "5.7" in http://download.gnome.org/sources/${PN}/5.7/${PN}-${PV}.tar.gz
2015-02-13 21:58:13 +00:00
dirver_regex = re . compile ( " (?P<dirver>[^/]*( \ d+ \ .)* \ d+([-_]r \ d+)*)/ " )
2015-02-13 21:58:11 +00:00
m = dirver_regex . search ( path )
2014-11-28 01:12:03 +00:00
if m :
2015-02-13 21:58:11 +00:00
pn = d . getVar ( ' PN ' , True )
2014-11-28 01:12:03 +00:00
dirver = m . group ( ' dirver ' )
2015-02-13 21:58:11 +00:00
dirver_pn_regex = re . compile ( " %s \ d? " % ( re . escape ( pn ) ) )
if not dirver_pn_regex . search ( dirver ) :
2015-07-15 00:30:58 +00:00
return ( self . _check_latest_version_by_dir ( dirver ,
package , package_regex , current_version , ud , d ) , ' ' )
2015-02-13 21:58:11 +00:00
uri = bb . fetch . encodeurl ( [ ud . type , ud . host , path , ud . user , ud . pswd , { } ] )
2014-11-28 01:12:03 +00:00
else :
2015-02-13 21:58:11 +00:00
uri = regex_uri
2014-11-05 18:10:29 +00:00
2015-07-15 00:30:58 +00:00
return ( self . _check_latest_version ( uri , package , package_regex ,
current_version , ud , d ) , ' ' )