2016-04-30 19:41:00 +00:00
import argparse
2016-05-12 07:30:35 +00:00
import http . client
2016-04-30 19:41:00 +00:00
import json
import logging
import os
import subprocess
2016-05-12 07:30:35 +00:00
import urllib . parse
2016-04-30 19:41:00 +00:00
from bblayers . action import ActionPlugin
logger = logging . getLogger ( ' bitbake-layers ' )
def plugin_init ( plugins ) :
return LayerIndexPlugin ( )
class LayerIndexPlugin ( ActionPlugin ) :
""" Subcommands for interacting with the layer index.
This class inherits ActionPlugin to get do_add_layer .
"""
def get_json_data ( self , apiurl ) :
proxy_settings = os . environ . get ( " http_proxy " , None )
conn = None
2016-05-12 07:30:35 +00:00
_parsedurl = urllib . parse . urlparse ( apiurl )
2016-04-30 19:41:00 +00:00
path = _parsedurl . path
query = _parsedurl . query
def parse_url ( url ) :
2016-05-12 07:30:35 +00:00
parsedurl = urllib . parse . urlparse ( url )
2016-04-30 19:41:00 +00:00
if parsedurl . netloc [ 0 ] == ' [ ' :
host , port = parsedurl . netloc [ 1 : ] . split ( ' ] ' , 1 )
if ' : ' in port :
port = port . rsplit ( ' : ' , 1 ) [ 1 ]
else :
port = None
else :
if parsedurl . netloc . count ( ' : ' ) == 1 :
( host , port ) = parsedurl . netloc . split ( " : " )
else :
host = parsedurl . netloc
port = None
return ( host , 80 if port is None else int ( port ) )
if proxy_settings is None :
host , port = parse_url ( apiurl )
2016-05-12 07:30:35 +00:00
conn = http . client . HTTPConnection ( host , port )
2016-04-30 19:41:00 +00:00
conn . request ( " GET " , path + " ? " + query )
else :
host , port = parse_url ( proxy_settings )
2016-05-12 07:30:35 +00:00
conn = http . client . HTTPConnection ( host , port )
2016-04-30 19:41:00 +00:00
conn . request ( " GET " , apiurl )
r = conn . getresponse ( )
if r . status != 200 :
raise Exception ( " Failed to read " + path + " : %d %s " % ( r . status , r . reason ) )
return json . loads ( r . read ( ) )
def get_layer_deps ( self , layername , layeritems , layerbranches , layerdependencies , branchnum , selfname = False ) :
def layeritems_info_id ( items_name , layeritems ) :
litems_id = None
for li in layeritems :
if li [ ' name ' ] == items_name :
litems_id = li [ ' id ' ]
break
return litems_id
def layerbranches_info ( items_id , layerbranches ) :
lbranch = { }
for lb in layerbranches :
if lb [ ' layer ' ] == items_id and lb [ ' branch ' ] == branchnum :
lbranch [ ' id ' ] = lb [ ' id ' ]
lbranch [ ' vcs_subdir ' ] = lb [ ' vcs_subdir ' ]
break
return lbranch
def layerdependencies_info ( lb_id , layerdependencies ) :
ld_deps = [ ]
for ld in layerdependencies :
if ld [ ' layerbranch ' ] == lb_id and not ld [ ' dependency ' ] in ld_deps :
ld_deps . append ( ld [ ' dependency ' ] )
if not ld_deps :
logger . error ( " The dependency of layerDependencies is not found. " )
return ld_deps
def layeritems_info_name_subdir ( items_id , layeritems ) :
litems = { }
for li in layeritems :
if li [ ' id ' ] == items_id :
litems [ ' vcs_url ' ] = li [ ' vcs_url ' ]
litems [ ' name ' ] = li [ ' name ' ]
break
return litems
if selfname :
selfid = layeritems_info_id ( layername , layeritems )
lbinfo = layerbranches_info ( selfid , layerbranches )
if lbinfo :
selfsubdir = lbinfo [ ' vcs_subdir ' ]
else :
logger . error ( " %s is not found in the specified branch " % layername )
return
selfurl = layeritems_info_name_subdir ( selfid , layeritems ) [ ' vcs_url ' ]
if selfurl :
return selfurl , selfsubdir
else :
logger . error ( " Cannot get layer %s git repo and subdir " % layername )
return
ldict = { }
itemsid = layeritems_info_id ( layername , layeritems )
if not itemsid :
return layername , None
lbid = layerbranches_info ( itemsid , layerbranches )
if lbid :
lbid = layerbranches_info ( itemsid , layerbranches ) [ ' id ' ]
else :
logger . error ( " %s is not found in the specified branch " % layername )
return None , None
for dependency in layerdependencies_info ( lbid , layerdependencies ) :
lname = layeritems_info_name_subdir ( dependency , layeritems ) [ ' name ' ]
lurl = layeritems_info_name_subdir ( dependency , layeritems ) [ ' vcs_url ' ]
lsubdir = layerbranches_info ( dependency , layerbranches ) [ ' vcs_subdir ' ]
ldict [ lname ] = lurl , lsubdir
return None , ldict
def get_fetch_layer ( self , fetchdir , url , subdir , fetch_layer ) :
layername = self . get_layer_name ( url )
if os . path . splitext ( layername ) [ 1 ] == ' .git ' :
layername = os . path . splitext ( layername ) [ 0 ]
repodir = os . path . join ( fetchdir , layername )
layerdir = os . path . join ( repodir , subdir )
if not os . path . exists ( repodir ) :
if fetch_layer :
result = subprocess . call ( ' git clone %s %s ' % ( url , repodir ) , shell = True )
if result :
logger . error ( " Failed to download %s " % url )
return None , None
else :
return layername , layerdir
else :
logger . plain ( " Repository %s needs to be fetched " % url )
return layername , layerdir
elif os . path . exists ( layerdir ) :
return layername , layerdir
else :
logger . error ( " %s is not in %s " % ( url , subdir ) )
return None , None
def do_layerindex_fetch ( self , args ) :
""" Fetches a layer from a layer index along with its dependent layers, and adds them to conf/bblayers.conf.
"""
apiurl = self . tinfoil . config_data . getVar ( ' BBLAYERS_LAYERINDEX_URL ' , True )
if not apiurl :
logger . error ( " Cannot get BBLAYERS_LAYERINDEX_URL " )
return 1
else :
if apiurl [ - 1 ] != ' / ' :
apiurl + = ' / '
apiurl + = " api/ "
apilinks = self . get_json_data ( apiurl )
branches = self . get_json_data ( apilinks [ ' branches ' ] )
branchnum = 0
for branch in branches :
if branch [ ' name ' ] == args . branch :
branchnum = branch [ ' id ' ]
break
if branchnum == 0 :
validbranches = ' , ' . join ( [ branch [ ' name ' ] for branch in branches ] )
logger . error ( ' Invalid layer branch name " %s " . Valid branches: %s ' % ( args . branch , validbranches ) )
return 1
ignore_layers = [ ]
for collection in self . tinfoil . config_data . getVar ( ' BBFILE_COLLECTIONS ' , True ) . split ( ) :
lname = self . tinfoil . config_data . getVar ( ' BBLAYERS_LAYERINDEX_NAME_ %s ' % collection , True )
if lname :
ignore_layers . append ( lname )
if args . ignore :
ignore_layers . extend ( args . ignore . split ( ' , ' ) )
layeritems = self . get_json_data ( apilinks [ ' layerItems ' ] )
layerbranches = self . get_json_data ( apilinks [ ' layerBranches ' ] )
layerdependencies = self . get_json_data ( apilinks [ ' layerDependencies ' ] )
invaluenames = [ ]
repourls = { }
printlayers = [ ]
def query_dependencies ( layers , layeritems , layerbranches , layerdependencies , branchnum ) :
depslayer = [ ]
for layername in layers :
invaluename , layerdict = self . get_layer_deps ( layername , layeritems , layerbranches , layerdependencies , branchnum )
if layerdict :
repourls [ layername ] = self . get_layer_deps ( layername , layeritems , layerbranches , layerdependencies , branchnum , selfname = True )
for layer in layerdict :
if not layer in ignore_layers :
depslayer . append ( layer )
printlayers . append ( ( layername , layer , layerdict [ layer ] [ 0 ] , layerdict [ layer ] [ 1 ] ) )
if not layer in ignore_layers and not layer in repourls :
repourls [ layer ] = ( layerdict [ layer ] [ 0 ] , layerdict [ layer ] [ 1 ] )
if invaluename and not invaluename in invaluenames :
invaluenames . append ( invaluename )
return depslayer
depslayers = query_dependencies ( args . layername , layeritems , layerbranches , layerdependencies , branchnum )
while depslayers :
depslayer = query_dependencies ( depslayers , layeritems , layerbranches , layerdependencies , branchnum )
depslayers = depslayer
if invaluenames :
for invaluename in invaluenames :
logger . error ( ' Layer " %s " not found in layer index ' % invaluename )
return 1
logger . plain ( " %s %s %s %s " % ( " Layer " . ljust ( 19 ) , " Required by " . ljust ( 19 ) , " Git repository " . ljust ( 54 ) , " Subdirectory " ) )
logger . plain ( ' = ' * 115 )
for layername in args . layername :
layerurl = repourls [ layername ]
logger . plain ( " %s %s %s %s " % ( layername . ljust ( 20 ) , ' - ' . ljust ( 20 ) , layerurl [ 0 ] . ljust ( 55 ) , layerurl [ 1 ] ) )
printedlayers = [ ]
for layer , dependency , gitrepo , subdirectory in printlayers :
if dependency in printedlayers :
continue
logger . plain ( " %s %s %s %s " % ( dependency . ljust ( 20 ) , layer . ljust ( 20 ) , gitrepo . ljust ( 55 ) , subdirectory ) )
printedlayers . append ( dependency )
if repourls :
fetchdir = self . tinfoil . config_data . getVar ( ' BBLAYERS_FETCH_DIR ' , True )
if not fetchdir :
logger . error ( " Cannot get BBLAYERS_FETCH_DIR " )
return 1
if not os . path . exists ( fetchdir ) :
os . makedirs ( fetchdir )
addlayers = [ ]
for repourl , subdir in repourls . values ( ) :
name , layerdir = self . get_fetch_layer ( fetchdir , repourl , subdir , not args . show_only )
if not name :
# Error already shown
return 1
addlayers . append ( ( subdir , name , layerdir ) )
if not args . show_only :
for subdir , name , layerdir in set ( addlayers ) :
if os . path . exists ( layerdir ) :
if subdir :
logger . plain ( " Adding layer \" %s \" to conf/bblayers.conf " % subdir )
else :
logger . plain ( " Adding layer \" %s \" to conf/bblayers.conf " % name )
localargs = argparse . Namespace ( )
localargs . layerdir = layerdir
self . do_add_layer ( localargs )
else :
break
def do_layerindex_show_depends ( self , args ) :
""" Find layer dependencies from layer index.
"""
args . show_only = True
args . ignore = [ ]
self . do_layerindex_fetch ( args )
def register_commands ( self , sp ) :
parser_layerindex_fetch = self . add_command ( sp , ' layerindex-fetch ' , self . do_layerindex_fetch )
parser_layerindex_fetch . add_argument ( ' -n ' , ' --show-only ' , help = ' show dependencies and do nothing else ' , action = ' store_true ' )
parser_layerindex_fetch . add_argument ( ' -b ' , ' --branch ' , help = ' branch name to fetch (default %(default)s ) ' , default = ' master ' )
parser_layerindex_fetch . add_argument ( ' -i ' , ' --ignore ' , help = ' assume the specified layers do not need to be fetched/added (separate multiple layers with commas, no spaces) ' , metavar = ' LAYER ' )
parser_layerindex_fetch . add_argument ( ' layername ' , nargs = ' + ' , help = ' layer to fetch ' )
parser_layerindex_show_depends = self . add_command ( sp , ' layerindex-show-depends ' , self . do_layerindex_show_depends )
parser_layerindex_show_depends . add_argument ( ' -b ' , ' --branch ' , help = ' branch name to fetch (default %(default)s ) ' , default = ' master ' )
parser_layerindex_show_depends . add_argument ( ' layername ' , nargs = ' + ' , help = ' layer to query ' )