bitbake: tinfoil: rewrite as a wrapper around the UI
Rewrite tinfoil as a wrapper around the UI, instead of the earlier approach of starting up just enough of cooker to do what we want. This has several advantages: * It now works when bitbake is memory-resident instead of failing with "ERROR: Only one copy of bitbake should be run against a build directory". * We can now connect an actual UI, thus you get things like the recipe parsing / cache loading progress bar and parse error handling for free * We can now handle events generated by the server if we wish to do so * We can potentially extend this to do more stuff, e.g. actually running build operations - this needs to be made more practical before we can use it though (since you effectively have to become the UI yourself for this at the moment.) The downside is that tinfoil no longer has direct access to cooker, the global datastore, or the cache. To mitigate this I have extended data_smart to provide remote access capability for the datastore, and created "fake" cooker and cooker.recipecache / cooker.collection adapter objects in order to avoid breaking too many tinfoil-using scripts that might be out there (we've never officially documented tinfoil or BitBake's internal code, but we can still make accommodations where practical). I've at least gone far enough to support all of the utilities that use tinfoil in OE-Core with some changes, but I know there are scripts such as Chris Larson's "bb" out there that do make other calls into BitBake code that I'm not currently providing access to through the adapters. Part of the fix for [YOCTO #5470]. (Bitbake rev: 3bbf8d611c859f74d563778115677a04f5c4ab43) Signed-off-by: Paul Eggleton <paul.eggleton@linux.intel.com> Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
This commit is contained in:
Paul Eggleton
Richard Purdie
parent
e271d7dc60
commit
7d5c9860de
|
|
@ -28,8 +28,15 @@ and must not trigger events, directly or indirectly.
|
|||
Commands are queued in a CommandQueue
|
||||
"""
|
||||
|
||||
from collections import OrderedDict, defaultdict
|
||||
|
||||
import bb.event
|
||||
import bb.cooker
|
||||
import bb.remotedata
|
||||
|
||||
class DataStoreConnectionHandle(object):
|
||||
def __init__(self, dsindex=0):
|
||||
self.dsindex = dsindex
|
||||
|
||||
class CommandCompleted(bb.event.Event):
|
||||
pass
|
||||
|
|
@ -55,6 +62,7 @@ class Command:
|
|||
self.cooker = cooker
|
||||
self.cmds_sync = CommandsSync()
|
||||
self.cmds_async = CommandsAsync()
|
||||
self.remotedatastores = bb.remotedata.RemoteDatastores(cooker)
|
||||
|
||||
# FIXME Add lock for this
|
||||
self.currentAsyncCommand = None
|
||||
|
|
@ -298,6 +306,193 @@ class CommandsSync:
|
|||
command.cooker.updateConfigOpts(options, environment)
|
||||
updateConfig.needconfig = False
|
||||
|
||||
def parseConfiguration(self, command, params):
|
||||
"""Instruct bitbake to parse its configuration
|
||||
NOTE: it is only necessary to call this if you aren't calling any normal action
|
||||
(otherwise parsing is taken care of automatically)
|
||||
"""
|
||||
command.cooker.parseConfiguration()
|
||||
parseConfiguration.needconfig = False
|
||||
|
||||
def getLayerPriorities(self, command, params):
|
||||
ret = []
|
||||
# regex objects cannot be marshalled by xmlrpc
|
||||
for collection, pattern, regex, pri in command.cooker.bbfile_config_priorities:
|
||||
ret.append((collection, pattern, regex.pattern, pri))
|
||||
return ret
|
||||
getLayerPriorities.readonly = True
|
||||
|
||||
def getRecipes(self, command, params):
|
||||
try:
|
||||
mc = params[0]
|
||||
except IndexError:
|
||||
mc = ''
|
||||
return list(command.cooker.recipecaches[mc].pkg_pn.items())
|
||||
getRecipes.readonly = True
|
||||
|
||||
def getRecipeDepends(self, command, params):
|
||||
try:
|
||||
mc = params[0]
|
||||
except IndexError:
|
||||
mc = ''
|
||||
return list(command.cooker.recipecaches[mc].deps.items())
|
||||
getRecipeDepends.readonly = True
|
||||
|
||||
def getRecipeVersions(self, command, params):
|
||||
try:
|
||||
mc = params[0]
|
||||
except IndexError:
|
||||
mc = ''
|
||||
return command.cooker.recipecaches[mc].pkg_pepvpr
|
||||
getRecipeVersions.readonly = True
|
||||
|
||||
def getRuntimeDepends(self, command, params):
|
||||
ret = []
|
||||
try:
|
||||
mc = params[0]
|
||||
except IndexError:
|
||||
mc = ''
|
||||
rundeps = command.cooker.recipecaches[mc].rundeps
|
||||
for key, value in rundeps.items():
|
||||
if isinstance(value, defaultdict):
|
||||
value = dict(value)
|
||||
ret.append((key, value))
|
||||
return ret
|
||||
getRuntimeDepends.readonly = True
|
||||
|
||||
def getRuntimeRecommends(self, command, params):
|
||||
ret = []
|
||||
try:
|
||||
mc = params[0]
|
||||
except IndexError:
|
||||
mc = ''
|
||||
runrecs = command.cooker.recipecaches[mc].runrecs
|
||||
for key, value in runrecs.items():
|
||||
if isinstance(value, defaultdict):
|
||||
value = dict(value)
|
||||
ret.append((key, value))
|
||||
return ret
|
||||
getRuntimeRecommends.readonly = True
|
||||
|
||||
def getRecipeInherits(self, command, params):
|
||||
try:
|
||||
mc = params[0]
|
||||
except IndexError:
|
||||
mc = ''
|
||||
return command.cooker.recipecaches[mc].inherits
|
||||
getRecipeInherits.readonly = True
|
||||
|
||||
def getBbFilePriority(self, command, params):
|
||||
try:
|
||||
mc = params[0]
|
||||
except IndexError:
|
||||
mc = ''
|
||||
return command.cooker.recipecaches[mc].bbfile_priority
|
||||
getBbFilePriority.readonly = True
|
||||
|
||||
def getDefaultPreference(self, command, params):
|
||||
try:
|
||||
mc = params[0]
|
||||
except IndexError:
|
||||
mc = ''
|
||||
return command.cooker.recipecaches[mc].pkg_dp
|
||||
getDefaultPreference.readonly = True
|
||||
|
||||
def getSkippedRecipes(self, command, params):
|
||||
# Return list sorted by reverse priority order
|
||||
import bb.cache
|
||||
skipdict = OrderedDict(sorted(command.cooker.skiplist.items(),
|
||||
key=lambda x: (-command.cooker.collection.calc_bbfile_priority(bb.cache.virtualfn2realfn(x[0])[0]), x[0])))
|
||||
return list(skipdict.items())
|
||||
getSkippedRecipes.readonly = True
|
||||
|
||||
def getOverlayedRecipes(self, command, params):
|
||||
return list(command.cooker.collection.overlayed.items())
|
||||
getOverlayedRecipes.readonly = True
|
||||
|
||||
def getFileAppends(self, command, params):
|
||||
fn = params[0]
|
||||
return command.cooker.collection.get_file_appends(fn)
|
||||
getFileAppends.readonly = True
|
||||
|
||||
def getAllAppends(self, command, params):
|
||||
return command.cooker.collection.bbappends
|
||||
getAllAppends.readonly = True
|
||||
|
||||
def findProviders(self, command, params):
|
||||
return command.cooker.findProviders()
|
||||
findProviders.readonly = True
|
||||
|
||||
def findBestProvider(self, command, params):
|
||||
pn = params[0]
|
||||
return command.cooker.findBestProvider(pn)
|
||||
findBestProvider.readonly = True
|
||||
|
||||
def allProviders(self, command, params):
|
||||
try:
|
||||
mc = params[0]
|
||||
except IndexError:
|
||||
mc = ''
|
||||
return list(bb.providers.allProviders(command.cooker.recipecaches[mc]).items())
|
||||
allProviders.readonly = True
|
||||
|
||||
def getRuntimeProviders(self, command, params):
|
||||
rprovide = params[0]
|
||||
try:
|
||||
mc = params[1]
|
||||
except IndexError:
|
||||
mc = ''
|
||||
all_p = bb.providers.getRuntimeProviders(command.cooker.recipecaches[mc], rprovide)
|
||||
if all_p:
|
||||
best = bb.providers.filterProvidersRunTime(all_p, rprovide,
|
||||
command.cooker.data,
|
||||
command.cooker.recipecaches[mc])[0][0]
|
||||
else:
|
||||
best = None
|
||||
return all_p, best
|
||||
getRuntimeProviders.readonly = True
|
||||
|
||||
def dataStoreConnectorFindVar(self, command, params):
|
||||
dsindex = params[0]
|
||||
name = params[1]
|
||||
datastore = command.remotedatastores[dsindex]
|
||||
value = datastore._findVar(name)
|
||||
|
||||
if value:
|
||||
content = value.get('_content', None)
|
||||
if isinstance(content, bb.data_smart.DataSmart):
|
||||
# Value is a datastore (e.g. BB_ORIGENV) - need to handle this carefully
|
||||
idx = command.remotedatastores.check_store(content, True)
|
||||
return {'_content': DataStoreConnectionHandle(idx), '_connector_origtype': 'DataStoreConnectionHandle'}
|
||||
elif isinstance(content, set):
|
||||
return {'_content': list(content), '_connector_origtype': 'set'}
|
||||
return value
|
||||
dataStoreConnectorFindVar.readonly = True
|
||||
|
||||
def dataStoreConnectorGetKeys(self, command, params):
|
||||
dsindex = params[0]
|
||||
datastore = command.remotedatastores[dsindex]
|
||||
return list(datastore.keys())
|
||||
dataStoreConnectorGetKeys.readonly = True
|
||||
|
||||
def dataStoreConnectorGetVarHistory(self, command, params):
|
||||
dsindex = params[0]
|
||||
name = params[1]
|
||||
datastore = command.remotedatastores[dsindex]
|
||||
return datastore.varhistory.variable(name)
|
||||
dataStoreConnectorGetVarHistory.readonly = True
|
||||
|
||||
def dataStoreConnectorExpandPythonRef(self, command, params):
|
||||
dsindex = params[0]
|
||||
varname = params[1]
|
||||
expr = params[2]
|
||||
if dsindex:
|
||||
datastore = self.dataStores[dsindex]
|
||||
else:
|
||||
datastore = command.cooker.data
|
||||
varparse = bb.data_smart.VariableParse(varname, datastore)
|
||||
return varparse.python_sub(expr)
|
||||
|
||||
class CommandsAsync:
|
||||
"""
|
||||
A class of asynchronous commands
|
||||
|
|
|
|||
|
|
@ -583,13 +583,12 @@ class BBCooker:
|
|||
|
||||
def showVersions(self):
|
||||
|
||||
pkg_pn = self.recipecaches[''].pkg_pn
|
||||
(latest_versions, preferred_versions) = bb.providers.findProviders(self.data, self.recipecaches[''], pkg_pn)
|
||||
(latest_versions, preferred_versions) = self.findProviders()
|
||||
|
||||
logger.plain("%-35s %25s %25s", "Recipe Name", "Latest Version", "Preferred Version")
|
||||
logger.plain("%-35s %25s %25s\n", "===========", "==============", "=================")
|
||||
|
||||
for p in sorted(pkg_pn):
|
||||
for p in sorted(self.recipecaches[''].pkg_pn):
|
||||
pref = preferred_versions[p]
|
||||
latest = latest_versions[p]
|
||||
|
||||
|
|
@ -1084,6 +1083,20 @@ class BBCooker:
|
|||
if matches:
|
||||
bb.event.fire(bb.event.FilesMatchingFound(filepattern, matches), self.data)
|
||||
|
||||
def findProviders(self, mc=''):
|
||||
return bb.providers.findProviders(self.data, self.recipecaches[mc], self.recipecaches[mc].pkg_pn)
|
||||
|
||||
def findBestProvider(self, pn, mc=''):
|
||||
if pn in self.recipecaches[mc].providers:
|
||||
filenames = self.recipecaches[mc].providers[pn]
|
||||
eligible, foundUnique = bb.providers.filterProviders(filenames, pn, self.expanded_data, self.recipecaches[mc])
|
||||
filename = eligible[0]
|
||||
return None, None, None, filename
|
||||
elif pn in self.recipecaches[mc].pkg_pn:
|
||||
return bb.providers.findBestProvider(pn, self.data, self.recipecaches[mc], self.recipecaches[mc].pkg_pn)
|
||||
else:
|
||||
return None, None, None, None
|
||||
|
||||
def findConfigFiles(self, varname):
|
||||
"""
|
||||
Find config files which are appropriate values for varname.
|
||||
|
|
|
|||
|
|
@ -389,12 +389,8 @@ def bitbake_main(configParams, configuration):
|
|||
except:
|
||||
pass
|
||||
|
||||
|
||||
configuration.setConfigParameters(configParams)
|
||||
|
||||
ui_module = import_extension_module(bb.ui, configParams.ui, 'main')
|
||||
servermodule = import_extension_module(bb.server, configParams.servertype, 'BitBakeServer')
|
||||
|
||||
if configParams.server_only:
|
||||
if configParams.servertype != "xmlrpc":
|
||||
raise BBMainException("FATAL: If '--server-only' is defined, we must set the "
|
||||
|
|
@ -442,66 +438,11 @@ def bitbake_main(configParams, configuration):
|
|||
bb.msg.init_msgconfig(configParams.verbose, configuration.debug,
|
||||
configuration.debug_domains)
|
||||
|
||||
# Ensure logging messages get sent to the UI as events
|
||||
handler = bb.event.LogHandler()
|
||||
if not configParams.status_only:
|
||||
# In status only mode there are no logs and no UI
|
||||
logger.addHandler(handler)
|
||||
|
||||
# Clear away any spurious environment variables while we stoke up the cooker
|
||||
cleanedvars = bb.utils.clean_environment()
|
||||
|
||||
featureset = []
|
||||
if not configParams.server_only:
|
||||
# Collect the feature set for the UI
|
||||
featureset = getattr(ui_module, "featureSet", [])
|
||||
|
||||
if configParams.server_only:
|
||||
for param in ('prefile', 'postfile'):
|
||||
value = getattr(configParams, param)
|
||||
if value:
|
||||
setattr(configuration, "%s_server" % param, value)
|
||||
param = "%s_server" % param
|
||||
|
||||
if not configParams.remote_server:
|
||||
# we start a server with a given configuration
|
||||
server = start_server(servermodule, configParams, configuration, featureset)
|
||||
bb.event.ui_queue = []
|
||||
else:
|
||||
if os.getenv('BBSERVER') == 'autostart':
|
||||
if configParams.remote_server == 'autostart' or \
|
||||
not servermodule.check_connection(configParams.remote_server, timeout=2):
|
||||
configParams.bind = 'localhost:0'
|
||||
srv = start_server(servermodule, configParams, configuration, featureset)
|
||||
configParams.remote_server = '%s:%d' % tuple(configuration.interface)
|
||||
bb.event.ui_queue = []
|
||||
|
||||
# we start a stub server that is actually a XMLRPClient that connects to a real server
|
||||
server = servermodule.BitBakeXMLRPCClient(configParams.observe_only,
|
||||
configParams.xmlrpctoken)
|
||||
server.saveConnectionDetails(configParams.remote_server)
|
||||
|
||||
server, server_connection, ui_module = setup_bitbake(configParams, configuration)
|
||||
if server_connection is None and configParams.kill_server:
|
||||
return 0
|
||||
|
||||
if not configParams.server_only:
|
||||
try:
|
||||
server_connection = server.establishConnection(featureset)
|
||||
except Exception as e:
|
||||
bb.fatal("Could not connect to server %s: %s" % (configParams.remote_server, str(e)))
|
||||
|
||||
if configParams.kill_server:
|
||||
server_connection.connection.terminateServer()
|
||||
bb.event.ui_queue = []
|
||||
return 0
|
||||
|
||||
server_connection.setupEventQueue()
|
||||
|
||||
# Restore the environment in case the UI needs it
|
||||
for k in cleanedvars:
|
||||
os.environ[k] = cleanedvars[k]
|
||||
|
||||
logger.removeHandler(handler)
|
||||
|
||||
|
||||
if configParams.status_only:
|
||||
server_connection.terminate()
|
||||
return 0
|
||||
|
|
@ -520,3 +461,77 @@ def bitbake_main(configParams, configuration):
|
|||
return 0
|
||||
|
||||
return 1
|
||||
|
||||
def setup_bitbake(configParams, configuration, extrafeatures=None):
|
||||
# Ensure logging messages get sent to the UI as events
|
||||
handler = bb.event.LogHandler()
|
||||
if not configParams.status_only:
|
||||
# In status only mode there are no logs and no UI
|
||||
logger.addHandler(handler)
|
||||
|
||||
# Clear away any spurious environment variables while we stoke up the cooker
|
||||
cleanedvars = bb.utils.clean_environment()
|
||||
|
||||
if configParams.server_only:
|
||||
featureset = []
|
||||
ui_module = None
|
||||
else:
|
||||
ui_module = import_extension_module(bb.ui, configParams.ui, 'main')
|
||||
# Collect the feature set for the UI
|
||||
featureset = getattr(ui_module, "featureSet", [])
|
||||
|
||||
if configParams.server_only:
|
||||
for param in ('prefile', 'postfile'):
|
||||
value = getattr(configParams, param)
|
||||
if value:
|
||||
setattr(configuration, "%s_server" % param, value)
|
||||
param = "%s_server" % param
|
||||
|
||||
if extrafeatures:
|
||||
for feature in extrafeatures:
|
||||
if not feature in featureset:
|
||||
featureset.append(feature)
|
||||
|
||||
servermodule = import_extension_module(bb.server,
|
||||
configParams.servertype,
|
||||
'BitBakeServer')
|
||||
if configParams.remote_server:
|
||||
if os.getenv('BBSERVER') == 'autostart':
|
||||
if configParams.remote_server == 'autostart' or \
|
||||
not servermodule.check_connection(configParams.remote_server, timeout=2):
|
||||
configParams.bind = 'localhost:0'
|
||||
srv = start_server(servermodule, configParams, configuration, featureset)
|
||||
configParams.remote_server = '%s:%d' % tuple(configuration.interface)
|
||||
bb.event.ui_queue = []
|
||||
# we start a stub server that is actually a XMLRPClient that connects to a real server
|
||||
from bb.server.xmlrpc import BitBakeXMLRPCClient
|
||||
server = servermodule.BitBakeXMLRPCClient(configParams.observe_only,
|
||||
configParams.xmlrpctoken)
|
||||
server.saveConnectionDetails(configParams.remote_server)
|
||||
else:
|
||||
# we start a server with a given configuration
|
||||
server = start_server(servermodule, configParams, configuration, featureset)
|
||||
bb.event.ui_queue = []
|
||||
|
||||
if configParams.server_only:
|
||||
server_connection = None
|
||||
else:
|
||||
try:
|
||||
server_connection = server.establishConnection(featureset)
|
||||
except Exception as e:
|
||||
bb.fatal("Could not connect to server %s: %s" % (configParams.remote_server, str(e)))
|
||||
|
||||
if configParams.kill_server:
|
||||
server_connection.connection.terminateServer()
|
||||
bb.event.ui_queue = []
|
||||
return None, None, None
|
||||
|
||||
server_connection.setupEventQueue()
|
||||
|
||||
# Restore the environment in case the UI needs it
|
||||
for k in cleanedvars:
|
||||
os.environ[k] = cleanedvars[k]
|
||||
|
||||
logger.removeHandler(handler)
|
||||
|
||||
return server, server_connection, ui_module
|
||||
|
|
|
|||
|
|
@ -0,0 +1,74 @@
|
|||
"""
|
||||
BitBake 'remotedata' module
|
||||
|
||||
Provides support for using a datastore from the bitbake client
|
||||
"""
|
||||
|
||||
# Copyright (C) 2016 Intel Corporation
|
||||
#
|
||||
# This program is free software; you can redistribute it and/or modify
|
||||
# it under the terms of the GNU General Public License version 2 as
|
||||
# published by the Free Software Foundation.
|
||||
#
|
||||
# This program is distributed in the hope that it will be useful,
|
||||
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
# GNU General Public License for more details.
|
||||
#
|
||||
# You should have received a copy of the GNU General Public License along
|
||||
# with this program; if not, write to the Free Software Foundation, Inc.,
|
||||
# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
|
||||
|
||||
import bb.data
|
||||
|
||||
class RemoteDatastores:
|
||||
"""Used on the server side to manage references to server-side datastores"""
|
||||
def __init__(self, cooker):
|
||||
self.cooker = cooker
|
||||
self.datastores = {}
|
||||
self.locked = []
|
||||
self.nextindex = 1
|
||||
|
||||
def __len__(self):
|
||||
return len(self.datastores)
|
||||
|
||||
def __getitem__(self, key):
|
||||
if key is None:
|
||||
return self.cooker.data
|
||||
else:
|
||||
return self.datastores[key]
|
||||
|
||||
def items(self):
|
||||
return self.datastores.items()
|
||||
|
||||
def store(self, d, locked=False):
|
||||
"""
|
||||
Put a datastore into the collection. If locked=True then the datastore
|
||||
is understood to be managed externally and cannot be released by calling
|
||||
release().
|
||||
"""
|
||||
idx = self.nextindex
|
||||
self.datastores[idx] = d
|
||||
if locked:
|
||||
self.locked.append(idx)
|
||||
self.nextindex += 1
|
||||
return idx
|
||||
|
||||
def check_store(self, d, locked=False):
|
||||
"""
|
||||
Put a datastore into the collection if it's not already in there;
|
||||
in either case return the index
|
||||
"""
|
||||
for key, val in self.datastores.items():
|
||||
if val is d:
|
||||
idx = key
|
||||
break
|
||||
else:
|
||||
idx = self.store(d, locked)
|
||||
return idx
|
||||
|
||||
def release(self, idx):
|
||||
"""Discard a datastore in the collection"""
|
||||
if idx in self.locked:
|
||||
raise Exception('Tried to release locked datastore %d' % idx)
|
||||
del self.datastores[idx]
|
||||
|
|
@ -1,6 +1,6 @@
|
|||
# tinfoil: a simple wrapper around cooker for bitbake-based command-line utilities
|
||||
#
|
||||
# Copyright (C) 2012 Intel Corporation
|
||||
# Copyright (C) 2012-2016 Intel Corporation
|
||||
# Copyright (C) 2011 Mentor Graphics Corporation
|
||||
#
|
||||
# This program is free software; you can redistribute it and/or modify
|
||||
|
|
@ -17,47 +17,172 @@
|
|||
# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
|
||||
|
||||
import logging
|
||||
import warnings
|
||||
import os
|
||||
import sys
|
||||
import atexit
|
||||
import re
|
||||
from collections import OrderedDict, defaultdict
|
||||
|
||||
import bb.cache
|
||||
import bb.cooker
|
||||
import bb.providers
|
||||
import bb.utils
|
||||
from bb.cooker import state, BBCooker, CookerFeatures
|
||||
import bb.command
|
||||
from bb.cookerdata import CookerConfiguration, ConfigParameters
|
||||
from bb.main import setup_bitbake, BitBakeConfigParameters, BBMainException
|
||||
import bb.fetch2
|
||||
|
||||
|
||||
# We need this in order to shut down the connection to the bitbake server,
|
||||
# otherwise the process will never properly exit
|
||||
_server_connections = []
|
||||
def _terminate_connections():
|
||||
for connection in _server_connections:
|
||||
connection.terminate()
|
||||
atexit.register(_terminate_connections)
|
||||
|
||||
class TinfoilUIException(Exception):
|
||||
"""Exception raised when the UI returns non-zero from its main function"""
|
||||
def __init__(self, returncode):
|
||||
self.returncode = returncode
|
||||
def __repr__(self):
|
||||
return 'UI module main returned %d' % self.returncode
|
||||
|
||||
class TinfoilCommandFailed(Exception):
|
||||
"""Exception raised when run_command fails"""
|
||||
|
||||
class TinfoilDataStoreConnector:
|
||||
|
||||
def __init__(self, tinfoil, dsindex):
|
||||
self.tinfoil = tinfoil
|
||||
self.dsindex = dsindex
|
||||
def getVar(self, name):
|
||||
value = self.tinfoil.run_command('dataStoreConnectorFindVar', self.dsindex, name)
|
||||
if isinstance(value, dict):
|
||||
if '_connector_origtype' in value:
|
||||
value['_content'] = self.tinfoil._reconvert_type(value['_content'], value['_connector_origtype'])
|
||||
del value['_connector_origtype']
|
||||
|
||||
return value
|
||||
def getKeys(self):
|
||||
return set(self.tinfoil.run_command('dataStoreConnectorGetKeys', self.dsindex))
|
||||
def getVarHistory(self, name):
|
||||
return self.tinfoil.run_command('dataStoreConnectorGetVarHistory', self.dsindex, name)
|
||||
def expandPythonRef(self, varname, expr):
|
||||
ret = self.tinfoil.run_command('dataStoreConnectorExpandPythonRef', self.dsindex, varname, expr)
|
||||
return ret
|
||||
def setVar(self, varname, value):
|
||||
if self.dsindex is None:
|
||||
self.tinfoil.run_command('setVariable', varname, value)
|
||||
else:
|
||||
# Not currently implemented - indicate that setting should
|
||||
# be redirected to local side
|
||||
return True
|
||||
|
||||
class TinfoilCookerAdapter:
|
||||
"""
|
||||
Provide an adapter for existing code that expects to access a cooker object via Tinfoil,
|
||||
since now Tinfoil is on the client side it no longer has direct access.
|
||||
"""
|
||||
|
||||
class TinfoilCookerCollectionAdapter:
|
||||
""" cooker.collection adapter """
|
||||
def __init__(self, tinfoil):
|
||||
self.tinfoil = tinfoil
|
||||
def get_file_appends(self, fn):
|
||||
return self.tinfoil.run_command('getFileAppends', fn)
|
||||
def __getattr__(self, name):
|
||||
if name == 'overlayed':
|
||||
return self.tinfoil.get_overlayed_recipes()
|
||||
elif name == 'bbappends':
|
||||
return self.tinfoil.run_command('getAllAppends')
|
||||
else:
|
||||
raise AttributeError("%s instance has no attribute '%s'" % (self.__class__.__name__, name))
|
||||
|
||||
class TinfoilRecipeCacheAdapter:
|
||||
""" cooker.recipecache adapter """
|
||||
def __init__(self, tinfoil):
|
||||
self.tinfoil = tinfoil
|
||||
self._cache = {}
|
||||
|
||||
def get_pkg_pn_fn(self):
|
||||
pkg_pn = defaultdict(list, self.tinfoil.run_command('getRecipes') or [])
|
||||
pkg_fn = {}
|
||||
for pn, fnlist in pkg_pn.items():
|
||||
for fn in fnlist:
|
||||
pkg_fn[fn] = pn
|
||||
self._cache['pkg_pn'] = pkg_pn
|
||||
self._cache['pkg_fn'] = pkg_fn
|
||||
|
||||
def __getattr__(self, name):
|
||||
# Grab these only when they are requested since they aren't always used
|
||||
if name in self._cache:
|
||||
return self._cache[name]
|
||||
elif name == 'pkg_pn':
|
||||
self.get_pkg_pn_fn()
|
||||
return self._cache[name]
|
||||
elif name == 'pkg_fn':
|
||||
self.get_pkg_pn_fn()
|
||||
return self._cache[name]
|
||||
elif name == 'deps':
|
||||
attrvalue = defaultdict(list, self.tinfoil.run_command('getRecipeDepends') or [])
|
||||
elif name == 'rundeps':
|
||||
attrvalue = defaultdict(lambda: defaultdict(list), self.tinfoil.run_command('getRuntimeDepends') or [])
|
||||
elif name == 'runrecs':
|
||||
attrvalue = defaultdict(lambda: defaultdict(list), self.tinfoil.run_command('getRuntimeRecommends') or [])
|
||||
elif name == 'pkg_pepvpr':
|
||||
attrvalue = self.tinfoil.run_command('getRecipeVersions') or {}
|
||||
elif name == 'inherits':
|
||||
attrvalue = self.tinfoil.run_command('getRecipeInherits') or {}
|
||||
elif name == 'bbfile_priority':
|
||||
attrvalue = self.tinfoil.run_command('getBbFilePriority') or {}
|
||||
elif name == 'pkg_dp':
|
||||
attrvalue = self.tinfoil.run_command('getDefaultPreference') or {}
|
||||
else:
|
||||
raise AttributeError("%s instance has no attribute '%s'" % (self.__class__.__name__, name))
|
||||
|
||||
self._cache[name] = attrvalue
|
||||
return attrvalue
|
||||
|
||||
def __init__(self, tinfoil):
|
||||
self.tinfoil = tinfoil
|
||||
self.collection = self.TinfoilCookerCollectionAdapter(tinfoil)
|
||||
self.recipecaches = {}
|
||||
# FIXME all machines
|
||||
self.recipecaches[''] = self.TinfoilRecipeCacheAdapter(tinfoil)
|
||||
self._cache = {}
|
||||
def __getattr__(self, name):
|
||||
# Grab these only when they are requested since they aren't always used
|
||||
if name in self._cache:
|
||||
return self._cache[name]
|
||||
elif name == 'skiplist':
|
||||
attrvalue = self.tinfoil.get_skipped_recipes()
|
||||
elif name == 'bbfile_config_priorities':
|
||||
ret = self.tinfoil.run_command('getLayerPriorities')
|
||||
bbfile_config_priorities = []
|
||||
for collection, pattern, regex, pri in ret:
|
||||
bbfile_config_priorities.append((collection, pattern, re.compile(regex), pri))
|
||||
|
||||
attrvalue = bbfile_config_priorities
|
||||
else:
|
||||
raise AttributeError("%s instance has no attribute '%s'" % (self.__class__.__name__, name))
|
||||
|
||||
self._cache[name] = attrvalue
|
||||
return attrvalue
|
||||
|
||||
def findBestProvider(self, pn):
|
||||
return self.tinfoil.find_best_provider(pn)
|
||||
|
||||
|
||||
class Tinfoil:
|
||||
|
||||
def __init__(self, output=sys.stdout, tracking=False):
|
||||
# Needed to avoid deprecation warnings with python 2.6
|
||||
warnings.filterwarnings("ignore", category=DeprecationWarning)
|
||||
|
||||
# Set up logging
|
||||
self.logger = logging.getLogger('BitBake')
|
||||
self._log_hdlr = logging.StreamHandler(output)
|
||||
bb.msg.addDefaultlogFilter(self._log_hdlr)
|
||||
format = bb.msg.BBLogFormatter("%(levelname)s: %(message)s")
|
||||
if output.isatty():
|
||||
format.enable_color()
|
||||
self._log_hdlr.setFormatter(format)
|
||||
self.logger.addHandler(self._log_hdlr)
|
||||
|
||||
self.config = CookerConfiguration()
|
||||
configparams = TinfoilConfigParameters(parse_only=True)
|
||||
self.config.setConfigParameters(configparams)
|
||||
self.config.setServerRegIdleCallback(self.register_idle_function)
|
||||
features = []
|
||||
if tracking:
|
||||
features.append(CookerFeatures.BASEDATASTORE_TRACKING)
|
||||
self.cooker = BBCooker(self.config, features)
|
||||
self.config_data = self.cooker.data
|
||||
bb.providers.logger.setLevel(logging.ERROR)
|
||||
self.cooker_data = None
|
||||
|
||||
def register_idle_function(self, function, data):
|
||||
pass
|
||||
self.config_data = None
|
||||
self.cooker = None
|
||||
self.tracking = tracking
|
||||
self.ui_module = None
|
||||
self.server_connection = None
|
||||
|
||||
def __enter__(self):
|
||||
return self
|
||||
|
|
@ -65,30 +190,120 @@ class Tinfoil:
|
|||
def __exit__(self, type, value, traceback):
|
||||
self.shutdown()
|
||||
|
||||
def parseRecipes(self):
|
||||
sys.stderr.write("Parsing recipes..")
|
||||
self.logger.setLevel(logging.WARNING)
|
||||
def prepare(self, config_only=False, config_params=None, quiet=0):
|
||||
if self.tracking:
|
||||
extrafeatures = [bb.cooker.CookerFeatures.BASEDATASTORE_TRACKING]
|
||||
else:
|
||||
extrafeatures = []
|
||||
|
||||
try:
|
||||
while self.cooker.state in (state.initial, state.parsing):
|
||||
self.cooker.updateCache()
|
||||
except KeyboardInterrupt:
|
||||
self.cooker.shutdown()
|
||||
self.cooker.updateCache()
|
||||
sys.exit(2)
|
||||
if not config_params:
|
||||
config_params = TinfoilConfigParameters(config_only=config_only, quiet=quiet)
|
||||
|
||||
self.logger.setLevel(logging.INFO)
|
||||
sys.stderr.write("done.\n")
|
||||
cookerconfig = CookerConfiguration()
|
||||
cookerconfig.setConfigParameters(config_params)
|
||||
|
||||
self.cooker_data = self.cooker.recipecaches['']
|
||||
server, self.server_connection, ui_module = setup_bitbake(config_params,
|
||||
cookerconfig,
|
||||
extrafeatures)
|
||||
|
||||
def prepare(self, config_only = False):
|
||||
if not self.cooker_data:
|
||||
self.ui_module = ui_module
|
||||
|
||||
if self.server_connection:
|
||||
_server_connections.append(self.server_connection)
|
||||
if config_only:
|
||||
self.cooker.parseConfiguration()
|
||||
self.cooker_data = self.cooker.recipecaches['']
|
||||
config_params.updateToServer(self.server_connection.connection, os.environ.copy())
|
||||
self.run_command('parseConfiguration')
|
||||
else:
|
||||
self.parseRecipes()
|
||||
self.run_actions(config_params)
|
||||
|
||||
self.config_data = bb.data.init()
|
||||
connector = TinfoilDataStoreConnector(self, None)
|
||||
self.config_data.setVar('_remote_data', connector)
|
||||
self.cooker = TinfoilCookerAdapter(self)
|
||||
self.cooker_data = self.cooker.recipecaches['']
|
||||
else:
|
||||
raise Exception('Failed to start bitbake server')
|
||||
|
||||
def run_actions(self, config_params):
|
||||
"""
|
||||
Run the actions specified in config_params through the UI.
|
||||
"""
|
||||
ret = self.ui_module.main(self.server_connection.connection, self.server_connection.events, config_params)
|
||||
if ret:
|
||||
raise TinfoilUIException(ret)
|
||||
|
||||
def parseRecipes(self):
|
||||
"""
|
||||
Force a parse of all recipes. Normally you should specify
|
||||
config_only=False when calling prepare() instead of using this
|
||||
function; this function is designed for situations where you need
|
||||
to initialise Tinfoil and use it with config_only=True first and
|
||||
then conditionally call this function to parse recipes later.
|
||||
"""
|
||||
config_params = TinfoilConfigParameters(config_only=False)
|
||||
self.run_actions(config_params)
|
||||
|
||||
def run_command(self, command, *params):
|
||||
"""
|
||||
Run a command on the server (as implemented in bb.command).
|
||||
Note that there are two types of command - synchronous and
|
||||
asynchronous; in order to receive the results of asynchronous
|
||||
commands you will need to set an appropriate event mask
|
||||
using set_event_mask() and listen for the result using
|
||||
wait_event() - with the correct event mask you'll at least get
|
||||
bb.command.CommandCompleted and possibly other events before
|
||||
that depending on the command.
|
||||
"""
|
||||
if not self.server_connection:
|
||||
raise Exception('Not connected to server (did you call .prepare()?)')
|
||||
|
||||
commandline = [command]
|
||||
if params:
|
||||
commandline.extend(params)
|
||||
result = self.server_connection.connection.runCommand(commandline)
|
||||
if result[1]:
|
||||
raise TinfoilCommandFailed(result[1])
|
||||
return result[0]
|
||||
|
||||
def set_event_mask(self, eventlist):
|
||||
"""Set the event mask which will be applied within wait_event()"""
|
||||
if not self.server_connection:
|
||||
raise Exception('Not connected to server (did you call .prepare()?)')
|
||||
llevel, debug_domains = bb.msg.constructLogOptions()
|
||||
ret = self.run_command('setEventMask', self.server_connection.connection.getEventHandle(), llevel, debug_domains, eventlist)
|
||||
if not ret:
|
||||
raise Exception('setEventMask failed')
|
||||
|
||||
def wait_event(self, timeout=0):
|
||||
"""
|
||||
Wait for an event from the server for the specified time.
|
||||
A timeout of 0 means don't wait if there are no events in the queue.
|
||||
Returns the next event in the queue or None if the timeout was
|
||||
reached. Note that in order to recieve any events you will
|
||||
first need to set the internal event mask using set_event_mask()
|
||||
(otherwise whatever event mask the UI set up will be in effect).
|
||||
"""
|
||||
if not self.server_connection:
|
||||
raise Exception('Not connected to server (did you call .prepare()?)')
|
||||
return self.server_connection.events.waitEvent(timeout)
|
||||
|
||||
def get_overlayed_recipes(self):
|
||||
return defaultdict(list, self.run_command('getOverlayedRecipes'))
|
||||
|
||||
def get_skipped_recipes(self):
|
||||
return OrderedDict(self.run_command('getSkippedRecipes'))
|
||||
|
||||
def get_all_providers(self):
|
||||
return defaultdict(list, self.run_command('allProviders'))
|
||||
|
||||
def find_providers(self):
|
||||
return self.run_command('findProviders')
|
||||
|
||||
def find_best_provider(self, pn):
|
||||
return self.run_command('findBestProvider', pn)
|
||||
|
||||
def get_runtime_providers(self, rdep):
|
||||
return self.run_command('getRuntimeProviders', rdep)
|
||||
|
||||
def parse_recipe_file(self, fn, appends=True, appendlist=None, config_data=None):
|
||||
"""
|
||||
|
|
@ -126,22 +341,72 @@ class Tinfoil:
|
|||
envdata = parser.loadDataFull(fn, appendfiles)
|
||||
return envdata
|
||||
|
||||
def build_file(self, buildfile, task):
|
||||
"""
|
||||
Runs the specified task for just a single recipe (i.e. no dependencies).
|
||||
This is equivalent to bitbake -b.
|
||||
"""
|
||||
return self.run_command('buildFile', buildfile, task)
|
||||
|
||||
def shutdown(self):
|
||||
self.cooker.shutdown(force=True)
|
||||
self.cooker.post_serve()
|
||||
self.cooker.unlockBitbake()
|
||||
self.logger.removeHandler(self._log_hdlr)
|
||||
if self.server_connection:
|
||||
self.run_command('clientComplete')
|
||||
_server_connections.remove(self.server_connection)
|
||||
bb.event.ui_queue = []
|
||||
self.server_connection.terminate()
|
||||
self.server_connection = None
|
||||
|
||||
class TinfoilConfigParameters(ConfigParameters):
|
||||
def _reconvert_type(self, obj, origtypename):
|
||||
"""
|
||||
Convert an object back to the right type, in the case
|
||||
that marshalling has changed it (especially with xmlrpc)
|
||||
"""
|
||||
supported_types = {
|
||||
'set': set,
|
||||
'DataStoreConnectionHandle': bb.command.DataStoreConnectionHandle,
|
||||
}
|
||||
|
||||
def __init__(self, **options):
|
||||
origtype = supported_types.get(origtypename, None)
|
||||
if origtype is None:
|
||||
raise Exception('Unsupported type "%s"' % origtypename)
|
||||
if type(obj) == origtype:
|
||||
newobj = obj
|
||||
elif isinstance(obj, dict):
|
||||
# New style class
|
||||
newobj = origtype()
|
||||
for k,v in obj.items():
|
||||
setattr(newobj, k, v)
|
||||
else:
|
||||
# Assume we can coerce the type
|
||||
newobj = origtype(obj)
|
||||
|
||||
if isinstance(newobj, bb.command.DataStoreConnectionHandle):
|
||||
connector = TinfoilDataStoreConnector(self, newobj.dsindex)
|
||||
newobj = bb.data.init()
|
||||
newobj.setVar('_remote_data', connector)
|
||||
|
||||
return newobj
|
||||
|
||||
|
||||
class TinfoilConfigParameters(BitBakeConfigParameters):
|
||||
|
||||
def __init__(self, config_only, **options):
|
||||
self.initial_options = options
|
||||
# Apply some sane defaults
|
||||
if not 'parse_only' in options:
|
||||
self.initial_options['parse_only'] = not config_only
|
||||
#if not 'status_only' in options:
|
||||
# self.initial_options['status_only'] = config_only
|
||||
if not 'ui' in options:
|
||||
self.initial_options['ui'] = 'knotty'
|
||||
if not 'argv' in options:
|
||||
self.initial_options['argv'] = []
|
||||
|
||||
super(TinfoilConfigParameters, self).__init__()
|
||||
|
||||
def parseCommandLine(self, argv=sys.argv):
|
||||
class DummyOptions:
|
||||
def __init__(self, initial_options):
|
||||
for key, val in initial_options.items():
|
||||
setattr(self, key, val)
|
||||
|
||||
return DummyOptions(self.initial_options), None
|
||||
def parseCommandLine(self, argv=None):
|
||||
# We don't want any parameters parsed from the command line
|
||||
opts = super(TinfoilConfigParameters, self).parseCommandLine([])
|
||||
for key, val in self.initial_options.items():
|
||||
setattr(opts[0], key, val)
|
||||
return opts
|
||||
|
|
|
|||
|
|
@ -5,8 +5,6 @@ import sys
|
|||
import os
|
||||
import re
|
||||
|
||||
import bb.cache
|
||||
import bb.providers
|
||||
import bb.utils
|
||||
|
||||
from bblayers.common import LayerPlugin
|
||||
|
|
@ -122,15 +120,13 @@ skipped recipes will also be listed, with a " (skipped)" suffix.
|
|||
sys.exit(1)
|
||||
|
||||
pkg_pn = self.tinfoil.cooker.recipecaches[''].pkg_pn
|
||||
(latest_versions, preferred_versions) = bb.providers.findProviders(self.tinfoil.config_data, self.tinfoil.cooker.recipecaches[''], pkg_pn)
|
||||
allproviders = bb.providers.allProviders(self.tinfoil.cooker.recipecaches[''])
|
||||
(latest_versions, preferred_versions) = self.tinfoil.find_providers()
|
||||
allproviders = self.tinfoil.get_all_providers()
|
||||
|
||||
# Ensure we list skipped recipes
|
||||
# We are largely guessing about PN, PV and the preferred version here,
|
||||
# but we have no choice since skipped recipes are not fully parsed
|
||||
skiplist = list(self.tinfoil.cooker.skiplist.keys())
|
||||
skiplist.sort( key=lambda fileitem: self.tinfoil.cooker.collection.calc_bbfile_priority(fileitem) )
|
||||
skiplist.reverse()
|
||||
for fn in skiplist:
|
||||
recipe_parts = os.path.splitext(os.path.basename(fn))[0].split('_')
|
||||
p = recipe_parts[0]
|
||||
|
|
@ -265,10 +261,7 @@ Lists recipes with the bbappends that apply to them as subitems.
|
|||
def show_appends_for_pn(self, pn):
|
||||
filenames = self.tinfoil.cooker_data.pkg_pn[pn]
|
||||
|
||||
best = bb.providers.findBestProvider(pn,
|
||||
self.tinfoil.config_data,
|
||||
self.tinfoil.cooker_data,
|
||||
self.tinfoil.cooker_data.pkg_pn)
|
||||
best = self.tinfoil.find_best_provider(pn)
|
||||
best_filename = os.path.basename(best[3])
|
||||
|
||||
return self.show_appends_output(filenames, best_filename)
|
||||
|
|
@ -336,10 +329,7 @@ NOTE: .bbappend files can impact the dependencies.
|
|||
deps = self.tinfoil.cooker_data.deps[f]
|
||||
for pn in deps:
|
||||
if pn in self.tinfoil.cooker_data.pkg_pn:
|
||||
best = bb.providers.findBestProvider(pn,
|
||||
self.tinfoil.config_data,
|
||||
self.tinfoil.cooker_data,
|
||||
self.tinfoil.cooker_data.pkg_pn)
|
||||
best = self.tinfoil.find_best_provider(pn)
|
||||
self.check_cross_depends("DEPENDS", layername, f, best[3], args.filenames, ignore_layers)
|
||||
|
||||
# The RDPENDS
|
||||
|
|
@ -352,14 +342,11 @@ NOTE: .bbappend files can impact the dependencies.
|
|||
sorted_rdeps[k2] = 1
|
||||
all_rdeps = sorted_rdeps.keys()
|
||||
for rdep in all_rdeps:
|
||||
all_p = bb.providers.getRuntimeProviders(self.tinfoil.cooker_data, rdep)
|
||||
all_p, best = self.tinfoil.get_runtime_providers(rdep)
|
||||
if all_p:
|
||||
if f in all_p:
|
||||
# The recipe provides this one itself, ignore
|
||||
continue
|
||||
best = bb.providers.filterProvidersRunTime(all_p, rdep,
|
||||
self.tinfoil.config_data,
|
||||
self.tinfoil.cooker_data)[0][0]
|
||||
self.check_cross_depends("RDEPENDS", layername, f, best, args.filenames, ignore_layers)
|
||||
|
||||
# The RRECOMMENDS
|
||||
|
|
@ -372,14 +359,11 @@ NOTE: .bbappend files can impact the dependencies.
|
|||
sorted_rrecs[k2] = 1
|
||||
all_rrecs = sorted_rrecs.keys()
|
||||
for rrec in all_rrecs:
|
||||
all_p = bb.providers.getRuntimeProviders(self.tinfoil.cooker_data, rrec)
|
||||
all_p, best = self.tinfoil.get_runtime_providers(rrec)
|
||||
if all_p:
|
||||
if f in all_p:
|
||||
# The recipe provides this one itself, ignore
|
||||
continue
|
||||
best = bb.providers.filterProvidersRunTime(all_p, rrec,
|
||||
self.tinfoil.config_data,
|
||||
self.tinfoil.cooker_data)[0][0]
|
||||
self.check_cross_depends("RRECOMMENDS", layername, f, best, args.filenames, ignore_layers)
|
||||
|
||||
# The inherit class
|
||||
|
|
|
|||
Loading…
Reference in New Issue