bitbake: toasterui: add extra debug and development infos
We update and add logs throughout the code in order to help with development. The extra logging is turned off by default, but it can be enabled by using environment variables. All logging happens through the Python logging facilities. The toaster UI will save a log of all incoming events if the TOASTER_EVENTLOG variable is set. If TOASTER_SQLDEBUG is set all DB queries will be logged. If TOASTER_DEVEL is set and the django-fresh module is available, the module is enabled to allow auto-reload of pages when the source is changed. (Bitbake rev: 10c27450601b4d24bbb273bd0e053498807d1060) Signed-off-by: Alexandru DAMIAN <alexandru.damian@intel.com> Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
This commit is contained in:
Alexandru DAMIAN
Richard Purdie
parent
f99f2cdd69
commit
d086fa3aed
|
|
@ -33,6 +33,11 @@ from toaster.orm.models import Task_Dependency, Package_Dependency
|
|||
from toaster.orm.models import Recipe_Dependency
|
||||
from bb.msg import BBLogFormatter as format
|
||||
from django.db import models
|
||||
import logging
|
||||
|
||||
|
||||
logger = logging.getLogger("BitBake")
|
||||
|
||||
|
||||
class NotExisting(Exception):
|
||||
pass
|
||||
|
|
@ -115,7 +120,9 @@ class ORMWrapper(object):
|
|||
build_name=build_info['build_name'],
|
||||
bitbake_version=build_info['bitbake_version'])
|
||||
|
||||
logger.debug(1, "buildinfohelper: build is created %s" % build)
|
||||
if brbe is not None:
|
||||
logger.debug(1, "buildinfohelper: brbe is %s" % brbe)
|
||||
from bldcontrol.models import BuildEnvironment, BuildRequest
|
||||
br, be = brbe.split(":")
|
||||
|
||||
|
|
@ -605,6 +612,7 @@ class BuildInfoHelper(object):
|
|||
self.has_build_history = has_build_history
|
||||
self.tmp_dir = self.server.runCommand(["getVariable", "TMPDIR"])[0]
|
||||
self.brbe = self.server.runCommand(["getVariable", "TOASTER_BRBE"])[0]
|
||||
logger.debug(1, "buildinfohelper: Build info helper inited %s" % vars(self))
|
||||
|
||||
|
||||
def _configure_django(self):
|
||||
|
|
@ -1110,10 +1118,10 @@ class BuildInfoHelper(object):
|
|||
if 'build' in self.internal_state and 'backlog' in self.internal_state:
|
||||
if len(self.internal_state['backlog']):
|
||||
tempevent = self.internal_state['backlog'].pop()
|
||||
print "DEBUG: Saving stored event ", tempevent
|
||||
logger.debug(1, "buildinfohelper: Saving stored event %s " % tempevent)
|
||||
self.store_log_event(tempevent)
|
||||
else:
|
||||
print "ERROR: Events not saved: \n", self.internal_state['backlog']
|
||||
logger.error("buildinfohelper: Events not saved: %s" % self.internal_state['backlog'])
|
||||
del self.internal_state['backlog']
|
||||
|
||||
log_information = {}
|
||||
|
|
|
|||
|
|
@ -62,15 +62,6 @@ def _log_settings_from_server(server):
|
|||
|
||||
def main(server, eventHandler, params ):
|
||||
|
||||
includelogs, loglines = _log_settings_from_server(server)
|
||||
|
||||
# verify and warn
|
||||
build_history_enabled = True
|
||||
inheritlist, error = server.runCommand(["getVariable", "INHERIT"])
|
||||
if not "buildhistory" in inheritlist.split(" "):
|
||||
logger.warn("buildhistory is not enabled. Please enable INHERIT += \"buildhistory\" to see image details.")
|
||||
build_history_enabled = False
|
||||
|
||||
helper = uihelper.BBUIHelper()
|
||||
|
||||
console = logging.StreamHandler(sys.stdout)
|
||||
|
|
@ -80,6 +71,16 @@ def main(server, eventHandler, params ):
|
|||
console.setFormatter(format)
|
||||
logger.addHandler(console)
|
||||
|
||||
includelogs, loglines = _log_settings_from_server(server)
|
||||
|
||||
# verify and warn
|
||||
build_history_enabled = True
|
||||
inheritlist, error = server.runCommand(["getVariable", "INHERIT"])
|
||||
|
||||
if not "buildhistory" in inheritlist.split(" "):
|
||||
logger.warn("buildhistory is not enabled. Please enable INHERIT += \"buildhistory\" to see image details.")
|
||||
build_history_enabled = False
|
||||
|
||||
if not params.observe_only:
|
||||
logger.error("ToasterUI can only work in observer mode")
|
||||
return
|
||||
|
|
|
|||
|
|
@ -32,6 +32,10 @@ from toastermain import settings
|
|||
|
||||
from bbcontroller import BuildEnvironmentController, ShellCmdException, BuildSetupException, _getgitcheckoutdirectoryname
|
||||
|
||||
import logging
|
||||
logger = logging.getLogger("toaster")
|
||||
|
||||
|
||||
class LocalhostBEController(BuildEnvironmentController):
|
||||
""" Implementation of the BuildEnvironmentController for the localhost;
|
||||
this controller manages the default build directory,
|
||||
|
|
@ -56,8 +60,10 @@ class LocalhostBEController(BuildEnvironmentController):
|
|||
err = "command: %s \n%s" % (command, out)
|
||||
else:
|
||||
err = "command: %s \n%s" % (command, err)
|
||||
logger.debug("localhostbecontroller: shellcmd error %s" % err)
|
||||
raise ShellCmdException(err)
|
||||
else:
|
||||
logger.debug("localhostbecontroller: shellcmd success")
|
||||
return out
|
||||
|
||||
def _createdirpath(self, path):
|
||||
|
|
@ -90,7 +96,7 @@ class LocalhostBEController(BuildEnvironmentController):
|
|||
for i in self._shellcmd(cmd).split("\n"):
|
||||
if i.startswith("Bitbake server address"):
|
||||
port = i.split(" ")[-1]
|
||||
print "Found bitbake server port ", port
|
||||
logger.debug("localhostbecontroller: Found bitbake server port %s" % port)
|
||||
|
||||
def _toaster_ui_started(filepath):
|
||||
if not os.path.exists(filepath):
|
||||
|
|
@ -103,10 +109,10 @@ class LocalhostBEController(BuildEnvironmentController):
|
|||
|
||||
while not _toaster_ui_started(os.path.join(self.be.builddir, "toaster_ui.log")):
|
||||
import time
|
||||
print "DEBUG: Waiting server to start"
|
||||
logger.debug("localhostbecontroller: Waiting bitbake server to start")
|
||||
time.sleep(0.5)
|
||||
|
||||
print("DEBUG: Started server")
|
||||
logger.debug("localhostbecontroller: Started bitbake server")
|
||||
assert self.be.sourcedir and os.path.exists(self.be.builddir)
|
||||
self.be.bbaddress = "localhost"
|
||||
self.be.bbport = port
|
||||
|
|
@ -116,11 +122,11 @@ class LocalhostBEController(BuildEnvironmentController):
|
|||
def stopBBServer(self):
|
||||
assert self.pokydirname and os.path.exists(self.pokydirname)
|
||||
assert self.islayerset
|
||||
print self._shellcmd("bash -c \"source %s/oe-init-build-env %s && %s source toaster stop\"" %
|
||||
self._shellcmd("bash -c \"source %s/oe-init-build-env %s && %s source toaster stop\"" %
|
||||
(self.pokydirname, self.be.builddir, (lambda: "" if self.be.bbtoken is None else "BBTOKEN=%s" % self.be.bbtoken)()))
|
||||
self.be.bbstate = BuildEnvironment.SERVER_STOPPED
|
||||
self.be.save()
|
||||
print "Stopped server"
|
||||
logger.debug("localhostbecontroller: Stopped bitbake server")
|
||||
|
||||
def setLayers(self, bitbakes, layers):
|
||||
""" a word of attention: by convention, the first layer for any build will be poky! """
|
||||
|
|
@ -149,12 +155,13 @@ class LocalhostBEController(BuildEnvironmentController):
|
|||
raise BuildSetupException("More than one commit per git url, unsupported configuration: \n%s" % pprint.pformat(gitrepos))
|
||||
|
||||
|
||||
logger.debug("localhostbecontroller, our git repos are %s" % gitrepos)
|
||||
layerlist = []
|
||||
|
||||
# 2. checkout the repositories
|
||||
for giturl in gitrepos.keys():
|
||||
localdirname = os.path.join(self.be.sourcedir, _getgitcheckoutdirectoryname(giturl))
|
||||
print "DEBUG: giturl ", giturl ,"checking out in current directory", localdirname
|
||||
logger.debug("localhostbecontroller: giturl %s checking out in current directory %s" % (giturl, localdirname))
|
||||
|
||||
# make sure our directory is a git repository
|
||||
if os.path.exists(localdirname):
|
||||
|
|
@ -167,17 +174,17 @@ class LocalhostBEController(BuildEnvironmentController):
|
|||
|
||||
# branch magic name "HEAD" will inhibit checkout
|
||||
if commit != "HEAD":
|
||||
print "DEBUG: checking out commit ", commit, "to", localdirname
|
||||
logger.debug("localhostbecontroller: checking out commit %s to %s " % (commit, localdirname))
|
||||
self._shellcmd("git fetch --all && git checkout \"%s\"" % commit , localdirname)
|
||||
|
||||
# take the localdirname as poky dir if we can find the oe-init-build-env
|
||||
if self.pokydirname is None and os.path.exists(os.path.join(localdirname, "oe-init-build-env")):
|
||||
print "DEBUG: selected poky dir name", localdirname
|
||||
logger.debug("localhostbecontroller: selected poky dir name %s" % localdirname)
|
||||
self.pokydirname = localdirname
|
||||
|
||||
# make sure we have a working bitbake
|
||||
if not os.path.exists(os.path.join(self.pokydirname, 'bitbake')):
|
||||
print "DEBUG: checking bitbake into the poky dirname %s " % self.pokydirname
|
||||
logger.debug("localhostbecontroller: checking bitbake into the poky dirname %s " % self.pokydirname)
|
||||
self._shellcmd("git clone -b \"%s\" \"%s\" \"%s\" " % (bitbakes[0].commit, bitbakes[0].giturl, os.path.join(self.pokydirname, 'bitbake')))
|
||||
|
||||
# verify our repositories
|
||||
|
|
@ -189,7 +196,7 @@ class LocalhostBEController(BuildEnvironmentController):
|
|||
if name != "bitbake":
|
||||
layerlist.append(localdirpath.rstrip("/"))
|
||||
|
||||
print "DEBUG: current layer list ", layerlist
|
||||
logger.debug("localhostbecontroller: current layer list %s " % layerlist)
|
||||
|
||||
# 3. configure the build environment, so we have a conf/bblayers.conf
|
||||
assert self.pokydirname is not None
|
||||
|
|
|
|||
|
|
@ -1,7 +1,7 @@
|
|||
from django.core.management.base import NoArgsCommand, CommandError
|
||||
from django.db import transaction
|
||||
from bldcontrol.bbcontroller import getBuildEnvironmentController, ShellCmdException
|
||||
from bldcontrol.models import BuildRequest, BuildEnvironment
|
||||
from bldcontrol.models import BuildRequest, BuildEnvironment, BRError
|
||||
from orm.models import ToasterSetting
|
||||
import os
|
||||
|
||||
|
|
|
|||
|
|
@ -4,6 +4,9 @@ from orm.models import Build
|
|||
from bldcontrol.bbcontroller import getBuildEnvironmentController, ShellCmdException, BuildSetupException
|
||||
from bldcontrol.models import BuildRequest, BuildEnvironment, BRError
|
||||
import os
|
||||
import logging
|
||||
|
||||
logger = logging.getLogger("toaster")
|
||||
|
||||
class Command(NoArgsCommand):
|
||||
args = ""
|
||||
|
|
@ -32,6 +35,7 @@ class Command(NoArgsCommand):
|
|||
# select the build environment and the request to build
|
||||
br = self._selectBuildRequest()
|
||||
except IndexError as e:
|
||||
# logger.debug("runbuilds: No build request")
|
||||
return
|
||||
try:
|
||||
bec = self._selectBuildEnvironment()
|
||||
|
|
@ -39,10 +43,10 @@ class Command(NoArgsCommand):
|
|||
# we could not find a BEC; postpone the BR
|
||||
br.state = BuildRequest.REQ_QUEUED
|
||||
br.save()
|
||||
print "No build env"
|
||||
logger.debug("runbuilds: No build env")
|
||||
return
|
||||
|
||||
print "Build %s, Environment %s" % (br, bec.be)
|
||||
logger.debug("runbuilds: starting build %s, environment %s" % (br, bec.be))
|
||||
# let the build request know where it is being executed
|
||||
br.environment = bec.be
|
||||
br.save()
|
||||
|
|
@ -63,7 +67,7 @@ class Command(NoArgsCommand):
|
|||
task = None
|
||||
bbctrl.build(list(map(lambda x:x.target, br.brtarget_set.all())), task)
|
||||
|
||||
print "Build launched, exiting"
|
||||
logger.debug("runbuilds: Build launched, exiting")
|
||||
# disconnect from the server
|
||||
bbctrl.disconnect()
|
||||
|
||||
|
|
@ -71,7 +75,7 @@ class Command(NoArgsCommand):
|
|||
|
||||
|
||||
except Exception as e:
|
||||
print " EE Error executing shell command\n", e
|
||||
logger.error("runbuilds: Error executing shell command %s" % e)
|
||||
traceback.print_exc(e)
|
||||
BRError.objects.create(req = br,
|
||||
errtype = str(type(e)),
|
||||
|
|
@ -83,6 +87,7 @@ class Command(NoArgsCommand):
|
|||
bec.be.save()
|
||||
|
||||
|
||||
|
||||
def cleanup(self):
|
||||
from django.utils import timezone
|
||||
from datetime import timedelta
|
||||
|
|
|
|||
|
|
@ -21,11 +21,15 @@
|
|||
|
||||
# Django settings for Toaster project.
|
||||
|
||||
import os, re
|
||||
|
||||
DEBUG = True
|
||||
TEMPLATE_DEBUG = DEBUG
|
||||
|
||||
# Set to True to see the SQL queries in console
|
||||
SQL_DEBUG = False
|
||||
if os.environ.get("TOASTER_SQLDEBUG", None) is not None:
|
||||
SQL_DEBUG = True
|
||||
|
||||
|
||||
ADMINS = (
|
||||
|
|
@ -46,7 +50,6 @@ DATABASES = {
|
|||
}
|
||||
|
||||
# Reinterpret database settings if we have DATABASE_URL environment variable defined
|
||||
import os, re
|
||||
|
||||
if 'DATABASE_URL' in os.environ:
|
||||
dburl = os.environ['DATABASE_URL']
|
||||
|
|
@ -212,6 +215,9 @@ MIDDLEWARE_CLASSES = (
|
|||
# 'django.middleware.clickjacking.XFrameOptionsMiddleware',
|
||||
)
|
||||
|
||||
from os.path import dirname as DN
|
||||
SITE_ROOT=DN(DN(os.path.abspath(__file__)))
|
||||
|
||||
ROOT_URLCONF = 'toastermain.urls'
|
||||
|
||||
# Python dotted path to the WSGI application used by Django's runserver.
|
||||
|
|
@ -245,6 +251,19 @@ INSTALLED_APPS = (
|
|||
'south',
|
||||
)
|
||||
|
||||
|
||||
# Load django-fresh is TOASTER_DEVEL is set, and the module is available
|
||||
FRESH_ENABLED = False
|
||||
if os.environ.get('TOASTER_DEVEL', None) is not None:
|
||||
try:
|
||||
import fresh
|
||||
MIDDLEWARE_CLASSES = MIDDLEWARE_CLASSES + ("fresh.middleware.FreshMiddleware",)
|
||||
INSTALLED_APPS = INSTALLED_APPS + ('fresh',)
|
||||
FRESH_ENABLED = True
|
||||
except:
|
||||
pass
|
||||
|
||||
|
||||
SOUTH_TESTS_MIGRATE = False
|
||||
|
||||
# if we run in managed mode, we need user support
|
||||
|
|
@ -286,7 +305,7 @@ LOGGING = {
|
|||
},
|
||||
'formatters': {
|
||||
'datetime': {
|
||||
'format': 'DB %(asctime)s %(message)s'
|
||||
'format': '%(levelname)s %(asctime)s %(message)s'
|
||||
}
|
||||
},
|
||||
'handlers': {
|
||||
|
|
@ -302,6 +321,10 @@ LOGGING = {
|
|||
}
|
||||
},
|
||||
'loggers': {
|
||||
'toaster' : {
|
||||
'handlers': ['console'],
|
||||
'level': 'DEBUG',
|
||||
},
|
||||
'django.request': {
|
||||
'handlers': ['mail_admins'],
|
||||
'level': 'ERROR',
|
||||
|
|
|
|||
|
|
@ -44,11 +44,15 @@ urlpatterns = patterns('',
|
|||
)
|
||||
|
||||
import toastermain.settings
|
||||
|
||||
if toastermain.settings.FRESH_ENABLED:
|
||||
urlpatterns.insert(1, url(r'', include('fresh.urls')))
|
||||
|
||||
if toastermain.settings.MANAGED:
|
||||
urlpatterns = urlpatterns + [
|
||||
urlpatterns = [
|
||||
# Uncomment the next line to enable the admin:
|
||||
url(r'^admin/', include(admin.site.urls)),
|
||||
]
|
||||
] + urlpatterns
|
||||
# Automatically discover urls.py in various apps, beside our own
|
||||
# and map module directories to the patterns
|
||||
|
||||
|
|
@ -61,4 +65,4 @@ for t in os.walk(os.path.dirname(currentdir)):
|
|||
|
||||
if "urls.py" in t[2] and t[0] != currentdir:
|
||||
modulename = os.path.basename(t[0])
|
||||
urlpatterns.append( url(r'^' + modulename + '/', include ( modulename + '.urls')))
|
||||
urlpatterns.insert(0, url(r'^' + modulename + '/', include ( modulename + '.urls')))
|
||||
|
|
|
|||
Loading…
Reference in New Issue