2013-10-11 12:46:23 +00:00
#
2014-01-14 12:50:32 +00:00
# ex:ts=4:sw=4:sts=4:et
# -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*-
#
2013-10-11 12:46:23 +00:00
# BitBake Toaster Implementation
#
# Copyright (C) 2013 Intel Corporation
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License version 2 as
# published by the Free Software Foundation.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License along
# with this program; if not, write to the Free Software Foundation, Inc.,
# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
2016-03-08 18:09:02 +00:00
from __future__ import unicode_literals
2015-08-18 16:28:54 +00:00
from django . db import models , IntegrityError
2015-11-04 14:48:54 +00:00
from django . db . models import F , Q , Avg , Max , Sum
2014-08-08 14:03:03 +00:00
from django . utils import timezone
2016-03-08 18:09:02 +00:00
from django . utils . encoding import force_bytes
2014-08-08 14:03:03 +00:00
2015-08-04 19:46:29 +00:00
from django . core . urlresolvers import reverse
2014-10-09 11:37:30 +00:00
from django . core import validators
2014-12-05 15:14:20 +00:00
from django . conf import settings
2015-05-19 15:10:19 +00:00
import django . db . models . signals
2014-10-09 11:37:30 +00:00
2016-01-15 11:00:46 +00:00
import os . path
import re
2016-01-18 14:23:55 +00:00
import itertools
2016-01-15 11:00:46 +00:00
2015-08-18 16:28:52 +00:00
import logging
logger = logging . getLogger ( " toaster " )
2015-12-10 03:56:40 +00:00
if ' sqlite ' in settings . DATABASES [ ' default ' ] [ ' ENGINE ' ] :
from django . db import transaction , OperationalError
from time import sleep
_base_save = models . Model . save
def save ( self , * args , * * kwargs ) :
while True :
try :
with transaction . atomic ( ) :
return _base_save ( self , * args , * * kwargs )
except OperationalError as err :
if ' database is locked ' in str ( err ) :
logger . warning ( " %s , model: %s , args: %s , kwargs: %s " ,
err , self . __class__ , args , kwargs )
sleep ( 0.5 )
continue
raise
models . Model . save = save
2015-08-18 16:28:52 +00:00
2015-12-10 03:56:42 +00:00
# HACK: Monkey patch Django to fix 'database is locked' issue
from django . db . models . query import QuerySet
_base_insert = QuerySet . _insert
def _insert ( self , * args , * * kwargs ) :
with transaction . atomic ( using = self . db , savepoint = False ) :
return _base_insert ( self , * args , * * kwargs )
QuerySet . _insert = _insert
from django . utils import six
def _create_object_from_params ( self , lookup , params ) :
"""
Tries to create an object using passed params .
Used by get_or_create and update_or_create
"""
try :
obj = self . create ( * * params )
return obj , True
except IntegrityError :
exc_info = sys . exc_info ( )
try :
return self . get ( * * lookup ) , False
except self . model . DoesNotExist :
pass
six . reraise ( * exc_info )
QuerySet . _create_object_from_params = _create_object_from_params
# end of HACK
2014-10-09 11:37:30 +00:00
class GitURLValidator ( validators . URLValidator ) :
import re
regex = re . compile (
r ' ^(?:ssh|git|http|ftp)s?:// ' # http:// or https://
r ' (?:(?:[A-Z0-9](?:[A-Z0-9-] { 0,61}[A-Z0-9])? \ .)+(?:[A-Z] { 2,6} \ .?|[A-Z0-9-] { 2,} \ .?)| ' # domain...
r ' localhost| ' # localhost...
r ' \ d { 1,3} \ . \ d { 1,3} \ . \ d { 1,3} \ . \ d { 1,3}| ' # ...or ipv4
r ' \ [?[A-F0-9]*:[A-F0-9:]+ \ ]?) ' # ...or ipv6
r ' (?:: \ d+)? ' # optional port
r ' (?:/?|[/?] \ S+)$ ' , re . IGNORECASE )
def GitURLField ( * * kwargs ) :
r = models . URLField ( * * kwargs )
for i in xrange ( len ( r . validators ) ) :
if isinstance ( r . validators [ i ] , validators . URLValidator ) :
r . validators [ i ] = GitURLValidator ( )
return r
2014-08-08 14:03:03 +00:00
class ToasterSetting ( models . Model ) :
name = models . CharField ( max_length = 63 )
helptext = models . TextField ( )
value = models . CharField ( max_length = 255 )
2014-10-09 11:37:30 +00:00
def __unicode__ ( self ) :
2015-05-01 21:58:15 +00:00
return " Setting %s = %s " % ( self . name , self . value )
2014-10-09 11:37:30 +00:00
2014-06-30 14:58:36 +00:00
class ProjectManager ( models . Manager ) :
2014-08-08 14:03:03 +00:00
def create_project ( self , name , release ) :
2015-06-22 13:48:53 +00:00
if release is not None :
prj = self . model ( name = name , bitbake_version = release . bitbake_version , release = release )
else :
prj = self . model ( name = name , bitbake_version = None , release = None )
2014-06-30 14:58:36 +00:00
prj . save ( )
2014-08-08 14:03:03 +00:00
for defaultconf in ToasterSetting . objects . filter ( name__startswith = " DEFCONF_ " ) :
name = defaultconf . name [ 8 : ]
ProjectVariable . objects . create ( project = prj ,
name = name ,
value = defaultconf . value )
2015-06-22 13:48:53 +00:00
if release is None :
return prj
2014-11-14 17:07:06 +00:00
for rdl in release . releasedefaultlayer_set . all ( ) :
2015-02-13 13:08:26 +00:00
try :
2015-08-18 16:28:57 +00:00
lv = Layer_Version . objects . filter ( layer__name = rdl . layer_name , up_branch__name = release . branch_name ) [ 0 ] . get_equivalents_wpriority ( prj ) [ 0 ]
2015-02-13 13:08:26 +00:00
ProjectLayer . objects . create ( project = prj ,
2014-08-08 14:03:03 +00:00
layercommit = lv ,
optional = False )
2015-02-13 13:08:26 +00:00
except IndexError :
# we may have no valid layer version objects, and that's ok
pass
2014-06-30 14:58:36 +00:00
return prj
2015-09-03 00:25:08 +00:00
# return single object with is_default = True
2015-12-10 03:56:29 +00:00
def get_or_create_default_project ( self ) :
2015-09-03 00:25:08 +00:00
projects = super ( ProjectManager , self ) . filter ( is_default = True )
2015-12-10 03:56:29 +00:00
2015-09-03 00:25:08 +00:00
if len ( projects ) > 1 :
2015-12-10 03:56:29 +00:00
raise Exception ( ' Inconsistent project data: multiple ' +
' default projects (i.e. with is_default=True) ' )
2015-09-03 00:25:08 +00:00
elif len ( projects ) < 1 :
2015-12-10 03:56:29 +00:00
options = {
' name ' : ' Command line builds ' ,
' short_description ' : ' Project for builds started outside Toaster ' ,
' is_default ' : True
}
project = Project . objects . create ( * * options )
project . save ( )
return project
else :
return projects [ 0 ]
2014-06-30 14:58:36 +00:00
2014-06-03 15:26:17 +00:00
class Project ( models . Model ) :
2014-11-14 17:07:06 +00:00
search_allowed_fields = [ ' name ' , ' short_description ' , ' release__name ' , ' release__branch_name ' ]
2014-06-03 15:26:17 +00:00
name = models . CharField ( max_length = 100 )
2014-06-30 14:58:36 +00:00
short_description = models . CharField ( max_length = 50 , blank = True )
2015-05-14 15:44:27 +00:00
bitbake_version = models . ForeignKey ( ' BitbakeVersion ' , null = True )
release = models . ForeignKey ( " Release " , null = True )
2014-06-03 15:26:17 +00:00
created = models . DateTimeField ( auto_now_add = True )
updated = models . DateTimeField ( auto_now = True )
2014-06-30 14:58:36 +00:00
# This is a horrible hack; since Toaster has no "User" model available when
# running in interactive mode, we can't reference the field here directly
# Instead, we keep a possible null reference to the User id, as not to force
# hard links to possibly missing models
user_id = models . IntegerField ( null = True )
objects = ProjectManager ( )
2013-10-11 12:46:23 +00:00
2015-09-03 00:25:08 +00:00
# set to True for the project which is the default container
# for builds initiated by the command line etc.
is_default = models . BooleanField ( default = False )
2014-11-14 17:07:06 +00:00
def __unicode__ ( self ) :
2015-03-12 11:30:43 +00:00
return " %s (Release %s , BBV %s ) " % ( self . name , self . release , self . bitbake_version )
2014-07-15 12:01:56 +00:00
2015-02-26 21:42:00 +00:00
def get_current_machine_name ( self ) :
try :
return self . projectvariable_set . get ( name = " MACHINE " ) . value
except ( ProjectVariable . DoesNotExist , IndexError ) :
2015-11-13 13:48:34 +00:00
return None ;
2015-02-26 21:42:00 +00:00
def get_number_of_builds ( self ) :
2015-11-13 13:48:34 +00:00
""" Return the number of builds which have ended """
return self . build_set . filter ( ~ Q ( outcome = Build . IN_PROGRESS ) ) . count ( )
2015-02-26 21:42:00 +00:00
def get_last_build_id ( self ) :
try :
return Build . objects . filter ( project = self . id ) . order_by ( ' -completed_on ' ) [ 0 ] . id
except ( Build . DoesNotExist , IndexError ) :
return ( - 1 )
def get_last_outcome ( self ) :
build_id = self . get_last_build_id
if ( - 1 == build_id ) :
return ( " " )
try :
return Build . objects . filter ( id = self . get_last_build_id ) [ 0 ] . outcome
except ( Build . DoesNotExist , IndexError ) :
return ( " not_found " )
def get_last_target ( self ) :
build_id = self . get_last_build_id
if ( - 1 == build_id ) :
return ( " " )
try :
return Target . objects . filter ( build = build_id ) [ 0 ] . target
except ( Target . DoesNotExist , IndexError ) :
return ( " not_found " )
def get_last_errors ( self ) :
build_id = self . get_last_build_id
if ( - 1 == build_id ) :
return ( 0 )
try :
2015-06-17 16:30:34 +00:00
return Build . objects . filter ( id = build_id ) [ 0 ] . errors . count ( )
2015-02-26 21:42:00 +00:00
except ( Build . DoesNotExist , IndexError ) :
return ( " not_found " )
def get_last_warnings ( self ) :
build_id = self . get_last_build_id
if ( - 1 == build_id ) :
return ( 0 )
try :
2015-06-17 16:30:34 +00:00
return Build . objects . filter ( id = build_id ) [ 0 ] . warnings . count ( )
2015-02-26 21:42:00 +00:00
except ( Build . DoesNotExist , IndexError ) :
return ( " not_found " )
2016-01-15 11:00:46 +00:00
def get_last_build_extensions ( self ) :
"""
Get list of file name extensions for images produced by the most
recent build
"""
last_build = Build . objects . get ( pk = self . get_last_build_id ( ) )
return last_build . get_image_file_extensions ( )
2015-02-26 21:42:00 +00:00
def get_last_imgfiles ( self ) :
build_id = self . get_last_build_id
if ( - 1 == build_id ) :
return ( " " )
try :
return Variable . objects . filter ( build = build_id , variable_name = " IMAGE_FSTYPES " ) [ 0 ] . variable_value
except ( Variable . DoesNotExist , IndexError ) :
return ( " not_found " )
2015-09-29 09:50:10 +00:00
def get_all_compatible_layer_versions ( self ) :
""" Returns Queryset of all Layer_Versions which are compatible with
this project """
queryset = Layer_Version . objects . filter (
2016-03-09 13:01:24 +00:00
( Q ( up_branch__name = self . release . branch_name ) &
Q ( build = None ) &
Q ( project = None ) ) |
Q ( project = self ) )
2015-09-29 09:50:10 +00:00
return queryset
def get_project_layer_versions ( self , pk = False ) :
""" Returns the Layer_Versions currently added to this project """
2015-12-07 18:15:43 +00:00
layer_versions = self . projectlayer_set . all ( ) . values_list ( ' layercommit ' ,
flat = True )
2015-09-29 09:50:10 +00:00
if pk is False :
2015-12-07 18:15:43 +00:00
return Layer_Version . objects . filter ( pk__in = layer_versions )
2015-09-29 09:50:10 +00:00
else :
2015-12-07 18:15:43 +00:00
return layer_versions
2015-09-29 09:50:10 +00:00
2014-12-03 13:55:19 +00:00
2015-08-04 19:46:30 +00:00
def get_available_machines ( self ) :
""" Returns QuerySet of all Machines which are provided by the
Layers currently added to the Project """
2015-09-29 09:50:10 +00:00
queryset = Machine . objects . filter (
2015-10-07 03:21:25 +00:00
layer_version__in = self . get_project_layer_versions ( ) )
2015-09-29 09:50:10 +00:00
2015-08-04 19:46:30 +00:00
return queryset
def get_all_compatible_machines ( self ) :
""" Returns QuerySet of all the compatible machines available to the
project including ones from Layers not currently added """
2015-09-29 09:50:10 +00:00
queryset = Machine . objects . filter (
layer_version__in = self . get_all_compatible_layer_versions ( ) )
2015-08-04 19:46:30 +00:00
return queryset
def get_available_recipes ( self ) :
2015-09-29 09:50:10 +00:00
""" Returns QuerySet of all the recipes that are provided by layers
added to this project """
queryset = Recipe . objects . filter (
layer_version__in = self . get_project_layer_versions ( ) )
2015-08-04 19:46:30 +00:00
return queryset
def get_all_compatible_recipes ( self ) :
""" Returns QuerySet of all the compatible Recipes available to the
project including ones from Layers not currently added """
2015-09-29 09:50:10 +00:00
queryset = Recipe . objects . filter (
2015-10-01 21:19:57 +00:00
layer_version__in = self . get_all_compatible_layer_versions ( ) ) . exclude ( name__exact = ' ' )
2015-08-04 19:46:30 +00:00
return queryset
2014-07-15 12:01:56 +00:00
def schedule_build ( self ) :
2014-08-08 14:03:03 +00:00
from bldcontrol . models import BuildRequest , BRTarget , BRLayer , BRVariable , BRBitbake
2014-07-15 12:01:56 +00:00
br = BuildRequest . objects . create ( project = self )
2014-09-09 10:47:13 +00:00
try :
2014-08-08 14:03:03 +00:00
2014-09-09 10:47:13 +00:00
BRBitbake . objects . create ( req = br ,
giturl = self . bitbake_version . giturl ,
commit = self . bitbake_version . branch ,
dirpath = self . bitbake_version . dirpath )
2014-11-05 14:47:51 +00:00
for l in self . projectlayer_set . all ( ) . order_by ( " pk " ) :
2015-03-04 15:52:01 +00:00
commit = l . layercommit . get_vcs_reference ( )
print ( " ii Building layer " , l . layercommit . layer . name , " at vcs point " , commit )
2015-09-29 04:45:27 +00:00
BRLayer . objects . create ( req = br , name = l . layercommit . layer . name , giturl = l . layercommit . layer . vcs_url , commit = commit , dirpath = l . layercommit . dirpath , layer_version = l . layercommit )
2015-06-11 17:27:53 +00:00
br . state = BuildRequest . REQ_QUEUED
now = timezone . now ( )
br . build = Build . objects . create ( project = self ,
completed_on = now ,
started_on = now ,
)
2014-09-09 10:47:13 +00:00
for t in self . projecttarget_set . all ( ) :
BRTarget . objects . create ( req = br , target = t . target , task = t . task )
2015-09-22 09:34:52 +00:00
Target . objects . create ( build = br . build , target = t . target , task = t . task )
2015-06-11 17:27:53 +00:00
2014-09-09 10:47:13 +00:00
for v in self . projectvariable_set . all ( ) :
BRVariable . objects . create ( req = br , name = v . name , value = v . value )
2015-06-11 17:27:53 +00:00
try :
br . build . machine = self . projectvariable_set . get ( name = ' MACHINE ' ) . value
br . build . save ( )
except ProjectVariable . DoesNotExist :
pass
2014-09-09 10:47:13 +00:00
br . save ( )
2015-08-18 16:28:50 +00:00
except Exception :
# revert the build request creation since we're not done cleanly
2014-09-09 10:47:13 +00:00
br . delete ( )
2015-08-18 16:28:50 +00:00
raise
2014-07-15 12:01:56 +00:00
return br
2013-10-11 12:46:23 +00:00
class Build ( models . Model ) :
SUCCEEDED = 0
FAILED = 1
IN_PROGRESS = 2
BUILD_OUTCOME = (
( SUCCEEDED , ' Succeeded ' ) ,
( FAILED , ' Failed ' ) ,
( IN_PROGRESS , ' In Progress ' ) ,
)
2014-02-20 12:47:55 +00:00
search_allowed_fields = [ ' machine ' , ' cooker_log_path ' , " target__target " , " target__target_image_file__file_name " ]
2013-10-11 12:46:23 +00:00
2015-05-14 15:44:27 +00:00
project = models . ForeignKey ( Project ) # must have a project
2013-10-11 12:46:23 +00:00
machine = models . CharField ( max_length = 100 )
distro = models . CharField ( max_length = 100 )
distro_version = models . CharField ( max_length = 100 )
started_on = models . DateTimeField ( )
completed_on = models . DateTimeField ( )
outcome = models . IntegerField ( choices = BUILD_OUTCOME , default = IN_PROGRESS )
cooker_log_path = models . CharField ( max_length = 500 )
build_name = models . CharField ( max_length = 100 )
bitbake_version = models . CharField ( max_length = 50 )
2016-01-18 14:23:55 +00:00
@staticmethod
def get_recent ( project = None ) :
"""
Return recent builds as a list ; if project is set , only return
builds for that project
"""
builds = Build . objects . all ( )
if project :
builds = builds . filter ( project = project )
finished_criteria = Q ( outcome = Build . SUCCEEDED ) | Q ( outcome = Build . FAILED )
recent_builds = list ( itertools . chain (
builds . filter ( outcome = Build . IN_PROGRESS ) . order_by ( " -started_on " ) ,
builds . filter ( finished_criteria ) . order_by ( " -completed_on " ) [ : 3 ]
) )
# add percentage done property to each build; this is used
# to show build progress in mrb_section.html
for build in recent_builds :
build . percentDone = build . completeper ( )
return recent_builds
2014-08-27 16:24:42 +00:00
def completeper ( self ) :
tf = Task . objects . filter ( build = self )
tfc = tf . count ( )
if tfc > 0 :
2016-01-18 14:23:55 +00:00
completeper = tf . exclude ( order__isnull = True ) . count ( ) * 100 / tfc
2014-08-27 16:24:42 +00:00
else :
completeper = 0
return completeper
def eta ( self ) :
2014-09-09 10:47:13 +00:00
eta = timezone . now ( )
2014-08-27 16:24:42 +00:00
completeper = self . completeper ( )
if self . completeper ( ) > 0 :
2015-02-24 17:21:00 +00:00
eta + = ( ( eta - self . started_on ) * ( 100 - completeper ) ) / completeper
2014-08-27 16:24:42 +00:00
return eta
2016-01-15 11:00:46 +00:00
def get_image_file_extensions ( self ) :
"""
Get list of file name extensions for images produced by this build
"""
targets = Target . objects . filter ( build_id = self . id )
extensions = [ ]
# pattern to match against file path for building extension string
pattern = re . compile ( ' \ .([^ \ .]+?)$ ' )
for target in targets :
if ( not target . is_image ) :
continue
target_image_files = Target_Image_File . objects . filter ( target_id = target . id )
for target_image_file in target_image_files :
file_name = os . path . basename ( target_image_file . file_name )
suffix = ' '
continue_matching = True
# incrementally extract the suffix from the file path,
# checking it against the list of valid suffixes at each
# step; if the path is stripped of all potential suffix
# parts without matching a valid suffix, this returns all
# characters after the first '.' in the file name
while continue_matching :
matches = pattern . search ( file_name )
if None == matches :
continue_matching = False
suffix = re . sub ( ' ^ \ . ' , ' ' , suffix )
continue
else :
suffix = matches . group ( 1 ) + suffix
if suffix in Target_Image_File . SUFFIXES :
continue_matching = False
continue
else :
# reduce the file name and try to find the next
# segment from the path which might be part
# of the suffix
file_name = re . sub ( ' . ' + matches . group ( 1 ) , ' ' , file_name )
suffix = ' . ' + suffix
if not suffix in extensions :
extensions . append ( suffix )
return ' , ' . join ( extensions )
2014-08-27 16:24:42 +00:00
2014-03-21 12:35:50 +00:00
def get_sorted_target_list ( self ) :
tgts = Target . objects . filter ( build_id = self . id ) . order_by ( ' target ' ) ;
return ( tgts ) ;
2015-10-14 14:43:44 +00:00
def get_outcome_text ( self ) :
return Build . BUILD_OUTCOME [ int ( self . outcome ) ] [ 1 ]
2016-01-15 11:00:49 +00:00
@property
def failed_tasks ( self ) :
""" Get failed tasks for the build """
tasks = self . task_build . all ( )
return tasks . filter ( order__gt = 0 , outcome = Task . OUTCOME_FAILED )
2015-06-17 16:30:34 +00:00
@property
def errors ( self ) :
2015-10-15 12:45:13 +00:00
return ( self . logmessage_set . filter ( level = LogMessage . ERROR ) |
self . logmessage_set . filter ( level = LogMessage . EXCEPTION ) |
self . logmessage_set . filter ( level = LogMessage . CRITICAL ) )
2015-06-17 16:30:34 +00:00
@property
def warnings ( self ) :
return self . logmessage_set . filter ( level = LogMessage . WARNING )
2016-01-15 11:00:49 +00:00
@property
def timespent ( self ) :
return self . completed_on - self . started_on
2015-06-17 16:30:34 +00:00
@property
def timespent_seconds ( self ) :
2016-01-15 11:00:49 +00:00
return self . timespent . total_seconds ( )
@property
def target_labels ( self ) :
"""
Sorted ( a - z ) " target1:task, target2, target3 " etc . string for all
targets in this build
"""
targets = self . target_set . all ( )
2016-01-15 11:01:06 +00:00
target_labels = [ target . target +
( ' : ' + target . task if target . task else ' ' )
for target in targets ]
2016-01-15 11:00:49 +00:00
target_labels . sort ( )
return target_labels
2015-06-11 17:27:53 +00:00
def get_current_status ( self ) :
2015-10-14 14:43:44 +00:00
"""
get the status string from the build request if the build
has one , or the text for the build outcome if it doesn ' t
"""
2015-06-17 11:27:48 +00:00
from bldcontrol . models import BuildRequest
2015-10-14 14:43:44 +00:00
build_request = None
if hasattr ( self , ' buildrequest ' ) :
build_request = self . buildrequest
if ( build_request
and build_request . state != BuildRequest . REQ_INPROGRESS
and self . outcome == Build . IN_PROGRESS ) :
2015-06-17 11:27:48 +00:00
return self . buildrequest . get_state_display ( )
2015-10-14 14:43:44 +00:00
else :
return self . get_outcome_text ( )
2015-06-11 17:27:53 +00:00
2015-05-19 12:13:27 +00:00
def __str__ ( self ) :
2016-01-15 11:00:46 +00:00
return " %d %s %s " % ( self . id , self . project , " , " . join ( [ t . target for t in self . target_set . all ( ) ] ) )
2015-05-19 12:13:27 +00:00
2014-11-25 10:12:46 +00:00
2014-12-05 15:14:20 +00:00
# an Artifact is anything that results from a Build, and may be of interest to the user, and is not stored elsewhere
class BuildArtifact ( models . Model ) :
build = models . ForeignKey ( Build )
file_name = models . FilePathField ( )
file_size = models . IntegerField ( )
def get_local_file_name ( self ) :
try :
deploydir = Variable . objects . get ( build = self . build , variable_name = " DEPLOY_DIR " ) . variable_value
return self . file_name [ len ( deploydir ) + 1 : ]
except :
raise
return self . file_name
2016-02-23 12:17:04 +00:00
def get_basename ( self ) :
return os . path . basename ( self . file_name )
2014-12-05 15:14:20 +00:00
def is_available ( self ) :
2015-08-18 16:28:50 +00:00
return self . build . buildrequest . environment . has_artifact ( self . file_name )
2014-12-05 15:14:20 +00:00
2014-06-03 15:26:17 +00:00
class ProjectTarget ( models . Model ) :
project = models . ForeignKey ( Project )
target = models . CharField ( max_length = 100 )
2014-06-30 17:33:04 +00:00
task = models . CharField ( max_length = 100 , null = True )
2014-03-21 12:35:50 +00:00
2013-10-11 12:46:23 +00:00
class Target ( models . Model ) :
2014-02-20 12:47:55 +00:00
search_allowed_fields = [ ' target ' , ' file_name ' ]
2013-10-11 12:46:23 +00:00
build = models . ForeignKey ( Build )
target = models . CharField ( max_length = 100 )
2015-06-11 17:27:53 +00:00
task = models . CharField ( max_length = 100 , null = True )
2013-10-11 12:46:23 +00:00
is_image = models . BooleanField ( default = False )
2014-03-06 18:22:38 +00:00
image_size = models . IntegerField ( default = 0 )
license_manifest_path = models . CharField ( max_length = 500 , null = True )
2013-10-11 12:46:23 +00:00
2014-03-12 21:54:09 +00:00
def package_count ( self ) :
return Target_Installed_Package . objects . filter ( target_id__exact = self . id ) . count ( )
2014-11-14 17:07:06 +00:00
def __unicode__ ( self ) :
2013-10-11 12:46:23 +00:00
return self . target
2014-02-20 12:47:55 +00:00
class Target_Image_File ( models . Model ) :
2016-01-15 11:00:45 +00:00
# valid suffixes for image files produced by a build
SUFFIXES = {
' btrfs ' , ' cpio ' , ' cpio.gz ' , ' cpio.lz4 ' , ' cpio.lzma ' , ' cpio.xz ' ,
' cramfs ' , ' elf ' , ' ext2 ' , ' ext2.bz2 ' , ' ext2.gz ' , ' ext2.lzma ' , ' ext4 ' ,
' ext4.gz ' , ' ext3 ' , ' ext3.gz ' , ' hddimg ' , ' iso ' , ' jffs2 ' , ' jffs2.sum ' ,
' squashfs ' , ' squashfs-lzo ' , ' squashfs-xz ' , ' tar.bz2 ' , ' tar.lz4 ' ,
' tar.xz ' , ' tartar.gz ' , ' ubi ' , ' ubifs ' , ' vmdk '
}
2014-02-20 12:47:55 +00:00
target = models . ForeignKey ( Target )
2014-04-28 14:11:03 +00:00
file_name = models . FilePathField ( max_length = 254 )
2014-02-20 12:47:55 +00:00
file_size = models . IntegerField ( )
2016-02-23 12:17:04 +00:00
@property
def suffix ( self ) :
filename , suffix = os . path . splitext ( self . file_name )
suffix = suffix . lstrip ( ' . ' )
return suffix
2014-02-20 12:47:55 +00:00
class Target_File ( models . Model ) :
ITYPE_REGULAR = 1
ITYPE_DIRECTORY = 2
ITYPE_SYMLINK = 3
ITYPE_SOCKET = 4
ITYPE_FIFO = 5
ITYPE_CHARACTER = 6
ITYPE_BLOCK = 7
ITYPES = ( ( ITYPE_REGULAR , ' regular ' ) ,
( ITYPE_DIRECTORY , ' directory ' ) ,
( ITYPE_SYMLINK , ' symlink ' ) ,
( ITYPE_SOCKET , ' socket ' ) ,
( ITYPE_FIFO , ' fifo ' ) ,
( ITYPE_CHARACTER , ' character ' ) ,
( ITYPE_BLOCK , ' block ' ) ,
2014-03-06 18:22:38 +00:00
)
2014-02-20 12:47:55 +00:00
target = models . ForeignKey ( Target )
path = models . FilePathField ( )
size = models . IntegerField ( )
inodetype = models . IntegerField ( choices = ITYPES )
2014-03-06 18:22:38 +00:00
permission = models . CharField ( max_length = 16 )
2014-02-20 12:47:55 +00:00
owner = models . CharField ( max_length = 128 )
group = models . CharField ( max_length = 128 )
2014-03-06 18:22:38 +00:00
directory = models . ForeignKey ( ' Target_File ' , related_name = " directory_set " , null = True )
sym_target = models . ForeignKey ( ' Target_File ' , related_name = " symlink_set " , null = True )
2013-10-11 12:46:23 +00:00
2014-01-14 12:35:12 +00:00
2013-10-11 12:46:23 +00:00
class Task ( models . Model ) :
SSTATE_NA = 0
SSTATE_MISS = 1
SSTATE_FAILED = 2
SSTATE_RESTORED = 3
SSTATE_RESULT = (
( SSTATE_NA , ' Not Applicable ' ) , # For rest of tasks, but they still need checking.
2014-02-18 12:08:40 +00:00
( SSTATE_MISS , ' File not in cache ' ) , # the sstate object was not found
2013-10-11 12:46:23 +00:00
( SSTATE_FAILED , ' Failed ' ) , # there was a pkg, but the script failed
2014-02-14 22:58:20 +00:00
( SSTATE_RESTORED , ' Succeeded ' ) , # successfully restored
2013-10-11 12:46:23 +00:00
)
2013-11-14 10:52:58 +00:00
CODING_NA = 0
CODING_PYTHON = 2
CODING_SHELL = 3
2013-10-11 12:46:23 +00:00
TASK_CODING = (
2013-11-14 10:52:58 +00:00
( CODING_NA , ' N/A ' ) ,
2013-10-11 12:46:23 +00:00
( CODING_PYTHON , ' Python ' ) ,
( CODING_SHELL , ' Shell ' ) ,
)
2014-02-13 13:12:39 +00:00
OUTCOME_NA = - 1
2013-10-11 12:46:23 +00:00
OUTCOME_SUCCESS = 0
OUTCOME_COVERED = 1
2014-01-14 12:35:12 +00:00
OUTCOME_CACHED = 2
OUTCOME_PREBUILT = 3
2013-10-11 12:46:23 +00:00
OUTCOME_FAILED = 4
2014-02-13 13:12:39 +00:00
OUTCOME_EMPTY = 5
2013-10-11 12:46:23 +00:00
TASK_OUTCOME = (
2014-02-13 13:12:39 +00:00
( OUTCOME_NA , ' Not Available ' ) ,
2013-10-11 12:46:23 +00:00
( OUTCOME_SUCCESS , ' Succeeded ' ) ,
( OUTCOME_COVERED , ' Covered ' ) ,
2014-01-14 12:35:12 +00:00
( OUTCOME_CACHED , ' Cached ' ) ,
( OUTCOME_PREBUILT , ' Prebuilt ' ) ,
2013-10-11 12:46:23 +00:00
( OUTCOME_FAILED , ' Failed ' ) ,
2014-02-13 13:12:39 +00:00
( OUTCOME_EMPTY , ' Empty ' ) ,
2013-10-11 12:46:23 +00:00
)
2014-02-21 08:58:01 +00:00
TASK_OUTCOME_HELP = (
2014-03-16 13:09:34 +00:00
( OUTCOME_SUCCESS , ' This task successfully completed ' ) ,
2014-02-21 08:58:01 +00:00
( OUTCOME_COVERED , ' This task did not run because its output is provided by another task ' ) ,
( OUTCOME_CACHED , ' This task restored output from the sstate-cache directory or mirrors ' ) ,
( OUTCOME_PREBUILT , ' This task did not run because its outcome was reused from a previous build ' ) ,
( OUTCOME_FAILED , ' This task did not complete ' ) ,
2014-03-07 06:15:58 +00:00
( OUTCOME_EMPTY , ' This task has no executable content ' ) ,
2014-02-21 08:58:01 +00:00
( OUTCOME_NA , ' ' ) ,
)
2014-02-18 04:30:41 +00:00
2014-02-14 22:58:20 +00:00
search_allowed_fields = [ " recipe__name " , " recipe__version " , " task_name " , " logfile " ]
2014-01-07 13:10:42 +00:00
2015-08-18 16:28:51 +00:00
def __init__ ( self , * args , * * kwargs ) :
super ( Task , self ) . __init__ ( * args , * * kwargs )
try :
self . _helptext = HelpText . objects . get ( key = self . task_name , area = HelpText . VARIABLE , build = self . build ) . text
except HelpText . DoesNotExist :
self . _helptext = None
2014-01-14 12:35:12 +00:00
def get_related_setscene ( self ) :
2015-02-26 21:41:56 +00:00
return Task . objects . filter ( task_executed = True , build = self . build , recipe = self . recipe , task_name = self . task_name + " _setscene " )
2014-01-14 12:35:12 +00:00
2014-04-11 04:43:11 +00:00
def get_outcome_text ( self ) :
2015-08-18 16:28:51 +00:00
return Task . TASK_OUTCOME [ int ( self . outcome ) + 1 ] [ 1 ]
2014-04-11 04:43:11 +00:00
2014-02-21 08:58:01 +00:00
def get_outcome_help ( self ) :
2015-08-18 16:28:51 +00:00
return Task . TASK_OUTCOME_HELP [ int ( self . outcome ) ] [ 1 ]
2014-02-18 04:30:41 +00:00
2014-04-11 04:43:11 +00:00
def get_sstate_text ( self ) :
if self . sstate_result == Task . SSTATE_NA :
return ' '
else :
2015-08-18 16:28:51 +00:00
return Task . SSTATE_RESULT [ int ( self . sstate_result ) ] [ 1 ]
2014-04-11 04:43:11 +00:00
2014-02-13 13:36:54 +00:00
def get_executed_display ( self ) :
if self . task_executed :
return " Executed "
return " Not Executed "
2014-03-18 19:17:31 +00:00
def get_description ( self ) :
2015-02-17 17:24:29 +00:00
return self . _helptext
2014-03-18 19:17:31 +00:00
2013-10-11 12:46:23 +00:00
build = models . ForeignKey ( Build , related_name = ' task_build ' )
order = models . IntegerField ( null = True )
2014-01-14 12:35:12 +00:00
task_executed = models . BooleanField ( default = False ) # True means Executed, False means Not/Executed
2013-10-11 12:46:23 +00:00
outcome = models . IntegerField ( choices = TASK_OUTCOME , default = OUTCOME_NA )
sstate_checksum = models . CharField ( max_length = 100 , blank = True )
path_to_sstate_obj = models . FilePathField ( max_length = 500 , blank = True )
2015-02-26 21:41:56 +00:00
recipe = models . ForeignKey ( ' Recipe ' , related_name = ' tasks ' )
2013-10-11 12:46:23 +00:00
task_name = models . CharField ( max_length = 100 )
source_url = models . FilePathField ( max_length = 255 , blank = True )
work_directory = models . FilePathField ( max_length = 255 , blank = True )
2013-11-14 10:52:58 +00:00
script_type = models . IntegerField ( choices = TASK_CODING , default = CODING_NA )
2013-10-11 12:46:23 +00:00
line_number = models . IntegerField ( default = 0 )
2016-03-08 11:32:12 +00:00
# start/end times
started = models . DateTimeField ( null = True )
ended = models . DateTimeField ( null = True )
# in seconds; this is stored to enable sorting
2015-05-14 15:44:27 +00:00
elapsed_time = models . DecimalField ( max_digits = 8 , decimal_places = 2 , null = True )
2016-03-08 11:32:12 +00:00
# in bytes; note that disk_io is stored to enable sorting
disk_io = models . IntegerField ( null = True )
disk_io_read = models . IntegerField ( null = True )
disk_io_write = models . IntegerField ( null = True )
# in seconds
cpu_time_user = models . DecimalField ( max_digits = 8 , decimal_places = 2 , null = True )
cpu_time_system = models . DecimalField ( max_digits = 8 , decimal_places = 2 , null = True )
2013-10-11 12:46:23 +00:00
sstate_result = models . IntegerField ( choices = SSTATE_RESULT , default = SSTATE_NA )
message = models . CharField ( max_length = 240 )
logfile = models . FilePathField ( max_length = 255 , blank = True )
2014-04-11 04:43:11 +00:00
outcome_text = property ( get_outcome_text )
sstate_text = property ( get_sstate_text )
2015-02-24 17:20:58 +00:00
def __unicode__ ( self ) :
2016-01-15 11:00:46 +00:00
return " %d ( %d ) %s : %s " % ( self . pk , self . build . pk , self . recipe . name , self . task_name )
2015-02-26 21:41:56 +00:00
2013-10-11 12:46:23 +00:00
class Meta :
ordering = ( ' order ' , ' recipe ' , )
2014-02-13 13:12:39 +00:00
unique_together = ( ' build ' , ' recipe ' , ' task_name ' , )
2013-10-11 12:46:23 +00:00
class Task_Dependency ( models . Model ) :
task = models . ForeignKey ( Task , related_name = ' task_dependencies_task ' )
depends_on = models . ForeignKey ( Task , related_name = ' task_dependencies_depends ' )
2013-11-26 18:12:43 +00:00
class Package ( models . Model ) :
2015-05-14 15:10:50 +00:00
search_allowed_fields = [ ' name ' , ' version ' , ' revision ' , ' recipe__name ' , ' recipe__version ' , ' recipe__license ' , ' recipe__layer_version__layer__name ' , ' recipe__layer_version__branch ' , ' recipe__layer_version__commit ' , ' recipe__layer_version__local_path ' , ' installed_name ' ]
2015-09-29 04:45:18 +00:00
build = models . ForeignKey ( ' Build ' , null = True )
2013-10-11 12:46:23 +00:00
recipe = models . ForeignKey ( ' Recipe ' , null = True )
name = models . CharField ( max_length = 100 )
2014-01-17 17:58:05 +00:00
installed_name = models . CharField ( max_length = 100 , default = ' ' )
2013-10-11 12:46:23 +00:00
version = models . CharField ( max_length = 100 , blank = True )
revision = models . CharField ( max_length = 32 , blank = True )
2014-10-09 11:37:30 +00:00
summary = models . TextField ( blank = True )
2014-04-28 14:11:03 +00:00
description = models . TextField ( blank = True )
2013-10-11 12:46:23 +00:00
size = models . IntegerField ( default = 0 )
2013-11-26 18:12:43 +00:00
installed_size = models . IntegerField ( default = 0 )
2013-10-11 12:46:23 +00:00
section = models . CharField ( max_length = 80 , blank = True )
license = models . CharField ( max_length = 80 , blank = True )
2016-01-21 17:11:11 +00:00
@property
def is_locale_package ( self ) :
""" Returns True if this package is identifiable as a locale package """
if self . name . find ( ' locale ' ) != - 1 :
return True
return False
@property
def is_packagegroup ( self ) :
""" Returns True is this package is identifiable as a packagegroup """
if self . name . find ( ' packagegroup ' ) != - 1 :
return True
return False
2015-12-07 18:22:08 +00:00
class CustomImagePackage ( Package ) :
# CustomImageRecipe fields to track pacakges appended,
# included and excluded from a CustomImageRecipe
recipe_includes = models . ManyToManyField ( ' CustomImageRecipe ' ,
related_name = ' includes_set ' )
recipe_excludes = models . ManyToManyField ( ' CustomImageRecipe ' ,
related_name = ' excludes_set ' )
recipe_appends = models . ManyToManyField ( ' CustomImageRecipe ' ,
related_name = ' appends_set ' )
2014-03-19 12:24:47 +00:00
class Package_DependencyManager ( models . Manager ) :
use_for_related_fields = True
2015-12-10 03:56:31 +00:00
def get_queryset ( self ) :
return super ( Package_DependencyManager , self ) . get_queryset ( ) . exclude ( package_id = F ( ' depends_on__id ' ) )
2014-03-19 12:24:47 +00:00
2015-11-04 14:48:54 +00:00
def get_total_source_deps_size ( self ) :
""" Returns the total file size of all the packages that depend on
thispackage .
"""
return self . all ( ) . aggregate ( Sum ( ' depends_on__size ' ) )
2016-02-05 11:18:09 +00:00
def all_depends ( self ) :
""" Returns just the depends packages and not any other dep_type """
return self . filter ( Q ( dep_type = Package_Dependency . TYPE_RDEPENDS ) |
Q ( dep_type = Package_Dependency . TYPE_TRDEPENDS ) )
2013-11-26 18:12:43 +00:00
class Package_Dependency ( models . Model ) :
2013-10-11 12:46:23 +00:00
TYPE_RDEPENDS = 0
2014-01-23 17:47:41 +00:00
TYPE_TRDEPENDS = 1
2013-10-11 12:46:23 +00:00
TYPE_RRECOMMENDS = 2
2014-01-23 17:47:41 +00:00
TYPE_TRECOMMENDS = 3
TYPE_RSUGGESTS = 4
TYPE_RPROVIDES = 5
TYPE_RREPLACES = 6
TYPE_RCONFLICTS = 7
' TODO: bpackage should be changed to remove the DEPENDS_TYPE access '
2013-10-11 12:46:23 +00:00
DEPENDS_TYPE = (
2014-01-23 17:47:41 +00:00
( TYPE_RDEPENDS , " depends " ) ,
( TYPE_TRDEPENDS , " depends " ) ,
( TYPE_TRECOMMENDS , " recommends " ) ,
( TYPE_RRECOMMENDS , " recommends " ) ,
( TYPE_RSUGGESTS , " suggests " ) ,
( TYPE_RPROVIDES , " provides " ) ,
( TYPE_RREPLACES , " replaces " ) ,
( TYPE_RCONFLICTS , " conflicts " ) ,
2013-10-11 12:46:23 +00:00
)
2014-11-14 17:07:06 +00:00
""" Indexed by dep_type, in view order, key for short name and help
2014-01-23 17:47:41 +00:00
description which when viewed will be printf ' d with the
package name .
2014-11-14 17:07:06 +00:00
"""
2014-01-23 17:47:41 +00:00
DEPENDS_DICT = {
TYPE_RDEPENDS : ( " depends " , " %s is required to run %s " ) ,
TYPE_TRDEPENDS : ( " depends " , " %s is required to run %s " ) ,
TYPE_TRECOMMENDS : ( " recommends " , " %s extends the usability of %s " ) ,
TYPE_RRECOMMENDS : ( " recommends " , " %s extends the usability of %s " ) ,
TYPE_RSUGGESTS : ( " suggests " , " %s is suggested for installation with %s " ) ,
TYPE_RPROVIDES : ( " provides " , " %s is provided by %s " ) ,
TYPE_RREPLACES : ( " replaces " , " %s is replaced by %s " ) ,
TYPE_RCONFLICTS : ( " conflicts " , " %s conflicts with %s , which will not be installed if this package is not first removed " ) ,
}
2013-11-26 18:12:43 +00:00
package = models . ForeignKey ( Package , related_name = ' package_dependencies_source ' )
depends_on = models . ForeignKey ( Package , related_name = ' package_dependencies_target ' ) # soft dependency
2013-10-11 12:46:23 +00:00
dep_type = models . IntegerField ( choices = DEPENDS_TYPE )
2013-11-26 18:12:43 +00:00
target = models . ForeignKey ( Target , null = True )
2014-03-19 12:24:47 +00:00
objects = Package_DependencyManager ( )
2013-10-11 12:46:23 +00:00
2013-11-26 18:12:43 +00:00
class Target_Installed_Package ( models . Model ) :
target = models . ForeignKey ( Target )
2014-01-23 17:47:41 +00:00
package = models . ForeignKey ( Package , related_name = ' buildtargetlist_package ' )
2013-10-11 12:46:23 +00:00
2013-11-26 18:12:43 +00:00
class Package_File ( models . Model ) :
package = models . ForeignKey ( Package , related_name = ' buildfilelist_package ' )
2013-10-11 12:46:23 +00:00
path = models . FilePathField ( max_length = 255 , blank = True )
size = models . IntegerField ( )
class Recipe ( models . Model ) :
2015-05-14 15:10:50 +00:00
search_allowed_fields = [ ' name ' , ' version ' , ' file_path ' , ' section ' , ' summary ' , ' description ' , ' license ' , ' layer_version__layer__name ' , ' layer_version__branch ' , ' layer_version__commit ' , ' layer_version__local_path ' , ' layer_version__layer_source__name ' ]
2014-08-08 14:03:03 +00:00
layer_source = models . ForeignKey ( ' LayerSource ' , default = None , null = True ) # from where did we get this recipe
up_id = models . IntegerField ( null = True , default = None ) # id of entry in the source
up_date = models . DateTimeField ( null = True , default = None )
name = models . CharField ( max_length = 100 , blank = True ) # pn
version = models . CharField ( max_length = 100 , blank = True ) # pv
2013-10-11 12:46:23 +00:00
layer_version = models . ForeignKey ( ' Layer_Version ' , related_name = ' recipe_layer_version ' )
2014-10-09 11:37:30 +00:00
summary = models . TextField ( blank = True )
2014-04-28 14:11:03 +00:00
description = models . TextField ( blank = True )
2013-10-11 12:46:23 +00:00
section = models . CharField ( max_length = 100 , blank = True )
license = models . CharField ( max_length = 200 , blank = True )
homepage = models . URLField ( blank = True )
bugtracker = models . URLField ( blank = True )
file_path = models . FilePathField ( max_length = 255 )
2015-05-14 15:10:50 +00:00
pathflags = models . CharField ( max_length = 200 , blank = True )
2015-07-28 14:24:45 +00:00
is_image = models . BooleanField ( default = False )
2013-10-11 12:46:23 +00:00
2014-08-29 15:41:59 +00:00
def get_layersource_view_url ( self ) :
if self . layer_source is None :
return " "
url = self . layer_source . get_object_view ( self . layer_version . up_branch , " recipes " , self . name )
return url
2014-08-08 14:03:03 +00:00
def __unicode__ ( self ) :
return " Recipe " + self . name + " : " + self . version
2015-05-11 17:51:28 +00:00
def get_vcs_recipe_file_link_url ( self ) :
return self . layer_version . get_vcs_file_link_url ( self . file_path )
def get_description_or_summary ( self ) :
if self . description :
return self . description
elif self . summary :
return self . summary
else :
return " "
2014-11-05 14:47:51 +00:00
class Meta :
2015-05-14 15:10:50 +00:00
unique_together = ( ( " layer_version " , " file_path " , " pathflags " ) , )
2014-11-05 14:47:51 +00:00
2014-03-19 12:24:47 +00:00
class Recipe_DependencyManager ( models . Manager ) :
use_for_related_fields = True
2015-12-10 03:56:31 +00:00
def get_queryset ( self ) :
return super ( Recipe_DependencyManager , self ) . get_queryset ( ) . exclude ( recipe_id = F ( ' depends_on__id ' ) )
2013-10-11 12:46:23 +00:00
2016-01-08 11:17:18 +00:00
class Provides ( models . Model ) :
name = models . CharField ( max_length = 100 )
recipe = models . ForeignKey ( Recipe )
2013-10-11 12:46:23 +00:00
class Recipe_Dependency ( models . Model ) :
TYPE_DEPENDS = 0
TYPE_RDEPENDS = 1
DEPENDS_TYPE = (
( TYPE_DEPENDS , " depends " ) ,
( TYPE_RDEPENDS , " rdepends " ) ,
)
recipe = models . ForeignKey ( Recipe , related_name = ' r_dependencies_recipe ' )
depends_on = models . ForeignKey ( Recipe , related_name = ' r_dependencies_depends ' )
2016-01-08 11:17:18 +00:00
via = models . ForeignKey ( Provides , null = True , default = None )
2013-10-11 12:46:23 +00:00
dep_type = models . IntegerField ( choices = DEPENDS_TYPE )
2014-03-19 12:24:47 +00:00
objects = Recipe_DependencyManager ( )
2013-10-11 12:46:23 +00:00
2014-06-03 15:26:17 +00:00
2014-08-08 14:03:03 +00:00
class Machine ( models . Model ) :
2015-02-05 14:24:01 +00:00
search_allowed_fields = [ " name " , " description " , " layer_version__layer__name " ]
2014-08-08 14:03:03 +00:00
layer_source = models . ForeignKey ( ' LayerSource ' , default = None , null = True ) # from where did we get this machine
up_id = models . IntegerField ( null = True , default = None ) # id of entry in the source
up_date = models . DateTimeField ( null = True , default = None )
layer_version = models . ForeignKey ( ' Layer_Version ' )
name = models . CharField ( max_length = 255 )
description = models . CharField ( max_length = 255 )
2015-02-03 15:40:41 +00:00
def get_vcs_machine_file_link_url ( self ) :
path = ' conf/machine/ ' + self . name + ' .conf '
return self . layer_version . get_vcs_file_link_url ( path )
2014-08-08 14:03:03 +00:00
def __unicode__ ( self ) :
return " Machine " + self . name + " ( " + self . description + " ) "
class Meta :
unique_together = ( " layer_source " , " up_id " )
from django . db . models . base import ModelBase
class InheritanceMetaclass ( ModelBase ) :
def __call__ ( cls , * args , * * kwargs ) :
obj = super ( InheritanceMetaclass , cls ) . __call__ ( * args , * * kwargs )
return obj . get_object ( )
class LayerSource ( models . Model ) :
__metaclass__ = InheritanceMetaclass
class Meta :
unique_together = ( ( ' sourcetype ' , ' apiurl ' ) , )
TYPE_LOCAL = 0
TYPE_LAYERINDEX = 1
2014-11-14 17:07:06 +00:00
TYPE_IMPORTED = 2
2014-08-08 14:03:03 +00:00
SOURCE_TYPE = (
( TYPE_LOCAL , " local " ) ,
( TYPE_LAYERINDEX , " layerindex " ) ,
2014-11-14 17:07:06 +00:00
( TYPE_IMPORTED , " imported " ) ,
2014-08-08 14:03:03 +00:00
)
2014-11-14 17:07:06 +00:00
name = models . CharField ( max_length = 63 , unique = True )
2014-08-08 14:03:03 +00:00
sourcetype = models . IntegerField ( choices = SOURCE_TYPE )
apiurl = models . CharField ( max_length = 255 , null = True , default = None )
2015-08-18 16:28:51 +00:00
def __init__ ( self , * args , * * kwargs ) :
super ( LayerSource , self ) . __init__ ( * args , * * kwargs )
2015-03-31 08:58:11 +00:00
if self . sourcetype == LayerSource . TYPE_LOCAL :
self . __class__ = LocalLayerSource
elif self . sourcetype == LayerSource . TYPE_LAYERINDEX :
self . __class__ = LayerIndexLayerSource
elif self . sourcetype == LayerSource . TYPE_IMPORTED :
self . __class__ = ImportedLayerSource
2014-08-08 14:03:03 +00:00
elif self . sourcetype == None :
2014-11-14 17:07:06 +00:00
raise Exception ( " Unknown LayerSource-derived class. If you added a new layer source type, fill out all code stubs. " )
2015-08-18 16:28:51 +00:00
def update ( self ) :
"""
Updates the local database information from the upstream layer source
"""
raise Exception ( " Abstract, update() must be implemented by all LayerSource-derived classes (object is %s ) " % str ( vars ( self ) ) )
def save ( self , * args , * * kwargs ) :
2014-08-08 14:03:03 +00:00
return super ( LayerSource , self ) . save ( * args , * * kwargs )
def get_object ( self ) :
2015-03-31 08:58:11 +00:00
# preset an un-initilized object
if None == self . name :
self . name = " "
if None == self . apiurl :
self . apiurl = " "
if None == self . sourcetype :
self . sourcetype = LayerSource . TYPE_LOCAL
2014-11-14 17:07:06 +00:00
if self . sourcetype == LayerSource . TYPE_LOCAL :
self . __class__ = LocalLayerSource
elif self . sourcetype == LayerSource . TYPE_LAYERINDEX :
self . __class__ = LayerIndexLayerSource
elif self . sourcetype == LayerSource . TYPE_IMPORTED :
self . __class__ = ImportedLayerSource
else :
raise Exception ( " Unknown LayerSource type. If you added a new layer source type, fill out all code stubs. " )
2014-08-08 14:03:03 +00:00
return self
2014-11-14 17:07:06 +00:00
def __unicode__ ( self ) :
return " %s ( %s ) " % ( self . name , self . sourcetype )
2014-08-08 14:03:03 +00:00
class LocalLayerSource ( LayerSource ) :
class Meta ( LayerSource . _meta . __class__ ) :
proxy = True
def __init__ ( self , * args , * * kwargs ) :
super ( LocalLayerSource , self ) . __init__ ( args , kwargs )
self . sourcetype = LayerSource . TYPE_LOCAL
def update ( self ) :
2014-11-14 17:07:06 +00:00
"""
Fetches layer , recipe and machine information from local repository
"""
pass
class ImportedLayerSource ( LayerSource ) :
class Meta ( LayerSource . _meta . __class__ ) :
proxy = True
def __init__ ( self , * args , * * kwargs ) :
super ( ImportedLayerSource , self ) . __init__ ( args , kwargs )
self . sourcetype = LayerSource . TYPE_IMPORTED
def update ( self ) :
"""
2014-08-08 14:03:03 +00:00
Fetches layer , recipe and machine information from local repository
2014-11-14 17:07:06 +00:00
"""
2014-08-08 14:03:03 +00:00
pass
2014-11-14 17:07:06 +00:00
2014-08-08 14:03:03 +00:00
class LayerIndexLayerSource ( LayerSource ) :
class Meta ( LayerSource . _meta . __class__ ) :
proxy = True
def __init__ ( self , * args , * * kwargs ) :
super ( LayerIndexLayerSource , self ) . __init__ ( args , kwargs )
self . sourcetype = LayerSource . TYPE_LAYERINDEX
2014-08-29 15:41:59 +00:00
def get_object_view ( self , branch , objectype , upid ) :
return self . apiurl + " ../branch/ " + branch . name + " / " + objectype + " /?q= " + str ( upid )
2014-08-08 14:03:03 +00:00
def update ( self ) :
2014-11-14 17:07:06 +00:00
"""
2014-08-08 14:03:03 +00:00
Fetches layer , recipe and machine information from remote repository
2014-11-14 17:07:06 +00:00
"""
2014-08-08 14:03:03 +00:00
assert self . apiurl is not None
2015-03-09 11:23:47 +00:00
from django . db import transaction , connection
2014-08-08 14:03:03 +00:00
2015-08-28 15:37:38 +00:00
import urllib2 , urlparse , json
2014-11-21 13:58:51 +00:00
import os
proxy_settings = os . environ . get ( " http_proxy " , None )
2015-09-29 04:45:26 +00:00
oe_core_layer = ' openembedded-core '
2014-11-21 13:58:51 +00:00
2014-08-08 14:03:03 +00:00
def _get_json_response ( apiurl = self . apiurl ) :
2014-11-21 13:58:51 +00:00
_parsedurl = urlparse . urlparse ( apiurl )
path = _parsedurl . path
2015-08-28 15:37:38 +00:00
try :
res = urllib2 . urlopen ( apiurl )
except urllib2 . URLError as e :
raise Exception ( " Failed to read %s : %s " % ( path , e . reason ) )
return json . loads ( res . read ( ) )
2014-08-08 14:03:03 +00:00
# verify we can get the basic api
try :
apilinks = _get_json_response ( )
2014-09-09 10:47:13 +00:00
except Exception as e :
import traceback
2014-11-21 13:58:51 +00:00
if proxy_settings is not None :
2015-08-18 16:28:52 +00:00
logger . info ( " EE: Using proxy %s " % proxy_settings )
logger . warning ( " EE: could not connect to %s , skipping update: %s \n %s " % ( self . apiurl , e , traceback . format_exc ( e ) ) )
2014-08-08 14:03:03 +00:00
return
2015-05-20 14:44:16 +00:00
# update branches; only those that we already have names listed in the
# Releases table
2014-11-14 17:07:06 +00:00
whitelist_branch_names = map ( lambda x : x . branch_name , Release . objects . all ( ) )
2015-05-20 14:44:16 +00:00
if len ( whitelist_branch_names ) == 0 :
raise Exception ( " Failed to make list of branches to fetch " )
2014-08-08 14:03:03 +00:00
2015-08-18 16:28:52 +00:00
logger . debug ( " Fetching branches " )
2014-08-08 14:03:03 +00:00
branches_info = _get_json_response ( apilinks [ ' branches ' ]
+ " ?filter=name: %s " % " OR " . join ( whitelist_branch_names ) )
for bi in branches_info :
2014-08-29 15:41:59 +00:00
b , created = Branch . objects . get_or_create ( layer_source = self , name = bi [ ' name ' ] )
b . up_id = bi [ ' id ' ]
b . up_date = bi [ ' updated ' ]
b . name = bi [ ' name ' ]
b . short_description = bi [ ' short_description ' ]
b . save ( )
2014-08-08 14:03:03 +00:00
# update layers
layers_info = _get_json_response ( apilinks [ ' layerItems ' ] )
2016-02-10 11:51:44 +00:00
2014-08-08 14:03:03 +00:00
for li in layers_info :
2015-09-29 04:45:26 +00:00
# Special case for the openembedded-core layer
if li [ ' name ' ] == oe_core_layer :
try :
# If we have an existing openembedded-core for example
# from the toasterconf.json augment the info using the
# layerindex rather than duplicate it
oe_core_l = Layer . objects . get ( name = oe_core_layer )
# Take ownership of the layer as now coming from the
# layerindex
oe_core_l . layer_source = self
oe_core_l . up_id = li [ ' id ' ]
oe_core_l . summary = li [ ' summary ' ]
oe_core_l . description = li [ ' description ' ]
oe_core_l . save ( )
continue
2015-10-02 18:05:14 +00:00
except Layer . DoesNotExist :
2015-09-29 04:45:26 +00:00
pass
2014-11-05 14:47:51 +00:00
l , created = Layer . objects . get_or_create ( layer_source = self , name = li [ ' name ' ] )
l . up_id = li [ ' id ' ]
2014-08-29 15:41:59 +00:00
l . up_date = li [ ' updated ' ]
l . vcs_url = li [ ' vcs_url ' ]
2014-10-20 15:26:14 +00:00
l . vcs_web_url = li [ ' vcs_web_url ' ]
l . vcs_web_tree_base_url = li [ ' vcs_web_tree_base_url ' ]
2014-08-29 15:41:59 +00:00
l . vcs_web_file_base_url = li [ ' vcs_web_file_base_url ' ]
l . summary = li [ ' summary ' ]
l . description = li [ ' description ' ]
l . save ( )
2015-09-29 04:45:26 +00:00
2014-08-08 14:03:03 +00:00
# update layerbranches/layer_versions
2015-08-18 16:28:52 +00:00
logger . debug ( " Fetching layer information " )
2014-08-08 14:03:03 +00:00
layerbranches_info = _get_json_response ( apilinks [ ' layerBranches ' ]
2014-09-09 10:47:13 +00:00
+ " ?filter=branch: %s " % " OR " . join ( map ( lambda x : str ( x . up_id ) , [ i for i in Branch . objects . filter ( layer_source = self ) if i . up_id is not None ] ) )
2014-08-08 14:03:03 +00:00
)
2015-02-24 17:20:57 +00:00
2014-08-08 14:03:03 +00:00
for lbi in layerbranches_info :
2014-09-23 15:48:52 +00:00
lv , created = Layer_Version . objects . get_or_create ( layer_source = self ,
up_id = lbi [ ' id ' ] ,
layer = Layer . objects . get ( layer_source = self , up_id = lbi [ ' layer ' ] )
)
2014-08-29 15:41:59 +00:00
lv . up_date = lbi [ ' updated ' ]
lv . up_branch = Branch . objects . get ( layer_source = self , up_id = lbi [ ' branch ' ] )
lv . branch = lbi [ ' actual_branch ' ]
2015-01-08 13:15:13 +00:00
lv . commit = lbi [ ' actual_branch ' ]
2014-08-29 15:41:59 +00:00
lv . dirpath = lbi [ ' vcs_subdir ' ]
lv . save ( )
2014-09-23 15:48:52 +00:00
# update layer dependencies
layerdependencies_info = _get_json_response ( apilinks [ ' layerDependencies ' ] )
dependlist = { }
for ldi in layerdependencies_info :
try :
lv = Layer_Version . objects . get ( layer_source = self , up_id = ldi [ ' layerbranch ' ] )
except Layer_Version . DoesNotExist as e :
continue
if lv not in dependlist :
dependlist [ lv ] = [ ]
try :
dependlist [ lv ] . append ( Layer_Version . objects . get ( layer_source = self , layer__up_id = ldi [ ' dependency ' ] , up_branch = lv . up_branch ) )
2015-07-20 16:23:20 +00:00
except Layer_Version . DoesNotExist :
2015-08-18 16:28:54 +00:00
logger . warning ( " Cannot find layer version (ls: %s ), up_id: %s lv: %s " % ( self , ldi [ ' dependency ' ] , lv ) )
2014-09-23 15:48:52 +00:00
for lv in dependlist :
LayerVersionDependency . objects . filter ( layer_version = lv ) . delete ( )
for lvd in dependlist [ lv ] :
LayerVersionDependency . objects . get_or_create ( layer_version = lv , depends_on = lvd )
2014-08-08 14:03:03 +00:00
# update machines
2015-08-18 16:28:52 +00:00
logger . debug ( " Fetching machine information " )
2014-08-08 14:03:03 +00:00
machines_info = _get_json_response ( apilinks [ ' machines ' ]
+ " ?filter=layerbranch: %s " % " OR " . join ( map ( lambda x : str ( x . up_id ) , Layer_Version . objects . filter ( layer_source = self ) ) )
)
2015-03-09 11:23:47 +00:00
2014-08-08 14:03:03 +00:00
for mi in machines_info :
2014-09-23 15:48:52 +00:00
mo , created = Machine . objects . get_or_create ( layer_source = self , up_id = mi [ ' id ' ] , layer_version = Layer_Version . objects . get ( layer_source = self , up_id = mi [ ' layerbranch ' ] ) )
2014-08-29 15:41:59 +00:00
mo . up_date = mi [ ' updated ' ]
mo . name = mi [ ' name ' ]
mo . description = mi [ ' description ' ]
mo . save ( )
2015-03-09 11:23:47 +00:00
2014-08-08 14:03:03 +00:00
# update recipes; paginate by layer version / layer branch
2015-08-18 16:28:52 +00:00
logger . debug ( " Fetching target information " )
2014-08-08 14:03:03 +00:00
recipes_info = _get_json_response ( apilinks [ ' recipes ' ]
+ " ?filter=layerbranch: %s " % " OR " . join ( map ( lambda x : str ( x . up_id ) , Layer_Version . objects . filter ( layer_source = self ) ) )
)
for ri in recipes_info :
2014-11-05 14:47:51 +00:00
try :
ro , created = Recipe . objects . get_or_create ( layer_source = self , up_id = ri [ ' id ' ] , layer_version = Layer_Version . objects . get ( layer_source = self , up_id = ri [ ' layerbranch ' ] ) )
ro . up_date = ri [ ' updated ' ]
ro . name = ri [ ' pn ' ]
ro . version = ri [ ' pv ' ]
ro . summary = ri [ ' summary ' ]
ro . description = ri [ ' description ' ]
ro . section = ri [ ' section ' ]
ro . license = ri [ ' license ' ]
ro . homepage = ri [ ' homepage ' ]
ro . bugtracker = ri [ ' bugtracker ' ]
ro . file_path = ri [ ' filepath ' ] + " / " + ri [ ' filename ' ]
2015-07-28 14:24:45 +00:00
if ' inherits ' in ri :
ro . is_image = ' image ' in ri [ ' inherits ' ] . split ( )
2015-09-29 04:45:11 +00:00
else : # workaround for old style layer index
ro . is_image = " -image- " in ri [ ' pn ' ]
2014-11-05 14:47:51 +00:00
ro . save ( )
2015-08-18 16:28:54 +00:00
except IntegrityError as e :
logger . debug ( " Failed saving recipe, ignoring: %s ( %s : %s ) " % ( e , ro . layer_version , ri [ ' filepath ' ] + " / " + ri [ ' filename ' ] ) )
2015-10-01 21:19:56 +00:00
ro . delete ( )
2014-08-08 14:03:03 +00:00
class BitbakeVersion ( models . Model ) :
2014-10-09 11:37:30 +00:00
2014-08-08 14:03:03 +00:00
name = models . CharField ( max_length = 32 , unique = True )
2014-10-09 11:37:30 +00:00
giturl = GitURLField ( )
2014-08-08 14:03:03 +00:00
branch = models . CharField ( max_length = 32 )
dirpath = models . CharField ( max_length = 255 )
2014-10-09 11:37:30 +00:00
def __unicode__ ( self ) :
2015-03-12 11:30:43 +00:00
return " %s (Branch: %s ) " % ( self . name , self . branch )
2014-10-09 11:37:30 +00:00
2014-08-08 14:03:03 +00:00
class Release ( models . Model ) :
2014-11-14 17:07:06 +00:00
""" A release is a project template, used to pre-populate Project settings with a configuration set """
2014-08-08 14:03:03 +00:00
name = models . CharField ( max_length = 32 , unique = True )
description = models . CharField ( max_length = 255 )
bitbake_version = models . ForeignKey ( BitbakeVersion )
2014-11-14 17:07:06 +00:00
branch_name = models . CharField ( max_length = 50 , default = " " )
2014-11-05 14:47:51 +00:00
helptext = models . TextField ( null = True )
2014-08-08 14:03:03 +00:00
2014-11-14 17:07:06 +00:00
def __unicode__ ( self ) :
return " %s ( %s ) " % ( self . name , self . branch_name )
class ReleaseLayerSourcePriority ( models . Model ) :
""" Each release selects layers from the set up layer sources, ordered by priority """
release = models . ForeignKey ( " Release " )
layer_source = models . ForeignKey ( " LayerSource " )
priority = models . IntegerField ( default = 0 )
def __unicode__ ( self ) :
return " %s - %s : %d " % ( self . release . name , self . layer_source . name , self . priority )
class Meta :
unique_together = ( ( ' release ' , ' layer_source ' ) , )
2014-08-08 14:03:03 +00:00
class ReleaseDefaultLayer ( models . Model ) :
release = models . ForeignKey ( Release )
2014-11-14 17:07:06 +00:00
layer_name = models . CharField ( max_length = 100 , default = " " )
2014-08-08 14:03:03 +00:00
# Branch class is synced with layerindex.Branch, branches can only come from remote layer indexes
class Branch ( models . Model ) :
layer_source = models . ForeignKey ( ' LayerSource ' , null = True , default = True )
up_id = models . IntegerField ( null = True , default = None ) # id of branch in the source
up_date = models . DateTimeField ( null = True , default = None )
name = models . CharField ( max_length = 50 )
short_description = models . CharField ( max_length = 50 , blank = True )
class Meta :
verbose_name_plural = " Branches "
unique_together = ( ( ' layer_source ' , ' name ' ) , ( ' layer_source ' , ' up_id ' ) )
def __unicode__ ( self ) :
return self . name
# Layer class synced with layerindex.LayerItem
2013-10-11 12:46:23 +00:00
class Layer ( models . Model ) :
2014-08-08 14:03:03 +00:00
layer_source = models . ForeignKey ( LayerSource , null = True , default = None ) # from where did we got this layer
up_id = models . IntegerField ( null = True , default = None ) # id of layer in the remote source
up_date = models . DateTimeField ( null = True , default = None )
2013-10-11 12:46:23 +00:00
name = models . CharField ( max_length = 100 )
layer_index_url = models . URLField ( )
2014-10-09 11:37:30 +00:00
vcs_url = GitURLField ( default = None , null = True )
2014-10-20 15:26:14 +00:00
vcs_web_url = models . URLField ( null = True , default = None )
vcs_web_tree_base_url = models . URLField ( null = True , default = None )
2014-08-08 14:03:03 +00:00
vcs_web_file_base_url = models . URLField ( null = True , default = None )
2014-10-09 11:37:30 +00:00
summary = models . TextField ( help_text = ' One-line description of the layer ' , null = True , default = None )
2014-08-08 14:03:03 +00:00
description = models . TextField ( null = True , default = None )
def __unicode__ ( self ) :
2014-11-14 17:07:06 +00:00
return " %s / %s " % ( self . name , self . layer_source )
2014-08-08 14:03:03 +00:00
class Meta :
unique_together = ( ( " layer_source " , " up_id " ) , ( " layer_source " , " name " ) )
2013-10-11 12:46:23 +00:00
2014-08-08 14:03:03 +00:00
# LayerCommit class is synced with layerindex.LayerBranch
2013-10-11 12:46:23 +00:00
class Layer_Version ( models . Model ) :
2014-11-05 14:47:51 +00:00
search_allowed_fields = [ " layer__name " , " layer__summary " , " layer__description " , " layer__vcs_url " , " dirpath " , " up_branch__name " , " commit " , " branch " ]
2014-08-08 14:03:03 +00:00
build = models . ForeignKey ( Build , related_name = ' layer_version_build ' , default = None , null = True )
2013-10-11 12:46:23 +00:00
layer = models . ForeignKey ( Layer , related_name = ' layer_version_layer ' )
2014-08-08 14:03:03 +00:00
layer_source = models . ForeignKey ( LayerSource , null = True , default = None ) # from where did we get this Layer Version
up_id = models . IntegerField ( null = True , default = None ) # id of layerbranch in the remote source
up_date = models . DateTimeField ( null = True , default = None )
up_branch = models . ForeignKey ( Branch , null = True , default = None )
branch = models . CharField ( max_length = 80 ) # LayerBranch.actual_branch
commit = models . CharField ( max_length = 100 ) # LayerBranch.vcs_last_rev
dirpath = models . CharField ( max_length = 255 , null = True , default = None ) # LayerBranch.vcs_subdir
priority = models . IntegerField ( default = 0 ) # if -1, this is a default layer
2015-05-14 15:10:50 +00:00
local_path = models . FilePathField ( max_length = 1024 , default = " / " ) # where this layer was checked-out
2014-11-24 12:52:49 +00:00
project = models . ForeignKey ( ' Project ' , null = True , default = None ) # Set if this layer is project-specific; always set for imported layers, and project-set branches
2014-10-20 15:26:14 +00:00
# code lifted, with adaptations, from the layerindex-web application https://git.yoctoproject.org/cgit/cgit.cgi/layerindex-web/
def _handle_url_path ( self , base_url , path ) :
2015-02-03 15:40:41 +00:00
import re , posixpath
2014-10-20 15:26:14 +00:00
if base_url :
if self . dirpath :
if path :
extra_path = self . dirpath + ' / ' + path
# Normalise out ../ in path for usage URL
extra_path = posixpath . normpath ( extra_path )
# Minor workaround to handle case where subdirectory has been added between branches
# (should probably support usage URL per branch to handle this... sigh...)
if extra_path . startswith ( ' ../ ' ) :
extra_path = extra_path [ 3 : ]
else :
extra_path = self . dirpath
else :
extra_path = path
branchname = self . up_branch . name
url = base_url . replace ( ' % branch % ' , branchname )
# If there's a % in the path (e.g. a wildcard bbappend) we need to encode it
if extra_path :
extra_path = extra_path . replace ( ' % ' , ' % 25 ' )
if ' % path % ' in base_url :
if extra_path :
url = re . sub ( r ' \ [([^ \ ]]* % path % [^ \ ]]*) \ ] ' , ' \\ 1 ' , url )
else :
url = re . sub ( r ' \ [([^ \ ]]* % path % [^ \ ]]*) \ ] ' , ' ' , url )
return url . replace ( ' % path % ' , extra_path )
else :
return url + extra_path
return None
def get_vcs_link_url ( self ) :
if self . layer . vcs_web_url is None :
return None
return self . layer . vcs_web_url
def get_vcs_file_link_url ( self , file_path = " " ) :
2014-09-29 19:20:33 +00:00
if self . layer . vcs_web_file_base_url is None :
return None
2014-10-20 15:26:14 +00:00
return self . _handle_url_path ( self . layer . vcs_web_file_base_url , file_path )
2014-09-29 19:20:33 +00:00
2014-10-20 15:26:14 +00:00
def get_vcs_dirpath_link_url ( self ) :
if self . layer . vcs_web_tree_base_url is None :
return None
return self . _handle_url_path ( self . layer . vcs_web_tree_base_url , ' ' )
2014-09-29 19:20:33 +00:00
2014-11-14 17:07:06 +00:00
def get_equivalents_wpriority ( self , project ) :
2015-12-10 03:56:36 +00:00
layer_versions = project . get_all_compatible_layer_versions ( )
filtered = layer_versions . filter ( layer__name = self . layer . name )
return filtered . order_by ( " -layer_source__releaselayersourcepriority__priority " )
2014-11-14 17:07:06 +00:00
2015-01-08 13:15:13 +00:00
def get_vcs_reference ( self ) :
if self . branch is not None and len ( self . branch ) > 0 :
return self . branch
2015-02-02 17:57:36 +00:00
if self . up_branch is not None :
return self . up_branch . name
2015-09-29 04:45:28 +00:00
if self . commit is not None and len ( self . commit ) > 0 :
return self . commit
2015-10-23 21:57:18 +00:00
return ' N/A '
2014-09-29 19:20:33 +00:00
2015-08-04 19:46:29 +00:00
def get_detailspage_url ( self , project_id ) :
return reverse ( ' layerdetails ' , args = ( project_id , self . pk ) )
2015-10-02 18:05:12 +00:00
def get_alldeps ( self , project_id ) :
""" Get full list of unique layer dependencies. """
def gen_layerdeps ( lver , project ) :
for ldep in lver . dependencies . all ( ) :
yield ldep . depends_on
# get next level of deps recursively calling gen_layerdeps
for subdep in gen_layerdeps ( ldep . depends_on , project ) :
yield subdep
project = Project . objects . get ( pk = project_id )
result = [ ]
projectlvers = [ player . layercommit for player in project . projectlayer_set . all ( ) ]
for dep in gen_layerdeps ( self , project ) :
# filter out duplicates and layers already belonging to the project
if dep not in result + projectlvers :
result . append ( dep )
return sorted ( result , key = lambda x : x . layer . name )
2014-08-08 14:03:03 +00:00
def __unicode__ ( self ) :
2016-01-15 11:00:46 +00:00
return " %d %s (VCS %s , Project %s ) " % ( self . pk , str ( self . layer ) , self . get_vcs_reference ( ) , self . build . project if self . build is not None else " No project " )
2014-08-08 14:03:03 +00:00
class Meta :
unique_together = ( " layer_source " , " up_id " )
class LayerVersionDependency ( models . Model ) :
layer_source = models . ForeignKey ( LayerSource , null = True , default = None ) # from where did we got this layer
up_id = models . IntegerField ( null = True , default = None ) # id of layerbranch in the remote source
layer_version = models . ForeignKey ( Layer_Version , related_name = " dependencies " )
depends_on = models . ForeignKey ( Layer_Version , related_name = " dependees " )
class Meta :
unique_together = ( " layer_source " , " up_id " )
class ProjectLayer ( models . Model ) :
project = models . ForeignKey ( Project )
layercommit = models . ForeignKey ( Layer_Version , null = True )
optional = models . BooleanField ( default = True )
2013-10-11 12:46:23 +00:00
2014-11-14 17:07:06 +00:00
def __unicode__ ( self ) :
return " %s , %s " % ( self . project . name , self . layercommit )
2014-11-05 14:47:51 +00:00
class Meta :
unique_together = ( ( " project " , " layercommit " ) , )
2015-11-04 14:54:41 +00:00
class CustomImageRecipe ( Recipe ) :
2015-11-26 16:44:32 +00:00
search_allowed_fields = [ ' name ' ]
2015-11-04 14:54:41 +00:00
base_recipe = models . ForeignKey ( Recipe , related_name = ' based_on_recipe ' )
2015-09-29 04:45:18 +00:00
project = models . ForeignKey ( Project )
2016-02-05 11:44:57 +00:00
last_updated = models . DateTimeField ( null = True , default = None )
2015-09-29 04:45:18 +00:00
2016-02-05 11:41:43 +00:00
def get_last_successful_built_target ( self ) :
""" Return the last successful built target object if one exists
otherwise return None """
return Target . objects . filter ( Q ( build__outcome = Build . SUCCEEDED ) &
Q ( build__project = self . project ) &
Q ( target = self . name ) ) . last ( )
2016-02-05 11:59:17 +00:00
def update_package_list ( self ) :
""" Update the package list from the last good build of this
CustomImageRecipe
"""
# Check if we're aldready up-to-date or not
target = self . get_last_successful_built_target ( )
if target == None :
# So we've never actually built this Custom recipe but what about
# the recipe it's based on?
target = \
Target . objects . filter ( Q ( build__outcome = Build . SUCCEEDED ) &
Q ( build__project = self . project ) &
Q ( target = self . base_recipe . name ) ) . last ( )
if target == None :
return
if target . build . completed_on == self . last_updated :
return
self . includes_set . clear ( )
excludes_list = self . excludes_set . values_list ( ' name ' , flat = True )
appends_list = self . appends_set . values_list ( ' name ' , flat = True )
built_packages_list = \
target . target_installed_package_set . values_list ( ' package__name ' ,
flat = True )
for built_package in built_packages_list :
# Is the built package in the custom packages list?
if built_package in excludes_list :
continue
if built_package in appends_list :
continue
cust_img_p = \
CustomImagePackage . objects . get ( name = built_package )
self . includes_set . add ( cust_img_p )
self . last_updated = target . build . completed_on
self . save ( )
2015-12-07 18:23:32 +00:00
def get_all_packages ( self ) :
""" Get the included packages and any appended packages """
2016-02-05 11:59:17 +00:00
self . update_package_list ( )
2015-12-07 18:23:32 +00:00
return CustomImagePackage . objects . filter ( ( Q ( recipe_appends = self ) |
2016-02-05 11:59:17 +00:00
Q ( recipe_includes = self ) ) &
~ Q ( recipe_excludes = self ) )
2015-12-07 18:23:32 +00:00
2015-11-04 14:56:36 +00:00
def generate_recipe_file_contents ( self ) :
""" Generate the contents for the recipe file. """
# If we have no excluded packages we only need to _append
if self . excludes_set . count ( ) == 0 :
packages_conf = " IMAGE_INSTALL_append = \" "
for pkg in self . appends_set . all ( ) :
packages_conf + = pkg . name + ' '
else :
2016-01-28 18:01:39 +00:00
packages_conf = " IMAGE_FEATURES = \" \" \n IMAGE_INSTALL = \" "
2015-11-04 14:56:36 +00:00
# We add all the known packages to be built by this recipe apart
2016-02-05 12:37:08 +00:00
# from locale packages which are are controlled with IMAGE_LINGUAS.
for pkg in self . get_all_packages ( ) . exclude (
name__icontains = " locale " ) :
2015-11-04 14:56:36 +00:00
packages_conf + = pkg . name + ' '
packages_conf + = " \" "
base_recipe = open ( " %s / %s " %
( self . base_recipe . layer_version . dirpath ,
self . base_recipe . file_path ) , ' r ' ) . read ( )
2016-01-27 12:44:54 +00:00
# Add a special case for when the recipe we have based a custom image
# recipe on requires another recipe.
# For example:
# "require core-image-minimal.bb" is changed to:
# "require recipes-core/images/core-image-minimal.bb"
if " require " in base_recipe :
req_search = re . search ( r ' (require \ s+)(.+ \ .bb \ s*$) ' ,
base_recipe ,
re . MULTILINE )
require_filename = req_search . group ( 2 ) . strip ( )
corrected_location = Recipe . objects . filter (
Q ( layer_version = self . base_recipe . layer_version ) &
Q ( file_path__icontains = require_filename ) ) . last ( ) . file_path
new_require_line = " require %s " % corrected_location
base_recipe = \
base_recipe . replace ( req_search . group ( 0 ) , new_require_line )
2015-11-04 14:56:36 +00:00
info = { " date " : timezone . now ( ) . strftime ( " % Y- % m- %d % H: % M: % S " ) ,
" base_recipe " : base_recipe ,
" recipe_name " : self . name ,
" base_recipe_name " : self . base_recipe . name ,
" license " : self . license ,
" summary " : self . summary ,
" description " : self . description ,
" packages_conf " : packages_conf . strip ( ) ,
}
recipe_contents = ( " # Original recipe %(base_recipe_name)s \n "
" %(base_recipe)s \n \n "
" # Recipe %(recipe_name)s \n "
" # Customisation Generated by Toaster on %(date)s \n "
" SUMMARY = \" %(summary)s \" \n "
" DESCRIPTION = \" %(description)s \" \n "
" LICENSE = \" %(license)s \" \n "
" %(packages_conf)s " ) % info
return recipe_contents
2014-06-03 15:26:17 +00:00
class ProjectVariable ( models . Model ) :
project = models . ForeignKey ( Project )
name = models . CharField ( max_length = 100 )
value = models . TextField ( blank = True )
2013-10-11 12:46:23 +00:00
class Variable ( models . Model ) :
2014-01-07 13:10:42 +00:00
search_allowed_fields = [ ' variable_name ' , ' variable_value ' ,
2014-01-16 12:22:21 +00:00
' vhistory__file_name ' , " description " ]
2013-10-11 12:46:23 +00:00
build = models . ForeignKey ( Build , related_name = ' variable_build ' )
variable_name = models . CharField ( max_length = 100 )
variable_value = models . TextField ( blank = True )
changed = models . BooleanField ( default = False )
human_readable_name = models . CharField ( max_length = 200 )
description = models . TextField ( blank = True )
2013-11-14 13:56:30 +00:00
class VariableHistory ( models . Model ) :
2014-01-07 13:10:42 +00:00
variable = models . ForeignKey ( Variable , related_name = ' vhistory ' )
2014-02-20 12:47:55 +00:00
value = models . TextField ( blank = True )
2013-11-14 13:56:30 +00:00
file_name = models . FilePathField ( max_length = 255 )
line_number = models . IntegerField ( null = True )
2014-04-28 14:11:03 +00:00
operation = models . CharField ( max_length = 64 )
2013-10-11 12:46:23 +00:00
2014-04-03 10:16:23 +00:00
class HelpText ( models . Model ) :
VARIABLE = 0
HELPTEXT_AREA = ( ( VARIABLE , ' variable ' ) , )
build = models . ForeignKey ( Build , related_name = ' helptext_build ' )
area = models . IntegerField ( choices = HELPTEXT_AREA )
key = models . CharField ( max_length = 100 )
text = models . TextField ( )
2013-10-11 12:46:23 +00:00
class LogMessage ( models . Model ) :
2014-11-25 10:12:46 +00:00
EXCEPTION = - 1 # used to signal self-toaster-exceptions
2013-10-11 12:46:23 +00:00
INFO = 0
WARNING = 1
ERROR = 2
2015-10-15 12:45:13 +00:00
CRITICAL = 3
LOG_LEVEL = (
( INFO , " info " ) ,
( WARNING , " warn " ) ,
( ERROR , " error " ) ,
( CRITICAL , " critical " ) ,
( EXCEPTION , " toaster exception " )
)
2013-10-11 12:46:23 +00:00
build = models . ForeignKey ( Build )
2014-02-20 12:47:55 +00:00
task = models . ForeignKey ( Task , blank = True , null = True )
2013-10-11 12:46:23 +00:00
level = models . IntegerField ( choices = LOG_LEVEL , default = INFO )
2015-10-22 10:22:28 +00:00
message = models . TextField ( blank = True , null = True )
2013-10-11 12:46:23 +00:00
pathname = models . FilePathField ( max_length = 255 , blank = True )
lineno = models . IntegerField ( null = True )
2015-05-19 15:10:19 +00:00
2015-06-17 11:27:48 +00:00
def __str__ ( self ) :
2016-03-08 18:09:02 +00:00
return force_bytes ( ' %s %s %s ' % ( self . get_level_display ( ) , self . message , self . build ) )
2015-06-17 11:27:48 +00:00
2015-05-19 15:10:19 +00:00
def invalidate_cache ( * * kwargs ) :
from django . core . cache import cache
try :
cache . clear ( )
except Exception as e :
2015-08-18 16:28:52 +00:00
logger . warning ( " Problem with cache backend: Failed to clear cache: %s " % e )
2015-05-19 15:10:19 +00:00
django . db . models . signals . post_save . connect ( invalidate_cache )
django . db . models . signals . post_delete . connect ( invalidate_cache )
2016-01-29 14:40:41 +00:00
django . db . models . signals . m2m_changed . connect ( invalidate_cache )