2014-12-19 11:41:56 +00:00
# Development tool - deploy/undeploy command plugin
#
2016-02-19 09:38:57 +00:00
# Copyright (C) 2014-2016 Intel Corporation
2014-12-19 11:41:56 +00:00
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License version 2 as
# published by the Free Software Foundation.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License along
# with this program; if not, write to the Free Software Foundation, Inc.,
# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
2015-05-11 13:17:01 +00:00
""" Devtool plugin containing the deploy subcommands """
2014-12-19 11:41:56 +00:00
import os
import subprocess
import logging
2016-02-19 09:38:57 +00:00
import tempfile
import shutil
2016-02-19 09:38:58 +00:00
import argparse_oe
2015-09-22 16:21:24 +00:00
from devtool import exec_fakeroot , setup_tinfoil , check_workspace_recipe , DevtoolError
2014-12-19 11:41:56 +00:00
logger = logging . getLogger ( ' devtool ' )
2016-02-19 09:38:57 +00:00
deploylist_path = ' /.devtool '
2016-02-19 09:38:59 +00:00
def _prepare_remote_script ( deploy , verbose = False , dryrun = False , undeployall = False , nopreserve = False , nocheckspace = False ) :
2016-02-19 09:38:57 +00:00
"""
Prepare a shell script for running on the target to
deploy / undeploy files . We have to be careful what we put in this
script - only commands that are likely to be available on the
target are suitable ( the target might be constrained , e . g . using
busybox rather than bash with coreutils ) .
"""
lines = [ ]
lines . append ( ' #!/bin/sh ' )
lines . append ( ' set -e ' )
2016-02-19 09:38:58 +00:00
if undeployall :
# Yes, I know this is crude - but it does work
lines . append ( ' for entry in %s /*.list; do ' % deploylist_path )
lines . append ( ' [ ! -f $entry ] && exit ' )
lines . append ( ' set `basename $entry | sed " s/.list// " ` ' )
if dryrun :
if not deploy :
lines . append ( ' echo " Previously deployed files for $1: " ' )
2016-02-19 09:38:57 +00:00
lines . append ( ' manifest= " %s /$1.list " ' % deploylist_path )
2016-02-19 09:38:59 +00:00
lines . append ( ' preservedir= " %s /$1.preserve " ' % deploylist_path )
2016-02-19 09:38:57 +00:00
lines . append ( ' if [ -f $manifest ] ; then ' )
# Read manifest in reverse and delete files / remove empty dirs
lines . append ( ' sed \' 1!G;h;$!d \' $manifest | while read file ' )
lines . append ( ' do ' )
2016-02-19 09:38:58 +00:00
if dryrun :
lines . append ( ' if [ ! -d $file ] ; then ' )
lines . append ( ' echo $file ' )
lines . append ( ' fi ' )
else :
lines . append ( ' if [ -d $file ] ; then ' )
2016-02-19 09:38:59 +00:00
# Avoid deleting a preserved directory in case it has special perms
lines . append ( ' if [ ! -d $preservedir/$file ] ; then ' )
lines . append ( ' rmdir $file > /dev/null 2>&1 || true ' )
lines . append ( ' fi ' )
2016-02-19 09:38:58 +00:00
lines . append ( ' else ' )
lines . append ( ' rm $file ' )
lines . append ( ' fi ' )
2016-02-19 09:38:57 +00:00
lines . append ( ' done ' )
2016-02-19 09:38:58 +00:00
if not dryrun :
lines . append ( ' rm $manifest ' )
if not deploy and not dryrun :
2016-02-19 09:38:57 +00:00
# May as well remove all traces
lines . append ( ' rmdir `dirname $manifest` > /dev/null 2>&1 || true ' )
lines . append ( ' fi ' )
if deploy :
2016-02-19 09:38:59 +00:00
if not nocheckspace :
# Check for available space
# FIXME This doesn't take into account files spread across multiple
# partitions, but doing that is non-trivial
# Find the part of the destination path that exists
lines . append ( ' checkpath= " $2 " ' )
lines . append ( ' while [ " $checkpath " != " / " ] && [ ! -e $checkpath ] ' )
lines . append ( ' do ' )
lines . append ( ' checkpath=`dirname " $checkpath " ` ' )
lines . append ( ' done ' )
2016-09-30 19:53:40 +00:00
lines . append ( r ' freespace=$(df -P $checkpath | sed -nre " s/^( \ S+ \ s+) {3} ([0-9]+).*/ \ 2/p " ) ' )
2016-02-19 09:38:59 +00:00
# First line of the file is the total space
lines . append ( ' total=`head -n1 $3` ' )
lines . append ( ' if [ $total -gt $freespace ] ; then ' )
lines . append ( ' echo " ERROR: insufficient space on target (available $ {freespace} , needed $ {total} ) " ' )
lines . append ( ' exit 1 ' )
lines . append ( ' fi ' )
if not nopreserve :
# Preserve any files that exist. Note that this will add to the
# preserved list with successive deployments if the list of files
# deployed changes, but because we've deleted any previously
# deployed files at this point it will never preserve anything
# that was deployed, only files that existed prior to any deploying
# (which makes the most sense)
lines . append ( ' cat $3 | sed " 1d " | while read file fsize ' )
lines . append ( ' do ' )
lines . append ( ' if [ -e $file ] ; then ' )
lines . append ( ' dest= " $preservedir/$file " ' )
lines . append ( ' mkdir -p `dirname $dest` ' )
lines . append ( ' mv $file $dest ' )
lines . append ( ' fi ' )
lines . append ( ' done ' )
lines . append ( ' rm $3 ' )
2016-02-19 09:38:57 +00:00
lines . append ( ' mkdir -p `dirname $manifest` ' )
lines . append ( ' mkdir -p $2 ' )
if verbose :
lines . append ( ' tar xv -C $2 -f - | tee $manifest ' )
else :
lines . append ( ' tar xv -C $2 -f - > $manifest ' )
lines . append ( ' sed -i " s!^./!$2! " $manifest ' )
2016-02-19 09:38:59 +00:00
elif not dryrun :
# Put any preserved files back
lines . append ( ' if [ -d $preservedir ] ; then ' )
lines . append ( ' cd $preservedir ' )
lines . append ( ' find . -type f -exec mv {} / {} \ ; ' )
lines . append ( ' cd / ' )
lines . append ( ' rm -rf $preservedir ' )
lines . append ( ' fi ' )
2016-02-19 09:38:58 +00:00
if undeployall :
if not dryrun :
lines . append ( ' echo " NOTE: Successfully undeployed $1 " ' )
lines . append ( ' done ' )
2016-02-19 09:38:57 +00:00
# Delete the script itself
lines . append ( ' rm $0 ' )
lines . append ( ' ' )
return ' \n ' . join ( lines )
2014-12-19 11:41:56 +00:00
def deploy ( args , config , basepath , workspace ) :
2015-05-11 13:17:01 +00:00
""" Entry point for the devtool ' deploy ' subcommand """
2014-12-19 11:41:56 +00:00
import re
2016-02-19 09:38:59 +00:00
import math
2015-05-14 10:47:35 +00:00
import oe . recipeutils
2014-12-19 11:41:56 +00:00
2015-09-22 16:21:24 +00:00
check_workspace_recipe ( workspace , args . recipename , checksrc = False )
2014-12-19 11:41:56 +00:00
try :
host , destdir = args . target . split ( ' : ' )
except ValueError :
destdir = ' / '
else :
args . target = host
2016-02-19 09:38:57 +00:00
if not destdir . endswith ( ' / ' ) :
destdir + = ' / '
2014-12-19 11:41:56 +00:00
2015-09-23 10:05:23 +00:00
tinfoil = setup_tinfoil ( basepath = basepath )
2015-05-14 10:47:35 +00:00
try :
2015-08-05 14:48:00 +00:00
try :
rd = oe . recipeutils . parse_recipe_simple ( tinfoil . cooker , args . recipename , tinfoil . config_data )
except Exception as e :
raise DevtoolError ( ' Exception parsing recipe %s : %s ' %
( args . recipename , e ) )
recipe_outdir = rd . getVar ( ' D ' , True )
if not os . path . exists ( recipe_outdir ) or not os . listdir ( recipe_outdir ) :
raise DevtoolError ( ' No files to deploy - have you built the %s '
' recipe? If so, the install step has not installed '
' any files. ' % args . recipename )
filelist = [ ]
ftotalsize = 0
for root , _ , files in os . walk ( recipe_outdir ) :
for fn in files :
# Get the size in kiB (since we'll be comparing it to the output of du -k)
# MUST use lstat() here not stat() or getfilesize() since we don't want to
# dereference symlinks
fsize = int ( math . ceil ( float ( os . lstat ( os . path . join ( root , fn ) ) . st_size ) / 1024 ) )
ftotalsize + = fsize
# The path as it would appear on the target
fpath = os . path . join ( destdir , os . path . relpath ( root , recipe_outdir ) , fn )
filelist . append ( ( fpath , fsize ) )
if args . dry_run :
print ( ' Files to be deployed for %s on target %s : ' % ( args . recipename , args . target ) )
for item , _ in filelist :
print ( ' %s ' % item )
return 0
extraoptions = ' '
if args . no_host_check :
extraoptions + = ' -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no '
if not args . show_status :
extraoptions + = ' -q '
# In order to delete previously deployed files and have the manifest file on
# the target, we write out a shell script and then copy it to the target
# so we can then run it (piping tar output to it).
# (We cannot use scp here, because it doesn't preserve symlinks.)
tmpdir = tempfile . mkdtemp ( prefix = ' devtool ' )
try :
tmpscript = ' /tmp/devtool_deploy.sh '
tmpfilelist = os . path . join ( os . path . dirname ( tmpscript ) , ' devtool_deploy.list ' )
shellscript = _prepare_remote_script ( deploy = True ,
verbose = args . show_status ,
nopreserve = args . no_preserve ,
nocheckspace = args . no_check_space )
# Write out the script to a file
with open ( os . path . join ( tmpdir , os . path . basename ( tmpscript ) ) , ' w ' ) as f :
f . write ( shellscript )
# Write out the file list
with open ( os . path . join ( tmpdir , os . path . basename ( tmpfilelist ) ) , ' w ' ) as f :
f . write ( ' %d \n ' % ftotalsize )
for fpath , fsize in filelist :
f . write ( ' %s %d \n ' % ( fpath , fsize ) )
# Copy them to the target
ret = subprocess . call ( " scp %s %s /* %s : %s " % ( extraoptions , tmpdir , args . target , os . path . dirname ( tmpscript ) ) , shell = True )
if ret != 0 :
raise DevtoolError ( ' Failed to copy script to %s - rerun with -s to '
' get a complete error message ' % args . target )
finally :
shutil . rmtree ( tmpdir )
# Now run the script
ret = exec_fakeroot ( rd , ' tar cf - . | ssh %s %s \' sh %s %s %s %s \' ' % ( extraoptions , args . target , tmpscript , args . recipename , destdir , tmpfilelist ) , cwd = recipe_outdir , shell = True )
2016-02-19 09:38:57 +00:00
if ret != 0 :
2015-08-05 14:48:00 +00:00
raise DevtoolError ( ' Deploy failed - rerun with -s to get a complete '
' error message ' )
2014-12-19 11:41:56 +00:00
2015-08-05 14:48:00 +00:00
logger . info ( ' Successfully deployed %s ' % recipe_outdir )
2014-12-19 11:41:56 +00:00
2015-08-05 14:48:00 +00:00
files_list = [ ]
for root , _ , files in os . walk ( recipe_outdir ) :
for filename in files :
filename = os . path . relpath ( os . path . join ( root , filename ) , recipe_outdir )
files_list . append ( os . path . join ( destdir , filename ) )
finally :
tinfoil . shutdown ( )
2014-12-19 11:41:56 +00:00
return 0
def undeploy ( args , config , basepath , workspace ) :
2015-05-11 13:17:01 +00:00
""" Entry point for the devtool ' undeploy ' subcommand """
2016-02-19 09:38:58 +00:00
if args . all and args . recipename :
raise argparse_oe . ArgumentUsageError ( ' Cannot specify -a/--all with a recipe name ' , ' undeploy-target ' )
elif not args . recipename and not args . all :
raise argparse_oe . ArgumentUsageError ( ' If you don \' t specify a recipe, you must specify -a/--all ' , ' undeploy-target ' )
2015-03-08 12:25:56 +00:00
extraoptions = ' '
if args . no_host_check :
extraoptions + = ' -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no '
2015-03-08 13:03:33 +00:00
if not args . show_status :
extraoptions + = ' -q '
2015-03-08 12:25:56 +00:00
2016-02-19 09:38:57 +00:00
args . target = args . target . split ( ' : ' ) [ 0 ]
2014-12-19 11:41:56 +00:00
2016-02-19 09:38:57 +00:00
tmpdir = tempfile . mkdtemp ( prefix = ' devtool ' )
try :
tmpscript = ' /tmp/devtool_undeploy.sh '
2016-02-19 09:38:58 +00:00
shellscript = _prepare_remote_script ( deploy = False , dryrun = args . dry_run , undeployall = args . all )
2016-02-19 09:38:57 +00:00
# Write out the script to a file
with open ( os . path . join ( tmpdir , os . path . basename ( tmpscript ) ) , ' w ' ) as f :
f . write ( shellscript )
# Copy it to the target
ret = subprocess . call ( " scp %s %s /* %s : %s " % ( extraoptions , tmpdir , args . target , os . path . dirname ( tmpscript ) ) , shell = True )
if ret != 0 :
raise DevtoolError ( ' Failed to copy script to %s - rerun with -s to '
' get a complete error message ' % args . target )
finally :
shutil . rmtree ( tmpdir )
# Now run the script
ret = subprocess . call ( ' ssh %s %s \' sh %s %s \' ' % ( extraoptions , args . target , tmpscript , args . recipename ) , shell = True )
if ret != 0 :
2015-05-27 14:59:09 +00:00
raise DevtoolError ( ' Undeploy failed - rerun with -s to get a complete '
' error message ' )
2014-12-19 11:41:56 +00:00
2016-02-19 09:38:58 +00:00
if not args . all and not args . dry_run :
logger . info ( ' Successfully undeployed %s ' % args . recipename )
2016-02-19 09:38:57 +00:00
return 0
2014-12-19 11:41:56 +00:00
def register_commands ( subparsers , context ) :
2015-05-11 13:17:01 +00:00
""" Register devtool subcommands from the deploy plugin """
devtool: categorise and order subcommands in help output
The listing of subcommands in the --help output for devtool was starting
to get difficult to follow, with commands appearing in no particular
order (due to some being in separate modules and the order of those
modules being parsed). Logically grouping the subcommands as well as
being able to exercise some control over the order of the subcommands
and groups would help, if we do so without losing the dynamic nature of
the list (i.e. that it comes from the plugins). Argparse provides no
built-in way to handle this and really, really makes it a pain to add,
but with some subclassing and hacking it's now possible, and can be
extended by any plugin as desired.
To put a subcommand into a group, all you need to do is specify a group=
parameter in the call to subparsers.add_parser(). you can also specify
an order= parameter to make the subcommand sort higher or lower in the
list (higher order numbers appear first, so use negative numbers to
force items to the end if that's what you want). To add a new group, use
subparsers.add_subparser_group(), supplying the name, description and
optionally an order number for the group itself (again, higher numbers
appear first).
(From OE-Core rev: e1b9d31e6ea3c254ecfe940fe795af44761e0e69)
Signed-off-by: Paul Eggleton <paul.eggleton@linux.intel.com>
Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
2016-02-19 09:38:53 +00:00
parser_deploy = subparsers . add_parser ( ' deploy-target ' ,
help = ' Deploy recipe output files to live target machine ' ,
2016-02-19 09:38:59 +00:00
description = ' Deploys a recipe \' s build output (i.e. the output of the do_install task) to a live target machine over ssh. By default, any existing files will be preserved instead of being overwritten and will be restored if you run devtool undeploy-target. Note: this only deploys the recipe itself and not any runtime dependencies, so it is assumed that those have been installed on the target beforehand. ' ,
devtool: categorise and order subcommands in help output
The listing of subcommands in the --help output for devtool was starting
to get difficult to follow, with commands appearing in no particular
order (due to some being in separate modules and the order of those
modules being parsed). Logically grouping the subcommands as well as
being able to exercise some control over the order of the subcommands
and groups would help, if we do so without losing the dynamic nature of
the list (i.e. that it comes from the plugins). Argparse provides no
built-in way to handle this and really, really makes it a pain to add,
but with some subclassing and hacking it's now possible, and can be
extended by any plugin as desired.
To put a subcommand into a group, all you need to do is specify a group=
parameter in the call to subparsers.add_parser(). you can also specify
an order= parameter to make the subcommand sort higher or lower in the
list (higher order numbers appear first, so use negative numbers to
force items to the end if that's what you want). To add a new group, use
subparsers.add_subparser_group(), supplying the name, description and
optionally an order number for the group itself (again, higher numbers
appear first).
(From OE-Core rev: e1b9d31e6ea3c254ecfe940fe795af44761e0e69)
Signed-off-by: Paul Eggleton <paul.eggleton@linux.intel.com>
Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
2016-02-19 09:38:53 +00:00
group = ' testbuild ' )
2014-12-19 11:41:56 +00:00
parser_deploy . add_argument ( ' recipename ' , help = ' Recipe to deploy ' )
parser_deploy . add_argument ( ' target ' , help = ' Live target machine running an ssh server: user@hostname[:destdir] ' )
2015-03-08 12:25:56 +00:00
parser_deploy . add_argument ( ' -c ' , ' --no-host-check ' , help = ' Disable ssh host key checking ' , action = ' store_true ' )
2015-03-08 13:03:33 +00:00
parser_deploy . add_argument ( ' -s ' , ' --show-status ' , help = ' Show progress/status output ' , action = ' store_true ' )
2015-03-10 14:42:36 +00:00
parser_deploy . add_argument ( ' -n ' , ' --dry-run ' , help = ' List files to be deployed only ' , action = ' store_true ' )
2016-02-19 09:38:59 +00:00
parser_deploy . add_argument ( ' -p ' , ' --no-preserve ' , help = ' Do not preserve existing files ' , action = ' store_true ' )
parser_deploy . add_argument ( ' --no-check-space ' , help = ' Do not check for available space before deploying ' , action = ' store_true ' )
2014-12-19 11:41:56 +00:00
parser_deploy . set_defaults ( func = deploy )
devtool: categorise and order subcommands in help output
The listing of subcommands in the --help output for devtool was starting
to get difficult to follow, with commands appearing in no particular
order (due to some being in separate modules and the order of those
modules being parsed). Logically grouping the subcommands as well as
being able to exercise some control over the order of the subcommands
and groups would help, if we do so without losing the dynamic nature of
the list (i.e. that it comes from the plugins). Argparse provides no
built-in way to handle this and really, really makes it a pain to add,
but with some subclassing and hacking it's now possible, and can be
extended by any plugin as desired.
To put a subcommand into a group, all you need to do is specify a group=
parameter in the call to subparsers.add_parser(). you can also specify
an order= parameter to make the subcommand sort higher or lower in the
list (higher order numbers appear first, so use negative numbers to
force items to the end if that's what you want). To add a new group, use
subparsers.add_subparser_group(), supplying the name, description and
optionally an order number for the group itself (again, higher numbers
appear first).
(From OE-Core rev: e1b9d31e6ea3c254ecfe940fe795af44761e0e69)
Signed-off-by: Paul Eggleton <paul.eggleton@linux.intel.com>
Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
2016-02-19 09:38:53 +00:00
parser_undeploy = subparsers . add_parser ( ' undeploy-target ' ,
help = ' Undeploy recipe output files in live target machine ' ,
2016-02-19 09:38:55 +00:00
description = ' Un-deploys recipe output files previously deployed to a live target machine by devtool deploy-target. ' ,
devtool: categorise and order subcommands in help output
The listing of subcommands in the --help output for devtool was starting
to get difficult to follow, with commands appearing in no particular
order (due to some being in separate modules and the order of those
modules being parsed). Logically grouping the subcommands as well as
being able to exercise some control over the order of the subcommands
and groups would help, if we do so without losing the dynamic nature of
the list (i.e. that it comes from the plugins). Argparse provides no
built-in way to handle this and really, really makes it a pain to add,
but with some subclassing and hacking it's now possible, and can be
extended by any plugin as desired.
To put a subcommand into a group, all you need to do is specify a group=
parameter in the call to subparsers.add_parser(). you can also specify
an order= parameter to make the subcommand sort higher or lower in the
list (higher order numbers appear first, so use negative numbers to
force items to the end if that's what you want). To add a new group, use
subparsers.add_subparser_group(), supplying the name, description and
optionally an order number for the group itself (again, higher numbers
appear first).
(From OE-Core rev: e1b9d31e6ea3c254ecfe940fe795af44761e0e69)
Signed-off-by: Paul Eggleton <paul.eggleton@linux.intel.com>
Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
2016-02-19 09:38:53 +00:00
group = ' testbuild ' )
2016-02-19 09:38:58 +00:00
parser_undeploy . add_argument ( ' recipename ' , help = ' Recipe to undeploy (if not using -a/--all) ' , nargs = ' ? ' )
2014-12-19 11:41:56 +00:00
parser_undeploy . add_argument ( ' target ' , help = ' Live target machine running an ssh server: user@hostname ' )
2015-03-08 12:25:56 +00:00
parser_undeploy . add_argument ( ' -c ' , ' --no-host-check ' , help = ' Disable ssh host key checking ' , action = ' store_true ' )
2015-03-08 13:03:33 +00:00
parser_undeploy . add_argument ( ' -s ' , ' --show-status ' , help = ' Show progress/status output ' , action = ' store_true ' )
2016-02-19 09:38:58 +00:00
parser_undeploy . add_argument ( ' -a ' , ' --all ' , help = ' Undeploy all recipes deployed on the target ' , action = ' store_true ' )
2015-03-10 14:42:36 +00:00
parser_undeploy . add_argument ( ' -n ' , ' --dry-run ' , help = ' List files to be undeployed only ' , action = ' store_true ' )
2014-12-19 11:41:56 +00:00
parser_undeploy . set_defaults ( func = undeploy )