2016-02-29 11:48:26 +00:00
|
|
|
# Recipe creation tool - node.js NPM module support plugin
|
|
|
|
#
|
|
|
|
# Copyright (C) 2016 Intel Corporation
|
|
|
|
#
|
|
|
|
# This program is free software; you can redistribute it and/or modify
|
|
|
|
# it under the terms of the GNU General Public License version 2 as
|
|
|
|
# published by the Free Software Foundation.
|
|
|
|
#
|
|
|
|
# This program is distributed in the hope that it will be useful,
|
|
|
|
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
|
|
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
|
|
# GNU General Public License for more details.
|
|
|
|
#
|
|
|
|
# You should have received a copy of the GNU General Public License along
|
|
|
|
# with this program; if not, write to the Free Software Foundation, Inc.,
|
|
|
|
# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
|
|
|
|
|
2016-03-09 04:48:55 +00:00
|
|
|
import os
|
2016-02-29 11:48:26 +00:00
|
|
|
import logging
|
2016-03-09 04:48:55 +00:00
|
|
|
import subprocess
|
|
|
|
import tempfile
|
|
|
|
import shutil
|
2016-02-29 11:48:26 +00:00
|
|
|
import json
|
devtool: add: build nodejs-native if npm is needed and not available
If the user runs devtool add on an npm:// URL (or source tree that uses
node.js), and npm is not available, just build nodejs-native instead of
telling the user they need to do it; if that fails because there isn't
any such recipe (which would be the default, since it's not in OE-Core)
then produce a slightly more readable error message hinting at what the
user needs to do.
Note that this forces the use of nodejs-native rather than npm on the
host - this makes sense for two reasons: (1) we need it to be compatible
with nodejs for the target, and (2) we have to have a recipe for that
anyway, so allowing you to avoid having a recipe for the native version
isn't really beneficial.
There's a bit of a hack in here in order to allow this - for node.js
sources that aren't fetched via npm we don't know that they are that
until we've fetched and unpacked them, by which time we're inside
recipetool and have an active tinfoil instance that will prevent bitbake
being run. To avoid this being an issue, we allow recipetool to get to
the point where we know we need npm and then exit with a specific exit
code, at which point devtool can try to build it and then if that
succeeds, it will re-execute recipetool. This is definitely not ideal,
but it can't really be refactored and done properly until we do the
tinfoil2 refactoring; in the mean time though we still want to be
helpful to the user.
Fixes [YOCTO #10337].
(From OE-Core rev: f40662bde5aab158c4e4c3c3ff5e68665a4194a5)
Signed-off-by: Paul Eggleton <paul.eggleton@linux.intel.com>
Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
2016-10-04 09:31:16 +00:00
|
|
|
from recipetool.create import RecipeHandler, split_pkg_licenses, handle_license_vars, check_npm
|
2016-02-29 11:48:26 +00:00
|
|
|
|
|
|
|
logger = logging.getLogger('recipetool')
|
|
|
|
|
|
|
|
|
2016-03-09 04:48:55 +00:00
|
|
|
tinfoil = None
|
|
|
|
|
|
|
|
def tinfoil_init(instance):
|
|
|
|
global tinfoil
|
|
|
|
tinfoil = instance
|
|
|
|
|
|
|
|
|
2016-02-29 11:48:26 +00:00
|
|
|
class NpmRecipeHandler(RecipeHandler):
|
2016-03-09 04:48:55 +00:00
|
|
|
lockdownpath = None
|
|
|
|
|
2016-03-09 04:48:52 +00:00
|
|
|
def _handle_license(self, data):
|
|
|
|
'''
|
|
|
|
Handle the license value from an npm package.json file
|
|
|
|
'''
|
|
|
|
license = None
|
|
|
|
if 'license' in data:
|
|
|
|
license = data['license']
|
|
|
|
if isinstance(license, dict):
|
|
|
|
license = license.get('type', None)
|
2016-04-27 13:06:19 +00:00
|
|
|
return license
|
2016-03-09 04:48:52 +00:00
|
|
|
|
2016-03-09 04:48:55 +00:00
|
|
|
def _shrinkwrap(self, srctree, localfilesdir, extravalues, lines_before):
|
|
|
|
try:
|
2016-12-14 21:13:05 +00:00
|
|
|
runenv = dict(os.environ, PATH=tinfoil.config_data.getVar('PATH'))
|
2016-03-09 04:48:55 +00:00
|
|
|
bb.process.run('npm shrinkwrap', cwd=srctree, stderr=subprocess.STDOUT, env=runenv, shell=True)
|
|
|
|
except bb.process.ExecutionError as e:
|
|
|
|
logger.warn('npm shrinkwrap failed:\n%s' % e.stdout)
|
|
|
|
return
|
|
|
|
|
|
|
|
tmpfile = os.path.join(localfilesdir, 'npm-shrinkwrap.json')
|
|
|
|
shutil.move(os.path.join(srctree, 'npm-shrinkwrap.json'), tmpfile)
|
|
|
|
extravalues.setdefault('extrafiles', {})
|
|
|
|
extravalues['extrafiles']['npm-shrinkwrap.json'] = tmpfile
|
|
|
|
lines_before.append('NPM_SHRINKWRAP := "${THISDIR}/${PN}/npm-shrinkwrap.json"')
|
|
|
|
|
|
|
|
def _lockdown(self, srctree, localfilesdir, extravalues, lines_before):
|
2016-12-14 21:13:05 +00:00
|
|
|
runenv = dict(os.environ, PATH=tinfoil.config_data.getVar('PATH'))
|
2016-03-09 04:48:55 +00:00
|
|
|
if not NpmRecipeHandler.lockdownpath:
|
|
|
|
NpmRecipeHandler.lockdownpath = tempfile.mkdtemp('recipetool-npm-lockdown')
|
|
|
|
bb.process.run('npm install lockdown --prefix %s' % NpmRecipeHandler.lockdownpath,
|
|
|
|
cwd=srctree, stderr=subprocess.STDOUT, env=runenv, shell=True)
|
|
|
|
relockbin = os.path.join(NpmRecipeHandler.lockdownpath, 'node_modules', 'lockdown', 'relock.js')
|
|
|
|
if not os.path.exists(relockbin):
|
|
|
|
logger.warn('Could not find relock.js within lockdown directory; skipping lockdown')
|
|
|
|
return
|
|
|
|
try:
|
|
|
|
bb.process.run('node %s' % relockbin, cwd=srctree, stderr=subprocess.STDOUT, env=runenv, shell=True)
|
|
|
|
except bb.process.ExecutionError as e:
|
|
|
|
logger.warn('lockdown-relock failed:\n%s' % e.stdout)
|
|
|
|
return
|
|
|
|
|
|
|
|
tmpfile = os.path.join(localfilesdir, 'lockdown.json')
|
|
|
|
shutil.move(os.path.join(srctree, 'lockdown.json'), tmpfile)
|
|
|
|
extravalues.setdefault('extrafiles', {})
|
|
|
|
extravalues['extrafiles']['lockdown.json'] = tmpfile
|
|
|
|
lines_before.append('NPM_LOCKDOWN := "${THISDIR}/${PN}/lockdown.json"')
|
|
|
|
|
2016-09-05 03:58:01 +00:00
|
|
|
def _handle_dependencies(self, d, deps, lines_before, srctree):
|
|
|
|
import scriptutils
|
|
|
|
# If this isn't a single module we need to get the dependencies
|
|
|
|
# and add them to SRC_URI
|
|
|
|
def varfunc(varname, origvalue, op, newlines):
|
|
|
|
if varname == 'SRC_URI':
|
|
|
|
if not origvalue.startswith('npm://'):
|
|
|
|
src_uri = origvalue.split()
|
|
|
|
changed = False
|
|
|
|
for dep, depdata in deps.items():
|
|
|
|
version = self.get_node_version(dep, depdata, d)
|
|
|
|
if version:
|
|
|
|
url = 'npm://registry.npmjs.org;name=%s;version=%s;subdir=node_modules/%s' % (dep, version, dep)
|
|
|
|
scriptutils.fetch_uri(d, url, srctree)
|
|
|
|
src_uri.append(url)
|
|
|
|
changed = True
|
|
|
|
if changed:
|
|
|
|
return src_uri, None, -1, True
|
|
|
|
return origvalue, None, 0, True
|
|
|
|
updated, newlines = bb.utils.edit_metadata(lines_before, ['SRC_URI'], varfunc)
|
|
|
|
if updated:
|
|
|
|
del lines_before[:]
|
|
|
|
for line in newlines:
|
|
|
|
# Hack to avoid newlines that edit_metadata inserts
|
|
|
|
if line.endswith('\n'):
|
|
|
|
line = line[:-1]
|
|
|
|
lines_before.append(line)
|
|
|
|
return updated
|
|
|
|
|
|
|
|
def _replace_license_vars(self, srctree, lines_before, handled, extravalues, d):
|
|
|
|
for item in handled:
|
|
|
|
if isinstance(item, tuple):
|
|
|
|
if item[0] == 'license':
|
|
|
|
del item
|
|
|
|
break
|
|
|
|
|
|
|
|
calledvars = []
|
|
|
|
def varfunc(varname, origvalue, op, newlines):
|
|
|
|
if varname in ['LICENSE', 'LIC_FILES_CHKSUM']:
|
|
|
|
for i, e in enumerate(reversed(newlines)):
|
|
|
|
if not e.startswith('#'):
|
|
|
|
stop = i
|
|
|
|
while stop > 0:
|
|
|
|
newlines.pop()
|
|
|
|
stop -= 1
|
|
|
|
break
|
|
|
|
calledvars.append(varname)
|
|
|
|
if len(calledvars) > 1:
|
|
|
|
# The second time around, put the new license text in
|
|
|
|
insertpos = len(newlines)
|
|
|
|
handle_license_vars(srctree, newlines, handled, extravalues, d)
|
|
|
|
return None, None, 0, True
|
|
|
|
return origvalue, None, 0, True
|
|
|
|
updated, newlines = bb.utils.edit_metadata(lines_before, ['LICENSE', 'LIC_FILES_CHKSUM'], varfunc)
|
|
|
|
if updated:
|
|
|
|
del lines_before[:]
|
|
|
|
lines_before.extend(newlines)
|
|
|
|
else:
|
|
|
|
raise Exception('Did not find license variables')
|
|
|
|
|
2016-02-29 11:48:26 +00:00
|
|
|
def process(self, srctree, classes, lines_before, lines_after, handled, extravalues):
|
2016-03-09 04:48:55 +00:00
|
|
|
import bb.utils
|
2016-03-09 04:48:52 +00:00
|
|
|
import oe
|
|
|
|
from collections import OrderedDict
|
|
|
|
|
2016-02-29 11:48:26 +00:00
|
|
|
if 'buildsystem' in handled:
|
|
|
|
return False
|
|
|
|
|
2016-03-09 04:48:52 +00:00
|
|
|
def read_package_json(fn):
|
2016-06-29 03:12:03 +00:00
|
|
|
with open(fn, 'r', errors='surrogateescape') as f:
|
2016-03-09 04:48:52 +00:00
|
|
|
return json.loads(f.read())
|
|
|
|
|
2016-02-29 11:48:26 +00:00
|
|
|
files = RecipeHandler.checkfiles(srctree, ['package.json'])
|
|
|
|
if files:
|
devtool: add: build nodejs-native if npm is needed and not available
If the user runs devtool add on an npm:// URL (or source tree that uses
node.js), and npm is not available, just build nodejs-native instead of
telling the user they need to do it; if that fails because there isn't
any such recipe (which would be the default, since it's not in OE-Core)
then produce a slightly more readable error message hinting at what the
user needs to do.
Note that this forces the use of nodejs-native rather than npm on the
host - this makes sense for two reasons: (1) we need it to be compatible
with nodejs for the target, and (2) we have to have a recipe for that
anyway, so allowing you to avoid having a recipe for the native version
isn't really beneficial.
There's a bit of a hack in here in order to allow this - for node.js
sources that aren't fetched via npm we don't know that they are that
until we've fetched and unpacked them, by which time we're inside
recipetool and have an active tinfoil instance that will prevent bitbake
being run. To avoid this being an issue, we allow recipetool to get to
the point where we know we need npm and then exit with a specific exit
code, at which point devtool can try to build it and then if that
succeeds, it will re-execute recipetool. This is definitely not ideal,
but it can't really be refactored and done properly until we do the
tinfoil2 refactoring; in the mean time though we still want to be
helpful to the user.
Fixes [YOCTO #10337].
(From OE-Core rev: f40662bde5aab158c4e4c3c3ff5e68665a4194a5)
Signed-off-by: Paul Eggleton <paul.eggleton@linux.intel.com>
Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
2016-10-04 09:31:16 +00:00
|
|
|
check_npm(tinfoil.config_data)
|
|
|
|
|
2016-03-09 04:48:52 +00:00
|
|
|
data = read_package_json(files[0])
|
2016-02-29 11:48:26 +00:00
|
|
|
if 'name' in data and 'version' in data:
|
|
|
|
extravalues['PN'] = data['name']
|
|
|
|
extravalues['PV'] = data['version']
|
|
|
|
classes.append('npm')
|
|
|
|
handled.append('buildsystem')
|
|
|
|
if 'description' in data:
|
2016-05-29 22:20:59 +00:00
|
|
|
extravalues['SUMMARY'] = data['description']
|
2016-02-29 11:48:26 +00:00
|
|
|
if 'homepage' in data:
|
2016-05-29 22:20:59 +00:00
|
|
|
extravalues['HOMEPAGE'] = data['homepage']
|
2016-03-09 04:48:52 +00:00
|
|
|
|
2016-09-05 03:58:01 +00:00
|
|
|
deps = data.get('dependencies', {})
|
|
|
|
updated = self._handle_dependencies(tinfoil.config_data, deps, lines_before, srctree)
|
|
|
|
if updated:
|
|
|
|
# We need to redo the license stuff
|
|
|
|
self._replace_license_vars(srctree, lines_before, handled, extravalues, tinfoil.config_data)
|
|
|
|
|
2016-03-09 04:48:55 +00:00
|
|
|
# Shrinkwrap
|
|
|
|
localfilesdir = tempfile.mkdtemp(prefix='recipetool-npm')
|
|
|
|
self._shrinkwrap(srctree, localfilesdir, extravalues, lines_before)
|
|
|
|
|
|
|
|
# Lockdown
|
|
|
|
self._lockdown(srctree, localfilesdir, extravalues, lines_before)
|
|
|
|
|
2016-03-09 04:48:52 +00:00
|
|
|
# Split each npm module out to is own package
|
|
|
|
npmpackages = oe.package.npm_split_package_dirs(srctree)
|
|
|
|
for item in handled:
|
|
|
|
if isinstance(item, tuple):
|
|
|
|
if item[0] == 'license':
|
|
|
|
licvalues = item[1]
|
|
|
|
break
|
|
|
|
if licvalues:
|
|
|
|
# Augment the license list with information we have in the packages
|
|
|
|
licenses = {}
|
|
|
|
license = self._handle_license(data)
|
|
|
|
if license:
|
|
|
|
licenses['${PN}'] = license
|
2016-05-18 18:39:44 +00:00
|
|
|
for pkgname, pkgitem in npmpackages.items():
|
2016-03-09 04:48:52 +00:00
|
|
|
_, pdata = pkgitem
|
|
|
|
license = self._handle_license(pdata)
|
|
|
|
if license:
|
|
|
|
licenses[pkgname] = license
|
|
|
|
# Now write out the package-specific license values
|
|
|
|
# We need to strip out the json data dicts for this since split_pkg_licenses
|
|
|
|
# isn't expecting it
|
2016-05-18 18:39:44 +00:00
|
|
|
packages = OrderedDict((x,y[0]) for x,y in npmpackages.items())
|
2016-03-09 04:48:52 +00:00
|
|
|
packages['${PN}'] = ''
|
|
|
|
pkglicenses = split_pkg_licenses(licvalues, packages, lines_after, licenses)
|
|
|
|
all_licenses = list(set([item for pkglicense in pkglicenses.values() for item in pkglicense]))
|
|
|
|
# Go back and update the LICENSE value since we have a bit more
|
|
|
|
# information than when that was written out (and we know all apply
|
|
|
|
# vs. there being a choice, so we can join them with &)
|
|
|
|
for i, line in enumerate(lines_before):
|
|
|
|
if line.startswith('LICENSE = '):
|
|
|
|
lines_before[i] = 'LICENSE = "%s"' % ' & '.join(all_licenses)
|
|
|
|
break
|
|
|
|
|
2016-09-05 03:58:01 +00:00
|
|
|
# Need to move S setting after inherit npm
|
|
|
|
for i, line in enumerate(lines_before):
|
|
|
|
if line.startswith('S ='):
|
|
|
|
lines_before.pop(i)
|
|
|
|
lines_after.insert(0, '# Must be set after inherit npm since that itself sets S')
|
|
|
|
lines_after.insert(1, line)
|
|
|
|
break
|
|
|
|
|
2016-02-29 11:48:26 +00:00
|
|
|
return True
|
|
|
|
|
|
|
|
return False
|
|
|
|
|
2016-09-05 03:58:01 +00:00
|
|
|
# FIXME this is duplicated from lib/bb/fetch2/npm.py
|
|
|
|
def _parse_view(self, output):
|
|
|
|
'''
|
|
|
|
Parse the output of npm view --json; the last JSON result
|
|
|
|
is assumed to be the one that we're interested in.
|
|
|
|
'''
|
|
|
|
pdata = None
|
|
|
|
outdeps = {}
|
|
|
|
datalines = []
|
|
|
|
bracelevel = 0
|
|
|
|
for line in output.splitlines():
|
|
|
|
if bracelevel:
|
|
|
|
datalines.append(line)
|
|
|
|
elif '{' in line:
|
|
|
|
datalines = []
|
|
|
|
datalines.append(line)
|
|
|
|
bracelevel = bracelevel + line.count('{') - line.count('}')
|
|
|
|
if datalines:
|
|
|
|
pdata = json.loads('\n'.join(datalines))
|
|
|
|
return pdata
|
|
|
|
|
|
|
|
# FIXME this is effectively duplicated from lib/bb/fetch2/npm.py
|
|
|
|
# (split out from _getdependencies())
|
|
|
|
def get_node_version(self, pkg, version, d):
|
|
|
|
import bb.fetch2
|
|
|
|
pkgfullname = pkg
|
|
|
|
if version != '*' and not '/' in version:
|
|
|
|
pkgfullname += "@'%s'" % version
|
|
|
|
logger.debug(2, "Calling getdeps on %s" % pkg)
|
2016-12-14 21:13:05 +00:00
|
|
|
runenv = dict(os.environ, PATH=d.getVar('PATH'))
|
2016-09-05 03:58:01 +00:00
|
|
|
fetchcmd = "npm view %s --json" % pkgfullname
|
|
|
|
output, _ = bb.process.run(fetchcmd, stderr=subprocess.STDOUT, env=runenv, shell=True)
|
|
|
|
data = self._parse_view(output)
|
|
|
|
return data.get('version', None)
|
|
|
|
|
2016-02-29 11:48:26 +00:00
|
|
|
def register_recipe_handlers(handlers):
|
|
|
|
handlers.append((NpmRecipeHandler(), 60))
|