bitbake: fetch2: add the npmsw fetcher

This commit adds a new npmsw fetcher that fetches every npm dependencies
described in a npm shrinkwrap file:

  https://docs.npmjs.com/files/shrinkwrap.json.html

The main package must be fetched separately:

  SRC_URI = "npm://registry.url;package=foobar;version=1.0.0 \
             npmsw://${THISDIR}/npm-shrinkwrap.json"

Since a separation has been created between the package and its
dependencies, the package can also be fetched with a non npm fetcher
without impacting the general behavior:

  SRC_URI = "git://github.com/foo/bar.git;protocol=https \
             npmsw://${THISDIR}/npm-shrinkwrap.json"

(Bitbake rev: f5223be54450bf20e0bfbd53b372a7748a44b475)

Signed-off-by: Jean-Marie LEMETAYER <jean-marie.lemetayer@savoirfairelinux.com>
Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>

bitbake/lib/bb/fetch2/__init__.py

@@ -1894,6 +1894,7 @@ from . import osc
 from . import repo
 from . import clearcase
 from . import npm
+from . import npmsw
 
 methods.append(local.Local())
 methods.append(wget.Wget())
@@ -1912,3 +1913,4 @@ methods.append(osc.Osc())
 methods.append(repo.Repo())
 methods.append(clearcase.ClearCase())
 methods.append(npm.Npm())
+methods.append(npmsw.NpmShrinkWrap())

bitbake/lib/bb/fetch2/npmsw.py

@@ -0,0 +1,255 @@
+# Copyright (C) 2020 Savoir-Faire Linux
+#
+# SPDX-License-Identifier: GPL-2.0-only
+#
+"""
+BitBake 'Fetch' npm shrinkwrap implementation
+
+npm fetcher support the SRC_URI with format of:
+SRC_URI = "npmsw://some.registry.url;OptionA=xxx;OptionB=xxx;..."
+
+Supported SRC_URI options are:
+
+- dev
+   Set to 1 to also install devDependencies.
+
+- destsuffix
+    Specifies the directory to use to unpack the dependencies (default: ${S}).
+"""
+
+import json
+import os
+import re
+import bb
+from bb.fetch2 import Fetch
+from bb.fetch2 import FetchMethod
+from bb.fetch2 import ParameterError
+from bb.fetch2 import URI
+from bb.fetch2.npm import npm_integrity
+from bb.fetch2.npm import npm_localfile
+from bb.fetch2.npm import npm_unpack
+from bb.utils import is_semver
+
+def foreach_dependencies(shrinkwrap, callback=None, dev=False):
+    """
+        Run a callback for each dependencies of a shrinkwrap file.
+        The callback is using the format:
+            callback(name, params, deptree)
+        with:
+            name = the package name (string)
+            params = the package parameters (dictionary)
+            deptree = the package dependency tree (array of strings)
+    """
+    def _walk_deps(deps, deptree):
+        for name in deps:
+            subtree = [*deptree, name]
+            _walk_deps(deps[name].get("dependencies", {}), subtree)
+            if callback is not None:
+                if deps[name].get("dev", False) and not dev:
+                    continue
+                elif deps[name].get("bundled", False):
+                    continue
+                callback(name, deps[name], subtree)
+
+    _walk_deps(shrinkwrap.get("dependencies", {}), [])
+
+class NpmShrinkWrap(FetchMethod):
+    """Class to fetch all package from a shrinkwrap file"""
+
+    def supports(self, ud, d):
+        """Check if a given url can be fetched with npmsw"""
+        return ud.type in ["npmsw"]
+
+    def urldata_init(self, ud, d):
+        """Init npmsw specific variables within url data"""
+
+        # Get the 'shrinkwrap' parameter
+        ud.shrinkwrap_file = re.sub(r"^npmsw://", "", ud.url.split(";")[0])
+
+        # Get the 'dev' parameter
+        ud.dev = bb.utils.to_boolean(ud.parm.get("dev"), False)
+
+        # Resolve the dependencies
+        ud.deps = []
+
+        def _resolve_dependency(name, params, deptree):
+            url = None
+            localpath = None
+            extrapaths = []
+            destsubdirs = [os.path.join("node_modules", dep) for dep in deptree]
+            destsuffix = os.path.join(*destsubdirs)
+
+            integrity = params.get("integrity", None)
+            resolved = params.get("resolved", None)
+            version = params.get("version", None)
+
+            # Handle registry sources
+            if is_semver(version) and resolved and integrity:
+                localfile = npm_localfile(name, version)
+
+                uri = URI(resolved)
+                uri.params["downloadfilename"] = localfile
+
+                checksum_name, checksum_expected = npm_integrity(integrity)
+                uri.params[checksum_name] = checksum_expected
+
+                url = str(uri)
+
+                localpath = os.path.join(d.getVar("DL_DIR"), localfile)
+
+                # Create a resolve file to mimic the npm fetcher and allow
+                # re-usability of the downloaded file.
+                resolvefile = localpath + ".resolved"
+
+                bb.utils.mkdirhier(os.path.dirname(resolvefile))
+                with open(resolvefile, "w") as f:
+                    f.write(url)
+
+                extrapaths.append(resolvefile)
+
+            # Handle http tarball sources
+            elif version.startswith("http") and integrity:
+                localfile = os.path.join("npm2", os.path.basename(version))
+
+                uri = URI(version)
+                uri.params["downloadfilename"] = localfile
+
+                checksum_name, checksum_expected = npm_integrity(integrity)
+                uri.params[checksum_name] = checksum_expected
+
+                url = str(uri)
+
+                localpath = os.path.join(d.getVar("DL_DIR"), localfile)
+
+            # Handle git sources
+            elif version.startswith("git"):
+                regex = re.compile(r"""
+                    ^
+                    git\+
+                    (?P<protocol>[a-z]+)
+                    ://
+                    (?P<url>[^#]+)
+                    \#
+                    (?P<rev>[0-9a-f]+)
+                    $
+                    """, re.VERBOSE)
+
+                match = regex.match(version)
+
+                if not match:
+                    raise ParameterError("Invalid git url: %s" % version, ud.url)
+
+                groups = match.groupdict()
+
+                uri = URI("git://" + str(groups["url"]))
+                uri.params["protocol"] = str(groups["protocol"])
+                uri.params["rev"] = str(groups["rev"])
+                uri.params["destsuffix"] = destsuffix
+
+                url = str(uri)
+
+            # local tarball sources and local link sources are unsupported
+            else:
+                raise ParameterError("Unsupported dependency: %s" % name, ud.url)
+
+            ud.deps.append({
+                "url": url,
+                "localpath": localpath,
+                "extrapaths": extrapaths,
+                "destsuffix": destsuffix,
+            })
+
+        try:
+            with open(ud.shrinkwrap_file, "r") as f:
+                shrinkwrap = json.load(f)
+        except Exception as e:
+            raise ParameterError("Invalid shrinkwrap file: %s" % str(e), ud.url)
+
+        foreach_dependencies(shrinkwrap, _resolve_dependency, ud.dev)
+
+        # Avoid conflicts between the environment data and:
+        # - the proxy url revision
+        # - the proxy url checksum
+        data = bb.data.createCopy(d)
+        data.delVar("SRCREV")
+        data.delVarFlags("SRC_URI")
+
+        # This fetcher resolves multiple URIs from a shrinkwrap file and then
+        # forwards it to a proxy fetcher. The management of the donestamp file,
+        # the lockfile and the checksums are forwarded to the proxy fetcher.
+        ud.proxy = Fetch([dep["url"] for dep in ud.deps], data)
+        ud.needdonestamp = False
+
+    @staticmethod
+    def _foreach_proxy_method(ud, handle):
+        returns = []
+        for proxy_url in ud.proxy.urls:
+            proxy_ud = ud.proxy.ud[proxy_url]
+            proxy_d = ud.proxy.d
+            proxy_ud.setup_localpath(proxy_d)
+            returns.append(handle(proxy_ud.method, proxy_ud, proxy_d))
+        return returns
+
+    def verify_donestamp(self, ud, d):
+        """Verify the donestamp file"""
+        def _handle(m, ud, d):
+            return m.verify_donestamp(ud, d)
+        return all(self._foreach_proxy_method(ud, _handle))
+
+    def update_donestamp(self, ud, d):
+        """Update the donestamp file"""
+        def _handle(m, ud, d):
+            m.update_donestamp(ud, d)
+        self._foreach_proxy_method(ud, _handle)
+
+    def need_update(self, ud, d):
+        """Force a fetch, even if localpath exists ?"""
+        def _handle(m, ud, d):
+            return m.need_update(ud, d)
+        return all(self._foreach_proxy_method(ud, _handle))
+
+    def try_mirrors(self, fetch, ud, d, mirrors):
+        """Try to use a mirror"""
+        def _handle(m, ud, d):
+            return m.try_mirrors(fetch, ud, d, mirrors)
+        return all(self._foreach_proxy_method(ud, _handle))
+
+    def download(self, ud, d):
+        """Fetch url"""
+        ud.proxy.download()
+
+    def unpack(self, ud, rootdir, d):
+        """Unpack the downloaded dependencies"""
+        destdir = d.getVar("S")
+        destsuffix = ud.parm.get("destsuffix")
+        if destsuffix:
+            destdir = os.path.join(rootdir, destsuffix)
+
+        bb.utils.mkdirhier(destdir)
+        bb.utils.copyfile(ud.shrinkwrap_file,
+                          os.path.join(destdir, "npm-shrinkwrap.json"))
+
+        auto = [dep["url"] for dep in ud.deps if not dep["localpath"]]
+        manual = [dep for dep in ud.deps if dep["localpath"]]
+
+        if auto:
+            ud.proxy.unpack(destdir, auto)
+
+        for dep in manual:
+            depdestdir = os.path.join(destdir, dep["destsuffix"])
+            npm_unpack(dep["localpath"], depdestdir, d)
+
+    def clean(self, ud, d):
+        """Clean any existing full or partial download"""
+        ud.proxy.clean()
+
+        # Clean extra files
+        for dep in ud.deps:
+            for path in dep["extrapaths"]:
+                bb.utils.remove(path)
+
+    def done(self, ud, d):
+        """Is the download done ?"""
+        def _handle(m, ud, d):
+            return m.done(ud, d)
+        return all(self._foreach_proxy_method(ud, _handle))