meta/lib/oeqa: Test for bootimg-biosplusefi Source

Add unittests for bootimg-biosplusefi SourcePlugin in wic module.
First test check wic creation works correctly.
Second test uses qemu to boot image and checks that it has both
EFI and BIOS files in a single partition.

(From OE-Core rev: e0c3436241afca93f107e325d1b9ffcdebf706cd)

Signed-off-by: William Bourque <wbourque@gmail.com>
Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>

README.LSB

@@ -12,6 +12,9 @@ therefore provides compatibility with the following caveats:
   tests any longer. Users are recommended to group together to support
   maintenance of that layer. [http://git.yoctoproject.org/cgit/cgit.cgi/meta-qt4/]
 
+* GTK+ 2 is provided by meta-oe.  It is noted that GTK+ 2 is end of life
+  and use of GTK+ 2 isn't recommended.
+
 * mailx has been dropped since its no longer being developed upstream 
   and there are better, more modern replacements such as s-nail 
   (http://sdaoden.eu/code.html) or mailutils (http://mailutils.org/).

bitbake/bin/bitbake-selftest

@@ -25,6 +25,7 @@ tests = ["bb.tests.codeparser",
          "bb.tests.fetch",
          "bb.tests.parse",
          "bb.tests.persist_data",
+         "bb.tests.runqueue",
          "bb.tests.utils",
          "hashserv.tests",
          "layerindexlib.tests.layerindexobj",

bitbake/lib/bb/build.py

@@ -163,12 +163,35 @@ class LogTee(object):
 
     def __repr__(self):
         return '<LogTee {0}>'.format(self.name)
+
     def flush(self):
         self.outfile.flush()
 
+
+class StdoutNoopContextManager:
+    """
+    This class acts like sys.stdout, but adds noop __enter__ and __exit__ methods.
+    """
+    def __enter__(self):
+        return sys.stdout
+
+    def __exit__(self, *exc_info):
+        pass
+
+    def write(self, string):
+        return sys.stdout.write(string)
+
+    def flush(self):
+        sys.stdout.flush()
+
+    @property
+    def name(self):
+        return sys.stdout.name
+
+
 #
 # pythonexception allows the python exceptions generated to be raised
-# as the real exceptions (not FuncFailed) and without a backtrace at the 
+# as the real exceptions (not FuncFailed) and without a backtrace at the
 # origin of the failure.
 #
 def exec_func(func, d, dirs = None, pythonexception=False):
@@ -323,6 +346,42 @@ trap 'bb_exit_handler' 0
 set -e
 '''
 
+def create_progress_handler(func, progress, logfile, d):
+    if progress == 'percent':
+        # Use default regex
+        return bb.progress.BasicProgressHandler(d, outfile=logfile)
+    elif progress.startswith('percent:'):
+        # Use specified regex
+        return bb.progress.BasicProgressHandler(d, regex=progress.split(':', 1)[1], outfile=logfile)
+    elif progress.startswith('outof:'):
+        # Use specified regex
+        return bb.progress.OutOfProgressHandler(d, regex=progress.split(':', 1)[1], outfile=logfile)
+    elif progress.startswith("custom:"):
+        # Use a custom progress handler that was injected via OE_EXTRA_IMPORTS or __builtins__
+        import functools
+        from types import ModuleType
+
+        parts = progress.split(":", 2)
+        _, cls, otherargs = parts[0], parts[1], (parts[2] or None) if parts[2:] else None
+        if cls:
+            def resolve(x, y):
+                if not x:
+                    return None
+                if isinstance(x, ModuleType):
+                    return getattr(x, y, None)
+                return x.get(y)
+            cls_obj = functools.reduce(resolve, cls.split("."), bb.utils._context)
+            if not cls_obj:
+                # Fall-back on __builtins__
+                cls_obj = functools.reduce(lambda x, y: x.get(y), cls.split("."), __builtins__)
+            if cls_obj:
+                return cls_obj(d, outfile=logfile, otherargs=otherargs)
+            bb.warn('%s: unknown custom progress handler in task progress varflag value "%s", ignoring' % (func, cls))
+    else:
+        bb.warn('%s: invalid task progress varflag value "%s", ignoring' % (func, progress))
+
+    return logfile
+
 def exec_func_shell(func, d, runfile, cwd=None):
     """Execute a shell function from the metadata
 
@@ -360,23 +419,13 @@ exit $ret
             cmd = [fakerootcmd, runfile]
 
     if bb.msg.loggerDefaultVerbose:
-        logfile = LogTee(logger, sys.stdout)
+        logfile = LogTee(logger, StdoutNoopContextManager())
     else:
-        logfile = sys.stdout
+        logfile = StdoutNoopContextManager()
 
     progress = d.getVarFlag(func, 'progress')
     if progress:
-        if progress == 'percent':
-            # Use default regex
-            logfile = bb.progress.BasicProgressHandler(d, outfile=logfile)
-        elif progress.startswith('percent:'):
-            # Use specified regex
-            logfile = bb.progress.BasicProgressHandler(d, regex=progress.split(':', 1)[1], outfile=logfile)
-        elif progress.startswith('outof:'):
-            # Use specified regex
-            logfile = bb.progress.OutOfProgressHandler(d, regex=progress.split(':', 1)[1], outfile=logfile)
-        else:
-            bb.warn('%s: invalid task progress varflag value "%s", ignoring' % (func, progress))
+        logfile = create_progress_handler(func, progress, logfile, d)
 
     fifobuffer = bytearray()
     def readfifo(data):
@@ -428,7 +477,7 @@ exit $ret
             bb.debug(2, "Executing shell function %s" % func)
 
             try:
-                with open(os.devnull, 'r+') as stdin:
+                with open(os.devnull, 'r+') as stdin, logfile:
                     bb.process.run(cmd, shell=False, stdin=stdin, log=logfile, extrafiles=[(fifo,readfifo)])
             except bb.process.CmdError:
                 logfn = d.getVar('BB_LOGFILE')

bitbake/lib/bb/cache.py

@@ -83,20 +83,20 @@ class CoreRecipeInfo(RecipeInfoCommon):
         self.appends = self.listvar('__BBAPPEND', metadata)
         self.nocache = self.getvar('BB_DONT_CACHE', metadata)
 
-        self.skipreason = self.getvar('__SKIPPED', metadata)
-        if self.skipreason:
-            self.pn = self.getvar('PN', metadata) or bb.parse.vars_from_file(filename,metadata)[0]
-            self.skipped = True
-            self.provides  = self.depvar('PROVIDES', metadata)
-            self.rprovides = self.depvar('RPROVIDES', metadata)
-            return
-
-        self.tasks = metadata.getVar('__BBTASKS', False)
-
-        self.pn = self.getvar('PN', metadata)
+        self.provides  = self.depvar('PROVIDES', metadata)
+        self.rprovides = self.depvar('RPROVIDES', metadata)
+        self.pn = self.getvar('PN', metadata) or bb.parse.vars_from_file(filename,metadata)[0]
         self.packages = self.listvar('PACKAGES', metadata)
         if not self.packages:
             self.packages.append(self.pn)
+        self.packages_dynamic = self.listvar('PACKAGES_DYNAMIC', metadata)
+
+        self.skipreason = self.getvar('__SKIPPED', metadata)
+        if self.skipreason:
+            self.skipped = True
+            return
+
+        self.tasks = metadata.getVar('__BBTASKS', False)
 
         self.basetaskhashes = self.taskvar('BB_BASEHASH', self.tasks, metadata)
         self.hashfilename = self.getvar('BB_HASHFILENAME', metadata)
@@ -113,11 +113,8 @@ class CoreRecipeInfo(RecipeInfoCommon):
         self.stampclean = self.getvar('STAMPCLEAN', metadata)
         self.stamp_extrainfo = self.flaglist('stamp-extra-info', self.tasks, metadata)
         self.file_checksums = self.flaglist('file-checksums', self.tasks, metadata, True)
-        self.packages_dynamic = self.listvar('PACKAGES_DYNAMIC', metadata)
         self.depends          = self.depvar('DEPENDS', metadata)
-        self.provides         = self.depvar('PROVIDES', metadata)
         self.rdepends         = self.depvar('RDEPENDS', metadata)
-        self.rprovides        = self.depvar('RPROVIDES', metadata)
         self.rrecommends      = self.depvar('RRECOMMENDS', metadata)
         self.rprovides_pkg    = self.pkgvar('RPROVIDES', self.packages, metadata)
         self.rdepends_pkg     = self.pkgvar('RDEPENDS', self.packages, metadata)
@@ -237,7 +234,7 @@ def virtualfn2realfn(virtualfn):
     Convert a virtual file name to a real one + the associated subclass keyword
     """
     mc = ""
-    if virtualfn.startswith('multiconfig:'):
+    if virtualfn.startswith('mc:'):
         elems = virtualfn.split(':')
         mc = elems[1]
         virtualfn = ":".join(elems[2:])
@@ -258,7 +255,7 @@ def realfn2virtual(realfn, cls, mc):
     if cls:
         realfn = "virtual:" + cls + ":" + realfn
     if mc:
-        realfn = "multiconfig:" + mc + ":" + realfn
+        realfn = "mc:" + mc + ":" + realfn
     return realfn
 
 def variant2virtual(realfn, variant):
@@ -267,11 +264,11 @@ def variant2virtual(realfn, variant):
     """
     if variant == "":
         return realfn
-    if variant.startswith("multiconfig:"):
+    if variant.startswith("mc:"):
         elems = variant.split(":")
         if elems[2]:
-            return "multiconfig:" + elems[1] + ":virtual:" + ":".join(elems[2:]) + ":" + realfn
-        return "multiconfig:" + elems[1] + ":" + realfn
+            return "mc:" + elems[1] + ":virtual:" + ":".join(elems[2:]) + ":" + realfn
+        return "mc:" + elems[1] + ":" + realfn
     return "virtual:" + variant + ":" + realfn
 
 def parse_recipe(bb_data, bbfile, appends, mc=''):
@@ -349,7 +346,7 @@ class NoCache(object):
             bb_data = self.databuilder.mcdata[mc].createCopy()
             newstores = parse_recipe(bb_data, bbfile, appends, mc)
             for ns in newstores:
-                datastores["multiconfig:%s:%s" % (mc, ns)] = newstores[ns]
+                datastores["mc:%s:%s" % (mc, ns)] = newstores[ns]
 
         return datastores
 
@@ -399,6 +396,15 @@ class Cache(NoCache):
         else:
             logger.debug(1, "Cache file %s not found, building..." % self.cachefile)
 
+        # We don't use the symlink, its just for debugging convinience
+        symlink = os.path.join(self.cachedir, "bb_cache.dat")
+        if os.path.exists(symlink):
+            bb.utils.remove(symlink)
+        try:
+            os.symlink(os.path.basename(self.cachefile), symlink)
+        except OSError:
+            pass
+
     def load_cachefile(self):
         cachesize = 0
         previous_progress = 0

bitbake/lib/bb/cooker.py

@@ -1,4 +1,3 @@
-#!/usr/bin/env python
 #
 # Copyright (C) 2003, 2004  Chris Larson
 # Copyright (C) 2003, 2004  Phil Blundell
@@ -378,8 +377,9 @@ class BBCooker:
         if CookerFeatures.BASEDATASTORE_TRACKING in self.featureset:
             self.disableDataTracking()
 
-        self.data.renameVar("__depends", "__base_depends")
-        self.add_filewatch(self.data.getVar("__base_depends", False), self.configwatcher)
+        for mc in self.databuilder.mcdata.values():
+            mc.renameVar("__depends", "__base_depends")
+            self.add_filewatch(mc.getVar("__base_depends", False), self.configwatcher)
 
         self.baseconfig_valid = True
         self.parsecache_valid = False
@@ -494,6 +494,7 @@ class BBCooker:
         """
         fn = None
         envdata = None
+        mc = ''
         if not pkgs_to_build:
             pkgs_to_build = []
 
@@ -502,6 +503,12 @@ class BBCooker:
             self.enableDataTracking()
             self.reset()
 
+        def mc_base(p):
+            if p.startswith('mc:'):
+                s = p.split(':')
+                if len(s) == 2:
+                    return s[1]
+            return None
 
         if buildfile:
             # Parse the configuration here. We need to do it explicitly here since
@@ -512,18 +519,16 @@ class BBCooker:
             fn = self.matchFile(fn)
             fn = bb.cache.realfn2virtual(fn, cls, mc)
         elif len(pkgs_to_build) == 1:
-            ignore = self.data.getVar("ASSUME_PROVIDED") or ""
-            if pkgs_to_build[0] in set(ignore.split()):
-                bb.fatal("%s is in ASSUME_PROVIDED" % pkgs_to_build[0])
+            mc = mc_base(pkgs_to_build[0])
+            if not mc:
+                ignore = self.data.getVar("ASSUME_PROVIDED") or ""
+                if pkgs_to_build[0] in set(ignore.split()):
+                    bb.fatal("%s is in ASSUME_PROVIDED" % pkgs_to_build[0])
 
-            taskdata, runlist = self.buildTaskData(pkgs_to_build, None, self.configuration.abort, allowincomplete=True)
+                taskdata, runlist = self.buildTaskData(pkgs_to_build, None, self.configuration.abort, allowincomplete=True)
 
-            mc = runlist[0][0]
-            fn = runlist[0][3]
-        else:
-            envdata = self.data
-            data.expandKeys(envdata)
-            parse.ast.runAnonFuncs(envdata)
+                mc = runlist[0][0]
+                fn = runlist[0][3]
 
         if fn:
             try:
@@ -532,6 +537,12 @@ class BBCooker:
             except Exception as e:
                 parselog.exception("Unable to read %s", fn)
                 raise
+        else:
+            if not mc in self.databuilder.mcdata:
+                bb.fatal('Not multiconfig named "%s" found' % mc)
+            envdata = self.databuilder.mcdata[mc]
+            data.expandKeys(envdata)
+            parse.ast.runAnonFuncs(envdata)
 
         # Display history
         with closing(StringIO()) as env:
@@ -571,10 +582,10 @@ class BBCooker:
         wildcard = False
 
         # Wild card expansion:
-        # Replace string such as "multiconfig:*:bash"
-        # into "multiconfig:A:bash multiconfig:B:bash bash"
+        # Replace string such as "mc:*:bash"
+        # into "mc:A:bash mc:B:bash bash"
         for k in targetlist:
-            if k.startswith("multiconfig:"):
+            if k.startswith("mc:"):
                 if wildcard:
                     bb.fatal('multiconfig conflict')
                 if k.split(":")[1] == "*":
@@ -607,7 +618,7 @@ class BBCooker:
         runlist = []
         for k in fulltargetlist:
             mc = ""
-            if k.startswith("multiconfig:"):
+            if k.startswith("mc:"):
                 mc = k.split(":")[1]
                 k = ":".join(k.split(":")[2:])
             ktask = task
@@ -626,9 +637,13 @@ class BBCooker:
             runlist.append([mc, k, ktask, fn])
             bb.event.fire(bb.event.TreeDataPreparationProgress(current, len(fulltargetlist)), self.data)
 
+        havemc = False
+        for mc in self.multiconfigs:
+            if taskdata[mc].get_mcdepends():
+                havemc = True
 
         # No need to do check providers if there are no mcdeps or not an mc build
-        if len(self.multiconfigs) > 1:
+        if havemc or len(self.multiconfigs) > 1:
             seen = set()
             new = True
             # Make sure we can provide the multiconfig dependency
@@ -688,7 +703,7 @@ class BBCooker:
     @staticmethod
     def add_mc_prefix(mc, pn):
         if mc:
-            return "multiconfig:%s:%s" % (mc, pn)
+            return "mc:%s:%s" % (mc, pn)
         return pn
 
     def buildDependTree(self, rq, taskdata):
@@ -1465,7 +1480,7 @@ class BBCooker:
         ntargets = []
         for target in runlist:
             if target[0]:
-                ntargets.append("multiconfig:%s:%s:%s" % (target[0], target[1], target[2]))
+                ntargets.append("mc:%s:%s:%s" % (target[0], target[1], target[2]))
             ntargets.append("%s:%s" % (target[1], target[2]))
 
         for mc in self.multiconfigs:
@@ -1588,6 +1603,9 @@ class BBCooker:
         for pkg in pkgs_to_build:
             if pkg in ignore:
                 parselog.warning("Explicit target \"%s\" is in ASSUME_PROVIDED, ignoring" % pkg)
+            if pkg.startswith("multiconfig:"):
+                pkgs_to_build.remove(pkg)
+                pkgs_to_build.append(pkg.replace("multiconfig:", "mc:"))
 
         if 'world' in pkgs_to_build:
             pkgs_to_build.remove('world')
@@ -1595,7 +1613,7 @@ class BBCooker:
                 bb.providers.buildWorldTargetList(self.recipecaches[mc], task)
                 for t in self.recipecaches[mc].world_target:
                     if mc:
-                        t = "multiconfig:" + mc + ":" + t
+                        t = "mc:" + mc + ":" + t
                     pkgs_to_build.append(t)
 
         if 'universe' in pkgs_to_build:
@@ -1614,7 +1632,7 @@ class BBCooker:
                             bb.debug(1, "Skipping %s for universe tasks as task %s doesn't exist" % (t, task))
                             continue
                     if mc:
-                        t = "multiconfig:" + mc + ":" + t
+                        t = "mc:" + mc + ":" + t
                     pkgs_to_build.append(t)
 
         return pkgs_to_build

bitbake/lib/bb/cookerdata.py

@@ -1,4 +1,3 @@
-#!/usr/bin/env python
 #
 # Copyright (C) 2003, 2004  Chris Larson
 # Copyright (C) 2003, 2004  Phil Blundell
@@ -122,6 +121,7 @@ class CookerConfiguration(object):
         self.profile = False
         self.nosetscene = False
         self.setsceneonly = False
+        self.skipsetscene = False
         self.invalidate_stamp = False
         self.dump_signatures = []
         self.dry_run = False
@@ -342,14 +342,24 @@ class CookerDataBuilder(object):
             data = parse_config_file(layerconf, data)
 
             layers = (data.getVar('BBLAYERS') or "").split()
+            broken_layers = []
 
             data = bb.data.createCopy(data)
             approved = bb.utils.approved_variables()
+
+            # Check whether present layer directories exist
             for layer in layers:
                 if not os.path.isdir(layer):
-                    parselog.critical("Layer directory '%s' does not exist! "
-                                      "Please check BBLAYERS in %s" % (layer, layerconf))
-                    sys.exit(1)
+                    broken_layers.append(layer)
+
+            if broken_layers:
+                parselog.critical("The following layer directories do not exist:")
+                for layer in broken_layers:
+                    parselog.critical("   %s", layer)
+                parselog.critical("Please check BBLAYERS in %s" % (layerconf))
+                sys.exit(1)
+
+            for layer in layers:
                 parselog.debug(2, "Adding layer %s", layer)
                 if 'HOME' in approved and '~' in layer:
                     layer = os.path.expanduser(layer)

bitbake/lib/bb/event.py

@@ -124,6 +124,7 @@ def fire_class_handlers(event, d):
 ui_queue = []
 @atexit.register
 def print_ui_queue():
+    global ui_queue
     """If we're exiting before a UI has been spawned, display any queued
     LogRecords to the console."""
     logger = logging.getLogger("BitBake")
@@ -168,6 +169,7 @@ def print_ui_queue():
             logger.removeHandler(stderr)
         else:
             logger.removeHandler(stdout)
+        ui_queue = []
 
 def fire_ui_handlers(event, d):
     global _thread_lock
@@ -402,23 +404,6 @@ class RecipeTaskPreProcess(RecipeEvent):
 class RecipeParsed(RecipeEvent):
     """ Recipe Parsing Complete """
 
-class StampUpdate(Event):
-    """Trigger for any adjustment of the stamp files to happen"""
-
-    def __init__(self, targets, stampfns):
-        self._targets = targets
-        self._stampfns = stampfns
-        Event.__init__(self)
-
-    def getStampPrefix(self):
-        return self._stampfns
-
-    def getTargets(self):
-        return self._targets
-
-    stampPrefix = property(getStampPrefix)
-    targets = property(getTargets)
-
 class BuildBase(Event):
     """Base class for bitbake build events"""
 

bitbake/lib/bb/fetch2/__init__.py

@@ -843,6 +843,11 @@ def runfetchcmd(cmd, d, quiet=False, cleanup=None, log=None, workdir=None):
         if val:
             cmd = 'export ' + var + '=\"%s\"; %s' % (val, cmd)
 
+    # Ensure that a _PYTHON_SYSCONFIGDATA_NAME value set by a recipe
+    # (for example via python3native.bbclass since warrior) is not set for
+    # host Python (otherwise tools like git-make-shallow will fail)
+    cmd = 'unset _PYTHON_SYSCONFIGDATA_NAME; ' + cmd
+
     # Disable pseudo as it may affect ssh, potentially causing it to hang.
     cmd = 'export PSEUDO_DISABLED=1; ' + cmd
 

bitbake/lib/bb/fetch2/clearcase.py

@@ -54,6 +54,8 @@ import shutil
 import bb
 from   bb.fetch2 import FetchMethod
 from   bb.fetch2 import FetchError
+from   bb.fetch2 import MissingParameterError
+from   bb.fetch2 import ParameterError
 from   bb.fetch2 import runfetchcmd
 from   bb.fetch2 import logger
 
@@ -79,7 +81,7 @@ class ClearCase(FetchMethod):
         if 'protocol' in ud.parm:
             ud.proto = ud.parm['protocol']
         if not ud.proto in ('http', 'https'):
-            raise fetch2.ParameterError("Invalid protocol type", ud.url)
+            raise ParameterError("Invalid protocol type", ud.url)
 
         ud.vob = ''
         if 'vob' in ud.parm:

bitbake/lib/bb/main.py

@@ -1,4 +1,3 @@
-#!/usr/bin/env python
 #
 # Copyright (C) 2003, 2004  Chris Larson
 # Copyright (C) 2003, 2004  Phil Blundell
@@ -255,6 +254,11 @@ class BitBakeConfigParameters(cookerdata.ConfigParameters):
                           help="Do not run any setscene tasks. sstate will be ignored and "
                                "everything needed, built.")
 
+        parser.add_option("", "--skip-setscene", action="store_true",
+                          dest="skipsetscene", default=False,
+                          help="Skip setscene tasks if they would be executed. Tasks previously "
+                               "restored from sstate will be kept, unlike --no-setscene")
+
         parser.add_option("", "--setscene-only", action="store_true",
                           dest="setsceneonly", default=False,
                           help="Only run setscene tasks, don't run any real tasks.")
@@ -448,12 +452,7 @@ def setup_bitbake(configParams, configuration, extrafeatures=None):
                             bb.utils.unlockfile(lock)
                         raise bb.server.process.ProcessTimeout("Bitbake still shutting down as socket exists but no lock?")
                 if not configParams.server_only:
-                    try:
-                        server_connection = bb.server.process.connectProcessServer(sockname, featureset)
-                    except EOFError:
-                        # The server may have been shutting down but not closed the socket yet. If that happened,
-                        # ignore it.
-                        pass
+                    server_connection = bb.server.process.connectProcessServer(sockname, featureset)
 
                 if server_connection or configParams.server_only:
                     break
@@ -464,12 +463,13 @@ def setup_bitbake(configParams, configuration, extrafeatures=None):
                     raise
                 retries -= 1
                 tryno = 8 - retries
-                if isinstance(e, (bb.server.process.ProcessTimeout, BrokenPipeError)):
+                if isinstance(e, (bb.server.process.ProcessTimeout, BrokenPipeError, EOFError)):
                     logger.info("Retrying server connection (#%d)..." % tryno)
                 else:
                     logger.info("Retrying server connection (#%d)... (%s)" % (tryno, traceback.format_exc()))
             if not retries:
-                bb.fatal("Unable to connect to bitbake server, or start one")
+                bb.fatal("Unable to connect to bitbake server, or start one (server startup failures would be in bitbake-cookerdaemon.log).")
+            bb.event.print_ui_queue()
             if retries < 5:
                 time.sleep(5)
 
@@ -491,7 +491,7 @@ def setup_bitbake(configParams, configuration, extrafeatures=None):
 def lockBitbake():
     topdir = bb.cookerdata.findTopdir()
     if not topdir:
-        bb.error("Unable to find conf/bblayers.conf or conf/bitbake.conf. BBAPTH is unset and/or not in a build directory?")
+        bb.error("Unable to find conf/bblayers.conf or conf/bitbake.conf. BBPATH is unset and/or not in a build directory?")
         raise BBMainFatal
     lockfile = topdir + "/bitbake.lock"
     return topdir, bb.utils.lockfile(lockfile, False, False)

bitbake/lib/bb/monitordisk.py

@@ -1,4 +1,3 @@
-#!/usr/bin/env python
 #
 # Copyright (C) 2012 Robert Yang
 #

bitbake/lib/bb/namedtuple_with_abc.py

@@ -1,5 +1,4 @@
 # http://code.activestate.com/recipes/577629-namedtupleabc-abstract-base-class-mix-in-for-named/
-#!/usr/bin/env python
 # Copyright (c) 2011 Jan Kaliszewski (zuo). Available under the MIT License.
 #
 # SPDX-License-Identifier: MIT

bitbake/lib/bb/parse/parse_py/BBHandler.py

@@ -1,4 +1,3 @@
-#!/usr/bin/env python
 """
    class for handling .bb files
 

bitbake/lib/bb/parse/parse_py/ConfHandler.py

@@ -1,4 +1,3 @@
-#!/usr/bin/env python
 """
    class for handling configuration data files
 

bitbake/lib/bb/parse/parse_py/__init__.py

@@ -1,4 +1,3 @@
-#!/usr/bin/env python
 """
 BitBake Parsers
 

bitbake/lib/bb/progress.py

@@ -13,6 +13,7 @@ import time
 import inspect
 import bb.event
 import bb.build
+from bb.build import StdoutNoopContextManager
 
 class ProgressHandler(object):
     """
@@ -27,7 +28,14 @@ class ProgressHandler(object):
         if outfile:
             self._outfile = outfile
         else:
-            self._outfile = sys.stdout
+            self._outfile = StdoutNoopContextManager()
+
+    def __enter__(self):
+        self._outfile.__enter__()
+        return self
+
+    def __exit__(self, *excinfo):
+        self._outfile.__exit__(*excinfo)
 
     def _fire_progress(self, taskprogress, rate=None):
         """Internal function to fire the progress event"""
@@ -147,6 +155,12 @@ class MultiStageProgressReporter(object):
             self._stage_total = None
             self._callers = []
 
+    def __enter__(self):
+        return self
+
+    def __exit__(self, *excinfo):
+        pass
+
     def _fire_progress(self, taskprogress):
         bb.event.fire(bb.build.TaskProgress(taskprogress), self._data)
 

bitbake/lib/bb/siggen.py

@@ -49,7 +49,9 @@ class SignatureGenerator(object):
         return self.taskhash[task]
 
     def get_taskhash(self, fn, task, deps, dataCache):
-        return "0"
+        k = fn + "." + task
+        self.taskhash[k] = hashlib.sha256(k.encode("utf-8")).hexdigest()
+        return self.taskhash[k]
 
     def writeout_file_checksum_cache(self):
         """Write/update the file checksum cache onto disk"""
@@ -179,7 +181,7 @@ class SignatureGeneratorBasic(SignatureGenerator):
     def get_taskhash(self, fn, task, deps, dataCache):
 
         mc = ''
-        if fn.startswith('multiconfig:'):
+        if fn.startswith('mc:'):
             mc = fn.split(':')[1]
         k = fn + "." + task
 
@@ -194,7 +196,7 @@ class SignatureGeneratorBasic(SignatureGenerator):
                 depmc = pkgname.split(':')[1]
                 if mc != depmc:
                     continue
-            if dep.startswith("multiconfig:") and not mc:
+            if dep.startswith("mc:") and not mc:
                 continue
             depname = dataCache.pkg_fn[pkgname]
             if not self.rundep_check(fn, recipename, task, dep, depname, dataCache):
@@ -412,13 +414,13 @@ def list_inline_diff(oldlist, newlist, colors=None):
 
 def clean_basepath(a):
     mc = None
-    if a.startswith("multiconfig:"):
+    if a.startswith("mc:"):
         _, mc, a = a.split(":", 2)
     b = a.rsplit("/", 2)[1] + '/' + a.rsplit("/", 2)[2]
     if a.startswith("virtual:"):
         b = b + ":" + a.rsplit(":", 1)[0]
     if mc:
-        b = b + ":multiconfig:" + mc
+        b = b + ":mc:" + mc
     return b
 
 def clean_basepaths(a):
@@ -643,9 +645,9 @@ def compare_sigfiles(a, b, recursecb=None, color=False, collapsed=False):
     a_taint = a_data.get('taint', None)
     b_taint = b_data.get('taint', None)
     if a_taint != b_taint:
-        if a_taint.startswith('nostamp:'):
+        if a_taint and a_taint.startswith('nostamp:'):
             a_taint = a_taint.replace('nostamp:', 'nostamp(uuid4):')
-        if b_taint.startswith('nostamp:'):
+        if b_taint and b_taint.startswith('nostamp:'):
             b_taint = b_taint.replace('nostamp:', 'nostamp(uuid4):')
         output.append(color_format("{color_title}Taint (by forced/invalidated task) changed{color_default} from %s to %s") % (a_taint, b_taint))
 

bitbake/lib/bb/taskdata.py

@@ -1,4 +1,3 @@
-#!/usr/bin/env python
 """
 BitBake 'TaskData' implementation
 
@@ -81,7 +80,7 @@ class TaskData:
         def add_mcdepends(task):
             for dep in task_deps['mcdepends'][task].split():
                 if len(dep.split(':')) != 5:
-                    bb.msg.fatal("TaskData", "Error for %s:%s[%s], multiconfig dependency %s does not contain exactly four  ':' characters.\n Task '%s' should be specified in the form 'multiconfig:fromMC:toMC:packagename:task'" % (fn, task, 'mcdepends', dep, 'mcdepends'))
+                    bb.msg.fatal("TaskData", "Error for %s:%s[%s], multiconfig dependency %s does not contain exactly four  ':' characters.\n Task '%s' should be specified in the form 'mc:fromMC:toMC:packagename:task'" % (fn, task, 'mcdepends', dep, 'mcdepends'))
                 if dep not in self.mcdepends:
                     self.mcdepends.append(dep)
 

bitbake/lib/bb/tests/event.py

@@ -561,14 +561,6 @@ class EventClassesTest(unittest.TestCase):
         self.assertEqual(event.fn(1), callback(1))
         self.assertEqual(event.pid, EventClassesTest._worker_pid)
 
-    def test_StampUpdate(self):
-        targets = ["foo", "bar"]
-        stampfns = [lambda:"foobar"]
-        event = bb.event.StampUpdate(targets, stampfns)
-        self.assertEqual(event.targets, targets)
-        self.assertEqual(event.stampPrefix, stampfns)
-        self.assertEqual(event.pid, EventClassesTest._worker_pid)
-
     def test_BuildBase(self):
         """ Test base class for bitbake build events """
         name = "foo"

bitbake/lib/bb/tests/fetch.py

@@ -899,6 +899,7 @@ class FetcherNetworkTest(FetcherTest):
         if os.path.exists(os.path.join(repo_path, 'bitbake-gitsm-test1')):
             self.assertTrue(os.path.exists(os.path.join(repo_path, 'bitbake-gitsm-test1', 'bitbake')), msg='submodule of submodule missing')
 
+    @skipIfNoNetwork()
     def test_git_submodule_dbus_broker(self):
         # The following external repositories have show failures in fetch and unpack operations
         # We want to avoid regressions!
@@ -916,6 +917,7 @@ class FetcherNetworkTest(FetcherTest):
         self.assertTrue(os.path.exists(os.path.join(repo_path, '.git/modules/subprojects/c-sundry/config')), msg='Missing submodule config "subprojects/c-sundry"')
         self.assertTrue(os.path.exists(os.path.join(repo_path, '.git/modules/subprojects/c-utf8/config')), msg='Missing submodule config "subprojects/c-utf8"')
 
+    @skipIfNoNetwork()
     def test_git_submodule_CLI11(self):
         url = "gitsm://github.com/CLIUtils/CLI11;protocol=git;rev=bd4dc911847d0cde7a6b41dfa626a85aab213baf"
         fetcher = bb.fetch.Fetch([url], self.d)
@@ -929,6 +931,7 @@ class FetcherNetworkTest(FetcherTest):
         self.assertTrue(os.path.exists(os.path.join(repo_path, '.git/modules/extern/json/config')), msg='Missing submodule config "extern/json"')
         self.assertTrue(os.path.exists(os.path.join(repo_path, '.git/modules/extern/sanitizers/config')), msg='Missing submodule config "extern/sanitizers"')
 
+    @skipIfNoNetwork()
     def test_git_submodule_update_CLI11(self):
         """ Prevent regression on update detection not finding missing submodule, or modules without needed commits """
         url = "gitsm://github.com/CLIUtils/CLI11;protocol=git;rev=cf6a99fa69aaefe477cc52e3ef4a7d2d7fa40714"
@@ -948,6 +951,7 @@ class FetcherNetworkTest(FetcherTest):
         self.assertTrue(os.path.exists(os.path.join(repo_path, '.git/modules/extern/json/config')), msg='Missing submodule config "extern/json"')
         self.assertTrue(os.path.exists(os.path.join(repo_path, '.git/modules/extern/sanitizers/config')), msg='Missing submodule config "extern/sanitizers"')
 
+    @skipIfNoNetwork()
     def test_git_submodule_aktualizr(self):
         url = "gitsm://github.com/advancedtelematic/aktualizr;branch=master;protocol=git;rev=d00d1a04cc2366d1a5f143b84b9f507f8bd32c44"
         fetcher = bb.fetch.Fetch([url], self.d)
@@ -964,6 +968,7 @@ class FetcherNetworkTest(FetcherTest):
         self.assertTrue(os.path.exists(os.path.join(repo_path, '.git/modules/third_party/googletest/config')), msg='Missing submodule config "third_party/googletest/config"')
         self.assertTrue(os.path.exists(os.path.join(repo_path, '.git/modules/third_party/HdrHistogram_c/config')), msg='Missing submodule config "third_party/HdrHistogram_c/config"')
 
+    @skipIfNoNetwork()
     def test_git_submodule_iotedge(self):
         """ Prevent regression on deeply nested submodules not being checked out properly, even though they were fetched. """
 

bitbake/lib/bb/tests/runqueue-tests/classes/base.bbclass

@@ -0,0 +1,242 @@
+SLOWTASKS ??= ""
+SSTATEVALID ??= ""
+
+def stamptask(d):
+    import time
+
+    thistask = d.expand("${PN}:${BB_CURRENTTASK}")
+    with open(d.expand("${TOPDIR}/%s.run") % thistask, "a+") as f:
+        f.write("\n")
+
+    if d.getVar("BB_CURRENT_MC") != "default":
+        thistask = d.expand("${BB_CURRENT_MC}:${PN}:${BB_CURRENTTASK}")
+    if thistask in d.getVar("SLOWTASKS").split():
+        bb.note("Slowing task %s" % thistask)
+        time.sleep(0.5)
+
+    with open(d.expand("${TOPDIR}/task.log"), "a+") as f:
+        f.write(thistask + "\n")
+
+python do_fetch() {
+    # fetch
+    stamptask(d)
+}
+python do_unpack() {
+    # unpack
+    stamptask(d)
+}
+python do_patch() {
+    # patch
+    stamptask(d)
+}
+python do_populate_lic() {
+    # populate_lic
+    stamptask(d)
+}
+python do_prepare_recipe_sysroot() {
+    # prepare_recipe_sysroot
+    stamptask(d)
+}
+python do_configure() {
+    # configure
+    stamptask(d)
+}
+python do_compile() {
+    # compile
+    stamptask(d)
+}
+python do_install() {
+    # install
+    stamptask(d)
+}
+python do_populate_sysroot() {
+    # populate_sysroot
+    stamptask(d)
+}
+python do_package() {
+    # package
+    stamptask(d)
+}
+python do_package_write_ipk() {
+    # package_write_ipk
+    stamptask(d)
+}
+python do_package_write_rpm() {
+    # package_write_rpm
+    stamptask(d)
+}
+python do_packagedata() {
+    # packagedata
+    stamptask(d)
+}
+python do_package_qa() {
+    # package_qa
+    stamptask(d)
+}
+python do_build() {
+    # build
+    stamptask(d)
+}
+do_prepare_recipe_sysroot[deptask] = "do_populate_sysroot"
+do_package[deptask] += "do_packagedata"
+do_build[recrdeptask] += "do_deploy"
+do_build[recrdeptask] += "do_package_write_ipk"
+do_build[recrdeptask] += "do_package_write_rpm"
+do_package_qa[rdeptask] = "do_packagedata"
+do_populate_lic_deploy[recrdeptask] += "do_populate_lic do_deploy"
+
+DEBIANRDEP = "do_packagedata"
+oo_package_write_ipk[rdeptask] = "${DEBIANRDEP}"
+do_package_write_rpm[rdeptask] = "${DEBIANRDEP}"
+
+addtask fetch
+addtask unpack after do_fetch
+addtask patch after do_unpack
+addtask prepare_recipe_sysroot after do_patch
+addtask configure after do_prepare_recipe_sysroot
+addtask compile after do_configure
+addtask install after do_compile
+addtask populate_sysroot after do_install
+addtask package after do_install
+addtask package_write_ipk after do_packagedata do_package
+addtask package_write_rpm after do_packagedata do_package
+addtask packagedata after do_package
+addtask package_qa after do_package
+addtask build after do_package_qa do_package_write_rpm do_package_write_ipk do_populate_sysroot
+
+python do_package_setscene() {
+    stamptask(d)
+}
+python do_package_qa_setscene() {
+    stamptask(d)
+}
+python do_package_write_ipk_setscene() {
+    stamptask(d)
+}
+python do_package_write_rpm_setscene() {
+    stamptask(d)
+}
+python do_packagedata_setscene() {
+    stamptask(d)
+}
+python do_populate_lic_setscene() {
+    stamptask(d)
+}
+python do_populate_sysroot_setscene() {
+    stamptask(d)
+}
+
+addtask package_setscene
+addtask package_qa_setscene
+addtask package_write_ipk_setscene
+addtask package_write_rpm_setscene
+addtask packagedata_setscene
+addtask populate_lic_setscene
+addtask populate_sysroot_setscene
+
+BB_SETSCENE_DEPVALID = "setscene_depvalid"
+
+def setscene_depvalid(task, taskdependees, notneeded, d, log=None):
+    # taskdependees is a dict of tasks which depend on task, each being a 3 item list of [PN, TASKNAME, FILENAME]
+    # task is included in taskdependees too
+    # Return - False - We need this dependency
+    #        - True - We can skip this dependency
+    import re
+
+    def logit(msg, log):
+        if log is not None:
+            log.append(msg)
+        else:
+            bb.debug(2, msg)
+
+    logit("Considering setscene task: %s" % (str(taskdependees[task])), log)
+
+    def isNativeCross(x):
+        return x.endswith("-native") or "-cross-" in x or "-crosssdk" in x or x.endswith("-cross")
+
+    # We only need to trigger populate_lic through direct dependencies
+    if taskdependees[task][1] == "do_populate_lic":
+        return True
+
+    # We only need to trigger packagedata through direct dependencies
+    # but need to preserve packagedata on packagedata links
+    if taskdependees[task][1] == "do_packagedata":
+        for dep in taskdependees:
+            if taskdependees[dep][1] == "do_packagedata":
+                return False
+        return True
+
+    for dep in taskdependees:
+        logit("  considering dependency: %s" % (str(taskdependees[dep])), log)
+        if task == dep:
+            continue
+        if dep in notneeded:
+            continue
+        # do_package_write_* and do_package doesn't need do_package
+        if taskdependees[task][1] == "do_package" and taskdependees[dep][1] in ['do_package', 'do_package_write_ipk', 'do_package_write_rpm', 'do_packagedata', 'do_package_qa']:
+            continue
+        # do_package_write_* need do_populate_sysroot as they're mainly postinstall dependencies
+        if taskdependees[task][1] == "do_populate_sysroot" and taskdependees[dep][1] in ['do_package_write_ipk', 'do_package_write_rpm']:
+            return False
+        # do_package/packagedata/package_qa don't need do_populate_sysroot
+        if taskdependees[task][1] == "do_populate_sysroot" and taskdependees[dep][1] in ['do_package', 'do_packagedata', 'do_package_qa']:
+            continue
+        # Native/Cross packages don't exist and are noexec anyway
+        if isNativeCross(taskdependees[dep][0]) and taskdependees[dep][1] in ['do_package_write_ipk', 'do_package_write_rpm', 'do_packagedata', 'do_package', 'do_package_qa']:
+            continue
+
+        # This is due to the [depends] in useradd.bbclass complicating matters
+        # The logic *is* reversed here due to the way hard setscene dependencies are injected
+        if (taskdependees[task][1] == 'do_package' or taskdependees[task][1] == 'do_populate_sysroot') and taskdependees[dep][0].endswith(('shadow-native', 'shadow-sysroot', 'base-passwd', 'pseudo-native')) and taskdependees[dep][1] == 'do_populate_sysroot':
+            continue
+
+        # Consider sysroot depending on sysroot tasks
+        if taskdependees[task][1] == 'do_populate_sysroot' and taskdependees[dep][1] == 'do_populate_sysroot':
+            # Native/Cross populate_sysroot need their dependencies
+            if isNativeCross(taskdependees[task][0]) and isNativeCross(taskdependees[dep][0]):
+                return False
+            # Target populate_sysroot depended on by cross tools need to be installed
+            if isNativeCross(taskdependees[dep][0]):
+                return False
+            # Native/cross tools depended upon by target sysroot are not needed
+            # Add an exception for shadow-native as required by useradd.bbclass
+            if isNativeCross(taskdependees[task][0]) and taskdependees[task][0] != 'shadow-native':
+                continue
+            # Target populate_sysroot need their dependencies
+            return False
+
+
+        if taskdependees[dep][1] == "do_populate_lic":
+            continue
+
+        # Safe fallthrough default
+        logit(" Default setscene dependency fall through due to dependency: %s" % (str(taskdependees[dep])), log)
+        return False
+    return True
+
+BB_HASHCHECK_FUNCTION = "sstate_checkhashes"
+
+def sstate_checkhashes(sq_fn, sq_task, sq_hash, sq_hashfn, d, siginfo=False, *, sq_unihash=None):
+
+    ret = []
+    missed = []
+
+    valid = d.getVar("SSTATEVALID").split()
+
+    for task in range(len(sq_fn)):
+        n = os.path.basename(sq_fn[task]).rsplit(".", 1)[0] + ":" + sq_task[task]
+        if n in valid:
+            bb.note("SState: Found valid sstate for %s" % n)
+            ret.append(task)
+        elif os.path.exists(d.expand("${TOPDIR}/%s.run" % n.replace("do_", ""))):
+            bb.note("SState: Found valid sstate for %s (already run)" % n)
+            ret.append(task)
+        else:
+            missed.append(task)
+            bb.note("SState: Found no valid sstate for %s" % n)
+
+    if hasattr(bb.parse.siggen, "checkhashes"):
+        bb.parse.siggen.checkhashes(missed, ret, sq_fn, sq_task, sq_hash, sq_hashfn, d)
+
+    return ret
+

bitbake/lib/bb/tests/runqueue-tests/classes/image.bbclass

@@ -0,0 +1,5 @@
+do_rootfs[recrdeptask] += "do_package_write_deb do_package_qa"
+do_rootfs[recrdeptask] += "do_package_write_ipk do_package_qa"
+do_rootfs[recrdeptask] += "do_package_write_rpm do_package_qa
+do_rootfs[recrdeptask] += "do_packagedata"
+do_rootfs[recrdeptask] += "do_populate_lic"

bitbake/lib/bb/tests/runqueue-tests/classes/native.bbclass

@@ -0,0 +1,2 @@
+RECIPERDEPTASK = "do_populate_sysroot"
+do_populate_sysroot[rdeptask] = "${RECIPERDEPTASK}"

bitbake/lib/bb/tests/runqueue-tests/conf/bitbake.conf

@@ -0,0 +1,16 @@
+CACHE = "${TOPDIR}/cache"
+THISDIR = "${@os.path.dirname(d.getVar('FILE'))}"
+COREBASE := "${@os.path.normpath(os.path.dirname(d.getVar('FILE')+'/../../'))}"
+BBFILES = "${COREBASE}/recipes/*.bb"
+PROVIDES = "${PN}"
+PN = "${@bb.parse.vars_from_file(d.getVar('FILE', False),d)[0]}"
+PF = "${BB_CURRENT_MC}:${PN}"
+export PATH
+TMPDIR ??= "${TOPDIR}"
+STAMP = "${TMPDIR}/stamps/${PN}"
+T = "${TMPDIR}/workdir/${PN}/temp"
+BB_NUMBER_THREADS = "4"
+
+BB_HASHBASE_WHITELIST = "BB_CURRENT_MC"
+
+include conf/multiconfig/${BB_CURRENT_MC}.conf

bitbake/lib/bb/tests/runqueue-tests/conf/multiconfig/mc1.conf

@@ -0,0 +1 @@
+TMPDIR = "${TOPDIR}/mc1/"

bitbake/lib/bb/tests/runqueue-tests/conf/multiconfig/mc2.conf

@@ -0,0 +1 @@
+TMPDIR = "${TOPDIR}/mc2/"

bitbake/lib/bb/tests/runqueue-tests/recipes/b1.bb

@@ -0,0 +1 @@
+DEPENDS = "a1"

bitbake/lib/bb/tests/runqueue-tests/recipes/d1.bb

@@ -0,0 +1,3 @@
+DEPENDS = "a1"
+
+do_package_setscene[depends] = "a1:do_populate_sysroot_setscene"

bitbake/lib/bb/tests/runqueue.py

@@ -0,0 +1,228 @@
+#
+# BitBake Tests for runqueue task processing
+#
+# Copyright (C) 2019 Richard Purdie
+#
+# SPDX-License-Identifier: GPL-2.0-only
+#
+
+import unittest
+import bb
+import os
+import tempfile
+import subprocess
+
+#
+# TODO:
+# Add tests on task ordering (X happens before Y after Z)
+#
+
+class RunQueueTests(unittest.TestCase):
+
+    alltasks = ['package', 'fetch', 'unpack', 'patch', 'prepare_recipe_sysroot', 'configure',
+                'compile', 'install', 'packagedata', 'package_qa', 'package_write_rpm', 'package_write_ipk',
+                'populate_sysroot', 'build']
+    a1_sstatevalid = "a1:do_package a1:do_package_qa a1:do_packagedata a1:do_package_write_ipk a1:do_package_write_rpm a1:do_populate_lic a1:do_populate_sysroot"
+    b1_sstatevalid = "b1:do_package b1:do_package_qa b1:do_packagedata b1:do_package_write_ipk b1:do_package_write_rpm b1:do_populate_lic b1:do_populate_sysroot"
+
+    def run_bitbakecmd(self, cmd, builddir, sstatevalid="", slowtasks="", extraenv=None):
+        env = os.environ.copy()
+        env["BBPATH"] = os.path.realpath(os.path.join(os.path.dirname(__file__), "runqueue-tests"))
+        env["BB_ENV_EXTRAWHITE"] = "SSTATEVALID SLOWTASKS"
+        env["SSTATEVALID"] = sstatevalid
+        env["SLOWTASKS"] = slowtasks
+        if extraenv:
+            for k in extraenv:
+                env[k] = extraenv[k]
+                env["BB_ENV_EXTRAWHITE"] = env["BB_ENV_EXTRAWHITE"] + " " + k
+        try:
+            output = subprocess.check_output(cmd, env=env, stderr=subprocess.STDOUT,universal_newlines=True, cwd=builddir)
+        except subprocess.CalledProcessError as e:
+            self.fail("Command %s failed with %s" % (cmd, e.output))
+        tasks = []
+        with open(builddir + "/task.log", "r") as f:
+            tasks = [line.rstrip() for line in f]
+        return tasks
+
+    def test_no_setscenevalid(self):
+        with tempfile.TemporaryDirectory(prefix="runqueuetest") as tempdir:
+            cmd = ["bitbake", "a1"]
+            sstatevalid = ""
+            tasks = self.run_bitbakecmd(cmd, tempdir, sstatevalid)
+            expected = ['a1:' + x for x in self.alltasks]
+            self.assertEqual(set(tasks), set(expected))
+
+    def test_single_setscenevalid(self):
+        with tempfile.TemporaryDirectory(prefix="runqueuetest") as tempdir:
+            cmd = ["bitbake", "a1"]
+            sstatevalid = "a1:do_package"
+            tasks = self.run_bitbakecmd(cmd, tempdir, sstatevalid)
+            expected = ['a1:package_setscene', 'a1:fetch', 'a1:unpack', 'a1:patch', 'a1:prepare_recipe_sysroot', 'a1:configure',
+                        'a1:compile', 'a1:install', 'a1:packagedata', 'a1:package_qa', 'a1:package_write_rpm', 'a1:package_write_ipk',
+                        'a1:populate_sysroot', 'a1:build']
+            self.assertEqual(set(tasks), set(expected))
+
+    def test_intermediate_setscenevalid(self):
+        with tempfile.TemporaryDirectory(prefix="runqueuetest") as tempdir:
+            cmd = ["bitbake", "a1"]
+            sstatevalid = "a1:do_package a1:do_populate_sysroot"
+            tasks = self.run_bitbakecmd(cmd, tempdir, sstatevalid)
+            expected = ['a1:package_setscene', 'a1:packagedata', 'a1:package_qa', 'a1:package_write_rpm', 'a1:package_write_ipk',
+                        'a1:populate_sysroot_setscene', 'a1:build']
+            self.assertEqual(set(tasks), set(expected))
+
+    def test_intermediate_notcovered(self):
+        with tempfile.TemporaryDirectory(prefix="runqueuetest") as tempdir:
+            cmd = ["bitbake", "a1"]
+            sstatevalid = "a1:do_package_qa a1:do_packagedata a1:do_package_write_ipk a1:do_package_write_rpm a1:do_populate_lic a1:do_populate_sysroot"
+            tasks = self.run_bitbakecmd(cmd, tempdir, sstatevalid)
+            expected = ['a1:package_write_ipk_setscene', 'a1:package_write_rpm_setscene', 'a1:packagedata_setscene',
+                        'a1:package_qa_setscene', 'a1:build', 'a1:populate_sysroot_setscene']
+            self.assertEqual(set(tasks), set(expected))
+
+    def test_all_setscenevalid(self):
+        with tempfile.TemporaryDirectory(prefix="runqueuetest") as tempdir:
+            cmd = ["bitbake", "a1"]
+            sstatevalid = self.a1_sstatevalid
+            tasks = self.run_bitbakecmd(cmd, tempdir, sstatevalid)
+            expected = ['a1:package_write_ipk_setscene', 'a1:package_write_rpm_setscene', 'a1:packagedata_setscene',
+                        'a1:package_qa_setscene', 'a1:build', 'a1:populate_sysroot_setscene']
+            self.assertEqual(set(tasks), set(expected))
+
+    def test_no_settasks(self):
+        with tempfile.TemporaryDirectory(prefix="runqueuetest") as tempdir:
+            cmd = ["bitbake", "a1", "-c", "patch"]
+            sstatevalid = self.a1_sstatevalid
+            tasks = self.run_bitbakecmd(cmd, tempdir, sstatevalid)
+            expected = ['a1:fetch', 'a1:unpack', 'a1:patch']
+            self.assertEqual(set(tasks), set(expected))
+
+    def test_mix_covered_notcovered(self):
+        with tempfile.TemporaryDirectory(prefix="runqueuetest") as tempdir:
+            cmd = ["bitbake", "a1:do_patch", "a1:do_populate_sysroot"]
+            sstatevalid = self.a1_sstatevalid
+            tasks = self.run_bitbakecmd(cmd, tempdir, sstatevalid)
+            expected = ['a1:fetch', 'a1:unpack', 'a1:patch', 'a1:populate_sysroot_setscene']
+            self.assertEqual(set(tasks), set(expected))
+
+
+    # Test targets with intermediate setscene tasks alongside a target with no intermediate setscene tasks
+    def test_mixed_direct_tasks_setscene_tasks(self):
+        with tempfile.TemporaryDirectory(prefix="runqueuetest") as tempdir:
+            cmd = ["bitbake", "c1:do_patch", "a1"]
+            sstatevalid = self.a1_sstatevalid
+            tasks = self.run_bitbakecmd(cmd, tempdir, sstatevalid)
+            expected = ['c1:fetch', 'c1:unpack', 'c1:patch', 'a1:package_write_ipk_setscene', 'a1:package_write_rpm_setscene', 'a1:packagedata_setscene',
+                        'a1:package_qa_setscene', 'a1:build', 'a1:populate_sysroot_setscene']
+            self.assertEqual(set(tasks), set(expected))
+
+    # This test slows down the execution of do_package_setscene until after other real tasks have
+    # started running which tests for a bug where tasks were being lost from the buildable list of real
+    # tasks if they weren't in tasks_covered or tasks_notcovered
+    def test_slow_setscene(self):
+        with tempfile.TemporaryDirectory(prefix="runqueuetest") as tempdir:
+            cmd = ["bitbake", "a1"]
+            sstatevalid = "a1:do_package"
+            slowtasks = "a1:package_setscene"
+            tasks = self.run_bitbakecmd(cmd, tempdir, sstatevalid, slowtasks)
+            expected = ['a1:package_setscene', 'a1:fetch', 'a1:unpack', 'a1:patch', 'a1:prepare_recipe_sysroot', 'a1:configure',
+                        'a1:compile', 'a1:install', 'a1:packagedata', 'a1:package_qa', 'a1:package_write_rpm', 'a1:package_write_ipk',
+                        'a1:populate_sysroot', 'a1:build']
+            self.assertEqual(set(tasks), set(expected))
+
+    def test_setscenewhitelist(self):
+        with tempfile.TemporaryDirectory(prefix="runqueuetest") as tempdir:
+            cmd = ["bitbake", "a1"]
+            extraenv = {
+                "BB_SETSCENE_ENFORCE" : "1",
+                "BB_SETSCENE_ENFORCE_WHITELIST" : "a1:do_package_write_rpm a1:do_build"
+            }
+            sstatevalid = "a1:do_package a1:do_package_qa a1:do_packagedata a1:do_package_write_ipk a1:do_populate_lic a1:do_populate_sysroot"
+            tasks = self.run_bitbakecmd(cmd, tempdir, sstatevalid, extraenv=extraenv)
+            expected = ['a1:packagedata_setscene', 'a1:package_qa_setscene', 'a1:package_write_ipk_setscene',
+                        'a1:populate_sysroot_setscene', 'a1:package_setscene']
+            self.assertEqual(set(tasks), set(expected))
+
+    # Tests for problems with dependencies between setscene tasks
+    def test_no_setscenevalid_harddeps(self):
+        with tempfile.TemporaryDirectory(prefix="runqueuetest") as tempdir:
+            cmd = ["bitbake", "d1"]
+            sstatevalid = ""
+            tasks = self.run_bitbakecmd(cmd, tempdir, sstatevalid)
+            expected = ['a1:package', 'a1:fetch', 'a1:unpack', 'a1:patch', 'a1:prepare_recipe_sysroot', 'a1:configure',
+                        'a1:compile', 'a1:install', 'a1:packagedata', 'a1:package_write_rpm', 'a1:package_write_ipk',
+                        'a1:populate_sysroot', 'd1:package', 'd1:fetch', 'd1:unpack', 'd1:patch', 'd1:prepare_recipe_sysroot', 'd1:configure',
+                        'd1:compile', 'd1:install', 'd1:packagedata', 'd1:package_qa', 'd1:package_write_rpm', 'd1:package_write_ipk',
+                        'd1:populate_sysroot', 'd1:build']
+            self.assertEqual(set(tasks), set(expected))
+
+    def test_no_setscenevalid_withdeps(self):
+        with tempfile.TemporaryDirectory(prefix="runqueuetest") as tempdir:
+            cmd = ["bitbake", "b1"]
+            sstatevalid = ""
+            tasks = self.run_bitbakecmd(cmd, tempdir, sstatevalid)
+            expected = ['a1:' + x for x in self.alltasks] + ['b1:' + x for x in self.alltasks]
+            expected.remove('a1:build')
+            expected.remove('a1:package_qa')
+            self.assertEqual(set(tasks), set(expected))
+
+    def test_single_a1_setscenevalid_withdeps(self):
+        with tempfile.TemporaryDirectory(prefix="runqueuetest") as tempdir:
+            cmd = ["bitbake", "b1"]
+            sstatevalid = "a1:do_package"
+            tasks = self.run_bitbakecmd(cmd, tempdir, sstatevalid)
+            expected = ['a1:package_setscene', 'a1:fetch', 'a1:unpack', 'a1:patch', 'a1:prepare_recipe_sysroot', 'a1:configure',
+                        'a1:compile', 'a1:install', 'a1:packagedata', 'a1:package_write_rpm', 'a1:package_write_ipk',
+                        'a1:populate_sysroot'] + ['b1:' + x for x in self.alltasks]
+            self.assertEqual(set(tasks), set(expected))
+
+    def test_single_b1_setscenevalid_withdeps(self):
+        with tempfile.TemporaryDirectory(prefix="runqueuetest") as tempdir:
+            cmd = ["bitbake", "b1"]
+            sstatevalid = "b1:do_package"
+            tasks = self.run_bitbakecmd(cmd, tempdir, sstatevalid)
+            expected = ['a1:package', 'a1:fetch', 'a1:unpack', 'a1:patch', 'a1:prepare_recipe_sysroot', 'a1:configure',
+                        'a1:compile', 'a1:install', 'a1:packagedata', 'a1:package_write_rpm', 'a1:package_write_ipk',
+                        'a1:populate_sysroot', 'b1:package_setscene'] + ['b1:' + x for x in self.alltasks]
+            expected.remove('b1:package')
+            self.assertEqual(set(tasks), set(expected))
+
+    def test_intermediate_setscenevalid_withdeps(self):
+        with tempfile.TemporaryDirectory(prefix="runqueuetest") as tempdir:
+            cmd = ["bitbake", "b1"]
+            sstatevalid = "a1:do_package a1:do_populate_sysroot b1:do_package"
+            tasks = self.run_bitbakecmd(cmd, tempdir, sstatevalid)
+            expected = ['a1:package_setscene', 'a1:packagedata', 'a1:package_write_rpm', 'a1:package_write_ipk',
+                        'a1:populate_sysroot_setscene', 'b1:package_setscene'] + ['b1:' + x for x in self.alltasks]
+            expected.remove('b1:package')
+            self.assertEqual(set(tasks), set(expected))
+
+    def test_all_setscenevalid_withdeps(self):
+        with tempfile.TemporaryDirectory(prefix="runqueuetest") as tempdir:
+            cmd = ["bitbake", "b1"]
+            sstatevalid = self.a1_sstatevalid + " " + self.b1_sstatevalid
+            tasks = self.run_bitbakecmd(cmd, tempdir, sstatevalid)
+            expected = ['a1:package_write_ipk_setscene', 'a1:package_write_rpm_setscene', 'a1:packagedata_setscene',
+                        'b1:build', 'a1:populate_sysroot_setscene', 'b1:package_write_ipk_setscene', 'b1:package_write_rpm_setscene',
+                        'b1:packagedata_setscene', 'b1:package_qa_setscene', 'b1:populate_sysroot_setscene']
+            self.assertEqual(set(tasks), set(expected))
+
+    def test_multiconfig_setscene_optimise(self):
+        with tempfile.TemporaryDirectory(prefix="runqueuetest") as tempdir:
+            extraenv = {
+                "BBMULTICONFIG" : "mc1 mc2",
+                "BB_SIGNATURE_HANDLER" : "basic"
+            }
+            cmd = ["bitbake", "b1", "mc:mc1:b1", "mc:mc2:b1"]
+            setscenetasks = ['package_write_ipk_setscene', 'package_write_rpm_setscene', 'packagedata_setscene',
+                             'populate_sysroot_setscene', 'package_qa_setscene']
+            sstatevalid = ""
+            tasks = self.run_bitbakecmd(cmd, tempdir, sstatevalid, extraenv=extraenv)
+            expected = ['a1:' + x for x in self.alltasks] + ['b1:' + x for x in self.alltasks] + \
+                       ['mc1:b1:' + x for x in setscenetasks] + ['mc1:a1:' + x for x in setscenetasks] + \
+                       ['mc2:b1:' + x for x in setscenetasks] + ['mc2:a1:' + x for x in setscenetasks] + \
+                       ['mc1:b1:build', 'mc2:b1:build']
+            for x in ['mc1:a1:package_qa_setscene', 'mc2:a1:package_qa_setscene', 'a1:build', 'a1:package_qa']:
+                expected.remove(x)
+            self.assertEqual(set(tasks), set(expected))
+

bitbake/lib/bb/tests/utils.py

@@ -29,6 +29,10 @@ class VerCmpString(unittest.TestCase):
         self.assertTrue(result < 0)
         result = bb.utils.vercmp_string('1.1', '1.0+1.1-beta1')
         self.assertTrue(result > 0)
+        result = bb.utils.vercmp_string('1a', '1a1')
+        self.assertTrue(result < 0)
+        result = bb.utils.vercmp_string('1a1', '1a')
+        self.assertTrue(result > 0)
         result = bb.utils.vercmp_string('1.', '1.1')
         self.assertTrue(result < 0)
         result = bb.utils.vercmp_string('1.1', '1.')

bitbake/lib/bb/ui/knotty.py

@@ -300,8 +300,8 @@ class TerminalFilter(object):
                         if start_time:
                             pbar.start_time = start_time
                     pbar.setmessage('%s:%s' % (tasknum, pbar.msg.split(':', 1)[1]))
+                    pbar.setextra(rate)
                     if progress > -1:
-                        pbar.setextra(rate)
                         content = pbar.update(progress)
                     else:
                         content = pbar.update(1)
@@ -660,7 +660,6 @@ def main(server, eventHandler, params, tf = TerminalFilter):
             # ignore
             if isinstance(event, (bb.event.BuildBase,
                                   bb.event.MetadataEvent,
-                                  bb.event.StampUpdate,
                                   bb.event.ConfigParsed,
                                   bb.event.MultiConfigParsed,
                                   bb.event.RecipeParsed,

bitbake/lib/bb/ui/uihelper.py

@@ -40,7 +40,7 @@ class BBUIHelper:
             self.running_pids.remove(event.pid)
             self.failed_tasks.append( { 'title' : "%s %s" % (event._package, event._task)})
             self.needUpdate = True
-        elif isinstance(event, bb.runqueue.runQueueTaskStarted) or isinstance(event, bb.runqueue.sceneQueueTaskStarted):
+        elif isinstance(event, bb.runqueue.runQueueTaskStarted):
             self.tasknumber_current = event.stats.completed + event.stats.active + event.stats.failed + 1
             self.tasknumber_total = event.stats.total
             self.needUpdate = True

bitbake/lib/pyinotify.py

@@ -1,5 +1,4 @@
-#!/usr/bin/env python
-
+#
 # pyinotify.py - python interface to inotify
 # Copyright (c) 2005-2015 Sebastien Martini <seb@dbzteam.org>
 #

bitbake/lib/toaster/orm/fixtures/oe-core.xml

@@ -53,7 +53,7 @@
   <object model="orm.release" pk="4">
     <field type="CharField" name="name">thud</field>
     <field type="CharField" name="description">Openembedded Thud</field>
-    <field rel="ManyToOneRel" to="orm.bitbakeversion" name="bitbake_version">1</field>
+    <field rel="ManyToOneRel" to="orm.bitbakeversion" name="bitbake_version">4</field>
     <field type="CharField" name="branch_name">thud</field>
     <field type="TextField" name="helptext">Toaster will run your builds using the tip of the &lt;a href=\"http://cgit.openembedded.org/openembedded-core/log/?h=thud\"&gt;OpenEmbedded Thud&lt;/a&gt; branch.</field>
   </object>

bitbake/lib/toaster/orm/fixtures/poky.xml

@@ -58,7 +58,7 @@
   <object model="orm.release" pk="4">
     <field type="CharField" name="name">thud</field>
     <field type="CharField" name="description">Yocto Project 2.6 "Thud"</field>
-    <field rel="ManyToOneRel" to="orm.bitbakeversion" name="bitbake_version">1</field>
+    <field rel="ManyToOneRel" to="orm.bitbakeversion" name="bitbake_version">4</field>
     <field type="CharField" name="branch_name">thud</field>
     <field type="TextField" name="helptext">Toaster will run your builds using the tip of the &lt;a href="http://git.yoctoproject.org/cgit/cgit.cgi/poky/log/?h=thud"&gt;Yocto Project Thud branch&lt;/a&gt;.</field>
   </object>

bitbake/lib/toaster/tests/browser/selenium_helpers.py

@@ -1,4 +1,4 @@
-#! /usr/bin/env python
+#! /usr/bin/env python3
 #
 # BitBake Toaster Implementation
 #

bitbake/lib/toaster/tests/browser/selenium_helpers_base.py

@@ -1,4 +1,4 @@
-#! /usr/bin/env python
+#! /usr/bin/env python3
 #
 # BitBake Toaster Implementation
 #

bitbake/lib/toaster/tests/browser/test_all_builds_page.py

@@ -1,4 +1,4 @@
-#! /usr/bin/env python
+#! /usr/bin/env python3
 #
 # BitBake Toaster Implementation
 #

bitbake/lib/toaster/tests/browser/test_all_projects_page.py

@@ -1,4 +1,4 @@
-#! /usr/bin/env python
+#! /usr/bin/env python3
 #
 # BitBake Toaster Implementation
 #

bitbake/lib/toaster/tests/browser/test_builddashboard_page.py

@@ -1,4 +1,4 @@
-#! /usr/bin/env python
+#! /usr/bin/env python3
 #
 # BitBake Toaster Implementation
 #

bitbake/lib/toaster/tests/browser/test_builddashboard_page_artifacts.py

@@ -1,4 +1,4 @@
-#! /usr/bin/env python
+#! /usr/bin/env python3
 #
 # BitBake Toaster Implementation
 #

bitbake/lib/toaster/tests/browser/test_builddashboard_page_recipes.py

@@ -1,4 +1,4 @@
-#! /usr/bin/env python
+#! /usr/bin/env python3
 #
 # BitBake Toaster Implementation
 #

bitbake/lib/toaster/tests/browser/test_builddashboard_page_tasks.py

@@ -1,4 +1,4 @@
-#! /usr/bin/env python
+#! /usr/bin/env python3
 #
 # BitBake Toaster Implementation
 #

bitbake/lib/toaster/tests/browser/test_js_unit_tests.py

@@ -1,4 +1,4 @@
-#! /usr/bin/env python
+#! /usr/bin/env python3
 #
 # BitBake Toaster Implementation
 #

bitbake/lib/toaster/tests/browser/test_landing_page.py

@@ -1,4 +1,4 @@
-#! /usr/bin/env python
+#! /usr/bin/env python3
 #
 # BitBake Toaster Implementation
 #

bitbake/lib/toaster/tests/browser/test_layerdetails_page.py

@@ -1,4 +1,4 @@
-#! /usr/bin/env python
+#! /usr/bin/env python3
 #
 # BitBake Toaster Implementation
 #

bitbake/lib/toaster/tests/browser/test_most_recent_builds_states.py

@@ -1,4 +1,4 @@
-#! /usr/bin/env python
+#! /usr/bin/env python3
 #
 # BitBake Toaster Implementation
 #

bitbake/lib/toaster/tests/browser/test_new_custom_image_page.py

@@ -1,4 +1,4 @@
-#! /usr/bin/env python
+#! /usr/bin/env python3
 #
 # BitBake Toaster Implementation
 #

bitbake/lib/toaster/tests/browser/test_new_project_page.py

@@ -1,4 +1,4 @@
-#! /usr/bin/env python
+#! /usr/bin/env python3
 #
 # BitBake Toaster Implementation
 #

bitbake/lib/toaster/tests/browser/test_project_builds_page.py

@@ -1,4 +1,4 @@
-#! /usr/bin/env python
+#! /usr/bin/env python3
 #
 # BitBake Toaster Implementation
 #

bitbake/lib/toaster/tests/browser/test_project_config_page.py

@@ -1,4 +1,4 @@
-#! /usr/bin/env python
+#! /usr/bin/env python3
 #
 # BitBake Toaster Implementation
 #

bitbake/lib/toaster/tests/browser/test_project_page.py

@@ -1,4 +1,4 @@
-#! /usr/bin/env python
+#! /usr/bin/env python3
 #
 # BitBake Toaster Implementation
 #

bitbake/lib/toaster/tests/browser/test_sample.py

@@ -1,4 +1,4 @@
-#! /usr/bin/env python
+#! /usr/bin/env python3
 #
 # BitBake Toaster Implementation
 #

bitbake/lib/toaster/tests/browser/test_task_page.py

@@ -1,4 +1,4 @@
-#! /usr/bin/env python
+#! /usr/bin/env python3
 #
 # BitBake Toaster Implementation
 #

bitbake/lib/toaster/tests/browser/test_toastertable_ui.py

@@ -1,4 +1,4 @@
-#! /usr/bin/env python
+#! /usr/bin/env python3
 #
 # BitBake Toaster Implementation
 #

bitbake/lib/toaster/tests/builds/buildtest.py

@@ -1,4 +1,4 @@
-#! /usr/bin/env python
+#! /usr/bin/env python3
 #
 # BitBake Toaster Implementation
 #

bitbake/lib/toaster/tests/builds/test_core_image_min.py

@@ -1,4 +1,4 @@
-#! /usr/bin/env python
+#! /usr/bin/env python3
 #
 # BitBake Toaster Implementation
 #

bitbake/lib/toaster/tests/commands/test_loaddata.py

@@ -1,4 +1,4 @@
-#! /usr/bin/env python
+#! /usr/bin/env python3
 #
 # BitBake Toaster Implementation
 #

bitbake/lib/toaster/tests/commands/test_lsupdates.py

@@ -1,4 +1,4 @@
-#! /usr/bin/env python
+#! /usr/bin/env python3
 #
 # BitBake Toaster Implementation
 #

bitbake/lib/toaster/tests/commands/test_runbuilds.py

@@ -1,4 +1,4 @@
-#! /usr/bin/env python
+#! /usr/bin/env python3
 #
 # BitBake Toaster Implementation
 #

bitbake/lib/toaster/tests/eventreplay/__init__.py

@@ -1,4 +1,4 @@
-#! /usr/bin/env python
+#! /usr/bin/env python3
 #
 # BitBake Toaster Implementation
 #

bitbake/lib/toaster/tests/functional/functional_helpers.py

@@ -1,4 +1,4 @@
-#! /usr/bin/env python
+#! /usr/bin/env python3
 #
 # BitBake Toaster functional tests implementation
 #

bitbake/lib/toaster/tests/functional/test_functional_basic.py

@@ -1,4 +1,4 @@
-#! /usr/bin/env python
+#! /usr/bin/env python3
 #
 # BitBake Toaster functional tests implementation
 #

bitbake/lib/toaster/tests/views/test_views.py

@@ -1,4 +1,4 @@
-#! /usr/bin/env python
+#! /usr/bin/env python3
 #
 # BitBake Toaster Implementation
 #

bitbake/lib/toaster/toastermain/management/commands/checksocket.py

@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
 #
 # BitBake Toaster Implementation
 #

documentation/Makefile

@@ -156,7 +156,7 @@ TARFILES = dev-style.css dev-manual.html \
 TARFILES = dev-style.css dev-manual.html figures/buildhistory-web.png \
            figures/dev-title.png figures/buildhistory.png \
            figures/recipe-workflow.png figures/bitbake-build-flow.png \
-           figures/multiconfig_files.png
+           figures/multiconfig_files.png figures/cute-files-npm-example.png
 	endif
 
 MANUALS = $(DOC)/$(DOC).html
@@ -260,7 +260,7 @@ TARFILES = mega-manual.html mega-style.css \
 	figures/sdk-devtool-modify-flow.png \
 	figures/sdk-devtool-upgrade-flow.png figures/bitbake-build-flow.png figures/bypqs-title.png \
 	figures/overview-manual-title.png figures/sdk-autotools-flow.png figures/sdk-makefile-flow.png \
-	figures/bb_multiconfig_files.png figures/bitbake-title.png
+	figures/bb_multiconfig_files.png figures/bitbake-title.png figures/cute-files-npm-example.png
 	endif
 
 MANUALS = $(DOC)/$(DOC).html

documentation/dev-manual/dev-manual-common-tasks.xml

@@ -4197,11 +4197,27 @@
             built by layer recipes.
             It is recommended to keep recipes up-to-date with upstream
             version releases.
+        </para>
+
+        <para>
+            While several methods exist that allow you upgrade a recipe,
+            you might consider checking on the upgrade status of a recipe
+            first.
+            You can do so using the
+            <filename>devtool check-upgrade-status</filename> command.
+            See the
+            "<ulink url='&YOCTO_DOCS_REF_URL;#devtool-checking-on-the-upgrade-status-of-a-recipe'>Checking on the Upgrade Status of a Recipe</ulink>"
+            section in the Yocto Project Reference Manual for more information.
+        </para>
+
+        <para>
+            The remainder of this section describes three ways you can
+            upgrade a recipe.
             You can use the Automated Upgrade Helper (AUH) to set up
             automatic version upgrades.
             Alternatively, you can use <filename>devtool upgrade</filename>
             to set up semi-automatic version upgrades.
-            Finally, you can even manually upgrade a recipe by editing the
+            Finally, you can manually upgrade a recipe by editing the
             recipe itself.
         </para>
 
@@ -4319,12 +4335,6 @@
                         directory you just created for AUH.
                         Make these following configurations:
                         <itemizedlist>
-                            <listitem><para>
-                                Enable "distrodata" as follows:
-                                <literallayout class='monospaced'>
-     INHERIT =+ "distrodata"
-                                </literallayout>
-                                </para></listitem>
                             <listitem><para>
                                 If you want to enable
                                 <ulink url='&YOCTO_DOCS_DEV_URL;#maintaining-build-output-quality'>Build History</ulink>,
@@ -7119,7 +7129,8 @@
             This problem is solved with the OpenEmbedded build system by
             running the code through QEMU, which allows precisely that.
             Unfortunately, QEMU does not always work perfectly as mentioned
-            in the xxx section.
+            in the
+            "<link linkend='known-issues'>Known Issues</link>" section.
         </para>
 
         <section id='enabling-the-generation-of-introspection-data'>
@@ -8964,11 +8975,17 @@
                     <link linkend='handling-optional-module-packaging'>Handling optional module packaging</link>
                     </para></listitem>
                 <listitem><para>
-                    <link linkend='using-runtime-package-management'>Using Runtime Package Management</link>
+                    <link linkend='using-runtime-package-management'>Using runtime package management</link>
+                    </para></listitem>
+                <listitem><para>
+                    <link linkend='generating-and-using-signed-packages'>Generating and using signed packages</link>
                     </para></listitem>
                 <listitem><para>
                     <link linkend='testing-packages-with-ptest'>Setting up and running package test (ptest)</link>
                     </para></listitem>
+                <listitem><para>
+                    <link linkend='creating-node-package-manager-npm-packages'>Creating node package manager (NPM) packages</link>
+                    </para></listitem>
             </itemizedlist>
         </para>
 
@@ -10397,6 +10414,292 @@
                 </para>
             </section>
         </section>
+
+        <section id='creating-node-package-manager-npm-packages'>
+            <title>Creating Node Package Manager (NPM) Packages</title>
+
+            <para>
+                <ulink url='https://en.wikipedia.org/wiki/Npm_(software)'>NPM</ulink>
+                is a package manager for the JavaScript programming
+                language.
+                The Yocto Project supports the NPM
+                <ulink url='&YOCTO_DOCS_BB_URL;#bb-fetchers'>fetcher</ulink>.
+                You can use this fetcher in combination with
+                <ulink url='&YOCTO_DOCS_REF_URL;#ref-devtool-reference'><filename>devtool</filename></ulink>
+                to create recipes that produce NPM packages.
+            </para>
+
+            <para>
+                Two workflows exist that allow you to create NPM packages
+                using <filename>devtool</filename>: the NPM registry modules
+                method and the NPM project code method.
+                <note>
+                    While it is possible to create NPM recipes manually,
+                    using <filename>devtool</filename> is far simpler.
+                </note>
+                Additionally, some requirements and caveats exist.
+            </para>
+
+            <section id='npm-package-creation-requirements'>
+                <title>Requirements and Caveats</title>
+
+                <para>
+                    You need to be aware of the following before using
+                    <filename>devtool</filename> to create NPM packages:
+                    <itemizedlist>
+                        <listitem><para>
+                            Of the two methods that you can use
+                            <filename>devtool</filename> to create NPM
+                            packages, the registry approach is slightly
+                            simpler.
+                            However, you might consider the project
+                            approach because you do not have to publish
+                            your module in the NPM registry
+                            (<ulink url='https://docs.npmjs.com/misc/registry'><filename>npm-registry</filename></ulink>),
+                            which is NPM's public registry.
+                            </para></listitem>
+                        <listitem><para>
+                            Be familiar with
+                            <ulink url='&YOCTO_DOCS_REF_URL;#ref-devtool-reference'><filename>devtool</filename></ulink>.
+                            </para></listitem>
+                        <listitem><para>
+                            The NPM host tools need the native
+                            <filename>nodejs-npm</filename> package, which
+                            is part of the OpenEmbedded environment.
+                            You need to get the package by cloning the
+                            <ulink url='https://github.com/openembedded/meta-openembedded'></ulink>
+                            repository out of GitHub.
+                            Be sure to add the path to your local copy to
+                            your <filename>bblayers.conf</filename> file.
+                            </para></listitem>
+                        <listitem><para>
+                            <filename>devtool</filename> cannot detect
+                            native libraries in module dependencies.
+                            Consequently, you must manually add packages
+                            to your recipe.
+                            </para></listitem>
+                        <listitem><para>
+                            While deploying NPM packages,
+                            <filename>devtool</filename> cannot determine
+                            which dependent packages are missing on the
+                            target (e.g. the node runtime
+                            <filename>nodejs</filename>).
+                            Consequently, you need to find out what
+                            files are missing and be sure they are on the
+                            target.
+                            </para></listitem>
+                        <listitem><para>
+                            Although you might not need NPM to run your
+                            node package, it is useful to have NPM on your
+                            target.
+                            The NPM package name is
+                            <filename>nodejs-npm</filename>.
+                            </para></listitem>
+                    </itemizedlist>
+                </para>
+            </section>
+
+            <section id='npm-using-the-registry-modules-method'>
+                <title>Using the Registry Modules Method</title>
+
+                <para>
+                    This section presents an example that uses the
+                    <filename>cute-files</filename> module, which is a
+                    file browser web application.
+                    <note>
+                        You must know the <filename>cute-files</filename>
+                        module version.
+                    </note>
+                </para>
+
+                <para>
+                    The first thing you need to do is use
+                    <filename>devtool</filename> and the NPM fetcher to
+                    create the recipe:
+                    <literallayout class='monospaced'>
+     $ devtool add "npm://registry.npmjs.org;name=cute-files;version=1.0.2"
+                    </literallayout>
+                    The <filename>devtool add</filename> command runs
+                    <filename>recipetool create</filename> and uses the
+                    same fetch URI to download each dependency and capture
+                    license details where possible.
+                    The result is a generated recipe.
+                </para>
+
+                <para>
+                    The recipe file is fairly simple and contains every
+                    license that <filename>recipetool</filename> finds
+                    and includes the licenses in the recipe's
+                    <ulink url='&YOCTO_DOCS_REF_URL;#var-LIC_FILES_CHKSUM'><filename>LIC_FILES_CHKSUM</filename></ulink>
+                    variables.
+                    You need to examine the variables and look for those
+                    with "unknown" in the
+                    <ulink url='&YOCTO_DOCS_REF_URL;#var-LICENSE'><filename>LICENSE</filename></ulink>
+                    field.
+                    You need to track down the license information for
+                    "unknown" modules and manually add the information to the
+                    recipe.
+                </para>
+
+                <para>
+                    <filename>recipetool</filename> creates "shrinkwrap" and
+                    "lockdown" files for your recipe.
+                    Shrinkwrap files capture the version of all dependent
+                    modules.
+                    Many packages do not provide shrinkwrap files.
+                    <filename>recipetool</filename> create a shrinkwrap
+                    file as it runs.
+                    You can replace the shrinkwrap file with your own file
+                    by setting the <filename>NPM_SHRINKWRAP</filename>
+                    variable.
+                </para>
+
+                <para>
+                    Lockdown files contain the checksum for each module
+                    to determine if your users download the same files when
+                    building with a recipe.
+                    Lockdown files ensure that dependencies have not been
+                    changed and that your NPM registry is still providing
+                    the same file.
+                    <note>
+                        A package is created for each sub-module.
+                        This policy is the only practical way to have the
+                        licenses for all of the dependencies represented
+                        in the license manifest of the image.
+                    </note>
+                </para>
+
+                <para>
+                    The <filename>devtool edit-recipe</filename> command
+                    lets you take a look at the recipe:
+                    <literallayout class='monospaced'>
+     $ devtool edit-recipe cute-files
+     SUMMARY = "Turn any folder on your computer into a cute file browser, available on the local network."
+     LICENSE = "BSD-3-Clause &amp; Unknown &amp; MIT &amp; ISC"
+     LIC_FILES_CHKSUM = "file://LICENSE;md5=71d98c0a1db42956787b1909c74a86ca \
+                         file://node_modules/content-disposition/LICENSE;md5=c6e0ce1e688c5ff16db06b7259e9cd20 \
+                         file://node_modules/express/LICENSE;md5=5513c00a5c36cd361da863dd9aa8875d \
+                         ...
+
+     SRC_URI = "npm://registry.npmjs.org;name=cute-files;version=${PV}"
+     NPM_SHRINKWRAP := "${THISDIR}/${PN}/npm-shrinkwrap.json"
+     NPM_LOCKDOWN := "${THISDIR}/${PN}/lockdown.json"
+     inherit npm
+     # Must be set after inherit npm since that itself sets S
+     S = "${WORKDIR}/npmpkg"
+
+     LICENSE_${PN}-content-disposition = "MIT"
+     ...
+     LICENSE_${PN}-express = "MIT"
+     LICENSE_${PN} = "MIT"
+                    </literallayout>
+                    Three key points exist in the previous example:
+                    <itemizedlist>
+                        <listitem><para>
+                            <ulink url='&YOCTO_DOCS_REF_URL;#var-SRC_URI'><filename>SRC_URI</filename></ulink>
+                            uses the NPM scheme so that the NPM fetcher
+                            is used.
+                            </para></listitem>
+                        <listitem><para>
+                            <filename>recipetool</filename> collects all
+                            the license information.
+                            If a sub-module's license is unavailable,
+                            the sub-module's name appears in the comments.
+                            </para></listitem>
+                        <listitem><para>
+                            The <filename>inherit npm</filename> statement
+                            causes the
+                            <ulink url='&YOCTO_DOCS_REF_URL;#ref-classes-npm'><filename>npm</filename></ulink>
+                            class to package up all the modules.
+                            </para></listitem>
+                    </itemizedlist>
+                </para>
+
+                <para>
+                    You can run the following command to build the
+                    <filename>cute-files</filename> package:
+                    <literallayout class='monospaced'>
+     $ devtool build cute-files
+                    </literallayout>
+                    Remember that <filename>nodejs</filename> must be
+                    installed on the target before your package.
+                </para>
+
+                <para>
+                    Assuming 192.168.7.2 for the target's IP address, use
+                    the following command to deploy your package:
+                    <literallayout class='monospaced'>
+     $ devtool deploy-target -s cute-files root@192.168.7.2
+                    </literallayout>
+                    Once the package is installed on the target, you can
+                    test the application:
+                    <note>
+                        Because of a know issue, you cannot simply run
+                        <filename>cute-files</filename> as you would if you
+                        had run <filename>npm install</filename>.
+                    </note>
+                    <literallayout class='monospaced'>
+     $ cd /usr/lib/node_modules/cute-files
+     $ node cute-files.js
+                    </literallayout>
+                    On a browser, go to
+                    <filename>http://192.168.7.2:3000</filename> and you
+                    see the following:
+                    <imagedata fileref="figures/cute-files-npm-example.png" align="center" width="6in" depth="4in" />
+                </para>
+
+                <para>
+                    You can find the recipe in
+                    <filename>workspace/recipes/cute-files</filename>.
+                    You can use the recipe in any layer you choose.
+                </para>
+            </section>
+
+            <section id='npm-using-the-npm-projects-method'>
+                <title>Using the NPM Projects Code Method</title>
+
+                <para>
+                    Although it is useful to package modules already in the
+                    NPM registry, adding <filename>node.js</filename> projects
+                    under development is a more common developer use case.
+                </para>
+
+                <para>
+                    This section covers the NPM projects code method, which is
+                    very similar to the "registry" approach described in the
+                    previous section.
+                    In the NPM projects method, you provide
+                    <filename>devtool</filename> with an URL that points to the
+                    source files.
+                </para>
+
+                <para>
+                    Replicating the same example, (i.e.
+                    <filename>cute-files</filename>) use the following command:
+                    <literallayout class='monospaced'>
+     $ devtool add https://github.com/martinaglv/cute-files.git
+                    </literallayout>
+                    The recipe this command generates is very similar to the
+                    recipe created in the previous section.
+                    However, the <filename>SRC_URI</filename> looks like the
+                    following:
+                    <literallayout class='monospaced'>
+     SRC_URI = "git://github.com/martinaglv/cute-files.git;protocol=https \
+               npm://registry.npmjs.org;name=commander;version=2.9.0;subdir=node_modules/commander \
+               npm://registry.npmjs.org;name=express;version=4.14.0;subdir=node_modules/express \
+               npm://registry.npmjs.org;name=content-disposition;version=0.3.0;subdir=node_modules/content-disposition \
+               "
+                    </literallayout>
+                    In this example, the main module is taken from the Git
+                    repository and dependents are taken from the NPM registry.
+                    Other than those differences, the recipe is basically the
+                    same between the two methods.
+                    You can build and deploy the package exactly as described
+                    in the previous section that uses the registry modules
+                    method.
+                </para>
+            </section>
+        </section>
     </section>
 
     <section id='efficiently-fetching-source-files-during-a-build'>
@@ -10519,7 +10822,7 @@
             <title>Using systemd Exclusively</title>
 
             <para>
-                Set the these variables in your distribution configuration
+                Set these variables in your distribution configuration
                 file as follows:
                 <literallayout class='monospaced'>
      DISTRO_FEATURES_append = " systemd"

documentation/overview-manual/overview-manual-concepts.xml

@@ -924,17 +924,18 @@
                 <title>Source Control Managers (Optional)</title>
 
                 <para>
-                    Another place the build system can get source files from is
-                    through a Source Control Manager (SCM) such as Git or
-                    Subversion.
+                    Another place from which the build system can get source
+                    files is with
+                    <ulink url='&YOCTO_DOCS_BB_URL;#bb-fetchers'>fetchers</ulink>
+                    employing various Source Control Managers (SCMs) such as
+                    Git or Subversion.
                     In such cases, a repository is cloned or checked out.
                     The
                     <ulink url='&YOCTO_DOCS_REF_URL;#ref-tasks-fetch'><filename>do_fetch</filename></ulink>
                     task inside BitBake uses
                     the <ulink url='&YOCTO_DOCS_REF_URL;#var-SRC_URI'><filename>SRC_URI</filename></ulink>
                     variable and the argument's prefix to determine the correct
-                    <ulink url='&YOCTO_DOCS_BB_URL;#bb-fetchers'><filename>fetcher</filename></ulink>
-                    module.
+                    fetcher module.
                     <note>
                         For information on how to have the OpenEmbedded build
                         system generate tarballs for Git repositories and place

documentation/overview-manual/overview-manual.xml

@@ -36,6 +36,11 @@
                 <date>May 2018</date>
                 <revremark>The initial document released with the Yocto Project 2.5 Release.</revremark>
             </revision>
+            <revision>
+                <revnumber>2.6</revnumber>
+                <date>November 2018</date>
+                <revremark>Released with the Yocto Project 2.7 Release.</revremark>
+            </revision>
             <revision>
                 <revnumber>2.7</revnumber>
                 <date>May 2019</date>

documentation/ref-manual/migration.xml

@@ -2566,9 +2566,7 @@
             <link linkend='oe-core'>OE-Core</link>.
             The change includes <filename>package_regex.inc</filename> and
             <filename>distro_alias.inc</filename>, which are typically enabled
-            when using the
-            <link linkend='ref-classes-distrodata'><filename>distrodata</filename></link>
-            class.
+            when using the <filename>distrodata</filename> class.
             Additionally, the contents of
             <filename>upstream_tracking.inc</filename> has now been split out
             to the relevant recipes.

documentation/ref-manual/ref-classes.xml

@@ -335,18 +335,6 @@
     </para>
 </section>
 
-<section id='ref-classes-bugzilla'>
-    <title><filename>bugzilla.bbclass</filename></title>
-
-    <para>
-        The <filename>bugzilla</filename> class supports setting up an
-        instance of Bugzilla in which you can automatically files bug reports
-        in response to build failures.
-        For this class to work, you need to enable the XML-RPC interface in
-        the instance of Bugzilla.
-    </para>
-</section>
-
 <section id='ref-classes-buildhistory'>
     <title><filename>buildhistory.bbclass</filename></title>
 
@@ -714,39 +702,6 @@
     </para>
 </section>
 
-<section id='ref-classes-distrodata'>
-    <title><filename>distrodata.bbclass</filename></title>
-
-    <para>
-        The <filename>distrodata</filename> class
-        provides for automatic checking for upstream recipe updates.
-        The class creates a comma-separated value (CSV) spreadsheet that
-        contains information about the recipes.
-        The information provides the
-        <link linkend='ref-tasks-distrodata'><filename>do_distrodata</filename></link>
-        and
-        <filename>do_distro_check</filename> tasks, which do upstream checking
-        and also verify if a package is used in multiple major distributions.
-    </para>
-
-    <para>
-        The class is not included by default.
-        To use it, you must set the
-        <link linkend='var-INHERIT'><filename>INHERIT</filename></link>
-        variable:
-        <literallayout class='monospaced'>
-     INHERIT+= "distrodata"
-        </literallayout>
-    </para>
-
-    <para>
-        The <filename>distrodata</filename> class also provides the
-        <link linkend='ref-tasks-checkpkg'><filename>do_checkpkg</filename></link>
-        task, which can be used against a simple recipe or against an
-        image to get all its recipe information.
-    </para>
-</section>
-
 <section id='ref-classes-distutils'>
     <title><filename>distutils*.bbclass</filename></title>
 
@@ -776,11 +731,6 @@
         some of the <filename>distutils*</filename> classes to provide common
         Python2 support.
     </para>
-
-    <para>
-	    The <filename>distutils-tools</filename> class supports recipes for
-        additional "distutils" tools.
-    </para>
 </section>
 
 <section id='ref-classes-distutils3'>
@@ -2335,13 +2285,16 @@ This check was removed for YP 2.3 release
     <title><filename>npm.bbclass</filename></title>
 
     <para>
-        Provides support for building Node.js software fetched using the npm
-        package manager.
+        Provides support for building Node.js software fetched using the
+        <ulink url='https://en.wikipedia.org/wiki/Npm_(software)'>node package manager (NPM)</ulink>.
         <note>
             Currently, recipes inheriting this class must use the
             <filename>npm://</filename> fetcher to have dependencies fetched
             and packaged automatically.
         </note>
+        For information on how to create NPM packages, see the
+        "<ulink url='&YOCTO_DOCS_DEV_URL;#creating-node-package-manager-npm-packages'>Creating Node Package Manager (NPM) Packages</ulink>"
+        section in the Yocto Project Development Tasks Manual.
     </para>
 </section>
 

documentation/ref-manual/ref-devtool-reference.xml

@@ -34,7 +34,7 @@
             You can run <filename>devtool --help</filename> to see all
             the commands:
             <literallayout class='monospaced'>
-     $ devtool --help
+     $ devtool -h
      NOTE: Starting bitbake server...
      usage: devtool [--basepath BASEPATH] [--bbpath BBPATH] [-d] [-q]
                     [--color COLOR] [-h]
@@ -43,50 +43,48 @@
      OpenEmbedded development tool
 
      options:
-       --basepath BASEPATH  Base directory of SDK / build directory
-       --bbpath BBPATH      Explicitly specify the BBPATH, rather than getting it
-                            from the metadata
-       -d, --debug          Enable debug output
-       -q, --quiet          Print only errors
-       --color COLOR        Colorize output (where COLOR is auto, always, never)
-       -h, --help           show this help message and exit
+       --basepath BASEPATH   Base directory of SDK / build directory
+       --bbpath BBPATH       Explicitly specify the BBPATH, rather than getting it
+                             from the metadata
+       -d, --debug           Enable debug output
+       -q, --quiet           Print only errors
+       --color COLOR         Colorize output (where COLOR is auto, always, never)
+       -h, --help            show this help message and exit
 
      subcommands:
        Beginning work on a recipe:
-         add                  Add a new recipe
-         modify               Modify the source for an existing recipe
-         upgrade              Upgrade an existing recipe
+         add                   Add a new recipe
+         modify                Modify the source for an existing recipe
+         upgrade               Upgrade an existing recipe
        Getting information:
-         status               Show workspace status
-         search               Search available recipes
-         latest-version       Report the latest version of an existing recipe
+         status                Show workspace status
+         search                Search available recipes
+         latest-version        Report the latest version of an existing recipe
+         check-upgrade-status  Report upgradability for multiple (or all) recipes
        Working on a recipe in the workspace:
-         build                Build a recipe
-         rename               Rename a recipe file in the workspace
-         edit-recipe          Edit a recipe file
-         find-recipe          Find a recipe file
-         configure-help       Get help on configure script options
-         update-recipe        Apply changes from external source tree to recipe
-         reset                Remove a recipe from your workspace
-         finish               Finish working on a recipe in your workspace
+         build                 Build a recipe
+         rename                Rename a recipe file in the workspace
+         edit-recipe           Edit a recipe file
+         find-recipe           Find a recipe file
+         configure-help        Get help on configure script options
+         update-recipe         Apply changes from external source tree to recipe
+         reset                 Remove a recipe from your workspace
+         finish                Finish working on a recipe in your workspace
        Testing changes on target:
-         deploy-target        Deploy recipe output files to live target machine
-         undeploy-target      Undeploy recipe output files in live target machine
-         build-image          Build image including workspace recipe packages
+         deploy-target         Deploy recipe output files to live target machine
+         undeploy-target       Undeploy recipe output files in live target machine
+         build-image           Build image including workspace recipe packages
        Advanced:
-         create-workspace     Set up workspace in an alternative location
-         export               Export workspace into a tar archive
-         import               Import exported tar archive into workspace
-         extract              Extract the source for an existing recipe
-         sync                 Synchronize the source tree for an existing recipe
+         create-workspace      Set up workspace in an alternative location
+         export                Export workspace into a tar archive
+         import                Import exported tar archive into workspace
+         extract               Extract the source for an existing recipe
+         sync                  Synchronize the source tree for an existing recipe
      Use devtool &lt;subcommand&gt; --help to get help on a specific command
             </literallayout>
-        </para>
-
-        <para>
-                As directed in the general help output, you can get more
-                syntax on a specific command by providing the command
-                name and using <filename>--help</filename>:
+            As directed in the general help output, you can get more syntax
+            on a specific command by providing the command name and using
+            "--help":
             <literallayout class='monospaced'>
      $ devtool add --help
      NOTE: Starting bitbake server...
@@ -429,6 +427,108 @@
         </para>
     </section>
 
+    <section id='devtool-checking-on-the-upgrade-status-of-a-recipe'>
+        <title>Checking on the Upgrade Status of a Recipe</title>
+
+        <para>
+            Upstream recipes change over time.
+            Consequently, you might find that you need to determine if you
+            can upgrade a recipe to a newer version.
+        </para>
+
+        <para>
+            To check on the upgrade status of a recipe, use the
+            <filename>devtool check-upgrade-status</filename> command.
+            The command displays a table of your current recipe versions,
+            the latest upstream versions, the email address of the recipe's
+            maintainer, and any additional information such as commit hash
+            strings and reasons you might not be able to upgrade a particular
+            recipe.
+            <note><title>NOTES:</title>
+                <itemizedlist>
+                    <listitem><para>
+                        For the <filename>oe-core</filename> layer, recipe
+                        maintainers come from the
+                        <ulink url='http://git.yoctoproject.org/cgit/cgit.cgi/poky/tree/meta/conf/distro/include/maintainers.inc'><filename>maintainers.inc</filename></ulink>
+                        file.
+                        </para></listitem>
+                    <listitem><para>
+                        If the recipe is using the
+                        <ulink url='&YOCTO_DOCS_BB_URL;#git-fetcher'>Git fetcher</ulink>
+                        rather than a tarball, the commit hash points to the
+                        commit that matches the recipe's latest version tag.
+                        </para></listitem>
+                </itemizedlist>
+            </note>
+        </para>
+
+        <para>
+            As with all <filename>devtool</filename> commands, you can get
+            help on the individual command:
+            <literallayout class='monospaced'>
+     $ devtool check-upgrade-status -h
+     NOTE: Starting bitbake server...
+     usage: devtool check-upgrade-status [-h] [--all] [recipe [recipe ...]]
+
+     Prints a table of recipes together with versions currently provided by
+     recipes, and latest upstream versions, when there is a later version available
+
+     arguments:
+       recipe      Name of the recipe to report (omit to report upgrade info for
+                   all recipes)
+
+     options:
+      -h, --help  show this help message and exit
+       --all, -a   Show all recipes, not just recipes needing upgrade
+            </literallayout>
+        </para>
+
+        <para>
+            Unless you provide a specific recipe name on the command line,
+            the command checks all recipes in all configured layers.
+        </para>
+
+        <para>
+            Following is a partial example table that reports on all the
+            recipes.
+            Notice the reported reason for not upgrading the
+            <filename>base-passwd</filename> recipe.
+            In this example, while a new version is available upstream,
+            you do not want to use it because the dependency on
+            <filename>cdebconf</filename> is not easily satisfied.
+            <note>
+                When a reason for not upgrading displays, the reason is
+                usually written into the recipe using the
+                <filename>RECIPE_NO_UPDATE_REASON</filename> variable.
+                See the
+                <ulink url='http://git.yoctoproject.org/cgit/cgit.cgi/poky/tree/meta/recipes-core/base-passwd/base-passwd_3.5.29.bb'><filename>base-passwd.bb</filename></ulink>
+                recipe for an example.
+            </note>
+            <literallayout class='monospaced'>
+     $ devtool check-upgrade-status
+         ...
+         NOTE: acpid                     2.0.30          2.0.31
+     Ross Burton &lt;ross.burton@intel.com&gt;
+         NOTE: u-boot-fw-utils           2018.11         2019.01
+     Marek Vasut &lt;marek.vasut@gmail.com&gt;
+     d3689267f92c5956e09cc7d1baa4700141662bff
+         NOTE: u-boot-tools              2018.11         2019.01
+     Marek Vasut &lt;marek.vasut@gmail.com&gt;
+     d3689267f92c5956e09cc7d1baa4700141662bff
+          .
+          .
+          .
+         NOTE: base-passwd               3.5.29          3.5.45
+     Anuj Mittal &lt;anuj.mittal@intel.com&gt;  cannot be updated due to: Version
+     3.5.38 requires cdebconf for update-passwd utility
+         NOTE: busybox                   1.29.2          1.30.0
+     Andrej Valek &lt;andrej.valek@siemens.com&gt;
+         NOTE: dbus-test                 1.12.10         1.12.12
+     Chen Qi &lt;Qi.Chen@windriver.com&gt;
+            </literallayout>
+        </para>
+    </section>
+
     <section id='devtool-upgrading-a-recipe'>
         <title>Upgrading a Recipe</title>
 
@@ -443,6 +543,13 @@
             section of the Yocto Project Development Tasks Manual.
             This section overviews the <filename>devtool upgrade</filename>
             command.
+            <note>
+                Before you upgrade a recipe, you can check on its upgrade
+                status.
+                See the
+                "<link linkend='devtool-checking-on-the-upgrade-status-of-a-recipe'>Checking on the Upgrade Status of a Recipe</link>"
+                for more information.
+            </note>
         </para>
 
         <para>

documentation/ref-manual/ref-features.xml

@@ -76,8 +76,6 @@
                     </para></listitem>
                 <listitem><para><emphasis>ext2:</emphasis> Hardware HDD or Microdrive
                     </para></listitem>
-                <listitem><para><emphasis>irda:</emphasis> Hardware has IrDA support
-                    </para></listitem>
                 <listitem><para><emphasis>keyboard:</emphasis> Hardware has a keyboard
                     </para></listitem>
                 <listitem><para><emphasis>pcbios:</emphasis> Support for booting through BIOS
@@ -190,8 +188,6 @@
                     support.</para></listitem>
                 <listitem><para><emphasis>ipv6:</emphasis> Include IPv6 support.
                     </para></listitem>
-                <listitem><para><emphasis>irda:</emphasis> Include IrDA support.
-                    </para></listitem>
                 <listitem><para><emphasis>keyboard:</emphasis> Include keyboard
                     support (e.g. keymaps will be loaded during boot).
                     </para></listitem>

documentation/ref-manual/ref-tasks.xml

@@ -158,32 +158,6 @@
         </para>
     </section>
 
-    <section id='ref-tasks-distrodata'>
-        <title><filename>do_distrodata</filename></title>
-
-        <para>
-            Provides information about the recipe.
-        </para>
-
-        <para>
-            The <filename>distrodata</filename> task is included as part of the
-            <link linkend='ref-classes-distrodata'><filename>distrodata</filename></link>
-            class.
-        </para>
-
-        <para>
-            To build the <filename>distrodata</filename> task, use the
-            <filename>bitbake</filename> command with the "-c" option and
-            task name:
-            <literallayout class='monospaced'>
-     $ bitbake core-image-minimal -c distrodata
-            </literallayout>
-            By default, the results are stored in
-            <link linkend='var-LOG_DIR'><filename>$LOG_DIR</filename></link>
-            (e.g. <filename>$BUILD_DIR/tmp/log</filename>).
-        </para>
-    </section>
-
     <section id='ref-tasks-fetch'>
         <title><filename>do_fetch</filename></title>
 
@@ -192,7 +166,8 @@
             This task uses the
             <link linkend='var-SRC_URI'><filename>SRC_URI</filename></link>
             variable and the argument's prefix to determine the correct
-            fetcher module.
+            <ulink url='&YOCTO_DOCS_BB_URL;#bb-fetchers'>fetcher</ulink>
+            module.
         </para>
     </section>
 
@@ -635,9 +610,18 @@
         </para>
 
         <para>
-            The <filename>checkpkg</filename> task is included as part of the
-            <link linkend='ref-classes-distrodata'><filename>distrodata</filename></link>
-            class.
+            To check the upstream version and status of a recipe, use the
+            following devtool commands:
+            <literallayout class='monospaced'>
+     $ devtool latest-version
+     $ devtool check-upgrade-status
+            </literallayout>
+            See the
+            "<link linkend='ref-devtool-reference'><filename>devtool</filename> Quick Reference</link>"
+            chapter for more information on <filename>devtool</filename>.
+            See the
+            "<ulink url='&YOCTO_DOCS_REF_URL;#devtool-checking-on-the-upgrade-status-of-a-recipe'>Checking on the Upgrade Status of a Recipe</ulink>"
+            section for information on checking the upgrade status of a recipe.
         </para>
 
         <para>

documentation/ref-manual/ref-variables.xml

@@ -1358,7 +1358,7 @@
                     For example, the following line specifies three
                     configuration files:
                     <literallayout class='monospaced'>
-     BBMULTIFONFIG = "configA configB configC"
+     BBMULTICONFIG = "configA configB configC"
                     </literallayout>
                     Each configuration file you use must reside in the
                     <link linkend='build-directory'>Build Directory</link>
@@ -11694,8 +11694,7 @@
                     or
                     <link linkend='ref-classes-setuptools'><filename>setuptools</filename></link>
                     classes, specifies the major Python version being built.
-                    For Python 2.x, <filename>PYTHON_PN</filename> would
-                    be "python2".  For Python 3.x, the variable would be
+                    For Python 3.x, <filename>PYTHON_PN</filename> would be
                     "python3".
                     You do not have to set this variable as the
                     OpenEmbedded build system automatically sets it for you.
@@ -16629,16 +16628,16 @@
             <glossdef>
                 <para role="glossdeffirst">
 <!--                <para role="glossdeffirst"><imagedata fileref="figures/define-generic.png" /> -->
-                    When the
-                    <link linkend='ref-classes-distrodata'><filename>distrodata</filename></link>
-                    class is enabled globally, you can perform a per-recipe
-                    check for what the latest upstream source code version is
-                    by calling
+                    You can perform a per-recipe check for what the latest
+                    upstream source code version is by calling
                     <filename>bitbake -c checkpkg</filename> <replaceable>recipe</replaceable>.
                     If the recipe source code is provided from Git
                     repositories, the OpenEmbedded build system determines the
                     latest upstream version by picking the latest tag from the
                     list of all repository tags.
+                </para>
+
+                <para>
                     You can use the
                     <filename>UPSTREAM_CHECK_GITTAGREGEX</filename>
                     variable to provide a regular expression to filter only the
@@ -16658,11 +16657,8 @@
             <glossdef>
                 <para role="glossdeffirst">
 <!--                <para role="glossdeffirst"><imagedata fileref="figures/define-generic.png" /> -->
-                    When the
-                    <link linkend='ref-classes-distrodata'><filename>distrodata</filename></link>
-                    class is enabled globally, use the
-                    <filename>UPSTREAM_CHECK_REGEX</filename> variable to
-                    specify a different regular expression instead of the
+                    Use the <filename>UPSTREAM_CHECK_REGEX</filename> variable
+                    to specify a different regular expression instead of the
                     default one when the package checking system is parsing
                     the page found using
                     <link linkend='var-UPSTREAM_CHECK_URI'><filename>UPSTREAM_CHECK_URI</filename></link>.
@@ -16680,12 +16676,9 @@
             <glossdef>
                 <para role="glossdeffirst">
 <!--                <para role="glossdeffirst"><imagedata fileref="figures/define-generic.png" /> -->
-                    When the
-                    <link linkend='ref-classes-distrodata'><filename>distrodata</filename></link>
-                    class is enabled globally, you can perform a per-recipe
-                    check for what the latest upstream source code version is
-                    by calling <filename>bitbake -c checkpkg</filename>
-                    <replaceable>recipe</replaceable>.
+                    You can perform a per-recipe check for what the latest
+                    upstream source code version is by calling
+                    <filename>bitbake -c checkpkg</filename> <replaceable>recipe</replaceable>.
                     If the source code is provided from tarballs, the latest
                     version is determined by fetching the directory listing
                     where the tarball is and attempting to find a later tarball.

documentation/sdk-manual/sdk-extensible.xml

@@ -618,7 +618,23 @@
                                 The result is that the command sets up both
                                 the source code and an append file within the
                                 workspace while the recipe remains in its
-                                original location.
+                                original location.</para>
+
+                                <para>Additionally, if you have any non-patch
+                                local files (i.e. files referred to with
+                                <filename>file://</filename> entries in
+                                <filename>SRC_URI</filename> statement excluding
+                                <filename>*.patch/</filename> or
+                                <filename>*.diff</filename>), these files are
+                                copied to an
+                                <filename>oe-local-files</filename> folder
+                                under the newly created source tree.
+                                Copying the files here gives you a convenient
+                                area from which you can modify the files.
+                                Any changes or additions you make to those
+                                files are incorporated into the build the next
+                                time you build the software just as are other
+                                changes you might have made to the source.
                                 </para></listitem>
                             <listitem><para>
                                 <emphasis>Middle</emphasis>:
@@ -633,10 +649,10 @@
                                 </para>
 
                                 <para>The following command tells
-                                <filename>devtool</filename> what recipe with
+                                <filename>devtool</filename> the recipe with
                                 which to work and, in this case, identifies a
                                 local area for the extracted source files that
-                                is outside of the default
+                                exists outside of the default
                                 <filename>devtool</filename> workspace:
                                 <literallayout class='monospaced'>
      $ devtool modify <replaceable>recipe srctree</replaceable>
@@ -650,8 +666,12 @@
                                 the recipe's <filename>SRC_URI</filename>
                                 statements to locate the source files and any
                                 associated patch files.
-                                Once the files are located, the command by
-                                default extracts them into
+                                Non-patch files are copied to an
+                                <filename>oe-local-files</filename> folder
+                                under the newly created source tree.</para>
+
+                                <para>Once the files are located, the command
+                                by default extracts them into
                                 <replaceable>srctree</replaceable>.</para>
 
                                 <para>Within workspace,
@@ -685,9 +705,21 @@
                                 </literallayout>
                                 </para>
 
-                                <para>Once the command finishes, it creates only
-                                an append file for the recipe in the
-                                <filename>devtool</filename> workspace.
+                                <para>If an <filename>oe-local-files</filename>
+                                subdirectory happens to exist and it contains
+                                non-patch files, the files are used.
+                                However, if the subdirectory does not exist and
+                                you run the <filename>devtool finish</filename>
+                                command, any non-patch files that might exist
+                                next to the recipe are removed because it
+                                appears to <filename>devtool</filename> that
+                                you have deleted those files.</para>
+
+                                <para>Once the
+                                <filename>devtool modify</filename> command
+                                finishes, it creates only an append file for
+                                the recipe in the <filename>devtool</filename>
+                                workspace.
                                 The recipe and the source code remain in their
                                 original locations.
                                 </para></listitem>
@@ -778,7 +810,12 @@
                         original recipe in the original layer or the command
                         creates a <filename>.bbappend</filename> file in a
                         different layer as provided by
-                        <replaceable>layer</replaceable>.</para>
+                        <replaceable>layer</replaceable>.
+                        Any work you did in the
+                        <filename>oe-local-files</filename> directory is
+                        preserved in the original files next to the recipe
+                        during the <filename>devtool finish</filename>
+                        command.</para>
 
                         <para>As a final process of the
                         <filename>devtool finish</filename> command, the state
@@ -825,7 +862,9 @@
                 versioning schemes, extract code into or out of the
                 <filename>devtool</filename>
                 <ulink url='&YOCTO_DOCS_REF_URL;#devtool-the-workspace-layer-structure'>workspace</ulink>,
-                and work with any source file forms that the fetchers support.
+                and work with any source file forms that the
+                <ulink url='&YOCTO_DOCS_BB_URL;#bb-fetchers'>fetchers</ulink>
+                support.
             </para>
 
             <para>
@@ -896,7 +935,23 @@
                         files from other developers.
                         The result is that the command sets up the source
                         code, the new version of the recipe, and an append file
-                        all within the workspace.
+                        all within the workspace.</para>
+
+                        <para>Additionally, if you have any non-patch
+                        local files (i.e. files referred to with
+                        <filename>file://</filename> entries in
+                        <filename>SRC_URI</filename> statement excluding
+                        <filename>*.patch/</filename> or
+                        <filename>*.diff</filename>), these files are
+                        copied to an
+                        <filename>oe-local-files</filename> folder
+                        under the newly created source tree.
+                        Copying the files here gives you a convenient
+                        area from which you can modify the files.
+                        Any changes or additions you make to those
+                        files are incorporated into the build the next
+                        time you build the software just as are other
+                        changes you might have made to the source.
                         </para></listitem>
                     <listitem><para>
                         <emphasis>Resolve any Conflicts created by the Upgrade</emphasis>:
@@ -974,10 +1029,18 @@
                         Git repository, moves the new recipe to a more
                         permanent layer, and then resets the recipe so that
                         the recipe is built normally rather than from the
-                        workspace.
+                        workspace.</para>
+
+                        <para>Any work you did in the
+                        <filename>oe-local-files</filename> directory is
+                        preserved in the original files next to the recipe
+                        during the <filename>devtool finish</filename>
+                        command.</para>
+
+                        <para>
                         If you specify a destination layer that is the same as
                         the original source, then the old version of the
-                        recipe and associated files will be removed prior to
+                        recipe and associated files are removed prior to
                         adding the new version.
                         <literallayout class='monospaced'>
      $ devtool finish <replaceable>recipe layer</replaceable>

meta-poky/conf/distro/include/poky-distro-alt-test-config.inc

@@ -0,0 +1,2 @@
+DISTRO_FEATURES_append = " pam"
+PREFERRED_VERSION_linux-yocto_linuxstdbase ?= "4.19%"

meta-poky/conf/distro/poky-lsb.conf

@@ -4,8 +4,7 @@ require conf/distro/poky.conf
 DISTRO = "poky-lsb"
 DISTROOVERRIDES = "poky:linuxstdbase"
 
-DISTRO_FEATURES_append = " pam largefile opengl"
-PREFERRED_PROVIDER_virtual/libx11 = "libx11"
+DISTRO_FEATURES_append = " pam"
 
 # Ensure the kernel nfs server is enabled
 KERNEL_FEATURES_append_pn-linux-yocto = " features/nfsd/nfsd-enable.scc"

meta-poky/conf/distro/poky.conf

@@ -56,10 +56,13 @@ SANITY_TESTED_DISTROS ?= " \
             ubuntu-18.04 \n \
             fedora-28 \n \
             fedora-29 \n \
+            fedora-30 \n \
             centos-7 \n \
             debian-8 \n \
             debian-9 \n \
+            debian-10 \n \
             opensuse-42.3 \n \
+            opensuseleap-15.1 \n \
             "
 #
 # OELAYOUT_ABI allows us to notify users when the format of TMPDIR changes in

meta-poky/conf/local.conf.sample

@@ -34,8 +34,8 @@
 #MACHINE ?= "mpc8315e-rdb"
 #MACHINE ?= "edgerouter"
 #
-# This sets the default machine to be qemux86 if no other machine is selected:
-MACHINE ??= "qemux86"
+# This sets the default machine to be qemux86-64 if no other machine is selected:
+MACHINE ??= "qemux86-64"
 
 #
 # Where to place downloads

meta-poky/conf/local.conf.sample.extended

@@ -321,7 +321,6 @@ DISTRO_FEATURES_remove = "x11"
 # show an example of how it can be done
 # You'll also need your fortran recipe to depend on libgfortran
 #FORTRAN_forcevariable = ",fortran"
-#RUNTIMETARGET_append_pn-gcc-runtime = " libquadmath"
 
 #
 # Kernel image features

meta-poky/recipes-core/tiny-init/tiny-init.bb

@@ -28,3 +28,4 @@ do_install() {
 }
 
 FILES_${PN} = "/init ${sysconfdir}/rc.local.sample"
+RCONFLICTS_${PN} = "systemd"

meta-selftest/recipes-test/fortran/files/hello.f95

@@ -0,0 +1,5 @@
+program helloworld
+
+      print * , "Hello World!"
+
+end program helloworld

meta-selftest/recipes-test/fortran/fortran-helloworld.bb

@@ -0,0 +1,25 @@
+SUMMARY = "Fortran Hello World"
+LICENSE = "MIT"
+LIC_FILES_CHKSUM = "file://${COREBASE}/meta/COPYING.MIT;md5=3da9cfbcb788c80a0384361b4de20420"
+
+DEPENDS = "libgfortran"
+
+SRC_URI = "file://hello.f95"
+
+# These set flags that Fortran doesn't support
+SECURITY_CFLAGS = ""
+SECURITY_LDFLAGS = ""
+
+do_compile() {
+	${HOST_PREFIX}gfortran ${HOST_CC_ARCH}${TOOLCHAIN_OPTIONS} ${LDFLAGS} ${WORKDIR}/hello.f95 -o ${B}/fortran-hello
+}
+
+do_install() {
+	install -d ${D}${bindir}
+	install ${B}/fortran-hello ${D}${bindir}
+}
+
+python () {
+    if not d.getVar("FORTRAN"):
+        raise bb.parse.SkipRecipe("Fortran isn't enabled")
+}

meta-selftest/recipes-test/license/incompatible-license-alias.bb

@@ -0,0 +1,3 @@
+SUMMARY = "Recipe with an alias of an SPDX license"
+DESCRIPTION = "Is licensed with an alias of an SPDX license to be used for testing"
+LICENSE = "GPLv3"

meta-selftest/recipes-test/license/incompatible-license.bb

@@ -0,0 +1,3 @@
+SUMMARY = "Recipe with an SPDX license"
+DESCRIPTION = "Is licensed with an SPDX license to be used for testing"
+LICENSE = "GPL-3.0"

meta-selftest/recipes-test/license/incompatible-nonspdx-license.bb

@@ -0,0 +1,3 @@
+SUMMARY = "Recipe with a non-SPDX license"
+DESCRIPTION = "Is licensed with a non-SPDX license to be used for testing"
+LICENSE = "FooLicense"

meta-selftest/recipes-test/multiconfig/multiconfig-image-packager_0.1.bb

@@ -8,7 +8,7 @@ MCIMGTYPE_virtclass-mcextend-tiny = "cpio.gz"
 
 MC_DEPLOY_DIR_IMAGE = "${TOPDIR}/tmp-mc-${MCNAME}/deploy/images/${MCMACHINE}"
 
-do_install[mcdepends] += "multiconfig::${MCNAME}:core-image-minimal:do_image_complete multiconfig::${MCNAME}:virtual/kernel:do_deploy"
+do_install[mcdepends] += "mc::${MCNAME}:core-image-minimal:do_image_complete mc::${MCNAME}:virtual/kernel:do_deploy"
 
 do_install () {
     install -d ${D}/var/lib/machines/${MCNAME}

meta-selftest/wic/test_biosplusefi_plugin.wks

@@ -0,0 +1,6 @@
+# short-description: This file is used in oe-selftest wic module to test biosplusefi plugin
+
+part /boot --source bootimg-biosplusefi --sourceparams="loader=grub-efi" --active --align 1024 --use-uuid
+part / --source rootfs --fstype=ext4 --align 1024 --use-uuid
+
+bootloader  --timeout=0 --append="console=ttyS0,115200n8"

meta-yocto-bsp/conf/machine/beaglebone-yocto.conf

@@ -18,9 +18,9 @@ IMAGE_FSTYPES += "tar.bz2 jffs2 wic wic.bmap"
 EXTRA_IMAGECMD_jffs2 = "-lnp "
 WKS_FILE ?= "beaglebone-yocto.wks"
 IMAGE_INSTALL_append = " kernel-devicetree kernel-image-zimage"
-do_image_wic[depends] += "mtools-native:do_populate_sysroot dosfstools-native:do_populate_sysroot"
+do_image_wic[depends] += "mtools-native:do_populate_sysroot dosfstools-native:do_populate_sysroot u-boot:do_deploy"
 
-SERIAL_CONSOLES ?= "115200;ttyS0 115200;ttyO0"
+SERIAL_CONSOLES ?= "115200;ttyS0 115200;ttyO0 115200;ttyAMA0"
 SERIAL_CONSOLES_CHECK = "${SERIAL_CONSOLES}"
 
 PREFERRED_PROVIDER_virtual/kernel ?= "linux-yocto"
@@ -39,3 +39,21 @@ UBOOT_LOADADDRESS = "0x80008000"
 MACHINE_FEATURES = "usbgadget usbhost vfat alsa"
 
 IMAGE_BOOT_FILES ?= "u-boot.${UBOOT_SUFFIX} MLO zImage am335x-bone.dtb am335x-boneblack.dtb am335x-bonegreen.dtb"
+
+# support runqemu
+EXTRA_IMAGEDEPENDS += "qemu-native qemu-helper-native"
+IMAGE_CLASSES += "qemuboot"
+QB_DEFAULT_FSTYPE = "wic"
+QB_FSINFO = "wic:no-kernel-in-fs"
+QB_KERNEL_ROOT = "/dev/vda2"
+QB_SYSTEM_NAME = "qemu-system-arm"
+QB_MEM = "-m 512"
+QB_MACHINE = "-machine virt"
+QB_CPU = "-cpu cortex-a15"
+QB_KERNEL_CMDLINE_APPEND = "console=ttyAMA0 systemd.mask=systemd-networkd"
+QB_OPT_APPEND = "-device virtio-rng-device"
+QB_TAP_OPT = "-netdev tap,id=net0,ifname=@TAP@,script=no,downscript=no"
+QB_NETWORK_DEVICE = "-device virtio-net-device,netdev=net0,mac=@MAC@"
+QB_ROOTFS_OPT = "-drive id=disk0,file=@ROOTFS@,if=none,format=raw -device virtio-blk-device,drive=disk0"
+QB_SERIAL_OPT = ""
+QB_TCPSERIAL_OPT = "-device virtio-serial-device -chardev socket,id=virtcon,port=@PORT@,host=127.0.0.1 -device virtconsole,chardev=virtcon"

meta/classes/base.bbclass

@@ -10,9 +10,13 @@ inherit utility-tasks
 inherit metadata_scm
 inherit logging
 
-OE_IMPORTS += "os sys time oe.path oe.utils oe.types oe.package oe.packagegroup oe.sstatesig oe.lsb oe.cachedpath oe.license"
+OE_EXTRA_IMPORTS ?= ""
+
+OE_IMPORTS += "os sys time oe.path oe.utils oe.types oe.package oe.packagegroup oe.sstatesig oe.lsb oe.cachedpath oe.license ${OE_EXTRA_IMPORTS}"
 OE_IMPORTS[type] = "list"
 
+PACKAGECONFIG_CONFARGS ??= ""
+
 def oe_import(d):
     import sys
 
@@ -639,6 +643,18 @@ python () {
     if needsrcrev:
         d.setVar("SRCPV", "${@bb.fetch2.get_srcrev(d)}")
 
+        # Gather all named SRCREVs to add to the sstate hash calculation
+        # This anonymous python snippet is called multiple times so we
+        # need to be careful to not double up the appends here and cause
+        # the base hash to mismatch the task hash
+        for uri in srcuri.split():
+            parm = bb.fetch.decodeurl(uri)[5]
+            uri_names = parm.get("name", "").split(",")
+            for uri_name in filter(None, uri_names):
+                srcrev_name = "SRCREV_{}".format(uri_name)
+                if srcrev_name not in (d.getVarFlag("do_fetch", "vardeps") or "").split():
+                    d.appendVarFlag("do_fetch", "vardeps", " {}".format(srcrev_name))
+
     set_packagetriplet(d)
 
     # 'multimachine' handling

meta/classes/binconfig.bbclass

@@ -40,15 +40,6 @@ binconfig_package_preprocess () {
 		    -e 's:${STAGING_DIR_HOST}${prefix}:${prefix}:' \
                     $config
 	done
-	for lafile in `find ${PKGD} -type f -name "*.la"` ; do
-		sed -i \
-		    -e 's:${STAGING_BASELIBDIR}:${base_libdir}:g;' \
-		    -e 's:${STAGING_LIBDIR}:${libdir}:g;' \
-		    -e 's:${STAGING_INCDIR}:${includedir}:g;' \
-		    -e 's:${STAGING_DATADIR}:${datadir}:' \
-		    -e 's:${STAGING_DIR_HOST}${prefix}:${prefix}:' \
-		    $lafile
-	done	    
 }
 
 SYSROOT_PREPROCESS_FUNCS += "binconfig_sysroot_preprocess"

meta/classes/bluetooth.bbclass

@@ -1,14 +0,0 @@
-# Avoid code duplication in bluetooth-dependent recipes.
-
-# Define a variable that expands to the recipe (package) providing core
-# bluetooth support on the platform:
-# "" if bluetooth is not in DISTRO_FEATURES
-# else "bluez5" if bluez5 is in DISTRO_FEATURES
-# else "bluez4"
-
-# Use this with:
-#  inherit bluetooth
-#  PACKAGECONFIG ??= "${@bb.utils.contains('DISTRO_FEATURES', 'bluetooth', '${BLUEZ}', '', d)}
-#  PACKAGECONFIG[bluez4] = "--enable-bluez4,--disable-bluez4,bluez4"
-
-BLUEZ ?= "${@bb.utils.contains('DISTRO_FEATURES', 'bluetooth', bb.utils.contains('DISTRO_FEATURES', 'bluez5', 'bluez5', 'bluez4', d), '', d)}"

meta/classes/buildhistory.bbclass

@@ -60,15 +60,34 @@ SSTATEPOSTUNPACKFUNCS[vardepvalueexclude] .= "| buildhistory_emit_outputsigs"
 # When extending build history, derive your class from buildhistory.bbclass
 # and extend this list here with the additional files created by the derived
 # class.
-BUILDHISTORY_PRESERVE = "latest latest_srcrev"
+BUILDHISTORY_PRESERVE = "latest latest_srcrev sysroot"
 
 PATCH_GIT_USER_EMAIL ?= "buildhistory@oe"
 PATCH_GIT_USER_NAME ?= "OpenEmbedded"
 
+#
+# Write out the contents of the sysroot
+#
+buildhistory_emit_sysroot() {
+	mkdir --parents ${BUILDHISTORY_DIR_PACKAGE}
+	case ${CLASSOVERRIDE} in
+	class-native|class-cross|class-crosssdk)
+		BASE=${SYSROOT_DESTDIR}/${STAGING_DIR_NATIVE}
+		;;
+	*)
+		BASE=${SYSROOT_DESTDIR}
+		;;
+	esac
+	buildhistory_list_files_no_owners $BASE ${BUILDHISTORY_DIR_PACKAGE}/sysroot
+}
+
 #
 # Write out metadata about this package for comparison when writing future packages
 #
 python buildhistory_emit_pkghistory() {
+    if d.getVar('BB_CURRENTTASK') in ['populate_sysroot', 'populate_sysroot_setscene']:
+        bb.build.exec_func("buildhistory_emit_sysroot", d)
+
     if not d.getVar('BB_CURRENTTASK') in ['packagedata', 'packagedata_setscene']:
         return 0
 
@@ -529,6 +548,20 @@ buildhistory_list_files() {
 	fi | sort -k5 | sed 's/ * -> $//' > $2 )
 }
 
+buildhistory_list_files_no_owners() {
+	# List the files in the specified directory, but exclude date/time etc.
+	# Also don't output the ownership data, but instead output just - - so
+	# that the same parsing code as for _list_files works.
+	# This is somewhat messy, but handles where the size is not printed for device files under pseudo
+	( cd $1
+	find_cmd='find . ! -path . -printf "%M -          -          %10s %p -> %l\n"'
+	if [ "$3" = "fakeroot" ] ; then
+		eval ${FAKEROOTENV} ${FAKEROOTCMD} "$find_cmd"
+	else
+		eval "$find_cmd"
+	fi | sort -k5 | sed 's/ * -> $//' > $2 )
+}
+
 buildhistory_list_pkg_files() {
 	# Create individual files-in-package for each recipe's package
 	for pkgdir in $(find ${PKGDEST}/* -maxdepth 0 -type d); do
@@ -839,11 +872,15 @@ python buildhistory_eventhandler() {
             if e.data.getVar("BUILDHISTORY_COMMIT") == "1":
                 bb.note("Writing buildhistory")
                 bb.build.exec_func("buildhistory_write_sigs", d)
+                import time
+                start=time.time()
                 localdata = bb.data.createCopy(e.data)
                 localdata.setVar('BUILDHISTORY_BUILD_FAILURES', str(e._failures))
                 interrupted = getattr(e, '_interrupted', 0)
                 localdata.setVar('BUILDHISTORY_BUILD_INTERRUPTED', str(interrupted))
                 bb.build.exec_func("buildhistory_commit", localdata)
+                stop=time.time()
+                bb.note("Writing buildhistory took: %s seconds" % round(stop-start))
             else:
                 bb.note("No commit since BUILDHISTORY_COMMIT != '1'")
 }

meta/classes/cmake.bbclass

@@ -26,14 +26,16 @@ python() {
     if not d.getVar('OECMAKE_C_COMPILER'):
         cc_list = d.getVar('CC').split()
         if cc_list[0] == 'ccache':
-            d.setVar('OECMAKE_C_COMPILER', '%s %s' % (cc_list[0], cc_list[1]))
+            d.setVar('OECMAKE_C_COMPILER_LAUNCHER', cc_list[0])
+            d.setVar('OECMAKE_C_COMPILER', cc_list[1])
         else:
             d.setVar('OECMAKE_C_COMPILER', cc_list[0])
 
     if not d.getVar('OECMAKE_CXX_COMPILER'):
         cxx_list = d.getVar('CXX').split()
         if cxx_list[0] == 'ccache':
-            d.setVar('OECMAKE_CXX_COMPILER', '%s %s' % (cxx_list[0], cxx_list[1]))
+            d.setVar('OECMAKE_CXX_COMPILER_LAUNCHER', cxx_list[0])
+            d.setVar('OECMAKE_CXX_COMPILER', cxx_list[1])
         else:
             d.setVar('OECMAKE_CXX_COMPILER', cxx_list[0])
 }
@@ -49,6 +51,9 @@ OECMAKE_CXX_LINK_FLAGS ?= "${HOST_CC_ARCH} ${TOOLCHAIN_OPTIONS} ${CXXFLAGS} ${LD
 CXXFLAGS += "${HOST_CC_ARCH} ${TOOLCHAIN_OPTIONS}"
 CFLAGS += "${HOST_CC_ARCH} ${TOOLCHAIN_OPTIONS}"
 
+OECMAKE_C_COMPILER_LAUNCHER ?= ""
+OECMAKE_CXX_COMPILER_LAUNCHER ?= ""
+
 OECMAKE_RPATH ?= ""
 OECMAKE_PERLNATIVE_DIR ??= ""
 OECMAKE_EXTRA_ROOT_PATH ?= ""
@@ -86,6 +91,8 @@ set( CMAKE_SYSTEM_NAME `echo ${TARGET_OS} | sed -e 's/^./\u&/' -e 's/^\(Linux\).
 set( CMAKE_SYSTEM_PROCESSOR ${@map_target_arch_to_uname_arch(d.getVar('TARGET_ARCH'))} )
 set( CMAKE_C_COMPILER ${OECMAKE_C_COMPILER} )
 set( CMAKE_CXX_COMPILER ${OECMAKE_CXX_COMPILER} )
+set( CMAKE_C_COMPILER_LAUNCHER ${OECMAKE_C_COMPILER_LAUNCHER} )
+set( CMAKE_CXX_COMPILER_LAUNCHER ${OECMAKE_CXX_COMPILER_LAUNCHER} )
 set( CMAKE_ASM_COMPILER ${OECMAKE_C_COMPILER} )
 set( CMAKE_AR ${OECMAKE_AR} CACHE FILEPATH "Archiver" )
 set( CMAKE_C_FLAGS "${OECMAKE_C_FLAGS}" CACHE STRING "CFLAGS" )
@@ -112,7 +119,7 @@ set( ENV{QT_CONF_PATH} ${WORKDIR}/qt.conf )
 # directory as rpath by default
 set( CMAKE_INSTALL_RPATH ${OECMAKE_RPATH} )
 
-# Use native cmake modules
+# Use our cmake modules
 list(APPEND CMAKE_MODULE_PATH "${STAGING_DATADIR}/cmake/Modules/")
 
 # add for non /usr/lib libdir, e.g. /usr/lib64
@@ -154,15 +161,15 @@ cmake_do_configure() {
 	  $oecmake_sitefile \
 	  ${OECMAKE_SOURCEPATH} \
 	  -DCMAKE_INSTALL_PREFIX:PATH=${prefix} \
-	  -DCMAKE_INSTALL_BINDIR:PATH=${@os.path.relpath(d.getVar('bindir'), d.getVar('prefix'))} \
-	  -DCMAKE_INSTALL_SBINDIR:PATH=${@os.path.relpath(d.getVar('sbindir'), d.getVar('prefix'))} \
-	  -DCMAKE_INSTALL_LIBEXECDIR:PATH=${@os.path.relpath(d.getVar('libexecdir'), d.getVar('prefix'))} \
+	  -DCMAKE_INSTALL_BINDIR:PATH=${@os.path.relpath(d.getVar('bindir'), d.getVar('prefix') + '/')} \
+	  -DCMAKE_INSTALL_SBINDIR:PATH=${@os.path.relpath(d.getVar('sbindir'), d.getVar('prefix') + '/')} \
+	  -DCMAKE_INSTALL_LIBEXECDIR:PATH=${@os.path.relpath(d.getVar('libexecdir'), d.getVar('prefix') + '/')} \
 	  -DCMAKE_INSTALL_SYSCONFDIR:PATH=${sysconfdir} \
-	  -DCMAKE_INSTALL_SHAREDSTATEDIR:PATH=${@os.path.relpath(d.getVar('sharedstatedir'), d.  getVar('prefix'))} \
+	  -DCMAKE_INSTALL_SHAREDSTATEDIR:PATH=${@os.path.relpath(d.getVar('sharedstatedir'), d.  getVar('prefix') + '/')} \
 	  -DCMAKE_INSTALL_LOCALSTATEDIR:PATH=${localstatedir} \
-	  -DCMAKE_INSTALL_LIBDIR:PATH=${@os.path.relpath(d.getVar('libdir'), d.getVar('prefix'))} \
-	  -DCMAKE_INSTALL_INCLUDEDIR:PATH=${@os.path.relpath(d.getVar('includedir'), d.getVar('prefix'))} \
-	  -DCMAKE_INSTALL_DATAROOTDIR:PATH=${@os.path.relpath(d.getVar('datadir'), d.getVar('prefix'))} \
+	  -DCMAKE_INSTALL_LIBDIR:PATH=${@os.path.relpath(d.getVar('libdir'), d.getVar('prefix') + '/')} \
+	  -DCMAKE_INSTALL_INCLUDEDIR:PATH=${@os.path.relpath(d.getVar('includedir'), d.getVar('prefix') + '/')} \
+	  -DCMAKE_INSTALL_DATAROOTDIR:PATH=${@os.path.relpath(d.getVar('datadir'), d.getVar('prefix') + '/')} \
 	  -DCMAKE_INSTALL_SO_NO_EXE=0 \
 	  -DCMAKE_TOOLCHAIN_FILE=${WORKDIR}/toolchain.cmake \
 	  -DCMAKE_NO_SYSTEM_FROM_IMPORTED=1 \