meta/lib/oeqa: Test for bootimg-biosplusefi Source

Add unittests for bootimg-biosplusefi SourcePlugin in wic module.
First test check wic creation works correctly.
Second test uses qemu to boot image and checks that it has both
EFI and BIOS files in a single partition.

(From OE-Core rev: e0c3436241afca93f107e325d1b9ffcdebf706cd)

Signed-off-by: William Bourque <wbourque@gmail.com>
Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>

README.LSB

@@ -12,6 +12,9 @@ therefore provides compatibility with the following caveats:
   tests any longer. Users are recommended to group together to support
   maintenance of that layer. [http://git.yoctoproject.org/cgit/cgit.cgi/meta-qt4/]
 
+* GTK+ 2 is provided by meta-oe.  It is noted that GTK+ 2 is end of life
+  and use of GTK+ 2 isn't recommended.
+
 * mailx has been dropped since its no longer being developed upstream 
   and there are better, more modern replacements such as s-nail 
   (http://sdaoden.eu/code.html) or mailutils (http://mailutils.org/).

bitbake/bin/bitbake-selftest

@@ -25,6 +25,7 @@ tests = ["bb.tests.codeparser",
          "bb.tests.fetch",
          "bb.tests.parse",
          "bb.tests.persist_data",
+         "bb.tests.runqueue",
          "bb.tests.utils",
          "hashserv.tests",
          "layerindexlib.tests.layerindexobj",

bitbake/lib/bb/cache.py

@@ -83,20 +83,20 @@ class CoreRecipeInfo(RecipeInfoCommon):
         self.appends = self.listvar('__BBAPPEND', metadata)
         self.nocache = self.getvar('BB_DONT_CACHE', metadata)
 
-        self.skipreason = self.getvar('__SKIPPED', metadata)
-        if self.skipreason:
-            self.pn = self.getvar('PN', metadata) or bb.parse.vars_from_file(filename,metadata)[0]
-            self.skipped = True
-            self.provides  = self.depvar('PROVIDES', metadata)
-            self.rprovides = self.depvar('RPROVIDES', metadata)
-            return
-
-        self.tasks = metadata.getVar('__BBTASKS', False)
-
-        self.pn = self.getvar('PN', metadata)
+        self.provides  = self.depvar('PROVIDES', metadata)
+        self.rprovides = self.depvar('RPROVIDES', metadata)
+        self.pn = self.getvar('PN', metadata) or bb.parse.vars_from_file(filename,metadata)[0]
         self.packages = self.listvar('PACKAGES', metadata)
         if not self.packages:
             self.packages.append(self.pn)
+        self.packages_dynamic = self.listvar('PACKAGES_DYNAMIC', metadata)
+
+        self.skipreason = self.getvar('__SKIPPED', metadata)
+        if self.skipreason:
+            self.skipped = True
+            return
+
+        self.tasks = metadata.getVar('__BBTASKS', False)
 
         self.basetaskhashes = self.taskvar('BB_BASEHASH', self.tasks, metadata)
         self.hashfilename = self.getvar('BB_HASHFILENAME', metadata)
@@ -113,11 +113,8 @@ class CoreRecipeInfo(RecipeInfoCommon):
         self.stampclean = self.getvar('STAMPCLEAN', metadata)
         self.stamp_extrainfo = self.flaglist('stamp-extra-info', self.tasks, metadata)
         self.file_checksums = self.flaglist('file-checksums', self.tasks, metadata, True)
-        self.packages_dynamic = self.listvar('PACKAGES_DYNAMIC', metadata)
         self.depends          = self.depvar('DEPENDS', metadata)
-        self.provides         = self.depvar('PROVIDES', metadata)
         self.rdepends         = self.depvar('RDEPENDS', metadata)
-        self.rprovides        = self.depvar('RPROVIDES', metadata)
         self.rrecommends      = self.depvar('RRECOMMENDS', metadata)
         self.rprovides_pkg    = self.pkgvar('RPROVIDES', self.packages, metadata)
         self.rdepends_pkg     = self.pkgvar('RDEPENDS', self.packages, metadata)
@@ -399,6 +396,15 @@ class Cache(NoCache):
         else:
             logger.debug(1, "Cache file %s not found, building..." % self.cachefile)
 
+        # We don't use the symlink, its just for debugging convinience
+        symlink = os.path.join(self.cachedir, "bb_cache.dat")
+        if os.path.exists(symlink):
+            bb.utils.remove(symlink)
+        try:
+            os.symlink(os.path.basename(self.cachefile), symlink)
+        except OSError:
+            pass
+
     def load_cachefile(self):
         cachesize = 0
         previous_progress = 0

bitbake/lib/bb/cooker.py

@@ -1,4 +1,3 @@
-#!/usr/bin/env python
 #
 # Copyright (C) 2003, 2004  Chris Larson
 # Copyright (C) 2003, 2004  Phil Blundell

bitbake/lib/bb/cookerdata.py

@@ -1,4 +1,3 @@
-#!/usr/bin/env python
 #
 # Copyright (C) 2003, 2004  Chris Larson
 # Copyright (C) 2003, 2004  Phil Blundell
@@ -122,6 +121,7 @@ class CookerConfiguration(object):
         self.profile = False
         self.nosetscene = False
         self.setsceneonly = False
+        self.skipsetscene = False
         self.invalidate_stamp = False
         self.dump_signatures = []
         self.dry_run = False

bitbake/lib/bb/event.py

@@ -124,6 +124,7 @@ def fire_class_handlers(event, d):
 ui_queue = []
 @atexit.register
 def print_ui_queue():
+    global ui_queue
     """If we're exiting before a UI has been spawned, display any queued
     LogRecords to the console."""
     logger = logging.getLogger("BitBake")
@@ -168,6 +169,7 @@ def print_ui_queue():
             logger.removeHandler(stderr)
         else:
             logger.removeHandler(stdout)
+        ui_queue = []
 
 def fire_ui_handlers(event, d):
     global _thread_lock
@@ -402,23 +404,6 @@ class RecipeTaskPreProcess(RecipeEvent):
 class RecipeParsed(RecipeEvent):
     """ Recipe Parsing Complete """
 
-class StampUpdate(Event):
-    """Trigger for any adjustment of the stamp files to happen"""
-
-    def __init__(self, targets, stampfns):
-        self._targets = targets
-        self._stampfns = stampfns
-        Event.__init__(self)
-
-    def getStampPrefix(self):
-        return self._stampfns
-
-    def getTargets(self):
-        return self._targets
-
-    stampPrefix = property(getStampPrefix)
-    targets = property(getTargets)
-
 class BuildBase(Event):
     """Base class for bitbake build events"""
 

bitbake/lib/bb/fetch2/clearcase.py

@@ -54,6 +54,8 @@ import shutil
 import bb
 from   bb.fetch2 import FetchMethod
 from   bb.fetch2 import FetchError
+from   bb.fetch2 import MissingParameterError
+from   bb.fetch2 import ParameterError
 from   bb.fetch2 import runfetchcmd
 from   bb.fetch2 import logger
 
@@ -79,7 +81,7 @@ class ClearCase(FetchMethod):
         if 'protocol' in ud.parm:
             ud.proto = ud.parm['protocol']
         if not ud.proto in ('http', 'https'):
-            raise fetch2.ParameterError("Invalid protocol type", ud.url)
+            raise ParameterError("Invalid protocol type", ud.url)
 
         ud.vob = ''
         if 'vob' in ud.parm:

bitbake/lib/bb/main.py

@@ -1,4 +1,3 @@
-#!/usr/bin/env python
 #
 # Copyright (C) 2003, 2004  Chris Larson
 # Copyright (C) 2003, 2004  Phil Blundell
@@ -255,6 +254,11 @@ class BitBakeConfigParameters(cookerdata.ConfigParameters):
                           help="Do not run any setscene tasks. sstate will be ignored and "
                                "everything needed, built.")
 
+        parser.add_option("", "--skip-setscene", action="store_true",
+                          dest="skipsetscene", default=False,
+                          help="Skip setscene tasks if they would be executed. Tasks previously "
+                               "restored from sstate will be kept, unlike --no-setscene")
+
         parser.add_option("", "--setscene-only", action="store_true",
                           dest="setsceneonly", default=False,
                           help="Only run setscene tasks, don't run any real tasks.")
@@ -448,12 +452,7 @@ def setup_bitbake(configParams, configuration, extrafeatures=None):
                             bb.utils.unlockfile(lock)
                         raise bb.server.process.ProcessTimeout("Bitbake still shutting down as socket exists but no lock?")
                 if not configParams.server_only:
-                    try:
-                        server_connection = bb.server.process.connectProcessServer(sockname, featureset)
-                    except EOFError:
-                        # The server may have been shutting down but not closed the socket yet. If that happened,
-                        # ignore it.
-                        pass
+                    server_connection = bb.server.process.connectProcessServer(sockname, featureset)
 
                 if server_connection or configParams.server_only:
                     break
@@ -464,12 +463,13 @@ def setup_bitbake(configParams, configuration, extrafeatures=None):
                     raise
                 retries -= 1
                 tryno = 8 - retries
-                if isinstance(e, (bb.server.process.ProcessTimeout, BrokenPipeError)):
+                if isinstance(e, (bb.server.process.ProcessTimeout, BrokenPipeError, EOFError)):
                     logger.info("Retrying server connection (#%d)..." % tryno)
                 else:
                     logger.info("Retrying server connection (#%d)... (%s)" % (tryno, traceback.format_exc()))
             if not retries:
-                bb.fatal("Unable to connect to bitbake server, or start one")
+                bb.fatal("Unable to connect to bitbake server, or start one (server startup failures would be in bitbake-cookerdaemon.log).")
+            bb.event.print_ui_queue()
             if retries < 5:
                 time.sleep(5)
 

bitbake/lib/bb/monitordisk.py

@@ -1,4 +1,3 @@
-#!/usr/bin/env python
 #
 # Copyright (C) 2012 Robert Yang
 #

bitbake/lib/bb/namedtuple_with_abc.py

@@ -1,5 +1,4 @@
 # http://code.activestate.com/recipes/577629-namedtupleabc-abstract-base-class-mix-in-for-named/
-#!/usr/bin/env python
 # Copyright (c) 2011 Jan Kaliszewski (zuo). Available under the MIT License.
 #
 # SPDX-License-Identifier: MIT

bitbake/lib/bb/parse/parse_py/BBHandler.py

@@ -1,4 +1,3 @@
-#!/usr/bin/env python
 """
    class for handling .bb files
 

bitbake/lib/bb/parse/parse_py/ConfHandler.py

@@ -1,4 +1,3 @@
-#!/usr/bin/env python
 """
    class for handling configuration data files
 

bitbake/lib/bb/parse/parse_py/__init__.py

@@ -1,4 +1,3 @@
-#!/usr/bin/env python
 """
 BitBake Parsers
 

bitbake/lib/bb/siggen.py

@@ -49,7 +49,9 @@ class SignatureGenerator(object):
         return self.taskhash[task]
 
     def get_taskhash(self, fn, task, deps, dataCache):
-        return "0"
+        k = fn + "." + task
+        self.taskhash[k] = hashlib.sha256(k.encode("utf-8")).hexdigest()
+        return self.taskhash[k]
 
     def writeout_file_checksum_cache(self):
         """Write/update the file checksum cache onto disk"""
@@ -643,9 +645,9 @@ def compare_sigfiles(a, b, recursecb=None, color=False, collapsed=False):
     a_taint = a_data.get('taint', None)
     b_taint = b_data.get('taint', None)
     if a_taint != b_taint:
-        if a_taint.startswith('nostamp:'):
+        if a_taint and a_taint.startswith('nostamp:'):
             a_taint = a_taint.replace('nostamp:', 'nostamp(uuid4):')
-        if b_taint.startswith('nostamp:'):
+        if b_taint and b_taint.startswith('nostamp:'):
             b_taint = b_taint.replace('nostamp:', 'nostamp(uuid4):')
         output.append(color_format("{color_title}Taint (by forced/invalidated task) changed{color_default} from %s to %s") % (a_taint, b_taint))
 

bitbake/lib/bb/taskdata.py

@@ -1,4 +1,3 @@
-#!/usr/bin/env python
 """
 BitBake 'TaskData' implementation
 

bitbake/lib/bb/tests/event.py

@@ -561,14 +561,6 @@ class EventClassesTest(unittest.TestCase):
         self.assertEqual(event.fn(1), callback(1))
         self.assertEqual(event.pid, EventClassesTest._worker_pid)
 
-    def test_StampUpdate(self):
-        targets = ["foo", "bar"]
-        stampfns = [lambda:"foobar"]
-        event = bb.event.StampUpdate(targets, stampfns)
-        self.assertEqual(event.targets, targets)
-        self.assertEqual(event.stampPrefix, stampfns)
-        self.assertEqual(event.pid, EventClassesTest._worker_pid)
-
     def test_BuildBase(self):
         """ Test base class for bitbake build events """
         name = "foo"

bitbake/lib/bb/tests/fetch.py

@@ -899,6 +899,7 @@ class FetcherNetworkTest(FetcherTest):
         if os.path.exists(os.path.join(repo_path, 'bitbake-gitsm-test1')):
             self.assertTrue(os.path.exists(os.path.join(repo_path, 'bitbake-gitsm-test1', 'bitbake')), msg='submodule of submodule missing')
 
+    @skipIfNoNetwork()
     def test_git_submodule_dbus_broker(self):
         # The following external repositories have show failures in fetch and unpack operations
         # We want to avoid regressions!
@@ -916,6 +917,7 @@ class FetcherNetworkTest(FetcherTest):
         self.assertTrue(os.path.exists(os.path.join(repo_path, '.git/modules/subprojects/c-sundry/config')), msg='Missing submodule config "subprojects/c-sundry"')
         self.assertTrue(os.path.exists(os.path.join(repo_path, '.git/modules/subprojects/c-utf8/config')), msg='Missing submodule config "subprojects/c-utf8"')
 
+    @skipIfNoNetwork()
     def test_git_submodule_CLI11(self):
         url = "gitsm://github.com/CLIUtils/CLI11;protocol=git;rev=bd4dc911847d0cde7a6b41dfa626a85aab213baf"
         fetcher = bb.fetch.Fetch([url], self.d)
@@ -929,6 +931,7 @@ class FetcherNetworkTest(FetcherTest):
         self.assertTrue(os.path.exists(os.path.join(repo_path, '.git/modules/extern/json/config')), msg='Missing submodule config "extern/json"')
         self.assertTrue(os.path.exists(os.path.join(repo_path, '.git/modules/extern/sanitizers/config')), msg='Missing submodule config "extern/sanitizers"')
 
+    @skipIfNoNetwork()
     def test_git_submodule_update_CLI11(self):
         """ Prevent regression on update detection not finding missing submodule, or modules without needed commits """
         url = "gitsm://github.com/CLIUtils/CLI11;protocol=git;rev=cf6a99fa69aaefe477cc52e3ef4a7d2d7fa40714"
@@ -948,6 +951,7 @@ class FetcherNetworkTest(FetcherTest):
         self.assertTrue(os.path.exists(os.path.join(repo_path, '.git/modules/extern/json/config')), msg='Missing submodule config "extern/json"')
         self.assertTrue(os.path.exists(os.path.join(repo_path, '.git/modules/extern/sanitizers/config')), msg='Missing submodule config "extern/sanitizers"')
 
+    @skipIfNoNetwork()
     def test_git_submodule_aktualizr(self):
         url = "gitsm://github.com/advancedtelematic/aktualizr;branch=master;protocol=git;rev=d00d1a04cc2366d1a5f143b84b9f507f8bd32c44"
         fetcher = bb.fetch.Fetch([url], self.d)
@@ -964,6 +968,7 @@ class FetcherNetworkTest(FetcherTest):
         self.assertTrue(os.path.exists(os.path.join(repo_path, '.git/modules/third_party/googletest/config')), msg='Missing submodule config "third_party/googletest/config"')
         self.assertTrue(os.path.exists(os.path.join(repo_path, '.git/modules/third_party/HdrHistogram_c/config')), msg='Missing submodule config "third_party/HdrHistogram_c/config"')
 
+    @skipIfNoNetwork()
     def test_git_submodule_iotedge(self):
         """ Prevent regression on deeply nested submodules not being checked out properly, even though they were fetched. """
 

bitbake/lib/bb/tests/runqueue-tests/classes/base.bbclass

@@ -0,0 +1,242 @@
+SLOWTASKS ??= ""
+SSTATEVALID ??= ""
+
+def stamptask(d):
+    import time
+
+    thistask = d.expand("${PN}:${BB_CURRENTTASK}")
+    with open(d.expand("${TOPDIR}/%s.run") % thistask, "a+") as f:
+        f.write("\n")
+
+    if d.getVar("BB_CURRENT_MC") != "default":
+        thistask = d.expand("${BB_CURRENT_MC}:${PN}:${BB_CURRENTTASK}")
+    if thistask in d.getVar("SLOWTASKS").split():
+        bb.note("Slowing task %s" % thistask)
+        time.sleep(0.5)
+
+    with open(d.expand("${TOPDIR}/task.log"), "a+") as f:
+        f.write(thistask + "\n")
+
+python do_fetch() {
+    # fetch
+    stamptask(d)
+}
+python do_unpack() {
+    # unpack
+    stamptask(d)
+}
+python do_patch() {
+    # patch
+    stamptask(d)
+}
+python do_populate_lic() {
+    # populate_lic
+    stamptask(d)
+}
+python do_prepare_recipe_sysroot() {
+    # prepare_recipe_sysroot
+    stamptask(d)
+}
+python do_configure() {
+    # configure
+    stamptask(d)
+}
+python do_compile() {
+    # compile
+    stamptask(d)
+}
+python do_install() {
+    # install
+    stamptask(d)
+}
+python do_populate_sysroot() {
+    # populate_sysroot
+    stamptask(d)
+}
+python do_package() {
+    # package
+    stamptask(d)
+}
+python do_package_write_ipk() {
+    # package_write_ipk
+    stamptask(d)
+}
+python do_package_write_rpm() {
+    # package_write_rpm
+    stamptask(d)
+}
+python do_packagedata() {
+    # packagedata
+    stamptask(d)
+}
+python do_package_qa() {
+    # package_qa
+    stamptask(d)
+}
+python do_build() {
+    # build
+    stamptask(d)
+}
+do_prepare_recipe_sysroot[deptask] = "do_populate_sysroot"
+do_package[deptask] += "do_packagedata"
+do_build[recrdeptask] += "do_deploy"
+do_build[recrdeptask] += "do_package_write_ipk"
+do_build[recrdeptask] += "do_package_write_rpm"
+do_package_qa[rdeptask] = "do_packagedata"
+do_populate_lic_deploy[recrdeptask] += "do_populate_lic do_deploy"
+
+DEBIANRDEP = "do_packagedata"
+oo_package_write_ipk[rdeptask] = "${DEBIANRDEP}"
+do_package_write_rpm[rdeptask] = "${DEBIANRDEP}"
+
+addtask fetch
+addtask unpack after do_fetch
+addtask patch after do_unpack
+addtask prepare_recipe_sysroot after do_patch
+addtask configure after do_prepare_recipe_sysroot
+addtask compile after do_configure
+addtask install after do_compile
+addtask populate_sysroot after do_install
+addtask package after do_install
+addtask package_write_ipk after do_packagedata do_package
+addtask package_write_rpm after do_packagedata do_package
+addtask packagedata after do_package
+addtask package_qa after do_package
+addtask build after do_package_qa do_package_write_rpm do_package_write_ipk do_populate_sysroot
+
+python do_package_setscene() {
+    stamptask(d)
+}
+python do_package_qa_setscene() {
+    stamptask(d)
+}
+python do_package_write_ipk_setscene() {
+    stamptask(d)
+}
+python do_package_write_rpm_setscene() {
+    stamptask(d)
+}
+python do_packagedata_setscene() {
+    stamptask(d)
+}
+python do_populate_lic_setscene() {
+    stamptask(d)
+}
+python do_populate_sysroot_setscene() {
+    stamptask(d)
+}
+
+addtask package_setscene
+addtask package_qa_setscene
+addtask package_write_ipk_setscene
+addtask package_write_rpm_setscene
+addtask packagedata_setscene
+addtask populate_lic_setscene
+addtask populate_sysroot_setscene
+
+BB_SETSCENE_DEPVALID = "setscene_depvalid"
+
+def setscene_depvalid(task, taskdependees, notneeded, d, log=None):
+    # taskdependees is a dict of tasks which depend on task, each being a 3 item list of [PN, TASKNAME, FILENAME]
+    # task is included in taskdependees too
+    # Return - False - We need this dependency
+    #        - True - We can skip this dependency
+    import re
+
+    def logit(msg, log):
+        if log is not None:
+            log.append(msg)
+        else:
+            bb.debug(2, msg)
+
+    logit("Considering setscene task: %s" % (str(taskdependees[task])), log)
+
+    def isNativeCross(x):
+        return x.endswith("-native") or "-cross-" in x or "-crosssdk" in x or x.endswith("-cross")
+
+    # We only need to trigger populate_lic through direct dependencies
+    if taskdependees[task][1] == "do_populate_lic":
+        return True
+
+    # We only need to trigger packagedata through direct dependencies
+    # but need to preserve packagedata on packagedata links
+    if taskdependees[task][1] == "do_packagedata":
+        for dep in taskdependees:
+            if taskdependees[dep][1] == "do_packagedata":
+                return False
+        return True
+
+    for dep in taskdependees:
+        logit("  considering dependency: %s" % (str(taskdependees[dep])), log)
+        if task == dep:
+            continue
+        if dep in notneeded:
+            continue
+        # do_package_write_* and do_package doesn't need do_package
+        if taskdependees[task][1] == "do_package" and taskdependees[dep][1] in ['do_package', 'do_package_write_ipk', 'do_package_write_rpm', 'do_packagedata', 'do_package_qa']:
+            continue
+        # do_package_write_* need do_populate_sysroot as they're mainly postinstall dependencies
+        if taskdependees[task][1] == "do_populate_sysroot" and taskdependees[dep][1] in ['do_package_write_ipk', 'do_package_write_rpm']:
+            return False
+        # do_package/packagedata/package_qa don't need do_populate_sysroot
+        if taskdependees[task][1] == "do_populate_sysroot" and taskdependees[dep][1] in ['do_package', 'do_packagedata', 'do_package_qa']:
+            continue
+        # Native/Cross packages don't exist and are noexec anyway
+        if isNativeCross(taskdependees[dep][0]) and taskdependees[dep][1] in ['do_package_write_ipk', 'do_package_write_rpm', 'do_packagedata', 'do_package', 'do_package_qa']:
+            continue
+
+        # This is due to the [depends] in useradd.bbclass complicating matters
+        # The logic *is* reversed here due to the way hard setscene dependencies are injected
+        if (taskdependees[task][1] == 'do_package' or taskdependees[task][1] == 'do_populate_sysroot') and taskdependees[dep][0].endswith(('shadow-native', 'shadow-sysroot', 'base-passwd', 'pseudo-native')) and taskdependees[dep][1] == 'do_populate_sysroot':
+            continue
+
+        # Consider sysroot depending on sysroot tasks
+        if taskdependees[task][1] == 'do_populate_sysroot' and taskdependees[dep][1] == 'do_populate_sysroot':
+            # Native/Cross populate_sysroot need their dependencies
+            if isNativeCross(taskdependees[task][0]) and isNativeCross(taskdependees[dep][0]):
+                return False
+            # Target populate_sysroot depended on by cross tools need to be installed
+            if isNativeCross(taskdependees[dep][0]):
+                return False
+            # Native/cross tools depended upon by target sysroot are not needed
+            # Add an exception for shadow-native as required by useradd.bbclass
+            if isNativeCross(taskdependees[task][0]) and taskdependees[task][0] != 'shadow-native':
+                continue
+            # Target populate_sysroot need their dependencies
+            return False
+
+
+        if taskdependees[dep][1] == "do_populate_lic":
+            continue
+
+        # Safe fallthrough default
+        logit(" Default setscene dependency fall through due to dependency: %s" % (str(taskdependees[dep])), log)
+        return False
+    return True
+
+BB_HASHCHECK_FUNCTION = "sstate_checkhashes"
+
+def sstate_checkhashes(sq_fn, sq_task, sq_hash, sq_hashfn, d, siginfo=False, *, sq_unihash=None):
+
+    ret = []
+    missed = []
+
+    valid = d.getVar("SSTATEVALID").split()
+
+    for task in range(len(sq_fn)):
+        n = os.path.basename(sq_fn[task]).rsplit(".", 1)[0] + ":" + sq_task[task]
+        if n in valid:
+            bb.note("SState: Found valid sstate for %s" % n)
+            ret.append(task)
+        elif os.path.exists(d.expand("${TOPDIR}/%s.run" % n.replace("do_", ""))):
+            bb.note("SState: Found valid sstate for %s (already run)" % n)
+            ret.append(task)
+        else:
+            missed.append(task)
+            bb.note("SState: Found no valid sstate for %s" % n)
+
+    if hasattr(bb.parse.siggen, "checkhashes"):
+        bb.parse.siggen.checkhashes(missed, ret, sq_fn, sq_task, sq_hash, sq_hashfn, d)
+
+    return ret
+

bitbake/lib/bb/tests/runqueue-tests/classes/image.bbclass

@@ -0,0 +1,5 @@
+do_rootfs[recrdeptask] += "do_package_write_deb do_package_qa"
+do_rootfs[recrdeptask] += "do_package_write_ipk do_package_qa"
+do_rootfs[recrdeptask] += "do_package_write_rpm do_package_qa
+do_rootfs[recrdeptask] += "do_packagedata"
+do_rootfs[recrdeptask] += "do_populate_lic"

bitbake/lib/bb/tests/runqueue-tests/classes/native.bbclass

@@ -0,0 +1,2 @@
+RECIPERDEPTASK = "do_populate_sysroot"
+do_populate_sysroot[rdeptask] = "${RECIPERDEPTASK}"

bitbake/lib/bb/tests/runqueue-tests/conf/bitbake.conf

@@ -0,0 +1,16 @@
+CACHE = "${TOPDIR}/cache"
+THISDIR = "${@os.path.dirname(d.getVar('FILE'))}"
+COREBASE := "${@os.path.normpath(os.path.dirname(d.getVar('FILE')+'/../../'))}"
+BBFILES = "${COREBASE}/recipes/*.bb"
+PROVIDES = "${PN}"
+PN = "${@bb.parse.vars_from_file(d.getVar('FILE', False),d)[0]}"
+PF = "${BB_CURRENT_MC}:${PN}"
+export PATH
+TMPDIR ??= "${TOPDIR}"
+STAMP = "${TMPDIR}/stamps/${PN}"
+T = "${TMPDIR}/workdir/${PN}/temp"
+BB_NUMBER_THREADS = "4"
+
+BB_HASHBASE_WHITELIST = "BB_CURRENT_MC"
+
+include conf/multiconfig/${BB_CURRENT_MC}.conf

bitbake/lib/bb/tests/runqueue-tests/conf/multiconfig/mc1.conf

@@ -0,0 +1 @@
+TMPDIR = "${TOPDIR}/mc1/"

bitbake/lib/bb/tests/runqueue-tests/conf/multiconfig/mc2.conf

@@ -0,0 +1 @@
+TMPDIR = "${TOPDIR}/mc2/"

bitbake/lib/bb/tests/runqueue-tests/recipes/b1.bb

@@ -0,0 +1 @@
+DEPENDS = "a1"

bitbake/lib/bb/tests/runqueue-tests/recipes/d1.bb

@@ -0,0 +1,3 @@
+DEPENDS = "a1"
+
+do_package_setscene[depends] = "a1:do_populate_sysroot_setscene"

bitbake/lib/bb/tests/runqueue.py

@@ -0,0 +1,228 @@
+#
+# BitBake Tests for runqueue task processing
+#
+# Copyright (C) 2019 Richard Purdie
+#
+# SPDX-License-Identifier: GPL-2.0-only
+#
+
+import unittest
+import bb
+import os
+import tempfile
+import subprocess
+
+#
+# TODO:
+# Add tests on task ordering (X happens before Y after Z)
+#
+
+class RunQueueTests(unittest.TestCase):
+
+    alltasks = ['package', 'fetch', 'unpack', 'patch', 'prepare_recipe_sysroot', 'configure',
+                'compile', 'install', 'packagedata', 'package_qa', 'package_write_rpm', 'package_write_ipk',
+                'populate_sysroot', 'build']
+    a1_sstatevalid = "a1:do_package a1:do_package_qa a1:do_packagedata a1:do_package_write_ipk a1:do_package_write_rpm a1:do_populate_lic a1:do_populate_sysroot"
+    b1_sstatevalid = "b1:do_package b1:do_package_qa b1:do_packagedata b1:do_package_write_ipk b1:do_package_write_rpm b1:do_populate_lic b1:do_populate_sysroot"
+
+    def run_bitbakecmd(self, cmd, builddir, sstatevalid="", slowtasks="", extraenv=None):
+        env = os.environ.copy()
+        env["BBPATH"] = os.path.realpath(os.path.join(os.path.dirname(__file__), "runqueue-tests"))
+        env["BB_ENV_EXTRAWHITE"] = "SSTATEVALID SLOWTASKS"
+        env["SSTATEVALID"] = sstatevalid
+        env["SLOWTASKS"] = slowtasks
+        if extraenv:
+            for k in extraenv:
+                env[k] = extraenv[k]
+                env["BB_ENV_EXTRAWHITE"] = env["BB_ENV_EXTRAWHITE"] + " " + k
+        try:
+            output = subprocess.check_output(cmd, env=env, stderr=subprocess.STDOUT,universal_newlines=True, cwd=builddir)
+        except subprocess.CalledProcessError as e:
+            self.fail("Command %s failed with %s" % (cmd, e.output))
+        tasks = []
+        with open(builddir + "/task.log", "r") as f:
+            tasks = [line.rstrip() for line in f]
+        return tasks
+
+    def test_no_setscenevalid(self):
+        with tempfile.TemporaryDirectory(prefix="runqueuetest") as tempdir:
+            cmd = ["bitbake", "a1"]
+            sstatevalid = ""
+            tasks = self.run_bitbakecmd(cmd, tempdir, sstatevalid)
+            expected = ['a1:' + x for x in self.alltasks]
+            self.assertEqual(set(tasks), set(expected))
+
+    def test_single_setscenevalid(self):
+        with tempfile.TemporaryDirectory(prefix="runqueuetest") as tempdir:
+            cmd = ["bitbake", "a1"]
+            sstatevalid = "a1:do_package"
+            tasks = self.run_bitbakecmd(cmd, tempdir, sstatevalid)
+            expected = ['a1:package_setscene', 'a1:fetch', 'a1:unpack', 'a1:patch', 'a1:prepare_recipe_sysroot', 'a1:configure',
+                        'a1:compile', 'a1:install', 'a1:packagedata', 'a1:package_qa', 'a1:package_write_rpm', 'a1:package_write_ipk',
+                        'a1:populate_sysroot', 'a1:build']
+            self.assertEqual(set(tasks), set(expected))
+
+    def test_intermediate_setscenevalid(self):
+        with tempfile.TemporaryDirectory(prefix="runqueuetest") as tempdir:
+            cmd = ["bitbake", "a1"]
+            sstatevalid = "a1:do_package a1:do_populate_sysroot"
+            tasks = self.run_bitbakecmd(cmd, tempdir, sstatevalid)
+            expected = ['a1:package_setscene', 'a1:packagedata', 'a1:package_qa', 'a1:package_write_rpm', 'a1:package_write_ipk',
+                        'a1:populate_sysroot_setscene', 'a1:build']
+            self.assertEqual(set(tasks), set(expected))
+
+    def test_intermediate_notcovered(self):
+        with tempfile.TemporaryDirectory(prefix="runqueuetest") as tempdir:
+            cmd = ["bitbake", "a1"]
+            sstatevalid = "a1:do_package_qa a1:do_packagedata a1:do_package_write_ipk a1:do_package_write_rpm a1:do_populate_lic a1:do_populate_sysroot"
+            tasks = self.run_bitbakecmd(cmd, tempdir, sstatevalid)
+            expected = ['a1:package_write_ipk_setscene', 'a1:package_write_rpm_setscene', 'a1:packagedata_setscene',
+                        'a1:package_qa_setscene', 'a1:build', 'a1:populate_sysroot_setscene']
+            self.assertEqual(set(tasks), set(expected))
+
+    def test_all_setscenevalid(self):
+        with tempfile.TemporaryDirectory(prefix="runqueuetest") as tempdir:
+            cmd = ["bitbake", "a1"]
+            sstatevalid = self.a1_sstatevalid
+            tasks = self.run_bitbakecmd(cmd, tempdir, sstatevalid)
+            expected = ['a1:package_write_ipk_setscene', 'a1:package_write_rpm_setscene', 'a1:packagedata_setscene',
+                        'a1:package_qa_setscene', 'a1:build', 'a1:populate_sysroot_setscene']
+            self.assertEqual(set(tasks), set(expected))
+
+    def test_no_settasks(self):
+        with tempfile.TemporaryDirectory(prefix="runqueuetest") as tempdir:
+            cmd = ["bitbake", "a1", "-c", "patch"]
+            sstatevalid = self.a1_sstatevalid
+            tasks = self.run_bitbakecmd(cmd, tempdir, sstatevalid)
+            expected = ['a1:fetch', 'a1:unpack', 'a1:patch']
+            self.assertEqual(set(tasks), set(expected))
+
+    def test_mix_covered_notcovered(self):
+        with tempfile.TemporaryDirectory(prefix="runqueuetest") as tempdir:
+            cmd = ["bitbake", "a1:do_patch", "a1:do_populate_sysroot"]
+            sstatevalid = self.a1_sstatevalid
+            tasks = self.run_bitbakecmd(cmd, tempdir, sstatevalid)
+            expected = ['a1:fetch', 'a1:unpack', 'a1:patch', 'a1:populate_sysroot_setscene']
+            self.assertEqual(set(tasks), set(expected))
+
+
+    # Test targets with intermediate setscene tasks alongside a target with no intermediate setscene tasks
+    def test_mixed_direct_tasks_setscene_tasks(self):
+        with tempfile.TemporaryDirectory(prefix="runqueuetest") as tempdir:
+            cmd = ["bitbake", "c1:do_patch", "a1"]
+            sstatevalid = self.a1_sstatevalid
+            tasks = self.run_bitbakecmd(cmd, tempdir, sstatevalid)
+            expected = ['c1:fetch', 'c1:unpack', 'c1:patch', 'a1:package_write_ipk_setscene', 'a1:package_write_rpm_setscene', 'a1:packagedata_setscene',
+                        'a1:package_qa_setscene', 'a1:build', 'a1:populate_sysroot_setscene']
+            self.assertEqual(set(tasks), set(expected))
+
+    # This test slows down the execution of do_package_setscene until after other real tasks have
+    # started running which tests for a bug where tasks were being lost from the buildable list of real
+    # tasks if they weren't in tasks_covered or tasks_notcovered
+    def test_slow_setscene(self):
+        with tempfile.TemporaryDirectory(prefix="runqueuetest") as tempdir:
+            cmd = ["bitbake", "a1"]
+            sstatevalid = "a1:do_package"
+            slowtasks = "a1:package_setscene"
+            tasks = self.run_bitbakecmd(cmd, tempdir, sstatevalid, slowtasks)
+            expected = ['a1:package_setscene', 'a1:fetch', 'a1:unpack', 'a1:patch', 'a1:prepare_recipe_sysroot', 'a1:configure',
+                        'a1:compile', 'a1:install', 'a1:packagedata', 'a1:package_qa', 'a1:package_write_rpm', 'a1:package_write_ipk',
+                        'a1:populate_sysroot', 'a1:build']
+            self.assertEqual(set(tasks), set(expected))
+
+    def test_setscenewhitelist(self):
+        with tempfile.TemporaryDirectory(prefix="runqueuetest") as tempdir:
+            cmd = ["bitbake", "a1"]
+            extraenv = {
+                "BB_SETSCENE_ENFORCE" : "1",
+                "BB_SETSCENE_ENFORCE_WHITELIST" : "a1:do_package_write_rpm a1:do_build"
+            }
+            sstatevalid = "a1:do_package a1:do_package_qa a1:do_packagedata a1:do_package_write_ipk a1:do_populate_lic a1:do_populate_sysroot"
+            tasks = self.run_bitbakecmd(cmd, tempdir, sstatevalid, extraenv=extraenv)
+            expected = ['a1:packagedata_setscene', 'a1:package_qa_setscene', 'a1:package_write_ipk_setscene',
+                        'a1:populate_sysroot_setscene', 'a1:package_setscene']
+            self.assertEqual(set(tasks), set(expected))
+
+    # Tests for problems with dependencies between setscene tasks
+    def test_no_setscenevalid_harddeps(self):
+        with tempfile.TemporaryDirectory(prefix="runqueuetest") as tempdir:
+            cmd = ["bitbake", "d1"]
+            sstatevalid = ""
+            tasks = self.run_bitbakecmd(cmd, tempdir, sstatevalid)
+            expected = ['a1:package', 'a1:fetch', 'a1:unpack', 'a1:patch', 'a1:prepare_recipe_sysroot', 'a1:configure',
+                        'a1:compile', 'a1:install', 'a1:packagedata', 'a1:package_write_rpm', 'a1:package_write_ipk',
+                        'a1:populate_sysroot', 'd1:package', 'd1:fetch', 'd1:unpack', 'd1:patch', 'd1:prepare_recipe_sysroot', 'd1:configure',
+                        'd1:compile', 'd1:install', 'd1:packagedata', 'd1:package_qa', 'd1:package_write_rpm', 'd1:package_write_ipk',
+                        'd1:populate_sysroot', 'd1:build']
+            self.assertEqual(set(tasks), set(expected))
+
+    def test_no_setscenevalid_withdeps(self):
+        with tempfile.TemporaryDirectory(prefix="runqueuetest") as tempdir:
+            cmd = ["bitbake", "b1"]
+            sstatevalid = ""
+            tasks = self.run_bitbakecmd(cmd, tempdir, sstatevalid)
+            expected = ['a1:' + x for x in self.alltasks] + ['b1:' + x for x in self.alltasks]
+            expected.remove('a1:build')
+            expected.remove('a1:package_qa')
+            self.assertEqual(set(tasks), set(expected))
+
+    def test_single_a1_setscenevalid_withdeps(self):
+        with tempfile.TemporaryDirectory(prefix="runqueuetest") as tempdir:
+            cmd = ["bitbake", "b1"]
+            sstatevalid = "a1:do_package"
+            tasks = self.run_bitbakecmd(cmd, tempdir, sstatevalid)
+            expected = ['a1:package_setscene', 'a1:fetch', 'a1:unpack', 'a1:patch', 'a1:prepare_recipe_sysroot', 'a1:configure',
+                        'a1:compile', 'a1:install', 'a1:packagedata', 'a1:package_write_rpm', 'a1:package_write_ipk',
+                        'a1:populate_sysroot'] + ['b1:' + x for x in self.alltasks]
+            self.assertEqual(set(tasks), set(expected))
+
+    def test_single_b1_setscenevalid_withdeps(self):
+        with tempfile.TemporaryDirectory(prefix="runqueuetest") as tempdir:
+            cmd = ["bitbake", "b1"]
+            sstatevalid = "b1:do_package"
+            tasks = self.run_bitbakecmd(cmd, tempdir, sstatevalid)
+            expected = ['a1:package', 'a1:fetch', 'a1:unpack', 'a1:patch', 'a1:prepare_recipe_sysroot', 'a1:configure',
+                        'a1:compile', 'a1:install', 'a1:packagedata', 'a1:package_write_rpm', 'a1:package_write_ipk',
+                        'a1:populate_sysroot', 'b1:package_setscene'] + ['b1:' + x for x in self.alltasks]
+            expected.remove('b1:package')
+            self.assertEqual(set(tasks), set(expected))
+
+    def test_intermediate_setscenevalid_withdeps(self):
+        with tempfile.TemporaryDirectory(prefix="runqueuetest") as tempdir:
+            cmd = ["bitbake", "b1"]
+            sstatevalid = "a1:do_package a1:do_populate_sysroot b1:do_package"
+            tasks = self.run_bitbakecmd(cmd, tempdir, sstatevalid)
+            expected = ['a1:package_setscene', 'a1:packagedata', 'a1:package_write_rpm', 'a1:package_write_ipk',
+                        'a1:populate_sysroot_setscene', 'b1:package_setscene'] + ['b1:' + x for x in self.alltasks]
+            expected.remove('b1:package')
+            self.assertEqual(set(tasks), set(expected))
+
+    def test_all_setscenevalid_withdeps(self):
+        with tempfile.TemporaryDirectory(prefix="runqueuetest") as tempdir:
+            cmd = ["bitbake", "b1"]
+            sstatevalid = self.a1_sstatevalid + " " + self.b1_sstatevalid
+            tasks = self.run_bitbakecmd(cmd, tempdir, sstatevalid)
+            expected = ['a1:package_write_ipk_setscene', 'a1:package_write_rpm_setscene', 'a1:packagedata_setscene',
+                        'b1:build', 'a1:populate_sysroot_setscene', 'b1:package_write_ipk_setscene', 'b1:package_write_rpm_setscene',
+                        'b1:packagedata_setscene', 'b1:package_qa_setscene', 'b1:populate_sysroot_setscene']
+            self.assertEqual(set(tasks), set(expected))
+
+    def test_multiconfig_setscene_optimise(self):
+        with tempfile.TemporaryDirectory(prefix="runqueuetest") as tempdir:
+            extraenv = {
+                "BBMULTICONFIG" : "mc1 mc2",
+                "BB_SIGNATURE_HANDLER" : "basic"
+            }
+            cmd = ["bitbake", "b1", "mc:mc1:b1", "mc:mc2:b1"]
+            setscenetasks = ['package_write_ipk_setscene', 'package_write_rpm_setscene', 'packagedata_setscene',
+                             'populate_sysroot_setscene', 'package_qa_setscene']
+            sstatevalid = ""
+            tasks = self.run_bitbakecmd(cmd, tempdir, sstatevalid, extraenv=extraenv)
+            expected = ['a1:' + x for x in self.alltasks] + ['b1:' + x for x in self.alltasks] + \
+                       ['mc1:b1:' + x for x in setscenetasks] + ['mc1:a1:' + x for x in setscenetasks] + \
+                       ['mc2:b1:' + x for x in setscenetasks] + ['mc2:a1:' + x for x in setscenetasks] + \
+                       ['mc1:b1:build', 'mc2:b1:build']
+            for x in ['mc1:a1:package_qa_setscene', 'mc2:a1:package_qa_setscene', 'a1:build', 'a1:package_qa']:
+                expected.remove(x)
+            self.assertEqual(set(tasks), set(expected))
+

bitbake/lib/bb/tests/utils.py

@@ -29,6 +29,10 @@ class VerCmpString(unittest.TestCase):
         self.assertTrue(result < 0)
         result = bb.utils.vercmp_string('1.1', '1.0+1.1-beta1')
         self.assertTrue(result > 0)
+        result = bb.utils.vercmp_string('1a', '1a1')
+        self.assertTrue(result < 0)
+        result = bb.utils.vercmp_string('1a1', '1a')
+        self.assertTrue(result > 0)
         result = bb.utils.vercmp_string('1.', '1.1')
         self.assertTrue(result < 0)
         result = bb.utils.vercmp_string('1.1', '1.')

bitbake/lib/bb/ui/knotty.py

@@ -660,7 +660,6 @@ def main(server, eventHandler, params, tf = TerminalFilter):
             # ignore
             if isinstance(event, (bb.event.BuildBase,
                                   bb.event.MetadataEvent,
-                                  bb.event.StampUpdate,
                                   bb.event.ConfigParsed,
                                   bb.event.MultiConfigParsed,
                                   bb.event.RecipeParsed,

bitbake/lib/bb/ui/uihelper.py

@@ -40,7 +40,7 @@ class BBUIHelper:
             self.running_pids.remove(event.pid)
             self.failed_tasks.append( { 'title' : "%s %s" % (event._package, event._task)})
             self.needUpdate = True
-        elif isinstance(event, bb.runqueue.runQueueTaskStarted) or isinstance(event, bb.runqueue.sceneQueueTaskStarted):
+        elif isinstance(event, bb.runqueue.runQueueTaskStarted):
             self.tasknumber_current = event.stats.completed + event.stats.active + event.stats.failed + 1
             self.tasknumber_total = event.stats.total
             self.needUpdate = True

bitbake/lib/pyinotify.py

@@ -1,5 +1,4 @@
-#!/usr/bin/env python
-
+#
 # pyinotify.py - python interface to inotify
 # Copyright (c) 2005-2015 Sebastien Martini <seb@dbzteam.org>
 #

bitbake/lib/toaster/tests/browser/selenium_helpers.py

@@ -1,4 +1,4 @@
-#! /usr/bin/env python
+#! /usr/bin/env python3
 #
 # BitBake Toaster Implementation
 #

bitbake/lib/toaster/tests/browser/selenium_helpers_base.py

@@ -1,4 +1,4 @@
-#! /usr/bin/env python
+#! /usr/bin/env python3
 #
 # BitBake Toaster Implementation
 #

bitbake/lib/toaster/tests/browser/test_all_builds_page.py

@@ -1,4 +1,4 @@
-#! /usr/bin/env python
+#! /usr/bin/env python3
 #
 # BitBake Toaster Implementation
 #

bitbake/lib/toaster/tests/browser/test_all_projects_page.py

@@ -1,4 +1,4 @@
-#! /usr/bin/env python
+#! /usr/bin/env python3
 #
 # BitBake Toaster Implementation
 #

bitbake/lib/toaster/tests/browser/test_builddashboard_page.py

@@ -1,4 +1,4 @@
-#! /usr/bin/env python
+#! /usr/bin/env python3
 #
 # BitBake Toaster Implementation
 #

bitbake/lib/toaster/tests/browser/test_builddashboard_page_artifacts.py

@@ -1,4 +1,4 @@
-#! /usr/bin/env python
+#! /usr/bin/env python3
 #
 # BitBake Toaster Implementation
 #

bitbake/lib/toaster/tests/browser/test_builddashboard_page_recipes.py

@@ -1,4 +1,4 @@
-#! /usr/bin/env python
+#! /usr/bin/env python3
 #
 # BitBake Toaster Implementation
 #

bitbake/lib/toaster/tests/browser/test_builddashboard_page_tasks.py

@@ -1,4 +1,4 @@
-#! /usr/bin/env python
+#! /usr/bin/env python3
 #
 # BitBake Toaster Implementation
 #

bitbake/lib/toaster/tests/browser/test_js_unit_tests.py

@@ -1,4 +1,4 @@
-#! /usr/bin/env python
+#! /usr/bin/env python3
 #
 # BitBake Toaster Implementation
 #

bitbake/lib/toaster/tests/browser/test_landing_page.py

@@ -1,4 +1,4 @@
-#! /usr/bin/env python
+#! /usr/bin/env python3
 #
 # BitBake Toaster Implementation
 #

bitbake/lib/toaster/tests/browser/test_layerdetails_page.py

@@ -1,4 +1,4 @@
-#! /usr/bin/env python
+#! /usr/bin/env python3
 #
 # BitBake Toaster Implementation
 #

bitbake/lib/toaster/tests/browser/test_most_recent_builds_states.py

@@ -1,4 +1,4 @@
-#! /usr/bin/env python
+#! /usr/bin/env python3
 #
 # BitBake Toaster Implementation
 #

bitbake/lib/toaster/tests/browser/test_new_custom_image_page.py

@@ -1,4 +1,4 @@
-#! /usr/bin/env python
+#! /usr/bin/env python3
 #
 # BitBake Toaster Implementation
 #

bitbake/lib/toaster/tests/browser/test_new_project_page.py

@@ -1,4 +1,4 @@
-#! /usr/bin/env python
+#! /usr/bin/env python3
 #
 # BitBake Toaster Implementation
 #

bitbake/lib/toaster/tests/browser/test_project_builds_page.py

@@ -1,4 +1,4 @@
-#! /usr/bin/env python
+#! /usr/bin/env python3
 #
 # BitBake Toaster Implementation
 #

bitbake/lib/toaster/tests/browser/test_project_config_page.py

@@ -1,4 +1,4 @@
-#! /usr/bin/env python
+#! /usr/bin/env python3
 #
 # BitBake Toaster Implementation
 #

bitbake/lib/toaster/tests/browser/test_project_page.py

@@ -1,4 +1,4 @@
-#! /usr/bin/env python
+#! /usr/bin/env python3
 #
 # BitBake Toaster Implementation
 #

bitbake/lib/toaster/tests/browser/test_sample.py

@@ -1,4 +1,4 @@
-#! /usr/bin/env python
+#! /usr/bin/env python3
 #
 # BitBake Toaster Implementation
 #

bitbake/lib/toaster/tests/browser/test_task_page.py

@@ -1,4 +1,4 @@
-#! /usr/bin/env python
+#! /usr/bin/env python3
 #
 # BitBake Toaster Implementation
 #

bitbake/lib/toaster/tests/browser/test_toastertable_ui.py

@@ -1,4 +1,4 @@
-#! /usr/bin/env python
+#! /usr/bin/env python3
 #
 # BitBake Toaster Implementation
 #

bitbake/lib/toaster/tests/builds/buildtest.py

@@ -1,4 +1,4 @@
-#! /usr/bin/env python
+#! /usr/bin/env python3
 #
 # BitBake Toaster Implementation
 #

bitbake/lib/toaster/tests/builds/test_core_image_min.py

@@ -1,4 +1,4 @@
-#! /usr/bin/env python
+#! /usr/bin/env python3
 #
 # BitBake Toaster Implementation
 #

bitbake/lib/toaster/tests/commands/test_loaddata.py

@@ -1,4 +1,4 @@
-#! /usr/bin/env python
+#! /usr/bin/env python3
 #
 # BitBake Toaster Implementation
 #

bitbake/lib/toaster/tests/commands/test_lsupdates.py

@@ -1,4 +1,4 @@
-#! /usr/bin/env python
+#! /usr/bin/env python3
 #
 # BitBake Toaster Implementation
 #

bitbake/lib/toaster/tests/commands/test_runbuilds.py

@@ -1,4 +1,4 @@
-#! /usr/bin/env python
+#! /usr/bin/env python3
 #
 # BitBake Toaster Implementation
 #

bitbake/lib/toaster/tests/eventreplay/__init__.py

@@ -1,4 +1,4 @@
-#! /usr/bin/env python
+#! /usr/bin/env python3
 #
 # BitBake Toaster Implementation
 #

bitbake/lib/toaster/tests/functional/functional_helpers.py

@@ -1,4 +1,4 @@
-#! /usr/bin/env python
+#! /usr/bin/env python3
 #
 # BitBake Toaster functional tests implementation
 #

bitbake/lib/toaster/tests/functional/test_functional_basic.py

@@ -1,4 +1,4 @@
-#! /usr/bin/env python
+#! /usr/bin/env python3
 #
 # BitBake Toaster functional tests implementation
 #

bitbake/lib/toaster/tests/views/test_views.py

@@ -1,4 +1,4 @@
-#! /usr/bin/env python
+#! /usr/bin/env python3
 #
 # BitBake Toaster Implementation
 #

bitbake/lib/toaster/toastermain/management/commands/checksocket.py

@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
 #
 # BitBake Toaster Implementation
 #

documentation/dev-manual/dev-manual-common-tasks.xml

@@ -7129,7 +7129,8 @@
             This problem is solved with the OpenEmbedded build system by
             running the code through QEMU, which allows precisely that.
             Unfortunately, QEMU does not always work perfectly as mentioned
-            in the xxx section.
+            in the
+            "<link linkend='known-issues'>Known Issues</link>" section.
         </para>
 
         <section id='enabling-the-generation-of-introspection-data'>

documentation/overview-manual/overview-manual.xml

@@ -36,6 +36,11 @@
                 <date>May 2018</date>
                 <revremark>The initial document released with the Yocto Project 2.5 Release.</revremark>
             </revision>
+            <revision>
+                <revnumber>2.6</revnumber>
+                <date>November 2018</date>
+                <revremark>Released with the Yocto Project 2.7 Release.</revremark>
+            </revision>
             <revision>
                 <revnumber>2.7</revnumber>
                 <date>May 2019</date>

documentation/ref-manual/ref-variables.xml

@@ -1358,7 +1358,7 @@
                     For example, the following line specifies three
                     configuration files:
                     <literallayout class='monospaced'>
-     BBMULTIFONFIG = "configA configB configC"
+     BBMULTICONFIG = "configA configB configC"
                     </literallayout>
                     Each configuration file you use must reside in the
                     <link linkend='build-directory'>Build Directory</link>
@@ -11694,8 +11694,7 @@
                     or
                     <link linkend='ref-classes-setuptools'><filename>setuptools</filename></link>
                     classes, specifies the major Python version being built.
-                    For Python 2.x, <filename>PYTHON_PN</filename> would
-                    be "python2".  For Python 3.x, the variable would be
+                    For Python 3.x, <filename>PYTHON_PN</filename> would be
                     "python3".
                     You do not have to set this variable as the
                     OpenEmbedded build system automatically sets it for you.

documentation/sdk-manual/sdk-extensible.xml

@@ -618,7 +618,23 @@
                                 The result is that the command sets up both
                                 the source code and an append file within the
                                 workspace while the recipe remains in its
-                                original location.
+                                original location.</para>
+
+                                <para>Additionally, if you have any non-patch
+                                local files (i.e. files referred to with
+                                <filename>file://</filename> entries in
+                                <filename>SRC_URI</filename> statement excluding
+                                <filename>*.patch/</filename> or
+                                <filename>*.diff</filename>), these files are
+                                copied to an
+                                <filename>oe-local-files</filename> folder
+                                under the newly created source tree.
+                                Copying the files here gives you a convenient
+                                area from which you can modify the files.
+                                Any changes or additions you make to those
+                                files are incorporated into the build the next
+                                time you build the software just as are other
+                                changes you might have made to the source.
                                 </para></listitem>
                             <listitem><para>
                                 <emphasis>Middle</emphasis>:
@@ -633,10 +649,10 @@
                                 </para>
 
                                 <para>The following command tells
-                                <filename>devtool</filename> what recipe with
+                                <filename>devtool</filename> the recipe with
                                 which to work and, in this case, identifies a
                                 local area for the extracted source files that
-                                is outside of the default
+                                exists outside of the default
                                 <filename>devtool</filename> workspace:
                                 <literallayout class='monospaced'>
      $ devtool modify <replaceable>recipe srctree</replaceable>
@@ -650,8 +666,12 @@
                                 the recipe's <filename>SRC_URI</filename>
                                 statements to locate the source files and any
                                 associated patch files.
-                                Once the files are located, the command by
-                                default extracts them into
+                                Non-patch files are copied to an
+                                <filename>oe-local-files</filename> folder
+                                under the newly created source tree.</para>
+
+                                <para>Once the files are located, the command
+                                by default extracts them into
                                 <replaceable>srctree</replaceable>.</para>
 
                                 <para>Within workspace,
@@ -685,9 +705,21 @@
                                 </literallayout>
                                 </para>
 
-                                <para>Once the command finishes, it creates only
-                                an append file for the recipe in the
-                                <filename>devtool</filename> workspace.
+                                <para>If an <filename>oe-local-files</filename>
+                                subdirectory happens to exist and it contains
+                                non-patch files, the files are used.
+                                However, if the subdirectory does not exist and
+                                you run the <filename>devtool finish</filename>
+                                command, any non-patch files that might exist
+                                next to the recipe are removed because it
+                                appears to <filename>devtool</filename> that
+                                you have deleted those files.</para>
+
+                                <para>Once the
+                                <filename>devtool modify</filename> command
+                                finishes, it creates only an append file for
+                                the recipe in the <filename>devtool</filename>
+                                workspace.
                                 The recipe and the source code remain in their
                                 original locations.
                                 </para></listitem>
@@ -778,7 +810,12 @@
                         original recipe in the original layer or the command
                         creates a <filename>.bbappend</filename> file in a
                         different layer as provided by
-                        <replaceable>layer</replaceable>.</para>
+                        <replaceable>layer</replaceable>.
+                        Any work you did in the
+                        <filename>oe-local-files</filename> directory is
+                        preserved in the original files next to the recipe
+                        during the <filename>devtool finish</filename>
+                        command.</para>
 
                         <para>As a final process of the
                         <filename>devtool finish</filename> command, the state
@@ -898,7 +935,23 @@
                         files from other developers.
                         The result is that the command sets up the source
                         code, the new version of the recipe, and an append file
-                        all within the workspace.
+                        all within the workspace.</para>
+
+                        <para>Additionally, if you have any non-patch
+                        local files (i.e. files referred to with
+                        <filename>file://</filename> entries in
+                        <filename>SRC_URI</filename> statement excluding
+                        <filename>*.patch/</filename> or
+                        <filename>*.diff</filename>), these files are
+                        copied to an
+                        <filename>oe-local-files</filename> folder
+                        under the newly created source tree.
+                        Copying the files here gives you a convenient
+                        area from which you can modify the files.
+                        Any changes or additions you make to those
+                        files are incorporated into the build the next
+                        time you build the software just as are other
+                        changes you might have made to the source.
                         </para></listitem>
                     <listitem><para>
                         <emphasis>Resolve any Conflicts created by the Upgrade</emphasis>:
@@ -976,10 +1029,18 @@
                         Git repository, moves the new recipe to a more
                         permanent layer, and then resets the recipe so that
                         the recipe is built normally rather than from the
-                        workspace.
+                        workspace.</para>
+
+                        <para>Any work you did in the
+                        <filename>oe-local-files</filename> directory is
+                        preserved in the original files next to the recipe
+                        during the <filename>devtool finish</filename>
+                        command.</para>
+
+                        <para>
                         If you specify a destination layer that is the same as
                         the original source, then the old version of the
-                        recipe and associated files will be removed prior to
+                        recipe and associated files are removed prior to
                         adding the new version.
                         <literallayout class='monospaced'>
      $ devtool finish <replaceable>recipe layer</replaceable>

meta-poky/conf/distro/include/poky-distro-alt-test-config.inc

@@ -0,0 +1,2 @@
+DISTRO_FEATURES_append = " pam"
+PREFERRED_VERSION_linux-yocto_linuxstdbase ?= "4.19%"

meta-poky/conf/distro/poky-lsb.conf

@@ -4,8 +4,7 @@ require conf/distro/poky.conf
 DISTRO = "poky-lsb"
 DISTROOVERRIDES = "poky:linuxstdbase"
 
-DISTRO_FEATURES_append = " pam largefile opengl"
-PREFERRED_PROVIDER_virtual/libx11 = "libx11"
+DISTRO_FEATURES_append = " pam"
 
 # Ensure the kernel nfs server is enabled
 KERNEL_FEATURES_append_pn-linux-yocto = " features/nfsd/nfsd-enable.scc"

meta-poky/conf/distro/poky.conf

@@ -56,10 +56,13 @@ SANITY_TESTED_DISTROS ?= " \
             ubuntu-18.04 \n \
             fedora-28 \n \
             fedora-29 \n \
+            fedora-30 \n \
             centos-7 \n \
             debian-8 \n \
             debian-9 \n \
+            debian-10 \n \
             opensuse-42.3 \n \
+            opensuseleap-15.1 \n \
             "
 #
 # OELAYOUT_ABI allows us to notify users when the format of TMPDIR changes in

meta-poky/conf/local.conf.sample.extended

@@ -321,7 +321,6 @@ DISTRO_FEATURES_remove = "x11"
 # show an example of how it can be done
 # You'll also need your fortran recipe to depend on libgfortran
 #FORTRAN_forcevariable = ",fortran"
-#RUNTIMETARGET_append_pn-gcc-runtime = " libquadmath"
 
 #
 # Kernel image features

meta-selftest/recipes-test/fortran/files/hello.f95

@@ -0,0 +1,5 @@
+program helloworld
+
+      print * , "Hello World!"
+
+end program helloworld

meta-selftest/recipes-test/fortran/fortran-helloworld.bb

@@ -0,0 +1,25 @@
+SUMMARY = "Fortran Hello World"
+LICENSE = "MIT"
+LIC_FILES_CHKSUM = "file://${COREBASE}/meta/COPYING.MIT;md5=3da9cfbcb788c80a0384361b4de20420"
+
+DEPENDS = "libgfortran"
+
+SRC_URI = "file://hello.f95"
+
+# These set flags that Fortran doesn't support
+SECURITY_CFLAGS = ""
+SECURITY_LDFLAGS = ""
+
+do_compile() {
+	${HOST_PREFIX}gfortran ${HOST_CC_ARCH}${TOOLCHAIN_OPTIONS} ${LDFLAGS} ${WORKDIR}/hello.f95 -o ${B}/fortran-hello
+}
+
+do_install() {
+	install -d ${D}${bindir}
+	install ${B}/fortran-hello ${D}${bindir}
+}
+
+python () {
+    if not d.getVar("FORTRAN"):
+        raise bb.parse.SkipRecipe("Fortran isn't enabled")
+}

meta-selftest/wic/test_biosplusefi_plugin.wks

@@ -0,0 +1,6 @@
+# short-description: This file is used in oe-selftest wic module to test biosplusefi plugin
+
+part /boot --source bootimg-biosplusefi --sourceparams="loader=grub-efi" --active --align 1024 --use-uuid
+part / --source rootfs --fstype=ext4 --align 1024 --use-uuid
+
+bootloader  --timeout=0 --append="console=ttyS0,115200n8"

meta-yocto-bsp/conf/machine/beaglebone-yocto.conf

@@ -20,7 +20,7 @@ WKS_FILE ?= "beaglebone-yocto.wks"
 IMAGE_INSTALL_append = " kernel-devicetree kernel-image-zimage"
 do_image_wic[depends] += "mtools-native:do_populate_sysroot dosfstools-native:do_populate_sysroot u-boot:do_deploy"
 
-SERIAL_CONSOLES ?= "115200;ttyS0 115200;ttyO0"
+SERIAL_CONSOLES ?= "115200;ttyS0 115200;ttyO0 115200;ttyAMA0"
 SERIAL_CONSOLES_CHECK = "${SERIAL_CONSOLES}"
 
 PREFERRED_PROVIDER_virtual/kernel ?= "linux-yocto"
@@ -39,3 +39,21 @@ UBOOT_LOADADDRESS = "0x80008000"
 MACHINE_FEATURES = "usbgadget usbhost vfat alsa"
 
 IMAGE_BOOT_FILES ?= "u-boot.${UBOOT_SUFFIX} MLO zImage am335x-bone.dtb am335x-boneblack.dtb am335x-bonegreen.dtb"
+
+# support runqemu
+EXTRA_IMAGEDEPENDS += "qemu-native qemu-helper-native"
+IMAGE_CLASSES += "qemuboot"
+QB_DEFAULT_FSTYPE = "wic"
+QB_FSINFO = "wic:no-kernel-in-fs"
+QB_KERNEL_ROOT = "/dev/vda2"
+QB_SYSTEM_NAME = "qemu-system-arm"
+QB_MEM = "-m 512"
+QB_MACHINE = "-machine virt"
+QB_CPU = "-cpu cortex-a15"
+QB_KERNEL_CMDLINE_APPEND = "console=ttyAMA0 systemd.mask=systemd-networkd"
+QB_OPT_APPEND = "-device virtio-rng-device"
+QB_TAP_OPT = "-netdev tap,id=net0,ifname=@TAP@,script=no,downscript=no"
+QB_NETWORK_DEVICE = "-device virtio-net-device,netdev=net0,mac=@MAC@"
+QB_ROOTFS_OPT = "-drive id=disk0,file=@ROOTFS@,if=none,format=raw -device virtio-blk-device,drive=disk0"
+QB_SERIAL_OPT = ""
+QB_TCPSERIAL_OPT = "-device virtio-serial-device -chardev socket,id=virtcon,port=@PORT@,host=127.0.0.1 -device virtconsole,chardev=virtcon"

meta/classes/bluetooth.bbclass

@@ -1,14 +0,0 @@
-# Avoid code duplication in bluetooth-dependent recipes.
-
-# Define a variable that expands to the recipe (package) providing core
-# bluetooth support on the platform:
-# "" if bluetooth is not in DISTRO_FEATURES
-# else "bluez5" if bluez5 is in DISTRO_FEATURES
-# else "bluez4"
-
-# Use this with:
-#  inherit bluetooth
-#  PACKAGECONFIG ??= "${@bb.utils.contains('DISTRO_FEATURES', 'bluetooth', '${BLUEZ}', '', d)}
-#  PACKAGECONFIG[bluez4] = "--enable-bluez4,--disable-bluez4,bluez4"
-
-BLUEZ ?= "${@bb.utils.contains('DISTRO_FEATURES', 'bluetooth', bb.utils.contains('DISTRO_FEATURES', 'bluez5', 'bluez5', 'bluez4', d), '', d)}"

meta/classes/buildhistory.bbclass

@@ -60,15 +60,34 @@ SSTATEPOSTUNPACKFUNCS[vardepvalueexclude] .= "| buildhistory_emit_outputsigs"
 # When extending build history, derive your class from buildhistory.bbclass
 # and extend this list here with the additional files created by the derived
 # class.
-BUILDHISTORY_PRESERVE = "latest latest_srcrev"
+BUILDHISTORY_PRESERVE = "latest latest_srcrev sysroot"
 
 PATCH_GIT_USER_EMAIL ?= "buildhistory@oe"
 PATCH_GIT_USER_NAME ?= "OpenEmbedded"
 
+#
+# Write out the contents of the sysroot
+#
+buildhistory_emit_sysroot() {
+	mkdir --parents ${BUILDHISTORY_DIR_PACKAGE}
+	case ${CLASSOVERRIDE} in
+	class-native|class-cross|class-crosssdk)
+		BASE=${SYSROOT_DESTDIR}/${STAGING_DIR_NATIVE}
+		;;
+	*)
+		BASE=${SYSROOT_DESTDIR}
+		;;
+	esac
+	buildhistory_list_files_no_owners $BASE ${BUILDHISTORY_DIR_PACKAGE}/sysroot
+}
+
 #
 # Write out metadata about this package for comparison when writing future packages
 #
 python buildhistory_emit_pkghistory() {
+    if d.getVar('BB_CURRENTTASK') in ['populate_sysroot', 'populate_sysroot_setscene']:
+        bb.build.exec_func("buildhistory_emit_sysroot", d)
+
     if not d.getVar('BB_CURRENTTASK') in ['packagedata', 'packagedata_setscene']:
         return 0
 
@@ -529,6 +548,20 @@ buildhistory_list_files() {
 	fi | sort -k5 | sed 's/ * -> $//' > $2 )
 }
 
+buildhistory_list_files_no_owners() {
+	# List the files in the specified directory, but exclude date/time etc.
+	# Also don't output the ownership data, but instead output just - - so
+	# that the same parsing code as for _list_files works.
+	# This is somewhat messy, but handles where the size is not printed for device files under pseudo
+	( cd $1
+	find_cmd='find . ! -path . -printf "%M -          -          %10s %p -> %l\n"'
+	if [ "$3" = "fakeroot" ] ; then
+		eval ${FAKEROOTENV} ${FAKEROOTCMD} "$find_cmd"
+	else
+		eval "$find_cmd"
+	fi | sort -k5 | sed 's/ * -> $//' > $2 )
+}
+
 buildhistory_list_pkg_files() {
 	# Create individual files-in-package for each recipe's package
 	for pkgdir in $(find ${PKGDEST}/* -maxdepth 0 -type d); do

meta/classes/cve-check.bbclass

@@ -26,7 +26,7 @@ CVE_PRODUCT ??= "${BPN}"
 CVE_VERSION ??= "${PV}"
 
 CVE_CHECK_DB_DIR ?= "${DL_DIR}/CVE_CHECK"
-CVE_CHECK_DB_FILE ?= "${CVE_CHECK_DB_DIR}/nvd-json.db"
+CVE_CHECK_DB_FILE ?= "${CVE_CHECK_DB_DIR}/nvdcve_1.0.db"
 
 CVE_CHECK_LOG ?= "${T}/cve.log"
 CVE_CHECK_TMP_FILE ?= "${TMPDIR}/cve_check"
@@ -37,32 +37,33 @@ CVE_CHECK_COPY_FILES ??= "1"
 CVE_CHECK_CREATE_MANIFEST ??= "1"
 
 # Whitelist for packages (PN)
-CVE_CHECK_PN_WHITELIST = "\
-    glibc-locale \
-"
+CVE_CHECK_PN_WHITELIST ?= ""
 
-# Whitelist for CVE and version of package
-CVE_CHECK_CVE_WHITELIST = "{\
-    'CVE-2014-2524': ('6.3','5.2',), \
-}"
+# Whitelist for CVE. If a CVE is found, then it is considered patched.
+# The value is a string containing space separated CVE values:
+# 
+# CVE_CHECK_WHITELIST = 'CVE-2014-2524 CVE-2018-1234'
+# 
+CVE_CHECK_WHITELIST ?= ""
 
 python do_cve_check () {
     """
     Check recipe for patched and unpatched CVEs
     """
 
-    if os.path.exists(d.getVar("CVE_CHECK_TMP_FILE")):
+    if os.path.exists(d.getVar("CVE_CHECK_DB_FILE")):
         patched_cves = get_patches_cves(d)
         patched, unpatched = check_cves(d, patched_cves)
         if patched or unpatched:
             cve_data = get_cve_info(d, patched + unpatched)
             cve_write_data(d, patched, unpatched, cve_data)
     else:
-        bb.note("Failed to update CVE database, skipping CVE check")
+        bb.note("No CVE database found, skipping CVE check")
+
 }
 
 addtask cve_check after do_unpack before do_build
-do_cve_check[depends] = "cve-update-db:do_populate_cve_db"
+do_cve_check[depends] = "cve-update-db-native:do_populate_cve_db"
 do_cve_check[nostamp] = "1"
 
 python cve_check_cleanup () {
@@ -170,52 +171,87 @@ def check_cves(d, patched_cves):
 
     cves_unpatched = []
     # CVE_PRODUCT can contain more than one product (eg. curl/libcurl)
-    bpn = d.getVar("CVE_PRODUCT").split()
+    products = d.getVar("CVE_PRODUCT").split()
     # If this has been unset then we're not scanning for CVEs here (for example, image recipes)
-    if len(bpn) == 0:
+    if not products:
         return ([], [])
     pv = d.getVar("CVE_VERSION").split("+git")[0]
-    cve_whitelist = ast.literal_eval(d.getVar("CVE_CHECK_CVE_WHITELIST"))
 
     # If the recipe has been whitlisted we return empty lists
     if d.getVar("PN") in d.getVar("CVE_CHECK_PN_WHITELIST").split():
         bb.note("Recipe has been whitelisted, skipping check")
         return ([], [])
 
+    old_cve_whitelist =  d.getVar("CVE_CHECK_CVE_WHITELIST")
+    if old_cve_whitelist:
+        bb.warn("CVE_CHECK_CVE_WHITELIST is deprecated, please use CVE_CHECK_WHITELIST.")
+    cve_whitelist = d.getVar("CVE_CHECK_WHITELIST").split()
+
     import sqlite3
     db_file = d.getVar("CVE_CHECK_DB_FILE")
     conn = sqlite3.connect(db_file)
-    c = conn.cursor()
 
-    query = """SELECT * FROM PRODUCTS WHERE
-               (PRODUCT IS '{0}' AND VERSION = '{1}' AND OPERATOR IS '=') OR
-               (PRODUCT IS '{0}' AND OPERATOR IS '<=');"""
-    for idx in range(len(bpn)):
-        for row in c.execute(query.format(bpn[idx],pv)):
-            cve = row[1]
-            version = row[4]
+    for product in products:
+        c = conn.cursor()
+        if ":" in product:
+            vendor, product = product.split(":", 1)
+            c.execute("SELECT * FROM PRODUCTS WHERE PRODUCT IS ? AND VENDOR IS ?", (product, vendor))
+        else:
+            c.execute("SELECT * FROM PRODUCTS WHERE PRODUCT IS ?", (product,))
 
-            try:
-                discardVersion = LooseVersion(version) < LooseVersion(pv)
-            except:
-                discardVersion = True
+        for row in c:
+            cve = row[0]
+            version_start = row[3]
+            operator_start = row[4]
+            version_end = row[5]
+            operator_end = row[6]
 
-            if pv in cve_whitelist.get(cve,[]):
-                bb.note("%s-%s has been whitelisted for %s" % (bpn[idx], pv, cve))
+            if cve in cve_whitelist:
+                bb.note("%s-%s has been whitelisted for %s" % (product, pv, cve))
             elif cve in patched_cves:
                 bb.note("%s has been patched" % (cve))
-            elif discardVersion:
-                bb.debug(2, "Do not consider version %s " % (version))
             else:
-                cves_unpatched.append(cve)
-                bb.debug(2, "%s-%s is not patched for %s" % (bpn[idx], pv, cve))
+                to_append = False
+                if (operator_start == '=' and pv == version_start):
+                    cves_unpatched.append(cve)
+                else:
+                    if operator_start:
+                        try:
+                            to_append_start =  (operator_start == '>=' and LooseVersion(pv) >= LooseVersion(version_start))
+                            to_append_start |= (operator_start == '>' and LooseVersion(pv) > LooseVersion(version_start))
+                        except:
+                            bb.note("%s: Failed to compare %s %s %s for %s" %
+                                    (product, pv, operator_start, version_start, cve))
+                            to_append_start = False
+                    else:
+                        to_append_start = False
+
+                    if operator_end:
+                        try:
+                            to_append_end  = (operator_end == '<=' and LooseVersion(pv) <= LooseVersion(version_end))
+                            to_append_end |= (operator_end == '<' and LooseVersion(pv) < LooseVersion(version_end))
+                        except:
+                            bb.note("%s: Failed to compare %s %s %s for %s" %
+                                    (product, pv, operator_end, version_end, cve))
+                            to_append_end = False
+                    else:
+                        to_append_end = False
+
+                    if operator_start and operator_end:
+                        to_append = to_append_start and to_append_end
+                    else:
+                        to_append = to_append_start or to_append_end
+
+                if to_append:
+                    cves_unpatched.append(cve)
+                bb.debug(2, "%s-%s is not patched for %s" % (product, pv, cve))
     conn.close()
 
     return (list(patched_cves), cves_unpatched)
 
 def get_cve_info(d, cves):
     """
-    Get CVE information from the database used by cve-check-tool.
+    Get CVE information from the database.
 
     Unfortunately the only way to get CVE info is set the output to
     html (hard to parse) or query directly the database.

meta/classes/go-ptest.bbclass

@@ -0,0 +1,54 @@
+inherit go ptest
+
+do_compile_ptest_base() {
+	export TMPDIR="${GOTMPDIR}"
+	rm -f ${B}/.go_compiled_tests.list
+	go_list_package_tests | while read pkg; do
+		cd ${B}/src/$pkg
+		${GO} test ${GOPTESTBUILDFLAGS} $pkg
+		find . -mindepth 1 -maxdepth 1 -type f -name '*.test' -exec echo $pkg/{} \; | \
+			sed -e's,/\./,/,'>> ${B}/.go_compiled_tests.list
+	done
+	do_compile_ptest
+}
+
+do_compile_ptest_base[dirs] =+ "${GOTMPDIR}"
+
+go_make_ptest_wrapper() {
+	cat >${D}${PTEST_PATH}/run-ptest <<EOF
+#!/bin/sh
+RC=0
+run_test() (
+    cd "\$1"
+    ((((./\$2 ${GOPTESTFLAGS}; echo \$? >&3) | sed -r -e"s,^(PASS|SKIP|FAIL)\$,\\1: \$1/\$2," >&4) 3>&1) | (read rc; exit \$rc)) 4>&1
+    exit \$?)
+EOF
+
+}
+
+do_install_ptest_base() {
+	test -f "${B}/.go_compiled_tests.list" || exit 0
+	install -d ${D}${PTEST_PATH}
+	go_stage_testdata
+	go_make_ptest_wrapper
+	havetests=""
+	while read test; do
+		testdir=`dirname $test`
+		testprog=`basename $test`
+		install -d ${D}${PTEST_PATH}/$testdir
+		install -m 0755 ${B}/src/$test ${D}${PTEST_PATH}/$test
+	echo "run_test $testdir $testprog || RC=1" >> ${D}${PTEST_PATH}/run-ptest
+		havetests="yes"
+	done < ${B}/.go_compiled_tests.list
+	if [ -n "$havetests" ]; then
+		echo "exit \$RC" >> ${D}${PTEST_PATH}/run-ptest
+		chmod +x ${D}${PTEST_PATH}/run-ptest
+	else
+		rm -rf ${D}${PTEST_PATH}
+	fi
+	do_install_ptest
+	chown -R root:root ${D}${PTEST_PATH}
+}
+
+INSANE_SKIP_${PN}-ptest += "ldflags"
+

meta/classes/go.bbclass

@@ -1,4 +1,4 @@
-inherit goarch ptest
+inherit goarch
 
 GO_PARALLEL_BUILD ?= "${@oe.utils.parallel_make_argument(d, '-p %d')}"
 
@@ -7,6 +7,7 @@ GOROOT_class-nativesdk = "${STAGING_DIR_TARGET}${libdir}/go"
 GOROOT = "${STAGING_LIBDIR}/go"
 export GOROOT
 export GOROOT_FINAL = "${libdir}/go"
+export GOCACHE = "${B}/.cache"
 
 export GOARCH = "${TARGET_GOARCH}"
 export GOOS = "${TARGET_GOOS}"
@@ -113,19 +114,6 @@ go_do_compile() {
 do_compile[dirs] =+ "${GOTMPDIR}"
 do_compile[cleandirs] = "${B}/bin ${B}/pkg"
 
-do_compile_ptest_base() {
-	export TMPDIR="${GOTMPDIR}"
-	rm -f ${B}/.go_compiled_tests.list
-	go_list_package_tests | while read pkg; do
-		cd ${B}/src/$pkg
-		${GO} test ${GOPTESTBUILDFLAGS} $pkg
-		find . -mindepth 1 -maxdepth 1 -type f -name '*.test' -exec echo $pkg/{} \; | \
-			sed -e's,/\./,/,'>> ${B}/.go_compiled_tests.list
-	done
-	do_compile_ptest
-}
-do_compile_ptest_base[dirs] =+ "${GOTMPDIR}"
-
 go_do_install() {
 	install -d ${D}${libdir}/go/src/${GO_IMPORT}
 	tar -C ${S}/src/${GO_IMPORT} -cf - --exclude-vcs --exclude '*.test' --exclude 'testdata' . | \
@@ -138,18 +126,6 @@ go_do_install() {
 	fi
 }
 
-go_make_ptest_wrapper() {
-	cat >${D}${PTEST_PATH}/run-ptest <<EOF
-#!/bin/sh
-RC=0
-run_test() (
-    cd "\$1"
-    ((((./\$2 ${GOPTESTFLAGS}; echo \$? >&3) | sed -r -e"s,^(PASS|SKIP|FAIL)\$,\\1: \$1/\$2," >&4) 3>&1) | (read rc; exit \$rc)) 4>&1
-    exit \$?)
-EOF
-
-}
-
 go_stage_testdata() {
 	oldwd="$PWD"
 	cd ${S}/src
@@ -164,37 +140,12 @@ go_stage_testdata() {
 	cd "$oldwd"
 }
 
-do_install_ptest_base() {
-	test -f "${B}/.go_compiled_tests.list" || exit 0
-	install -d ${D}${PTEST_PATH}
-	go_stage_testdata
-	go_make_ptest_wrapper
-	havetests=""
-	while read test; do
-		testdir=`dirname $test`
-		testprog=`basename $test`
-		install -d ${D}${PTEST_PATH}/$testdir
-		install -m 0755 ${B}/src/$test ${D}${PTEST_PATH}/$test
-	echo "run_test $testdir $testprog || RC=1" >> ${D}${PTEST_PATH}/run-ptest
-		havetests="yes"
-	done < ${B}/.go_compiled_tests.list
-	if [ -n "$havetests" ]; then
-		echo "exit \$RC" >> ${D}${PTEST_PATH}/run-ptest
-		chmod +x ${D}${PTEST_PATH}/run-ptest
-	else
-		rm -rf ${D}${PTEST_PATH}
-	fi
-	do_install_ptest
-	chown -R root:root ${D}${PTEST_PATH}
-}
-
 EXPORT_FUNCTIONS do_unpack do_configure do_compile do_install
 
 FILES_${PN}-dev = "${libdir}/go/src"
 FILES_${PN}-staticdev = "${libdir}/go/pkg"
 
 INSANE_SKIP_${PN} += "ldflags"
-INSANE_SKIP_${PN}-ptest += "ldflags"
 
 # Add -buildmode=pie to GOBUILDFLAGS to satisfy "textrel" QA checking, but mips
 # doesn't support -buildmode=pie, so skip the QA checking for mips and its

meta/classes/grub-efi.bbclass

@@ -9,9 +9,16 @@ efi_populate() {
 
 	GRUB_IMAGE="grub-efi-bootia32.efi"
 	DEST_IMAGE="bootia32.efi"
-	if [ "${TARGET_ARCH}" = "x86_64" ]; then
-		GRUB_IMAGE="grub-efi-bootx64.efi"
-		DEST_IMAGE="bootx64.efi"
+	if [ -n "${MLPREFIX}" ]; then
+		if [ "${TARGET_ARCH_MULTILIB_ORIGINAL}" = "x86_64" ]; then
+			GRUB_IMAGE="grub-efi-bootx64.efi"
+			DEST_IMAGE="bootx64.efi"
+		fi
+	else
+		if [ "${TARGET_ARCH}" = "x86_64" ]; then
+			GRUB_IMAGE="grub-efi-bootx64.efi"
+			DEST_IMAGE="bootx64.efi"
+		fi
 	fi
 	install -m 0644 ${DEPLOY_DIR_IMAGE}/${GRUB_IMAGE} ${DEST}${EFIDIR}/${DEST_IMAGE}
 	EFIPATH=$(echo "${EFIDIR}" | sed 's/\//\\/g')

meta/classes/gtk-icon-cache.bbclass

@@ -6,7 +6,7 @@ PACKAGE_WRITE_DEPS += "gtk+3-native gdk-pixbuf-native"
 
 gtk_icon_cache_postinst() {
 if [ "x$D" != "x" ]; then
-	$INTERCEPT_DIR/postinst_intercept update_icon_cache ${PKG} \
+	$INTERCEPT_DIR/postinst_intercept update_gtk_icon_cache ${PKG} \
 		mlprefix=${MLPREFIX} \
 		libdir_native=${libdir_native}
 else
@@ -24,7 +24,7 @@ fi
 
 gtk_icon_cache_postrm() {
 if [ "x$D" != "x" ]; then
-	$INTERCEPT_DIR/postinst_intercept update_icon_cache ${PKG} \
+	$INTERCEPT_DIR/postinst_intercept update_gtk_icon_cache ${PKG} \
 		mlprefix=${MLPREFIX} \
 		libdir=${libdir}
 else

meta/classes/icecc.bbclass

@@ -57,7 +57,7 @@ ICECC_ENV_VERSION = "2"
 # See: https://github.com/icecc/icecream/issues/190
 export ICECC_CARET_WORKAROUND ??= "0"
 
-export ICECC_REMOTE_CPP ??= "1"
+export ICECC_REMOTE_CPP ??= "0"
 
 ICECC_CFLAGS = ""
 CFLAGS += "${ICECC_CFLAGS}"

meta/classes/image.bbclass

@@ -328,7 +328,8 @@ addtask do_image_qa_setscene
 
 def setup_debugfs_variables(d):
     d.appendVar('IMAGE_ROOTFS', '-dbg')
-    d.appendVar('IMAGE_LINK_NAME', '-dbg')
+    if d.getVar('IMAGE_LINK_NAME'):
+        d.appendVar('IMAGE_LINK_NAME', '-dbg')
     d.appendVar('IMAGE_NAME','-dbg')
     d.setVar('IMAGE_BUILDING_DEBUGFS', 'true')
     debugfs_image_fstypes = d.getVar('IMAGE_FSTYPES_DEBUGFS')
@@ -666,10 +667,11 @@ reproducible_final_image_task () {
 }
 
 systemd_preset_all () {
-    systemctl --root="${IMAGE_ROOTFS}" --preset-mode=enable-only preset-all
+    if [ -e ${IMAGE_ROOTFS}${root_prefix}/lib/systemd/systemd ]; then
+	systemctl --root="${IMAGE_ROOTFS}" --preset-mode=enable-only preset-all
+    fi
 }
 
-IMAGE_EXTRADEPENDS += "${@ 'systemd-systemctl-native' if bb.utils.contains('DISTRO_FEATURES', 'systemd', True, False, d) and not bb.utils.contains('IMAGE_FEATURES', 'stateless-rootfs', True, False, d) else ''}"
 IMAGE_PREPROCESS_COMMAND_append = " ${@ 'systemd_preset_all;' if bb.utils.contains('DISTRO_FEATURES', 'systemd', True, False, d) and not bb.utils.contains('IMAGE_FEATURES', 'stateless-rootfs', True, False, d) else ''} reproducible_final_image_task; "
 
 CVE_PRODUCT = ""

meta/classes/image_types_wic.bbclass

@@ -3,7 +3,10 @@
 WICVARS ?= "\
            BBLAYERS IMGDEPLOYDIR DEPLOY_DIR_IMAGE FAKEROOTCMD IMAGE_BASENAME IMAGE_BOOT_FILES \
            IMAGE_LINK_NAME IMAGE_ROOTFS INITRAMFS_FSTYPES INITRD INITRD_LIVE ISODIR RECIPE_SYSROOT_NATIVE \
-           ROOTFS_SIZE STAGING_DATADIR STAGING_DIR STAGING_LIBDIR TARGET_SYS"
+           ROOTFS_SIZE STAGING_DATADIR STAGING_DIR STAGING_LIBDIR TARGET_SYS \
+           KERNEL_IMAGETYPE MACHINE INITRAMFS_IMAGE INITRAMFS_IMAGE_BUNDLE INITRAMFS_LINK_NAME"
+
+inherit ${@bb.utils.contains('INITRAMFS_IMAGE_BUNDLE', '1', 'kernel-artifact-names', '', d)}
 
 WKS_FILE ??= "${IMAGE_BASENAME}.${MACHINE}.wks"
 WKS_FILES ?= "${WKS_FILE} ${IMAGE_BASENAME}.wks"

meta/classes/insane.bbclass

@@ -874,15 +874,14 @@ def package_qa_check_host_user(path, name, d, elf, messages):
         if exc.errno != errno.ENOENT:
             raise
     else:
-        rootfs_path = path[len(dest):]
         check_uid = int(d.getVar('HOST_USER_UID'))
         if stat.st_uid == check_uid:
-            package_qa_add_message(messages, "host-user-contaminated", "%s: %s is owned by uid %d, which is the same as the user running bitbake. This may be due to host contamination" % (pn, rootfs_path, check_uid))
+            package_qa_add_message(messages, "host-user-contaminated", "%s: %s is owned by uid %d, which is the same as the user running bitbake. This may be due to host contamination" % (pn, package_qa_clean_path(path, d, name), check_uid))
             return False
 
         check_gid = int(d.getVar('HOST_USER_GID'))
         if stat.st_gid == check_gid:
-            package_qa_add_message(messages, "host-user-contaminated", "%s: %s is owned by gid %d, which is the same as the user running bitbake. This may be due to host contamination" % (pn, rootfs_path, check_gid))
+            package_qa_add_message(messages, "host-user-contaminated", "%s: %s is owned by gid %d, which is the same as the user running bitbake. This may be due to host contamination" % (pn, package_qa_clean_path(path, d, name), check_gid))
             return False
     return True
 

meta/classes/meson.bbclass

@@ -30,11 +30,6 @@ MESONOPTS = " --prefix ${prefix} \
               -Dcpp_args='${BUILD_CPPFLAGS} ${BUILD_CXXFLAGS}' \
               -Dcpp_link_args='${BUILD_LDFLAGS}'"
 
-MESON_TOOLCHAIN_ARGS = "${HOST_CC_ARCH}${TOOLCHAIN_OPTIONS}"
-MESON_C_ARGS = "${MESON_TOOLCHAIN_ARGS} ${CFLAGS}"
-MESON_CPP_ARGS = "${MESON_TOOLCHAIN_ARGS} ${CXXFLAGS}"
-MESON_LINK_ARGS = "${MESON_TOOLCHAIN_ARGS} ${LDFLAGS}"
-
 EXTRA_OEMESON_append = " ${PACKAGECONFIG_CONFARGS}"
 
 MESON_CROSS_FILE = ""
@@ -78,7 +73,7 @@ def meson_endian(prefix, d):
         bb.fatal("Cannot determine endianism for %s-%s" % (arch, os))
 
 addtask write_config before do_configure
-do_write_config[vardeps] += "MESON_C_ARGS MESON_CPP_ARGS MESON_LINK_ARGS CC CXX LD AR NM STRIP READELF"
+do_write_config[vardeps] += "CC CXX LD AR NM STRIP READELF CFLAGS CXXFLAGS LDFLAGS"
 do_write_config() {
     # This needs to be Py to split the args into single-element lists
     cat >${WORKDIR}/meson.cross <<EOF
@@ -95,10 +90,10 @@ llvm-config = 'llvm-config8.0.0'
 
 [properties]
 needs_exe_wrapper = true
-c_args = ${@meson_array('MESON_C_ARGS', d)}
-c_link_args = ${@meson_array('MESON_LINK_ARGS', d)}
-cpp_args = ${@meson_array('MESON_CPP_ARGS', d)}
-cpp_link_args = ${@meson_array('MESON_LINK_ARGS', d)}
+c_args = ${@meson_array('CFLAGS', d)}
+c_link_args = ${@meson_array('LDFLAGS', d)}
+cpp_args = ${@meson_array('CXXFLAGS', d)}
+cpp_link_args = ${@meson_array('LDFLAGS', d)}
 gtkdoc_exe_wrapper = '${B}/gtkdoc-qemuwrapper'
 
 [host_machine]
@@ -135,6 +130,7 @@ override_native_tools() {
     export CXX="${BUILD_CXX}"
     export LD="${BUILD_LD}"
     export AR="${BUILD_AR}"
+    export STRIP="${BUILD_STRIP}"
     # These contain *target* flags but will be used as *native* flags.  The
     # correct native flags will be passed via -Dc_args and so on, unset them so
     # they don't interfere with tools invoked by Meson (such as g-ir-scanner)

meta/classes/multilib.bbclass

@@ -125,8 +125,55 @@ python __anonymous () {
     clsextend.map_variable("USERADD_PACKAGES")
     clsextend.map_variable("SYSTEMD_PACKAGES")
     clsextend.map_variable("UPDATERCPN")
+
+    reset_alternative_priority(d)
 }
 
+def reset_alternative_priority(d):
+    if not bb.data.inherits_class('update-alternatives', d):
+        return
+
+    # There might be multiple multilibs at the same time, e.g., lib32 and
+    # lib64, each of them should have a different priority.
+    multilib_variants = d.getVar('MULTILIB_VARIANTS')
+    bbextendvariant = d.getVar('BBEXTENDVARIANT')
+    reset_gap = multilib_variants.split().index(bbextendvariant) + 1
+
+    # ALTERNATIVE_PRIORITY = priority
+    alt_priority_recipe = d.getVar('ALTERNATIVE_PRIORITY')
+    # Reset ALTERNATIVE_PRIORITY when found
+    if alt_priority_recipe:
+        reset_priority = int(alt_priority_recipe) - reset_gap
+        bb.debug(1, '%s: Setting ALTERNATIVE_PRIORITY to %s' % (d.getVar('PN'), reset_priority))
+        d.setVar('ALTERNATIVE_PRIORITY', reset_priority)
+
+    handled_pkgs = []
+    for pkg in (d.getVar('PACKAGES') or "").split():
+        # ALTERNATIVE_PRIORITY_pkg = priority
+        alt_priority_pkg = d.getVar('ALTERNATIVE_PRIORITY_%s' % pkg)
+        # Reset ALTERNATIVE_PRIORITY_pkg when found
+        if alt_priority_pkg:
+            reset_priority = int(alt_priority_pkg) - reset_gap
+            if not pkg in handled_pkgs:
+                handled_pkgs.append(pkg)
+                bb.debug(1, '%s: Setting ALTERNATIVE_PRIORITY_%s to %s' % (pkg, pkg, reset_priority))
+                d.setVar('ALTERNATIVE_PRIORITY_%s' % pkg, reset_priority)
+
+        for alt_name in (d.getVar('ALTERNATIVE_%s' % pkg) or "").split():
+            # ALTERNATIVE_PRIORITY_pkg[tool]  = priority
+            alt_priority_pkg_name = d.getVarFlag('ALTERNATIVE_PRIORITY_%s' % pkg, alt_name)
+            # ALTERNATIVE_PRIORITY[tool] = priority
+            alt_priority_name = d.getVarFlag('ALTERNATIVE_PRIORITY', alt_name)
+
+            if alt_priority_pkg_name:
+                reset_priority = int(alt_priority_pkg_name) - reset_gap
+                bb.debug(1, '%s: Setting ALTERNATIVE_PRIORITY_%s[%s] to %s' % (pkg, pkg, alt_name, reset_priority))
+                d.setVarFlag('ALTERNATIVE_PRIORITY_%s' % pkg, alt_name, reset_priority)
+            elif alt_priority_name:
+                reset_priority = int(alt_priority_name) - reset_gap
+                bb.debug(1, '%s: Setting ALTERNATIVE_PRIORITY[%s] to %s' % (pkg, alt_name, reset_priority))
+                d.setVarFlag('ALTERNATIVE_PRIORITY', alt_name, reset_priority)
+
 PACKAGEFUNCS_append = " do_package_qa_multilib"
 
 python do_package_qa_multilib() {

meta/classes/multilib_global.bbclass

@@ -172,21 +172,27 @@ python multilib_virtclass_handler_global () {
     if bb.data.inherits_class('kernel', e.data) or \
             bb.data.inherits_class('module-base', e.data) or \
             d.getVar('BPN') in non_ml_recipes:
+
+            # We need to avoid expanding KERNEL_VERSION which we can do by deleting it
+            # from a copy of the datastore
+            localdata = bb.data.createCopy(d)
+            localdata.delVar("KERNEL_VERSION")
+
             variants = (e.data.getVar("MULTILIB_VARIANTS") or "").split()
 
             import oe.classextend
             clsextends = []
             for variant in variants:
-                clsextends.append(oe.classextend.ClassExtender(variant, e.data))
+                clsextends.append(oe.classextend.ClassExtender(variant, localdata))
 
             # Process PROVIDES
-            origprovs = provs = e.data.getVar("PROVIDES") or ""
+            origprovs = provs = localdata.getVar("PROVIDES") or ""
             for clsextend in clsextends:
                 provs = provs + " " + clsextend.map_variable("PROVIDES", setvar=False)
             e.data.setVar("PROVIDES", provs)
 
             # Process RPROVIDES
-            origrprovs = rprovs = e.data.getVar("RPROVIDES") or ""
+            origrprovs = rprovs = localdata.getVar("RPROVIDES") or ""
             for clsextend in clsextends:
                 rprovs = rprovs + " " + clsextend.map_variable("RPROVIDES", setvar=False)
             if rprovs.strip():
@@ -194,7 +200,7 @@ python multilib_virtclass_handler_global () {
 
             # Process RPROVIDES_${PN}...
             for pkg in (e.data.getVar("PACKAGES") or "").split():
-                origrprovs = rprovs = e.data.getVar("RPROVIDES_%s" % pkg) or ""
+                origrprovs = rprovs = localdata.getVar("RPROVIDES_%s" % pkg) or ""
                 for clsextend in clsextends:
                     rprovs = rprovs + " " + clsextend.map_variable("RPROVIDES_%s" % pkg, setvar=False)
                     rprovs = rprovs + " " + clsextend.extname + "-" + pkg
@@ -202,5 +208,4 @@ python multilib_virtclass_handler_global () {
 }
 
 addhandler multilib_virtclass_handler_global
-multilib_virtclass_handler_global[eventmask] = "bb.event.RecipeParsed"
-
+multilib_virtclass_handler_global[eventmask] = "bb.event.RecipeTaskPreProcess"

meta/classes/multilib_script.bbclass

@@ -17,18 +17,18 @@ multilibscript_rename() {
 python () {
     # Do nothing if multilib isn't being used
     if not d.getVar("MULTILIB_VARIANTS"):
-       return
+        return
     # Do nothing for native/cross
     if bb.data.inherits_class('native', d) or bb.data.inherits_class('cross', d):
-       return
+        return
 
     for entry in (d.getVar("MULTILIB_SCRIPTS", False) or "").split():
-       pkg, script = entry.split(":")
-       epkg = d.expand(pkg)
-       scriptname = os.path.basename(script)
-       d.appendVar("ALTERNATIVE_" + epkg, " " + scriptname + " ")
-       d.setVarFlag("ALTERNATIVE_LINK_NAME", scriptname, script)
-       d.setVarFlag("ALTERNATIVE_TARGET", scriptname, script + "-${MULTILIB_SUFFIX}")
-       d.appendVar("multilibscript_rename",  "\n	mv ${PKGD}" + script + " ${PKGD}" + script + "-${MULTILIB_SUFFIX}")
-       d.appendVar("FILES_" + epkg, " " + script + "-${MULTILIB_SUFFIX}")
+        pkg, script = entry.split(":")
+        epkg = d.expand(pkg)
+        scriptname = os.path.basename(script)
+        d.appendVar("ALTERNATIVE_" + epkg, " " + scriptname + " ")
+        d.setVarFlag("ALTERNATIVE_LINK_NAME", scriptname, script)
+        d.setVarFlag("ALTERNATIVE_TARGET", scriptname, script + "-${MULTILIB_SUFFIX}")
+        d.appendVar("multilibscript_rename",  "\n	mv ${PKGD}" + script + " ${PKGD}" + script + "-${MULTILIB_SUFFIX}")
+        d.appendVar("FILES_" + epkg, " " + script + "-${MULTILIB_SUFFIX}")
 }

meta/classes/package.bbclass

@@ -40,6 +40,7 @@
 
 inherit packagedata
 inherit chrpath
+inherit package_pkgdata
 
 # Need the package_qa_handle_error() in insane.bbclass
 inherit insane
@@ -1216,7 +1217,8 @@ python populate_packages () {
                 src = os.path.join(src, p)
                 dest = os.path.join(dest, p)
                 fstat = cpath.stat(src)
-                os.mkdir(dest, fstat.st_mode)
+                os.mkdir(dest)
+                os.chmod(dest, fstat.st_mode)
                 os.chown(dest, fstat.st_uid, fstat.st_gid)
                 if p not in seen:
                     seen.append(p)
@@ -1356,12 +1358,16 @@ python emit_pkgdata() {
     import json
 
     def process_postinst_on_target(pkg, mlprefix):
+        pkgval = d.getVar('PKG_%s' % pkg)
+        if pkgval is None:
+            pkgval = pkg
+
         defer_fragment = """
 if [ -n "$D" ]; then
     $INTERCEPT_DIR/postinst_intercept delay_to_first_boot %s mlprefix=%s
     exit 0
 fi
-""" % (pkg, mlprefix)
+""" % (pkgval, mlprefix)
 
         postinst = d.getVar('pkg_postinst_%s' % pkg)
         postinst_ontarget = d.getVar('pkg_postinst_ontarget_%s' % pkg)
@@ -1570,7 +1576,7 @@ python package_do_filedeps() {
         d.setVar("FILERPROVIDESFLIST_" + pkg, " ".join(provides_files[pkg]))
 }
 
-SHLIBSDIRS = "${PKGDATA_DIR}/${MLPREFIX}shlibs2"
+SHLIBSDIRS = "${WORKDIR_PKGDATA}/${MLPREFIX}shlibs2"
 SHLIBSWORKDIR = "${PKGDESTWORK}/${MLPREFIX}shlibs2"
 
 python package_do_shlibs() {
@@ -1728,10 +1734,7 @@ python package_do_shlibs() {
 
     needed = {}
 
-    # Take shared lock since we're only reading, not writing
-    lf = bb.utils.lockfile(d.expand("${PACKAGELOCK}"), True)
     shlib_provider = oe.package.read_shlib_providers(d)
-    bb.utils.unlockfile(lf)
 
     for pkg in shlib_pkgs:
         private_libs = d.getVar('PRIVATE_LIBS_' + pkg) or d.getVar('PRIVATE_LIBS') or ""
@@ -1917,9 +1920,6 @@ python package_do_pkgconfig () {
                 f.write('%s\n' % p)
             f.close()
 
-    # Take shared lock since we're only reading, not writing
-    lf = bb.utils.lockfile(d.expand("${PACKAGELOCK}"), True)
-
     # Go from least to most specific since the last one found wins
     for dir in reversed(shlibs_dirs):
         if not os.path.exists(dir):
@@ -1935,8 +1935,6 @@ python package_do_pkgconfig () {
                 for l in lines:
                     pkgconfig_provided[pkg].append(l.rstrip())
 
-    bb.utils.unlockfile(lf)
-
     for pkg in packages.split():
         deps = []
         for n in pkgconfig_needed[pkg]:
@@ -2133,6 +2131,7 @@ def gen_packagevar(d):
 PACKAGE_PREPROCESS_FUNCS ?= ""
 # Functions for setting up PKGD
 PACKAGEBUILDPKGD ?= " \
+                package_prepare_pkgdata \
                 perform_packagecopy \
                 ${PACKAGE_PREPROCESS_FUNCS} \
                 split_and_strip_files \
@@ -2260,12 +2259,8 @@ do_packagedata () {
 addtask packagedata before do_build after do_package
 
 SSTATETASKS += "do_packagedata"
-# PACKAGELOCK protects readers of PKGDATA_DIR against writes
-# whilst code is reading in do_package
-PACKAGELOCK = "${STAGING_DIR}/package-output.lock"
 do_packagedata[sstate-inputdirs] = "${PKGDESTWORK}"
 do_packagedata[sstate-outputdirs] = "${PKGDATA_DIR}"
-do_packagedata[sstate-lockfile] = "${PACKAGELOCK}"
 do_packagedata[stamp-extra-info] = "${MACHINE_ARCH}"
 
 python do_packagedata_setscene () {

meta/classes/package_ipk.bbclass

@@ -14,7 +14,7 @@ OPKG_ARGS += "--force_postinstall --prefer-arch-to-version"
 OPKG_ARGS += "${@['', '--no-install-recommends'][d.getVar("NO_RECOMMENDATIONS") == "1"]}"
 OPKG_ARGS += "${@['', '--add-exclude ' + ' --add-exclude '.join((d.getVar('PACKAGE_EXCLUDE') or "").split())][(d.getVar("PACKAGE_EXCLUDE") or "").strip() != ""]}"
 
-OPKGLIBDIR = "${localstatedir}/lib"
+OPKGLIBDIR ??= "${localstatedir}/lib"
 
 python do_package_ipk () {
     workdir = d.getVar('WORKDIR')

meta/classes/package_pkgdata.bbclass

@@ -0,0 +1,167 @@
+WORKDIR_PKGDATA = "${WORKDIR}/pkgdata-sysroot"
+
+def package_populate_pkgdata_dir(pkgdatadir, d):
+    import glob
+
+    postinsts = []
+    seendirs = set()
+    stagingdir = d.getVar("PKGDATA_DIR")
+    pkgarchs = ['${MACHINE_ARCH}']
+    pkgarchs = pkgarchs + list(reversed(d.getVar("PACKAGE_EXTRA_ARCHS").split()))
+    pkgarchs.append('allarch')
+
+    bb.utils.mkdirhier(pkgdatadir)
+    for pkgarch in pkgarchs:
+        for manifest in glob.glob(d.expand("${SSTATE_MANIFESTS}/manifest-%s-*.packagedata" % pkgarch)):
+            with open(manifest, "r") as f:
+                for l in f:
+                    l = l.strip()
+                    dest = l.replace(stagingdir, "")
+                    if l.endswith("/"):
+                        staging_copydir(l, pkgdatadir, dest, seendirs)
+                        continue
+                    try:
+                        staging_copyfile(l, pkgdatadir, dest, postinsts, seendirs)
+                    except FileExistsError:
+                        continue
+
+python package_prepare_pkgdata() {
+    import copy
+    import glob
+
+    taskdepdata = d.getVar("BB_TASKDEPDATA", False)
+    mytaskname = d.getVar("BB_RUNTASK")
+    if mytaskname.endswith("_setscene"):
+        mytaskname = mytaskname.replace("_setscene", "")
+    workdir = d.getVar("WORKDIR")
+    pn = d.getVar("PN")
+    stagingdir = d.getVar("PKGDATA_DIR")
+    pkgdatadir = d.getVar("WORKDIR_PKGDATA")
+
+    # Detect bitbake -b usage
+    nodeps = d.getVar("BB_LIMITEDDEPS") or False
+    if nodeps:
+        staging_package_populate_pkgdata_dir(pkgdatadir, d)
+        return
+
+    start = None
+    configuredeps = []
+    for dep in taskdepdata:
+        data = taskdepdata[dep]
+        if data[1] == mytaskname and data[0] == pn:
+            start = dep
+            break
+    if start is None:
+        bb.fatal("Couldn't find ourself in BB_TASKDEPDATA?")
+
+    # We need to figure out which sysroot files we need to expose to this task.
+    # This needs to match what would get restored from sstate, which is controlled
+    # ultimately by calls from bitbake to setscene_depvalid().
+    # That function expects a setscene dependency tree. We build a dependency tree
+    # condensed to inter-sstate task dependencies, similar to that used by setscene
+    # tasks. We can then call into setscene_depvalid() and decide
+    # which dependencies we can "see" and should expose in the recipe specific sysroot.
+    setscenedeps = copy.deepcopy(taskdepdata)
+
+    start = set([start])
+
+    sstatetasks = d.getVar("SSTATETASKS").split()
+    # Add recipe specific tasks referenced by setscene_depvalid()
+    sstatetasks.append("do_stash_locale")
+
+    # If start is an sstate task (like do_package) we need to add in its direct dependencies
+    # else the code below won't recurse into them.
+    for dep in set(start):
+        for dep2 in setscenedeps[dep][3]:
+            start.add(dep2)
+        start.remove(dep)
+
+    # Create collapsed do_populate_sysroot -> do_populate_sysroot tree
+    for dep in taskdepdata:
+        data = setscenedeps[dep]
+        if data[1] not in sstatetasks:
+            for dep2 in setscenedeps:
+                data2 = setscenedeps[dep2]
+                if dep in data2[3]:
+                    data2[3].update(setscenedeps[dep][3])
+                    data2[3].remove(dep)
+            if dep in start:
+                start.update(setscenedeps[dep][3])
+                start.remove(dep)
+            del setscenedeps[dep]
+
+    # Remove circular references
+    for dep in setscenedeps:
+        if dep in setscenedeps[dep][3]:
+            setscenedeps[dep][3].remove(dep)
+
+    # Direct dependencies should be present and can be depended upon
+    for dep in set(start):
+        if setscenedeps[dep][1] == "do_packagedata":
+            if dep not in configuredeps:
+                configuredeps.append(dep)
+
+    msgbuf = []
+    # Call into setscene_depvalid for each sub-dependency and only copy sysroot files
+    # for ones that would be restored from sstate.
+    done = list(start)
+    next = list(start)
+    while next:
+        new = []
+        for dep in next:
+            data = setscenedeps[dep]
+            for datadep in data[3]:
+                if datadep in done:
+                    continue
+                taskdeps = {}
+                taskdeps[dep] = setscenedeps[dep][:2]
+                taskdeps[datadep] = setscenedeps[datadep][:2]
+                retval = setscene_depvalid(datadep, taskdeps, [], d, msgbuf)
+                done.append(datadep)
+                new.append(datadep)
+                if retval:
+                    msgbuf.append("Skipping setscene dependency %s" % datadep)
+                    continue
+                if datadep not in configuredeps and setscenedeps[datadep][1] == "do_packagedata":
+                    configuredeps.append(datadep)
+                    msgbuf.append("Adding dependency on %s" % setscenedeps[datadep][0])
+                else:
+                    msgbuf.append("Following dependency on %s" % setscenedeps[datadep][0])
+        next = new
+
+    # This logging is too verbose for day to day use sadly
+    #bb.debug(2, "\n".join(msgbuf))
+
+    seendirs = set()
+    postinsts = []
+    multilibs = {}
+    manifests = {}
+
+    msg_adding = []
+
+    for dep in configuredeps:
+        c = setscenedeps[dep][0]
+        msg_adding.append(c)
+
+        manifest, d2 = oe.sstatesig.find_sstate_manifest(c, setscenedeps[dep][2], "packagedata", d, multilibs)
+        destsysroot = pkgdatadir
+
+        if manifest:
+            targetdir = destsysroot
+            with open(manifest, "r") as f:
+                manifests[dep] = manifest
+                for l in f:
+                    l = l.strip()
+                    dest = targetdir + l.replace(stagingdir, "")
+                    if l.endswith("/"):
+                        staging_copydir(l, targetdir, dest, seendirs)
+                        continue
+                    staging_copyfile(l, targetdir, dest, postinsts, seendirs)
+
+    bb.note("Installed into pkgdata-sysroot: %s" % str(msg_adding))
+
+}
+package_prepare_pkgdata[cleandirs] = "${WORKDIR_PKGDATA}"
+package_prepare_pkgdata[vardepsexclude] += "MACHINE_ARCH PACKAGE_EXTRA_ARCHS SDK_ARCH BUILD_ARCH SDK_OS BB_TASKDEPDATA"
+
+

meta/classes/package_rpm.bbclass

@@ -36,7 +36,7 @@ def write_rpm_perfiledata(srcname, d):
     pkgd = d.getVar('PKGD')
 
     def dump_filerdeps(varname, outfile, d):
-        outfile.write("#!/usr/bin/env python\n\n")
+        outfile.write("#!/usr/bin/env python3\n\n")
         outfile.write("# Dependency table\n")
         outfile.write('deps = {\n')
         for pkg in packages.split():

meta/classes/rootfs-postcommands.bbclass

@@ -260,7 +260,7 @@ python write_image_manifest () {
     with open(manifest_name, 'w+') as image_manifest:
         image_manifest.write(format_pkg_list(pkgs, "ver"))
 
-    if os.path.exists(manifest_name):
+    if os.path.exists(manifest_name) and link_name:
         manifest_link = deploy_dir + "/" + link_name + ".manifest"
         if os.path.lexists(manifest_link):
             os.remove(manifest_link)
@@ -328,7 +328,7 @@ python write_image_test_data() {
     searchString = "%s/"%(d.getVar("TOPDIR")).replace("//","/")
     export2json(d, testdata_name, searchString=searchString, replaceString="")
 
-    if os.path.exists(testdata_name):
+    if os.path.exists(testdata_name) and link_name:
         testdata_link = os.path.join(deploy_dir, "%s.testdata.json" % link_name)
         if os.path.lexists(testdata_link):
             os.remove(testdata_link)

meta/classes/rootfs_ipk.bbclass

@@ -21,7 +21,7 @@ OPKG_PREPROCESS_COMMANDS = ""
 
 OPKG_POSTPROCESS_COMMANDS = ""
 
-OPKGLIBDIR = "${localstatedir}/lib"
+OPKGLIBDIR ??= "${localstatedir}/lib"
 
 MULTILIBRE_ALLOW_REP = "${OPKGLIBDIR}/opkg|/usr/lib/opkg"
 

meta/classes/sstate.bbclass

@@ -927,6 +927,10 @@ def sstate_checkhashes(sq_fn, sq_task, sq_hash, sq_hashfn, d, siginfo=False, *,
 
             bb.event.fire(bb.event.ProcessFinished(msg), d)
 
+    # Likely checking an individual task hash again for multiconfig sharing of sstate tasks so skip reporting
+    if len(sq_fn) == 1:
+        return ret
+
     inheritlist = d.getVar("INHERIT")
     if "toaster" in inheritlist:
         evdata = {'missed': [], 'found': []};

meta/classes/staging.bbclass

@@ -261,12 +261,10 @@ python extend_recipe_sysroot() {
     workdir = d.getVar("WORKDIR")
     #bb.warn(str(taskdepdata))
     pn = d.getVar("PN")
-    mc = d.getVar("BB_CURRENT_MC")
     stagingdir = d.getVar("STAGING_DIR")
     sharedmanifests = d.getVar("COMPONENTS_DIR") + "/manifests"
     recipesysroot = d.getVar("RECIPE_SYSROOT")
     recipesysrootnative = d.getVar("RECIPE_SYSROOT_NATIVE")
-    current_variant = d.getVar("BBEXTENDVARIANT")
 
     # Detect bitbake -b usage
     nodeps = d.getVar("BB_LIMITEDDEPS") or False
@@ -452,11 +450,6 @@ python extend_recipe_sysroot() {
     msg_adding = []
 
     for dep in configuredeps:
-        if mc != 'default':
-            # We should not care about other multiconfigs
-            depmc = dep.split(':')[1]
-            if depmc != mc:
-                continue
         c = setscenedeps[dep][0]
         if c not in installed:
             continue
@@ -584,17 +577,6 @@ python do_prepare_recipe_sysroot () {
 }
 addtask do_prepare_recipe_sysroot before do_configure after do_fetch
 
-# Clean out the recipe specific sysroots before do_fetch
-# (use a prefunc so we can order before extend_recipe_sysroot if it gets added)
-python clean_recipe_sysroot() {
-    # We remove these stamps since we're removing any content they'd have added with
-    # cleandirs. This removes the sigdata too, likely not a big deal,
-    oe.path.remove(d.getVar("STAMP") + "*addto_recipe_sysroot*")
-    return
-}
-clean_recipe_sysroot[cleandirs] += "${RECIPE_SYSROOT} ${RECIPE_SYSROOT_NATIVE}"
-do_fetch[prefuncs] += "clean_recipe_sysroot"
-
 python staging_taskhandler() {
     bbtasks = e.tasklist
     for task in bbtasks:

meta/classes/uboot-sign.bbclass

@@ -66,7 +66,7 @@ concat_dtb_helper() {
 		install ${UBOOT_BINARY} ${DEPLOYDIR}/${UBOOT_IMAGE}
 	elif [ -e "${DEPLOYDIR}/${UBOOT_NODTB_IMAGE}" -a -e "$deployed_uboot_dtb_binary" ]; then
 		cd ${DEPLOYDIR}
-		cat ${UBOOT_NODTB_IMAGE} $deployed_uboot_dtb_binary | tee ${UBOOT_BINARY} > ${UBOOT_IMAGE}
+		cat ${UBOOT_NODTB_IMAGE} $deployed_uboot_dtb_binary | tee ${B}/${CONFIG_B_PATH}/${UBOOT_BINARY} > ${UBOOT_IMAGE}
 	else
 		bbwarn "Failure while adding public key to u-boot binary. Verified boot won't be available."
 	fi
@@ -77,10 +77,12 @@ concat_dtb() {
 		mkdir -p ${DEPLOYDIR}
 		if [ -n "${UBOOT_CONFIG}" ]; then
 			for config in ${UBOOT_MACHINE}; do
+				CONFIG_B_PATH="${config}"
 				cd ${B}/${config}
 				concat_dtb_helper
 			done
 		else
+			CONFIG_B_PATH=""
 			cd ${B}
 			concat_dtb_helper
 		fi

meta/classes/update-alternatives.bbclass

@@ -284,8 +284,11 @@ python populate_packages_updatealternatives () {
 
             bb.note('adding update-alternatives calls to postinst/prerm for %s' % pkg)
             bb.note('%s' % alt_setup_links)
-            postinst = d.getVar('pkg_postinst_%s' % pkg) or '#!/bin/sh\n'
-            postinst += alt_setup_links
+            postinst = d.getVar('pkg_postinst_%s' % pkg)
+            if postinst:
+                postinst = alt_setup_links + postinst
+            else:
+                postinst = '#!/bin/sh\n' + alt_setup_links
             d.setVar('pkg_postinst_%s' % pkg, postinst)
 
             bb.note('%s' % alt_remove_links)