conf.py/poky.yaml: Move version information to poky.yaml and read in conf.py

Merge in the changes from master allowing conf.py to use information from
poky.yaml. This allows the head version mapped to X.999 on the website to
have the version information displayed clearly and correctly.

(From yocto-docs rev: 9dbd3e8726b5eebd067d01973540724ea3e246c4)

Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>

bitbake/bin/bitbake-server

@@ -26,7 +26,7 @@ readypipeinfd = int(sys.argv[3])
 logfile = sys.argv[4]
 lockname = sys.argv[5]
 sockname = sys.argv[6]
-timeout = sys.argv[7]
+timeout = float(sys.argv[7])
 xmlrpcinterface = (sys.argv[8], int(sys.argv[9]))
 if xmlrpcinterface[0] == "None":
     xmlrpcinterface = (None, xmlrpcinterface[1])

bitbake/doc/Makefile

@@ -0,0 +1,35 @@
+# Minimal makefile for Sphinx documentation
+#
+
+# You can set these variables from the command line, and also
+# from the environment for the first two.
+SPHINXOPTS    ?=
+SPHINXBUILD   ?= sphinx-build
+SOURCEDIR     = .
+BUILDDIR      = _build
+DESTDIR       = final
+
+ifeq ($(shell if which $(SPHINXBUILD) >/dev/null 2>&1; then echo 1; else echo 0; fi),0)
+$(error "The '$(SPHINXBUILD)' command was not found. Make sure you have Sphinx installed")
+endif
+
+# Put it first so that "make" without argument is like "make help".
+help:
+	@$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)
+
+.PHONY: help Makefile clean publish
+
+publish: Makefile html singlehtml
+	rm -rf $(BUILDDIR)/$(DESTDIR)/
+	mkdir -p $(BUILDDIR)/$(DESTDIR)/
+	cp -r $(BUILDDIR)/html/* $(BUILDDIR)/$(DESTDIR)/
+	cp $(BUILDDIR)/singlehtml/index.html $(BUILDDIR)/$(DESTDIR)/singleindex.html
+	sed -i -e 's@index.html#@singleindex.html#@g' $(BUILDDIR)/$(DESTDIR)/singleindex.html
+
+clean:
+	@rm -rf $(BUILDDIR)
+
+# Catch-all target: route all unknown targets to Sphinx using the new
+# "make mode" option.  $(O) is meant as a shortcut for $(SPHINXOPTS).
+%: Makefile
+	@$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)

bitbake/lib/bb/__init__.py

@@ -49,7 +49,7 @@ class BBLoggerMixin(object):
         if not bb.event.worker_pid:
             if self.name in bb.msg.loggerDefaultDomains and loglevel > (bb.msg.loggerDefaultDomains[self.name]):
                 return
-            if loglevel > bb.msg.loggerDefaultLogLevel:
+            if loglevel < bb.msg.loggerDefaultLogLevel:
                 return
         return self.log(loglevel, msg, *args, **kwargs)
 

bitbake/lib/bb/cache.py

@@ -26,7 +26,7 @@ import re
 
 logger = logging.getLogger("BitBake.Cache")
 
-__cache_version__ = "153"
+__cache_version__ = "154"
 
 def getCacheFile(path, filename, mc, data_hash):
     mcspec = ''
@@ -94,6 +94,7 @@ class CoreRecipeInfo(RecipeInfoCommon):
         if not self.packages:
             self.packages.append(self.pn)
         self.packages_dynamic = self.listvar('PACKAGES_DYNAMIC', metadata)
+        self.rprovides_pkg = self.pkgvar('RPROVIDES', self.packages, metadata)
 
         self.skipreason = self.getvar('__SKIPPED', metadata)
         if self.skipreason:
@@ -120,7 +121,6 @@ class CoreRecipeInfo(RecipeInfoCommon):
         self.depends          = self.depvar('DEPENDS', metadata)
         self.rdepends         = self.depvar('RDEPENDS', metadata)
         self.rrecommends      = self.depvar('RRECOMMENDS', metadata)
-        self.rprovides_pkg    = self.pkgvar('RPROVIDES', self.packages, metadata)
         self.rdepends_pkg     = self.pkgvar('RDEPENDS', self.packages, metadata)
         self.rrecommends_pkg  = self.pkgvar('RRECOMMENDS', self.packages, metadata)
         self.inherits         = self.getvar('__inherit_cache', metadata, expand=False)

bitbake/lib/bb/cooker.py

@@ -73,7 +73,9 @@ class SkippedPackage:
             self.pn = info.pn
             self.skipreason = info.skipreason
             self.provides = info.provides
-            self.rprovides = info.rprovides
+            self.rprovides = info.packages + info.rprovides
+            for package in info.packages:
+                self.rprovides += info.rprovides_pkg[package]
         elif reason:
             self.skipreason = reason
 
@@ -2207,18 +2209,18 @@ class CookerParser(object):
         except bb.BBHandledException as exc:
             self.error += 1
             logger.error('Failed to parse recipe: %s' % exc.recipe)
-            self.shutdown(clean=False)
+            self.shutdown(clean=False, force=True)
             return False
         except ParsingFailure as exc:
             self.error += 1
             logger.error('Unable to parse %s: %s' %
                      (exc.recipe, bb.exceptions.to_string(exc.realexception)))
-            self.shutdown(clean=False)
+            self.shutdown(clean=False, force=True)
             return False
         except bb.parse.ParseError as exc:
             self.error += 1
             logger.error(str(exc))
-            self.shutdown(clean=False)
+            self.shutdown(clean=False, force=True)
             return False
         except bb.data_smart.ExpansionError as exc:
             self.error += 1
@@ -2227,7 +2229,7 @@ class CookerParser(object):
             tb = list(itertools.dropwhile(lambda e: e.filename.startswith(bbdir), exc.traceback))
             logger.error('ExpansionError during parsing %s', value.recipe,
                          exc_info=(etype, value, tb))
-            self.shutdown(clean=False)
+            self.shutdown(clean=False, force=True)
             return False
         except Exception as exc:
             self.error += 1
@@ -2239,7 +2241,7 @@ class CookerParser(object):
                 # Most likely, an exception occurred during raising an exception
                 import traceback
                 logger.error('Exception during parse: %s' % traceback.format_exc())
-            self.shutdown(clean=False)
+            self.shutdown(clean=False, force=True)
             return False
 
         self.current += 1

bitbake/lib/bb/data_smart.py

@@ -28,7 +28,7 @@ logger = logging.getLogger("BitBake.Data")
 
 __setvar_keyword__ = ["_append", "_prepend", "_remove"]
 __setvar_regexp__ = re.compile(r'(?P<base>.*?)(?P<keyword>_append|_prepend|_remove)(_(?P<add>[^A-Z]*))?$')
-__expand_var_regexp__ = re.compile(r"\${[a-zA-Z0-9\-_+./~]+?}")
+__expand_var_regexp__ = re.compile(r"\${[a-zA-Z0-9\-_+./~:]+?}")
 __expand_python_regexp__ = re.compile(r"\${@.+?}")
 __whitespace_split__ = re.compile(r'(\s)')
 __override_regexp__ = re.compile(r'[a-z0-9]+')
@@ -481,6 +481,7 @@ class DataSmart(MutableMapping):
 
     def setVar(self, var, value, **loginfo):
         #print("var=" + str(var) + "  val=" + str(value))
+        var = var.replace(":", "_")
         self.expand_cache = {}
         parsing=False
         if 'parsing' in loginfo:
@@ -589,6 +590,8 @@ class DataSmart(MutableMapping):
         """
         Rename the variable key to newkey
         """
+        key = key.replace(":", "_")
+        newkey = newkey.replace(":", "_")
         if key == newkey:
             bb.warn("Calling renameVar with equivalent keys (%s) is invalid" % key)
             return
@@ -637,6 +640,7 @@ class DataSmart(MutableMapping):
         self.setVar(var + "_prepend", value, ignore=True, parsing=True)
 
     def delVar(self, var, **loginfo):
+        var = var.replace(":", "_")
         self.expand_cache = {}
 
         loginfo['detail'] = ""
@@ -664,6 +668,7 @@ class DataSmart(MutableMapping):
                          override = None
 
     def setVarFlag(self, var, flag, value, **loginfo):
+        var = var.replace(":", "_")
         self.expand_cache = {}
 
         if 'op' not in loginfo:
@@ -687,6 +692,7 @@ class DataSmart(MutableMapping):
             self.dict["__exportlist"]["_content"].add(var)
 
     def getVarFlag(self, var, flag, expand=True, noweakdefault=False, parsing=False, retparser=False):
+        var = var.replace(":", "_")
         if flag == "_content":
             cachename = var
         else:
@@ -814,6 +820,7 @@ class DataSmart(MutableMapping):
         return value
 
     def delVarFlag(self, var, flag, **loginfo):
+        var = var.replace(":", "_")
         self.expand_cache = {}
 
         local_var, _ = self._findVar(var)
@@ -831,6 +838,7 @@ class DataSmart(MutableMapping):
             del self.dict[var][flag]
 
     def appendVarFlag(self, var, flag, value, **loginfo):
+        var = var.replace(":", "_")
         loginfo['op'] = 'append'
         loginfo['flag'] = flag
         self.varhistory.record(**loginfo)
@@ -838,6 +846,7 @@ class DataSmart(MutableMapping):
         self.setVarFlag(var, flag, newvalue, ignore=True)
 
     def prependVarFlag(self, var, flag, value, **loginfo):
+        var = var.replace(":", "_")
         loginfo['op'] = 'prepend'
         loginfo['flag'] = flag
         self.varhistory.record(**loginfo)
@@ -845,6 +854,7 @@ class DataSmart(MutableMapping):
         self.setVarFlag(var, flag, newvalue, ignore=True)
 
     def setVarFlags(self, var, flags, **loginfo):
+        var = var.replace(":", "_")
         self.expand_cache = {}
         infer_caller_details(loginfo)
         if not var in self.dict:
@@ -859,6 +869,7 @@ class DataSmart(MutableMapping):
             self.dict[var][i] = flags[i]
 
     def getVarFlags(self, var, expand = False, internalflags=False):
+        var = var.replace(":", "_")
         local_var, _ = self._findVar(var)
         flags = {}
 
@@ -875,6 +886,7 @@ class DataSmart(MutableMapping):
 
 
     def delVarFlags(self, var, **loginfo):
+        var = var.replace(":", "_")
         self.expand_cache = {}
         if not var in self.dict:
             self._makeShadowCopy(var)
@@ -1005,7 +1017,7 @@ class DataSmart(MutableMapping):
             else:
                 data.update({key:value})
 
-            varflags = d.getVarFlags(key, internalflags = True)
+            varflags = d.getVarFlags(key, internalflags = True, expand=["vardepvalue"])
             if not varflags:
                 continue
             for f in varflags:

bitbake/lib/bb/fetch2/__init__.py

@@ -853,11 +853,6 @@ def runfetchcmd(cmd, d, quiet=False, cleanup=None, log=None, workdir=None):
         if val:
             cmd = 'export ' + var + '=\"%s\"; %s' % (val, cmd)
 
-    # Ensure that a _PYTHON_SYSCONFIGDATA_NAME value set by a recipe
-    # (for example via python3native.bbclass since warrior) is not set for
-    # host Python (otherwise tools like git-make-shallow will fail)
-    cmd = 'unset _PYTHON_SYSCONFIGDATA_NAME; ' + cmd
-
     # Disable pseudo as it may affect ssh, potentially causing it to hang.
     cmd = 'export PSEUDO_DISABLED=1; ' + cmd
 

bitbake/lib/bb/fetch2/git.py

@@ -141,6 +141,10 @@ class Git(FetchMethod):
             ud.proto = 'file'
         else:
             ud.proto = "git"
+        if ud.host == "github.com" and ud.proto == "git":
+            # github stopped supporting git protocol
+            # https://github.blog/2021-09-01-improving-git-protocol-security-github/#no-more-unauthenticated-git
+            ud.proto = "https"
 
         if not ud.proto in ('git', 'file', 'ssh', 'http', 'https', 'rsync'):
             raise bb.fetch2.ParameterError("Invalid protocol type", ud.url)
@@ -379,6 +383,35 @@ class Git(FetchMethod):
             if missing_rev:
                 raise bb.fetch2.FetchError("Unable to find revision %s even from upstream" % missing_rev)
 
+        if self._contains_lfs(ud, d, ud.clonedir) and self._need_lfs(ud):
+            # Unpack temporary working copy, use it to run 'git checkout' to force pre-fetching
+            # of all LFS blobs needed at the the srcrev.
+            #
+            # It would be nice to just do this inline here by running 'git-lfs fetch'
+            # on the bare clonedir, but that operation requires a working copy on some
+            # releases of Git LFS.
+            tmpdir = tempfile.mkdtemp(dir=d.getVar('DL_DIR'))
+            try:
+                # Do the checkout. This implicitly involves a Git LFS fetch.
+                Git.unpack(self, ud, tmpdir, d)
+
+                # Scoop up a copy of any stuff that Git LFS downloaded. Merge them into
+                # the bare clonedir.
+                #
+                # As this procedure is invoked repeatedly on incremental fetches as
+                # a recipe's SRCREV is bumped throughout its lifetime, this will
+                # result in a gradual accumulation of LFS blobs in <ud.clonedir>/lfs
+                # corresponding to all the blobs reachable from the different revs
+                # fetched across time.
+                #
+                # Only do this if the unpack resulted in a .git/lfs directory being
+                # created; this only happens if at least one blob needed to be
+                # downloaded.
+                if os.path.exists(os.path.join(tmpdir, "git", ".git", "lfs")):
+                    runfetchcmd("tar -cf - lfs | tar -xf - -C %s" % ud.clonedir, d, workdir="%s/git/.git" % tmpdir)
+            finally:
+                bb.utils.remove(tmpdir, recurse=True)
+
     def build_mirror_data(self, ud, d):
         if ud.shallow and ud.write_shallow_tarballs:
             if not os.path.exists(ud.fullshallow):
@@ -474,7 +507,7 @@ class Git(FetchMethod):
         if os.path.exists(destdir):
             bb.utils.prunedir(destdir)
 
-        need_lfs = ud.parm.get("lfs", "1") == "1"
+        need_lfs = self._need_lfs(ud)
 
         if not need_lfs:
             ud.basecmd = "GIT_LFS_SKIP_SMUDGE=1 " + ud.basecmd
@@ -563,6 +596,9 @@ class Git(FetchMethod):
             raise bb.fetch2.FetchError("The command '%s' gave output with more then 1 line unexpectedly, output: '%s'" % (cmd, output))
         return output.split()[0] != "0"
 
+    def _need_lfs(self, ud):
+        return ud.parm.get("lfs", "1") == "1"
+
     def _contains_lfs(self, ud, d, wd):
         """
         Check if the repository has 'lfs' (large file) content
@@ -573,8 +609,14 @@ class Git(FetchMethod):
         else:
             branchname = "master"
 
-        cmd = "%s grep lfs origin/%s:.gitattributes | wc -l" % (
-            ud.basecmd, ud.branches[ud.names[0]])
+        # The bare clonedir doesn't use the remote names; it has the branch immediately.
+        if wd == ud.clonedir:
+            refname = ud.branches[ud.names[0]]
+        else:
+            refname = "origin/%s" % ud.branches[ud.names[0]]
+
+        cmd = "%s grep lfs %s:.gitattributes | wc -l" % (
+            ud.basecmd, refname)
 
         try:
             output = runfetchcmd(cmd, d, quiet=True, workdir=wd)

bitbake/lib/bb/fetch2/wget.py

@@ -316,7 +316,7 @@ class Wget(FetchMethod):
             except (TypeError, ImportError, IOError, netrc.NetrcParseError):
                 pass
 
-            with opener.open(r) as response:
+            with opener.open(r, timeout=30) as response:
                 pass
         except urllib.error.URLError as e:
             if try_again:

bitbake/lib/bb/parse/ast.py

@@ -97,6 +97,7 @@ class DataNode(AstNode):
     def eval(self, data):
         groupd = self.groupd
         key = groupd["var"]
+        key = key.replace(":", "_")
         loginfo = {
             'variable': key,
             'file': self.filename,
@@ -207,6 +208,7 @@ class ExportFuncsNode(AstNode):
     def eval(self, data):
 
         for func in self.n:
+            func = func.replace(":", "_")
             calledfunc = self.classname + "_" + func
 
             if data.getVar(func, False) and not data.getVarFlag(func, 'export_func', False):

bitbake/lib/bb/parse/parse_py/BBHandler.py

@@ -22,7 +22,7 @@ from .ConfHandler import include, init
 # For compatibility
 bb.deprecate_import(__name__, "bb.parse", ["vars_from_file"])
 
-__func_start_regexp__    = re.compile(r"(((?P<py>python)|(?P<fr>fakeroot))\s*)*(?P<func>[\w\.\-\+\{\}\$]+)?\s*\(\s*\)\s*{$" )
+__func_start_regexp__    = re.compile(r"(((?P<py>python(?=(\s|\()))|(?P<fr>fakeroot(?=\s)))\s*)*(?P<func>[\w\.\-\+\{\}\$:]+)?\s*\(\s*\)\s*{$" )
 __inherit_regexp__       = re.compile(r"inherit\s+(.+)" )
 __export_func_regexp__   = re.compile(r"EXPORT_FUNCTIONS\s+(.+)" )
 __addtask_regexp__       = re.compile(r"addtask\s+(?P<func>\w+)\s*((before\s*(?P<before>((.*(?=after))|(.*))))|(after\s*(?P<after>((.*(?=before))|(.*)))))*")

bitbake/lib/bb/parse/parse_py/ConfHandler.py

@@ -20,7 +20,7 @@ from bb.parse import ParseError, resolve_file, ast, logger, handle
 __config_regexp__  = re.compile( r"""
     ^
     (?P<exp>export\s+)?
-    (?P<var>[a-zA-Z0-9\-_+.${}/~]+?)
+    (?P<var>[a-zA-Z0-9\-_+.${}/~:]+?)
     (\[(?P<flag>[a-zA-Z0-9\-_+.]+)\])?
 
     \s* (

bitbake/lib/bb/providers.py

@@ -151,7 +151,7 @@ def findPreferredProvider(pn, cfgData, dataCache, pkg_pn = None, item = None):
         if item:
             itemstr = " (for item %s)" % item
         if preferred_file is None:
-            logger.info("preferred version %s of %s not available%s", pv_str, pn, itemstr)
+            logger.warning("preferred version %s of %s not available%s", pv_str, pn, itemstr)
             available_vers = []
             for file_set in pkg_pn:
                 for f in file_set:
@@ -163,7 +163,7 @@ def findPreferredProvider(pn, cfgData, dataCache, pkg_pn = None, item = None):
                         available_vers.append(ver_str)
             if available_vers:
                 available_vers.sort()
-                logger.info("versions of %s available: %s", pn, ' '.join(available_vers))
+                logger.warning("versions of %s available: %s", pn, ' '.join(available_vers))
         else:
             logger.debug(1, "selecting %s as PREFERRED_VERSION %s of package %s%s", preferred_file, pv_str, pn, itemstr)
 

bitbake/lib/bb/runqueue.py

@@ -1942,6 +1942,10 @@ class RunQueueExecute:
             logger.error("Scenequeue had holdoff tasks: %s" % pprint.pformat(self.holdoff_tasks))
             err = True
 
+        for tid in self.scenequeue_covered.intersection(self.scenequeue_notcovered):
+            # No task should end up in both covered and uncovered, that is a bug.
+            logger.error("Setscene task %s in both covered and notcovered." % tid)
+
         for tid in self.rqdata.runq_setscene_tids:
             if tid not in self.scenequeue_covered and tid not in self.scenequeue_notcovered:
                 err = True
@@ -2430,6 +2434,9 @@ class RunQueueExecute:
 
         for dep in sorted(self.sqdata.sq_deps[task]):
             if fail and task in self.sqdata.sq_harddeps and dep in self.sqdata.sq_harddeps[task]:
+                if dep in self.scenequeue_covered or dep in self.scenequeue_notcovered:
+                    # dependency could be already processed, e.g. noexec setscene task
+                    continue
                 logger.debug(2, "%s was unavailable and is a hard dependency of %s so skipping" % (task, dep))
                 self.sq_task_failoutright(dep)
                 continue
@@ -2791,6 +2798,7 @@ def update_scenequeue_data(tids, sqdata, rqdata, rq, cooker, stampcache, sqrq, s
     sqdata.valid |= rq.validate_hashes(tocheck, cooker.data, len(sqdata.stamppresent), False, summary=summary)
 
     sqdata.hashes = {}
+    sqrq.sq_deferred = {}
     for mc in sorted(sqdata.multiconfigs):
         for tid in sorted(sqdata.sq_revdeps):
             if mc_from_tid(tid) != mc:
@@ -2803,6 +2811,9 @@ def update_scenequeue_data(tids, sqdata, rqdata, rq, cooker, stampcache, sqrq, s
                 continue
             if tid in sqrq.scenequeue_notcovered:
                 continue
+            if tid in sqrq.scenequeue_covered:
+                continue
+
             sqdata.outrightfail.add(tid)
 
             h = pending_hash_index(tid, rqdata)

bitbake/lib/bb/server/process.py

@@ -509,7 +509,7 @@ class BitBakeServer(object):
         os.set_inheritable(self.bitbake_lock.fileno(), True)
         os.set_inheritable(self.readypipein, True)
         serverscript = os.path.realpath(os.path.dirname(__file__) + "/../../../bin/bitbake-server")
-        os.execl(sys.executable, "bitbake-server", serverscript, "decafbad", str(self.bitbake_lock.fileno()), str(self.readypipein), self.logfile, self.bitbake_lock.name, self.sockname,  str(self.server_timeout), str(self.xmlrpcinterface[0]), str(self.xmlrpcinterface[1]))
+        os.execl(sys.executable, "bitbake-server", serverscript, "decafbad", str(self.bitbake_lock.fileno()), str(self.readypipein), self.logfile, self.bitbake_lock.name, self.sockname,  str(self.server_timeout or 0), str(self.xmlrpcinterface[0]), str(self.xmlrpcinterface[1]))
 
 def execServer(lockfd, readypipeinfd, lockname, sockname, server_timeout, xmlrpcinterface):
 

bitbake/lib/bb/tests/codeparser.py

@@ -111,9 +111,9 @@ ${D}${libdir}/pkgconfig/*.pc
         self.assertExecs(set(["sed"]))
 
     def test_parameter_expansion_modifiers(self):
-        # - and + are also valid modifiers for parameter expansion, but are
+        # -,+ and : are also valid modifiers for parameter expansion, but are
         # valid characters in bitbake variable names, so are not included here
-        for i in ('=', ':-', ':=', '?', ':?', ':+', '#', '%', '##', '%%'):
+        for i in ('=', '?', '#', '%', '##', '%%'):
             name = "foo%sbar" % i
             self.parseExpression("${%s}" % name)
             self.assertNotIn(name, self.references)

bitbake/lib/bb/tests/fetch.py

@@ -939,7 +939,7 @@ class FetcherNetworkTest(FetcherTest):
 
     @skipIfNoNetwork()
     def test_git_submodule_CLI11(self):
-        url = "gitsm://github.com/CLIUtils/CLI11;protocol=git;rev=bd4dc911847d0cde7a6b41dfa626a85aab213baf"
+        url = "gitsm://github.com/CLIUtils/CLI11;protocol=git;rev=bd4dc911847d0cde7a6b41dfa626a85aab213baf;branch=main"
         fetcher = bb.fetch.Fetch([url], self.d)
         fetcher.download()
         # Previous cwd has been deleted
@@ -954,12 +954,12 @@ class FetcherNetworkTest(FetcherTest):
     @skipIfNoNetwork()
     def test_git_submodule_update_CLI11(self):
         """ Prevent regression on update detection not finding missing submodule, or modules without needed commits """
-        url = "gitsm://github.com/CLIUtils/CLI11;protocol=git;rev=cf6a99fa69aaefe477cc52e3ef4a7d2d7fa40714"
+        url = "gitsm://github.com/CLIUtils/CLI11;protocol=git;rev=cf6a99fa69aaefe477cc52e3ef4a7d2d7fa40714;branch=main"
         fetcher = bb.fetch.Fetch([url], self.d)
         fetcher.download()
 
         # CLI11 that pulls in a newer nlohmann-json
-        url = "gitsm://github.com/CLIUtils/CLI11;protocol=git;rev=49ac989a9527ee9bb496de9ded7b4872c2e0e5ca"
+        url = "gitsm://github.com/CLIUtils/CLI11;protocol=git;rev=49ac989a9527ee9bb496de9ded7b4872c2e0e5ca;branch=main"
         fetcher = bb.fetch.Fetch([url], self.d)
         fetcher.download()
         # Previous cwd has been deleted
@@ -993,7 +993,7 @@ class FetcherNetworkTest(FetcherTest):
         """ Prevent regression on deeply nested submodules not being checked out properly, even though they were fetched. """
 
         # This repository also has submodules where the module (name), path and url do not align
-        url = "gitsm://github.com/azure/iotedge.git;protocol=git;rev=d76e0316c6f324345d77c48a83ce836d09392699"
+        url = "gitsm://github.com/azure/iotedge.git;protocol=git;rev=d76e0316c6f324345d77c48a83ce836d09392699;branch=main"
         fetcher = bb.fetch.Fetch([url], self.d)
         fetcher.download()
         # Previous cwd has been deleted
@@ -1180,7 +1180,7 @@ class FetchLatestVersionTest(FetcherTest):
         ("presentproto", "git://git.yoctoproject.org/bbfetchtests-presentproto", "24f3a56e541b0a9e6c6ee76081f441221a120ef9", "")
             : "1.0",
         # version pattern "pkg_name-vX.Y.Z"
-        ("dtc", "git://git.qemu.org/dtc.git", "65cc4d2748a2c2e6f27f1cf39e07a5dbabd80ebf", "")
+        ("dtc", "git://git.yoctoproject.org/bbfetchtests-dtc.git", "65cc4d2748a2c2e6f27f1cf39e07a5dbabd80ebf", "")
             : "1.4.0",
         # combination version pattern
         ("sysprof", "git://gitlab.gnome.org/GNOME/sysprof.git;protocol=https", "cd44ee6644c3641507fb53b8a2a69137f2971219", "")
@@ -2051,13 +2051,14 @@ class GitLfsTest(FetcherTest):
             cwd = self.gitdir
         return bb.process.run(cmd, cwd=cwd)[0]
 
-    def fetch(self, uri=None):
+    def fetch(self, uri=None, download=True):
         uris = self.d.getVar('SRC_URI').split()
         uri = uris[0]
         d = self.d
 
         fetcher = bb.fetch2.Fetch(uris, d)
-        fetcher.download()
+        if download:
+            fetcher.download()
         ud = fetcher.ud[uri]
         return fetcher, ud
 
@@ -2067,16 +2068,21 @@ class GitLfsTest(FetcherTest):
         uri = 'git://%s;protocol=file;subdir=${S};lfs=1' % self.srcdir
         self.d.setVar('SRC_URI', uri)
 
-        fetcher, ud = self.fetch()
+        # Careful: suppress initial attempt at downloading until
+        # we know whether git-lfs is installed.
+        fetcher, ud = self.fetch(uri=None, download=False)
         self.assertIsNotNone(ud.method._find_git_lfs)
 
-        # If git-lfs can be found, the unpack should be successful
-        ud.method._find_git_lfs = lambda d: True
-        shutil.rmtree(self.gitdir, ignore_errors=True)
-        fetcher.unpack(self.d.getVar('WORKDIR'))
+        # If git-lfs can be found, the unpack should be successful. Only
+        # attempt this with the real live copy of git-lfs installed.
+        if ud.method._find_git_lfs(self.d):
+            fetcher.download()
+            shutil.rmtree(self.gitdir, ignore_errors=True)
+            fetcher.unpack(self.d.getVar('WORKDIR'))
 
         # If git-lfs cannot be found, the unpack should throw an error
         with self.assertRaises(bb.fetch2.FetchError):
+            fetcher.download()
             ud.method._find_git_lfs = lambda d: False
             shutil.rmtree(self.gitdir, ignore_errors=True)
             fetcher.unpack(self.d.getVar('WORKDIR'))
@@ -2087,10 +2093,16 @@ class GitLfsTest(FetcherTest):
         uri = 'git://%s;protocol=file;subdir=${S};lfs=0' % self.srcdir
         self.d.setVar('SRC_URI', uri)
 
+        # In contrast to test_lfs_enabled(), allow the implicit download
+        # done by self.fetch() to occur here. The point of this test case
+        # is to verify that the fetcher can survive even if the source
+        # repository has Git LFS usage configured.
         fetcher, ud = self.fetch()
         self.assertIsNotNone(ud.method._find_git_lfs)
 
-        # If git-lfs can be found, the unpack should be successful
+        # If git-lfs can be found, the unpack should be successful. A
+        # live copy of git-lfs is not required for this case, so
+        # unconditionally forge its presence.
         ud.method._find_git_lfs = lambda d: True
         shutil.rmtree(self.gitdir, ignore_errors=True)
         fetcher.unpack(self.d.getVar('WORKDIR'))

documentation/adt-manual/adt-command.rst

@@ -1,180 +0,0 @@
-.. SPDX-License-Identifier: CC-BY-SA-2.0-UK
-
-**********************
-Using the Command Line
-**********************
-
-Recall that earlier the manual discussed how to use an existing
-toolchain tarball that had been installed into the default installation
-directory, ``/opt/poky/DISTRO``, which is outside of the :term:`Build Directory`
-(see the section
-"`Using a Cross-Toolchain
-Tarball) <#using-an-existing-toolchain-tarball>`__". And, that sourcing
-your architecture-specific environment setup script initializes a
-suitable cross-toolchain development environment.
-
-During this setup, locations for the compiler, QEMU scripts, QEMU
-binary, a special version of ``pkgconfig`` and other useful utilities
-are added to the ``PATH`` variable. Also, variables to assist
-``pkgconfig`` and ``autotools`` are also defined so that, for example,
-``configure.sh`` can find pre-generated test results for tests that need
-target hardware on which to run. You can see the "`Setting Up the
-Cross-Development
-Environment <#setting-up-the-cross-development-environment>`__" section
-for the list of cross-toolchain environment variables established by the
-script.
-
-Collectively, these conditions allow you to easily use the toolchain
-outside of the OpenEmbedded build environment on both Autotools-based
-projects and Makefile-based projects. This chapter provides information
-for both these types of projects.
-
-Autotools-Based Projects
-========================
-
-Once you have a suitable cross-toolchain installed, it is very easy to
-develop a project outside of the OpenEmbedded build system. This section
-presents a simple "Helloworld" example that shows how to set up,
-compile, and run the project.
-
-Creating and Running a Project Based on GNU Autotools
------------------------------------------------------
-
-Follow these steps to create a simple Autotools-based project:
-
-1.  *Create your directory:* Create a clean directory for your project
-    and then make that directory your working location: $ mkdir
-    $HOME/helloworld $ cd $HOME/helloworld
-
-2.  *Populate the directory:* Create ``hello.c``, ``Makefile.am``, and
-    ``configure.in`` files as follows:
-
-    -  For ``hello.c``, include these lines: #include <stdio.h> main() {
-       printf("Hello World!\n"); }
-
-    -  For ``Makefile.am``, include these lines: bin_PROGRAMS = hello
-       hello_SOURCES = hello.c
-
-    -  For ``configure.in``, include these lines: AC_INIT(hello.c)
-       AM_INIT_AUTOMAKE(hello,0.1) AC_PROG_CC AC_PROG_INSTALL
-       AC_OUTPUT(Makefile)
-
-3.  *Source the cross-toolchain environment setup file:* Installation of
-    the cross-toolchain creates a cross-toolchain environment setup
-    script in the directory that the ADT was installed. Before you can
-    use the tools to develop your project, you must source this setup
-    script. The script begins with the string "environment-setup" and
-    contains the machine architecture, which is followed by the string
-    "poky-linux". Here is an example that sources a script from the
-    default ADT installation directory that uses the 32-bit Intel x86
-    Architecture and the DISTRO_NAME Yocto Project release: $ source
-    /opt/poky/DISTRO/environment-setup-i586-poky-linux
-
-4.  *Generate the local aclocal.m4 files and create the configure
-    script:* The following GNU Autotools generate the local
-    ``aclocal.m4`` files and create the configure script: $ aclocal $
-    autoconf
-
-5.  *Generate files needed by GNU coding standards:* GNU coding
-    standards require certain files in order for the project to be
-    compliant. This command creates those files: $ touch NEWS README
-    AUTHORS ChangeLog
-
-6.  *Generate the configure file:* This command generates the
-    ``configure``: $ automake -a
-
-7.  *Cross-compile the project:* This command compiles the project using
-    the cross-compiler. The
-    :term:`CONFIGURE_FLAGS`
-    environment variable provides the minimal arguments for GNU
-    configure: $ ./configure ${CONFIGURE_FLAGS}
-
-8.  *Make and install the project:* These two commands generate and
-    install the project into the destination directory: $ make $ make
-    install DESTDIR=./tmp
-
-9.  *Verify the installation:* This command is a simple way to verify
-    the installation of your project. Running the command prints the
-    architecture on which the binary file can run. This architecture
-    should be the same architecture that the installed cross-toolchain
-    supports. $ file ./tmp/usr/local/bin/hello
-
-10. *Execute your project:* To execute the project in the shell, simply
-    enter the name. You could also copy the binary to the actual target
-    hardware and run the project there as well: $ ./hello As expected,
-    the project displays the "Hello World!" message.
-
-Passing Host Options
---------------------
-
-For an Autotools-based project, you can use the cross-toolchain by just
-passing the appropriate host option to ``configure.sh``. The host option
-you use is derived from the name of the environment setup script found
-in the directory in which you installed the cross-toolchain. For
-example, the host option for an ARM-based target that uses the GNU EABI
-is ``armv5te-poky-linux-gnueabi``. You will notice that the name of the
-script is ``environment-setup-armv5te-poky-linux-gnueabi``. Thus, the
-following command works to update your project and rebuild it using the
-appropriate cross-toolchain tools: $ ./configure
---host=armv5te-poky-linux-gnueabi \\ --with-libtool-sysroot=sysroot_dir
-
-.. note::
-
-   If the
-   configure
-   script results in problems recognizing the
-   --with-libtool-sysroot=
-   sysroot-dir
-   option, regenerate the script to enable the support by doing the
-   following and then run the script again:
-   ::
-
-           $ libtoolize --automake
-           $ aclocal -I ${OECORE_NATIVE_SYSROOT}/usr/share/aclocal \
-              [-I dir_containing_your_project-specific_m4_macros]
-           $ autoconf
-           $ autoheader
-           $ automake -a
-                      
-
-Makefile-Based Projects
-=======================
-
-For Makefile-based projects, the cross-toolchain environment variables
-established by running the cross-toolchain environment setup script are
-subject to general ``make`` rules.
-
-To illustrate this, consider the following four cross-toolchain
-environment variables:
-:term:`CC`\ =i586-poky-linux-gcc -m32
--march=i586 --sysroot=/opt/poky/1.8/sysroots/i586-poky-linux
-:term:`LD`\ =i586-poky-linux-ld
---sysroot=/opt/poky/1.8/sysroots/i586-poky-linux
-:term:`CFLAGS`\ =-O2 -pipe -g
--feliminate-unused-debug-types
-:term:`CXXFLAGS`\ =-O2 -pipe -g
--feliminate-unused-debug-types Now, consider the following three cases:
-
--  *Case 1 - No Variables Set in the ``Makefile``:* Because these
-   variables are not specifically set in the ``Makefile``, the variables
-   retain their values based on the environment.
-
--  *Case 2 - Variables Set in the ``Makefile``:* Specifically setting
-   variables in the ``Makefile`` during the build results in the
-   environment settings of the variables being overwritten.
-
--  *Case 3 - Variables Set when the ``Makefile`` is Executed from the
-   Command Line:* Executing the ``Makefile`` from the command line
-   results in the variables being overwritten with command-line content
-   regardless of what is being set in the ``Makefile``. In this case,
-   environment variables are not considered unless you use the "-e" flag
-   during the build: $ make -e file If you use this flag, then the
-   environment values of the variables override any variables
-   specifically set in the ``Makefile``.
-
-.. note::
-
-   For the list of variables set up by the cross-toolchain environment
-   setup script, see the "
-   Setting Up the Cross-Development Environment
-   " section.

documentation/adt-manual/adt-intro.rst

@@ -1,138 +0,0 @@
-.. SPDX-License-Identifier: CC-BY-SA-2.0-UK
-
-*****************************************
-The Application Development Toolkit (ADT)
-*****************************************
-
-Part of the Yocto Project development solution is an Application
-Development Toolkit (ADT). The ADT provides you with a custom-built,
-cross-development platform suited for developing a user-targeted product
-application.
-
-Fundamentally, the ADT consists of the following:
-
--  An architecture-specific cross-toolchain and matching sysroot both
-   built by the :term:`OpenEmbedded Build System`.
-   The toolchain and
-   sysroot are based on a `Metadata <&YOCTO_DOCS_DEV_URL;#metadata>`__
-   configuration and extensions, which allows you to cross-develop on
-   the host machine for the target hardware.
-
--  The Eclipse IDE Yocto Plug-in.
-
--  The Quick EMUlator (QEMU), which lets you simulate target hardware.
-
--  Various user-space tools that greatly enhance your application
-   development experience.
-
-The Cross-Development Toolchain
-===============================
-
-The `Cross-Development
-Toolchain <&YOCTO_DOCS_DEV_URL;#cross-development-toolchain>`__ consists
-of a cross-compiler, cross-linker, and cross-debugger that are used to
-develop user-space applications for targeted hardware. This toolchain is
-created either by running the ADT Installer script, a toolchain
-installer script, or through a :term:`Build Directory`
-that is based on
-your Metadata configuration or extension for your targeted device. The
-cross-toolchain works with a matching target sysroot.
-
-Sysroot
-=======
-
-The matching target sysroot contains needed headers and libraries for
-generating binaries that run on the target architecture. The sysroot is
-based on the target root filesystem image that is built by the
-OpenEmbedded build system and uses the same Metadata configuration used
-to build the cross-toolchain.
-
-.. _eclipse-overview:
-
-Eclipse Yocto Plug-in
-=====================
-
-The Eclipse IDE is a popular development environment and it fully
-supports development using the Yocto Project. When you install and
-configure the Eclipse Yocto Project Plug-in into the Eclipse IDE, you
-maximize your Yocto Project experience. Installing and configuring the
-Plug-in results in an environment that has extensions specifically
-designed to let you more easily develop software. These extensions allow
-for cross-compilation, deployment, and execution of your output into a
-QEMU emulation session. You can also perform cross-debugging and
-profiling. The environment also supports a suite of tools that allows
-you to perform remote profiling, tracing, collection of power data,
-collection of latency data, and collection of performance data.
-
-For information about the application development workflow that uses the
-Eclipse IDE and for a detailed example of how to install and configure
-the Eclipse Yocto Project Plug-in, see the "`Working Within
-Eclipse <&YOCTO_DOCS_DEV_URL;#adt-eclipse>`__" section of the Yocto
-Project Development Manual.
-
-The QEMU Emulator
-=================
-
-The QEMU emulator allows you to simulate your hardware while running
-your application or image. QEMU is made available a number of ways:
-
--  If you use the ADT Installer script to install ADT, you can specify
-   whether or not to install QEMU.
-
--  If you have cloned the ``poky`` Git repository to create a
-   :term:`Source Directory` and you have
-   sourced the environment setup script, QEMU is installed and
-   automatically available.
-
--  If you have downloaded a Yocto Project release and unpacked it to
-   create a :term:`Source Directory`
-   and you have sourced the environment setup script, QEMU is installed
-   and automatically available.
-
--  If you have installed the cross-toolchain tarball and you have
-   sourced the toolchain's setup environment script, QEMU is also
-   installed and automatically available.
-
-User-Space Tools
-================
-
-User-space tools are included as part of the Yocto Project. You will
-find these tools helpful during development. The tools include
-LatencyTOP, PowerTOP, OProfile, Perf, SystemTap, and Lttng-ust. These
-tools are common development tools for the Linux platform.
-
--  *LatencyTOP:* LatencyTOP focuses on latency that causes skips in
-   audio, stutters in your desktop experience, or situations that
-   overload your server even when you have plenty of CPU power left.
-
--  *PowerTOP:* Helps you determine what software is using the most
-   power. You can find out more about PowerTOP at
-   https://01.org/powertop/.
-
--  *OProfile:* A system-wide profiler for Linux systems that is capable
-   of profiling all running code at low overhead. You can find out more
-   about OProfile at http://oprofile.sourceforge.net/about/. For
-   examples on how to setup and use this tool, see the
-   "`OProfile <&YOCTO_DOCS_PROF_URL;#profile-manual-oprofile>`__"
-   section in the Yocto Project Profiling and Tracing Manual.
-
--  *Perf:* Performance counters for Linux used to keep track of certain
-   types of hardware and software events. For more information on these
-   types of counters see https://perf.wiki.kernel.org/. For
-   examples on how to setup and use this tool, see the
-   "`perf <&YOCTO_DOCS_PROF_URL;#profile-manual-perf>`__" section in the
-   Yocto Project Profiling and Tracing Manual.
-
--  *SystemTap:* A free software infrastructure that simplifies
-   information gathering about a running Linux system. This information
-   helps you diagnose performance or functional problems. SystemTap is
-   not available as a user-space tool through the Eclipse IDE Yocto
-   Plug-in. See http://sourceware.org/systemtap for more
-   information on SystemTap. For examples on how to setup and use this
-   tool, see the
-   "`SystemTap <&YOCTO_DOCS_PROF_URL;#profile-manual-systemtap>`__"
-   section in the Yocto Project Profiling and Tracing Manual.
-
--  *Lttng-ust:* A User-space Tracer designed to provide detailed
-   information on user-space activity. See http://lttng.org/ust
-   for more information on Lttng-ust.

documentation/adt-manual/adt-manual-intro.rst

@@ -1,24 +0,0 @@
-.. SPDX-License-Identifier: CC-BY-SA-2.0-UK
-
-************
-Introduction
-************
-
-Welcome to the Yocto Project Application Developer's Guide. This manual
-provides information that lets you begin developing applications using
-the Yocto Project.
-
-The Yocto Project provides an application development environment based
-on an Application Development Toolkit (ADT) and the availability of
-stand-alone cross-development toolchains and other tools. This manual
-describes the ADT and how you can configure and install it, how to
-access and use the cross-development toolchains, how to customize the
-development packages installation, how to use command-line development
-for both Autotools-based and Makefile-based projects, and an
-introduction to the Eclipse IDE Yocto Plug-in.
-
-.. note::
-
-   The ADT is distribution-neutral and does not require the Yocto
-   Project reference distribution, which is called Poky. This manual,
-   however, uses examples that use the Poky distribution.

documentation/adt-manual/adt-manual.rst

@@ -1,17 +0,0 @@
-.. SPDX-License-Identifier: CC-BY-SA-2.0-UK
-
-===========================================
-Yocto Project Application Developer's Guide
-===========================================
-
-|
-
-.. toctree::
-   :caption: Table of Contents
-   :numbered:
-
-   adt-manual-intro
-   adt-intro
-   adt-prepare
-   adt-package
-   adt-command

documentation/adt-manual/adt-package.rst

@@ -1,70 +0,0 @@
-.. SPDX-License-Identifier: CC-BY-SA-2.0-UK
-
-************************************************************
-Optionally Customizing the Development Packages Installation
-************************************************************
-
-Because the Yocto Project is suited for embedded Linux development, it
-is likely that you will need to customize your development packages
-installation. For example, if you are developing a minimal image, then
-you might not need certain packages (e.g. graphics support packages).
-Thus, you would like to be able to remove those packages from your
-target sysroot.
-
-Package Management Systems
-==========================
-
-The OpenEmbedded build system supports the generation of sysroot files
-using three different Package Management Systems (PMS):
-
--  *OPKG:* A less well known PMS whose use originated in the
-   OpenEmbedded and OpenWrt embedded Linux projects. This PMS works with
-   files packaged in an ``.ipk`` format. See
-   http://en.wikipedia.org/wiki/Opkg for more information about
-   OPKG.
-
--  *RPM:* A more widely known PMS intended for GNU/Linux distributions.
-   This PMS works with files packaged in an ``.rpm`` format. The build
-   system currently installs through this PMS by default. See
-   http://en.wikipedia.org/wiki/RPM_Package_Manager for more
-   information about RPM.
-
--  *Debian:* The PMS for Debian-based systems is built on many PMS
-   tools. The lower-level PMS tool ``dpkg`` forms the base of the Debian
-   PMS. For information on dpkg see
-   http://en.wikipedia.org/wiki/Dpkg.
-
-Configuring the PMS
-===================
-
-Whichever PMS you are using, you need to be sure that the
-:term:`PACKAGE_CLASSES`
-variable in the ``conf/local.conf`` file is set to reflect that system.
-The first value you choose for the variable specifies the package file
-format for the root filesystem at sysroot. Additional values specify
-additional formats for convenience or testing. See the
-``conf/local.conf`` configuration file for details.
-
-.. note::
-
-   For build performance information related to the PMS, see the "
-   package.bbclass
-   " section in the Yocto Project Reference Manual.
-
-As an example, consider a scenario where you are using OPKG and you want
-to add the ``libglade`` package to the target sysroot.
-
-First, you should generate the IPK file for the ``libglade`` package and
-add it into a working ``opkg`` repository. Use these commands: $ bitbake
-libglade $ bitbake package-index
-
-Next, source the cross-toolchain environment setup script found in the
-:term:`Source Directory`. Follow
-that by setting up the installation destination to point to your sysroot
-as sysroot_dir. Finally, have an OPKG configuration file conf_file that
-corresponds to the ``opkg`` repository you have just created. The
-following command forms should now work: $ opkg-cl –f conf_file -o
-sysroot_dir update $ opkg-cl –f cconf_file -o sysroot_dir \\
---force-overwrite install libglade $ opkg-cl –f cconf_file -o
-sysroot_dir \\ --force-overwrite install libglade-dbg $ opkg-cl –f
-conf_file> -osysroot_dir> \\ --force-overwrite install libglade-dev

documentation/adt-manual/adt-prepare.rst

@@ -1,752 +0,0 @@
-.. SPDX-License-Identifier: CC-BY-SA-2.0-UK
-
-*************************************
-Preparing for Application Development
-*************************************
-
-In order to develop applications, you need set up your host development
-system. Several ways exist that allow you to install cross-development
-tools, QEMU, the Eclipse Yocto Plug-in, and other tools. This chapter
-describes how to prepare for application development.
-
-.. _installing-the-adt:
-
-Installing the ADT and Toolchains
-=================================
-
-The following list describes installation methods that set up varying
-degrees of tool availability on your system. Regardless of the
-installation method you choose, you must ``source`` the cross-toolchain
-environment setup script, which establishes several key environment
-variables, before you use a toolchain. See the "`Setting Up the
-Cross-Development
-Environment <#setting-up-the-cross-development-environment>`__" section
-for more information.
-
-.. note::
-
-   Avoid mixing installation methods when installing toolchains for
-   different architectures. For example, avoid using the ADT Installer
-   to install some toolchains and then hand-installing cross-development
-   toolchains by running the toolchain installer for different
-   architectures. Mixing installation methods can result in situations
-   where the ADT Installer becomes unreliable and might not install the
-   toolchain.
-
-   If you must mix installation methods, you might avoid problems by
-   deleting ``/var/lib/opkg``, thus purging the ``opkg`` package
-   metadata.
-
--  *Use the ADT installer script:* This method is the recommended way to
-   install the ADT because it automates much of the process for you. For
-   example, you can configure the installation to install the QEMU
-   emulator and the user-space NFS, specify which root filesystem
-   profiles to download, and define the target sysroot location.
-
--  *Use an existing toolchain:* Using this method, you select and
-   download an architecture-specific toolchain installer and then run
-   the script to hand-install the toolchain. If you use this method, you
-   just get the cross-toolchain and QEMU - you do not get any of the
-   other mentioned benefits had you run the ADT Installer script.
-
--  *Use the toolchain from within the Build Directory:* If you already
-   have a :term:`Build Directory`,
-   you can build the cross-toolchain within the directory. However, like
-   the previous method mentioned, you only get the cross-toolchain and
-   QEMU - you do not get any of the other benefits without taking
-   separate steps.
-
-Using the ADT Installer
------------------------
-
-To run the ADT Installer, you need to get the ADT Installer tarball, be
-sure you have the necessary host development packages that support the
-ADT Installer, and then run the ADT Installer Script.
-
-For a list of the host packages needed to support ADT installation and
-use, see the "ADT Installer Extras" lists in the "`Required Packages for
-the Host Development
-System <&YOCTO_DOCS_REF_URL;#required-packages-for-the-host-development-system>`__"
-section of the Yocto Project Reference Manual.
-
-Getting the ADT Installer Tarball
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-
-The ADT Installer is contained in the ADT Installer tarball. You can get
-the tarball using either of these methods:
-
--  *Download the Tarball:* You can download the tarball from
-   ` <&YOCTO_ADTINSTALLER_DL_URL;>`__ into any directory.
-
--  *Build the Tarball:* You can use
-   :term:`BitBake` to generate the
-   tarball inside an existing :term:`Build Directory`.
-
-   If you use BitBake to generate the ADT Installer tarball, you must
-   ``source`` the environment setup script
-   (````` <&YOCTO_DOCS_REF_URL;#structure-core-script>`__ or
-   ```oe-init-build-env-memres`` <&YOCTO_DOCS_REF_URL;#structure-memres-core-script>`__)
-   located in the Source Directory before running the ``bitbake``
-   command that creates the tarball.
-
-   The following example commands establish the
-   :term:`Source Directory`, check out the
-   current release branch, set up the build environment while also
-   creating the default Build Directory, and run the ``bitbake`` command
-   that results in the tarball
-   ``poky/build/tmp/deploy/sdk/adt_installer.tar.bz2``:
-
-   .. note::
-
-      Before using BitBake to build the ADT tarball, be sure to make
-      sure your
-      local.conf
-      file is properly configured. See the "
-      User Configuration
-      " section in the Yocto Project Reference Manual for general
-      configuration information.
-
-   $ cd ~ $ git clone git://git.yoctoproject.org/poky $ cd poky $ git
-   checkout -b DISTRO_NAME origin/DISTRO_NAME $ source oe-init-build-env $
-   bitbake adt-installer
-
-Configuring and Running the ADT Installer Script
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-
-Before running the ADT Installer script, you need to unpack the tarball.
-You can unpack the tarball in any directory you wish. For example, this
-command copies the ADT Installer tarball from where it was built into
-the home directory and then unpacks the tarball into a top-level
-directory named ``adt-installer``: $ cd ~ $ cp
-poky/build/tmp/deploy/sdk/adt_installer.tar.bz2 $HOME $ tar -xjf
-adt_installer.tar.bz2 Unpacking it creates the directory
-``adt-installer``, which contains the ADT Installer script
-(``adt_installer``) and its configuration file (``adt_installer.conf``).
-
-Before you run the script, however, you should examine the ADT Installer
-configuration file and be sure you are going to get what you want. Your
-configurations determine which kernel and filesystem image are
-downloaded.
-
-The following list describes the configurations you can define for the
-ADT Installer. For configuration values and restrictions, see the
-comments in the ``adt-installer.conf`` file:
-
--  ``YOCTOADT_REPO``: This area includes the IPKG-based packages and the
-   root filesystem upon which the installation is based. If you want to
-   set up your own IPKG repository pointed to by ``YOCTOADT_REPO``, you
-   need to be sure that the directory structure follows the same layout
-   as the reference directory set up at
-   http://adtrepo.yoctoproject.org. Also, your repository needs
-   to be accessible through HTTP.
-
--  ``YOCTOADT_TARGETS``: The machine target architectures for which you
-   want to set up cross-development environments.
-
--  ``YOCTOADT_QEMU``: Indicates whether or not to install the emulator
-   QEMU.
-
--  ``YOCTOADT_NFS_UTIL``: Indicates whether or not to install user-mode
-   NFS. If you plan to use the Eclipse IDE Yocto plug-in against QEMU,
-   you should install NFS.
-
-   .. note::
-
-      To boot QEMU images using our userspace NFS server, you need to be
-      running
-      portmap
-      or
-      rpcbind
-      . If you are running
-      rpcbind
-      , you will also need to add the
-      -i
-      option when
-      rpcbind
-      starts up. Please make sure you understand the security
-      implications of doing this. You might also have to modify your
-      firewall settings to allow NFS booting to work.
-
--  ``YOCTOADT_ROOTFS_``\ arch: The root filesystem images you want to
-   download from the ``YOCTOADT_IPKG_REPO`` repository.
-
--  ``YOCTOADT_TARGET_SYSROOT_IMAGE_``\ arch: The particular root
-   filesystem used to extract and create the target sysroot. The value
-   of this variable must have been specified with
-   ``YOCTOADT_ROOTFS_``\ arch. For example, if you downloaded both
-   ``minimal`` and ``sato-sdk`` images by setting
-   ``YOCTOADT_ROOTFS_``\ arch to "minimal sato-sdk", then
-   ``YOCTOADT_ROOTFS_``\ arch must be set to either "minimal" or
-   "sato-sdk".
-
--  ``YOCTOADT_TARGET_SYSROOT_LOC_``\ arch: The location on the
-   development host where the target sysroot is created.
-
-After you have configured the ``adt_installer.conf`` file, run the
-installer using the following command: $ cd adt-installer $
-./adt_installer Once the installer begins to run, you are asked to enter
-the location for cross-toolchain installation. The default location is
-``/opt/poky/``\ release. After either accepting the default location or
-selecting your own location, you are prompted to run the installation
-script interactively or in silent mode. If you want to closely monitor
-the installation, choose "I" for interactive mode rather than "S" for
-silent mode. Follow the prompts from the script to complete the
-installation.
-
-Once the installation completes, the ADT, which includes the
-cross-toolchain, is installed in the selected installation directory.
-You will notice environment setup files for the cross-toolchain in the
-installation directory, and image tarballs in the ``adt-installer``
-directory according to your installer configurations, and the target
-sysroot located according to the ``YOCTOADT_TARGET_SYSROOT_LOC_``\ arch
-variable also in your configuration file.
-
-.. _using-an-existing-toolchain-tarball:
-
-Using a Cross-Toolchain Tarball
--------------------------------
-
-If you want to simply install a cross-toolchain by hand, you can do so
-by running the toolchain installer. The installer includes the pre-built
-cross-toolchain, the ``runqemu`` script, and support files. If you use
-this method to install the cross-toolchain, you might still need to
-install the target sysroot by installing and extracting it separately.
-For information on how to install the sysroot, see the "`Extracting the
-Root Filesystem <#extracting-the-root-filesystem>`__" section.
-
-Follow these steps:
-
-1. *Get your toolchain installer using one of the following methods:*
-
-   -  Go to ` <&YOCTO_TOOLCHAIN_DL_URL;>`__ and find the folder that
-      matches your host development system (i.e. ``i686`` for 32-bit
-      machines or ``x86_64`` for 64-bit machines).
-
-      Go into that folder and download the toolchain installer whose
-      name includes the appropriate target architecture. The toolchains
-      provided by the Yocto Project are based off of the
-      ``core-image-sato`` image and contain libraries appropriate for
-      developing against that image. For example, if your host
-      development system is a 64-bit x86 system and you are going to use
-      your cross-toolchain for a 32-bit x86 target, go into the
-      ``x86_64`` folder and download the following installer:
-      poky-glibc-x86_64-core-image-sato-i586-toolchain-DISTRO.sh
-
-   -  Build your own toolchain installer. For cases where you cannot use
-      an installer from the download area, you can build your own as
-      described in the "`Optionally Building a Toolchain
-      Installer <#optionally-building-a-toolchain-installer>`__"
-      section.
-
-2. *Once you have the installer, run it to install the toolchain:*
-
-   .. note::
-
-      You must change the permissions on the toolchain installer script
-      so that it is executable.
-
-   The following command shows how to run the installer given a
-   toolchain tarball for a 64-bit x86 development host system and a
-   32-bit x86 target architecture. The example assumes the toolchain
-   installer is located in ``~/Downloads/``. $
-   ~/Downloads/poky-glibc-x86_64-core-image-sato-i586-toolchain-DISTRO.sh
-   The first thing the installer prompts you for is the directory into
-   which you want to install the toolchain. The default directory used
-   is ``/opt/poky/DISTRO``. If you do not have write permissions for the
-   directory into which you are installing the toolchain, the toolchain
-   installer notifies you and exits. Be sure you have write permissions
-   in the directory and run the installer again.
-
-   When the script finishes, the cross-toolchain is installed. You will
-   notice environment setup files for the cross-toolchain in the
-   installation directory.
-
-.. _using-the-toolchain-from-within-the-build-tree:
-
-Using BitBake and the Build Directory
--------------------------------------
-
-A final way of making the cross-toolchain available is to use BitBake to
-generate the toolchain within an existing :term:`Build Directory`.
-This method does
-not install the toolchain into the default ``/opt`` directory. As with
-the previous method, if you need to install the target sysroot, you must
-do that separately as well.
-
-Follow these steps to generate the toolchain into the Build Directory:
-
-1. *Set up the Build Environment:* Source the OpenEmbedded build
-   environment setup script (i.e.
-   ````` <&YOCTO_DOCS_REF_URL;#structure-core-script>`__ or
-   ```oe-init-build-env-memres`` <&YOCTO_DOCS_REF_URL;#structure-memres-core-script>`__)
-   located in the :term:`Source Directory`.
-
-2. *Check your Local Configuration File:* At this point, you should be
-   sure that the :term:`MACHINE`
-   variable in the ``local.conf`` file found in the ``conf`` directory
-   of the Build Directory is set for the target architecture. Comments
-   within the ``local.conf`` file list the values you can use for the
-   ``MACHINE`` variable. If you do not change the ``MACHINE`` variable,
-   the OpenEmbedded build system uses ``qemux86`` as the default target
-   machine when building the cross-toolchain.
-
-   .. note::
-
-      You can populate the Build Directory with the cross-toolchains for
-      more than a single architecture. You just need to edit the
-      MACHINE
-      variable in the
-      local.conf
-      file and re-run the
-      bitbake
-      command.
-
-3. *Make Sure Your Layers are Enabled:* Examine the
-   ``conf/bblayers.conf`` file and make sure that you have enabled all
-   the compatible layers for your target machine. The OpenEmbedded build
-   system needs to be aware of each layer you want included when
-   building images and cross-toolchains. For information on how to
-   enable a layer, see the "`Enabling Your
-   Layer <&YOCTO_DOCS_DEV_URL;#enabling-your-layer>`__" section in the
-   Yocto Project Development Manual.
-
-4. *Generate the Cross-Toolchain:* Run ``bitbake meta-ide-support`` to
-   complete the cross-toolchain generation. Once the ``bitbake`` command
-   finishes, the cross-toolchain is generated and populated within the
-   Build Directory. You will notice environment setup files for the
-   cross-toolchain that contain the string "``environment-setup``" in
-   the Build Directory's ``tmp`` folder.
-
-   Be aware that when you use this method to install the toolchain, you
-   still need to separately extract and install the sysroot filesystem.
-   For information on how to do this, see the "`Extracting the Root
-   Filesystem <#extracting-the-root-filesystem>`__" section.
-
-Setting Up the Cross-Development Environment
-============================================
-
-Before you can develop using the cross-toolchain, you need to set up the
-cross-development environment by sourcing the toolchain's environment
-setup script. If you used the ADT Installer or hand-installed
-cross-toolchain, then you can find this script in the directory you
-chose for installation. For this release, the default installation
-directory is ````. If you installed the toolchain in the
-:term:`Build Directory`, you can find the
-environment setup script for the toolchain in the Build Directory's
-``tmp`` directory.
-
-Be sure to run the environment setup script that matches the
-architecture for which you are developing. Environment setup scripts
-begin with the string "``environment-setup``" and include as part of
-their name the architecture. For example, the toolchain environment
-setup script for a 64-bit IA-based architecture installed in the default
-installation directory would be the following:
-YOCTO_ADTPATH_DIR/environment-setup-x86_64-poky-linux When you run the
-setup script, many environment variables are defined:
-:term:`SDKTARGETSYSROOT` -
-The path to the sysroot used for cross-compilation
-:term:`PKG_CONFIG_PATH` - The
-path to the target pkg-config files
-:term:`CONFIG_SITE` - A GNU
-autoconf site file preconfigured for the target
-:term:`CC` - The minimal command and
-arguments to run the C compiler
-:term:`CXX` - The minimal command and
-arguments to run the C++ compiler
-:term:`CPP` - The minimal command and
-arguments to run the C preprocessor
-:term:`AS` - The minimal command and
-arguments to run the assembler :term:`LD`
-- The minimal command and arguments to run the linker
-:term:`GDB` - The minimal command and
-arguments to run the GNU Debugger
-:term:`STRIP` - The minimal command and
-arguments to run 'strip', which strips symbols
-:term:`RANLIB` - The minimal command
-and arguments to run 'ranlib'
-:term:`OBJCOPY` - The minimal command
-and arguments to run 'objcopy'
-:term:`OBJDUMP` - The minimal command
-and arguments to run 'objdump' :term:`AR`
-- The minimal command and arguments to run 'ar'
-:term:`NM` - The minimal command and
-arguments to run 'nm'
-:term:`TARGET_PREFIX` - The
-toolchain binary prefix for the target tools
-:term:`CROSS_COMPILE` - The
-toolchain binary prefix for the target tools
-:term:`CONFIGURE_FLAGS` - The
-minimal arguments for GNU configure
-:term:`CFLAGS` - Suggested C flags
-:term:`CXXFLAGS` - Suggested C++
-flags :term:`LDFLAGS` - Suggested
-linker flags when you use CC to link
-:term:`CPPFLAGS` - Suggested
-preprocessor flags
-
-Securing Kernel and Filesystem Images
-=====================================
-
-You will need to have a kernel and filesystem image to boot using your
-hardware or the QEMU emulator. Furthermore, if you plan on booting your
-image using NFS or you want to use the root filesystem as the target
-sysroot, you need to extract the root filesystem.
-
-Getting the Images
-------------------
-
-To get the kernel and filesystem images, you either have to build them
-or download pre-built versions. For an example of how to build these
-images, see the "`Buiding
-Images <&YOCTO_DOCS_QS_URL;#qs-buiding-images>`__" section of the Yocto
-Project Quick Start. For an example of downloading pre-build versions,
-see the "`Example Using Pre-Built Binaries and
-QEMU <#using-pre-built>`__" section.
-
-The Yocto Project ships basic kernel and filesystem images for several
-architectures (``x86``, ``x86-64``, ``mips``, ``powerpc``, and ``arm``)
-that you can use unaltered in the QEMU emulator. These kernel images
-reside in the release area - ` <&YOCTO_MACHINES_DL_URL;>`__ and are
-ideal for experimentation using Yocto Project. For information on the
-image types you can build using the OpenEmbedded build system, see the
-":ref:`ref-manual/ref-images:Images`" chapter in the Yocto
-Project Reference Manual.
-
-If you are planning on developing against your image and you are not
-building or using one of the Yocto Project development images (e.g.
-``core-image-*-dev``), you must be sure to include the development
-packages as part of your image recipe.
-
-If you plan on remotely deploying and debugging your application from
-within the Eclipse IDE, you must have an image that contains the Yocto
-Target Communication Framework (TCF) agent (``tcf-agent``). You can do
-this by including the ``eclipse-debug`` image feature.
-
-.. note::
-
-   See the "
-   Image Features
-   " section in the Yocto Project Reference Manual for information on
-   image features.
-
-To include the ``eclipse-debug`` image feature, modify your
-``local.conf`` file in the :term:`Build Directory`
-so that the
-:term:`EXTRA_IMAGE_FEATURES`
-variable includes the "eclipse-debug" feature. After modifying the
-configuration file, you can rebuild the image. Once the image is
-rebuilt, the ``tcf-agent`` will be included in the image and is launched
-automatically after the boot.
-
-Extracting the Root Filesystem
-------------------------------
-
-If you install your toolchain by hand or build it using BitBake and you
-need a root filesystem, you need to extract it separately. If you use
-the ADT Installer to install the ADT, the root filesystem is
-automatically extracted and installed.
-
-Here are some cases where you need to extract the root filesystem:
-
--  You want to boot the image using NFS.
-
--  You want to use the root filesystem as the target sysroot. For
-   example, the Eclipse IDE environment with the Eclipse Yocto Plug-in
-   installed allows you to use QEMU to boot under NFS.
-
--  You want to develop your target application using the root filesystem
-   as the target sysroot.
-
-To extract the root filesystem, first ``source`` the cross-development
-environment setup script to establish necessary environment variables.
-If you built the toolchain in the Build Directory, you will find the
-toolchain environment script in the ``tmp`` directory. If you installed
-the toolchain by hand, the environment setup script is located in
-``/opt/poky/DISTRO``.
-
-After sourcing the environment script, use the ``runqemu-extract-sdk``
-command and provide the filesystem image.
-
-Following is an example. The second command sets up the environment. In
-this case, the setup script is located in the ``/opt/poky/DISTRO``
-directory. The third command extracts the root filesystem from a
-previously built filesystem that is located in the ``~/Downloads``
-directory. Furthermore, this command extracts the root filesystem into
-the ``qemux86-sato`` directory: $ cd ~ $ source
-/opt/poky/DISTRO/environment-setup-i586-poky-linux $ runqemu-extract-sdk
-\\ ~/Downloads/core-image-sato-sdk-qemux86-2011091411831.rootfs.tar.bz2
-\\ $HOME/qemux86-sato You could now point to the target sysroot at
-``qemux86-sato``.
-
-Optionally Building a Toolchain Installer
-=========================================
-
-As an alternative to locating and downloading a toolchain installer, you
-can build the toolchain installer if you have a :term:`Build Directory`.
-
-.. note::
-
-   Although not the preferred method, it is also possible to use
-   bitbake meta-toolchain
-   to build the toolchain installer. If you do use this method, you must
-   separately install and extract the target sysroot. For information on
-   how to install the sysroot, see the "
-   Extracting the Root Filesystem
-   " section.
-
-To build the toolchain installer and populate the SDK image, use the
-following command: $ bitbake image -c populate_sdk The command results
-in a toolchain installer that contains the sysroot that matches your
-target root filesystem.
-
-Another powerful feature is that the toolchain is completely
-self-contained. The binaries are linked against their own copy of
-``libc``, which results in no dependencies on the target system. To
-achieve this, the pointer to the dynamic loader is configured at install
-time since that path cannot be dynamically altered. This is the reason
-for a wrapper around the ``populate_sdk`` archive.
-
-Another feature is that only one set of cross-canadian toolchain
-binaries are produced per architecture. This feature takes advantage of
-the fact that the target hardware can be passed to ``gcc`` as a set of
-compiler options. Those options are set up by the environment script and
-contained in variables such as :term:`CC`
-and :term:`LD`. This reduces the space
-needed for the tools. Understand, however, that a sysroot is still
-needed for every target since those binaries are target-specific.
-
-Remember, before using any BitBake command, you must source the build
-environment setup script (i.e.
-````` <&YOCTO_DOCS_REF_URL;#structure-core-script>`__ or
-```oe-init-build-env-memres`` <&YOCTO_DOCS_REF_URL;#structure-memres-core-script>`__)
-located in the Source Directory and you must make sure your
-``conf/local.conf`` variables are correct. In particular, you need to be
-sure the :term:`MACHINE` variable
-matches the architecture for which you are building and that the
-:term:`SDKMACHINE` variable is
-correctly set if you are building a toolchain designed to run on an
-architecture that differs from your current development host machine
-(i.e. the build machine).
-
-When the ``bitbake`` command completes, the toolchain installer will be
-in ``tmp/deploy/sdk`` in the Build Directory.
-
-.. note::
-
-   By default, this toolchain does not build static binaries. If you
-   want to use the toolchain to build these types of libraries, you need
-   to be sure your image has the appropriate static development
-   libraries. Use the
-   IMAGE_INSTALL
-   variable inside your
-   local.conf
-   file to install the appropriate library packages. Following is an
-   example using
-   glibc
-   static development libraries:
-   ::
-
-           IMAGE_INSTALL_append = " glibc-staticdev"
-                  
-
-Optionally Using an External Toolchain
-======================================
-
-You might want to use an external toolchain as part of your development.
-If this is the case, the fundamental steps you need to accomplish are as
-follows:
-
--  Understand where the installed toolchain resides. For cases where you
-   need to build the external toolchain, you would need to take separate
-   steps to build and install the toolchain.
-
--  Make sure you add the layer that contains the toolchain to your
-   ``bblayers.conf`` file through the
-   :term:`BBLAYERS` variable.
-
--  Set the
-   :term:`EXTERNAL_TOOLCHAIN`
-   variable in your ``local.conf`` file to the location in which you
-   installed the toolchain.
-
-A good example of an external toolchain used with the Yocto Project is
-Mentor Graphics Sourcery G++ Toolchain. You can see information on how
-to use that particular layer in the ``README`` file at
-http://github.com/MentorEmbedded/meta-sourcery/. You can find
-further information by reading about the
-:term:`TCMODE` variable in the Yocto
-Project Reference Manual's variable glossary.
-
-.. _using-pre-built:
-
-Example Using Pre-Built Binaries and QEMU
-=========================================
-
-If hardware, libraries and services are stable, you can get started by
-using a pre-built binary of the filesystem image, kernel, and toolchain
-and run it using the QEMU emulator. This scenario is useful for
-developing application software.
-
-|Using a Pre-Built Image|
-
-For this scenario, you need to do several things:
-
--  Install the appropriate stand-alone toolchain tarball.
-
--  Download the pre-built image that will boot with QEMU. You need to be
-   sure to get the QEMU image that matches your target machine's
-   architecture (e.g. x86, ARM, etc.).
-
--  Download the filesystem image for your target machine's architecture.
-
--  Set up the environment to emulate the hardware and then start the
-   QEMU emulator.
-
-Installing the Toolchain
-------------------------
-
-You can download a tarball installer, which includes the pre-built
-toolchain, the ``runqemu`` script, and support files from the
-appropriate directory under ` <&YOCTO_TOOLCHAIN_DL_URL;>`__. Toolchains
-are available for 32-bit and 64-bit x86 development systems from the
-``i686`` and ``x86_64`` directories, respectively. The toolchains the
-Yocto Project provides are based off the ``core-image-sato`` image and
-contain libraries appropriate for developing against that image. Each
-type of development system supports five or more target architectures.
-
-The names of the tarball installer scripts are such that a string
-representing the host system appears first in the filename and then is
-immediately followed by a string representing the target architecture.
-
-::
-
-        poky-glibc-host_system-image_type-arch-toolchain-release_version.sh
-
-        Where:
-            host_system is a string representing your development system:
-
-                       i686 or x86_64.
-
-            image_type is a string representing the image you wish to
-                   develop a Software Development Toolkit (SDK) for use against.
-                   The Yocto Project builds toolchain installers using the
-                   following BitBake command:
-
-                       bitbake core-image-sato -c populate_sdk
-
-            arch is a string representing the tuned target architecture:
-
-                       i586, x86_64, powerpc, mips, armv7a or armv5te
-
-            release_version is a string representing the release number of the
-                   Yocto Project:
-
-                       DISTRO, DISTRO+snapshot
-               
-
-For example, the following toolchain installer is for a 64-bit
-development host system and a i586-tuned target architecture based off
-the SDK for ``core-image-sato``:
-poky-glibc-x86_64-core-image-sato-i586-toolchain-DISTRO.sh
-
-Toolchains are self-contained and by default are installed into
-``/opt/poky``. However, when you run the toolchain installer, you can
-choose an installation directory.
-
-The following command shows how to run the installer given a toolchain
-tarball for a 64-bit x86 development host system and a 32-bit x86 target
-architecture. You must change the permissions on the toolchain installer
-script so that it is executable.
-
-The example assumes the toolchain installer is located in
-``~/Downloads/``.
-
-.. note::
-
-   If you do not have write permissions for the directory into which you
-   are installing the toolchain, the toolchain installer notifies you
-   and exits. Be sure you have write permissions in the directory and
-   run the installer again.
-
-$ ~/Downloads/poky-glibc-x86_64-core-image-sato-i586-toolchain-DISTRO.sh
-
-For more information on how to install tarballs, see the "`Using a
-Cross-Toolchain
-Tarball <&YOCTO_DOCS_ADT_URL;#using-an-existing-toolchain-tarball>`__"
-and "`Using BitBake and the Build
-Directory <&YOCTO_DOCS_ADT_URL;#using-the-toolchain-from-within-the-build-tree>`__"
-sections in the Yocto Project Application Developer's Guide.
-
-Downloading the Pre-Built Linux Kernel
---------------------------------------
-
-You can download the pre-built Linux kernel suitable for running in the
-QEMU emulator from ` <&YOCTO_QEMU_DL_URL;>`__. Be sure to use the kernel
-that matches the architecture you want to simulate. Download areas exist
-for the five supported machine architectures: ``qemuarm``, ``qemumips``,
-``qemuppc``, ``qemux86``, and ``qemux86-64``.
-
-Most kernel files have one of the following forms: \*zImage-qemuarch.bin
-vmlinux-qemuarch.bin Where: arch is a string representing the target
-architecture: x86, x86-64, ppc, mips, or arm.
-
-You can learn more about downloading a Yocto Project kernel in the
-"`Yocto Project Kernel <&YOCTO_DOCS_DEV_URL;#local-kernel-files>`__"
-bulleted item in the Yocto Project Development Manual.
-
-Downloading the Filesystem
---------------------------
-
-You can also download the filesystem image suitable for your target
-architecture from ` <&YOCTO_QEMU_DL_URL;>`__. Again, be sure to use the
-filesystem that matches the architecture you want to simulate.
-
-The filesystem image has two tarball forms: ``ext3`` and ``tar``. You
-must use the ``ext3`` form when booting an image using the QEMU
-emulator. The ``tar`` form can be flattened out in your host development
-system and used for build purposes with the Yocto Project.
-core-image-profile-qemuarch.ext3 core-image-profile-qemuarch.tar.bz2
-Where: profile is the filesystem image's profile: lsb, lsb-dev, lsb-sdk,
-lsb-qt3, minimal, minimal-dev, sato, sato-dev, or sato-sdk. For
-information on these types of image profiles, see the
-":ref:`ref-manual/ref-images:Images`" chapter in the Yocto
-Project Reference Manual. arch is a string representing the target
-architecture: x86, x86-64, ppc, mips, or arm.
-
-Setting Up the Environment and Starting the QEMU Emulator
----------------------------------------------------------
-
-Before you start the QEMU emulator, you need to set up the emulation
-environment. The following command form sets up the emulation
-environment. $ source
-YOCTO_ADTPATH_DIR/environment-setup-arch-poky-linux-if Where: arch is a
-string representing the target architecture: i586, x86_64, ppc603e,
-mips, or armv5te. if is a string representing an embedded application
-binary interface. Not all setup scripts include this string.
-
-Finally, this command form invokes the QEMU emulator $ runqemu qemuarch
-kernel-image filesystem-image Where: qemuarch is a string representing
-the target architecture: qemux86, qemux86-64, qemuppc, qemumips, or
-qemuarm. kernel-image is the architecture-specific kernel image.
-filesystem-image is the .ext3 filesystem image.
-
-Continuing with the example, the following two commands setup the
-emulation environment and launch QEMU. This example assumes the root
-filesystem (``.ext3`` file) and the pre-built kernel image file both
-reside in your home directory. The kernel and filesystem are for a
-32-bit target architecture. $ cd $HOME $ source
-YOCTO_ADTPATH_DIR/environment-setup-i586-poky-linux $ runqemu qemux86
-bzImage-qemux86.bin \\ core-image-sato-qemux86.ext3
-
-The environment in which QEMU launches varies depending on the
-filesystem image and on the target architecture. For example, if you
-source the environment for the ARM target architecture and then boot the
-minimal QEMU image, the emulator comes up in a new shell in command-line
-mode. However, if you boot the SDK image, QEMU comes up with a GUI.
-
-.. note::
-
-   Booting the PPC image results in QEMU launching in the same shell in
-   command-line mode.
-
-.. |Using a Pre-Built Image| image:: figures/using-a-pre-built-image.png

documentation/conf.py

@@ -15,8 +15,27 @@
 import os
 import sys
 import datetime
+try:
+    import yaml
+except ImportError:
+    sys.stderr.write("The Yocto Project Sphinx documentation requires PyYAML.\
+    \nPlease make sure to install pyyaml python package.\n")
+    sys.exit(1)
 
-current_version = "dev"
+# current_version = "dev"
+# bitbake_version = "" # Leave empty for development branch
+# Obtain versions from poky.yaml instead
+with open("poky.yaml") as data:
+    buff = data.read()
+    subst_vars = yaml.safe_load(buff)
+    if "DOCCONF_VERSION" not in subst_vars:
+        sys.stderr.write("Please set DOCCONF_VERSION in poky.yaml")
+        sys.exit(1)
+    current_version = subst_vars["DOCCONF_VERSION"]
+    if "BITBAKE_SERIES" not in subst_vars:
+        sys.stderr.write("Please set BITBAKE_SERIES in poky.yaml")
+        sys.exit(1)
+    bitbake_version = subst_vars["BITBAKE_SERIES"]
 
 # String used in sidebar
 version = 'Version: ' + current_version
@@ -53,8 +72,7 @@ templates_path = ['_templates']
 # List of patterns, relative to source directory, that match files and
 # directories to ignore when looking for source files.
 # This pattern also affects html_static_path and html_extra_path.
-exclude_patterns = ['_build', 'Thumbs.db', '.DS_Store', 'boilerplate.rst',
-                    'adt-manual/*.rst']
+exclude_patterns = ['_build', 'Thumbs.db', '.DS_Store', 'boilerplate.rst']
 
 # master document name. The default changed from contents to index. so better
 # set it ourselves.
@@ -83,7 +101,7 @@ extlinks = {
 
 # Intersphinx config to use cross reference with Bitbake user manual
 intersphinx_mapping = {
-    'bitbake': ('https://docs.yoctoproject.org/bitbake/', None)
+    'bitbake': ('https://docs.yoctoproject.org/bitbake/1.48', None)
 }
 
 # -- Options for HTML output -------------------------------------------------

documentation/poky.yaml

@@ -1,63 +1,17 @@
-DISTRO : "3.1"
-DISTRO_COMPRESSED : "31"
-DISTRO_NAME_NO_CAP : "dunfell"
-DISTRO_NAME : "Dunfell"
-DISTRO_NAME_NO_CAP_MINUS_ONE : "zeus"
-DISTRO_NAME_MINUS_ONE : "Zeus"
-YOCTO_DOC_VERSION : "3.1"
-YOCTO_DOC_VERSION_MINUS_ONE : "3.0.2"
-DISTRO_REL_TAG : "yocto-3.1"
-METAINTELVERSION : "12.0"
-REL_MONTH_YEAR : "April 2020"
-META_INTEL_REL_TAG : "&METAINTELVERSION;-&DISTRO_NAME_NO_CAP;-&YOCTO_DOC_VERSION;"
-POKYVERSION : "23.0.0"
-POKYVERSION_COMPRESSED : "2300"
+DISTRO : "3.2.4"
+DISTRO_NAME_NO_CAP : "gatesgarth"
+DISTRO_NAME : "Gatesgarth"
+DISTRO_NAME_NO_CAP_MINUS_ONE : "dunfell"
+YOCTO_DOC_VERSION : "3.2.4"
+YOCTO_DOC_VERSION_MINUS_ONE : "3.1.7"
+DISTRO_REL_TAG : "yocto-3.2.4"
+DOCCONF_VERSION : "3.2.4"
+BITBAKE_SERIES : "1.48"
+POKYVERSION : "24.0.4"
 YOCTO_POKY : "poky-&DISTRO_NAME_NO_CAP;-&POKYVERSION;"
-COPYRIGHT_YEAR : "2010-2020"
-ORGNAME : "The Yocto Project"
-ORGEMAIL : "docs@lists.yoctoproject.org"
 YOCTO_DL_URL : "https://downloads.yoctoproject.org"
-YOCTO_HOME_URL : "https://www.yoctoproject.org"
-YOCTO_LISTS_URL : "https://lists.yoctoproject.org"
-YOCTO_BUGZILLA_URL : "https://bugzilla.yoctoproject.org"
-YOCTO_WIKI_URL : "https://wiki.yoctoproject.org"
 YOCTO_AB_URL : "https://autobuilder.yoctoproject.org"
-YOCTO_GIT_URL : "https://git.yoctoproject.org"
-YOCTO_ADTREPO_URL : "http://adtrepo.yoctoproject.org"
-OE_HOME_URL : "https://www.openembedded.org"
-OE_LISTS_URL : "https://lists.openembedded.org"
-OE_DOCS_URL : "https://docs.openembedded.org"
-OH_HOME_URL : "http://o-hand.com"
-BITBAKE_HOME_URL : "http://developer.berlios.de/projects/bitbake/"
-YOCTO_DOCS_URL : "&YOCTO_HOME_URL;/docs"
-YOCTO_SOURCES_URL : "&YOCTO_HOME_URL;/sources/"
-YOCTO_AB_PORT_URL : "https://autobuilder.yocto.io/"
-YOCTO_AB_NIGHTLY_URL : "&YOCTO_AB_PORT_URL;/pub/nightly/"
-YOCTO_POKY_URL : "&YOCTO_DL_URL;/releases/poky/"
 YOCTO_RELEASE_DL_URL : "&YOCTO_DL_URL;/releases/yocto/yocto-&DISTRO;"
-YOCTO_TOOLCHAIN_DL_URL : "&YOCTO_RELEASE_DL_URL;/toolchain/"
-YOCTO_ADTINSTALLER_DL_URL : "&YOCTO_RELEASE_DL_URL;/adt-installer"
-YOCTO_POKY_DL_URL : "&YOCTO_RELEASE_DL_URL;/&YOCTO_POKY;.tar.bz2"
-YOCTO_MACHINES_DL_URL : "&YOCTO_RELEASE_DL_URL;/machines"
-YOCTO_QEMU_DL_URL : "&YOCTO_MACHINES_DL_URL;/qemu"
-YOCTO_PYTHON-i686_DL_URL : "&YOCTO_DL_URL;/releases/miscsupport/python-nativesdk-standalone-i686.tar.bz2"
-YOCTO_PYTHON-x86_64_DL_URL : "&YOCTO_DL_URL;/releases/miscsupport/python-nativesdk-standalone-x86_64.tar.bz2"
-YOCTO_DOCS_QS_URL : "&YOCTO_DOCS_URL;/&YOCTO_DOC_VERSION;/yocto-project-qs/yocto-project-qs.html"
-YOCTO_DOCS_ADT_URL : "&YOCTO_DOCS_URL;/&YOCTO_DOC_VERSION;/adt-manual/adt-manual.html"
-YOCTO_DOCS_REF_URL : "&YOCTO_DOCS_URL;/&YOCTO_DOC_VERSION;/ref-manual/ref-manual.html"
-YOCTO_DOCS_BSP_URL : "&YOCTO_DOCS_URL;/&YOCTO_DOC_VERSION;/bsp-guide/bsp-guide.html"
-YOCTO_DOCS_DEV_URL : "&YOCTO_DOCS_URL;/&YOCTO_DOC_VERSION;/dev-manual/dev-manual.html"
-YOCTO_DOCS_KERNEL_DEV_URL : "&YOCTO_DOCS_URL;/&YOCTO_DOC_VERSION;/kernel-dev/kernel-dev.html"
-YOCTO_DOCS_PROF_URL : "&YOCTO_DOCS_URL;/&YOCTO_DOC_VERSION;/profile-manual/profile-manual.html"
-YOCTO_DOCS_MM_URL : "&YOCTO_DOCS_URL;/&YOCTO_DOC_VERSION;/mega-manual/mega-manual.html"
-YOCTO_DOCS_BB_URL : "&YOCTO_DOCS_URL;/&YOCTO_DOC_VERSION;/bitbake-user-manual/bitbake-user-manual.html"
-YOCTO_DOCS_TOAST_URL : "&YOCTO_DOCS_URL;/&YOCTO_DOC_VERSION;/toaster-manual/toaster-manual.html"
-YOCTO_DOCS_SDK_URL : "&YOCTO_DOCS_URL;/&YOCTO_DOC_VERSION;/sdk-manual/sdk-manual.html"
-YOCTO_DOCS_OM_URL : "&YOCTO_DOCS_URL;/&YOCTO_DOC_VERSION;/overview-manual/overview-manual.html"
-YOCTO_DOCS_BRIEF_URL : "&YOCTO_DOCS_URL;/&YOCTO_DOC_VERSION;/brief-yoctoprojectqs/brief-yoctoprojectqs.html"
-YOCTO_ADTPATH_DIR : "/opt/poky/&DISTRO;"
-YOCTO_POKY_TARBALL : "&YOCTO_POKY;.tar.bz2"
-OE_INIT_PATH : "&YOCTO_POKY;/oe-init-build-env"
 UBUNTU_HOST_PACKAGES_ESSENTIAL : "gawk wget git-core diffstat unzip texinfo gcc-multilib \
      build-essential chrpath socat cpio python3 python3-pip python3-pexpect \
      xz-utils debianutils iputils-ping python3-git python3-jinja2 libegl1-mesa libsdl1.2-dev \

documentation/releases.rst

@@ -4,6 +4,16 @@
  Current Release Manuals
 =========================
 
+*******************************
+3.2 'gatesgarth' Release Series
+*******************************
+
+- :yocto_docs:`3.2 Documentation </3.2>`
+- :yocto_docs:`3.2.1 Documentation </3.2.1>`
+- :yocto_docs:`3.2.2 Documentation </3.2.2>`
+- :yocto_docs:`3.2.3 Documentation </3.2.3>`
+- :yocto_docs:`3.2.4 Documentation </3.2.4>`
+
 ****************************
 3.1 'dunfell' Release Series
 ****************************
@@ -11,6 +21,11 @@
 - :yocto_docs:`3.1 Documentation </3.1>`
 - :yocto_docs:`3.1.1 Documentation </3.1.1>`
 - :yocto_docs:`3.1.2 Documentation </3.1.2>`
+- :yocto_docs:`3.1.3 Documentation </3.1.3>`
+- :yocto_docs:`3.1.4 Documentation </3.1.4>`
+- :yocto_docs:`3.1.5 Documentation </3.1.5>`
+- :yocto_docs:`3.1.6 Documentation </3.1.6>`
+- :yocto_docs:`3.1.7 Documentation </3.1.7>`
 
 ==========================
  Previous Release Manuals
@@ -24,6 +39,7 @@
 - :yocto_docs:`3.0.1 Documentation </3.0.1>`
 - :yocto_docs:`3.0.2 Documentation </3.0.2>`
 - :yocto_docs:`3.0.3 Documentation </3.0.3>`
+- :yocto_docs:`3.0.4 Documentation </3.0.4>`
 
 ****************************
 2.7 'warrior' Release Series

documentation/sphinx-static/switchers.js

@@ -2,7 +2,8 @@
   'use strict';
 
   var all_versions = {
-    'dev': 'dev (3.2)',
+    'dev': 'dev (3.3)',
+    '3.2': '3.2',
     '3.1.3': '3.1.3',
     '3.0.4': '3.0.4',
     '2.7.4': '2.7.4',

meta-poky/conf/distro/poky.conf

@@ -1,6 +1,6 @@
 DISTRO = "poky"
 DISTRO_NAME = "Poky (Yocto Project Reference Distro)"
-DISTRO_VERSION = "3.2"
+DISTRO_VERSION = "3.2.4"
 DISTRO_CODENAME = "gatesgarth"
 SDK_VENDOR = "-pokysdk"
 SDK_VERSION = "${@d.getVar('DISTRO_VERSION').replace('snapshot-${DATE}', 'snapshot')}"

meta-selftest/lib/pseudo_pyc_test1.py

@@ -0,0 +1 @@
+STRING = "pseudo_pyc_test1"

meta-selftest/lib/pseudo_pyc_test2.py

@@ -0,0 +1 @@
+STRING = "pseudo_pyc_test2"

meta-selftest/recipes-test/pseudo-pyc-test/pseudo-pyc-test.bb

@@ -0,0 +1,15 @@
+SUMMARY = "pseudo env test"
+LICENSE = "MIT"
+LIC_FILES_CHKSUM = "file://${COREBASE}/meta/COPYING.MIT;md5=3da9cfbcb788c80a0384361b4de20420"
+
+INHIBIT_DEFAULT_DEPS = "1"
+
+python do_compile() {
+    import pseudo_pyc_test1
+    print(pseudo_pyc_test1.STRING)
+}
+
+python do_install() {
+    import pseudo_pyc_test2
+    print(pseudo_pyc_test2.STRING)
+}

meta-yocto-bsp/recipes-kernel/linux/linux-yocto_5.4.bbappend

@@ -7,8 +7,8 @@ KMACHINE_genericx86 ?= "common-pc"
 KMACHINE_genericx86-64 ?= "common-pc-64"
 KMACHINE_beaglebone-yocto ?= "beaglebone"
 
-SRCREV_machine_genericx86 ?= "cfcdd63145c0d741e57ee3e3e58f794229c6c09c"
-SRCREV_machine_genericx86-64 ?= "cfcdd63145c0d741e57ee3e3e58f794229c6c09c"
+SRCREV_machine_genericx86 ?= "31db2b47ac7d8508080fbb7344399b501216de66"
+SRCREV_machine_genericx86-64 ?= "31db2b47ac7d8508080fbb7344399b501216de66"
 SRCREV_machine_edgerouter ?= "706efec4c1e270ec5dda92275898cd465dfdc7dd"
 SRCREV_machine_beaglebone-yocto ?= "706efec4c1e270ec5dda92275898cd465dfdc7dd"
 
@@ -17,7 +17,7 @@ COMPATIBLE_MACHINE_genericx86-64 = "genericx86-64"
 COMPATIBLE_MACHINE_edgerouter = "edgerouter"
 COMPATIBLE_MACHINE_beaglebone-yocto = "beaglebone-yocto"
 
-LINUX_VERSION_genericx86 = "5.4.69"
-LINUX_VERSION_genericx86-64 = "5.4.69"
+LINUX_VERSION_genericx86 = "5.4.94"
+LINUX_VERSION_genericx86-64 = "5.4.94"
 LINUX_VERSION_edgerouter = "5.4.58"
 LINUX_VERSION_beaglebone-yocto = "5.4.58"

meta/classes/archiver.bbclass

@@ -590,6 +590,7 @@ addtask do_dumpdata
 addtask do_ar_recipe
 addtask do_deploy_archives
 do_build[recrdeptask] += "do_deploy_archives"
+do_rootfs[recrdeptask] += "do_deploy_archives"
 do_populate_sdk[recrdeptask] += "do_deploy_archives"
 
 python () {

meta/classes/base.bbclass

@@ -231,6 +231,7 @@ python base_eventhandler() {
     if isinstance(e, bb.event.ConfigParsed):
         if not d.getVar("NATIVELSBSTRING", False):
             d.setVar("NATIVELSBSTRING", lsb_distro_identifier(d))
+        d.setVar("ORIGNATIVELSBSTRING", d.getVar("NATIVELSBSTRING", False))
         d.setVar('BB_VERSION', bb.__version__)
 
     # There might be no bb.event.ConfigParsed event if bitbake server is

meta/classes/buildhistory.bbclass

@@ -674,13 +674,16 @@ IMAGE_POSTPROCESS_COMMAND[vardepsexclude] += "buildhistory_get_imageinfo"
 POPULATE_SDK_POST_TARGET_COMMAND_append = " buildhistory_list_installed_sdk_target;"
 POPULATE_SDK_POST_TARGET_COMMAND_append = " buildhistory_get_sdk_installed_target;"
 POPULATE_SDK_POST_TARGET_COMMAND[vardepvalueexclude] .= "| buildhistory_list_installed_sdk_target;| buildhistory_get_sdk_installed_target;"
+POPULATE_SDK_POST_TARGET_COMMAND[vardepsexclude] += "buildhistory_list_installed_sdk_target buildhistory_get_sdk_installed_target"
 
 POPULATE_SDK_POST_HOST_COMMAND_append = " buildhistory_list_installed_sdk_host;"
 POPULATE_SDK_POST_HOST_COMMAND_append = " buildhistory_get_sdk_installed_host;"
 POPULATE_SDK_POST_HOST_COMMAND[vardepvalueexclude] .= "| buildhistory_list_installed_sdk_host;| buildhistory_get_sdk_installed_host;"
+POPULATE_SDK_POST_HOST_COMMAND[vardepsexclude] += "buildhistory_list_installed_sdk_host buildhistory_get_sdk_installed_host"
 
 SDK_POSTPROCESS_COMMAND_append = " buildhistory_get_sdkinfo ; buildhistory_get_extra_sdkinfo; "
 SDK_POSTPROCESS_COMMAND[vardepvalueexclude] .= "| buildhistory_get_sdkinfo ; buildhistory_get_extra_sdkinfo; "
+SDK_POSTPROCESS_COMMAND[vardepsexclude] += "buildhistory_get_sdkinfo buildhistory_get_extra_sdkinfo"
 
 python buildhistory_write_sigs() {
     if not "task" in (d.getVar('BUILDHISTORY_FEATURES') or "").split():
@@ -855,7 +858,7 @@ END
 }
 
 python buildhistory_eventhandler() {
-    if e.data.getVar('BUILDHISTORY_FEATURES').strip():
+    if (e.data.getVar('BUILDHISTORY_FEATURES') or "").strip():
         reset = e.data.getVar("BUILDHISTORY_RESET")
         olddir = e.data.getVar("BUILDHISTORY_OLD_DIR")
         if isinstance(e, bb.event.BuildStarted):

meta/classes/cve-check.bbclass

@@ -53,6 +53,16 @@ CVE_CHECK_PN_WHITELIST ?= ""
 #
 CVE_CHECK_WHITELIST ?= ""
 
+# Layers to be excluded
+CVE_CHECK_LAYER_EXCLUDELIST ??= ""
+
+# Layers to be included 
+CVE_CHECK_LAYER_INCLUDELIST ??= ""
+
+
+# set to "alphabetical" for version using single alphabetical character as increament release
+CVE_VERSION_SUFFIX ??= ""
+
 python cve_save_summary_handler () {
     import shutil
     import datetime
@@ -206,7 +216,11 @@ def check_cves(d, patched_cves):
     """
     Connect to the NVD database and find unpatched cves.
     """
-    from distutils.version import LooseVersion
+    from oe.cve_check import Version
+
+    pn = d.getVar("PN")
+    real_pv = d.getVar("PV")
+    suffix = d.getVar("CVE_VERSION_SUFFIX")
 
     cves_unpatched = []
     # CVE_PRODUCT can contain more than one product (eg. curl/libcurl)
@@ -217,7 +231,7 @@ def check_cves(d, patched_cves):
     pv = d.getVar("CVE_VERSION").split("+git")[0]
 
     # If the recipe has been whitlisted we return empty lists
-    if d.getVar("PN") in d.getVar("CVE_CHECK_PN_WHITELIST").split():
+    if pn in d.getVar("CVE_CHECK_PN_WHITELIST").split():
         bb.note("Recipe has been whitelisted, skipping check")
         return ([], [], [])
 
@@ -260,8 +274,8 @@ def check_cves(d, patched_cves):
                 else:
                     if operator_start:
                         try:
-                            vulnerable_start =  (operator_start == '>=' and LooseVersion(pv) >= LooseVersion(version_start))
-                            vulnerable_start |= (operator_start == '>' and LooseVersion(pv) > LooseVersion(version_start))
+                            vulnerable_start =  (operator_start == '>=' and Version(pv,suffix) >= Version(version_start,suffix))
+                            vulnerable_start |= (operator_start == '>' and Version(pv,suffix) > Version(version_start,suffix))
                         except:
                             bb.warn("%s: Failed to compare %s %s %s for %s" %
                                     (product, pv, operator_start, version_start, cve))
@@ -271,8 +285,8 @@ def check_cves(d, patched_cves):
 
                     if operator_end:
                         try:
-                            vulnerable_end  = (operator_end == '<=' and LooseVersion(pv) <= LooseVersion(version_end))
-                            vulnerable_end |= (operator_end == '<' and LooseVersion(pv) < LooseVersion(version_end))
+                            vulnerable_end  = (operator_end == '<=' and Version(pv,suffix) <= Version(version_end,suffix) )
+                            vulnerable_end |= (operator_end == '<' and Version(pv,suffix) < Version(version_end,suffix) )
                         except:
                             bb.warn("%s: Failed to compare %s %s %s for %s" %
                                     (product, pv, operator_end, version_end, cve))
@@ -286,12 +300,12 @@ def check_cves(d, patched_cves):
                         vulnerable = vulnerable_start or vulnerable_end
 
                 if vulnerable:
-                    bb.note("%s-%s is vulnerable to %s" % (product, pv, cve))
+                    bb.note("%s-%s is vulnerable to %s" % (pn, real_pv, cve))
                     cves_unpatched.append(cve)
                     break
 
             if not vulnerable:
-                bb.note("%s-%s is not vulnerable to %s" % (product, pv, cve))
+                bb.note("%s-%s is not vulnerable to %s" % (pn, real_pv, cve))
                 # TODO: not patched but not vulnerable
                 patched_cves.add(cve)
 
@@ -327,7 +341,20 @@ def cve_write_data(d, patched, unpatched, whitelisted, cve_data):
     CVE manifest if enabled.
     """
 
+
     cve_file = d.getVar("CVE_CHECK_LOG")
+    fdir_name  = d.getVar("FILE_DIRNAME")
+    layer = fdir_name.split("/")[-3]
+
+    include_layers = d.getVar("CVE_CHECK_LAYER_INCLUDELIST").split()
+    exclude_layers = d.getVar("CVE_CHECK_LAYER_EXCLUDELIST").split()
+
+    if exclude_layers and layer in exclude_layers:
+        return
+
+    if include_layers and layer not in include_layers:
+        return
+
     nvd_link = "https://web.nvd.nist.gov/view/vuln/detail?vulnId="
     write_string = ""
     unpatched_cves = []
@@ -337,6 +364,7 @@ def cve_write_data(d, patched, unpatched, whitelisted, cve_data):
         is_patched = cve in patched
         if is_patched and (d.getVar("CVE_CHECK_REPORT_PATCHED") != "1"):
             continue
+        write_string += "LAYER: %s\n" % layer
         write_string += "PACKAGE NAME: %s\n" % d.getVar("PN")
         write_string += "PACKAGE VERSION: %s%s\n" % (d.getVar("EXTENDPE"), d.getVar("PV"))
         write_string += "CVE: %s\n" % cve

meta/classes/devshell.bbclass

@@ -128,6 +128,7 @@ def devpyshell(d):
                     more = i.runsource(source, "<pyshell>")
                     if not more:
                         buf = []
+                    sys.stderr.flush()
                     prompt(more)
             except KeyboardInterrupt:
                 i.write("\nKeyboardInterrupt\n")

meta/classes/distutils-common-base.bbclass

@@ -11,7 +11,7 @@ export LDCXXSHARED  = "${CXX} -shared"
 export CCSHARED  = "-fPIC -DPIC"
 # LINKFORSHARED are the flags passed to the $(CC) command that links
 # the python executable
-export LINKFORSHARED = "{SECURITY_CFLAGS} -Xlinker -export-dynamic"
+export LINKFORSHARED = "${SECURITY_CFLAGS} -Xlinker -export-dynamic"
 
 FILES_${PN} += "${libdir}/* ${libdir}/${PYTHON_DIR}/*"
 

meta/classes/distutils3-base.bbclass

@@ -1,5 +1,5 @@
 DEPENDS  += "${@["${PYTHON_PN}-native ${PYTHON_PN}", ""][(d.getVar('PACKAGES') == '')]}"
 RDEPENDS_${PN} += "${@['', '${PYTHON_PN}-core']['${CLASSOVERRIDE}' == 'class-target']}"
 
-inherit distutils-common-base python3native
+inherit distutils-common-base python3native python3targetconfig
 

meta/classes/externalsrc.bbclass

@@ -68,6 +68,7 @@ python () {
             url_data = fetch.ud[url]
             parm = url_data.parm
             if (url_data.type == 'file' or
+                    url_data.type == 'npmsw' or
                     'type' in parm and parm['type'] == 'kmeta'):
                 local_srcuri.append(url)
 
@@ -190,6 +191,7 @@ def srctree_hash_files(d, srcdir=None):
     import shutil
     import subprocess
     import tempfile
+    import hashlib
 
     s_dir = srcdir or d.getVar('EXTERNALSRC')
     git_dir = None
@@ -197,6 +199,10 @@ def srctree_hash_files(d, srcdir=None):
     try:
         git_dir = os.path.join(s_dir,
             subprocess.check_output(['git', '-C', s_dir, 'rev-parse', '--git-dir'], stderr=subprocess.DEVNULL).decode("utf-8").rstrip())
+        top_git_dir = os.path.join(s_dir, subprocess.check_output(['git', '-C', d.getVar("TOPDIR"), 'rev-parse', '--git-dir'],
+            stderr=subprocess.DEVNULL).decode("utf-8").rstrip())
+        if git_dir == top_git_dir:
+            git_dir = None
     except subprocess.CalledProcessError:
         pass
 
@@ -210,7 +216,17 @@ def srctree_hash_files(d, srcdir=None):
             env = os.environ.copy()
             env['GIT_INDEX_FILE'] = tmp_index.name
             subprocess.check_output(['git', 'add', '-A', '.'], cwd=s_dir, env=env)
-            sha1 = subprocess.check_output(['git', 'write-tree'], cwd=s_dir, env=env).decode("utf-8")
+            git_sha1 = subprocess.check_output(['git', 'write-tree'], cwd=s_dir, env=env).decode("utf-8")
+            submodule_helper = subprocess.check_output(['git', 'submodule--helper', 'list'], cwd=s_dir, env=env).decode("utf-8")
+            for line in submodule_helper.splitlines():
+                module_dir = os.path.join(s_dir, line.rsplit(maxsplit=1)[1])
+                if os.path.isdir(module_dir):
+                    proc = subprocess.Popen(['git', 'add', '-A', '.'], cwd=module_dir, env=env, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
+                    proc.communicate()
+                    proc = subprocess.Popen(['git', 'write-tree'], cwd=module_dir, env=env, stdout=subprocess.PIPE, stderr=subprocess.DEVNULL)
+                    stdout, _ = proc.communicate()
+                    git_sha1 += stdout.decode("utf-8")
+            sha1 = hashlib.sha1(git_sha1.encode("utf-8")).hexdigest()
         with open(oe_hash_file, 'w') as fobj:
             fobj.write(sha1)
         ret = oe_hash_file + ':True'

meta/classes/go.bbclass

@@ -40,6 +40,7 @@ GO_RPATH_LINK_class-native = "${@'-Wl,-rpath-link=${STAGING_LIBDIR_NATIVE}/go/pk
 GO_EXTLDFLAGS ?= "${HOST_CC_ARCH}${TOOLCHAIN_OPTIONS} ${GO_RPATH_LINK} ${LDFLAGS}"
 GO_LINKMODE ?= ""
 GO_LINKMODE_class-nativesdk = "--linkmode=external"
+GO_LINKMODE_class-native = "--linkmode=external"
 GO_LDFLAGS ?= '-ldflags="${GO_RPATH} ${GO_LINKMODE} -extldflags '${GO_EXTLDFLAGS}'"'
 export GOBUILDFLAGS ?= "-v ${GO_LDFLAGS} -trimpath"
 export GOPATH_OMIT_IN_ACTIONID ?= "1"

meta/classes/image-live.bbclass

@@ -30,7 +30,7 @@ do_bootimg[depends] += "dosfstools-native:do_populate_sysroot \
                         virtual/kernel:do_deploy \
                         ${MLPREFIX}syslinux:do_populate_sysroot \
                         syslinux-native:do_populate_sysroot \
-                        ${PN}:do_image_${@d.getVar('LIVE_ROOTFS_TYPE').replace('-', '_')} \
+                        ${@'%s:do_image_%s' % (d.getVar('PN'), d.getVar('LIVE_ROOTFS_TYPE').replace('-', '_')) if d.getVar('ROOTFS') else ''} \
                         "
 
 

meta/classes/image.bbclass

@@ -112,7 +112,7 @@ def rootfs_command_variables(d):
             'IMAGE_PREPROCESS_COMMAND','RPM_PREPROCESS_COMMANDS','RPM_POSTPROCESS_COMMANDS','DEB_PREPROCESS_COMMANDS','DEB_POSTPROCESS_COMMANDS']
 
 python () {
-    variables = rootfs_command_variables(d) + sdk_command_variables(d)
+    variables = rootfs_command_variables(d)
     for var in variables:
         if d.getVar(var, False):
             d.setVarFlag(var, 'func', '1')
@@ -180,6 +180,8 @@ IMAGE_LOCALES_ARCHIVE ?= '1'
 # aren't yet available.
 PSEUDO_PASSWD = "${IMAGE_ROOTFS}:${STAGING_DIR_NATIVE}"
 
+PSEUDO_IGNORE_PATHS .= ",${WORKDIR}/intercept_scripts,${WORKDIR}/oe-rootfs-repo,${WORKDIR}/sstate-build-image_complete"
+
 PACKAGE_EXCLUDE ??= ""
 PACKAGE_EXCLUDE[type] = "list"
 

meta/classes/image_types.bbclass

@@ -108,19 +108,9 @@ IMAGE_CMD_squashfs-xz = "mksquashfs ${IMAGE_ROOTFS} ${IMGDEPLOYDIR}/${IMAGE_NAME
 IMAGE_CMD_squashfs-lzo = "mksquashfs ${IMAGE_ROOTFS} ${IMGDEPLOYDIR}/${IMAGE_NAME}${IMAGE_NAME_SUFFIX}.squashfs-lzo ${EXTRA_IMAGECMD} -noappend -comp lzo"
 IMAGE_CMD_squashfs-lz4 = "mksquashfs ${IMAGE_ROOTFS} ${IMGDEPLOYDIR}/${IMAGE_NAME}${IMAGE_NAME_SUFFIX}.squashfs-lz4 ${EXTRA_IMAGECMD} -noappend -comp lz4"
 
-# By default, tar from the host is used, which can be quite old. If
-# you need special parameters (like --xattrs) which are only supported
-# by GNU tar upstream >= 1.27, then override that default:
-# IMAGE_CMD_TAR = "tar --xattrs --xattrs-include=*"
-# do_image_tar[depends] += "tar-replacement-native:do_populate_sysroot"
-# EXTRANATIVEPATH += "tar-native"
-#
-# The GNU documentation does not specify whether --xattrs-include is necessary.
-# In practice, it turned out to be not needed when creating archives and
-# required when extracting, but it seems prudent to use it in both cases.
 IMAGE_CMD_TAR ?= "tar"
 # ignore return code 1 "file changed as we read it" as other tasks(e.g. do_image_wic) may be hardlinking rootfs
-IMAGE_CMD_tar = "${IMAGE_CMD_TAR} --numeric-owner -cf ${IMGDEPLOYDIR}/${IMAGE_NAME}${IMAGE_NAME_SUFFIX}.tar -C ${IMAGE_ROOTFS} . || [ $? -eq 1 ]"
+IMAGE_CMD_tar = "${IMAGE_CMD_TAR} --sort=name --format=posix --numeric-owner -cf ${IMGDEPLOYDIR}/${IMAGE_NAME}${IMAGE_NAME_SUFFIX}.tar -C ${IMAGE_ROOTFS} . || [ $? -eq 1 ]"
 
 do_image_cpio[cleandirs] += "${WORKDIR}/cpio_append"
 IMAGE_CMD_cpio () {

meta/classes/image_types_wic.bbclass

@@ -3,7 +3,7 @@
 WICVARS ?= "\
            BBLAYERS IMGDEPLOYDIR DEPLOY_DIR_IMAGE FAKEROOTCMD IMAGE_BASENAME IMAGE_EFI_BOOT_FILES IMAGE_BOOT_FILES \
            IMAGE_LINK_NAME IMAGE_ROOTFS INITRAMFS_FSTYPES INITRD INITRD_LIVE ISODIR RECIPE_SYSROOT_NATIVE \
-           ROOTFS_SIZE STAGING_DATADIR STAGING_DIR STAGING_LIBDIR TARGET_SYS \
+           ROOTFS_SIZE STAGING_DATADIR STAGING_DIR STAGING_LIBDIR TARGET_SYS HOSTTOOLS_DIR \
            KERNEL_IMAGETYPE MACHINE INITRAMFS_IMAGE INITRAMFS_IMAGE_BUNDLE INITRAMFS_LINK_NAME APPEND \
            ASSUME_PROVIDED PSEUDO_IGNORE_PATHS"
 
@@ -29,11 +29,17 @@ WIC_CREATE_EXTRA_ARGS ?= ""
 IMAGE_CMD_wic () {
 	out="${IMGDEPLOYDIR}/${IMAGE_NAME}"
 	build_wic="${WORKDIR}/build-wic"
+	tmp_wic="${WORKDIR}/tmp-wic"
 	wks="${WKS_FULL_PATH}"
+	if [ -e "$tmp_wic" ]; then
+		# Ensure we don't have any junk leftover from a previously interrupted
+		# do_image_wic execution
+		rm -rf "$tmp_wic"
+	fi
 	if [ -z "$wks" ]; then
 		bbfatal "No kickstart files from WKS_FILES were found: ${WKS_FILES}. Please set WKS_FILE or WKS_FILES appropriately."
 	fi
-	BUILDDIR="${TOPDIR}" PSEUDO_UNLOAD=1 wic create "$wks" --vars "${STAGING_DIR}/${MACHINE}/imgdata/" -e "${IMAGE_BASENAME}" -o "$build_wic/" ${WIC_CREATE_EXTRA_ARGS}
+	BUILDDIR="${TOPDIR}" PSEUDO_UNLOAD=1 wic create "$wks" --vars "${STAGING_DIR}/${MACHINE}/imgdata/" -e "${IMAGE_BASENAME}" -o "$build_wic/" -w "$tmp_wic" ${WIC_CREATE_EXTRA_ARGS}
 	mv "$build_wic/$(basename "${wks%.wks}")"*.direct "$out${IMAGE_NAME_SUFFIX}.wic"
 }
 IMAGE_CMD_wic[vardepsexclude] = "WKS_FULL_PATH WKS_FILES TOPDIR"

meta/classes/insane.bbclass

@@ -175,7 +175,7 @@ def package_qa_check_useless_rpaths(file, name, d, elf, messages):
             if rpath_eq(rpath, libdir) or rpath_eq(rpath, base_libdir):
                 # The dynamic linker searches both these places anyway.  There is no point in
                 # looking there again.
-                package_qa_add_message(messages, "useless-rpaths", "%s: %s contains probably-redundant RPATH %s" % (name, package_qa_clean_path(file, d), rpath))
+                package_qa_add_message(messages, "useless-rpaths", "%s: %s contains probably-redundant RPATH %s" % (name, package_qa_clean_path(file, d, name), rpath))
 
 QAPATHTEST[dev-so] = "package_qa_check_dev"
 def package_qa_check_dev(path, name, d, elf, messages):
@@ -184,8 +184,8 @@ def package_qa_check_dev(path, name, d, elf, messages):
     """
 
     if not name.endswith("-dev") and not name.endswith("-dbg") and not name.endswith("-ptest") and not name.startswith("nativesdk-") and path.endswith(".so") and os.path.islink(path):
-        package_qa_add_message(messages, "dev-so", "non -dev/-dbg/nativesdk- package contains symlink .so: %s path '%s'" % \
-                 (name, package_qa_clean_path(path,d)))
+        package_qa_add_message(messages, "dev-so", "non -dev/-dbg/nativesdk- package %s contains symlink .so '%s'" % \
+                 (name, package_qa_clean_path(path, d, name)))
 
 QAPATHTEST[dev-elf] = "package_qa_check_dev_elf"
 def package_qa_check_dev_elf(path, name, d, elf, messages):
@@ -195,8 +195,8 @@ def package_qa_check_dev_elf(path, name, d, elf, messages):
     install link-time .so files that are linker scripts.
     """
     if name.endswith("-dev") and path.endswith(".so") and not os.path.islink(path) and elf:
-        package_qa_add_message(messages, "dev-elf", "-dev package contains non-symlink .so: %s path '%s'" % \
-                 (name, package_qa_clean_path(path,d)))
+        package_qa_add_message(messages, "dev-elf", "-dev package %s contains non-symlink .so '%s'" % \
+                 (name, package_qa_clean_path(path, d, name)))
 
 QAPATHTEST[staticdev] = "package_qa_check_staticdev"
 def package_qa_check_staticdev(path, name, d, elf, messages):
@@ -209,7 +209,7 @@ def package_qa_check_staticdev(path, name, d, elf, messages):
 
     if not name.endswith("-pic") and not name.endswith("-staticdev") and not name.endswith("-ptest") and path.endswith(".a") and not path.endswith("_nonshared.a") and not '/usr/lib/debug-static/' in path and not '/.debug-static/' in path:
         package_qa_add_message(messages, "staticdev", "non -staticdev package contains static .a library: %s path '%s'" % \
-                 (name, package_qa_clean_path(path,d)))
+                 (name, package_qa_clean_path(path,d, name)))
 
 QAPATHTEST[mime] = "package_qa_check_mime"
 def package_qa_check_mime(path, name, d, elf, messages):

meta/classes/kernel-devicetree.bbclass

@@ -1,8 +1,11 @@
 # Support for device tree generation
-PACKAGES_append = " \
-    ${KERNEL_PACKAGE_NAME}-devicetree \
-    ${@[d.getVar('KERNEL_PACKAGE_NAME') + '-image-zimage-bundle', ''][d.getVar('KERNEL_DEVICETREE_BUNDLE') != '1']} \
-"
+python () {
+    if not bb.data.inherits_class('nopackages', d):
+        d.appendVar("PACKAGES", " ${KERNEL_PACKAGE_NAME}-devicetree")
+        if d.getVar('KERNEL_DEVICETREE_BUNDLE') == '1':
+            d.appendVar("PACKAGES", " ${KERNEL_PACKAGE_NAME}-image-zimage-bundle")
+}
+
 FILES_${KERNEL_PACKAGE_NAME}-devicetree = "/${KERNEL_IMAGEDEST}/*.dtb /${KERNEL_IMAGEDEST}/*.dtbo"
 FILES_${KERNEL_PACKAGE_NAME}-image-zimage-bundle = "/${KERNEL_IMAGEDEST}/zImage-*.dtb.bin"
 

meta/classes/kernel-module-split.bbclass

@@ -121,6 +121,10 @@ python split_kernel_module_packages () {
         files = "%s /etc/modules-load.d/%s.conf /etc/modprobe.d/%s.conf" % (files, basename, basename)
         d.setVar('FILES_%s' % pkg, files)
 
+        conffiles = d.getVar('CONFFILES_%s' % pkg)
+        conffiles = "%s /etc/modules-load.d/%s.conf /etc/modprobe.d/%s.conf" % (conffiles, basename, basename)
+        d.setVar('CONFFILES_%s' % pkg, conffiles)
+
         if "description" in vals:
             old_desc = d.getVar('DESCRIPTION_' + pkg) or ""
             d.setVar('DESCRIPTION_' + pkg, old_desc + "; " + vals["description"])

meta/classes/kernel.bbclass

@@ -90,6 +90,8 @@ python __anonymous () {
     imagedest = d.getVar('KERNEL_IMAGEDEST')
 
     for type in types.split():
+        if bb.data.inherits_class('nopackages', d):
+            continue
         typelower = type.lower()
         d.appendVar('PACKAGES', ' %s-image-%s' % (kname, typelower))
         d.setVar('FILES_' + kname + '-image-' + typelower, '/' + imagedest + '/' + type + '-${KERNEL_VERSION_NAME}' + ' /' + imagedest + '/' + type)
@@ -383,6 +385,10 @@ do_compile_kernelmodules() {
 		# other kernel modules and will look at this
 		# file to do symbol lookups
 		cp ${B}/Module.symvers ${STAGING_KERNEL_BUILDDIR}/
+		# 5.10+ kernels have module.lds that we need to copy for external module builds
+		if [ -e "${B}/scripts/module.lds" ]; then
+			install -Dm 0644 ${B}/scripts/module.lds ${STAGING_KERNEL_BUILDDIR}/scripts/module.lds
+		fi
 	else
 		bbnote "no modules to compile"
 	fi
@@ -586,7 +592,7 @@ addtask savedefconfig after do_configure
 
 inherit cml1
 
-KCONFIG_CONFIG_COMMAND_append = " HOSTLDFLAGS='${BUILD_LDFLAGS}'"
+KCONFIG_CONFIG_COMMAND_append = " LD='${KERNEL_LD}' HOSTLDFLAGS='${BUILD_LDFLAGS}'"
 
 EXPORT_FUNCTIONS do_compile do_install do_configure
 
@@ -739,7 +745,7 @@ kernel_do_deploy() {
 	fi
 
 	if [ ! -z "${INITRAMFS_IMAGE}" -a x"${INITRAMFS_IMAGE_BUNDLE}" = x1 ]; then
-		for imageType in ${KERNEL_IMAGETYPES} ; do
+		for imageType in ${KERNEL_IMAGETYPE_FOR_MAKE} ; do
 			if [ "$imageType" = "fitImage" ] ; then
 				continue
 			fi

meta/classes/license.bbclass

@@ -31,6 +31,7 @@ python do_populate_lic() {
             f.write("%s: %s\n" % (key, info[key]))
 }
 
+PSEUDO_IGNORE_PATHS .= ",${@','.join(((d.getVar('COMMON_LICENSE_DIR') or '') + ' ' + (d.getVar('LICENSE_PATH') or '')).split())}"
 # it would be better to copy them in do_install_append, but find_license_filesa is python
 python perform_packagecopy_prepend () {
     enabled = oe.data.typed_value('LICENSE_CREATE_PACKAGE', d)

meta/classes/license_image.bbclass

@@ -125,7 +125,6 @@ def write_license_files(d, license_manifest, pkg_dic, rootfs=True):
 
                 licenses = os.listdir(pkg_license_dir)
                 for lic in licenses:
-                    rootfs_license = os.path.join(rootfs_license_dir, lic)
                     pkg_license = os.path.join(pkg_license_dir, lic)
                     pkg_rootfs_license = os.path.join(pkg_rootfs_license_dir, lic)
 
@@ -144,6 +143,8 @@ def write_license_files(d, license_manifest, pkg_dic, rootfs=True):
                                 bad_licenses) == False:
                             continue
 
+                        # Make sure we use only canonical name for the license file
+                        rootfs_license = os.path.join(rootfs_license_dir, "generic_%s" % generic_lic)
                         if not os.path.exists(rootfs_license):
                             oe.path.copyhardlink(pkg_license, rootfs_license)
 
@@ -209,7 +210,8 @@ def license_deployed_manifest(d):
             os.unlink(lic_manifest_symlink_dir)
 
         # create the image dir symlink
-        os.symlink(lic_manifest_dir, lic_manifest_symlink_dir)
+        if lic_manifest_dir != lic_manifest_symlink_dir:
+            os.symlink(lic_manifest_dir, lic_manifest_symlink_dir)
 
 def get_deployed_dependencies(d):
     """

meta/classes/linuxloader.bbclass

@@ -1,6 +1,6 @@
 def get_musl_loader_arch(d):
     import re
-    ldso_arch = None
+    ldso_arch = "NotSupported"
 
     targetarch = d.getVar("TARGET_ARCH")
     if targetarch.startswith("microblaze"):
@@ -32,7 +32,7 @@ def get_musl_loader(d):
 def get_glibc_loader(d):
     import re
 
-    dynamic_loader = None
+    dynamic_loader = "NotSupported"
     targetarch = d.getVar("TARGET_ARCH")
     if targetarch in ["powerpc", "microblaze"]:
         dynamic_loader = "${base_libdir}/ld.so.1"
@@ -58,7 +58,7 @@ def get_linuxloader(d):
     overrides = d.getVar("OVERRIDES").split(":")
 
     if "libc-baremetal" in overrides:
-        return None
+        return "NotSupported"
 
     if "libc-musl" in overrides:
         dynamic_loader = get_musl_loader(d)

meta/classes/npm.bbclass

@@ -17,8 +17,10 @@
 #  NPM_INSTALL_DEV:
 #       Set to 1 to also install devDependencies.
 
+inherit python3native
+
 DEPENDS_prepend = "nodejs-native "
-RDEPENDS_${PN}_prepend = "nodejs "
+RDEPENDS_${PN}_append_class-target = " nodejs"
 
 NPM_INSTALL_DEV ?= "0"
 
@@ -130,11 +132,17 @@ python npm_do_configure() {
     cached_manifest.pop("dependencies", None)
     cached_manifest.pop("devDependencies", None)
 
-    with open(orig_shrinkwrap_file, "r") as f:
-        orig_shrinkwrap = json.load(f)
+    has_shrinkwrap_file = True
 
-    cached_shrinkwrap = copy.deepcopy(orig_shrinkwrap)
-    cached_shrinkwrap.pop("dependencies", None)
+    try:
+        with open(orig_shrinkwrap_file, "r") as f:
+            orig_shrinkwrap = json.load(f)
+    except IOError:
+        has_shrinkwrap_file = False
+
+    if has_shrinkwrap_file:
+       cached_shrinkwrap = copy.deepcopy(orig_shrinkwrap)
+       cached_shrinkwrap.pop("dependencies", None)
 
     # Manage the dependencies
     progress = OutOfProgressHandler(d, r"^(\d+)/(\d+)$")
@@ -165,8 +173,10 @@ python npm_do_configure() {
             progress.write("%d/%d" % (progress_done, progress_total))
 
     dev = bb.utils.to_boolean(d.getVar("NPM_INSTALL_DEV"), False)
-    foreach_dependencies(orig_shrinkwrap, _count_dependency, dev)
-    foreach_dependencies(orig_shrinkwrap, _cache_dependency, dev)
+
+    if has_shrinkwrap_file:
+        foreach_dependencies(orig_shrinkwrap, _count_dependency, dev)
+        foreach_dependencies(orig_shrinkwrap, _cache_dependency, dev)
 
     # Configure the main package
     with tempfile.TemporaryDirectory() as tmpdir:
@@ -181,16 +191,19 @@ python npm_do_configure() {
                 cached_manifest[depkey] = {}
             cached_manifest[depkey][name] = version
 
-    _update_manifest("dependencies")
+    if has_shrinkwrap_file:
+        _update_manifest("dependencies")
 
     if dev:
-        _update_manifest("devDependencies")
+        if has_shrinkwrap_file:
+            _update_manifest("devDependencies")
 
     with open(cached_manifest_file, "w") as f:
         json.dump(cached_manifest, f, indent=2)
 
-    with open(cached_shrinkwrap_file, "w") as f:
-        json.dump(cached_shrinkwrap, f, indent=2)
+    if has_shrinkwrap_file:
+        with open(cached_shrinkwrap_file, "w") as f:
+            json.dump(cached_shrinkwrap, f, indent=2)
 }
 
 python npm_do_compile() {
@@ -237,9 +250,7 @@ python npm_do_compile() {
         sysroot = d.getVar("RECIPE_SYSROOT_NATIVE")
         nodedir = os.path.join(sysroot, d.getVar("prefix_native").strip("/"))
         configs.append(("nodedir", nodedir))
-        bindir = os.path.join(sysroot, d.getVar("bindir_native").strip("/"))
-        pythondir = os.path.join(bindir, "python-native", "python")
-        configs.append(("python", pythondir))
+        configs.append(("python", d.getVar("PYTHON")))
 
         # Add node-pre-gyp configuration
         args.append(("target_arch", d.getVar("NPM_ARCH")))

meta/classes/package.bbclass

@@ -2340,7 +2340,7 @@ python do_package () {
     # cache.  This is useful if an item this class depends on changes in a
     # way that the output of this class changes.  rpmdeps is a good example
     # as any change to rpmdeps requires this to be rerun.
-    # PACKAGE_BBCLASS_VERSION = "2"
+    # PACKAGE_BBCLASS_VERSION = "4"
 
     # Init cachedpath
     global cpath
@@ -2446,6 +2446,7 @@ python do_packagedata () {
 
     bb.build.exec_func("packagedata_translate_pr_autoinc", d)
 }
+do_packagedata[cleandirs] += "${WORKDIR}/pkgdata-pdata-input"
 
 # Translate the EXTENDPRAUTO and AUTOINC to the final values
 packagedata_translate_pr_autoinc() {

meta/classes/package_rpm.bbclass

@@ -687,6 +687,7 @@ python do_package_rpm () {
     cmd = cmd + " --define '_binary_payload w6T.xzdio'"
     cmd = cmd + " --define '_source_payload w6T.xzdio'"
     cmd = cmd + " --define 'clamp_mtime_to_source_date_epoch 1'"
+    cmd = cmd + " --define 'use_source_date_epoch_as_buildtime 1'"
     cmd = cmd + " --define '_buildhost reproducible'"
     if perfiledeps:
         cmd = cmd + " --define '__find_requires " + outdepends + "'"

meta/classes/populate_sdk_base.bbclass

@@ -178,7 +178,7 @@ do_populate_sdk[sstate-inputdirs] = "${SDKDEPLOYDIR}"
 do_populate_sdk[sstate-outputdirs] = "${SDK_DEPLOY}"
 do_populate_sdk[stamp-extra-info] = "${MACHINE_ARCH}${SDKMACHINE}"
 
-PSEUDO_IGNORE_PATHS .= ",${SDKDEPLOYDIR}"
+PSEUDO_IGNORE_PATHS .= ",${SDKDEPLOYDIR},${WORKDIR}/oe-sdk-repo,${WORKDIR}/sstate-build-populate_sdk"
 
 fakeroot create_sdk_files() {
 	cp ${COREBASE}/scripts/relocate_sdk.py ${SDK_OUTPUT}/${SDKPATH}/
@@ -329,6 +329,13 @@ def sdk_variables(d):
 
 do_populate_sdk[vardeps] += "${@sdk_variables(d)}"
 
+python () {
+    variables = sdk_command_variables(d)
+    for var in variables:
+        if d.getVar(var, False):
+            d.setVarFlag(var, 'func', '1')
+}
+
 do_populate_sdk[file-checksums] += "${TOOLCHAIN_SHAR_REL_TMPL}:True \
                                     ${TOOLCHAIN_SHAR_EXT_TMPL}:True"
 

meta/classes/populate_sdk_ext.bbclass

@@ -24,6 +24,7 @@ SDK_INCLUDE_NATIVESDK ?= "0"
 SDK_INCLUDE_BUILDTOOLS ?= '1'
 
 SDK_RECRDEP_TASKS ?= ""
+SDK_CUSTOM_TEMPLATECONF ?= "0"
 
 SDK_LOCAL_CONF_WHITELIST ?= ""
 SDK_LOCAL_CONF_BLACKLIST ?= "CONF_VERSION \
@@ -199,6 +200,9 @@ python copy_buildsystem () {
     buildsystem = oe.copy_buildsystem.BuildSystem('extensible SDK', d)
     baseoutpath = d.getVar('SDK_OUTPUT') + '/' + d.getVar('SDKPATH')
 
+    #check if custome templateconf path is set
+    use_custom_templateconf = d.getVar('SDK_CUSTOM_TEMPLATECONF')
+
     # Determine if we're building a derivative extensible SDK (from devtool build-sdk)
     derivative = (d.getVar('SDK_DERIVATIVE') or '') == '1'
     if derivative:
@@ -247,7 +251,9 @@ python copy_buildsystem () {
 
     # Create a layer for new recipes / appends
     bbpath = d.getVar('BBPATH')
-    bb.process.run(['devtool', '--bbpath', bbpath, '--basepath', baseoutpath, 'create-workspace', '--create-only', os.path.join(baseoutpath, 'workspace')])
+    env = os.environ.copy()
+    env['PYTHONDONTWRITEBYTECODE'] = '1'
+    bb.process.run(['devtool', '--bbpath', bbpath, '--basepath', baseoutpath, 'create-workspace', '--create-only', os.path.join(baseoutpath, 'workspace')], env=env)
 
     # Create bblayers.conf
     bb.utils.mkdirhier(baseoutpath + '/conf')
@@ -360,6 +366,9 @@ python copy_buildsystem () {
             # Hide the config information from bitbake output (since it's fixed within the SDK)
             f.write('BUILDCFG_HEADER = ""\n\n')
 
+            # Write METADATA_REVISION
+            f.write('METADATA_REVISION = "%s"\n\n' % d.getVar('METADATA_REVISION'))
+
             f.write('# Provide a flag to indicate we are in the EXT_SDK Context\n')
             f.write('WITHIN_EXT_SDK = "1"\n\n')
 
@@ -390,7 +399,7 @@ python copy_buildsystem () {
         shutil.copyfile(builddir + '/cache/bb_unihashes.dat', baseoutpath + '/cache/bb_unihashes.dat')
 
     # Use templateconf.cfg file from builddir if exists
-    if os.path.exists(builddir + '/conf/templateconf.cfg'):
+    if os.path.exists(builddir + '/conf/templateconf.cfg') and use_custom_templateconf == '1':
         shutil.copyfile(builddir + '/conf/templateconf.cfg', baseoutpath + '/conf/templateconf.cfg')
     else:
         # Write a templateconf.cfg

meta/classes/python3native.bbclass

@@ -17,8 +17,6 @@ export STAGING_LIBDIR
 export PYTHON_LIBRARY="${STAGING_LIBDIR}/lib${PYTHON_DIR}${PYTHON_ABI}.so"
 export PYTHON_INCLUDE_DIR="${STAGING_INCDIR}/${PYTHON_DIR}${PYTHON_ABI}"
 
-export _PYTHON_SYSCONFIGDATA_NAME="_sysconfigdata"
-
 # suppress host user's site-packages dirs.
 export PYTHONNOUSERSITE = "1"
 

meta/classes/python3targetconfig.bbclass

@@ -0,0 +1,17 @@
+inherit python3native
+
+EXTRA_PYTHON_DEPENDS ?= ""
+EXTRA_PYTHON_DEPENDS_class-target = "python3"
+DEPENDS_append = " ${EXTRA_PYTHON_DEPENDS}"
+
+do_configure_prepend_class-target() {
+        export _PYTHON_SYSCONFIGDATA_NAME="_sysconfigdata"
+}
+
+do_compile_prepend_class-target() {
+        export _PYTHON_SYSCONFIGDATA_NAME="_sysconfigdata"
+}
+
+do_install_prepend_class-target() {
+        export _PYTHON_SYSCONFIGDATA_NAME="_sysconfigdata"
+}

meta/classes/qemuboot.bbclass

@@ -43,7 +43,7 @@
 #                          QB_NETWORK_DEVICE_prepend might be used, since Qemu enumerates the eth*
 #                          devices in reverse order to -device arguments.
 #
-# QB_TAP_OPT: netowrk option for 'tap' mode, e.g.,
+# QB_TAP_OPT: network option for 'tap' mode, e.g.,
 #             "-netdev tap,id=net0,ifname=@TAP@,script=no,downscript=no"
 #              Note, runqemu will replace "@TAP@" with the one which is used, such as tap0, tap1 ...
 #

meta/classes/report-error.bbclass

@@ -6,6 +6,8 @@
 #
 # Licensed under the MIT license, see COPYING.MIT for details
 
+inherit base
+
 ERR_REPORT_DIR ?= "${LOG_DIR}/error-report"
 
 def errorreport_getdata(e):
@@ -64,6 +66,8 @@ python errorreport_handler () {
             data['failures'] = []
             data['component'] = " ".join(e.getPkgs())
             data['branch_commit'] = str(base_detect_branch(e.data)) + ": " + str(base_detect_revision(e.data))
+            data['bitbake_version'] = e.data.getVar("BB_VERSION")
+            data['layer_version'] = get_layers_branch_rev(e.data)
             data['local_conf'] = get_conf_data(e, 'local.conf')
             data['auto_conf'] = get_conf_data(e, 'auto.conf')
             lock = bb.utils.lockfile(datafile + '.lock')

meta/classes/rootfs_deb.bbclass

@@ -7,7 +7,7 @@ ROOTFS_PKGMANAGE = "dpkg apt"
 do_rootfs[depends] += "dpkg-native:do_populate_sysroot apt-native:do_populate_sysroot"
 do_populate_sdk[depends] += "dpkg-native:do_populate_sysroot apt-native:do_populate_sysroot bzip2-native:do_populate_sysroot"
 do_rootfs[recrdeptask] += "do_package_write_deb do_package_qa"
-do_rootfs[vardeps] += "PACKAGE_FEED_URIS"
+do_rootfs[vardeps] += "PACKAGE_FEED_URIS PACKAGE_FEED_BASE_PATHS PACKAGE_FEED_ARCHS"
 
 do_rootfs[lockfiles] += "${DEPLOY_DIR_DEB}/deb.lock"
 do_populate_sdk[lockfiles] += "${DEPLOY_DIR_DEB}/deb.lock"
@@ -32,4 +32,8 @@ python () {
          d.setVar('DEB_SDK_ARCH', 'amd64')
     elif darch == "arm":
          d.setVar('DEB_SDK_ARCH', 'armel')
+    elif darch == "aarch64":
+         d.setVar('DEB_SDK_ARCH', 'arm64')
+    else:
+         bb.fatal("Unhandled SDK_ARCH %s" % darch)
 }

meta/classes/rootfs_ipk.bbclass

@@ -11,7 +11,7 @@ ROOTFS_PKGMANAGE = "opkg ${EXTRAOPKGCONFIG}"
 do_rootfs[depends] += "opkg-native:do_populate_sysroot opkg-utils-native:do_populate_sysroot"
 do_populate_sdk[depends] += "opkg-native:do_populate_sysroot opkg-utils-native:do_populate_sysroot"
 do_rootfs[recrdeptask] += "do_package_write_ipk do_package_qa"
-do_rootfs[vardeps] += "PACKAGE_FEED_URIS"
+do_rootfs[vardeps] += "PACKAGE_FEED_URIS PACKAGE_FEED_BASE_PATHS PACKAGE_FEED_ARCHS"
 
 do_rootfs[lockfiles] += "${WORKDIR}/ipk.lock"
 do_populate_sdk[lockfiles] += "${WORKDIR}/ipk.lock"

meta/classes/rootfs_rpm.bbclass

@@ -24,7 +24,7 @@ do_rootfs[depends] += "${RPMROOTFSDEPENDS}"
 do_populate_sdk[depends] += "${RPMROOTFSDEPENDS}"
 
 do_rootfs[recrdeptask] += "do_package_write_rpm do_package_qa"
-do_rootfs[vardeps] += "PACKAGE_FEED_URIS"
+do_rootfs[vardeps] += "PACKAGE_FEED_URIS PACKAGE_FEED_BASE_PATHS PACKAGE_FEED_ARCHS"
 
 python () {
     if d.getVar('BUILD_IMAGES_FROM_FEEDS'):

meta/classes/sanity.bbclass

@@ -703,6 +703,23 @@ def check_sanity_version_change(status, d):
     if (tmpdirmode & stat.S_ISUID):
         status.addresult("TMPDIR is setuid, please don't build in a setuid directory")
 
+    # Check that a user isn't building in a path in PSEUDO_IGNORE_PATHS
+    pseudoignorepaths = d.getVar('PSEUDO_IGNORE_PATHS', expand=True).split(",")
+    workdir = d.getVar('WORKDIR', expand=True)
+    for i in pseudoignorepaths:
+        if i and workdir.startswith(i):
+            status.addresult("You are building in a path included in PSEUDO_IGNORE_PATHS " + str(i) + " please locate the build outside this path.\n")
+
+    # Check if PSEUDO_IGNORE_PATHS and and paths under pseudo control overlap
+    pseudoignorepaths = d.getVar('PSEUDO_IGNORE_PATHS', expand=True).split(",")
+    pseudo_control_dir = "${D},${PKGD},${PKGDEST},${IMAGEROOTFS},${SDK_OUTPUT}"
+    pseudocontroldir = d.expand(pseudo_control_dir).split(",")
+    for i in pseudoignorepaths:
+        for j in pseudocontroldir:
+            if i and j:
+                if j.startswith(i):
+                    status.addresult("A path included in PSEUDO_IGNORE_PATHS " + str(i) + " and the path " + str(j) + " overlap and this will break pseudo permission and ownership tracking. Please set the path " + str(j) + " to a different directory which does not overlap with pseudo controlled directories. \n")
+
     # Some third-party software apparently relies on chmod etc. being suid root (!!)
     import stat
     suid_check_bins = "chown chmod mknod".split()
@@ -787,6 +804,11 @@ def check_sanity_everybuild(status, d):
     if "." in paths or "./" in paths or "" in paths:
         status.addresult("PATH contains '.', './' or '' (empty element), which will break the build, please remove this.\nParsed PATH is " + str(paths) + "\n")
 
+    #Check if bitbake is present in PATH environment variable
+    bb_check = bb.utils.which(d.getVar('PATH'), 'bitbake')
+    if not bb_check:
+        bb.warn("bitbake binary is not found in PATH, did you source the script?")
+
     # Check whether 'inherit' directive is found (used for a class to inherit)
     # in conf file it's supposed to be uppercase INHERIT
     inherit = d.getVar('inherit')
@@ -860,13 +882,18 @@ def check_sanity_everybuild(status, d):
         except:
             pass
 
-    oeroot = d.getVar('COREBASE')
-    if oeroot.find('+') != -1:
-        status.addresult("Error, you have an invalid character (+) in your COREBASE directory path. Please move the installation to a directory which doesn't include any + characters.")
-    if oeroot.find('@') != -1:
-        status.addresult("Error, you have an invalid character (@) in your COREBASE directory path. Please move the installation to a directory which doesn't include any @ characters.")
-    if oeroot.find(' ') != -1:
-        status.addresult("Error, you have a space in your COREBASE directory path. Please move the installation to a directory which doesn't include a space since autotools doesn't support this.")
+    for checkdir in ['COREBASE', 'TMPDIR']:
+        val = d.getVar(checkdir)
+        if val.find('..') != -1:
+            status.addresult("Error, you have '..' in your %s directory path. Please ensure the variable contains an absolute path as this can break some recipe builds in obtuse ways." % checkdir)
+        if val.find('+') != -1:
+            status.addresult("Error, you have an invalid character (+) in your %s directory path. Please move the installation to a directory which doesn't include any + characters." % checkdir)
+        if val.find('@') != -1:
+            status.addresult("Error, you have an invalid character (@) in your %s directory path. Please move the installation to a directory which doesn't include any @ characters." % checkdir)
+        if val.find(' ') != -1:
+            status.addresult("Error, you have a space in your %s directory path. Please move the installation to a directory which doesn't include a space since autotools doesn't support this." % checkdir)
+        if val.find('%') != -1:
+            status.addresult("Error, you have an invalid character (%) in your %s directory path which causes problems with python string formatting. Please move the installation to a directory which doesn't include any % characters." % checkdir)
 
     # Check the format of MIRRORS, PREMIRRORS and SSTATE_MIRRORS
     import re

meta/classes/scons.bbclass

@@ -5,7 +5,6 @@ DEPENDS += "python3-scons-native"
 EXTRA_OESCONS ?= ""
 
 do_configure() {
-	unset _PYTHON_SYSCONFIGDATA_NAME
 	if [ -n "${CONFIGURESTAMPFILE}" ]; then
 		if [ -e "${CONFIGURESTAMPFILE}" -a "`cat ${CONFIGURESTAMPFILE}`" != "${BB_TASKHASH}" -a "${CLEANBROKEN}" != "1" ]; then
 			${STAGING_BINDIR_NATIVE}/scons --clean PREFIX=${prefix} prefix=${prefix} ${EXTRA_OESCONS}
@@ -17,13 +16,11 @@ do_configure() {
 }
 
 scons_do_compile() {
-	unset _PYTHON_SYSCONFIGDATA_NAME
 	${STAGING_BINDIR_NATIVE}/scons ${PARALLEL_MAKE} PREFIX=${prefix} prefix=${prefix} ${EXTRA_OESCONS} || \
 	die "scons build execution failed."
 }
 
 scons_do_install() {
-	unset _PYTHON_SYSCONFIGDATA_NAME
 	${STAGING_BINDIR_NATIVE}/scons install_root=${D}${prefix} PREFIX=${prefix} prefix=${prefix} ${EXTRA_OESCONS} install || \
 	die "scons install execution failed."
 }

meta/classes/sstate.bbclass

@@ -72,6 +72,7 @@ BB_HASHFILENAME = "False ${SSTATE_PKGSPEC} ${SSTATE_SWSPEC}"
 
 SSTATE_ARCHS = " \
     ${BUILD_ARCH} \
+    ${BUILD_ARCH}_${ORIGNATIVELSBSTRING} \
     ${BUILD_ARCH}_${SDK_ARCH}_${SDK_OS} \
     ${BUILD_ARCH}_${TARGET_ARCH} \
     ${SDK_ARCH}_${SDK_OS} \
@@ -80,6 +81,7 @@ SSTATE_ARCHS = " \
     ${PACKAGE_ARCH} \
     ${PACKAGE_EXTRA_ARCHS} \
     ${MACHINE_ARCH}"
+SSTATE_ARCHS[vardepsexclude] = "ORIGNATIVELSBSTRING"
 
 SSTATE_MANMACH ?= "${SSTATE_PKGARCH}"
 
@@ -121,6 +123,8 @@ SSTATE_HASHEQUIV_REPORT_TASKDATA[doc] = "Report additional useful data to the \
 python () {
     if bb.data.inherits_class('native', d):
         d.setVar('SSTATE_PKGARCH', d.getVar('BUILD_ARCH', False))
+        if d.getVar("PN") == "pseudo-native":
+            d.appendVar('SSTATE_PKGARCH', '_${ORIGNATIVELSBSTRING}')
     elif bb.data.inherits_class('crosssdk', d):
         d.setVar('SSTATE_PKGARCH', d.expand("${BUILD_ARCH}_${SDK_ARCH}_${SDK_OS}"))
     elif bb.data.inherits_class('cross', d):

meta/classes/staging.bbclass

@@ -27,11 +27,15 @@ SYSROOT_DIRS_BLACKLIST = " \
     ${mandir} \
     ${docdir} \
     ${infodir} \
+    ${datadir}/X11/locale \
     ${datadir}/applications \
+    ${datadir}/bash-completion \
     ${datadir}/fonts \
     ${datadir}/gtk-doc/html \
+    ${datadir}/installed-tests \
     ${datadir}/locale \
     ${datadir}/pixmaps \
+    ${datadir}/terminfo \
     ${libdir}/${BPN}/ptest \
 "
 

meta/classes/systemd.bbclass

@@ -174,7 +174,8 @@ python systemd_populate_packages() {
                 if path_found != '':
                     systemd_add_files_and_parse(pkg_systemd, path_found, service, keys)
                 else:
-                    bb.fatal("SYSTEMD_SERVICE_%s value %s does not exist" % (pkg_systemd, service))
+                    bb.fatal("Didn't find service unit '{0}', specified in SYSTEMD_SERVICE_{1}. {2}".format(
+                        service, pkg_systemd, "Also looked for service unit '{0}'.".format(base) if base is not None else ""))
 
     def systemd_create_presets(pkg, action):
         presetf = oe.path.join(d.getVar("PKGD"), d.getVar("systemd_unitdir"), "system-preset/98-%s.preset" % pkg)

meta/classes/testimage.bbclass

@@ -367,6 +367,7 @@ def testimage_main(d):
     package_extraction(d, tc.suites)
 
     results = None
+    complete = False
     orig_sigterm_handler = signal.signal(signal.SIGTERM, sigterm_exception)
     try:
         # We need to check if runqemu ends unexpectedly
@@ -378,6 +379,7 @@ def testimage_main(d):
         except ValueError:
             pass
         results = tc.runTests()
+        complete = True
     except (KeyboardInterrupt, BlockingIOError) as err:
         if isinstance(err, KeyboardInterrupt):
             bb.error('testimage interrupted, shutting down...')
@@ -385,20 +387,21 @@ def testimage_main(d):
             bb.error('runqemu failed, shutting down...')
         if results:
             results.stop()
-            results = None
+        results = tc.results
     finally:
         signal.signal(signal.SIGTERM, orig_sigterm_handler)
         tc.target.stop()
 
     # Show results (if we have them)
-    if not results:
+    if results:
+        configuration = get_testimage_configuration(d, 'runtime', machine)
+        results.logDetails(get_testimage_json_result_dir(d),
+                        configuration,
+                        get_testimage_result_id(configuration),
+                        dump_streams=d.getVar('TESTREPORT_FULLLOGS'))
+        results.logSummary(pn)
+    if not results or not complete:
         bb.fatal('%s - FAILED - tests were interrupted during execution' % pn, forcelog=True)
-    configuration = get_testimage_configuration(d, 'runtime', machine)
-    results.logDetails(get_testimage_json_result_dir(d),
-                       configuration,
-                       get_testimage_result_id(configuration),
-                       dump_streams=d.getVar('TESTREPORT_FULLLOGS'))
-    results.logSummary(pn)
     if not results.wasSuccessful():
         bb.fatal('%s - FAILED - check the task log and the ssh log' % pn, forcelog=True)
 

meta/conf/abi_version.conf

@@ -12,4 +12,4 @@ OELAYOUT_ABI = "14"
 # a reset of the equivalence, for example when reproducibility issues break the
 # existing match data. Distros can also append to this value for the same effect.
 #
-HASHEQUIV_HASH_VERSION  = "3"
+HASHEQUIV_HASH_VERSION  = "4"

meta/conf/bitbake.conf

@@ -685,17 +685,21 @@ SRC_URI = ""
 PSEUDO_LOCALSTATEDIR ?= "${WORKDIR}/pseudo/"
 PSEUDO_PASSWD ?= "${STAGING_DIR_TARGET}:${PSEUDO_SYSROOT}"
 PSEUDO_SYSROOT = "${COMPONENTS_DIR}/${BUILD_ARCH}/pseudo-native"
-PSEUDO_IGNORE_PATHS = "/usr/,/etc/,/lib,/dev/,${T},${WORKDIR}/recipe-sysroot,${SSTATE_DIR},${STAMPS_DIR},${WORKDIR}/pkgdata-sysroot,${TMPDIR}/sstate-control,${DEPLOY_DIR},${WORKDIR}/deploy-,${TMPDIR}/buildstats,${WORKDIR}/sstate-build-package_,${WORKDIR}/sstate-install-package_,${WORKDIR}/sstate-build-image_complete,${TMPDIR}/sysroots-components,${BUILDHISTORY_DIR},${TMPDIR}/pkgdata,${TOPDIR}/cache,${COREBASE}/scripts,${COREBASE}/meta,${CCACHE_DIR}"
+PSEUDO_IGNORE_PATHS = "/usr/,/etc/,/lib,/dev/,/run/,${T},${WORKDIR}/recipe-sysroot,${SSTATE_DIR},${STAMPS_DIR}"
+PSEUDO_IGNORE_PATHS .= ",${TMPDIR}/sstate-control,${TMPDIR}/buildstats,${TMPDIR}/sysroots-components,${TMPDIR}/pkgdata"
+PSEUDO_IGNORE_PATHS .= ",${WORKDIR}/deploy-,${WORKDIR}/sstate-build-package_,${WORKDIR}/sstate-install-package_,${WORKDIR}/pkgdata-sysroot"
+PSEUDO_IGNORE_PATHS .= ",${DEPLOY_DIR},${BUILDHISTORY_DIR},${TOPDIR}/cache,${COREBASE}/scripts,${CCACHE_DIR}"
 
 export PSEUDO_DISABLED = "1"
 #export PSEUDO_PREFIX = "${STAGING_DIR_NATIVE}${prefix_native}"
 #export PSEUDO_BINDIR = "${STAGING_DIR_NATIVE}${bindir_native}"
 #export PSEUDO_LIBDIR = "${STAGING_DIR_NATIVE}$PSEUDOBINDIR/../lib/pseudo/lib
-FAKEROOTBASEENV = "PSEUDO_BINDIR=${PSEUDO_SYSROOT}${bindir_native} PSEUDO_LIBDIR=${PSEUDO_SYSROOT}${prefix_native}/lib/pseudo/lib PSEUDO_PREFIX=${PSEUDO_SYSROOT}${prefix_native} PSEUDO_IGNORE_PATHS=${PSEUDO_IGNORE_PATHS} PSEUDO_DISABLED=1"
+FAKEROOTBASEENV = "PSEUDO_BINDIR=${PSEUDO_SYSROOT}${bindir_native} PSEUDO_LIBDIR=${PSEUDO_SYSROOT}${prefix_native}/lib/pseudo/lib PSEUDO_PREFIX=${PSEUDO_SYSROOT}${prefix_native} PSEUDO_IGNORE_PATHS=${@oe.path.canonicalize(d.getVar('PSEUDO_IGNORE_PATHS'))} PSEUDO_DISABLED=1 PYTHONDONTWRITEBYTECODE=1"
 FAKEROOTCMD = "${PSEUDO_SYSROOT}${bindir_native}/pseudo"
-FAKEROOTENV = "PSEUDO_PREFIX=${PSEUDO_SYSROOT}${prefix_native} PSEUDO_LOCALSTATEDIR=${PSEUDO_LOCALSTATEDIR} PSEUDO_PASSWD=${PSEUDO_PASSWD} PSEUDO_NOSYMLINKEXP=1 PSEUDO_IGNORE_PATHS=${PSEUDO_IGNORE_PATHS} PSEUDO_DISABLED=0"
+FAKEROOTENV = "PSEUDO_PREFIX=${PSEUDO_SYSROOT}${prefix_native} PSEUDO_LOCALSTATEDIR=${PSEUDO_LOCALSTATEDIR} PSEUDO_PASSWD=${PSEUDO_PASSWD} PSEUDO_NOSYMLINKEXP=1 PSEUDO_IGNORE_PATHS=${@oe.path.canonicalize(d.getVar('PSEUDO_IGNORE_PATHS'))} PSEUDO_DISABLED=0"
 FAKEROOTNOENV = "PSEUDO_UNLOAD=1"
 FAKEROOTDIRS = "${PSEUDO_LOCALSTATEDIR}"
+FAKEROOTLOGS = "${WORKDIR}/pseudo/pseudo.log"
 PREFERRED_PROVIDER_virtual/fakeroot-native ?= "pseudo-native"
 
 ##################################################################

meta/conf/conf-notes.txt

@@ -5,7 +5,9 @@ You can now run 'bitbake <target>'
 
 Common targets are:
     core-image-minimal
+    core-image-full-cmdline
     core-image-sato
+    core-image-weston
     meta-toolchain
     meta-ide-support
 

meta/conf/distro/include/ptest-packagelists.inc

@@ -60,6 +60,7 @@ PTESTS_FAST_remove_mips64 = "qemu-ptest"
 #    bash-ptest \ # Test outcomes are non-deterministic by design
 #    ifupdown-ptest \ # Tested separately in lib/oeqa/selftest/cases/imagefeatures.py
 #    mdadm-ptest \ # Tests rely on non-deterministic sleep() amounts
+#    libinput-ptest \ # Tests need an unloaded system to be reliable
 #"
 
 PTESTS_SLOW = "\
@@ -72,7 +73,6 @@ PTESTS_SLOW = "\
     glib-2.0-ptest \
     gstreamer1.0-ptest \
     libevent-ptest \
-    libinput-ptest \
     lttng-tools-ptest \
     openssh-ptest \
     openssl-ptest \

meta/conf/distro/include/yocto-uninative.inc

@@ -6,9 +6,9 @@
 # to the distro running on the build machine.
 #
 
-UNINATIVE_MAXGLIBCVERSION = "2.32"
+UNINATIVE_MAXGLIBCVERSION = "2.33"
 
-UNINATIVE_URL ?= "http://downloads.yoctoproject.org/releases/uninative/2.9/"
-UNINATIVE_CHECKSUM[aarch64] ?= "9f25a667aee225b1dd65c4aea73e01983e825b1cb9b56937932a1ee328b45f81"
-UNINATIVE_CHECKSUM[i686] ?= "cae5d73245d95b07cf133b780ba3f6c8d0adca3ffc4e7e7fab999961d5e24d36"
-UNINATIVE_CHECKSUM[x86_64] ?= "d07916b95c419c81541a19c8ef0ed8cbd78ae18437ff28a4c8a60ef40518e423"
+UNINATIVE_URL ?= "http://downloads.yoctoproject.org/releases/uninative/3.0/"
+UNINATIVE_CHECKSUM[aarch64] ?= "1c668909098c5b56132067adc69a249cb771f4560428e5822de903a12d97bf33"
+UNINATIVE_CHECKSUM[i686] ?= "e6cc2fc056234cffa6a2ff084cce27d544ea3f487a62b5e253351cefd4421900"
+UNINATIVE_CHECKSUM[x86_64] ?= "5ec5a9276046e7eceeac749a18b175667384e1f445cd4526300a41404d985a5b"

meta/conf/layer.conf

@@ -104,4 +104,4 @@ SSTATE_EXCLUDEDEPS_SYSROOT += ".*->autoconf-archive-native"
 # We need to keep bitbake tools in PATH
 # Avoid empty path entries
 BITBAKEPATH := "${@os.path.dirname(bb.utils.which(d.getVar('PATH'),'bitbake'))}"
-PATH := "${@'${BITBAKEPATH}:' if '${BITBAKEPATH}' is not '' else ''}${HOSTTOOLS_DIR}"
+PATH := "${@'${BITBAKEPATH}:' if '${BITBAKEPATH}' != '' else ''}${HOSTTOOLS_DIR}"

meta/conf/machine/include/qemu.inc

@@ -21,7 +21,7 @@ RDEPENDS_${KERNEL_PACKAGE_NAME}-base = ""
 # Use a common kernel recipe for all QEMU machines
 PREFERRED_PROVIDER_virtual/kernel ??= "linux-yocto"
 
-EXTRA_IMAGEDEPENDS += "qemu-native qemu-helper-native"
+EXTRA_IMAGEDEPENDS += "qemu-system-native qemu-helper-native"
 
 # Provide the nfs server kernel module for all qemu images
 KERNEL_FEATURES_append_pn-linux-yocto = " features/nfsd/nfsd-enable.scc"

meta/files/common-licenses/bzip2-1.0.4

@@ -0,0 +1,43 @@
+
+--------------------------------------------------------------------------
+
+This program, "bzip2", the associated library "libbzip2", and all
+documentation, are copyright (C) 1996-2006 Julian R Seward.  All
+rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions
+are met:
+
+1. Redistributions of source code must retain the above copyright
+   notice, this list of conditions and the following disclaimer.
+
+2. The origin of this software must not be misrepresented; you must 
+   not claim that you wrote the original software.  If you use this 
+   software in a product, an acknowledgment in the product 
+   documentation would be appreciated but is not required.
+
+3. Altered source versions must be plainly marked as such, and must
+   not be misrepresented as being the original software.
+
+4. The name of the author may not be used to endorse or promote 
+   products derived from this software without specific prior written 
+   permission.
+
+THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS
+OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY
+DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE
+GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
+WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
+NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+Julian Seward, Cambridge, UK.
+jseward@bzip.org
+bzip2/libbzip2 version 1.0.4 of 20 December 2006
+
+--------------------------------------------------------------------------

meta/files/fs-perms-persistent-log.txt

@@ -41,7 +41,7 @@ ${includedir}		0755	root	root	true	0644	root	root
 ${oldincludedir}	0755	root	root	true	0644	root	root
 
 # Cleanup debug src
-/usr/src/debug		0755	root	root	true	-	root	root
+/usr/src/debug		0755	root	root	true	0644	root	root
 
 # Items from base-files
 # Links

meta/files/fs-perms.txt

@@ -41,7 +41,7 @@ ${includedir}		0755	root	root	true	0644	root	root
 ${oldincludedir}	0755	root	root	true	0644	root	root
 
 # Cleanup debug src
-/usr/src/debug		0755	root	root	true	-	root	root
+/usr/src/debug		0755	root	root	true	0644	root	root
 
 # Items from base-files
 # Links

meta/files/toolchain-shar-extract.sh

@@ -95,7 +95,7 @@ while getopts ":yd:npDRSl" OPT; do
 		listcontents=1
 		;;
 	*)
-		echo "Usage: $(basename $0) [-y] [-d <dir>]"
+		echo "Usage: $(basename "$0") [-y] [-d <dir>]"
 		echo "  -y         Automatic yes to all prompts"
 		echo "  -d <dir>   Install the SDK to <dir>"
 		echo "======== Extensible SDK only options ============"
@@ -111,17 +111,17 @@ while getopts ":yd:npDRSl" OPT; do
 	esac
 done
 
-payload_offset=$(($(grep -na -m1 "^MARKER:$" $0|cut -d':' -f1) + 1))
+payload_offset=$(($(grep -na -m1 "^MARKER:$" "$0"|cut -d':' -f1) + 1))
 if [ "$listcontents" = "1" ] ; then
     if [ @SDK_ARCHIVE_TYPE@ = "zip" ]; then
-        tail -n +$payload_offset $0 > sdk.zip
+        tail -n +$payload_offset "$0" > sdk.zip
         if unzip -l sdk.zip;then
             rm sdk.zip
         else
             rm sdk.zip && exit 1
         fi
     else
-        tail -n +$payload_offset $0| tar tvJ || exit 1
+        tail -n +$payload_offset "$0"| tar tvJ || exit 1
     fi
     exit
 fi
@@ -242,14 +242,14 @@ fi
 
 printf "Extracting SDK..."
 if [ @SDK_ARCHIVE_TYPE@ = "zip" ]; then
-    tail -n +$payload_offset $0 > sdk.zip
+    tail -n +$payload_offset "$0" > sdk.zip
     if $SUDO_EXEC unzip $EXTRA_TAR_OPTIONS sdk.zip -d $target_sdk_dir;then
         rm sdk.zip
     else
         rm sdk.zip && exit 1
     fi
 else
-    tail -n +$payload_offset $0| $SUDO_EXEC tar mxJ -C $target_sdk_dir --checkpoint=.2500 $EXTRA_TAR_OPTIONS || exit 1
+    tail -n +$payload_offset "$0"| $SUDO_EXEC tar mxJ -C $target_sdk_dir --checkpoint=.2500 $EXTRA_TAR_OPTIONS || exit 1
 fi
 echo "done"
 

meta/files/toolchain-shar-relocate.sh

@@ -55,8 +55,11 @@ fi
 for replace in "$target_sdk_dir -maxdepth 1" "$native_sysroot"; do
 	$SUDO_EXEC find $replace -type f
 done | xargs -n100 file | grep ":.*\(ASCII\|script\|source\).*text" | \
-    awk -F':' '{printf "\"%s\"\n", $1}' | \
-    grep -Ev "$target_sdk_dir/(environment-setup-*|relocate_sdk*|${0##*/})" | \
+    awk -F': ' '{printf "\"%s\"\n", $1}' | \
+    grep -Fv -e "$target_sdk_dir/environment-setup-" \
+             -e "$target_sdk_dir/relocate_sdk" \
+             -e "$target_sdk_dir/post-relocate-setup" \
+             -e "$target_sdk_dir/${0##*/}" | \
     xargs -n100 $SUDO_EXEC sed -i \
         -e "s:$DEFAULT_INSTALL_DIR:$target_sdk_dir:g" \
         -e "s:^#! */usr/bin/perl.*:#! /usr/bin/env perl:g" \

meta/lib/oe/copy_buildsystem.py

@@ -20,7 +20,7 @@ def _smart_copy(src, dest):
     mode = os.stat(src).st_mode
     if stat.S_ISDIR(mode):
         bb.utils.mkdirhier(dest)
-        cmd = "tar --exclude='.git' --xattrs --xattrs-include='*' -chf - -C %s -p . \
+        cmd = "tar --exclude='.git' --exclude='__pycache__' --xattrs --xattrs-include='*' -chf - -C %s -p . \
         | tar --xattrs --xattrs-include='*' -xf - -C %s" % (src, dest)
         subprocess.check_output(cmd, shell=True, stderr=subprocess.STDOUT)
     else:
@@ -259,7 +259,7 @@ def create_locked_sstate_cache(lockedsigs, input_sstate_cache, output_sstate_cac
     bb.note('Generating sstate-cache...')
 
     nativelsbstring = d.getVar('NATIVELSBSTRING')
-    bb.process.run("gen-lockedsig-cache %s %s %s %s %s" % (lockedsigs, input_sstate_cache, output_sstate_cache, nativelsbstring, filterfile or ''))
+    bb.process.run("PYTHONDONTWRITEBYTECODE=1 gen-lockedsig-cache %s %s %s %s %s" % (lockedsigs, input_sstate_cache, output_sstate_cache, nativelsbstring, filterfile or ''))
     if fixedlsbstring and nativelsbstring != fixedlsbstring:
         nativedir = output_sstate_cache + '/' + nativelsbstring
         if os.path.isdir(nativedir):
@@ -286,7 +286,7 @@ def check_sstate_task_list(d, targets, filteroutfile, cmdprefix='', cwd=None, lo
         logparam = '-l %s' % logfile
     else:
         logparam = ''
-    cmd = "%sBB_SETSCENE_ENFORCE=1 PSEUDO_DISABLED=1 oe-check-sstate %s -s -o %s %s" % (cmdprefix, targets, filteroutfile, logparam)
+    cmd = "%sPYTHONDONTWRITEBYTECODE=1 BB_SETSCENE_ENFORCE=1 PSEUDO_DISABLED=1 oe-check-sstate %s -s -o %s %s" % (cmdprefix, targets, filteroutfile, logparam)
     env = dict(d.getVar('BB_ORIGENV', False))
     env.pop('BUILDDIR', '')
     env.pop('BBPATH', '')

meta/lib/oe/cve_check.py

@@ -0,0 +1,65 @@
+import collections
+import re
+import itertools
+import functools
+
+_Version = collections.namedtuple(
+    "_Version", ["release", "patch_l", "pre_l", "pre_v"]
+)
+
+@functools.total_ordering
+class Version():
+
+    def __init__(self, version, suffix=None):
+
+        suffixes = ["alphabetical", "patch"]
+
+        if str(suffix) == "alphabetical":
+            version_pattern =  r"""r?v?(?:(?P<release>[0-9]+(?:[-\.][0-9]+)*)(?P<patch>[-_\.]?(?P<patch_l>[a-z]))?(?P<pre>[-_\.]?(?P<pre_l>(rc|alpha|beta|pre|preview|dev))[-_\.]?(?P<pre_v>[0-9]+)?)?)(.*)?"""
+        elif str(suffix) == "patch":
+            version_pattern =  r"""r?v?(?:(?P<release>[0-9]+(?:[-\.][0-9]+)*)(?P<patch>[-_\.]?(p|patch)(?P<patch_l>[0-9]+))?(?P<pre>[-_\.]?(?P<pre_l>(rc|alpha|beta|pre|preview|dev))[-_\.]?(?P<pre_v>[0-9]+)?)?)(.*)?"""
+        else:
+            version_pattern =  r"""r?v?(?:(?P<release>[0-9]+(?:[-\.][0-9]+)*)(?P<pre>[-_\.]?(?P<pre_l>(rc|alpha|beta|pre|preview|dev))[-_\.]?(?P<pre_v>[0-9]+)?)?)(.*)?"""
+        regex = re.compile(r"^\s*" + version_pattern + r"\s*$", re.VERBOSE | re.IGNORECASE)
+
+        match = regex.search(version)
+        if not match:
+            raise Exception("Invalid version: '{0}'".format(version))
+
+        self._version = _Version(
+            release=tuple(int(i) for i in match.group("release").replace("-",".").split(".")),
+            patch_l=match.group("patch_l") if str(suffix) in suffixes and match.group("patch_l") else "",
+            pre_l=match.group("pre_l"),
+            pre_v=match.group("pre_v")
+        )
+
+        self._key = _cmpkey(
+            self._version.release,
+            self._version.patch_l,
+            self._version.pre_l,
+            self._version.pre_v
+        )
+
+    def __eq__(self, other):
+        if not isinstance(other, Version):
+            return NotImplemented
+        return self._key == other._key
+
+    def __gt__(self, other):
+        if not isinstance(other, Version):
+            return NotImplemented
+        return self._key > other._key
+
+def _cmpkey(release, patch_l, pre_l, pre_v):
+    # remove leading 0
+    _release = tuple(
+        reversed(list(itertools.dropwhile(lambda x: x == 0, reversed(release))))
+    )
+
+    _patch = patch_l.upper()
+
+    if pre_l is None and pre_v is None:
+        _pre = float('inf')
+    else:
+        _pre = float(pre_v) if pre_v else float('-inf')
+    return _release, _patch, _pre

meta/lib/oe/package_manager/__init__.py

@@ -189,7 +189,7 @@ class PackageManager(object, metaclass=ABCMeta):
         bb.utils.remove(self.intercepts_dir, True)
         bb.utils.mkdirhier(self.intercepts_dir)
         for intercept in postinst_intercepts:
-            bb.utils.copyfile(intercept, os.path.join(self.intercepts_dir, os.path.basename(intercept)))
+            shutil.copy(intercept, os.path.join(self.intercepts_dir, os.path.basename(intercept)))
 
     @abstractmethod
     def _handle_intercept_failure(self, failed_script):

meta/lib/oe/package_manager/deb/__init__.py

@@ -312,6 +312,12 @@ class DpkgPM(OpkgDpkgPM):
         if not pkgs:
             return
 
+        os.environ['D'] = self.target_rootfs
+        os.environ['OFFLINE_ROOT'] = self.target_rootfs
+        os.environ['IPKG_OFFLINE_ROOT'] = self.target_rootfs
+        os.environ['OPKG_OFFLINE_ROOT'] = self.target_rootfs
+        os.environ['INTERCEPT_DIR'] = self.intercepts_dir
+
         if with_dependencies:
             os.environ['APT_CONFIG'] = self.apt_conf_file
             cmd = "%s purge %s" % (self.apt_get_cmd, ' '.join(pkgs))

meta/lib/oe/package_manager/deb/sdk.py

@@ -65,6 +65,8 @@ class DpkgSdk(Sdk):
 
         self.target_pm.install_complementary(self.d.getVar('SDKIMAGE_INSTALL_COMPLEMENTARY'))
 
+        self.target_pm.run_pre_post_installs()
+
         self.target_pm.run_intercepts(populate_sdk='target')
 
         execute_pre_post_process(self.d, self.d.getVar("POPULATE_SDK_POST_TARGET_COMMAND"))
@@ -78,6 +80,8 @@ class DpkgSdk(Sdk):
         self._populate_sysroot(self.host_pm, self.host_manifest)
         self.install_locales(self.host_pm)
 
+        self.host_pm.run_pre_post_installs()
+
         self.host_pm.run_intercepts(populate_sdk='host')
 
         execute_pre_post_process(self.d, self.d.getVar("POPULATE_SDK_POST_HOST_COMMAND"))

meta/lib/oe/patch.py

@@ -38,15 +38,19 @@ def runcmd(args, dir = None):
         args = [ pipes.quote(str(arg)) for arg in args ]
         cmd = " ".join(args)
         # print("cmd: %s" % cmd)
-        (exitstatus, output) = subprocess.getstatusoutput(cmd)
+        proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True)
+        stdout, stderr = proc.communicate()
+        stdout = stdout.decode('utf-8')
+        stderr = stderr.decode('utf-8')
+        exitstatus = proc.returncode
         if exitstatus != 0:
-            raise CmdError(cmd, exitstatus >> 8, output)
-        if " fuzz " in output and "Hunk " in output:
+            raise CmdError(cmd, exitstatus >> 8, "stdout: %s\nstderr: %s" % (stdout, stderr))
+        if " fuzz " in stdout and "Hunk " in stdout:
             # Drop patch fuzz info with header and footer to log file so
             # insane.bbclass can handle to throw error/warning
-            bb.note("--- Patch fuzz start ---\n%s\n--- Patch fuzz end ---" % format(output))
+            bb.note("--- Patch fuzz start ---\n%s\n--- Patch fuzz end ---" % format(stdout))
 
-        return output
+        return stdout
 
     finally:
         if dir:

meta/lib/oe/path.py

@@ -320,3 +320,24 @@ def which_wild(pathname, path=None, mode=os.F_OK, *, reverse=False, candidates=F
 
     return files
 
+def canonicalize(paths, sep=','):
+    """Given a string with paths (separated by commas by default), expand
+    each path using os.path.realpath() and return the resulting paths as a
+    string (separated using the same separator a the original string).
+    """
+    # Ignore paths containing "$" as they are assumed to be unexpanded bitbake
+    # variables. Normally they would be ignored, e.g., when passing the paths
+    # through the shell they would expand to empty strings. However, when they
+    # are passed through os.path.realpath(), it will cause them to be prefixed
+    # with the absolute path to the current directory and thus not be empty
+    # anymore.
+    #
+    # Also maintain trailing slashes, as the paths may actually be used as
+    # prefixes in sting compares later on, where the slashes then are important.
+    canonical_paths = []
+    for path in (paths or '').split(sep):
+        if '$' not in path:
+            trailing_slash = path.endswith('/') and '/' or ''
+            canonical_paths.append(os.path.realpath(path) + trailing_slash)
+
+    return sep.join(canonical_paths)

meta/lib/oe/prservice.py

@@ -3,10 +3,6 @@
 #
 
 def prserv_make_conn(d, check = False):
-    # Otherwise this fails when called from recipes which e.g. inherit python3native (which sets _PYTHON_SYSCONFIGDATA_NAME) with:
-    # No module named '_sysconfigdata'
-    if '_PYTHON_SYSCONFIGDATA_NAME' in os.environ:
-        del os.environ['_PYTHON_SYSCONFIGDATA_NAME']
     import prserv.serv
     host_params = list([_f for _f in (d.getVar("PRSERV_HOST") or '').split(':') if _f])
     try:

meta/lib/oe/recipeutils.py

@@ -409,7 +409,7 @@ def copy_recipe_files(d, tgt_dir, whole_dir=False, download=True, all_variants=F
                 fetch.download()
             for pth in fetch.localpaths():
                 if pth not in localpaths:
-                    localpaths.append(pth)
+                    localpaths.append(os.path.abspath(pth))
             uri_values.append(srcuri)
 
     fetch_urls(d)

meta/lib/oe/reproducible.py

@@ -47,7 +47,7 @@ def find_git_folder(d, sourcedir):
     return None
 
 def get_source_date_epoch_from_git(d, sourcedir):
-    if not "git://" in d.getVar('SRC_URI'):
+    if not "git://" in d.getVar('SRC_URI') and not "gitsm://" in d.getVar('SRC_URI'):
         return None
 
     gitpath = find_git_folder(d, sourcedir)

meta/lib/oe/sstatesig.py

@@ -439,7 +439,7 @@ def find_sstate_manifest(taskdata, taskdata2, taskname, d, multilibcache):
         d2 = multilibcache[variant]
 
     if taskdata.endswith("-native"):
-        pkgarchs = ["${BUILD_ARCH}"]
+        pkgarchs = ["${BUILD_ARCH}", "${BUILD_ARCH}_${ORIGNATIVELSBSTRING}"]
     elif taskdata.startswith("nativesdk-"):
         pkgarchs = ["${SDK_ARCH}_${SDK_OS}", "allarch"]
     elif "-cross-canadian" in taskdata:
@@ -557,9 +557,11 @@ def OEOuthashBasic(path, sigfile, task, d):
                     try:
                         update_hash(" %10s" % pwd.getpwuid(s.st_uid).pw_name)
                         update_hash(" %10s" % grp.getgrgid(s.st_gid).gr_name)
-                    except KeyError:
+                    except KeyError as e:
                         bb.warn("KeyError in %s" % path)
-                        raise
+                        msg = ("KeyError: %s\nPath %s is owned by uid %d, gid %d, which doesn't match "
+                            "any user/group on target. This may be due to host contamination." % (e, path, s.st_uid, s.st_gid))
+                        raise Exception(msg).with_traceback(e.__traceback__)
 
                 if include_timestamps:
                     update_hash(" %10d" % s.st_mtime)

meta/lib/oe/terminal.py

@@ -163,7 +163,12 @@ class Tmux(Terminal):
         # devshells, if it's already there, add a new window to it.
         window_name = 'devshell-%i' % os.getpid()
 
-        self.command = 'tmux new -c "{{cwd}}" -d -s {0} -n {0} "{{command}}"'.format(window_name)
+        self.command = 'tmux new -c "{{cwd}}" -d -s {0} -n {0} "{{command}}"'
+        if not check_tmux_version('1.9'):
+            # `tmux new-session -c` was added in 1.9;
+            # older versions fail with that flag
+            self.command = 'tmux new -d -s {0} -n {0} "{{command}}"'
+        self.command = self.command.format(window_name)
         Terminal.__init__(self, sh_cmd, title, env, d)
 
         attach_cmd = 'tmux att -t {0}'.format(window_name)
@@ -253,13 +258,18 @@ def spawn(name, sh_cmd, title=None, env=None, d=None):
         except OSError:
            return
 
+def check_tmux_version(desired):
+    vernum = check_terminal_version("tmux")
+    if vernum and LooseVersion(vernum) < desired:
+        return False
+    return vernum
+
 def check_tmux_pane_size(tmux):
     import subprocess as sub
     # On older tmux versions (<1.9), return false. The reason
     # is that there is no easy way to get the height of the active panel
     # on current window without nested formats (available from version 1.9)
-    vernum = check_terminal_version("tmux")
-    if vernum and LooseVersion(vernum) < '1.9':
+    if not check_tmux_version('1.9'):
         return False
     try:
         p = sub.Popen('%s list-panes -F "#{?pane_active,#{pane_height},}"' % tmux,

meta/lib/oe/utils.py

@@ -193,7 +193,7 @@ def parallel_make(d, makeinst=False):
 
         return int(v)
 
-    return None
+    return ''
 
 def parallel_make_argument(d, fmt, limit=None, makeinst=False):
     """

meta/lib/oeqa/core/context.py

@@ -31,6 +31,9 @@ class OETestContext(object):
         self._registry = {}
         self._registry['cases'] = collections.OrderedDict()
 
+        self.results = unittest.TestResult()
+        unittest.registerResult(self.results)
+
     def _read_modules_from_manifest(self, manifest):
         if not os.path.exists(manifest):
             raise OEQAMissingManifest("Manifest does not exist on %s" % manifest)
@@ -82,6 +85,7 @@ class OETestContext(object):
         self.skipTests(skips)
 
         self._run_start_time = time.time()
+        self._run_end_time = self._run_start_time
         if not processes:
             self.runner.buffer = True
         result = self.runner.run(self.prepareSuite(self.suites, processes))

meta/lib/oeqa/manual/oe-core.json

@@ -80,7 +80,7 @@
           "expected_results": ""
         },
         "7": {
-          "action": "Run command:./configure && make ",
+          "action": "Run command:./configure ${CONFIGUREOPTS} && make ",
           "expected_results": "Verify that \"matchbox-desktop\" binary file was created successfully under \"src/\" directory "
         },
         "8": {

meta/lib/oeqa/runtime/cases/df.py

@@ -4,12 +4,14 @@
 
 from oeqa.runtime.case import OERuntimeTestCase
 from oeqa.core.decorator.depends import OETestDepends
+from oeqa.core.decorator.data import skipIfDataVar, skipIfInDataVar
 from oeqa.runtime.decorator.package import OEHasPackage
 
 class DfTest(OERuntimeTestCase):
 
     @OETestDepends(['ssh.SSHTest.test_ssh'])
     @OEHasPackage(['coreutils', 'busybox'])
+    @skipIfInDataVar('IMAGE_FEATURES', 'read-only-rootfs', 'Test case df requires a writable rootfs')
     def test_df(self):
         cmd = "df -P / | sed -n '2p' | awk '{print $4}'"
         (status,output) = self.target.run(cmd)