build-appliance-image: Update to honister head revision

(From OE-Core rev: 70384dd958c57d1da924a66cffa35f80eb60d4b0)

Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>

.gitignore

@@ -31,8 +31,4 @@ pull-*/
 bitbake/lib/toaster/contrib/tts/backlog.txt
 bitbake/lib/toaster/contrib/tts/log/*
 bitbake/lib/toaster/contrib/tts/.cache/*
-bitbake/lib/bb/tests/runqueue-tests/bitbake-cookerdaemon.log
-_toaster_clones/
-downloads/
-sstate-cache/
-toaster.sqlite
+bitbake/lib/bb/tests/runqueue-tests/bitbake-cookerdaemon.log

.templateconf

@@ -1,2 +1,2 @@
 # Template settings
-TEMPLATECONF=${TEMPLATECONF:-meta-poky/conf/templates/default}
+TEMPLATECONF=${TEMPLATECONF:-meta-poky/conf}

MAINTAINERS.md

@@ -53,6 +53,7 @@ Maintainers needed
 * wic
 * Patchwork
 * Patchtest
+* Prelink-cross
 * Matchbox
 * Sato
 * Autobuilder

Makefile

@@ -0,0 +1,35 @@
+# Minimal makefile for Sphinx documentation
+#
+
+# You can set these variables from the command line, and also
+# from the environment for the first two.
+SPHINXOPTS    ?=
+SPHINXBUILD   ?= sphinx-build
+SOURCEDIR     = .
+BUILDDIR      = _build
+DESTDIR       = final
+
+ifeq ($(shell if which $(SPHINXBUILD) >/dev/null 2>&1; then echo 1; else echo 0; fi),0)
+$(error "The '$(SPHINXBUILD)' command was not found. Make sure you have Sphinx installed")
+endif
+
+# Put it first so that "make" without argument is like "make help".
+help:
+	@$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)
+
+.PHONY: help Makefile.sphinx clean publish
+
+publish: Makefile.sphinx html singlehtml
+	rm -rf $(BUILDDIR)/$(DESTDIR)/
+	mkdir -p $(BUILDDIR)/$(DESTDIR)/
+	cp -r $(BUILDDIR)/html/* $(BUILDDIR)/$(DESTDIR)/
+	cp $(BUILDDIR)/singlehtml/index.html $(BUILDDIR)/$(DESTDIR)/singleindex.html
+	sed -i -e 's@index.html#@singleindex.html#@g' $(BUILDDIR)/$(DESTDIR)/singleindex.html
+
+clean:
+	@rm -rf $(BUILDDIR)
+
+# Catch-all target: route all unknown targets to Sphinx using the new
+# "make mode" option.  $(O) is meant as a shortcut for $(SPHINXOPTS).
+%: Makefile.sphinx
+	@$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)

README.OE-Core.md

@@ -6,28 +6,24 @@ of OpenEmbedded. It is distro-less (can build a functional image with
 DISTRO = "nodistro") and contains only emulated machine support.
 
 For information about OpenEmbedded, see the OpenEmbedded website:
-    https://www.openembedded.org/
+    http://www.openembedded.org/
 
 The Yocto Project has extensive documentation about OE including a reference manual
 which can be found at:
-    https://docs.yoctoproject.org/
+    http://yoctoproject.org/documentation
 
 
 Contributing
 ------------
 
-Please refer to our contributor guide here: https://docs.yoctoproject.org/dev/contributor-guide/
-for full details on how to submit changes.
-
-As a quick guide, patches should be sent to openembedded-core@lists.openembedded.org
-The git command to do that would be:
-
-     git send-email -M -1 --to openembedded-core@lists.openembedded.org
+Please refer to
+http://www.openembedded.org/wiki/How_to_submit_a_patch_to_OpenEmbedded
+for guidelines on how to submit patches.
 
 Mailing list:
 
-    https://lists.openembedded.org/g/openembedded-core
+    http://lists.openembedded.org/mailman/listinfo/openembedded-core
 
 Source code:
 
-    https://git.openembedded.org/openembedded-core/
+    http://git.openembedded.org/openembedded-core/

bitbake/README

@@ -13,24 +13,19 @@ Bitbake plain documentation can be found under the doc directory or its integrat
 html version at the Yocto Project website:
     https://docs.yoctoproject.org
 
-Bitbake requires Python version 3.8 or newer.
-
 Contributing
 ------------
 
-Please refer to our contributor guide here: https://docs.yoctoproject.org/dev/contributor-guide/
-for full details on how to submit changes.
-
-As a quick guide, patches should be sent to bitbake-devel@lists.openembedded.org
-The git command to do that would be:
+Please refer to
+https://www.openembedded.org/wiki/How_to_submit_a_patch_to_OpenEmbedded
+for guidelines on how to submit patches, just note that the latter documentation is intended
+for OpenEmbedded (and its core) not bitbake patches (bitbake-devel@lists.openembedded.org)
+but in general main guidelines apply. Once the commit(s) have been created, the way to send
+the patch is through git-send-email. For example, to send the last commit (HEAD) on current
+branch, type:
 
     git send-email -M -1 --to bitbake-devel@lists.openembedded.org
 
-If you're sending a patch related to the BitBake manual, make sure you copy
-the Yocto Project documentation mailing list:
-
-    git send-email -M -1 --to bitbake-devel@lists.openembedded.org --cc docs@lists.yoctoproject.org
-
 Mailing list:
 
     https://lists.openembedded.org/g/bitbake-devel
@@ -39,25 +34,10 @@ Source code:
 
     https://git.openembedded.org/bitbake/
 
-Testing
--------
+Testing:
 
 Bitbake has a testsuite located in lib/bb/tests/ whichs aim to try and prevent regressions.
 You can run this with "bitbake-selftest". In particular the fetcher is well covered since
 it has so many corner cases. The datastore has many tests too. Testing with the testsuite is
 recommended before submitting patches, particularly to the fetcher and datastore. We also
 appreciate new test cases and may require them for more obscure issues.
-
-To run the tests "zstd" and "git" must be installed.
-
-The assumption is made that this testsuite is run from an initialized OpenEmbedded build
-environment (i.e. `source oe-init-build-env` is used). If this is not the case, run the
-testsuite as follows:
-
-    export PATH=$(pwd)/bin:$PATH
-    bin/bitbake-selftest
-
-The testsuite can alternatively be executed using pytest, e.g. obtained from PyPI (in this
-case, the PATH is configured automatically):
-
-    pytest

bitbake/bin/bitbake

@@ -25,9 +25,10 @@ except RuntimeError as exc:
 from bb import cookerdata
 from bb.main import bitbake_main, BitBakeConfigParameters, BBMainException
 
-bb.utils.check_system_locale()
+if sys.getfilesystemencoding() != "utf-8":
+    sys.exit("Please use a locale setting which supports UTF-8 (such as LANG=en_US.UTF-8).\nPython can't change the filesystem locale after loading so we need a UTF-8 when Python starts or things won't work.")
 
-__version__ = "2.6.0"
+__version__ = "1.52.0"
 
 if __name__ == "__main__":
     if __version__ != bb.__version__:

bitbake/bin/bitbake-diffsigs

@@ -11,7 +11,6 @@
 import os
 import sys
 import warnings
-
 warnings.simplefilter("default")
 import argparse
 import logging
@@ -28,7 +27,6 @@ logger = bb.msg.logger_create(myname)
 
 is_dump = myname == 'bitbake-dumpsig'
 
-
 def find_siginfo(tinfoil, pn, taskname, sigs=None):
     result = None
     tinfoil.set_event_mask(['bb.event.FindSigInfoResult',
@@ -54,7 +52,6 @@ def find_siginfo(tinfoil, pn, taskname, sigs=None):
         sys.exit(2)
     return result
 
-
 def find_siginfo_task(bbhandler, pn, taskname, sig1=None, sig2=None):
     """ Find the most recent signature files for the specified PN/task """
 
@@ -63,13 +60,13 @@ def find_siginfo_task(bbhandler, pn, taskname, sig1=None, sig2=None):
 
     if sig1 and sig2:
         sigfiles = find_siginfo(bbhandler, pn, taskname, [sig1, sig2])
-        if not sigfiles:
+        if len(sigfiles) == 0:
             logger.error('No sigdata files found matching %s %s matching either %s or %s' % (pn, taskname, sig1, sig2))
             sys.exit(1)
-        elif sig1 not in sigfiles:
+        elif not sig1 in sigfiles:
             logger.error('No sigdata files found matching %s %s with signature %s' % (pn, taskname, sig1))
             sys.exit(1)
-        elif sig2 not in sigfiles:
+        elif not sig2 in sigfiles:
             logger.error('No sigdata files found matching %s %s with signature %s' % (pn, taskname, sig2))
             sys.exit(1)
         latestfiles = [sigfiles[sig1], sigfiles[sig2]]
@@ -89,11 +86,11 @@ def recursecb(key, hash1, hash2):
     hashfiles = find_siginfo(tinfoil, key, None, hashes)
 
     recout = []
-    if not hashfiles:
+    if len(hashfiles) == 0:
         recout.append("Unable to find matching sigdata for %s with hashes %s or %s" % (key, hash1, hash2))
-    elif hash1 not in hashfiles:
+    elif not hash1 in hashfiles:
         recout.append("Unable to find matching sigdata for %s with hash %s" % (key, hash1))
-    elif hash2 not in hashfiles:
+    elif not hash2 in hashfiles:
         recout.append("Unable to find matching sigdata for %s with hash %s" % (key, hash2))
     else:
         out2 = bb.siggen.compare_sigfiles(hashfiles[hash1], hashfiles[hash2], recursecb, color=color)
@@ -113,36 +110,36 @@ parser.add_argument('-D', '--debug',
 
 if is_dump:
     parser.add_argument("-t", "--task",
-                        help="find the signature data file for the last run of the specified task",
-                        action="store", dest="taskargs", nargs=2, metavar=('recipename', 'taskname'))
+            help="find the signature data file for the last run of the specified task",
+            action="store", dest="taskargs", nargs=2, metavar=('recipename', 'taskname'))
 
     parser.add_argument("sigdatafile1",
-                        help="Signature file to dump. Not used when using -t/--task.",
-                        action="store", nargs='?', metavar="sigdatafile")
+            help="Signature file to dump. Not used when using -t/--task.",
+            action="store", nargs='?', metavar="sigdatafile")
 else:
     parser.add_argument('-c', '--color',
-                        help='Colorize the output (where %(metavar)s is %(choices)s)',
-                        choices=['auto', 'always', 'never'], default='auto', metavar='color')
+            help='Colorize the output (where %(metavar)s is %(choices)s)',
+            choices=['auto', 'always', 'never'], default='auto', metavar='color')
 
     parser.add_argument('-d', '--dump',
-                        help='Dump the last signature data instead of comparing (equivalent to using bitbake-dumpsig)',
-                        action='store_true')
+            help='Dump the last signature data instead of comparing (equivalent to using bitbake-dumpsig)',
+            action='store_true')
 
     parser.add_argument("-t", "--task",
-                        help="find the signature data files for the last two runs of the specified task and compare them",
-                        action="store", dest="taskargs", nargs=2, metavar=('recipename', 'taskname'))
+            help="find the signature data files for the last two runs of the specified task and compare them",
+            action="store", dest="taskargs", nargs=2, metavar=('recipename', 'taskname'))
 
     parser.add_argument("-s", "--signature",
-                        help="With -t/--task, specify the signatures to look for instead of taking the last two",
-                        action="store", dest="sigargs", nargs=2, metavar=('fromsig', 'tosig'))
+            help="With -t/--task, specify the signatures to look for instead of taking the last two",
+            action="store", dest="sigargs", nargs=2, metavar=('fromsig', 'tosig'))
 
     parser.add_argument("sigdatafile1",
-                        help="First signature file to compare (or signature file to dump, if second not specified). Not used when using -t/--task.",
-                        action="store", nargs='?')
+            help="First signature file to compare (or signature file to dump, if second not specified). Not used when using -t/--task.",
+            action="store", nargs='?')
 
     parser.add_argument("sigdatafile2",
-                        help="Second signature file to compare",
-                        action="store", nargs='?')
+            help="Second signature file to compare",
+            action="store", nargs='?')
 
 options = parser.parse_args()
 if is_dump:
@@ -160,8 +157,7 @@ if options.taskargs:
     with bb.tinfoil.Tinfoil() as tinfoil:
         tinfoil.prepare(config_only=True)
         if not options.dump and options.sigargs:
-            files = find_siginfo_task(tinfoil, options.taskargs[0], options.taskargs[1], options.sigargs[0],
-                                      options.sigargs[1])
+            files = find_siginfo_task(tinfoil, options.taskargs[0], options.taskargs[1], options.sigargs[0], options.sigargs[1])
         else:
             files = find_siginfo_task(tinfoil, options.taskargs[0], options.taskargs[1])
 
@@ -170,8 +166,7 @@ if options.taskargs:
             output = bb.siggen.dump_sigfile(files[-1])
         else:
             if len(files) < 2:
-                logger.error('Only one matching sigdata file found for the specified task (%s %s)' % (
-                    options.taskargs[0], options.taskargs[1]))
+                logger.error('Only one matching sigdata file found for the specified task (%s %s)' % (options.taskargs[0], options.taskargs[1]))
                 sys.exit(1)
 
             # Recurse into signature comparison

bitbake/bin/bitbake-getvar

@@ -25,7 +25,6 @@ if __name__ == "__main__":
     parser.add_argument('-u', '--unexpand', help='Do not expand the value (with --value)', action="store_true")
     parser.add_argument('-f', '--flag', help='Specify a variable flag to query (with --value)', default=None)
     parser.add_argument('--value', help='Only report the value, no history and no variable name', action="store_true")
-    parser.add_argument('-q', '--quiet', help='Silence bitbake server logging', action="store_true")
     args = parser.parse_args()
 
     if args.unexpand and not args.value:
@@ -36,7 +35,7 @@ if __name__ == "__main__":
         print("--flag only makes sense with --value")
         sys.exit(1)
 
-    with bb.tinfoil.Tinfoil(tracking=True, setup_logging=not args.quiet) as tinfoil:
+    with bb.tinfoil.Tinfoil(tracking=True) as tinfoil:
         if args.recipe:
             tinfoil.prepare(quiet=2)
             d = tinfoil.parse_recipe(args.recipe)

bitbake/bin/bitbake-layers

@@ -68,11 +68,11 @@ def main():
 
         registered = False
         for plugin in plugins:
-            if hasattr(plugin, 'tinfoil_init'):
-                plugin.tinfoil_init(tinfoil)
             if hasattr(plugin, 'register_commands'):
                 registered = True
                 plugin.register_commands(subparsers)
+            if hasattr(plugin, 'tinfoil_init'):
+                plugin.tinfoil_init(tinfoil)
 
         if not registered:
             logger.error("No commands registered - missing plugins?")

bitbake/bin/bitbake-prserv

@@ -1,7 +1,5 @@
 #!/usr/bin/env python3
 #
-# Copyright BitBake Contributors
-#
 # SPDX-License-Identifier: GPL-2.0-only
 #
 

bitbake/bin/bitbake-server

@@ -12,12 +12,11 @@ warnings.simplefilter("default")
 import logging
 sys.path.insert(0, os.path.join(os.path.dirname(os.path.dirname(sys.argv[0])), 'lib'))
 
-import bb
-
-bb.utils.check_system_locale()
+if sys.getfilesystemencoding() != "utf-8":
+    sys.exit("Please use a locale setting which supports UTF-8 (such as LANG=en_US.UTF-8).\nPython can't change the filesystem locale after loading so we need a UTF-8 when Python starts or things won't work.")
 
 # Users shouldn't be running this code directly
-if len(sys.argv) != 11 or not sys.argv[1].startswith("decafbad"):
+if len(sys.argv) != 10 or not sys.argv[1].startswith("decafbad"):
     print("bitbake-server is meant for internal execution by bitbake itself, please don't use it standalone.")
     sys.exit(1)
 
@@ -29,8 +28,7 @@ logfile = sys.argv[4]
 lockname = sys.argv[5]
 sockname = sys.argv[6]
 timeout = float(sys.argv[7])
-profile = bool(int(sys.argv[8]))
-xmlrpcinterface = (sys.argv[9], int(sys.argv[10]))
+xmlrpcinterface = (sys.argv[8], int(sys.argv[9]))
 if xmlrpcinterface[0] == "None":
     xmlrpcinterface = (None, xmlrpcinterface[1])
 
@@ -51,5 +49,5 @@ logger = logging.getLogger("BitBake")
 handler = bb.event.LogHandler()
 logger.addHandler(handler)
 
-bb.server.process.execServer(lockfd, readypipeinfd, lockname, sockname, timeout, xmlrpcinterface, profile)
+bb.server.process.execServer(lockfd, readypipeinfd, lockname, sockname, timeout, xmlrpcinterface)
 

bitbake/bin/bitbake-worker

@@ -1,7 +1,5 @@
 #!/usr/bin/env python3
 #
-# Copyright BitBake Contributors
-#
 # SPDX-License-Identifier: GPL-2.0-only
 #
 
@@ -24,7 +22,8 @@ import subprocess
 from multiprocessing import Lock
 from threading import Thread
 
-bb.utils.check_system_locale()
+if sys.getfilesystemencoding() != "utf-8":
+    sys.exit("Please use a locale setting which supports UTF-8 (such as LANG=en_US.UTF-8).\nPython can't change the filesystem locale after loading so we need a UTF-8 when Python starts or things won't work.")
 
 # Users shouldn't be running this code directly
 if len(sys.argv) != 2 or not sys.argv[1].startswith("decafbad"):
@@ -121,10 +120,11 @@ def worker_child_fire(event, d):
 
     data = b"<event>" + pickle.dumps(event) + b"</event>"
     try:
-        with bb.utils.lock_timeout(worker_pipe_lock):
-            while(len(data)):
-                written = worker_pipe.write(data)
-                data = data[written:]
+        worker_pipe_lock.acquire()
+        while(len(data)):
+            written = worker_pipe.write(data)
+            data = data[written:]
+        worker_pipe_lock.release()
     except IOError:
         sigterm_handler(None, None)
         raise
@@ -143,17 +143,7 @@ def sigterm_handler(signum, frame):
     os.killpg(0, signal.SIGTERM)
     sys.exit()
 
-def fork_off_task(cfg, data, databuilder, workerdata, extraconfigdata, runtask):
-
-    fn = runtask['fn']
-    task = runtask['task']
-    taskname = runtask['taskname']
-    taskhash = runtask['taskhash']
-    unihash = runtask['unihash']
-    appends = runtask['appends']
-    layername = runtask['layername']
-    taskdepdata = runtask['taskdepdata']
-    quieterrors = runtask['quieterrors']
+def fork_off_task(cfg, data, databuilder, workerdata, fn, task, taskname, taskhash, unihash, appends, taskdepdata, extraconfigdata, quieterrors=False, dry_run_exec=False):
     # We need to setup the environment BEFORE the fork, since
     # a fork() or exec*() activates PSEUDO...
 
@@ -162,10 +152,7 @@ def fork_off_task(cfg, data, databuilder, workerdata, extraconfigdata, runtask):
     fakeenv = {}
     umask = None
 
-    uid = os.getuid()
-    gid = os.getgid()
-
-    taskdep = runtask['taskdep']
+    taskdep = workerdata["taskdeps"][fn]
     if 'umask' in taskdep and taskname in taskdep['umask']:
         umask = taskdep['umask'][taskname]
     elif workerdata["umask"]:
@@ -177,24 +164,24 @@ def fork_off_task(cfg, data, databuilder, workerdata, extraconfigdata, runtask):
         except TypeError:
              pass
 
-    dry_run = cfg.dry_run or runtask['dry_run']
+    dry_run = cfg.dry_run or dry_run_exec
 
     # We can't use the fakeroot environment in a dry run as it possibly hasn't been built
     if 'fakeroot' in taskdep and taskname in taskdep['fakeroot'] and not dry_run:
         fakeroot = True
-        envvars = (runtask['fakerootenv'] or "").split()
+        envvars = (workerdata["fakerootenv"][fn] or "").split()
         for key, value in (var.split('=') for var in envvars):
             envbackup[key] = os.environ.get(key)
             os.environ[key] = value
             fakeenv[key] = value
 
-        fakedirs = (runtask['fakerootdirs'] or "").split()
+        fakedirs = (workerdata["fakerootdirs"][fn] or "").split()
         for p in fakedirs:
             bb.utils.mkdirhier(p)
         logger.debug2('Running %s:%s under fakeroot, fakedirs: %s' %
                         (fn, taskname, ', '.join(fakedirs)))
     else:
-        envvars = (runtask['fakerootnoenv'] or "").split()
+        envvars = (workerdata["fakerootnoenv"][fn] or "").split()
         for key, value in (var.split('=') for var in envvars):
             envbackup[key] = os.environ.get(key)
             os.environ[key] = value
@@ -245,11 +232,11 @@ def fork_off_task(cfg, data, databuilder, workerdata, extraconfigdata, runtask):
                 os.umask(umask)
 
             try:
+                bb_cache = bb.cache.NoCache(databuilder)
                 (realfn, virtual, mc) = bb.cache.virtualfn2realfn(fn)
                 the_data = databuilder.mcdata[mc]
                 the_data.setVar("BB_WORKERCONTEXT", "1")
                 the_data.setVar("BB_TASKDEPDATA", taskdepdata)
-                the_data.setVar('BB_CURRENTTASK', taskname.replace("do_", ""))
                 if cfg.limited_deps:
                     the_data.setVar("BB_LIMITEDDEPS", "1")
                 the_data.setVar("BUILDNAME", workerdata["buildname"])
@@ -263,20 +250,12 @@ def fork_off_task(cfg, data, databuilder, workerdata, extraconfigdata, runtask):
                     bb.parse.siggen.set_taskhashes(workerdata["newhashes"])
                 ret = 0
 
-                the_data = databuilder.parseRecipe(fn, appends, layername)
+                the_data = bb_cache.loadDataFull(fn, appends)
                 the_data.setVar('BB_TASKHASH', taskhash)
                 the_data.setVar('BB_UNIHASH', unihash)
-                bb.parse.siggen.setup_datacache_from_datastore(fn, the_data)
 
                 bb.utils.set_process_name("%s:%s" % (the_data.getVar("PN"), taskname.replace("do_", "")))
 
-                if not bb.utils.to_boolean(the_data.getVarFlag(taskname, 'network')):
-                    if bb.utils.is_local_uid(uid):
-                        logger.debug("Attempting to disable network for %s" % taskname)
-                        bb.utils.disable_network(uid, gid)
-                    else:
-                        logger.debug("Skipping disable network for %s since %s is not a local uid." % (taskname, uid))
-
                 # exported_vars() returns a generator which *cannot* be passed to os.environ.update() 
                 # successfully. We also need to unset anything from the environment which shouldn't be there 
                 exports = bb.data.exported_vars(the_data)
@@ -449,7 +428,7 @@ class BitbakeWorker(object):
     def handle_cookercfg(self, data):
         self.cookercfg = pickle.loads(data)
         self.databuilder = bb.cookerdata.CookerDataBuilder(self.cookercfg, worker=True)
-        self.databuilder.parseBaseConfiguration(worker=True)
+        self.databuilder.parseBaseConfiguration()
         self.data = self.databuilder.data
 
     def handle_extraconfigdata(self, data):
@@ -464,7 +443,6 @@ class BitbakeWorker(object):
         for mc in self.databuilder.mcdata:
             self.databuilder.mcdata[mc].setVar("PRSERV_HOST", self.workerdata["prhost"])
             self.databuilder.mcdata[mc].setVar("BB_HASHSERVE", self.workerdata["hashservaddr"])
-            self.databuilder.mcdata[mc].setVar("__bbclasstype", "recipe")
 
     def handle_newtaskhashes(self, data):
         self.workerdata["newhashes"] = pickle.loads(data)
@@ -482,15 +460,11 @@ class BitbakeWorker(object):
         sys.exit(0)
 
     def handle_runtask(self, data):
-        runtask = pickle.loads(data)
-
-        fn = runtask['fn']
-        task = runtask['task']
-        taskname = runtask['taskname']
-
+        fn, task, taskname, taskhash, unihash, quieterrors, appends, taskdepdata, dry_run_exec = pickle.loads(data)
         workerlog_write("Handling runtask %s %s %s\n" % (task, fn, taskname))
 
-        pid, pipein, pipeout = fork_off_task(self.cookercfg, self.data, self.databuilder, self.workerdata, self.extraconfigdata, runtask)
+        pid, pipein, pipeout = fork_off_task(self.cookercfg, self.data, self.databuilder, self.workerdata, fn, task, taskname, taskhash, unihash, appends, taskdepdata, self.extraconfigdata, quieterrors, dry_run_exec)
+
         self.build_pids[pid] = task
         self.build_pipes[pid] = runQueueWorkerPipe(pipein, pipeout)
 

bitbake/bin/git-make-shallow

@@ -1,7 +1,5 @@
 #!/usr/bin/env python3
 #
-# Copyright BitBake Contributors
-#
 # SPDX-License-Identifier: GPL-2.0-only
 #
 

bitbake/bin/toaster

@@ -33,7 +33,7 @@ databaseCheck()
     $MANAGE migrate --noinput || retval=1
 
     if [ $retval -eq 1 ]; then
-        echo "Failed migrations, halting system start" 1>&2
+        echo "Failed migrations, aborting system start" 1>&2
         return $retval
     fi
     # Make sure that checksettings can pick up any value for TEMPLATECONF
@@ -41,7 +41,7 @@ databaseCheck()
     $MANAGE checksettings --traceback || retval=1
 
     if [ $retval -eq 1 ]; then
-        printf "\nError while checking settings; exiting\n"
+        printf "\nError while checking settings; aborting\n"
         return $retval
     fi
 
@@ -248,7 +248,7 @@ fi
 # 3) the sqlite db if that is being used.
 # 4) pid's we need to clean up on exit/shutdown
 export TOASTER_DIR=$TOASTERDIR
-export BB_ENV_PASSTHROUGH_ADDITIONS="$BB_ENV_PASSTHROUGH_ADDITIONS TOASTER_DIR"
+export BB_ENV_EXTRAWHITE="$BB_ENV_EXTRAWHITE TOASTER_DIR"
 
 # Determine the action. If specified by arguments, fine, if not, toggle it
 if [ "$CMD" = "start" ] ; then

bitbake/contrib/hashserv/Dockerfile

@@ -1,7 +1,7 @@
 # SPDX-License-Identifier: MIT
 #
 # Copyright (c) 2021 Joshua Watt <JPEWhacker@gmail.com>
-#
+# 
 # Dockerfile to build a bitbake hash equivalence server container
 #
 # From the root of the bitbake repository, run:
@@ -15,9 +15,5 @@ RUN apk add --no-cache python3
 
 COPY bin/bitbake-hashserv /opt/bbhashserv/bin/
 COPY lib/hashserv /opt/bbhashserv/lib/hashserv/
-COPY lib/bb /opt/bbhashserv/lib/bb/
-COPY lib/codegen.py /opt/bbhashserv/lib/codegen.py
-COPY lib/ply /opt/bbhashserv/lib/ply/
-COPY lib/bs4 /opt/bbhashserv/lib/bs4/
 
 ENTRYPOINT ["/opt/bbhashserv/bin/bitbake-hashserv"]

bitbake/contrib/prserv/Dockerfile

@@ -1,62 +0,0 @@
-# SPDX-License-Identifier: MIT
-#
-# Copyright (c) 2022 Daniel Gomez <daniel@qtec.com>
-#
-# Dockerfile to build a bitbake PR service container
-#
-# From the root of the bitbake repository, run:
-#
-#   docker build -f contrib/prserv/Dockerfile . -t prserv
-#
-# Running examples:
-#
-# 1. PR Service in RW mode, port 18585:
-#
-# docker run --detach --tty \
-# --env PORT=18585 \
-# --publish 18585:18585 \
-# --volume $PWD:/var/lib/bbprserv \
-# prserv
-#
-# 2. PR Service in RO mode, default port (8585) and custom LOGFILE:
-#
-# docker run --detach --tty \
-# --env DBMODE="--read-only" \
-# --env LOGFILE=/var/lib/bbprserv/prservro.log \
-# --publish 8585:8585 \
-# --volume $PWD:/var/lib/bbprserv \
-# prserv
-#
-
-FROM alpine:3.14.4
-
-RUN apk add --no-cache python3
-
-COPY bin/bitbake-prserv /opt/bbprserv/bin/
-COPY lib/prserv /opt/bbprserv/lib/prserv/
-COPY lib/bb /opt/bbprserv/lib/bb/
-COPY lib/codegen.py /opt/bbprserv/lib/codegen.py
-COPY lib/ply /opt/bbprserv/lib/ply/
-COPY lib/bs4 /opt/bbprserv/lib/bs4/
-
-ENV PATH=$PATH:/opt/bbprserv/bin
-
-RUN mkdir -p /var/lib/bbprserv
-
-ENV DBFILE=/var/lib/bbprserv/prserv.sqlite3 \
-    LOGFILE=/var/lib/bbprserv/prserv.log \
-    LOGLEVEL=debug \
-    HOST=0.0.0.0 \
-    PORT=8585 \
-    DBMODE=""
-
-ENTRYPOINT [ "/bin/sh", "-c", \
-"bitbake-prserv \
---file=$DBFILE \
---log=$LOGFILE \
---loglevel=$LOGLEVEL \
---start \
---host=$HOST \
---port=$PORT \
-$DBMODE \
-&& tail -f $LOGFILE"]

bitbake/contrib/vim/indent/bitbake.vim

@@ -40,7 +40,7 @@ set cpo&vim
 
 let s:maxoff = 50	" maximum number of lines to look backwards for ()
 
-function! GetBBPythonIndent(lnum)
+function GetPythonIndent(lnum)
 
   " If this line is explicitly joined: If the previous line was also joined,
   " line it up with that one, otherwise add two 'shiftwidth'
@@ -257,7 +257,7 @@ let b:did_indent = 1
 setlocal indentkeys+=0\"
 
 
-function! BitbakeIndent(lnum)
+function BitbakeIndent(lnum)
     if !has('syntax_items')
         return -1
     endif
@@ -315,7 +315,7 @@ function! BitbakeIndent(lnum)
     endif
 
     if index(["bbPyDefRegion", "bbPyFuncRegion"], name) != -1
-        let ret = GetBBPythonIndent(a:lnum)
+        let ret = GetPythonIndent(a:lnum)
         " Should normally always be indented by at least one shiftwidth; but allow
         " return of -1 (defer to autoindent) or -2 (force indent to 0)
         if ret == 0

bitbake/doc/README

@@ -8,7 +8,7 @@ Manual Organization
 
 Folders exist for individual manuals as follows:
 
-* bitbake-user-manual      --- The BitBake User Manual 
+* bitbake-user-manual      - The BitBake User Manual 
 
 Each folder is self-contained regarding content and figures.
 
@@ -47,8 +47,8 @@ To install all required packages run:
 
 To build the documentation locally, run:
 
- $ cd doc
- $ make html
+ $ cd documentation
+ $ make -f Makefile.sphinx html
 
 The resulting HTML index page will be _build/html/index.html, and you
 can browse your own copy of the locally generated documentation with

bitbake/doc/_templates/footer.html

@@ -1,9 +0,0 @@
-<footer>
-  <hr/>
-  <div role="contentinfo">
-    <p>&copy; Copyright {{ copyright }}
-       <br>Last updated on {{ last_updated }} from the <a href="https://git.openembedded.org/bitbake/">bitbake</a> git repository.
-    </p>
-  </div>
-</footer>
-

bitbake/doc/bitbake-user-manual/bitbake-user-manual-execution.rst

@@ -79,8 +79,8 @@ directives.
 Prior to parsing configuration files, BitBake looks at certain
 variables, including:
 
--  :term:`BB_ENV_PASSTHROUGH`
--  :term:`BB_ENV_PASSTHROUGH_ADDITIONS`
+-  :term:`BB_ENV_WHITELIST`
+-  :term:`BB_ENV_EXTRAWHITE`
 -  :term:`BB_PRESERVE_ENV`
 -  :term:`BB_ORIGENV`
 -  :term:`BITBAKE_UI`
@@ -228,7 +228,7 @@ and then reload it.
 Where possible, subsequent BitBake commands reuse this cache of recipe
 information. The validity of this cache is determined by first computing
 a checksum of the base configuration data (see
-:term:`BB_HASHCONFIG_IGNORE_VARS`) and
+:term:`BB_HASHCONFIG_WHITELIST`) and
 then checking if the checksum matches. If that checksum matches what is
 in the cache and the recipe and class files have not changed, BitBake is
 able to use the cache. BitBake then reloads the cached information about
@@ -435,7 +435,7 @@ BitBake writes a shell script to
 executes the script. The generated shell script contains all the
 exported variables, and the shell functions with all variables expanded.
 Output from the shell script goes to the file
-``${``\ :term:`T`\ ``}/log.do_taskname.pid``. Looking at the expanded shell functions in
+``${T}/log.do_taskname.pid``. Looking at the expanded shell functions in
 the run file and the output in the log files is a useful debugging
 technique.
 
@@ -477,7 +477,7 @@ changes because it should not affect the output for target packages. The
 simplistic approach for excluding the working directory is to set it to
 some fixed value and create the checksum for the "run" script. BitBake
 goes one step better and uses the
-:term:`BB_BASEHASH_IGNORE_VARS` variable
+:term:`BB_HASHBASE_WHITELIST` variable
 to define a list of variables that should never be included when
 generating the signatures.
 
@@ -523,7 +523,7 @@ it cannot figure out dependencies.
 Thus far, this section has limited discussion to the direct inputs into
 a task. Information based on direct inputs is referred to as the
 "basehash" in the code. However, there is still the question of a task's
-indirect inputs --- the things that were already built and present in the
+indirect inputs - the things that were already built and present in the
 build directory. The checksum (or signature) for a particular task needs
 to add the hashes of all the tasks on which the particular task depends.
 Choosing which dependencies to add is a policy decision. However, the
@@ -534,11 +534,11 @@ At the code level, there are a variety of ways both the basehash and the
 dependent task hashes can be influenced. Within the BitBake
 configuration file, we can give BitBake some extra information to help
 it construct the basehash. The following statement effectively results
-in a list of global variable dependency excludes --- variables never
+in a list of global variable dependency excludes - variables never
 included in any checksum. This example uses variables from OpenEmbedded
 to help illustrate the concept::
 
-   BB_BASEHASH_IGNORE_VARS ?= "TMPDIR FILE PATH PWD BB_TASKHASH BBPATH DL_DIR \
+   BB_HASHBASE_WHITELIST ?= "TMPDIR FILE PATH PWD BB_TASKHASH BBPATH DL_DIR \
        SSTATE_DIR THISDIR FILESEXTRAPATHS FILE_DIRNAME HOME LOGNAME SHELL \
        USER FILESPATH STAGING_DIR_HOST STAGING_DIR_TARGET COREBASE PRSERV_HOST \
        PRSERV_DUMPDIR PRSERV_DUMPFILE PRSERV_LOCKDOWN PARALLEL_MAKE \
@@ -552,8 +552,8 @@ through dependency chains are more complex and are generally
 accomplished with a Python function. The code in
 ``meta/lib/oe/sstatesig.py`` shows two examples of this and also
 illustrates how you can insert your own policy into the system if so
-desired. This file defines the basic signature generator
-OpenEmbedded-Core uses: "OEBasicHash". By default, there
+desired. This file defines the two basic signature generators
+OpenEmbedded-Core uses: "OEBasic" and "OEBasicHash". By default, there
 is a dummy "noop" signature handler enabled in BitBake. This means that
 behavior is unchanged from previous versions. ``OE-Core`` uses the
 "OEBasicHash" signature handler by default through this setting in the
@@ -561,13 +561,14 @@ behavior is unchanged from previous versions. ``OE-Core`` uses the
 
   BB_SIGNATURE_HANDLER ?= "OEBasicHash"
 
-The main feature of the "OEBasicHash" :term:`BB_SIGNATURE_HANDLER` is that
-it adds the task hash to the stamp files. Thanks to this, any metadata
-change will change the task hash, automatically causing the task to be run
-again. This removes the need to bump :term:`PR` values, and changes to
-metadata automatically ripple across the build.
+The "OEBasicHash" :term:`BB_SIGNATURE_HANDLER` is the same as the "OEBasic"
+version but adds the task hash to the stamp files. This results in any
+metadata change that changes the task hash, automatically causing the
+task to be run again. This removes the need to bump
+:term:`PR` values, and changes to metadata automatically
+ripple across the build.
 
-It is also worth noting that the end result of signature
+It is also worth noting that the end result of these signature
 generators is to make some dependency and hash information available to
 the build. This information includes:
 
@@ -656,7 +657,7 @@ builds are when execute, bitbake also supports user defined
 configuration of the `Python
 logging <https://docs.python.org/3/library/logging.html>`__ facilities
 through the :term:`BB_LOGCONFIG` variable. This
-variable defines a JSON or YAML `logging
+variable defines a json or yaml `logging
 configuration <https://docs.python.org/3/library/logging.config.html>`__
 that will be intelligently merged into the default configuration. The
 logging configuration is merged using the following rules:
@@ -690,9 +691,9 @@ logging configuration is merged using the following rules:
    adds a filter called ``BitBake.defaultFilter``, both filters will be
    applied to the logger
 
-As a first example, you can create a ``hashequiv.json`` user logging
-configuration file to log all Hash Equivalence related messages of ``VERBOSE``
-or higher priority to a file called ``hashequiv.log``::
+As an example, consider the following user logging configuration file
+which logs all Hash Equivalence related messages of VERBOSE or higher to
+a file called ``hashequiv.log`` ::
 
    {
        "version": 1,
@@ -721,40 +722,3 @@ or higher priority to a file called ``hashequiv.log``::
            }
        }
    }
-
-Then set the :term:`BB_LOGCONFIG` variable in ``conf/local.conf``::
-
-   BB_LOGCONFIG = "hashequiv.json"
-
-Another example is this ``warn.json`` file to log all ``WARNING`` and
-higher priority messages to a ``warn.log`` file::
-
-  {
-      "version": 1,
-      "formatters": {
-          "warnlogFormatter": {
-              "()": "bb.msg.BBLogFormatter",
-              "format": "%(levelname)s: %(message)s"
-          }
-      },
-
-      "handlers": {
-          "warnlog": {
-              "class": "logging.FileHandler",
-              "formatter": "warnlogFormatter",
-              "level": "WARNING",
-              "filename": "warn.log"
-          }
-      },
-
-      "loggers": {
-          "BitBake": {
-              "handlers": ["warnlog"]
-          }
-      },
-
-      "@disable_existing_loggers": false
-  }
-
-Note that BitBake's helper classes for structured logging are implemented in
-``lib/bb/msg.py``.

bitbake/doc/bitbake-user-manual/bitbake-user-manual-fetching.rst

@@ -84,18 +84,18 @@ fetcher does know how to use HTTP as a transport.
 Here are some examples that show commonly used mirror definitions::
 
    PREMIRRORS ?= "\
-      bzr://.*/.\*  http://somemirror.org/sources/ \
-      cvs://.*/.\*  http://somemirror.org/sources/ \
-      git://.*/.\*  http://somemirror.org/sources/ \
-      hg://.*/.\*   http://somemirror.org/sources/ \
-      osc://.*/.\*  http://somemirror.org/sources/ \
-      p4://.*/.\*   http://somemirror.org/sources/ \
-     svn://.*/.\*   http://somemirror.org/sources/"
+      bzr://.*/.\*  http://somemirror.org/sources/ \\n \
+      cvs://.*/.\*  http://somemirror.org/sources/ \\n \
+      git://.*/.\*  http://somemirror.org/sources/ \\n \
+      hg://.*/.\*   http://somemirror.org/sources/ \\n \
+      osc://.*/.\*  http://somemirror.org/sources/ \\n \
+      p4://.*/.\*   http://somemirror.org/sources/ \\n \
+     svn://.*/.\*   http://somemirror.org/sources/ \\n"
 
    MIRRORS =+ "\
-      ftp://.*/.\*   http://somemirror.org/sources/ \
-      http://.*/.\*  http://somemirror.org/sources/ \
-      https://.*/.\* http://somemirror.org/sources/"
+      ftp://.*/.\*   http://somemirror.org/sources/ \\n \
+      http://.*/.\*  http://somemirror.org/sources/ \\n \
+      https://.*/.\* http://somemirror.org/sources/ \\n"
 
 It is useful to note that BitBake
 supports cross-URLs. It is possible to mirror a Git repository on an
@@ -167,9 +167,6 @@ govern the behavior of the unpack stage:
 -  *dos:* Applies to ``.zip`` and ``.jar`` files and specifies whether
    to use DOS line ending conversion on text files.
 
--  *striplevel:* Strip specified number of leading components (levels)
-   from file names on extraction
-
 -  *subdir:* Unpacks the specific URL to the specified subdirectory
    within the root directory.
 
@@ -229,11 +226,6 @@ downloaded file is useful for avoiding collisions in
 :term:`DL_DIR` when dealing with multiple files that
 have the same name.
 
-If a username and password are specified in the ``SRC_URI``, a Basic
-Authorization header will be added to each request, including across redirects.
-To instead limit the Authorization header to the first request, add
-"redirectauth=0" to the list of parameters.
-
 Some example URLs are as follows::
 
    SRC_URI = "http://oe.handhelds.org/not_there.aac"
@@ -396,19 +388,6 @@ This fetcher supports the following parameters:
    protocol is "file". You can also use "http", "https", "ssh" and
    "rsync".
 
-   .. note::
-
-     When ``protocol`` is "ssh", the URL expected in :term:`SRC_URI` differs
-     from the one that is typically passed to ``git clone`` command and provided
-     by the Git server to fetch from. For example, the URL returned by GitLab
-     server for ``mesa`` when cloning over SSH is
-     ``git@gitlab.freedesktop.org:mesa/mesa.git``, however the expected URL in
-     :term:`SRC_URI` is the following::
-
-       SRC_URI = "git://git@gitlab.freedesktop.org/mesa/mesa.git;branch=main;protocol=ssh;..."
-
-     Note the ``:`` character changed for a ``/`` before the path to the project.
-
 -  *"nocheckout":* Tells the fetcher to not checkout source code when
    unpacking when set to "1". Set this option for the URL where there is
    a custom routine to checkout code. The default is "0".
@@ -424,17 +403,17 @@ This fetcher supports the following parameters:
 
 -  *"nobranch":* Tells the fetcher to not check the SHA validation for
    the branch when set to "1". The default is "0". Set this option for
-   the recipe that refers to the commit that is valid for any namespace
-   (branch, tag, ...) instead of the branch.
+   the recipe that refers to the commit that is valid for a tag instead
+   of the branch.
 
 -  *"bareclone":* Tells the fetcher to clone a bare clone into the
    destination directory without checking out a working tree. Only the
    raw Git metadata is provided. This parameter implies the "nocheckout"
    parameter as well.
 
--  *"branch":* The branch(es) of the Git tree to clone. Unless
-   "nobranch" is set to "1", this is a mandatory parameter. The number of
-   branch parameters must match the number of name parameters.
+-  *"branch":* The branch(es) of the Git tree to clone. If unset, this
+   is assumed to be "master". The number of branch parameters much match
+   the number of name parameters.
 
 -  *"rev":* The revision to use for the checkout. The default is
    "master".
@@ -457,9 +436,8 @@ This fetcher supports the following parameters:
 
 Here are some example URLs::
 
-   SRC_URI = "git://github.com/fronteed/icheck.git;protocol=https;branch=${PV};tag=${PV}"
-   SRC_URI = "git://github.com/asciidoc/asciidoc-py;protocol=https;branch=main"
-   SRC_URI = "git://git@gitlab.freedesktop.org/mesa/mesa.git;branch=main;protocol=ssh;..."
+   SRC_URI = "git://git.oe.handhelds.org/git/vip.git;tag=version-1"
+   SRC_URI = "git://git.oe.handhelds.org/git/vip.git;protocol=http"
 
 .. note::
 
@@ -476,14 +454,6 @@ Here are some example URLs::
    easy to share metadata without removing passwords. SSH keys, ``~/.netrc``
    and ``~/.ssh/config`` files can be used as alternatives.
 
-Using tags with the git fetcher may cause surprising behaviour. Bitbake needs to
-resolve the tag to a specific revision and to do that, it has to connect to and use
-the upstream repository. This is because the revision the tags point at can change and
-we've seen cases of this happening in well known public repositories. This can mean
-many more network connections than expected and recipes may be reparsed at every build.
-Source mirrors will also be bypassed as the upstream repository is the only source
-of truth to resolve the revision accurately. For these reasons, whilst the fetcher
-can support tags, we recommend being specific about revisions in recipes.
 
 .. _gitsm-fetcher:
 
@@ -696,133 +666,6 @@ Here is an example URL::
 
 It can also be used when setting mirrors definitions using the :term:`PREMIRRORS` variable.
 
-.. _gcp-fetcher:
-
-GCP Fetcher (``gs://``)
---------------------------
-
-This submodule fetches data from a
-`Google Cloud Storage Bucket <https://cloud.google.com/storage/docs/buckets>`__.
-It uses the `Google Cloud Storage Python Client <https://cloud.google.com/python/docs/reference/storage/latest>`__
-to check the status of objects in the bucket and download them.
-The use of the Python client makes it substantially faster than using command
-line tools such as gsutil.
-
-The fetcher requires the Google Cloud Storage Python Client to be installed, along
-with the gsutil tool.
-
-The fetcher requires that the machine has valid credentials for accessing the
-chosen bucket. Instructions for authentication can be found in the
-`Google Cloud documentation <https://cloud.google.com/docs/authentication/provide-credentials-adc#local-dev>`__.
-
-If it used from the OpenEmbedded build system, the fetcher can be used for
-fetching sstate artifacts from a GCS bucket by specifying the
-``SSTATE_MIRRORS`` variable as shown below::
-
-   SSTATE_MIRRORS ?= "\
-       file://.* gs://<bucket name>/PATH \
-   "
-
-The fetcher can also be used in recipes::
-
-   SRC_URI = "gs://<bucket name>/<foo_container>/<bar_file>"
-
-However, the checksum of the file should be also be provided::
-
-   SRC_URI[sha256sum] = "<sha256 string>"
-
-.. _crate-fetcher:
-
-Crate Fetcher (``crate://``)
-----------------------------
-
-This submodule fetches code for
-`Rust language "crates" <https://doc.rust-lang.org/reference/glossary.html?highlight=crate#crate>`__
-corresponding to Rust libraries and programs to compile. Such crates are typically shared
-on https://crates.io/ but this fetcher supports other crate registries too.
-
-The format for the :term:`SRC_URI` setting must be::
-
-   SRC_URI = "crate://REGISTRY/NAME/VERSION"
-
-Here is an example URL::
-
-   SRC_URI = "crate://crates.io/glob/0.2.11"
-
-.. _npm-fetcher:
-
-NPM Fetcher (``npm://``)
-------------------------
-
-This submodule fetches source code from an
-`NPM <https://en.wikipedia.org/wiki/Npm_(software)>`__
-Javascript package registry.
-
-The format for the :term:`SRC_URI` setting must be::
-
-   SRC_URI = "npm://some.registry.url;ParameterA=xxx;ParameterB=xxx;..."
-
-This fetcher supports the following parameters:
-
--  *"package":* The NPM package name. This is a mandatory parameter.
-
--  *"version":* The NPM package version. This is a mandatory parameter.
-
--  *"downloadfilename":* Specifies the filename used when storing the downloaded file.
-
--  *"destsuffix":* Specifies the directory to use to unpack the package (default: ``npm``).
-
-Note that NPM fetcher only fetches the package source itself. The dependencies
-can be fetched through the `npmsw-fetcher`_.
-
-Here is an example URL with both fetchers::
-
-   SRC_URI = " \
-       npm://registry.npmjs.org/;package=cute-files;version=${PV} \
-       npmsw://${THISDIR}/${BPN}/npm-shrinkwrap.json \
-       "
-
-See :yocto_docs:`Creating Node Package Manager (NPM) Packages
-</dev-manual/packages.html#creating-node-package-manager-npm-packages>`
-in the Yocto Project manual for details about using
-:yocto_docs:`devtool <https://docs.yoctoproject.org/ref-manual/devtool-reference.html>`
-to automatically create a recipe from an NPM URL.
-
-.. _npmsw-fetcher:
-
-NPM shrinkwrap Fetcher (``npmsw://``)
--------------------------------------
-
-This submodule fetches source code from an
-`NPM shrinkwrap <https://docs.npmjs.com/cli/v8/commands/npm-shrinkwrap>`__
-description file, which lists the dependencies
-of an NPM package while locking their versions.
-
-The format for the :term:`SRC_URI` setting must be::
-
-   SRC_URI = "npmsw://some.registry.url;ParameterA=xxx;ParameterB=xxx;..."
-
-This fetcher supports the following parameters:
-
--  *"dev":* Set this parameter to ``1`` to install "devDependencies".
-
--  *"destsuffix":* Specifies the directory to use to unpack the dependencies
-   (``${S}`` by default).
-
-Note that the shrinkwrap file can also be provided by the recipe for
-the package which has such dependencies, for example::
-
-   SRC_URI = " \
-       npm://registry.npmjs.org/;package=cute-files;version=${PV} \
-       npmsw://${THISDIR}/${BPN}/npm-shrinkwrap.json \
-       "
-
-Such a file can automatically be generated using
-:yocto_docs:`devtool <https://docs.yoctoproject.org/ref-manual/devtool-reference.html>`
-as described in the :yocto_docs:`Creating Node Package Manager (NPM) Packages
-</dev-manual/packages.html#creating-node-package-manager-npm-packages>`
-section of the Yocto Project.
-
 Other Fetchers
 --------------
 
@@ -832,9 +675,9 @@ Fetch submodules also exist for the following:
 
 -  Mercurial (``hg://``)
 
--  OSC (``osc://``)
+-  npm (``npm://``)
 
--  S3 (``s3://``)
+-  OSC (``osc://``)
 
 -  Secure FTP (``sftp://``)
 

bitbake/doc/bitbake-user-manual/bitbake-user-manual-hello.rst

@@ -18,32 +18,28 @@ it.
 Obtaining BitBake
 =================
 
-See the :ref:`bitbake-user-manual/bitbake-user-manual-intro:obtaining bitbake` section for
+See the :ref:`bitbake-user-manual/bitbake-user-manual-hello:obtaining bitbake` section for
 information on how to obtain BitBake. Once you have the source code on
 your machine, the BitBake directory appears as follows::
 
    $ ls -al
-   total 108
-   drwxr-xr-x  9 fawkh 10000  4096 feb 24 12:10 .
-   drwx------ 36 fawkh 10000  4096 mar  2 17:00 ..
-   -rw-r--r--  1 fawkh 10000   365 feb 24 12:10 AUTHORS
-   drwxr-xr-x  2 fawkh 10000  4096 feb 24 12:10 bin
-   -rw-r--r--  1 fawkh 10000 16501 feb 24 12:10 ChangeLog
-   drwxr-xr-x  2 fawkh 10000  4096 feb 24 12:10 classes
-   drwxr-xr-x  2 fawkh 10000  4096 feb 24 12:10 conf
-   drwxr-xr-x  5 fawkh 10000  4096 feb 24 12:10 contrib
-   drwxr-xr-x  6 fawkh 10000  4096 feb 24 12:10 doc
-   drwxr-xr-x  8 fawkh 10000  4096 mar  2 16:26 .git
-   -rw-r--r--  1 fawkh 10000    31 feb 24 12:10 .gitattributes
-   -rw-r--r--  1 fawkh 10000   392 feb 24 12:10 .gitignore
-   drwxr-xr-x 13 fawkh 10000  4096 feb 24 12:11 lib
-   -rw-r--r--  1 fawkh 10000  1224 feb 24 12:10 LICENSE
-   -rw-r--r--  1 fawkh 10000 15394 feb 24 12:10 LICENSE.GPL-2.0-only
-   -rw-r--r--  1 fawkh 10000  1286 feb 24 12:10 LICENSE.MIT
-   -rw-r--r--  1 fawkh 10000   229 feb 24 12:10 MANIFEST.in
-   -rw-r--r--  1 fawkh 10000  2413 feb 24 12:10 README
-   -rw-r--r--  1 fawkh 10000    43 feb 24 12:10 toaster-requirements.txt
-   -rw-r--r--  1 fawkh 10000  2887 feb 24 12:10 TODO
+   total 100
+   drwxrwxr-x. 9 wmat wmat  4096 Jan 31 13:44 .
+   drwxrwxr-x. 3 wmat wmat  4096 Feb  4 10:45 ..
+   -rw-rw-r--. 1 wmat wmat   365 Nov 26 04:55 AUTHORS
+   drwxrwxr-x. 2 wmat wmat  4096 Nov 26 04:55 bin
+   drwxrwxr-x. 4 wmat wmat  4096 Jan 31 13:44 build
+   -rw-rw-r--. 1 wmat wmat 16501 Nov 26 04:55 ChangeLog
+   drwxrwxr-x. 2 wmat wmat  4096 Nov 26 04:55 classes
+   drwxrwxr-x. 2 wmat wmat  4096 Nov 26 04:55 conf
+   drwxrwxr-x. 3 wmat wmat  4096 Nov 26 04:55 contrib
+   -rw-rw-r--. 1 wmat wmat 17987 Nov 26 04:55 COPYING
+   drwxrwxr-x. 3 wmat wmat  4096 Nov 26 04:55 doc
+   -rw-rw-r--. 1 wmat wmat    69 Nov 26 04:55 .gitignore
+   -rw-rw-r--. 1 wmat wmat   849 Nov 26 04:55 HEADER
+   drwxrwxr-x. 5 wmat wmat  4096 Jan 31 13:44 lib
+   -rw-rw-r--. 1 wmat wmat   195 Nov 26 04:55 MANIFEST.in
+   -rw-rw-r--. 1 wmat wmat  2887 Nov 26 04:55 TODO
 
 At this point, you should have BitBake cloned to a directory that
 matches the previous listing except for dates and user names.
@@ -56,7 +52,7 @@ directory to where your local BitBake files are and run the following
 command::
 
   $ ./bin/bitbake --version
-  BitBake Build Tool Core version 2.3.1
+  BitBake Build Tool Core version 1.23.0, bitbake version 1.23.0
 
 The console output tells you what version
 you are running.
@@ -134,8 +130,23 @@ Following is the complete "Hello World" example.
     directory. Run the ``bitbake`` command and see what it does::
 
        $ bitbake
-       ERROR: The BBPATH variable is not set and bitbake did not find a conf/bblayers.conf file in the expected location.
+       The BBPATH variable is not set and bitbake did not
+       find a conf/bblayers.conf file in the expected location.
        Maybe you accidentally invoked bitbake from the wrong directory?
+       DEBUG: Removed the following variables from the environment:
+       GNOME_DESKTOP_SESSION_ID, XDG_CURRENT_DESKTOP,
+       GNOME_KEYRING_CONTROL, DISPLAY, SSH_AGENT_PID, LANG, no_proxy,
+       XDG_SESSION_PATH, XAUTHORITY, SESSION_MANAGER, SHLVL,
+       MANDATORY_PATH, COMPIZ_CONFIG_PROFILE, WINDOWID, EDITOR,
+       GPG_AGENT_INFO, SSH_AUTH_SOCK, GDMSESSION, GNOME_KEYRING_PID,
+       XDG_SEAT_PATH, XDG_CONFIG_DIRS, LESSOPEN, DBUS_SESSION_BUS_ADDRESS,
+       _, XDG_SESSION_COOKIE, DESKTOP_SESSION, LESSCLOSE, DEFAULTS_PATH,
+       UBUNTU_MENUPROXY, OLDPWD, XDG_DATA_DIRS, COLORTERM, LS_COLORS
+
+    The majority of this output is specific to environment variables that
+    are not directly relevant to BitBake. However, the very first
+    message regarding the :term:`BBPATH` variable and the
+    ``conf/bblayers.conf`` file is relevant.
 
     When you run BitBake, it begins looking for metadata files. The
     :term:`BBPATH` variable is what tells BitBake where
@@ -168,14 +179,20 @@ Following is the complete "Hello World" example.
     ``bitbake`` command again::
 
        $ bitbake
-       ERROR: Unable to parse /home/scott-lenovo/bitbake/lib/bb/parse/__init__.py
-       Traceback (most recent call last):
-       File "/home/scott-lenovo/bitbake/lib/bb/parse/__init__.py", line 127, in resolve_file(fn='conf/bitbake.conf', d=<bb.data_smart.DataSmart object at 0x7f22919a3df0>):
-             if not newfn:
-       >            raise IOError(errno.ENOENT, "file %s not found in %s" % (fn, bbpath))
-             fn = newfn
-       FileNotFoundError: [Errno 2] file conf/bitbake.conf not found in <projectdirectory>
+       ERROR: Traceback (most recent call last):
+         File "/home/scott-lenovo/bitbake/lib/bb/cookerdata.py", line 163, in wrapped
+           return func(fn, *args)
+         File "/home/scott-lenovo/bitbake/lib/bb/cookerdata.py", line 173, in parse_config_file
+           return bb.parse.handle(fn, data, include)
+         File "/home/scott-lenovo/bitbake/lib/bb/parse/__init__.py", line 99, in handle
+           return h['handle'](fn, data, include)
+         File "/home/scott-lenovo/bitbake/lib/bb/parse/parse_py/ConfHandler.py", line 120, in handle
+           abs_fn = resolve_file(fn, data)
+         File "/home/scott-lenovo/bitbake/lib/bb/parse/__init__.py", line 117, in resolve_file
+           raise IOError("file %s not found in %s" % (fn, bbpath))
+       IOError: file conf/bitbake.conf not found in /home/scott-lenovo/hello
 
+       ERROR: Unable to parse conf/bitbake.conf: file conf/bitbake.conf not found in /home/scott-lenovo/hello
 
     This sample output shows that BitBake could not find the
     ``conf/bitbake.conf`` file in the project directory. This file is
@@ -209,12 +226,12 @@ Following is the complete "Hello World" example.
 
     .. note::
 
-       Without a value for :term:`PN`, the variables :term:`STAMP`, :term:`T`, and :term:`B`, prevent more
-       than one recipe from working. You can fix this by either setting :term:`PN` to
+       Without a value for PN , the variables STAMP , T , and B , prevent more
+       than one recipe from working. You can fix this by either setting PN to
        have a value similar to what OpenEmbedded and BitBake use in the default
-       ``bitbake.conf`` file (see previous example). Or, by manually updating each
-       recipe to set :term:`PN`. You will also need to include :term:`PN` as part of the :term:`STAMP`,
-       :term:`T`, and :term:`B` variable definitions in the ``local.conf`` file.
+       bitbake.conf file (see previous example). Or, by manually updating each
+       recipe to set PN . You will also need to include PN as part of the STAMP
+       , T , and B variable definitions in the local.conf file.
 
     The ``TMPDIR`` variable establishes a directory that BitBake uses
     for build output and intermediate files other than the cached
@@ -237,14 +254,18 @@ Following is the complete "Hello World" example.
     exists, you can run the ``bitbake`` command again::
 
        $ bitbake
-       ERROR: Unable to parse /home/scott-lenovo/bitbake/lib/bb/parse/parse_py/BBHandler.py
-       Traceback (most recent call last):
-       File "/home/scott-lenovo/bitbake/lib/bb/parse/parse_py/BBHandler.py", line 67, in inherit(files=['base'], fn='configuration INHERITs', lineno=0, d=<bb.data_smart.DataSmart object at 0x7fab6815edf0>):
-             if not os.path.exists(file):
-       >            raise ParseError("Could not inherit file %s" % (file), fn, lineno)
-
-       bb.parse.ParseError: ParseError in configuration INHERITs: Could not inherit file classes/base.bbclass
+       ERROR: Traceback (most recent call last):
+         File "/home/scott-lenovo/bitbake/lib/bb/cookerdata.py", line 163, in wrapped
+           return func(fn, *args)
+         File "/home/scott-lenovo/bitbake/lib/bb/cookerdata.py", line 177, in _inherit
+           bb.parse.BBHandler.inherit(bbclass, "configuration INHERITs", 0, data)
+         File "/home/scott-lenovo/bitbake/lib/bb/parse/parse_py/BBHandler.py", line 92, in inherit
+           include(fn, file, lineno, d, "inherit")
+         File "/home/scott-lenovo/bitbake/lib/bb/parse/parse_py/ConfHandler.py", line 100, in include
+           raise ParseError("Could not %(error_out)s file %(fn)s" % vars(), oldfn, lineno)
+       ParseError: ParseError in configuration INHERITs: Could not inherit file classes/base.bbclass
 
+       ERROR: Unable to parse base: ParseError in configuration INHERITs: Could not inherit file classes/base.bbclass
 
     In the sample output,
     BitBake could not find the ``classes/base.bbclass`` file. You need
@@ -263,10 +284,7 @@ Following is the complete "Hello World" example.
       $ mkdir classes
 
     Move to the ``classes`` directory and then create the
-    ``base.bbclass`` file by inserting this single line::
-
-      addtask build
-
+    ``base.bbclass`` file by inserting this single line: addtask build
     The minimal task that BitBake runs is the ``do_build`` task. This is
     all the example needs in order to build the project. Of course, the
     ``base.bbclass`` can have much more depending on which build
@@ -310,19 +328,10 @@ Following is the complete "Hello World" example.
       BBFILES += "${LAYERDIR}/*.bb"
       BBFILE_COLLECTIONS += "mylayer"
       BBFILE_PATTERN_mylayer := "^${LAYERDIR_RE}/"
-      LAYERSERIES_CORENAMES = "hello_world_example"
-      LAYERSERIES_COMPAT_mylayer = "hello_world_example"
 
     For information on these variables, click on :term:`BBFILES`,
-    :term:`LAYERDIR`, :term:`BBFILE_COLLECTIONS`, :term:`BBFILE_PATTERN_mylayer <BBFILE_PATTERN>`
-    or :term:`LAYERSERIES_COMPAT` to go to the definitions in the glossary.
-
-    .. note::
-
-       We are setting both ``LAYERSERIES_CORENAMES`` and :term:`LAYERSERIES_COMPAT` in this particular case, because we
-       are using bitbake without OpenEmbedded.
-       You should usually just use :term:`LAYERSERIES_COMPAT` to specify the OE-Core versions for which your layer
-       is compatible, and add the meta-openembedded layer to your project.
+    :term:`LAYERDIR`, :term:`BBFILE_COLLECTIONS` or :term:`BBFILE_PATTERN_mylayer <BBFILE_PATTERN>`
+    to go to the definitions in the glossary.
 
     You need to create the recipe file next. Inside your layer at the
     top-level, use an editor and create a recipe file named
@@ -380,14 +389,12 @@ Following is the complete "Hello World" example.
     target::
 
        $ bitbake printhello
-       Loading cache: 100% |
-       Loaded 0 entries from dependency cache.
        Parsing recipes: 100% |##################################################################################|
+       Time: 00:00:00
        Parsing of 1 .bb files complete (0 cached, 1 parsed). 1 targets, 0 skipped, 0 masked, 0 errors.
        NOTE: Resolving any missing task queue dependencies
-       Initialising tasks: 100% |###############################################################################|
-       NOTE: No setscene tasks
-       NOTE: Executing Tasks
+       NOTE: Preparing RunQueue
+       NOTE: Executing RunQueue Tasks
        ********************
        *                  *
        *  Hello, World!   *

bitbake/doc/bitbake-user-manual/bitbake-user-manual-intro.rst

@@ -61,9 +61,10 @@ member Chris Larson split the project into two distinct pieces:
 
 Today, BitBake is the primary basis of the
 `OpenEmbedded <https://www.openembedded.org/>`__ project, which is being
-used to build and maintain Linux distributions such as the `Poky
-Reference Distribution <https://www.yoctoproject.org/software-item/poky/>`__,
-developed under the umbrella of the `Yocto Project <https://www.yoctoproject.org>`__.
+used to build and maintain Linux distributions such as the `Angstrom
+Distribution <http://www.angstrom-distribution.org/>`__, and which is
+also being used as the build tool for Linux projects such as the `Yocto
+Project <https://www.yoctoproject.org>`__.
 
 Prior to BitBake, no other build tool adequately met the needs of an
 aspiring embedded Linux distribution. All of the build systems used by
@@ -536,7 +537,7 @@ current working directory:
 -  ``pn-buildlist``: Shows a simple list of targets that are to be
    built.
 
-To stop depending on common depends, use the ``-I`` depend option and
+To stop depending on common depends, use the "-I" depend option and
 BitBake omits them from the graph. Leaving this information out can
 produce more readable graphs. This way, you can remove from the graph
 :term:`DEPENDS` from inherited classes such as ``base.bbclass``.

bitbake/doc/bitbake-user-manual/bitbake-user-manual-metadata.rst

@@ -104,15 +104,15 @@ Line Joining
 
 Outside of :ref:`functions <bitbake-user-manual/bitbake-user-manual-metadata:functions>`,
 BitBake joins any line ending in
-a backslash character ("\\") with the following line before parsing
-statements. The most common use for the "\\" character is to split
+a backslash character ("\") with the following line before parsing
+statements. The most common use for the "\" character is to split
 variable assignments over multiple lines, as in the following example::
 
    FOO = "bar \
           baz \
           qaz"
 
-Both the "\\" character and the newline
+Both the "\" character and the newline
 character that follow it are removed when joining lines. Thus, no
 newline characters end up in the value of ``FOO``.
 
@@ -125,7 +125,7 @@ Consider this additional example where the two assignments both assign
 
 .. note::
 
-   BitBake does not interpret escape sequences like "\\n" in variable
+   BitBake does not interpret escape sequences like "\n" in variable
    values. For these to have an effect, the value must be passed to some
    utility that interprets escape sequences, such as
    ``printf`` or ``echo -n``.
@@ -159,7 +159,7 @@ behavior::
    C = "qux"
    *At this point, ${A} equals "qux bar baz"*
    B = "norf"
-   *At this point, ${A} equals "norf baz"*
+   *At this point, ${A} equals "norf baz"\*
 
 Contrast this behavior with the
 :ref:`bitbake-user-manual/bitbake-user-manual-metadata:immediate variable
@@ -195,45 +195,22 @@ value. However, if ``A`` is not set, the variable is set to "aval".
 Setting a weak default value (??=)
 ----------------------------------
 
-The weak default value of a variable is the value which that variable
-will expand to if no value has been assigned to it via any of the other
-assignment operators. The "??=" operator takes effect immediately, replacing
-any previously defined weak default value. Here is an example::
+It is possible to use a "weaker" assignment than in the previous section
+by using the "??=" operator. This assignment behaves identical to "?="
+except that the assignment is made at the end of the parsing process
+rather than immediately. Consequently, when multiple "??=" assignments
+exist, the last one is used. Also, any "=" or "?=" assignment will
+override the value set with "??=". Here is an example::
 
-   W ??= "x"
-   A := "${W}" # Immediate variable expansion
-   W ??= "y"
-   B := "${W}" # Immediate variable expansion
-   W ??= "z"
-   C = "${W}"
-   W ?= "i"
+   A ??= "somevalue"
+   A ??= "someothervalue"
 
-After parsing we will have::
+If ``A`` is set before the above statements are
+parsed, the variable retains its value. If ``A`` is not set, the
+variable is set to "someothervalue".
 
-   A = "x"
-   B = "y"
-   C = "i"
-   W = "i"
-
-Appending and prepending non-override style will not substitute the weak
-default value, which means that after parsing::
-
-   W ??= "x"
-   W += "y"
-
-we will have::
-
-   W = " y"
-
-On the other hand, override-style appends/prepends/removes are applied after
-any active weak default value has been substituted::
-
-   W ??= "x"
-   W:append = "y"
-
-After parsing we will have::
-
-   W = "xy"
+Again, this assignment is a "lazy" or "weak" assignment because it does
+not occur until the end of the parsing process.
 
 Immediate variable expansion (:=)
 ---------------------------------
@@ -319,10 +296,6 @@ The variable ``D`` becomes "dvaladditional data".
 
    You must control all spacing when you use the override syntax.
 
-.. note::
-
-   The overrides are applied in this order, ":append", ":prepend", ":remove".
-
 It is also possible to append and prepend to shell functions and
 BitBake-style Python functions. See the ":ref:`bitbake-user-manual/bitbake-user-manual-metadata:shell functions`" and ":ref:`bitbake-user-manual/bitbake-user-manual-metadata:bitbake-style python functions`"
 sections for examples.
@@ -334,8 +307,7 @@ Removal (Override Style Syntax)
 
 You can remove values from lists using the removal override style
 syntax. Specifying a value for removal causes all occurrences of that
-value to be removed from the variable. Unlike ":append" and ":prepend",
-there is no need to add a leading or trailing space to the value.
+value to be removed from the variable.
 
 When you use this syntax, BitBake expects one or more strings.
 Surrounding spaces and spacing are preserved. Here is an example::
@@ -356,28 +328,6 @@ The variable ``FOO`` becomes
 Like ":append" and ":prepend", ":remove" is applied at variable
 expansion time.
 
-.. note::
-
-   The overrides are applied in this order, ":append", ":prepend", ":remove".
-   This implies it is not possible to re-append previously removed strings.
-   However, one can undo a ":remove" by using an intermediate variable whose
-   content is passed to the ":remove" so that modifying the intermediate
-   variable equals to keeping the string in::
-
-     FOOREMOVE = "123 456 789"
-     FOO:remove = "${FOOREMOVE}"
-     ...
-     FOOREMOVE = "123 789"
-
-   This expands to ``FOO:remove = "123 789"``.
-
-.. note::
-
-   Override application order may not match variable parse history, i.e.
-   the output of ``bitbake -e`` may contain ":remove" before ":append",
-   but the result will be removed string, because ":remove" is handled
-   last.
-
 Override Style Operation Advantages
 -----------------------------------
 
@@ -448,12 +398,6 @@ documentation to a BitBake variable as follows::
 
    CACHE[doc] = "The directory holding the cache of the metadata."
 
-.. note::
-
-   Variable flag names starting with an underscore (``_``) character
-   are allowed but are ignored by ``d.getVarFlags("VAR")``
-   in Python code. Such flag names are used internally by BitBake.
-
 Inline Python Variable Expansion
 --------------------------------
 
@@ -566,8 +510,8 @@ variable.
 
 .. note::
 
-   Overrides can only use lower-case characters, digits and dashes.
-   In particular, colons are not permitted in override names as they are used to
+   Overrides can only use lower-case characters. Additionally,
+   underscores are not permitted in override names as they are used to
    separate overrides from each other and from the variable name.
 
 -  *Selecting a Variable:* The :term:`OVERRIDES` variable is a
@@ -579,14 +523,14 @@ variable.
 
       OVERRIDES = "architecture:os:machine"
       TEST = "default"
-      TEST:os = "osspecific"
-      TEST:nooverride = "othercondvalue"
+      TEST_os = "osspecific"
+      TEST_nooverride = "othercondvalue"
 
    In this example, the :term:`OVERRIDES`
    variable lists three overrides: "architecture", "os", and "machine".
    The variable ``TEST`` by itself has a default value of "default". You
    select the os-specific version of the ``TEST`` variable by appending
-   the "os" override to the variable (i.e. ``TEST:os``).
+   the "os" override to the variable (i.e. ``TEST_os``).
 
    To better understand this, consider a practical example that assumes
    an OpenEmbedded metadata-based Linux kernel recipe file. The
@@ -623,7 +567,7 @@ variable.
 -  *Setting a Variable for a Single Task:* BitBake supports setting a
    variable just for the duration of a single task. Here is an example::
 
-      FOO:task-configure = "val 1"
+      FOO_task-configure = "val 1"
       FOO:task-compile = "val 2"
 
    In the
@@ -641,16 +585,6 @@ variable.
 
       EXTRA_OEMAKE:prepend:task-compile = "${PARALLEL_MAKE} "
 
-.. note::
-
-   Before BitBake 1.52 (Honister 3.4), the syntax for :term:`OVERRIDES`
-   used ``_`` instead of ``:``, so you will still find a lot of documentation
-   using ``_append``, ``_prepend``, and ``_remove``, for example.
-
-   For details, see the
-   :yocto_docs:`Overrides Syntax Changes </migration-guides/migration-3.4.html#override-syntax-changes>`
-   section in the Yocto Project manual migration notes.
-
 Key Expansion
 -------------
 
@@ -960,7 +894,7 @@ Regardless of the type of function, you can only define them in class
 Shell Functions
 ---------------
 
-Functions written in shell script are executed either directly as
+Functions written in shell script and executed either directly as
 functions, tasks, or both. They can also be called by other shell
 functions. Here is an example shell function definition::
 
@@ -1010,7 +944,7 @@ Running ``do_foo`` prints the following::
    Overrides and override-style operators can be applied to any shell
    function, not just :ref:`tasks <bitbake-user-manual/bitbake-user-manual-metadata:tasks>`.
 
-You can use the ``bitbake -e recipename`` command to view the final
+You can use the ``bitbake -e`` recipename command to view the final
 assembled function after all overrides have been applied.
 
 BitBake-Style Python Functions
@@ -1062,7 +996,7 @@ Running ``do_foo`` prints the following::
    recipename do_foo: second
    recipename do_foo: third
 
-You can use the ``bitbake -e recipename`` command to view
+You can use the ``bitbake -e`` recipename command to view
 the final assembled function after all overrides have been applied.
 
 Python Functions
@@ -1409,8 +1343,8 @@ the build machine cannot influence the build.
 .. note::
 
    By default, BitBake cleans the environment to include only those
-   things exported or listed in its passthrough list to ensure that the
-   build environment is reproducible and consistent. You can prevent this
+   things exported or listed in its whitelist to ensure that the build
+   environment is reproducible and consistent. You can prevent this
    "cleaning" by setting the :term:`BB_PRESERVE_ENV` variable.
 
 Consequently, if you do want something to get passed into the build task
@@ -1418,14 +1352,14 @@ environment, you must take these two steps:
 
 #. Tell BitBake to load what you want from the environment into the
    datastore. You can do so through the
-   :term:`BB_ENV_PASSTHROUGH` and
-   :term:`BB_ENV_PASSTHROUGH_ADDITIONS` variables. For
+   :term:`BB_ENV_WHITELIST` and
+   :term:`BB_ENV_EXTRAWHITE` variables. For
    example, assume you want to prevent the build system from accessing
-   your ``$HOME/.ccache`` directory. The following command adds the
-   the environment variable ``CCACHE_DIR`` to BitBake's passthrough
-   list to allow that variable into the datastore::
+   your ``$HOME/.ccache`` directory. The following command "whitelists"
+   the environment variable ``CCACHE_DIR`` causing BitBake to allow that
+   variable into the datastore::
 
-      export BB_ENV_PASSTHROUGH_ADDITIONS="$BB_ENV_PASSTHROUGH_ADDITIONS CCACHE_DIR"
+      export BB_ENV_EXTRAWHITE="$BB_ENV_EXTRAWHITE CCACHE_DIR"
 
 #. Tell BitBake to export what you have loaded into the datastore to the
    task environment of every running task. Loading something from the
@@ -1442,7 +1376,7 @@ environment, you must take these two steps:
       A side effect of the previous steps is that BitBake records the
       variable as a dependency of the build process in things like the
       setscene checksums. If doing so results in unnecessary rebuilds of
-      tasks, you can also flag the variable so that the setscene code
+      tasks, you can whitelist the variable so that the setscene code
       ignores the dependency when it creates checksums.
 
 Sometimes, it is useful to be able to obtain information from the
@@ -1496,35 +1430,12 @@ functionality of the task:
    directory listed is used as the current working directory for the
    task.
 
-- ``[file-checksums]``: Controls the file dependencies for a task. The
-  baseline file list is the set of files associated with
-  :term:`SRC_URI`. May be used to set additional dependencies on
-  files not associated with :term:`SRC_URI`.
-
-  The value set to the list is a file-boolean pair where the first
-  value is the file name and the second is whether or not it
-  physically exists on the filesystem. ::
-
-    do_configure[file-checksums] += "${MY_DIRPATH}/my-file.txt:True"
-
-  It is important to record any paths which the task looked at and
-  which didn't exist. This means that if these do exist at a later
-  time, the task can be rerun with the new additional files. The
-  "exists" True or False value after the path allows this to be
-  handled.
-
 -  ``[lockfiles]``: Specifies one or more lockfiles to lock while the
    task executes. Only one task may hold a lockfile, and any task that
    attempts to lock an already locked file will block until the lock is
    released. You can use this variable flag to accomplish mutual
    exclusion.
 
--  ``[network]``: When set to "1", allows a task to access the network. By
-   default, only the ``do_fetch`` task is granted network access. Recipes
-   shouldn't access the network outside of ``do_fetch`` as it usually
-   undermines fetcher source mirroring, image and licence manifests, software
-   auditing and supply chain security.
-
 -  ``[noexec]``: When set to "1", marks the task as being empty, with
    no execution required. You can use the ``[noexec]`` flag to set up
    tasks as dependency placeholders, or to disable tasks defined
@@ -1737,8 +1648,8 @@ user interfaces:
 
 .. _variants-class-extension-mechanism:
 
-Variants --- Class Extension Mechanism
-======================================
+Variants - Class Extension Mechanism
+====================================
 
 BitBake supports multiple incarnations of a recipe file via the
 :term:`BBCLASSEXTEND` variable.
@@ -1978,33 +1889,6 @@ looking at the source code of the ``bb`` module, which is in
 the commonly used functions ``bb.utils.contains()`` and
 ``bb.utils.mkdirhier()``, which come with docstrings.
 
-Extending Python Library Code
------------------------------
-
-If you wish to add your own Python library code (e.g. to provide
-functions/classes you can use from Python functions in the metadata)
-you can do so from any layer using the ``addpylib`` directive.
-This directive is typically added to your layer configuration (
-``conf/layer.conf``) although it will be handled in any ``.conf`` file.
-
-Usage is of the form::
-
-   addpylib <directory> <namespace>
-
-Where <directory> specifies the directory to add to the library path.
-The specified <namespace> is imported automatically, and if the imported
-module specifies an attribute named ``BBIMPORTS``, that list of
-sub-modules is iterated and imported too.
-
-Testing and Debugging BitBake Python code
------------------------------------------
-
-The OpenEmbedded build system implements a convenient ``pydevshell`` target which
-you can use to access the BitBake datastore and experiment with your own Python
-code. See :yocto_docs:`Using a Python Development Shell
-</dev-manual/python-development-shell.html#using-a-python-development-shell>` in the Yocto
-Project manual for details.
-
 Task Checksums and Setscene
 ===========================
 
@@ -2037,6 +1921,12 @@ The following list describes related variables:
    Specifies a function BitBake calls that determines whether BitBake
    requires a setscene dependency to be met.
 
+-  :term:`BB_STAMP_POLICY`: Defines the mode
+   for comparing timestamps of stamp files.
+
+-  :term:`BB_STAMP_WHITELIST`: Lists stamp
+   files that are looked at when the stamp policy is "whitelist".
+
 -  :term:`BB_TASKHASH`: Within an executing task,
    this variable holds the hash of the task as returned by the currently
    enabled signature generator.

bitbake/doc/bitbake-user-manual/bitbake-user-manual-ref-variables.rst

@@ -40,7 +40,8 @@ overview of their function and contents.
       Azure Storage Shared Access Signature, when using the
       :ref:`Azure Storage fetcher <bitbake-user-manual/bitbake-user-manual-fetching:fetchers>`
       This variable can be defined to be used by the fetcher to authenticate
-      and gain access to non-public artifacts::
+      and gain access to non-public artifacts.
+      ::
 
          AZ_SAS = ""se=2021-01-01&sp=r&sv=2018-11-09&sr=c&skoid=<skoid>&sig=<signature>""
 
@@ -92,33 +93,10 @@ overview of their function and contents.
       fetcher does not attempt to use the host listed in :term:`SRC_URI` after
       a successful fetch from the :term:`PREMIRRORS` occurs.
 
-   :term:`BB_BASEHASH_IGNORE_VARS`
-      Lists variables that are excluded from checksum and dependency data.
-      Variables that are excluded can therefore change without affecting
-      the checksum mechanism. A common example would be the variable for
-      the path of the build. BitBake's output should not (and usually does
-      not) depend on the directory in which it was built.
-
-   :term:`BB_CACHEDIR`
-      Specifies the code parser cache directory (distinct from :term:`CACHE`
-      and :term:`PERSISTENT_DIR` although they can be set to the same value
-      if desired). The default value is "${TOPDIR}/cache".
-
    :term:`BB_CHECK_SSL_CERTS`
       Specifies if SSL certificates should be checked when fetching. The default
       value is ``1`` and certificates are not checked if the value is set to ``0``.
 
-   :term:`BB_HASH_CODEPARSER_VALS`
-      Specifies values for variables to use when populating the codeparser cache.
-      This can be used selectively to set dummy values for variables to avoid
-      the codeparser cache growing on every parse. Variables that would typically
-      be included are those where the value is not significant for where the
-      codeparser cache is used (i.e. when calculating variable dependencies for
-      code fragments.) The value is space-separated without quoting values, for
-      example::
-
-         BB_HASH_CODEPARSER_VALS = "T=/ WORKDIR=/ DATE=1234 TIME=1234"
-
    :term:`BB_CONSOLELOG`
       Specifies the path to a log file into which BitBake's user interface
       writes output during the build.
@@ -160,7 +138,7 @@ overview of their function and contents.
          where:
 
             <action> is:
-               HALT:      Immediately halt the build when
+               ABORT:     Immediately abort the build when
                           a threshold is broken.
                STOPTASKS: Stop the build after the currently
                           executing tasks have finished when
@@ -191,13 +169,13 @@ overview of their function and contents.
 
       Here are some examples::
 
-         BB_DISKMON_DIRS = "HALT,${TMPDIR},1G,100K WARN,${SSTATE_DIR},1G,100K"
+         BB_DISKMON_DIRS = "ABORT,${TMPDIR},1G,100K WARN,${SSTATE_DIR},1G,100K"
          BB_DISKMON_DIRS = "STOPTASKS,${TMPDIR},1G"
-         BB_DISKMON_DIRS = "HALT,${TMPDIR},,100K"
+         BB_DISKMON_DIRS = "ABORT,${TMPDIR},,100K"
 
       The first example works only if you also set the
       :term:`BB_DISKMON_WARNINTERVAL`
-      variable. This example causes the build system to immediately halt
+      variable. This example causes the build system to immediately abort
       when either the disk space in ``${TMPDIR}`` drops below 1 Gbyte or
       the available free inodes drops below 100 Kbytes. Because two
       directories are provided with the variable, the build system also
@@ -211,7 +189,7 @@ overview of their function and contents.
       directory drops below 1 Gbyte. No disk monitoring occurs for the free
       inodes in this case.
 
-      The final example immediately halts the build when the number of
+      The final example immediately aborts the build when the number of
       free inodes in the ``${TMPDIR}`` directory drops below 100 Kbytes. No
       disk space monitoring for the directory itself occurs in this case.
 
@@ -258,23 +236,23 @@ overview of their function and contents.
       based on the interval occur each time a respective interval is
       reached beyond the initial warning (i.e. 1 Gbytes and 100 Kbytes).
 
-   :term:`BB_ENV_PASSTHROUGH`
-      Specifies the internal list of variables to allow through from
-      the external environment into BitBake's datastore. If the value of
-      this variable is not specified (which is the default), the following
-      list is used: :term:`BBPATH`, :term:`BB_PRESERVE_ENV`,
-      :term:`BB_ENV_PASSTHROUGH`, and :term:`BB_ENV_PASSTHROUGH_ADDITIONS`.
+   :term:`BB_ENV_EXTRAWHITE`
+      Specifies an additional set of variables to allow through (whitelist)
+      from the external environment into BitBake's datastore. This list of
+      variables are on top of the internal list set in
+      :term:`BB_ENV_WHITELIST`.
 
       .. note::
 
          You must set this variable in the external environment in order
          for it to work.
 
-   :term:`BB_ENV_PASSTHROUGH_ADDITIONS`
-      Specifies an additional set of variables to allow through from the
-      external environment into BitBake's datastore. This list of variables
-      are on top of the internal list set in
-      :term:`BB_ENV_PASSTHROUGH`.
+   :term:`BB_ENV_WHITELIST`
+      Specifies the internal whitelist of variables to allow through from
+      the external environment into BitBake's datastore. If the value of
+      this variable is not specified (which is the default), the following
+      list is used: :term:`BBPATH`, :term:`BB_PRESERVE_ENV`,
+      :term:`BB_ENV_WHITELIST`, and :term:`BB_ENV_EXTRAWHITE`.
 
       .. note::
 
@@ -303,69 +281,12 @@ overview of their function and contents.
 
          BB_GENERATE_MIRROR_TARBALLS = "1"
 
-   :term:`BB_GENERATE_SHALLOW_TARBALLS`
-      Setting this variable to "1" when :term:`BB_GIT_SHALLOW` is also set to
-      "1" causes bitbake to generate shallow mirror tarballs when fetching git
-      repositories. The number of commits included in the shallow mirror
-      tarballs is controlled by :term:`BB_GIT_SHALLOW_DEPTH`.
-
-      If both :term:`BB_GIT_SHALLOW` and :term:`BB_GENERATE_MIRROR_TARBALLS` are
-      enabled, bitbake will generate shallow mirror tarballs by default for git
-      repositories. This separate variable exists so that shallow tarball
-      generation can be enabled without needing to also enable normal mirror
-      generation if it is not desired.
-
-      For example usage, see :term:`BB_GIT_SHALLOW`.
-
-   :term:`BB_GIT_SHALLOW`
-      Setting this variable to "1" enables the support for fetching, using and
-      generating mirror tarballs of `shallow git repositories <https://riptutorial.com/git/example/4584/shallow-clone>`_.
-      The external `git-make-shallow <https://git.openembedded.org/bitbake/tree/bin/git-make-shallow>`_
-      script is used for shallow mirror tarball creation.
-
-      When :term:`BB_GIT_SHALLOW` is enabled, bitbake will attempt to fetch a shallow
-      mirror tarball. If the shallow mirror tarball cannot be fetched, it will
-      try to fetch the full mirror tarball and use that.
-
-      When a mirror tarball is not available, a full git clone will be performed
-      regardless of whether this variable is set or not. Support for shallow
-      clones is not currently implemented as git does not directly support
-      shallow cloning a particular git commit hash (it only supports cloning
-      from a tag or branch reference).
-
-      See also :term:`BB_GIT_SHALLOW_DEPTH` and
-      :term:`BB_GENERATE_SHALLOW_TARBALLS`.
-
-      Example usage::
-
-         BB_GIT_SHALLOW ?= "1"
-
-         # Keep only the top commit
-         BB_GIT_SHALLOW_DEPTH ?= "1"
-
-         # This defaults to enabled if both BB_GIT_SHALLOW and
-         # BB_GENERATE_MIRROR_TARBALLS are enabled
-         BB_GENERATE_SHALLOW_TARBALLS ?= "1"
-
-   :term:`BB_GIT_SHALLOW_DEPTH`
-      When used with :term:`BB_GENERATE_SHALLOW_TARBALLS`, this variable sets
-      the number of commits to include in generated shallow mirror tarballs.
-      With a depth of 1, only the commit referenced in :term:`SRCREV` is
-      included in the shallow mirror tarball. Increasing the depth includes
-      additional parent commits, working back through the commit history.
-
-      If this variable is unset, bitbake will default to a depth of 1 when
-      generating shallow mirror tarballs.
-
-      For example usage, see :term:`BB_GIT_SHALLOW`.
-
-   :term:`BB_GLOBAL_PYMODULES`
-      Specifies the list of Python modules to place in the global namespace.
-      It is intended that only the core layer should set this and it is meant
-      to be a very small list, typically just ``os`` and ``sys``.
-      :term:`BB_GLOBAL_PYMODULES` is expected to be set before the first
-      ``addpylib`` directive.
-      See also ":ref:`bitbake-user-manual/bitbake-user-manual-metadata:extending python library code`".
+   :term:`BB_HASHBASE_WHITELIST`
+      Lists variables that are excluded from checksum and dependency data.
+      Variables that are excluded can therefore change without affecting
+      the checksum mechanism. A common example would be the variable for
+      the path of the build. BitBake's output should not (and usually does
+      not) depend on the directory in which it was built.
 
    :term:`BB_HASHCHECK_FUNCTION`
       Specifies the name of the function to call during the "setscene" part
@@ -381,7 +302,7 @@ overview of their function and contents.
       However, the more accurate the data returned, the more efficient the
       build will be.
 
-   :term:`BB_HASHCONFIG_IGNORE_VARS`
+   :term:`BB_HASHCONFIG_WHITELIST`
       Lists variables that are excluded from base configuration checksum,
       which is used to determine if the cache can be reused.
 
@@ -397,35 +318,12 @@ overview of their function and contents.
       Specifies the Hash Equivalence server to use.
 
       If set to ``auto``, BitBake automatically starts its own server
-      over a UNIX domain socket. An option is to connect this server
-      to an upstream one, by setting :term:`BB_HASHSERVE_UPSTREAM`.
+      over a UNIX domain socket.
 
-      If set to ``unix://path``, BitBake will connect to an existing
-      hash server available over a UNIX domain socket.
-
-      If set to ``host:port``, BitBake will connect to a remote server on the
+      If set to ``host:port``, BitBake will use a remote server on the
       specified host. This allows multiple clients to share the same
       hash equivalence data.
 
-      The remote server can be started manually through
-      the ``bin/bitbake-hashserv`` script provided by BitBake,
-      which supports UNIX domain sockets too. This script also allows
-      to start the server in read-only mode, to avoid accepting
-      equivalences that correspond to Share State caches that are
-      only available on specific clients.
-
-   :term:`BB_HASHSERVE_UPSTREAM`
-      Specifies an upstream Hash Equivalence server.
-
-      This optional setting is only useful when a local Hash Equivalence
-      server is started (setting :term:`BB_HASHSERVE` to ``auto``),
-      and you wish the local server to query an upstream server for
-      Hash Equivalence data.
-
-      Example usage::
-
-         BB_HASHSERVE_UPSTREAM = "hashserv.yocto.io:8687"
-
    :term:`BB_INVALIDCONF`
       Used in combination with the ``ConfigParsed`` event to trigger
       re-parsing the base metadata (i.e. all the recipes). The
@@ -449,19 +347,6 @@ overview of their function and contents.
       If you want to force log files to take a specific name, you can set this
       variable in a configuration file.
 
-   :term:`BB_MULTI_PROVIDER_ALLOWED`
-      Allows you to suppress BitBake warnings caused when building two
-      separate recipes that provide the same output.
-
-      BitBake normally issues a warning when building two different recipes
-      where each provides the same output. This scenario is usually
-      something the user does not want. However, cases do exist where it
-      makes sense, particularly in the ``virtual/*`` namespace. You can use
-      this variable to suppress BitBake's warnings.
-
-      To use the variable, list provider names (e.g. recipe names,
-      ``virtual/kernel``, and so forth).
-
    :term:`BB_NICE_LEVEL`
       Allows BitBake to run at a specific priority (i.e. nice level).
       System permissions usually mean that BitBake can reduce its priority
@@ -488,9 +373,8 @@ overview of their function and contents.
 
    :term:`BB_ORIGENV`
       Contains a copy of the original external environment in which BitBake
-      was run. The copy is taken before any variable values configured to
-      pass through from the external environment are filtered into BitBake's
-      datastore.
+      was run. The copy is taken before any whitelisted variable values are
+      filtered into BitBake's datastore.
 
       .. note::
 
@@ -498,72 +382,21 @@ overview of their function and contents.
          queried using the normal datastore operations.
 
    :term:`BB_PRESERVE_ENV`
-      Disables environment filtering and instead allows all variables through
-      from the external environment into BitBake's datastore.
+      Disables whitelisting and instead allows all variables through from
+      the external environment into BitBake's datastore.
 
       .. note::
 
          You must set this variable in the external environment in order
          for it to work.
 
-   :term:`BB_PRESSURE_MAX_CPU`
-      Specifies a maximum CPU pressure threshold, above which BitBake's
-      scheduler will not start new tasks (providing there is at least
-      one active task). If no value is set, CPU pressure is not
-      monitored when starting tasks.
-
-      The pressure data is calculated based upon what Linux kernels since
-      version 4.20 expose under ``/proc/pressure``. The threshold represents
-      the difference in "total" pressure from the previous second. The
-      minimum value is 1.0 (extremely slow builds) and the maximum is
-      1000000 (a pressure value unlikely to ever be reached).
-
-      This threshold can be set in ``conf/local.conf`` as::
-
-         BB_PRESSURE_MAX_CPU = "500"
-
-   :term:`BB_PRESSURE_MAX_IO`
-      Specifies a maximum I/O pressure threshold, above which BitBake's
-      scheduler will not start new tasks (providing there is at least
-      one active task). If no value is set, I/O pressure is not
-      monitored when starting tasks.
-
-      The pressure data is calculated based upon what Linux kernels since
-      version 4.20 expose under ``/proc/pressure``. The threshold represents
-      the difference in "total" pressure from the previous second. The
-      minimum value is 1.0 (extremely slow builds) and the maximum is
-      1000000 (a pressure value unlikely to ever be reached).
-
-      At this point in time, experiments show that IO pressure tends to
-      be short-lived and regulating just the CPU with
-      :term:`BB_PRESSURE_MAX_CPU` can help to reduce it.
-
-   :term:`BB_PRESSURE_MAX_MEMORY`
-
-      Specifies a maximum memory pressure threshold, above which BitBake's
-      scheduler will not start new tasks (providing there is at least
-      one active task). If no value is set, memory pressure is not
-      monitored when starting tasks.
-
-      The pressure data is calculated based upon what Linux kernels since
-      version 4.20 expose under ``/proc/pressure``. The threshold represents
-      the difference in "total" pressure from the previous second. The
-      minimum value is 1.0 (extremely slow builds) and the maximum is
-      1000000 (a pressure value unlikely to ever be reached).
-
-      Memory pressure is experienced when time is spent swapping,
-      refaulting pages from the page cache or performing direct reclaim.
-      This is why memory pressure is rarely seen, but setting this variable
-      might be useful as a last resort to prevent OOM errors if they are
-      occurring during builds.
-
    :term:`BB_RUNFMT`
       Specifies the name of the executable script files (i.e. run files)
       saved into ``${``\ :term:`T`\ ``}``. By default, the
       :term:`BB_RUNFMT` variable is undefined and the run filenames get
       created using the following form::
 
-         run.{func}.{pid}
+         run.{task}.{pid}
 
       If you want to force run files to take a specific name, you can set this
       variable in a configuration file.
@@ -577,14 +410,14 @@ overview of their function and contents.
       Selects the name of the scheduler to use for the scheduling of
       BitBake tasks. Three options exist:
 
-      -  *basic* --- the basic framework from which everything derives. Using
+      -  *basic* - The basic framework from which everything derives. Using
          this option causes tasks to be ordered numerically as they are
          parsed.
 
-      -  *speed* --- executes tasks first that have more tasks depending on
+      -  *speed* - Executes tasks first that have more tasks depending on
          them. The "speed" option is the default.
 
-      -  *completion* --- causes the scheduler to try to complete a given
+      -  *completion* - Causes the scheduler to try to complete a given
          recipe once its build has started.
 
    :term:`BB_SCHEDULERS`
@@ -631,12 +464,35 @@ overview of their function and contents.
 
       The variable can be set using one of two policies:
 
-      -  *cache* --- retains the value the system obtained previously rather
+      -  *cache* - Retains the value the system obtained previously rather
          than querying the source control system each time.
 
-      -  *clear* --- queries the source controls system every time. With this
+      -  *clear* - Queries the source controls system every time. With this
          policy, there is no cache. The "clear" policy is the default.
 
+   :term:`BB_STAMP_POLICY`
+      Defines the mode used for how timestamps of stamp files are compared.
+      You can set the variable to one of the following modes:
+
+      -  *perfile* - Timestamp comparisons are only made between timestamps
+         of a specific recipe. This is the default mode.
+
+      -  *full* - Timestamp comparisons are made for all dependencies.
+
+      -  *whitelist* - Identical to "full" mode except timestamp
+         comparisons are made for recipes listed in the
+         :term:`BB_STAMP_WHITELIST` variable.
+
+      .. note::
+
+         Stamp policies are largely obsolete with the introduction of
+         setscene tasks.
+
+   :term:`BB_STAMP_WHITELIST`
+      Lists files whose stamp file timestamps are compared when the stamp
+      policy mode is set to "whitelist". For information on stamp policies,
+      see the :term:`BB_STAMP_POLICY` variable.
+
    :term:`BB_STRICT_CHECKSUM`
       Sets a more strict checksum mechanism for non-local URLs. Setting
       this variable to a value causes BitBake to report an error if it
@@ -760,7 +616,7 @@ overview of their function and contents.
       This variable is useful in situations where the same recipe appears
       in more than one layer. Setting this variable allows you to
       prioritize a layer against other layers that contain the same recipe
-      --- effectively letting you control the precedence for the multiple
+      - effectively letting you control the precedence for the multiple
       layers. The precedence established through this variable stands
       regardless of a recipe's version (:term:`PV` variable).
       For example, a layer that has a recipe with a higher :term:`PV` value but
@@ -822,7 +678,7 @@ overview of their function and contents.
          "
 
       This next example shows an error message that occurs because invalid
-      entries are found, which cause parsing to fail::
+      entries are found, which cause parsing to abort::
 
          ERROR: BBFILES_DYNAMIC entries must be of the form {!}<collection name>:<filename pattern>, not:
          /work/my-layer/bbappends/meta-security-isafw/*/*/*.bbappend
@@ -920,9 +776,9 @@ overview of their function and contents.
       section.
 
    :term:`BBPATH`
-      A colon-separated list used by BitBake to locate class (``.bbclass``)
-      and configuration (``.conf``) files. This variable is analogous to the
-      ``PATH`` variable.
+      Used by BitBake to locate class (``.bbclass``) and configuration
+      (``.conf``) files. This variable is analogous to the ``PATH``
+      variable.
 
       If you run BitBake from a directory outside of the build directory,
       you must be sure to set :term:`BBPATH` to point to the build directory.
@@ -1014,7 +870,7 @@ overview of their function and contents.
       ``bblayers.conf`` configuration file.
 
       To exclude a recipe from a world build using this variable, set the
-      variable to "1" in the recipe. Set it to "0" to add it back to world build.
+      variable to "1" in the recipe.
 
       .. note::
 
@@ -1072,11 +928,6 @@ overview of their function and contents.
       environment variable. The value is a colon-separated list of
       directories that are searched left-to-right in order.
 
-   :term:`FILE_LAYERNAME`
-      During parsing and task execution, this is set to the name of the
-      layer containing the recipe file. Code can use this to identify which
-      layer a recipe is from.
-
    :term:`GITDIR`
       The directory in which a local copy of a Git repository is stored
       when it is cloned.
@@ -1125,29 +976,6 @@ overview of their function and contents.
       variable is not available outside of ``layer.conf`` and references
       are expanded immediately when parsing of the file completes.
 
-   :term:`LAYERSERIES_COMPAT`
-      Lists the versions of the OpenEmbedded-Core (OE-Core) for which
-      a layer is compatible. Using the :term:`LAYERSERIES_COMPAT` variable
-      allows the layer maintainer to indicate which combinations of the
-      layer and OE-Core can be expected to work. The variable gives the
-      system a way to detect when a layer has not been tested with new
-      releases of OE-Core (e.g. the layer is not maintained).
-
-      To specify the OE-Core versions for which a layer is compatible, use
-      this variable in your layer's ``conf/layer.conf`` configuration file.
-      For the list, use the Yocto Project release name (e.g. "kirkstone",
-      "mickledore"). To specify multiple OE-Core versions for the layer, use
-      a space-separated list::
-
-         LAYERSERIES_COMPAT_layer_root_name = "kirkstone mickledore"
-
-      .. note::
-
-         Setting :term:`LAYERSERIES_COMPAT` is required by the Yocto Project
-         Compatible version 2 standard.
-         The OpenEmbedded build system produces a warning if the variable
-         is not set for any given layer.
-
    :term:`LAYERVERSION`
       Optionally specifies the version of a layer as a single number. You
       can use this variable within
@@ -1169,9 +997,22 @@ overview of their function and contents.
       upstream source, and then locations specified by :term:`MIRRORS` in that
       order.
 
+   :term:`MULTI_PROVIDER_WHITELIST`
+      Allows you to suppress BitBake warnings caused when building two
+      separate recipes that provide the same output.
+
+      BitBake normally issues a warning when building two different recipes
+      where each provides the same output. This scenario is usually
+      something the user does not want. However, cases do exist where it
+      makes sense, particularly in the ``virtual/*`` namespace. You can use
+      this variable to suppress BitBake's warnings.
+
+      To use the variable, list provider names (e.g. recipe names,
+      ``virtual/kernel``, and so forth).
+
    :term:`OVERRIDES`
-      A colon-separated list that BitBake uses to control what variables are
-      overridden after BitBake parses recipes and configuration files.
+      BitBake uses :term:`OVERRIDES` to control what variables are overridden
+      after BitBake parses recipes and configuration files.
 
       Following is a simple example that uses an overrides list based on
       machine architectures: OVERRIDES = "arm:x86:mips:powerpc" You can
@@ -1282,10 +1123,10 @@ overview of their function and contents.
       your configuration::
 
          PREMIRRORS:prepend = "\
-         git://.*/.* http://downloads.yoctoproject.org/mirror/sources/ \
-         ftp://.*/.* http://downloads.yoctoproject.org/mirror/sources/ \
-         http://.*/.* http://downloads.yoctoproject.org/mirror/sources/ \
-         https://.*/.* http://downloads.yoctoproject.org/mirror/sources/"
+         git://.*/.* http://www.yoctoproject.org/sources/ \n \
+         ftp://.*/.* http://www.yoctoproject.org/sources/ \n \
+         http://.*/.* http://www.yoctoproject.org/sources/ \n \
+         https://.*/.* http://www.yoctoproject.org/sources/ \n"
 
       These changes cause the build system to intercept Git, FTP, HTTP, and
       HTTPS requests and direct them to the ``http://`` sources mirror. You can
@@ -1433,106 +1274,70 @@ overview of their function and contents.
       The section in which packages should be categorized.
 
    :term:`SRC_URI`
-      The list of source files --- local or remote. This variable tells
+      The list of source files - local or remote. This variable tells
       BitBake which bits to pull for the build and how to pull them. For
       example, if the recipe or append file needs to fetch a single tarball
-      from the Internet, the recipe or append file uses a :term:`SRC_URI`
-      entry that specifies that tarball. On the other hand, if the recipe or
-      append file needs to fetch a tarball, apply two patches, and include
-      a custom file, the recipe or append file needs an :term:`SRC_URI`
-      variable that specifies all those sources.
+      from the Internet, the recipe or append file uses a :term:`SRC_URI` entry
+      that specifies that tarball. On the other hand, if the recipe or
+      append file needs to fetch a tarball and include a custom file, the
+      recipe or append file needs an :term:`SRC_URI` variable that specifies
+      all those sources.
 
-      The following list explains the available URI protocols. URI
-      protocols are highly dependent on particular BitBake Fetcher
-      submodules. Depending on the fetcher BitBake uses, various URL
-      parameters are employed. For specifics on the supported Fetchers, see
-      the :ref:`bitbake-user-manual/bitbake-user-manual-fetching:fetchers`
-      section.
+      The following list explains the available URI protocols:
 
-      -  ``az://``: Fetches files from an Azure Storage account using HTTPS.
+      -  ``file://`` : Fetches files, which are usually files shipped
+         with the metadata, from the local machine. The path is relative to
+         the :term:`FILESPATH` variable.
 
-      -  ``bzr://``: Fetches files from a Bazaar revision control
+      -  ``bzr://`` : Fetches files from a Bazaar revision control
          repository.
 
-      -  ``ccrc://``: Fetches files from a ClearCase repository.
-
-      -  ``cvs://``: Fetches files from a CVS revision control
+      -  ``git://`` : Fetches files from a Git revision control
          repository.
 
-      -  ``file://``: Fetches files, which are usually files shipped
-         with the Metadata, from the local machine.
-         The path is relative to the :term:`FILESPATH`
-         variable. Thus, the build system searches, in order, from the
-         following directories, which are assumed to be a subdirectories of
-         the directory in which the recipe file (``.bb``) or append file
-         (``.bbappend``) resides:
-
-         -  ``${BPN}``: the base recipe name without any special suffix
-            or version numbers.
-
-         -  ``${BP}`` - ``${BPN}-${PV}``: the base recipe name and
-            version but without any special package name suffix.
-
-         -  ``files``: files within a directory, which is named ``files``
-            and is also alongside the recipe or append file.
-
-      -  ``ftp://``: Fetches files from the Internet using FTP.
-
-      -  ``git://``: Fetches files from a Git revision control
-         repository.
-
-      -  ``gitsm://``: Fetches submodules from a Git revision control
-         repository.
-
-      -  ``hg://``: Fetches files from a Mercurial (``hg``) revision
-         control repository.
-
-      -  ``http://``: Fetches files from the Internet using HTTP.
-
-      -  ``https://``: Fetches files from the Internet using HTTPS.
-
-      -  ``npm://``: Fetches JavaScript modules from a registry.
-
-      -  ``osc://``: Fetches files from an OSC (OpenSUSE Build service)
+      -  ``osc://`` : Fetches files from an OSC (OpenSUSE Build service)
          revision control repository.
 
-      -  ``p4://``: Fetches files from a Perforce (``p4``) revision
+      -  ``repo://`` : Fetches files from a repo (Git) repository.
+
+      -  ``http://`` : Fetches files from the Internet using HTTP.
+
+      -  ``https://`` : Fetches files from the Internet using HTTPS.
+
+      -  ``ftp://`` : Fetches files from the Internet using FTP.
+
+      -  ``cvs://`` : Fetches files from a CVS revision control
+         repository.
+
+      -  ``hg://`` : Fetches files from a Mercurial (``hg``) revision
          control repository.
 
-      -  ``repo://``: Fetches files from a repo (Git) repository.
-
-      -  ``ssh://``: Fetches files from a secure shell.
-
-      -  ``svn://``: Fetches files from a Subversion (``svn``) revision
+      -  ``p4://`` : Fetches files from a Perforce (``p4``) revision
          control repository.
 
+      -  ``ssh://`` : Fetches files from a secure shell.
+
+      -  ``svn://`` : Fetches files from a Subversion (``svn``) revision
+         control repository.
+
+      -  ``az://`` : Fetches files from an Azure Storage account using HTTPS.
+
       Here are some additional options worth mentioning:
 
-      -  ``downloadfilename``: Specifies the filename used when storing
-         the downloaded file.
+      -  ``unpack`` : Controls whether or not to unpack the file if it is
+         an archive. The default action is to unpack the file.
 
-      -  ``name``: Specifies a name to be used for association with
-         :term:`SRC_URI` checksums or :term:`SRCREV` when you have more than one
-         file or git repository specified in :term:`SRC_URI`. For example::
-
-            SRC_URI = "git://example.com/foo.git;branch=main;name=first \
-                       git://example.com/bar.git;branch=main;name=second \
-                       http://example.com/file.tar.gz;name=third"
-
-            SRCREV_first = "f1d2d2f924e986ac86fdf7b36c94bcdf32beec15"
-            SRCREV_second = "e242ed3bffccdf271b7fbaf34ed72d089537b42f"
-            SRC_URI[third.sha256sum] = "13550350a8681c84c861aac2e5b440161c2b33a3e4f302ac680ca5b686de48de"
-
-      -  ``subdir``: Places the file (or extracts its contents) into the
+      -  ``subdir`` : Places the file (or extracts its contents) into the
          specified subdirectory. This option is useful for unusual tarballs
          or other archives that do not have their files already in a
          subdirectory within the archive.
 
-      -  ``subpath``: Limits the checkout to a specific subpath of the
-         tree when using the Git fetcher is used.
+      -  ``name`` : Specifies a name to be used for association with
+         :term:`SRC_URI` checksums when you have more than one file specified
+         in :term:`SRC_URI`.
 
-      -  ``unpack``: Controls whether or not to unpack the file if it is
-         an archive. The default action is to unpack the file.
+      -  ``downloadfilename`` : Specifies the filename used when storing
+         the downloaded file.
 
    :term:`SRCDATE`
       The date of the source code used to build the package. This variable

bitbake/doc/releases.rst

@@ -1,76 +1,32 @@
 .. SPDX-License-Identifier: CC-BY-2.5
 
-=================================
-BitBake Supported Release Manuals
-=================================
-
-*******************************
-Release Series 4.2 (mickledore)
-*******************************
-
-- :yocto_docs:`BitBake 2.4 User Manual </bitbake/2.4/>`
-
-******************************
-Release Series 4.0 (kirkstone)
-******************************
-
-- :yocto_docs:`BitBake 2.0 User Manual </bitbake/2.0/>`
+=========================
+ Current Release Manuals
+=========================
 
 ****************************
-Release Series 3.1 (dunfell)
+3.1 'dunfell' Release Series
 ****************************
 
-- :yocto_docs:`BitBake 1.46 User Manual </bitbake/1.46/>`
-
-================================
-BitBake Outdated Release Manuals
-================================
-
-*****************************
-Release Series 4.1 (langdale)
-*****************************
-
-- :yocto_docs:`BitBake 2.2 User Manual </bitbake/2.2/>`
-
-******************************
-Release Series 3.4 (honister)
-******************************
-
-- :yocto_docs:`BitBake 1.52 User Manual </bitbake/1.52/>`
-
-******************************
-Release Series 3.3 (hardknott)
-******************************
-
-- :yocto_docs:`BitBake 1.50 User Manual </bitbake/1.50/>`
-
-*******************************
-Release Series 3.2 (gatesgarth)
-*******************************
-
-- :yocto_docs:`BitBake 1.48 User Manual </bitbake/1.48/>`
-
-*******************************************
-Release Series 3.1 (dunfell first versions)
-*******************************************
-
 - :yocto_docs:`3.1 BitBake User Manual </3.1/bitbake-user-manual/bitbake-user-manual.html>`
 - :yocto_docs:`3.1.1 BitBake User Manual </3.1.1/bitbake-user-manual/bitbake-user-manual.html>`
 - :yocto_docs:`3.1.2 BitBake User Manual </3.1.2/bitbake-user-manual/bitbake-user-manual.html>`
-- :yocto_docs:`3.1.3 BitBake User Manual </3.1.3/bitbake-user-manual/bitbake-user-manual.html>`
+
+==========================
+ Previous Release Manuals
+==========================
 
 *************************
-Release Series 3.0 (zeus)
+3.0 'zeus' Release Series
 *************************
 
 - :yocto_docs:`3.0 BitBake User Manual </3.0/bitbake-user-manual/bitbake-user-manual.html>`
 - :yocto_docs:`3.0.1 BitBake User Manual </3.0.1/bitbake-user-manual/bitbake-user-manual.html>`
 - :yocto_docs:`3.0.2 BitBake User Manual </3.0.2/bitbake-user-manual/bitbake-user-manual.html>`
 - :yocto_docs:`3.0.3 BitBake User Manual </3.0.3/bitbake-user-manual/bitbake-user-manual.html>`
-- :yocto_docs:`3.0.4 BitBake User Manual </3.0.4/bitbake-user-manual/bitbake-user-manual.html>`
 
 ****************************
-Release Series 2.7 (warrior)
+2.7 'warrior' Release Series
 ****************************
 
 - :yocto_docs:`2.7 BitBake User Manual </2.7/bitbake-user-manual/bitbake-user-manual.html>`
@@ -80,7 +36,7 @@ Release Series 2.7 (warrior)
 - :yocto_docs:`2.7.4 BitBake User Manual </2.7.4/bitbake-user-manual/bitbake-user-manual.html>`
 
 *************************
-Release Series 2.6 (thud)
+2.6 'thud' Release Series
 *************************
 
 - :yocto_docs:`2.6 BitBake User Manual </2.6/bitbake-user-manual/bitbake-user-manual.html>`
@@ -90,16 +46,16 @@ Release Series 2.6 (thud)
 - :yocto_docs:`2.6.4 BitBake User Manual </2.6.4/bitbake-user-manual/bitbake-user-manual.html>`
 
 *************************
-Release Series 2.5 (sumo)
+2.5 'sumo' Release Series
 *************************
 
-- :yocto_docs:`2.5 Documentation </2.5>`
-- :yocto_docs:`2.5.1 Documentation </2.5.1>`
-- :yocto_docs:`2.5.2 Documentation </2.5.2>`
-- :yocto_docs:`2.5.3 Documentation </2.5.3>`
+- :yocto_docs:`2.5 BitBake User Manual </2.5/bitbake-user-manual/bitbake-user-manual.html>`
+- :yocto_docs:`2.5.1 BitBake User Manual </2.5.1/bitbake-user-manual/bitbake-user-manual.html>`
+- :yocto_docs:`2.5.2 BitBake User Manual </2.5.2/bitbake-user-manual/bitbake-user-manual.html>`
+- :yocto_docs:`2.5.3 BitBake User Manual </2.5.3/bitbake-user-manual/bitbake-user-manual.html>`
 
 **************************
-Release Series 2.4 (rocko)
+2.4 'rocko' Release Series
 **************************
 
 - :yocto_docs:`2.4 BitBake User Manual </2.4/bitbake-user-manual/bitbake-user-manual.html>`
@@ -109,7 +65,7 @@ Release Series 2.4 (rocko)
 - :yocto_docs:`2.4.4 BitBake User Manual </2.4.4/bitbake-user-manual/bitbake-user-manual.html>`
 
 *************************
-Release Series 2.3 (pyro)
+2.3 'pyro' Release Series
 *************************
 
 - :yocto_docs:`2.3 BitBake User Manual </2.3/bitbake-user-manual/bitbake-user-manual.html>`
@@ -119,7 +75,7 @@ Release Series 2.3 (pyro)
 - :yocto_docs:`2.3.4 BitBake User Manual </2.3.4/bitbake-user-manual/bitbake-user-manual.html>`
 
 **************************
-Release Series 2.2 (morty)
+2.2 'morty' Release Series
 **************************
 
 - :yocto_docs:`2.2 BitBake User Manual </2.2/bitbake-user-manual/bitbake-user-manual.html>`
@@ -128,7 +84,7 @@ Release Series 2.2 (morty)
 - :yocto_docs:`2.2.3 BitBake User Manual </2.2.3/bitbake-user-manual/bitbake-user-manual.html>`
 
 ****************************
-Release Series 2.1 (krogoth)
+2.1 'krogoth' Release Series
 ****************************
 
 - :yocto_docs:`2.1 BitBake User Manual </2.1/bitbake-user-manual/bitbake-user-manual.html>`
@@ -137,7 +93,7 @@ Release Series 2.1 (krogoth)
 - :yocto_docs:`2.1.3 BitBake User Manual </2.1.3/bitbake-user-manual/bitbake-user-manual.html>`
 
 ***************************
-Release Series 2.0 (jethro)
+2.0 'jethro' Release Series
 ***************************
 
 - :yocto_docs:`1.9 BitBake User Manual </1.9/bitbake-user-manual/bitbake-user-manual.html>`
@@ -147,7 +103,7 @@ Release Series 2.0 (jethro)
 - :yocto_docs:`2.0.3 BitBake User Manual </2.0.3/bitbake-user-manual/bitbake-user-manual.html>`
 
 *************************
-Release Series 1.8 (fido)
+1.8 'fido' Release Series
 *************************
 
 - :yocto_docs:`1.8 BitBake User Manual </1.8/bitbake-user-manual/bitbake-user-manual.html>`
@@ -155,7 +111,7 @@ Release Series 1.8 (fido)
 - :yocto_docs:`1.8.2 BitBake User Manual </1.8.2/bitbake-user-manual/bitbake-user-manual.html>`
 
 **************************
-Release Series 1.7 (dizzy)
+1.7 'dizzy' Release Series
 **************************
 
 - :yocto_docs:`1.7 BitBake User Manual </1.7/bitbake-user-manual/bitbake-user-manual.html>`
@@ -164,7 +120,7 @@ Release Series 1.7 (dizzy)
 - :yocto_docs:`1.7.3 BitBake User Manual </1.7.3/bitbake-user-manual/bitbake-user-manual.html>`
 
 **************************
-Release Series 1.6 (daisy)
+1.6 'daisy' Release Series
 **************************
 
 - :yocto_docs:`1.6 BitBake User Manual </1.6/bitbake-user-manual/bitbake-user-manual.html>`

bitbake/lib/bb/COW.py

@@ -3,8 +3,6 @@
 #
 # Copyright (C) 2006 Tim Ansell
 #
-# SPDX-License-Identifier: GPL-2.0-only
-#
 # Please Note:
 # Be careful when using mutable types (ie Dict and Lists) - operations involving these are SLOW.
 # Assign a file to __warn__ to get warnings about slow operations.

bitbake/lib/bb/__init__.py

@@ -9,11 +9,11 @@
 # SPDX-License-Identifier: GPL-2.0-only
 #
 
-__version__ = "2.6.0"
+__version__ = "1.52.0"
 
 import sys
-if sys.version_info < (3, 8, 0):
-    raise RuntimeError("Sorry, python 3.8.0 or later is required for this version of bitbake")
+if sys.version_info < (3, 6, 0):
+    raise RuntimeError("Sorry, python 3.6.0 or later is required for this version of bitbake")
 
 
 class BBHandledException(Exception):
@@ -60,10 +60,6 @@ class BBLoggerMixin(object):
                 return
             if loglevel < bb.msg.loggerDefaultLogLevel:
                 return
-
-        if not isinstance(level, int) or not isinstance(msg, str):
-            mainlogger.warning("Invalid arguments in bbdebug: %s" % repr((level, msg,) + args))
-
         return self.log(loglevel, msg, *args, **kwargs)
 
     def plain(self, msg, *args, **kwargs):
@@ -75,13 +71,6 @@ class BBLoggerMixin(object):
     def verbnote(self, msg, *args, **kwargs):
         return self.log(logging.INFO + 2, msg, *args, **kwargs)
 
-    def warnonce(self, msg, *args, **kwargs):
-        return self.log(logging.WARNING - 1, msg, *args, **kwargs)
-
-    def erroronce(self, msg, *args, **kwargs):
-        return self.log(logging.ERROR - 1, msg, *args, **kwargs)
-
-
 Logger = logging.getLoggerClass()
 class BBLogger(Logger, BBLoggerMixin):
     def __init__(self, name, *args, **kwargs):
@@ -168,15 +157,9 @@ def verbnote(*args):
 def warn(*args):
     mainlogger.warning(''.join(args))
 
-def warnonce(*args):
-    mainlogger.warnonce(''.join(args))
-
 def error(*args, **kwargs):
     mainlogger.error(''.join(args), extra=kwargs)
 
-def erroronce(*args):
-    mainlogger.erroronce(''.join(args))
-
 def fatal(*args, **kwargs):
     mainlogger.critical(''.join(args), extra=kwargs)
     raise BBHandledException()

bitbake/lib/bb/acl.py

@@ -1,215 +0,0 @@
-#! /usr/bin/env python3
-#
-# Copyright 2023 by Garmin Ltd. or its subsidiaries
-#
-# SPDX-License-Identifier: MIT
-
-
-import sys
-import ctypes
-import os
-import errno
-import pwd
-import grp
-
-libacl = ctypes.CDLL("libacl.so.1", use_errno=True)
-
-
-ACL_TYPE_ACCESS = 0x8000
-ACL_TYPE_DEFAULT = 0x4000
-
-ACL_FIRST_ENTRY = 0
-ACL_NEXT_ENTRY = 1
-
-ACL_UNDEFINED_TAG = 0x00
-ACL_USER_OBJ = 0x01
-ACL_USER = 0x02
-ACL_GROUP_OBJ = 0x04
-ACL_GROUP = 0x08
-ACL_MASK = 0x10
-ACL_OTHER = 0x20
-
-ACL_READ = 0x04
-ACL_WRITE = 0x02
-ACL_EXECUTE = 0x01
-
-acl_t = ctypes.c_void_p
-acl_entry_t = ctypes.c_void_p
-acl_permset_t = ctypes.c_void_p
-acl_perm_t = ctypes.c_uint
-
-acl_tag_t = ctypes.c_int
-
-libacl.acl_free.argtypes = [acl_t]
-
-
-def acl_free(acl):
-    libacl.acl_free(acl)
-
-
-libacl.acl_get_file.restype = acl_t
-libacl.acl_get_file.argtypes = [ctypes.c_char_p, ctypes.c_uint]
-
-
-def acl_get_file(path, typ):
-    acl = libacl.acl_get_file(os.fsencode(path), typ)
-    if acl is None:
-        err = ctypes.get_errno()
-        raise OSError(err, os.strerror(err), str(path))
-
-    return acl
-
-
-libacl.acl_get_entry.argtypes = [acl_t, ctypes.c_int, ctypes.c_void_p]
-
-
-def acl_get_entry(acl, entry_id):
-    entry = acl_entry_t()
-    ret = libacl.acl_get_entry(acl, entry_id, ctypes.byref(entry))
-    if ret < 0:
-        err = ctypes.get_errno()
-        raise OSError(err, os.strerror(err))
-
-    if ret == 0:
-        return None
-
-    return entry
-
-
-libacl.acl_get_tag_type.argtypes = [acl_entry_t, ctypes.c_void_p]
-
-
-def acl_get_tag_type(entry_d):
-    tag = acl_tag_t()
-    ret = libacl.acl_get_tag_type(entry_d, ctypes.byref(tag))
-    if ret < 0:
-        err = ctypes.get_errno()
-        raise OSError(err, os.strerror(err))
-    return tag.value
-
-
-libacl.acl_get_qualifier.restype = ctypes.c_void_p
-libacl.acl_get_qualifier.argtypes = [acl_entry_t]
-
-
-def acl_get_qualifier(entry_d):
-    ret = libacl.acl_get_qualifier(entry_d)
-    if ret is None:
-        err = ctypes.get_errno()
-        raise OSError(err, os.strerror(err))
-    return ctypes.c_void_p(ret)
-
-
-libacl.acl_get_permset.argtypes = [acl_entry_t, ctypes.c_void_p]
-
-
-def acl_get_permset(entry_d):
-    permset = acl_permset_t()
-    ret = libacl.acl_get_permset(entry_d, ctypes.byref(permset))
-    if ret < 0:
-        err = ctypes.get_errno()
-        raise OSError(err, os.strerror(err))
-
-    return permset
-
-
-libacl.acl_get_perm.argtypes = [acl_permset_t, acl_perm_t]
-
-
-def acl_get_perm(permset_d, perm):
-    ret = libacl.acl_get_perm(permset_d, perm)
-    if ret < 0:
-        err = ctypes.get_errno()
-        raise OSError(err, os.strerror(err))
-    return bool(ret)
-
-
-class Entry(object):
-    def __init__(self, tag, qualifier, mode):
-        self.tag = tag
-        self.qualifier = qualifier
-        self.mode = mode
-
-    def __str__(self):
-        typ = ""
-        qual = ""
-        if self.tag == ACL_USER:
-            typ = "user"
-            qual = pwd.getpwuid(self.qualifier).pw_name
-        elif self.tag == ACL_GROUP:
-            typ = "group"
-            qual = grp.getgrgid(self.qualifier).gr_name
-        elif self.tag == ACL_USER_OBJ:
-            typ = "user"
-        elif self.tag == ACL_GROUP_OBJ:
-            typ = "group"
-        elif self.tag == ACL_MASK:
-            typ = "mask"
-        elif self.tag == ACL_OTHER:
-            typ = "other"
-
-        r = "r" if self.mode & ACL_READ else "-"
-        w = "w" if self.mode & ACL_WRITE else "-"
-        x = "x" if self.mode & ACL_EXECUTE else "-"
-
-        return f"{typ}:{qual}:{r}{w}{x}"
-
-
-class ACL(object):
-    def __init__(self, acl):
-        self.acl = acl
-
-    def __del__(self):
-        acl_free(self.acl)
-
-    def entries(self):
-        entry_id = ACL_FIRST_ENTRY
-        while True:
-            entry = acl_get_entry(self.acl, entry_id)
-            if entry is None:
-                break
-
-            permset = acl_get_permset(entry)
-
-            mode = 0
-            for m in (ACL_READ, ACL_WRITE, ACL_EXECUTE):
-                if acl_get_perm(permset, m):
-                    mode |= m
-
-            qualifier = None
-            tag = acl_get_tag_type(entry)
-
-            if tag == ACL_USER or tag == ACL_GROUP:
-                qual = acl_get_qualifier(entry)
-                qualifier = ctypes.cast(qual, ctypes.POINTER(ctypes.c_int))[0]
-
-            yield Entry(tag, qualifier, mode)
-
-            entry_id = ACL_NEXT_ENTRY
-
-    @classmethod
-    def from_path(cls, path, typ):
-        acl = acl_get_file(path, typ)
-        return cls(acl)
-
-
-def main():
-    import argparse
-    import pwd
-    import grp
-    from pathlib import Path
-
-    parser = argparse.ArgumentParser()
-    parser.add_argument("path", help="File Path", type=Path)
-
-    args = parser.parse_args()
-
-    acl = ACL.from_path(args.path, ACL_TYPE_ACCESS)
-    for entry in acl.entries():
-        print(str(entry))
-
-    return 0
-
-
-if __name__ == "__main__":
-    sys.exit(main())

bitbake/lib/bb/asyncrpc/__init__.py

@@ -1,6 +1,4 @@
 #
-# Copyright BitBake Contributors
-#
 # SPDX-License-Identifier: GPL-2.0-only
 #
 
@@ -30,4 +28,4 @@ def chunkify(msg, max_chunk):
 
 
 from .client import AsyncClient, Client
-from .serv import AsyncServer, AsyncServerConnection, ClientError, ServerError
+from .serv import AsyncServer, AsyncServerConnection

bitbake/lib/bb/asyncrpc/client.py

@@ -1,6 +1,4 @@
 #
-# Copyright BitBake Contributors
-#
 # SPDX-License-Identifier: GPL-2.0-only
 #
 
@@ -31,17 +29,7 @@ class AsyncClient(object):
 
     async def connect_unix(self, path):
         async def connect_sock():
-            # AF_UNIX has path length issues so chdir here to workaround
-            cwd = os.getcwd()
-            try:
-                os.chdir(os.path.dirname(path))
-                # The socket must be opened synchronously so that CWD doesn't get
-                # changed out from underneath us so we pass as a sock into asyncio
-                sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM, 0)
-                sock.connect(os.path.basename(path))
-            finally:
-               os.chdir(cwd)
-            return await asyncio.open_unix_connection(sock=sock)
+            return await asyncio.open_unix_connection(path)
 
         self._connect_sock = connect_sock
 
@@ -160,8 +148,14 @@ class Client(object):
             setattr(self, m, self._get_downcall_wrapper(downcall))
 
     def connect_unix(self, path):
-        self.loop.run_until_complete(self.client.connect_unix(path))
-        self.loop.run_until_complete(self.client.connect())
+        # AF_UNIX has path length issues so chdir here to workaround
+        cwd = os.getcwd()
+        try:
+            os.chdir(os.path.dirname(path))
+            self.loop.run_until_complete(self.client.connect_unix(os.path.basename(path)))
+            self.loop.run_until_complete(self.client.connect())
+        finally:
+            os.chdir(cwd)
 
     @property
     def max_chunk(self):

bitbake/lib/bb/asyncrpc/serv.py

@@ -1,6 +1,4 @@
 #
-# Copyright BitBake Contributors
-#
 # SPDX-License-Identifier: GPL-2.0-only
 #
 
@@ -42,7 +40,7 @@ class AsyncServerConnection(object):
 
             # Read protocol and version
             client_protocol = await self.reader.readline()
-            if not client_protocol:
+            if client_protocol is None:
                 return
 
             (client_proto_name, client_proto_version) = client_protocol.decode('utf-8').rstrip().split()
@@ -59,7 +57,7 @@ class AsyncServerConnection(object):
             # an empty line to signal the end of the headers
             while True:
                 line = await self.reader.readline()
-                if not line:
+                if line is None:
                     return
 
                 line = line.decode('utf-8').rstrip()
@@ -153,13 +151,6 @@ class AsyncServer(object):
                 s.setsockopt(socket.SOL_TCP, socket.TCP_NODELAY, 1)
                 s.setsockopt(socket.SOL_TCP, socket.TCP_QUICKACK, 1)
 
-                # Enable keep alives. This prevents broken client connections
-                # from persisting on the server for long periods of time.
-                s.setsockopt(socket.SOL_SOCKET, socket.SO_KEEPALIVE, 1)
-                s.setsockopt(socket.IPPROTO_TCP, socket.TCP_KEEPIDLE, 30)
-                s.setsockopt(socket.IPPROTO_TCP, socket.TCP_KEEPINTVL, 15)
-                s.setsockopt(socket.IPPROTO_TCP, socket.TCP_KEEPCNT, 4)
-
             name = self.server.sockets[0].getsockname()
             if self.server.sockets[0].family == socket.AF_INET6:
                 self.address = "[%s]:%d" % (name[0], name[1])

bitbake/lib/bb/build.py

@@ -20,12 +20,10 @@ import itertools
 import time
 import re
 import stat
-import datetime
 import bb
 import bb.msg
 import bb.process
 import bb.progress
-from io import StringIO
 from bb import data, event, utils
 
 bblogger = logging.getLogger('BitBake')
@@ -178,9 +176,7 @@ class StdoutNoopContextManager:
 
     @property
     def name(self):
-        if "name" in dir(sys.stdout):
-            return sys.stdout.name
-        return "<mem>"
+        return sys.stdout.name
 
 
 def exec_func(func, d, dirs = None):
@@ -299,21 +295,9 @@ def exec_func_python(func, d, runfile, cwd=None):
         lineno = int(d.getVarFlag(func, "lineno", False))
         bb.methodpool.insert_method(func, text, fn, lineno - 1)
 
-        if verboseStdoutLogging:
-            sys.stdout.flush()
-            sys.stderr.flush()
-            currout = sys.stdout
-            currerr = sys.stderr
-            sys.stderr = sys.stdout = execio = StringIO()
         comp = utils.better_compile(code, func, "exec_func_python() autogenerated")
         utils.better_exec(comp, {"d": d}, code, "exec_func_python() autogenerated")
     finally:
-        if verboseStdoutLogging:
-            execio.flush()
-            logger.plain("%s" % execio.getvalue())
-            sys.stdout = currout
-            sys.stderr = currerr
-            execio.close()
         # We want any stdout/stderr to be printed before any other log messages to make debugging
         # more accurate. In some cases we seem to lose stdout/stderr entirely in logging tests without this.
         sys.stdout.flush()
@@ -456,11 +440,7 @@ exit $ret
         if fakerootcmd:
             cmd = [fakerootcmd, runfile]
 
-    # We only want to output to logger via LogTee if stdout is sys.__stdout__ (which will either
-    # be real stdout or subprocess PIPE or similar). In other cases we are being run "recursively",
-    # ie. inside another function, in which case stdout is already being captured so we don't
-    # want to Tee here as output would be printed twice, and out of order.
-    if verboseStdoutLogging and sys.stdout == sys.__stdout__:
+    if verboseStdoutLogging:
         logfile = LogTee(logger, StdoutNoopContextManager())
     else:
         logfile = StdoutNoopContextManager()
@@ -589,8 +569,10 @@ exit $ret
 def _task_data(fn, task, d):
     localdata = bb.data.createCopy(d)
     localdata.setVar('BB_FILENAME', fn)
+    localdata.setVar('BB_CURRENTTASK', task[3:])
     localdata.setVar('OVERRIDES', 'task-%s:%s' %
                      (task[3:].replace('_', '-'), d.getVar('OVERRIDES', False)))
+    localdata.finalize()
     bb.data.expandKeys(localdata)
     return localdata
 
@@ -601,7 +583,7 @@ def _exec_task(fn, task, d, quieterr):
     running it with its own local metadata, and with some useful variables set.
     """
     if not d.getVarFlag(task, 'task', False):
-        event.fire(TaskInvalid(task, fn, d), d)
+        event.fire(TaskInvalid(task, d), d)
         logger.error("No such task: %s" % task)
         return 1
 
@@ -637,8 +619,7 @@ def _exec_task(fn, task, d, quieterr):
     logorder = os.path.join(tempdir, 'log.task_order')
     try:
         with open(logorder, 'a') as logorderfile:
-            timestamp = datetime.datetime.now().strftime("%Y%m%d-%H%M%S.%f")
-            logorderfile.write('{0} {1} ({2}): {3}\n'.format(timestamp, task, os.getpid(), logbase))
+            logorderfile.write('{0} ({1}): {2}\n'.format(task, os.getpid(), logbase))
     except OSError:
         logger.exception("Opening log file '%s'", logorder)
         pass
@@ -734,23 +715,19 @@ def _exec_task(fn, task, d, quieterr):
                 logger.debug2("Zero size logfn %s, removing", logfn)
                 bb.utils.remove(logfn)
                 bb.utils.remove(loglink)
+    except bb.BBHandledException:
+        event.fire(TaskFailed(task, fn, logfn, localdata, True), localdata)
+        return 1
     except (Exception, SystemExit) as exc:
-        handled = False
-        if isinstance(exc, bb.BBHandledException):
-            handled = True
-
         if quieterr:
-            if not handled:
-                logger.warning(repr(exc))
             event.fire(TaskFailedSilent(task, fn, logfn, localdata), localdata)
         else:
             errprinted = errchk.triggered
             # If the output is already on stdout, we've printed the information in the
             # logs once already so don't duplicate
-            if verboseStdoutLogging or handled:
+            if verboseStdoutLogging:
                 errprinted = True
-            if not handled:
-                logger.error(repr(exc))
+            logger.error(repr(exc))
             event.fire(TaskFailed(task, fn, logfn, localdata, errprinted), localdata)
         return 1
 
@@ -791,7 +768,44 @@ def exec_task(fn, task, d, profile = False):
             event.fire(failedevent, d)
         return 1
 
-def _get_cleanmask(taskname, mcfn):
+def stamp_internal(taskname, d, file_name, baseonly=False, noextra=False):
+    """
+    Internal stamp helper function
+    Makes sure the stamp directory exists
+    Returns the stamp path+filename
+
+    In the bitbake core, d can be a CacheData and file_name will be set.
+    When called in task context, d will be a data store, file_name will not be set
+    """
+    taskflagname = taskname
+    if taskname.endswith("_setscene") and taskname != "do_setscene":
+        taskflagname = taskname.replace("_setscene", "")
+
+    if file_name:
+        stamp = d.stamp[file_name]
+        extrainfo = d.stamp_extrainfo[file_name].get(taskflagname) or ""
+    else:
+        stamp = d.getVar('STAMP')
+        file_name = d.getVar('BB_FILENAME')
+        extrainfo = d.getVarFlag(taskflagname, 'stamp-extra-info') or ""
+
+    if baseonly:
+        return stamp
+    if noextra:
+        extrainfo = ""
+
+    if not stamp:
+        return
+
+    stamp = bb.parse.siggen.stampfile(stamp, file_name, taskname, extrainfo)
+
+    stampdir = os.path.dirname(stamp)
+    if cached_mtime_noerror(stampdir) == 0:
+        bb.utils.mkdirhier(stampdir)
+
+    return stamp
+
+def stamp_cleanmask_internal(taskname, d, file_name):
     """
     Internal stamp helper function to generate stamp cleaning mask
     Returns the stamp path+filename
@@ -799,14 +813,31 @@ def _get_cleanmask(taskname, mcfn):
     In the bitbake core, d can be a CacheData and file_name will be set.
     When called in task context, d will be a data store, file_name will not be set
     """
-    cleanmask = bb.parse.siggen.stampcleanmask_mcfn(taskname, mcfn)
-    taskflagname = taskname.replace("_setscene", "")
-    if cleanmask:
-        return [cleanmask, cleanmask.replace(taskflagname, taskflagname + "_setscene")]
-    return []
+    taskflagname = taskname
+    if taskname.endswith("_setscene") and taskname != "do_setscene":
+        taskflagname = taskname.replace("_setscene", "")
 
-def clean_stamp_mcfn(task, mcfn):
-    cleanmask = _get_cleanmask(task, mcfn)
+    if file_name:
+        stamp = d.stampclean[file_name]
+        extrainfo = d.stamp_extrainfo[file_name].get(taskflagname) or ""
+    else:
+        stamp = d.getVar('STAMPCLEAN')
+        file_name = d.getVar('BB_FILENAME')
+        extrainfo = d.getVarFlag(taskflagname, 'stamp-extra-info') or ""
+
+    if not stamp:
+        return []
+
+    cleanmask = bb.parse.siggen.stampcleanmask(stamp, file_name, taskname, extrainfo)
+
+    return [cleanmask, cleanmask.replace(taskflagname, taskflagname + "_setscene")]
+
+def make_stamp(task, d, file_name = None):
+    """
+    Creates/updates a stamp for a given task
+    (d can be a data dict or dataCache)
+    """
+    cleanmask = stamp_cleanmask_internal(task, d, file_name)
     for mask in cleanmask:
         for name in glob.glob(mask):
             # Preserve sigdata files in the stamps directory
@@ -817,45 +848,24 @@ def clean_stamp_mcfn(task, mcfn):
                 continue
             os.unlink(name)
 
-def clean_stamp(task, d):
-    mcfn = d.getVar('BB_FILENAME')
-    clean_stamp_mcfn(task, mcfn)
-
-def make_stamp_mcfn(task, mcfn):
-
-    basestamp = bb.parse.siggen.stampfile_mcfn(task, mcfn)
-
-    stampdir = os.path.dirname(basestamp)
-    if cached_mtime_noerror(stampdir) == 0:
-        bb.utils.mkdirhier(stampdir)
-
-    clean_stamp_mcfn(task, mcfn)
-
+    stamp = stamp_internal(task, d, file_name)
     # Remove the file and recreate to force timestamp
     # change on broken NFS filesystems
-    if basestamp:
-        bb.utils.remove(basestamp)
-        open(basestamp, "w").close()
-
-def make_stamp(task, d):
-    """
-    Creates/updates a stamp for a given task
-    """
-    mcfn = d.getVar('BB_FILENAME')
-
-    make_stamp_mcfn(task, mcfn)
+    if stamp:
+        bb.utils.remove(stamp)
+        open(stamp, "w").close()
 
     # If we're in task context, write out a signature file for each task
     # as it completes
-    if not task.endswith("_setscene"):
-        stampbase = bb.parse.siggen.stampfile_base(mcfn)
-        bb.parse.siggen.dump_sigtask(mcfn, task, stampbase, True)
+    if not task.endswith("_setscene") and task != "do_setscene" and not file_name:
+        stampbase = stamp_internal(task, d, None, True)
+        file_name = d.getVar('BB_FILENAME')
+        bb.parse.siggen.dump_sigtask(file_name, task, stampbase, True)
 
-
-def find_stale_stamps(task, mcfn):
-    current = bb.parse.siggen.stampfile_mcfn(task, mcfn)
-    current2 = bb.parse.siggen.stampfile_mcfn(task + "_setscene", mcfn)
-    cleanmask = _get_cleanmask(task, mcfn)
+def find_stale_stamps(task, d, file_name=None):
+    current = stamp_internal(task, d, file_name)
+    current2 = stamp_internal(task + "_setscene", d, file_name)
+    cleanmask = stamp_cleanmask_internal(task, d, file_name)
     found = []
     for mask in cleanmask:
         for name in glob.glob(mask):
@@ -869,14 +879,38 @@ def find_stale_stamps(task, mcfn):
             found.append(name)
     return found
 
-def write_taint(task, d):
+def del_stamp(task, d, file_name = None):
+    """
+    Removes a stamp for a given task
+    (d can be a data dict or dataCache)
+    """
+    stamp = stamp_internal(task, d, file_name)
+    bb.utils.remove(stamp)
+
+def write_taint(task, d, file_name = None):
     """
     Creates a "taint" file which will force the specified task and its
     dependents to be re-run the next time by influencing the value of its
     taskhash.
+    (d can be a data dict or dataCache)
     """
-    mcfn = d.getVar('BB_FILENAME')
-    bb.parse.siggen.invalidate_task(task, mcfn)
+    import uuid
+    if file_name:
+        taintfn = d.stamp[file_name] + '.' + task + '.taint'
+    else:
+        taintfn = d.getVar('STAMP') + '.' + task + '.taint'
+    bb.utils.mkdirhier(os.path.dirname(taintfn))
+    # The specific content of the taint file is not really important,
+    # we just need it to be random, so a random UUID is used
+    with open(taintfn, 'w') as taintf:
+        taintf.write(str(uuid.uuid4()))
+
+def stampfile(taskname, d, file_name = None, noextra=False):
+    """
+    Return the stamp for a given task
+    (d can be a data dict or dataCache)
+    """
+    return stamp_internal(taskname, d, file_name, noextra=noextra)
 
 def add_tasks(tasklist, d):
     task_deps = d.getVar('_task_deps', False)

bitbake/lib/bb/cache.py

@@ -24,11 +24,10 @@ from collections.abc import Mapping
 import bb.utils
 from bb import PrefixLoggerAdapter
 import re
-import shutil
 
 logger = logging.getLogger("BitBake.Cache")
 
-__cache_version__ = "155"
+__cache_version__ = "154"
 
 def getCacheFile(path, filename, mc, data_hash):
     mcspec = ''
@@ -105,7 +104,7 @@ class CoreRecipeInfo(RecipeInfoCommon):
 
         self.tasks = metadata.getVar('__BBTASKS', False)
 
-        self.basetaskhashes = metadata.getVar('__siggen_basehashes', False) or {}
+        self.basetaskhashes = self.taskvar('BB_BASEHASH', self.tasks, metadata)
         self.hashfilename = self.getvar('BB_HASHFILENAME', metadata)
 
         self.task_deps = metadata.getVar('_task_deps', False) or {'tasks': [], 'parents': {}}
@@ -216,7 +215,7 @@ class CoreRecipeInfo(RecipeInfoCommon):
 
         # Collect files we may need for possible world-dep
         # calculations
-        if not bb.utils.to_boolean(self.not_world):
+        if not self.not_world:
             cachedata.possible_world.append(fn)
         #else:
         #    logger.debug2("EXCLUDE FROM WORLD: %s", fn)
@@ -238,106 +237,6 @@ class CoreRecipeInfo(RecipeInfoCommon):
         cachedata.fakerootlogs[fn] = self.fakerootlogs
         cachedata.extradepsfunc[fn] = self.extradepsfunc
 
-
-class SiggenRecipeInfo(RecipeInfoCommon):
-    __slots__ = ()
-
-    classname = "SiggenRecipeInfo"
-    cachefile = "bb_cache_" + classname +".dat"
-    # we don't want to show this information in graph files so don't set cachefields
-    #cachefields = []
-
-    def __init__(self, filename, metadata):
-        self.siggen_gendeps = metadata.getVar("__siggen_gendeps", False)
-        self.siggen_varvals = metadata.getVar("__siggen_varvals", False)
-        self.siggen_taskdeps = metadata.getVar("__siggen_taskdeps", False)
-
-    @classmethod
-    def init_cacheData(cls, cachedata):
-        cachedata.siggen_taskdeps = {}
-        cachedata.siggen_gendeps = {}
-        cachedata.siggen_varvals = {}
-
-    def add_cacheData(self, cachedata, fn):
-        cachedata.siggen_gendeps[fn] = self.siggen_gendeps
-        cachedata.siggen_varvals[fn] = self.siggen_varvals
-        cachedata.siggen_taskdeps[fn] = self.siggen_taskdeps
-
-    # The siggen variable data is large and impacts:
-    #  - bitbake's overall memory usage
-    #  - the amount of data sent over IPC between parsing processes and the server
-    #  - the size of the cache files on disk
-    #  - the size of "sigdata" hash information files on disk
-    # The data consists of strings (some large) or frozenset lists of variables
-    # As such, we a) deplicate the data here and b) pass references to the object at second
-    # access (e.g. over IPC or saving into pickle).
-
-    store = {}
-    save_map = {}
-    save_count = 1
-    restore_map = {}
-    restore_count = {}
-
-    @classmethod
-    def reset(cls):
-        # Needs to be called before starting new streamed data in a given process 
-        # (e.g. writing out the cache again)
-        cls.save_map = {}
-        cls.save_count = 1
-        cls.restore_map = {}
-
-    @classmethod
-    def _save(cls, deps):
-        ret = []
-        if not deps:
-            return deps
-        for dep in deps:
-            fs = deps[dep]
-            if fs is None:
-                ret.append((dep, None, None))
-            elif fs in cls.save_map:
-                ret.append((dep, None, cls.save_map[fs]))
-            else:
-                cls.save_map[fs] = cls.save_count
-                ret.append((dep, fs, cls.save_count))
-                cls.save_count = cls.save_count + 1
-        return ret
-
-    @classmethod
-    def _restore(cls, deps, pid):
-        ret = {}
-        if not deps:
-            return deps
-        if pid not in cls.restore_map:
-            cls.restore_map[pid] = {}
-        map = cls.restore_map[pid]
-        for dep, fs, mapnum in deps:
-            if fs is None and mapnum is None:
-                ret[dep] = None
-            elif fs is None:
-                ret[dep] = map[mapnum]
-            else:
-                try:
-                    fs = cls.store[fs]
-                except KeyError:
-                    cls.store[fs] = fs
-                map[mapnum] = fs
-                ret[dep] = fs
-        return ret
-
-    def __getstate__(self):
-        ret = {}
-        for key in ["siggen_gendeps", "siggen_taskdeps", "siggen_varvals"]:
-            ret[key] = self._save(self.__dict__[key])
-        ret['pid'] = os.getpid()
-        return ret
-
-    def __setstate__(self, state):
-        pid = state['pid']
-        for key in ["siggen_gendeps", "siggen_taskdeps", "siggen_varvals"]:
-            setattr(self, key, self._restore(state[key], pid))
-
-
 def virtualfn2realfn(virtualfn):
     """
     Convert a virtual file name to a real one + the associated subclass keyword
@@ -380,18 +279,96 @@ def variant2virtual(realfn, variant):
         return "mc:" + elems[1] + ":" + realfn
     return "virtual:" + variant + ":" + realfn
 
-#
-# Cooker calls cacheValid on its recipe list, then either calls loadCached
-# from it's main thread or parse from separate processes to generate an up to
-# date cache
-#
-class Cache(object):
+def parse_recipe(bb_data, bbfile, appends, mc=''):
+    """
+    Parse a recipe
+    """
+
+    chdir_back = False
+
+    bb_data.setVar("__BBMULTICONFIG", mc)
+
+    # expand tmpdir to include this topdir
+    bb_data.setVar('TMPDIR', bb_data.getVar('TMPDIR') or "")
+    bbfile_loc = os.path.abspath(os.path.dirname(bbfile))
+    oldpath = os.path.abspath(os.getcwd())
+    bb.parse.cached_mtime_noerror(bbfile_loc)
+
+    # The ConfHandler first looks if there is a TOPDIR and if not
+    # then it would call getcwd().
+    # Previously, we chdir()ed to bbfile_loc, called the handler
+    # and finally chdir()ed back, a couple of thousand times. We now
+    # just fill in TOPDIR to point to bbfile_loc if there is no TOPDIR yet.
+    if not bb_data.getVar('TOPDIR', False):
+        chdir_back = True
+        bb_data.setVar('TOPDIR', bbfile_loc)
+    try:
+        if appends:
+            bb_data.setVar('__BBAPPEND', " ".join(appends))
+        bb_data = bb.parse.handle(bbfile, bb_data)
+        if chdir_back:
+            os.chdir(oldpath)
+        return bb_data
+    except:
+        if chdir_back:
+            os.chdir(oldpath)
+        raise
+
+
+
+class NoCache(object):
+
+    def __init__(self, databuilder):
+        self.databuilder = databuilder
+        self.data = databuilder.data
+
+    def loadDataFull(self, virtualfn, appends):
+        """
+        Return a complete set of data for fn.
+        To do this, we need to parse the file.
+        """
+        logger.debug("Parsing %s (full)" % virtualfn)
+        (fn, virtual, mc) = virtualfn2realfn(virtualfn)
+        bb_data = self.load_bbfile(virtualfn, appends, virtonly=True)
+        return bb_data[virtual]
+
+    def load_bbfile(self, bbfile, appends, virtonly = False, mc=None):
+        """
+        Load and parse one .bb build file
+        Return the data and whether parsing resulted in the file being skipped
+        """
+
+        if virtonly:
+            (bbfile, virtual, mc) = virtualfn2realfn(bbfile)
+            bb_data = self.databuilder.mcdata[mc].createCopy()
+            bb_data.setVar("__ONLYFINALISE", virtual or "default")
+            datastores = parse_recipe(bb_data, bbfile, appends, mc)
+            return datastores
+
+        if mc is not None:
+            bb_data = self.databuilder.mcdata[mc].createCopy()
+            return parse_recipe(bb_data, bbfile, appends, mc)
+
+        bb_data = self.data.createCopy()
+        datastores = parse_recipe(bb_data, bbfile, appends)
+
+        for mc in self.databuilder.mcdata:
+            if not mc:
+                continue
+            bb_data = self.databuilder.mcdata[mc].createCopy()
+            newstores = parse_recipe(bb_data, bbfile, appends, mc)
+            for ns in newstores:
+                datastores["mc:%s:%s" % (mc, ns)] = newstores[ns]
+
+        return datastores
+
+class Cache(NoCache):
     """
     BitBake Cache implementation
     """
     def __init__(self, databuilder, mc, data_hash, caches_array):
-        self.databuilder = databuilder
-        self.data = databuilder.data
+        super().__init__(databuilder)
+        data = databuilder.data
 
         # Pass caches_array information into Cache Constructor
         # It will be used later for deciding whether we
@@ -399,7 +376,7 @@ class Cache(object):
         self.mc = mc
         self.logger = PrefixLoggerAdapter("Cache: %s: " % (mc if mc else "default"), logger)
         self.caches_array = caches_array
-        self.cachedir = self.data.getVar("CACHE")
+        self.cachedir = data.getVar("CACHE")
         self.clean = set()
         self.checked = set()
         self.depends_cache = {}
@@ -409,12 +386,20 @@ class Cache(object):
         self.filelist_regex = re.compile(r'(?:(?<=:True)|(?<=:False))\s+')
 
         if self.cachedir in [None, '']:
-            bb.fatal("Please ensure CACHE is set to the cache directory for BitBake to use")
+            self.has_cache = False
+            self.logger.info("Not using a cache. "
+                             "Set CACHE = <directory> to enable.")
+            return
+
+        self.has_cache = True
 
     def getCacheFile(self, cachefile):
         return getCacheFile(self.cachedir, cachefile, self.mc, self.data_hash)
 
     def prepare_cache(self, progress):
+        if not self.has_cache:
+            return 0
+
         loaded = 0
 
         self.cachefile = self.getCacheFile("bb_cache.dat")
@@ -453,6 +438,9 @@ class Cache(object):
         return loaded
 
     def cachesize(self):
+        if not self.has_cache:
+            return 0
+
         cachesize = 0
         for cache_class in self.caches_array:
             cachefile = self.getCacheFile(cache_class.cachefile)
@@ -514,11 +502,11 @@ class Cache(object):
 
         return len(self.depends_cache)
 
-    def parse(self, filename, appends, layername):
+    def parse(self, filename, appends):
         """Parse the specified filename, returning the recipe information"""
         self.logger.debug("Parsing %s", filename)
         infos = []
-        datastores = self.databuilder.parseRecipeVariants(filename, appends, mc=self.mc, layername=layername)
+        datastores = self.load_bbfile(filename, appends, mc=self.mc)
         depends = []
         variants = []
         # Process the "real" fn last so we can store variants list
@@ -540,19 +528,43 @@ class Cache(object):
 
         return infos
 
-    def loadCached(self, filename, appends):
+    def load(self, filename, appends):
         """Obtain the recipe information for the specified filename,
-        using cached values.
-        """
+        using cached values if available, otherwise parsing.
 
-        infos = []
-        # info_array item is a list of [CoreRecipeInfo, XXXRecipeInfo]
-        info_array = self.depends_cache[filename]
-        for variant in info_array[0].variants:
-            virtualfn = variant2virtual(filename, variant)
-            infos.append((virtualfn, self.depends_cache[virtualfn]))
+        Note that if it does parse to obtain the info, it will not
+        automatically add the information to the cache or to your
+        CacheData.  Use the add or add_info method to do so after
+        running this, or use loadData instead."""
+        cached = self.cacheValid(filename, appends)
+        if cached:
+            infos = []
+            # info_array item is a list of [CoreRecipeInfo, XXXRecipeInfo]
+            info_array = self.depends_cache[filename]
+            for variant in info_array[0].variants:
+                virtualfn = variant2virtual(filename, variant)
+                infos.append((virtualfn, self.depends_cache[virtualfn]))
+        else:
+            return self.parse(filename, appends, configdata, self.caches_array)
 
-        return infos
+        return cached, infos
+
+    def loadData(self, fn, appends, cacheData):
+        """Load the recipe info for the specified filename,
+        parsing and adding to the cache if necessary, and adding
+        the recipe information to the supplied CacheData instance."""
+        skipped, virtuals = 0, 0
+
+        cached, infos = self.load(fn, appends)
+        for virtualfn, info_array in infos:
+            if info_array[0].skipped:
+                self.logger.debug("Skipping %s: %s", virtualfn, info_array[0].skipreason)
+                skipped += 1
+            else:
+                self.add_info(virtualfn, info_array, cacheData, not cached)
+                virtuals += 1
+
+        return cached, skipped, virtuals
 
     def cacheValid(self, fn, appends):
         """
@@ -561,6 +573,10 @@ class Cache(object):
         """
         if fn not in self.checked:
             self.cacheValidUpdate(fn, appends)
+
+        # Is cache enabled?
+        if not self.has_cache:
+            return False
         if fn in self.clean:
             return True
         return False
@@ -570,6 +586,10 @@ class Cache(object):
         Is the cache valid for fn?
         Make thorough (slower) checks including timestamps.
         """
+        # Is cache enabled?
+        if not self.has_cache:
+            return False
+
         self.checked.add(fn)
 
         # File isn't in depends_cache
@@ -620,7 +640,7 @@ class Cache(object):
                 for f in flist:
                     if not f:
                         continue
-                    f, exist = f.rsplit(":", 1)
+                    f, exist = f.split(":")
                     if (exist == "True" and not os.path.exists(f)) or (exist == "False" and os.path.exists(f)):
                         self.logger.debug2("%s's file checksum list file %s changed",
                                              fn, f)
@@ -676,6 +696,10 @@ class Cache(object):
         Save the cache
         Called from the parser when complete (or exiting)
         """
+
+        if not self.has_cache:
+            return
+
         if self.cacheclean:
             self.logger.debug2("Cache is clean, not saving.")
             return
@@ -696,7 +720,6 @@ class Cache(object):
                             p.dump(info)
 
         del self.depends_cache
-        SiggenRecipeInfo.reset()
 
     @staticmethod
     def mtime(cachefile):
@@ -719,11 +742,26 @@ class Cache(object):
             if watcher:
                 watcher(info_array[0].file_depends)
 
+        if not self.has_cache:
+            return
+
         if (info_array[0].skipped or 'SRCREVINACTION' not in info_array[0].pv) and not info_array[0].nocache:
             if parsed:
                 self.cacheclean = False
             self.depends_cache[filename] = info_array
 
+    def add(self, file_name, data, cacheData, parsed=None):
+        """
+        Save data we need into the cache
+        """
+
+        realfn = virtualfn2realfn(file_name)[0]
+
+        info_array = []
+        for cache_class in self.caches_array:
+            info_array.append(cache_class(realfn, data))
+        self.add_info(file_name, info_array, cacheData, parsed)
+
 class MulticonfigCache(Mapping):
     def __init__(self, databuilder, data_hash, caches_array):
         def progress(p):
@@ -760,7 +798,6 @@ class MulticonfigCache(Mapping):
         loaded = 0
 
         for c in self.__caches.values():
-            SiggenRecipeInfo.reset()
             loaded += c.prepare_cache(progress)
             previous_progress = current_progress
 
@@ -838,10 +875,11 @@ class MultiProcessCache(object):
         self.cachedata = self.create_cachedata()
         self.cachedata_extras = self.create_cachedata()
 
-    def init_cache(self, cachedir, cache_file_name=None):
-        if not cachedir:
+    def init_cache(self, d, cache_file_name=None):
+        cachedir = (d.getVar("PERSISTENT_DIR") or
+                    d.getVar("CACHE"))
+        if cachedir in [None, '']:
             return
-
         bb.utils.mkdirhier(cachedir)
         self.cachefile = os.path.join(cachedir,
                                       cache_file_name or self.__class__.cache_file_name)
@@ -872,10 +910,6 @@ class MultiProcessCache(object):
         if not self.cachefile:
             return
 
-        have_data = any(self.cachedata_extras)
-        if not have_data:
-            return
-
         glf = bb.utils.lockfile(self.cachefile + ".lock", shared=True)
 
         i = os.getpid()
@@ -910,8 +944,6 @@ class MultiProcessCache(object):
 
         data = self.cachedata
 
-        have_data = False
-
         for f in [y for y in os.listdir(os.path.dirname(self.cachefile)) if y.startswith(os.path.basename(self.cachefile) + '-')]:
             f = os.path.join(os.path.dirname(self.cachefile), f)
             try:
@@ -926,14 +958,12 @@ class MultiProcessCache(object):
                 os.unlink(f)
                 continue
 
-            have_data = True
             self.merge_data(extradata, data)
             os.unlink(f)
 
-        if have_data:
-            with open(self.cachefile, "wb") as f:
-                p = pickle.Pickler(f, -1)
-                p.dump([data, self.__class__.CACHE_VERSION])
+        with open(self.cachefile, "wb") as f:
+            p = pickle.Pickler(f, -1)
+            p.dump([data, self.__class__.CACHE_VERSION])
 
         bb.utils.unlockfile(glf)
 
@@ -989,11 +1019,3 @@ class SimpleCache(object):
             p.dump([data, self.cacheversion])
 
         bb.utils.unlockfile(glf)
-
-    def copyfile(self, target):
-        if not self.cachefile:
-            return
-
-        glf = bb.utils.lockfile(self.cachefile + ".lock")
-        shutil.copy(self.cachefile, target)
-        bb.utils.unlockfile(glf)

bitbake/lib/bb/checksum.py

@@ -11,13 +11,10 @@ import os
 import stat
 import bb.utils
 import logging
-import re
 from bb.cache import MultiProcessCache
 
 logger = logging.getLogger("BitBake.Cache")
 
-filelist_regex = re.compile(r'(?:(?<=:True)|(?<=:False))\s+')
-
 # mtime cache (non-persistent)
 # based upon the assumption that files do not change during bitbake run
 class FileMtimeCache(object):
@@ -53,7 +50,6 @@ class FileChecksumCache(MultiProcessCache):
         MultiProcessCache.__init__(self)
 
     def get_checksum(self, f):
-        f = os.path.normpath(f)
         entry = self.cachedata[0].get(f)
         cmtime = self.mtime_cache.cached_mtime(f)
         if entry:
@@ -88,36 +84,22 @@ class FileChecksumCache(MultiProcessCache):
                 return None
             return checksum
 
-        #
-        # Changing the format of file-checksums is problematic as both OE and Bitbake have
-        # knowledge of them. We need to encode a new piece of data, the portion of the path
-        # we care about from a checksum perspective. This means that files that change subdirectory
-        # are tracked by the task hashes. To do this, we do something horrible and put a "/./" into
-        # the path. The filesystem handles it but it gives us a marker to know which subsection
-        # of the path to cache.
-        #
         def checksum_dir(pth):
             # Handle directories recursively
             if pth == "/":
                 bb.fatal("Refusing to checksum /")
-            pth = pth.rstrip("/")
             dirchecksums = []
             for root, dirs, files in os.walk(pth, topdown=True):
                 [dirs.remove(d) for d in list(dirs) if d in localdirsexclude]
                 for name in files:
-                    fullpth = os.path.join(root, name).replace(pth, os.path.join(pth, "."))
+                    fullpth = os.path.join(root, name)
                     checksum = checksum_file(fullpth)
                     if checksum:
                         dirchecksums.append((fullpth, checksum))
             return dirchecksums
 
         checksums = []
-        for pth in filelist_regex.split(filelist):
-            if not pth:
-                continue
-            pth = pth.strip()
-            if not pth:
-                continue
+        for pth in filelist.split():
             exist = pth.split(":")[1]
             if exist == "False":
                 continue

bitbake/lib/bb/codeparser.py

@@ -1,6 +1,4 @@
 #
-# Copyright BitBake Contributors
-#
 # SPDX-License-Identifier: GPL-2.0-only
 #
 
@@ -27,7 +25,6 @@ import ast
 import sys
 import codegen
 import logging
-import inspect
 import bb.pysh as pysh
 import bb.utils, bb.data
 import hashlib
@@ -59,39 +56,10 @@ def check_indent(codestr):
 
     return codestr
 
-modulecode_deps = {}
-
-def add_module_functions(fn, functions, namespace):
-    fstat = os.stat(fn)
-    fixedhash = fn + ":" + str(fstat.st_size) +  ":" + str(fstat.st_mtime)
-    for f in functions:
-        name = "%s.%s" % (namespace, f)
-        parser = PythonParser(name, logger)
-        try:
-            parser.parse_python(None, filename=fn, lineno=1, fixedhash=fixedhash+f)
-            #bb.warn("Cached %s" % f)
-        except KeyError:
-            lines, lineno = inspect.getsourcelines(functions[f])
-            src = "".join(lines)
-            parser.parse_python(src, filename=fn, lineno=lineno, fixedhash=fixedhash+f)
-            #bb.warn("Not cached %s" % f)
-        execs = parser.execs.copy()
-        # Expand internal module exec references
-        for e in parser.execs:
-            if e in functions:
-                execs.remove(e)
-                execs.add(namespace + "." + e)
-        modulecode_deps[name] = [parser.references.copy(), execs, parser.var_execs.copy(), parser.contains.copy()]
-        #bb.warn("%s: %s\nRefs:%s Execs: %s %s %s" % (name, src, parser.references, parser.execs, parser.var_execs, parser.contains))
-
-def update_module_dependencies(d):
-    for mod in modulecode_deps:
-        excludes = set((d.getVarFlag(mod, "vardepsexclude") or "").split())
-        if excludes:
-            modulecode_deps[mod] = [modulecode_deps[mod][0] - excludes, modulecode_deps[mod][1] - excludes, modulecode_deps[mod][2] - excludes, modulecode_deps[mod][3]]
-
 # A custom getstate/setstate using tuples is actually worth 15% cachesize by
 # avoiding duplication of the attribute names!
+
+
 class SetCache(object):
     def __init__(self):
         self.setcache = {}
@@ -184,12 +152,12 @@ class CodeParserCache(MultiProcessCache):
         self.shellcachelines[h] = cacheline
         return cacheline
 
-    def init_cache(self, cachedir):
+    def init_cache(self, d):
         # Check if we already have the caches
         if self.pythoncache:
             return
 
-        MultiProcessCache.init_cache(self, cachedir)
+        MultiProcessCache.init_cache(self, d)
 
         # cachedata gets re-assigned in the parent
         self.pythoncache = self.cachedata[0]
@@ -201,8 +169,8 @@ class CodeParserCache(MultiProcessCache):
 
 codeparsercache = CodeParserCache()
 
-def parser_cache_init(cachedir):
-    codeparsercache.init_cache(cachedir)
+def parser_cache_init(d):
+    codeparsercache.init_cache(d)
 
 def parser_cache_save():
     codeparsercache.save_extras()
@@ -227,10 +195,6 @@ class BufferedLogger(Logger):
                 self.target.handle(record)
         self.buffer = []
 
-class DummyLogger():
-    def flush(self):
-        return
-
 class PythonParser():
     getvars = (".getVar", ".appendVar", ".prependVar", "oe.utils.conditional")
     getvarflags = (".getVarFlag", ".appendVarFlag", ".prependVarFlag")
@@ -312,24 +276,16 @@ class PythonParser():
         self.contains = {}
         self.execs = set()
         self.references = set()
-        self._log = log
-        # Defer init as expensive
-        self.log = DummyLogger()
+        self.log = BufferedLogger('BitBake.Data.PythonParser', logging.DEBUG, log)
 
         self.unhandled_message = "in call of %s, argument '%s' is not a string literal"
         self.unhandled_message = "while parsing %s, %s" % (name, self.unhandled_message)
 
-    # For the python module code it is expensive to have the function text so it is
-    # uses a different fixedhash to cache against. We can take the hit on obtaining the
-    # text if it isn't in the cache.
-    def parse_python(self, node, lineno=0, filename="<string>", fixedhash=None):
-        if not fixedhash and (not node or not node.strip()):
+    def parse_python(self, node, lineno=0, filename="<string>"):
+        if not node or not node.strip():
             return
 
-        if fixedhash:
-            h = fixedhash
-        else:
-            h = bbhash(str(node))
+        h = bbhash(str(node))
 
         if h in codeparsercache.pythoncache:
             self.references = set(codeparsercache.pythoncache[h].refs)
@@ -347,12 +303,6 @@ class PythonParser():
                 self.contains[i] = set(codeparsercache.pythoncacheextras[h].contains[i])
             return
 
-        if fixedhash and not node:
-            raise KeyError
-
-        # Need to parse so take the hit on the real log buffer
-        self.log = BufferedLogger('BitBake.Data.PythonParser', logging.DEBUG, self._log)
-
         # We can't add to the linenumbers for compile, we can pad to the correct number of blank lines though
         node = "\n" * int(lineno) + node
         code = compile(check_indent(str(node)), filename, "exec",
@@ -371,11 +321,7 @@ class ShellParser():
         self.funcdefs = set()
         self.allexecs = set()
         self.execs = set()
-        self._name = name
-        self._log = log
-        # Defer init as expensive
-        self.log = DummyLogger()
-
+        self.log = BufferedLogger('BitBake.Data.%s' % name, logging.DEBUG, log)
         self.unhandled_template = "unable to handle non-literal command '%s'"
         self.unhandled_template = "while parsing %s, %s" % (name, self.unhandled_template)
 
@@ -394,9 +340,6 @@ class ShellParser():
             self.execs = set(codeparsercache.shellcacheextras[h].execs)
             return self.execs
 
-        # Need to parse so take the hit on the real log buffer
-        self.log = BufferedLogger('BitBake.Data.%s' % self._name, logging.DEBUG, self._log)
-
         self._parse_shell(value)
         self.execs = set(cmd for cmd in self.allexecs if cmd not in self.funcdefs)
 

bitbake/lib/bb/command.py

@@ -51,21 +51,20 @@ class Command:
     """
     A queue of asynchronous commands for bitbake
     """
-    def __init__(self, cooker, process_server):
+    def __init__(self, cooker):
         self.cooker = cooker
         self.cmds_sync = CommandsSync()
         self.cmds_async = CommandsAsync()
         self.remotedatastores = None
 
-        self.process_server = process_server
-        # Access with locking using process_server.{get/set/clear}_async_cmd()
+        # FIXME Add lock for this
         self.currentAsyncCommand = None
 
-    def runCommand(self, commandline, process_server, ro_only=False):
+    def runCommand(self, commandline, ro_only = False):
         command = commandline.pop(0)
 
         # Ensure cooker is ready for commands
-        if command not in ["updateConfig", "setFeatures", "ping"]:
+        if command != "updateConfig" and command != "setFeatures":
             try:
                 self.cooker.init_configdata()
                 if not self.remotedatastores:
@@ -85,8 +84,7 @@ class Command:
                 if not hasattr(command_method, 'readonly') or not getattr(command_method, 'readonly'):
                     return None, "Not able to execute not readonly commands in readonly mode"
             try:
-                if command != "ping":
-                    self.cooker.process_inotify_updates_apply()
+                self.cooker.process_inotify_updates()
                 if getattr(command_method, 'needconfig', True):
                     self.cooker.updateCacheSync()
                 result = command_method(self, commandline)
@@ -101,24 +99,24 @@ class Command:
                 return None, traceback.format_exc()
             else:
                 return result, None
+        if self.currentAsyncCommand is not None:
+            return None, "Busy (%s in progress)" % self.currentAsyncCommand[0]
         if command not in CommandsAsync.__dict__:
             return None, "No such command"
-        if not process_server.set_async_cmd((command, commandline)):
-            return None, "Busy (%s in progress)" % self.process_server.get_async_cmd()[0]
-        self.cooker.idleCallBackRegister(self.runAsyncCommand, process_server)
+        self.currentAsyncCommand = (command, commandline)
+        self.cooker.idleCallBackRegister(self.cooker.runCommands, self.cooker)
         return True, None
 
-    def runAsyncCommand(self, _, process_server, halt):
+    def runAsyncCommand(self):
         try:
-            self.cooker.process_inotify_updates_apply()
+            self.cooker.process_inotify_updates()
             if self.cooker.state in (bb.cooker.state.error, bb.cooker.state.shutdown, bb.cooker.state.forceshutdown):
                 # updateCache will trigger a shutdown of the parser
                 # and then raise BBHandledException triggering an exit
                 self.cooker.updateCache()
-                return bb.server.process.idleFinish("Cooker in error state")
-            cmd = process_server.get_async_cmd()
-            if cmd is not None:
-                (command, options) = cmd
+                return False
+            if self.currentAsyncCommand is not None:
+                (command, options) = self.currentAsyncCommand
                 commandmethod = getattr(CommandsAsync, command)
                 needcache = getattr( commandmethod, "needcache" )
                 if needcache and self.cooker.state != bb.cooker.state.running:
@@ -128,21 +126,24 @@ class Command:
                     commandmethod(self.cmds_async, self, options)
                     return False
             else:
-                return bb.server.process.idleFinish("Nothing to do, no async command?")
+                return False
         except KeyboardInterrupt as exc:
-            return bb.server.process.idleFinish("Interrupted")
+            self.finishAsyncCommand("Interrupted")
+            return False
         except SystemExit as exc:
             arg = exc.args[0]
             if isinstance(arg, str):
-                return bb.server.process.idleFinish(arg)
+                self.finishAsyncCommand(arg)
             else:
-                return bb.server.process.idleFinish("Exited with %s" % arg)
+                self.finishAsyncCommand("Exited with %s" % arg)
+            return False
         except Exception as exc:
             import traceback
             if isinstance(exc, bb.BBHandledException):
-                return bb.server.process.idleFinish("")
+                self.finishAsyncCommand("")
             else:
-                return bb.server.process.idleFinish(traceback.format_exc())
+                self.finishAsyncCommand(traceback.format_exc())
+            return False
 
     def finishAsyncCommand(self, msg=None, code=None):
         if msg or msg == "":
@@ -151,8 +152,8 @@ class Command:
             bb.event.fire(CommandExit(code), self.cooker.data)
         else:
             bb.event.fire(CommandCompleted(), self.cooker.data)
+        self.currentAsyncCommand = None
         self.cooker.finishcommand()
-        self.process_server.clear_async_cmd()
 
     def reset(self):
         if self.remotedatastores:
@@ -165,14 +166,6 @@ class CommandsSync:
     These must not influence any running synchronous command.
     """
 
-    def ping(self, command, params):
-        """
-        Allow a UI to check the server is still alive
-        """
-        return "Still alive!"
-    ping.needconfig = False
-    ping.readonly = True
-
     def stateShutdown(self, command, params):
         """
         Trigger cooker 'shutdown' mode
@@ -564,7 +557,6 @@ class CommandsSync:
                 appendfiles = command.cooker.collections[mc].get_file_appends(fn)
         else:
             appendfiles = []
-        layername = command.cooker.collections[mc].calc_bbfile_priority(fn)[2]
         # We are calling bb.cache locally here rather than on the server,
         # but that's OK because it doesn't actually need anything from
         # the server barring the global datastore (which we have a remote
@@ -572,10 +564,11 @@ class CommandsSync:
         if config_data:
             # We have to use a different function here if we're passing in a datastore
             # NOTE: we took a copy above, so we don't do it here again
-            envdata = command.cooker.databuilder._parse_recipe(config_data, fn, appendfiles, mc, layername)['']
+            envdata = bb.cache.parse_recipe(config_data, fn, appendfiles, mc)['']
         else:
             # Use the standard path
-            envdata = command.cooker.databuilder.parseRecipe(fn, appendfiles, layername)
+            parser = bb.cache.NoCache(command.cooker.databuilder)
+            envdata = parser.loadDataFull(fn, appendfiles)
         idx = command.remotedatastores.store(envdata)
         return DataStoreConnectionHandle(idx)
     parseRecipeFile.readonly = True
@@ -748,7 +741,7 @@ class CommandsAsync:
         """
         event = params[0]
         bb.event.fire(eval(event), command.cooker.data)
-        process_server.clear_async_cmd()
+        command.currentAsyncCommand = None
     triggerEvent.needcache = False
 
     def resetCooker(self, command, params):

bitbake/lib/bb/compress/_pipecompress.py

@@ -1,6 +1,4 @@
 #
-# Copyright BitBake Contributors
-#
 # SPDX-License-Identifier: GPL-2.0-only
 #
 # Helper library to implement streaming compression and decompression using an
@@ -51,7 +49,7 @@ def open_wrap(
             raise ValueError("Argument 'newline' not supported in binary mode")
 
     file_mode = mode.replace("t", "")
-    if isinstance(filename, (str, bytes, os.PathLike, int)):
+    if isinstance(filename, (str, bytes, os.PathLike)):
         binary_file = cls(filename, file_mode, **kwargs)
     elif hasattr(filename, "read") or hasattr(filename, "write"):
         binary_file = cls(None, file_mode, fileobj=filename, **kwargs)

bitbake/lib/bb/compress/lz4.py

@@ -1,6 +1,4 @@
 #
-# Copyright BitBake Contributors
-#
 # SPDX-License-Identifier: GPL-2.0-only
 #
 

bitbake/lib/bb/compress/zstd.py

@@ -1,6 +1,4 @@
 #
-# Copyright BitBake Contributors
-#
 # SPDX-License-Identifier: GPL-2.0-only
 #
 

bitbake/lib/bb/cookerdata.py

@@ -57,7 +57,7 @@ class ConfigParameters(object):
 
     def updateToServer(self, server, environment):
         options = {}
-        for o in ["halt", "force", "invalidate_stamp",
+        for o in ["abort", "force", "invalidate_stamp",
                   "dry_run", "dump_signatures",
                   "extra_assume_provided", "profile",
                   "prefile", "postfile", "server_timeout",
@@ -86,7 +86,7 @@ class ConfigParameters(object):
                 action['msg'] = "Only one target can be used with the --environment option."
             elif self.options.buildfile and len(self.options.pkgs_to_build) > 0:
                 action['msg'] = "No target should be used with the --environment and --buildfile options."
-            elif self.options.pkgs_to_build:
+            elif len(self.options.pkgs_to_build) > 0:
                 action['action'] = ["showEnvironmentTarget", self.options.pkgs_to_build]
             else:
                 action['action'] = ["showEnvironment", self.options.buildfile]
@@ -124,7 +124,7 @@ class CookerConfiguration(object):
         self.prefile = []
         self.postfile = []
         self.cmd = None
-        self.halt = True
+        self.abort = True
         self.force = False
         self.profile = False
         self.nosetscene = False
@@ -160,7 +160,12 @@ def catch_parse_error(func):
     def wrapped(fn, *args):
         try:
             return func(fn, *args)
-        except Exception as exc:
+        except IOError as exc:
+            import traceback
+            parselog.critical(traceback.format_exc())
+            parselog.critical("Unable to parse %s: %s" % (fn, exc))
+            raise bb.BBHandledException()
+        except bb.data_smart.ExpansionError as exc:
             import traceback
 
             bbdir = os.path.dirname(__file__) + os.sep
@@ -172,11 +177,14 @@ def catch_parse_error(func):
                     break
             parselog.critical("Unable to parse %s" % fn, exc_info=(exc_class, exc, tb))
             raise bb.BBHandledException()
+        except bb.parse.ParseError as exc:
+            parselog.critical(str(exc))
+            raise bb.BBHandledException()
     return wrapped
 
 @catch_parse_error
 def parse_config_file(fn, data, include=True):
-    return bb.parse.handle(fn, data, include, baseconfig=True)
+    return bb.parse.handle(fn, data, include)
 
 @catch_parse_error
 def _inherit(bbclass, data):
@@ -202,7 +210,7 @@ def findConfigFile(configfile, data):
 
 #
 # We search for a conf/bblayers.conf under an entry in BBPATH or in cwd working
-# up to /. If that fails, bitbake would fall back to cwd.
+# up to /. If that fails, we search for a conf/bitbake.conf in BBPATH.
 #
 
 def findTopdir():
@@ -215,8 +223,11 @@ def findTopdir():
     layerconf = findConfigFile("bblayers.conf", d)
     if layerconf:
         return os.path.dirname(os.path.dirname(layerconf))
-
-    return os.path.abspath(os.getcwd())
+    if bbpath:
+        bitbakeconf = bb.utils.which(bbpath, "conf/bitbake.conf")
+        if bitbakeconf:
+            return os.path.dirname(os.path.dirname(bitbakeconf))
+    return None
 
 class CookerDataBuilder(object):
 
@@ -239,14 +250,10 @@ class CookerDataBuilder(object):
         self.savedenv = bb.data.init()
         for k in cookercfg.env:
             self.savedenv.setVar(k, cookercfg.env[k])
-            if k in bb.data_smart.bitbake_renamed_vars:
-                bb.error('Shell environment variable %s has been renamed to %s' % (k, bb.data_smart.bitbake_renamed_vars[k]))
-                bb.fatal("Exiting to allow enviroment variables to be corrected")
 
         filtered_keys = bb.utils.approved_variables()
         bb.data.inheritFromOS(self.basedata, self.savedenv, filtered_keys)
         self.basedata.setVar("BB_ORIGENV", self.savedenv)
-        self.basedata.setVar("__bbclasstype", "global")
 
         if worker:
             self.basedata.setVar("BB_WORKERCONTEXT", "1")
@@ -254,15 +261,15 @@ class CookerDataBuilder(object):
         self.data = self.basedata
         self.mcdata = {}
 
-    def parseBaseConfiguration(self, worker=False):
-        mcdata = {}
+    def parseBaseConfiguration(self):
         data_hash = hashlib.sha256()
         try:
             self.data = self.parseConfigurationFiles(self.prefiles, self.postfiles)
 
-            if self.data.getVar("BB_WORKERCONTEXT", False) is None and not worker:
+            if self.data.getVar("BB_WORKERCONTEXT", False) is None:
                 bb.fetch.fetcher_init(self.data)
             bb.parse.init_parser(self.data)
+            bb.codeparser.parser_cache_init(self.data)
 
             bb.event.fire(bb.event.ConfigParsed(), self.data)
 
@@ -280,62 +287,40 @@ class CookerDataBuilder(object):
 
             bb.parse.init_parser(self.data)
             data_hash.update(self.data.get_hash().encode('utf-8'))
-            mcdata[''] = self.data
+            self.mcdata[''] = self.data
 
             multiconfig = (self.data.getVar("BBMULTICONFIG") or "").split()
             for config in multiconfig:
                 if config[0].isdigit():
                     bb.fatal("Multiconfig name '%s' is invalid as multiconfigs cannot start with a digit" % config)
-                parsed_mcdata = self.parseConfigurationFiles(self.prefiles, self.postfiles, config)
-                bb.event.fire(bb.event.ConfigParsed(), parsed_mcdata)
-                mcdata[config] = parsed_mcdata
-                data_hash.update(parsed_mcdata.get_hash().encode('utf-8'))
+                mcdata = self.parseConfigurationFiles(self.prefiles, self.postfiles, config)
+                bb.event.fire(bb.event.ConfigParsed(), mcdata)
+                self.mcdata[config] = mcdata
+                data_hash.update(mcdata.get_hash().encode('utf-8'))
             if multiconfig:
-                bb.event.fire(bb.event.MultiConfigParsed(mcdata), self.data)
+                bb.event.fire(bb.event.MultiConfigParsed(self.mcdata), self.data)
 
             self.data_hash = data_hash.hexdigest()
+        except (SyntaxError, bb.BBHandledException):
+            raise bb.BBHandledException()
         except bb.data_smart.ExpansionError as e:
             logger.error(str(e))
             raise bb.BBHandledException()
-
-        bb.codeparser.update_module_dependencies(self.data)
-
-        # Handle obsolete variable names
-        d = self.data
-        renamedvars = d.getVarFlags('BB_RENAMED_VARIABLES') or {}
-        renamedvars.update(bb.data_smart.bitbake_renamed_vars)
-        issues = False
-        for v in renamedvars:
-            if d.getVar(v) != None or d.hasOverrides(v):
-                issues = True
-                loginfo = {}
-                history = d.varhistory.get_variable_refs(v)
-                for h in history:
-                    for line in history[h]:
-                        loginfo = {'file' : h, 'line' : line}
-                        bb.data.data_smart._print_rename_error(v, loginfo, renamedvars)
-                if not history:
-                    bb.data.data_smart._print_rename_error(v, loginfo, renamedvars)
-        if issues:
+        except Exception:
+            logger.exception("Error parsing configuration files")
             raise bb.BBHandledException()
 
-        for mc in mcdata:
-            mcdata[mc].renameVar("__depends", "__base_depends")
-            mcdata[mc].setVar("__bbclasstype", "recipe")
-
         # Create a copy so we can reset at a later date when UIs disconnect
-        self.mcorigdata = mcdata
-        for mc in mcdata:
-            self.mcdata[mc] = bb.data.createCopy(mcdata[mc])
-        self.data = self.mcdata['']
+        self.origdata = self.data
+        self.data = bb.data.createCopy(self.origdata)
+        self.mcdata[''] = self.data
 
     def reset(self):
         # We may not have run parseBaseConfiguration() yet
-        if not hasattr(self, 'mcorigdata'):
+        if not hasattr(self, 'origdata'):
             return
-        for mc in self.mcorigdata:
-            self.mcdata[mc] = bb.data.createCopy(self.mcorigdata[mc])
-        self.data = self.mcdata['']
+        self.data = bb.data.createCopy(self.origdata)
+        self.mcdata[''] = self.data
 
     def _findLayerConf(self, data):
         return findConfigFile("bblayers.conf", data)
@@ -350,17 +335,12 @@ class CookerDataBuilder(object):
 
         layerconf = self._findLayerConf(data)
         if layerconf:
-            parselog.debug2("Found bblayers.conf (%s)", layerconf)
+            parselog.debug(2, "Found bblayers.conf (%s)", layerconf)
             # By definition bblayers.conf is in conf/ of TOPDIR.
             # We may have been called with cwd somewhere else so reset TOPDIR
             data.setVar("TOPDIR", os.path.dirname(os.path.dirname(layerconf)))
             data = parse_config_file(layerconf, data)
 
-            if not data.getVar("BB_CACHEDIR"):
-                data.setVar("BB_CACHEDIR", "${TOPDIR}/cache")
-
-            bb.codeparser.parser_cache_init(data.getVar("BB_CACHEDIR"))
-
             layers = (data.getVar('BBLAYERS') or "").split()
             broken_layers = []
 
@@ -382,10 +362,8 @@ class CookerDataBuilder(object):
                 parselog.critical("Please check BBLAYERS in %s" % (layerconf))
                 raise bb.BBHandledException()
 
-            layerseries = None
-            compat_entries = {}
             for layer in layers:
-                parselog.debug2("Adding layer %s", layer)
+                parselog.debug(2, "Adding layer %s", layer)
                 if 'HOME' in approved and '~' in layer:
                     layer = os.path.expanduser(layer)
                 if layer.endswith('/'):
@@ -396,27 +374,8 @@ class CookerDataBuilder(object):
                 data.expandVarref('LAYERDIR')
                 data.expandVarref('LAYERDIR_RE')
 
-                # Sadly we can't have nice things.
-                # Some layers think they're going to be 'clever' and copy the values from
-                # another layer, e.g. using ${LAYERSERIES_COMPAT_core}. The whole point of
-                # this mechanism is to make it clear which releases a layer supports and
-                # show when a layer master branch is bitrotting and is unmaintained.
-                # We therefore avoid people doing this here.
-                collections = (data.getVar('BBFILE_COLLECTIONS') or "").split()
-                for c in collections:
-                    compat_entry = data.getVar("LAYERSERIES_COMPAT_%s" % c)
-                    if compat_entry:
-                        compat_entries[c] = set(compat_entry.split())
-                        data.delVar("LAYERSERIES_COMPAT_%s" % c)
-                if not layerseries:
-                    layerseries = set((data.getVar("LAYERSERIES_CORENAMES") or "").split())
-                    if layerseries:
-                        data.delVar("LAYERSERIES_CORENAMES")
-
             data.delVar('LAYERDIR_RE')
             data.delVar('LAYERDIR')
-            for c in compat_entries:
-                data.setVar("LAYERSERIES_COMPAT_%s" % c, " ".join(sorted(compat_entries[c])))
 
             bbfiles_dynamic = (data.getVar('BBFILES_DYNAMIC') or "").split()
             collections = (data.getVar('BBFILE_COLLECTIONS') or "").split()
@@ -435,15 +394,13 @@ class CookerDataBuilder(object):
             if invalid:
                 bb.fatal("BBFILES_DYNAMIC entries must be of the form {!}<collection name>:<filename pattern>, not:\n    %s" % "\n    ".join(invalid))
 
+            layerseries = set((data.getVar("LAYERSERIES_CORENAMES") or "").split())
             collections_tmp = collections[:]
             for c in collections:
                 collections_tmp.remove(c)
                 if c in collections_tmp:
                     bb.fatal("Found duplicated BBFILE_COLLECTIONS '%s', check bblayers.conf or layer.conf to fix it." % c)
-
-                compat = set()
-                if c in compat_entries:
-                    compat = compat_entries[c]
+                compat = set((data.getVar("LAYERSERIES_COMPAT_%s" % c) or "").split())
                 if compat and not layerseries:
                     bb.fatal("No core layer found to work with layer '%s'. Missing entry in bblayers.conf?" % c)
                 if compat and not (compat & layerseries):
@@ -452,21 +409,13 @@ class CookerDataBuilder(object):
                 elif not compat and not data.getVar("BB_WORKERCONTEXT"):
                     bb.warn("Layer %s should set LAYERSERIES_COMPAT_%s in its conf/layer.conf file to list the core layer names it is compatible with." % (c, c))
 
-            data.setVar("LAYERSERIES_CORENAMES", " ".join(sorted(layerseries)))
-
         if not data.getVar("BBPATH"):
             msg = "The BBPATH variable is not set"
             if not layerconf:
                 msg += (" and bitbake did not find a conf/bblayers.conf file in"
                         " the expected location.\nMaybe you accidentally"
                         " invoked bitbake from the wrong directory?")
-            bb.fatal(msg)
-
-        if not data.getVar("TOPDIR"):
-            data.setVar("TOPDIR", os.path.abspath(os.getcwd()))
-        if not data.getVar("BB_CACHEDIR"):
-            data.setVar("BB_CACHEDIR", "${TOPDIR}/cache")
-        bb.codeparser.parser_cache_init(data.getVar("BB_CACHEDIR"))
+            raise SystemExit(msg)
 
         data = parse_config_file(os.path.join("conf", "bitbake.conf"), data)
 
@@ -479,7 +428,7 @@ class CookerDataBuilder(object):
         for bbclass in bbclasses:
             data = _inherit(bbclass, data)
 
-        # Normally we only register event handlers at the end of parsing .bb files
+        # Nomally we only register event handlers at the end of parsing .bb files
         # We register any handlers we've found so far here...
         for var in data.getVar('__BBHANDLERS', False) or []:
             handlerfn = data.getVarFlag(var, "filename", False)
@@ -493,55 +442,3 @@ class CookerDataBuilder(object):
 
         return data
 
-    @staticmethod
-    def _parse_recipe(bb_data, bbfile, appends, mc, layername):
-        bb_data.setVar("__BBMULTICONFIG", mc)
-        bb_data.setVar("FILE_LAYERNAME", layername)
-
-        bbfile_loc = os.path.abspath(os.path.dirname(bbfile))
-        bb.parse.cached_mtime_noerror(bbfile_loc)
-
-        if appends:
-            bb_data.setVar('__BBAPPEND', " ".join(appends))
-        bb_data = bb.parse.handle(bbfile, bb_data)
-        return bb_data
-
-    def parseRecipeVariants(self, bbfile, appends, virtonly=False, mc=None, layername=None):
-        """
-        Load and parse one .bb build file
-        Return the data and whether parsing resulted in the file being skipped
-        """
-
-        if virtonly:
-            (bbfile, virtual, mc) = bb.cache.virtualfn2realfn(bbfile)
-            bb_data = self.mcdata[mc].createCopy()
-            bb_data.setVar("__ONLYFINALISE", virtual or "default")
-            datastores = self._parse_recipe(bb_data, bbfile, appends, mc, layername)
-            return datastores
-
-        if mc is not None:
-            bb_data = self.mcdata[mc].createCopy()
-            return self._parse_recipe(bb_data, bbfile, appends, mc, layername)
-
-        bb_data = self.data.createCopy()
-        datastores = self._parse_recipe(bb_data, bbfile, appends, '', layername)
-
-        for mc in self.mcdata:
-            if not mc:
-                continue
-            bb_data = self.mcdata[mc].createCopy()
-            newstores = self._parse_recipe(bb_data, bbfile, appends, mc, layername)
-            for ns in newstores:
-                datastores["mc:%s:%s" % (mc, ns)] = newstores[ns]
-
-        return datastores
-
-    def parseRecipe(self, virtualfn, appends, layername):
-        """
-        Return a complete set of data for fn.
-        To do this, we need to parse the file.
-        """
-        logger.debug("Parsing %s (full)" % virtualfn)
-        (fn, virtual, mc) = bb.cache.virtualfn2realfn(virtualfn)
-        bb_data = self.parseRecipeVariants(virtualfn, appends, virtonly=True, layername=layername)
-        return bb_data[virtual]

bitbake/lib/bb/daemonize.py

@@ -1,6 +1,4 @@
 #
-# Copyright BitBake Contributors
-#
 # SPDX-License-Identifier: GPL-2.0-only
 #
 
@@ -76,26 +74,26 @@ def createDaemon(function, logfile):
     with open('/dev/null', 'r') as si:
         os.dup2(si.fileno(), sys.stdin.fileno())
 
-    with open(logfile, 'a+') as so:
-        try:
-            os.dup2(so.fileno(), sys.stdout.fileno())
-            os.dup2(so.fileno(), sys.stderr.fileno())
-        except io.UnsupportedOperation:
-            sys.stdout = so
+    try:
+        so = open(logfile, 'a+')
+        os.dup2(so.fileno(), sys.stdout.fileno())
+        os.dup2(so.fileno(), sys.stderr.fileno())
+    except io.UnsupportedOperation:
+        sys.stdout = open(logfile, 'a+')
 
-        # Have stdout and stderr be the same so log output matches chronologically
-        # and there aren't two separate buffers
-        sys.stderr = sys.stdout
+    # Have stdout and stderr be the same so log output matches chronologically
+    # and there aren't two seperate buffers
+    sys.stderr = sys.stdout
 
-        try:
-            function()
-        except Exception as e:
-            traceback.print_exc()
-        finally:
-            bb.event.print_ui_queue()
-            # os._exit() doesn't flush open files like os.exit() does. Manually flush
-            # stdout and stderr so that any logging output will be seen, particularly
-            # exception tracebacks.
-            sys.stdout.flush()
-            sys.stderr.flush()
-            os._exit(0)
+    try:
+        function()
+    except Exception as e:
+        traceback.print_exc()
+    finally:
+        bb.event.print_ui_queue()
+        # os._exit() doesn't flush open files like os.exit() does. Manually flush
+        # stdout and stderr so that any logging output will be seen, particularly
+        # exception tracebacks.
+        sys.stdout.flush()
+        sys.stderr.flush()
+        os._exit(0)

bitbake/lib/bb/data.py

@@ -4,16 +4,14 @@ BitBake 'Data' implementations
 Functions for interacting with the data structure used by the
 BitBake build tools.
 
-expandKeys and datastore iteration are the most expensive
-operations. Updating overrides is now "on the fly" but still based
-on the idea of the cookie monster introduced by zecke:
-"At night the cookie monster came by and
+The expandKeys and update_data are the most expensive
+operations. At night the cookie monster came by and
 suggested 'give me cookies on setting the variables and
 things will work out'. Taking this suggestion into account
 applying the skills from the not yet passed 'Entwurf und
 Analyse von Algorithmen' lecture and the cookie
 monster seems to be right. We will track setVar more carefully
-to have faster datastore operations."
+to have faster update_data and expandKeys operations.
 
 This is a trade-off between speed and memory again but
 the speed is more critical here.
@@ -28,6 +26,11 @@ the speed is more critical here.
 
 import sys, os, re
 import hashlib
+if sys.argv[0][-5:] == "pydoc":
+    path = os.path.dirname(os.path.dirname(sys.argv[1]))
+else:
+    path = os.path.dirname(os.path.dirname(sys.argv[0]))
+sys.path.insert(0, path)
 from itertools import groupby
 
 from bb import data_smart
@@ -67,6 +70,10 @@ def keys(d):
     """Return a list of keys in d"""
     return d.keys()
 
+
+__expand_var_regexp__ = re.compile(r"\${[^{}]+}")
+__expand_python_regexp__ = re.compile(r"\${@.+?}")
+
 def expand(s, d, varname = None):
     """Variable expansion using the data store"""
     return d.expand(s, varname)
@@ -114,8 +121,8 @@ def emit_var(var, o=sys.__stdout__, d = init(), all=False):
     if d.getVarFlag(var, 'python', False) and func:
         return False
 
-    export = bb.utils.to_boolean(d.getVarFlag(var, "export"))
-    unexport = bb.utils.to_boolean(d.getVarFlag(var, "unexport"))
+    export = d.getVarFlag(var, "export", False)
+    unexport = d.getVarFlag(var, "unexport", False)
     if not all and not export and not unexport and not func:
         return False
 
@@ -188,8 +195,8 @@ def emit_env(o=sys.__stdout__, d = init(), all=False):
 
 def exported_keys(d):
     return (key for key in d.keys() if not key.startswith('__') and
-                                      bb.utils.to_boolean(d.getVarFlag(key, 'export')) and
-                                      not bb.utils.to_boolean(d.getVarFlag(key, 'unexport')))
+                                      d.getVarFlag(key, 'export', False) and
+                                      not d.getVarFlag(key, 'unexport', False))
 
 def exported_vars(d):
     k = list(exported_keys(d))
@@ -261,71 +268,65 @@ def emit_func_python(func, o=sys.__stdout__, d = init()):
                newdeps |= set((d.getVarFlag(dep, "vardeps") or "").split())
         newdeps -= seen
 
-def build_dependencies(key, keys, mod_funcs, shelldeps, varflagsexcl, ignored_vars, d, codeparsedata):
-    def handle_contains(value, contains, exclusions, d):
-        newvalue = []
-        if value:
-            newvalue.append(str(value))
-        for k in sorted(contains):
-            if k in exclusions or k in ignored_vars:
-                continue
-            l = (d.getVar(k) or "").split()
-            for item in sorted(contains[k]):
-                for word in item.split():
-                    if not word in l:
-                        newvalue.append("\n%s{%s} = Unset" % (k, item))
-                        break
-                else:
-                    newvalue.append("\n%s{%s} = Set" % (k, item))
-        return "".join(newvalue)
-
-    def handle_remove(value, deps, removes, d):
-        for r in sorted(removes):
-            r2 = d.expandWithRefs(r, None)
-            value += "\n_remove of %s" % r
-            deps |= r2.references
-            deps = deps | (keys & r2.execs)
-        return value
+def update_data(d):
+    """Performs final steps upon the datastore, including application of overrides"""
+    d.finalize(parent = True)
 
+def build_dependencies(key, keys, shelldeps, varflagsexcl, d):
     deps = set()
     try:
-        if key in mod_funcs:
-            exclusions = set()
-            moddep = bb.codeparser.modulecode_deps[key]
-            value = handle_contains("", moddep[3], exclusions, d)
-            return frozenset((moddep[0] | keys & moddep[1]) - ignored_vars), value
-
         if key[-1] == ']':
             vf = key[:-1].split('[')
-            if vf[1] == "vardepvalueexclude":
-                return deps, ""
             value, parser = d.getVarFlag(vf[0], vf[1], False, retparser=True)
             deps |= parser.references
             deps = deps | (keys & parser.execs)
-            deps -= ignored_vars
-            return frozenset(deps), value
+            return deps, value
         varflags = d.getVarFlags(key, ["vardeps", "vardepvalue", "vardepsexclude", "exports", "postfuncs", "prefuncs", "lineno", "filename"]) or {}
         vardeps = varflags.get("vardeps")
-        exclusions = varflags.get("vardepsexclude", "").split()
+
+        def handle_contains(value, contains, d):
+            newvalue = ""
+            for k in sorted(contains):
+                l = (d.getVar(k) or "").split()
+                for item in sorted(contains[k]):
+                    for word in item.split():
+                        if not word in l:
+                            newvalue += "\n%s{%s} = Unset" % (k, item)
+                            break
+                    else:
+                        newvalue += "\n%s{%s} = Set" % (k, item)
+            if not newvalue:
+                return value
+            if not value:
+                return newvalue
+            return value + newvalue
+
+        def handle_remove(value, deps, removes, d):
+            for r in sorted(removes):
+                r2 = d.expandWithRefs(r, None)
+                value += "\n_remove of %s" % r
+                deps |= r2.references
+                deps = deps | (keys & r2.execs)
+            return value
 
         if "vardepvalue" in varflags:
             value = varflags.get("vardepvalue")
         elif varflags.get("func"):
             if varflags.get("python"):
-                value = codeparsedata.getVarFlag(key, "_content", False)
+                value = d.getVarFlag(key, "_content", False)
                 parser = bb.codeparser.PythonParser(key, logger)
                 parser.parse_python(value, filename=varflags.get("filename"), lineno=varflags.get("lineno"))
                 deps = deps | parser.references
                 deps = deps | (keys & parser.execs)
-                value = handle_contains(value, parser.contains, exclusions, d)
+                value = handle_contains(value, parser.contains, d)
             else:
-                value, parsedvar = codeparsedata.getVarFlag(key, "_content", False, retparser=True)
+                value, parsedvar = d.getVarFlag(key, "_content", False, retparser=True)
                 parser = bb.codeparser.ShellParser(key, logger)
                 parser.parse_shell(parsedvar.value)
                 deps = deps | shelldeps
                 deps = deps | parsedvar.references
                 deps = deps | (keys & parser.execs) | (keys & parsedvar.execs)
-                value = handle_contains(value, parsedvar.contains, exclusions, d)
+                value = handle_contains(value, parsedvar.contains, d)
                 if hasattr(parsedvar, "removes"):
                     value = handle_remove(value, deps, parsedvar.removes, d)
             if vardeps is None:
@@ -340,7 +341,7 @@ def build_dependencies(key, keys, mod_funcs, shelldeps, varflagsexcl, ignored_va
             value, parser = d.getVarFlag(key, "_content", False, retparser=True)
             deps |= parser.references
             deps = deps | (keys & parser.execs)
-            value = handle_contains(value, parser.contains, exclusions, d)
+            value = handle_contains(value, parser.contains, d)
             if hasattr(parser, "removes"):
                 value = handle_remove(value, deps, parser.removes, d)
 
@@ -360,50 +361,43 @@ def build_dependencies(key, keys, mod_funcs, shelldeps, varflagsexcl, ignored_va
                 deps |= set(varfdeps)
 
         deps |= set((vardeps or "").split())
-        deps -= set(exclusions)
-        deps -= ignored_vars
+        deps -= set(varflags.get("vardepsexclude", "").split())
     except bb.parse.SkipRecipe:
         raise
     except Exception as e:
         bb.warn("Exception during build_dependencies for %s" % key)
         raise
-    return frozenset(deps), value
+    return deps, value
     #bb.note("Variable %s references %s and calls %s" % (key, str(deps), str(execs)))
     #d.setVarFlag(key, "vardeps", deps)
 
-def generate_dependencies(d, ignored_vars):
+def generate_dependencies(d, whitelist):
 
-    mod_funcs = set(bb.codeparser.modulecode_deps.keys())
-    keys = set(key for key in d if not key.startswith("__")) | mod_funcs
-    shelldeps = set(key for key in d.getVar("__exportlist", False) if bb.utils.to_boolean(d.getVarFlag(key, "export")) and not bb.utils.to_boolean(d.getVarFlag(key, "unexport")))
+    keys = set(key for key in d if not key.startswith("__"))
+    shelldeps = set(key for key in d.getVar("__exportlist", False) if d.getVarFlag(key, "export", False) and not d.getVarFlag(key, "unexport", False))
     varflagsexcl = d.getVar('BB_SIGNATURE_EXCLUDE_FLAGS')
 
-    codeparserd = d.createCopy()
-    for forced in (d.getVar('BB_HASH_CODEPARSER_VALS') or "").split():
-        key, value = forced.split("=", 1)
-        codeparserd.setVar(key, value)
-
     deps = {}
     values = {}
 
     tasklist = d.getVar('__BBTASKS', False) or []
     for task in tasklist:
-        deps[task], values[task] = build_dependencies(task, keys, mod_funcs, shelldeps, varflagsexcl, ignored_vars, d, codeparserd)
+        deps[task], values[task] = build_dependencies(task, keys, shelldeps, varflagsexcl, d)
         newdeps = deps[task]
         seen = set()
         while newdeps:
-            nextdeps = newdeps
+            nextdeps = newdeps - whitelist
             seen |= nextdeps
             newdeps = set()
             for dep in nextdeps:
                 if dep not in deps:
-                    deps[dep], values[dep] = build_dependencies(dep, keys, mod_funcs, shelldeps, varflagsexcl, ignored_vars, d, codeparserd)
+                    deps[dep], values[dep] = build_dependencies(dep, keys, shelldeps, varflagsexcl, d)
                 newdeps |=  deps[dep]
             newdeps -= seen
         #print "For %s: %s" % (task, str(deps[task]))
     return tasklist, deps, values
 
-def generate_dependency_hash(tasklist, gendeps, lookupcache, ignored_vars, fn):
+def generate_dependency_hash(tasklist, gendeps, lookupcache, whitelist, fn):
     taskdeps = {}
     basehash = {}
 
@@ -412,10 +406,9 @@ def generate_dependency_hash(tasklist, gendeps, lookupcache, ignored_vars, fn):
 
         if data is None:
             bb.error("Task %s from %s seems to be empty?!" % (task, fn))
-            data = []
-        else:
-            data = [data]
+            data = ''
 
+        gendeps[task] -= whitelist
         newdeps = gendeps[task]
         seen = set()
         while newdeps:
@@ -423,24 +416,27 @@ def generate_dependency_hash(tasklist, gendeps, lookupcache, ignored_vars, fn):
             seen |= nextdeps
             newdeps = set()
             for dep in nextdeps:
+                if dep in whitelist:
+                    continue
+                gendeps[dep] -= whitelist
                 newdeps |= gendeps[dep]
             newdeps -= seen
 
         alldeps = sorted(seen)
         for dep in alldeps:
-            data.append(dep)
+            data = data + dep
             var = lookupcache[dep]
             if var is not None:
-                data.append(str(var))
+                data = data + str(var)
         k = fn + ":" + task
-        basehash[k] = hashlib.sha256("".join(data).encode("utf-8")).hexdigest()
-        taskdeps[task] = frozenset(seen)
+        basehash[k] = hashlib.sha256(data.encode("utf-8")).hexdigest()
+        taskdeps[task] = alldeps
 
     return taskdeps, basehash
 
 def inherits_class(klass, d):
     val = d.getVar('__inherit_cache', False) or []
-    needle = '/%s.bbclass' % klass
+    needle = os.path.join('classes', '%s.bbclass' % klass)
     for v in val:
         if v.endswith(needle):
             return True

bitbake/lib/bb/data_smart.py

@@ -16,10 +16,7 @@ BitBake build tools.
 #
 # Based on functions from the base bb module, Copyright 2003 Holger Schurig
 
-import builtins
-import copy
-import re
-import sys
+import copy, re, sys, traceback
 from collections.abc import MutableMapping
 import logging
 import hashlib
@@ -32,22 +29,10 @@ logger = logging.getLogger("BitBake.Data")
 __setvar_keyword__ = [":append", ":prepend", ":remove"]
 __setvar_regexp__ = re.compile(r'(?P<base>.*?)(?P<keyword>:append|:prepend|:remove)(:(?P<add>[^A-Z]*))?$')
 __expand_var_regexp__ = re.compile(r"\${[a-zA-Z0-9\-_+./~:]+?}")
-__expand_python_regexp__ = re.compile(r"\${@(?:{.*?}|.)+?}")
+__expand_python_regexp__ = re.compile(r"\${@.+?}")
 __whitespace_split__ = re.compile(r'(\s)')
 __override_regexp__ = re.compile(r'[a-z0-9]+')
 
-bitbake_renamed_vars = {
-    "BB_ENV_WHITELIST": "BB_ENV_PASSTHROUGH",
-    "BB_ENV_EXTRAWHITE": "BB_ENV_PASSTHROUGH_ADDITIONS",
-    "BB_HASHBASE_WHITELIST": "BB_BASEHASH_IGNORE_VARS",
-    "BB_HASHCONFIG_WHITELIST": "BB_HASHCONFIG_IGNORE_VARS",
-    "BB_HASHTASK_WHITELIST": "BB_TASKHASH_IGNORE_TASKS",
-    "BB_SETSCENE_ENFORCE_WHITELIST": "BB_SETSCENE_ENFORCE_IGNORE_TASKS",
-    "MULTI_PROVIDER_WHITELIST": "BB_MULTI_PROVIDER_ALLOWED",
-    "BB_STAMP_WHITELIST": "is a deprecated variable and support has been removed",
-    "BB_STAMP_POLICY": "is a deprecated variable and support has been removed",
-}
-
 def infer_caller_details(loginfo, parent = False, varval = True):
     """Save the caller the trouble of specifying everything."""
     # Save effort.
@@ -95,11 +80,10 @@ def infer_caller_details(loginfo, parent = False, varval = True):
             loginfo['func'] = func
 
 class VariableParse:
-    def __init__(self, varname, d, unexpanded_value = None, val = None):
+    def __init__(self, varname, d, val = None):
         self.varname = varname
         self.d = d
         self.value = val
-        self.unexpanded_value = unexpanded_value
 
         self.references = set()
         self.execs = set()
@@ -123,11 +107,6 @@ class VariableParse:
             else:
                 code = match.group()[3:-1]
 
-            # Do not run code that contains one or more unexpanded variables
-            # instead return the code with the characters we removed put back
-            if __expand_var_regexp__.findall(code):
-                return "${@" + code + "}"
-
             if self.varname:
                 varname = 'Var <%s>' % self.varname
             else:
@@ -153,21 +132,16 @@ class VariableParse:
             value = utils.better_eval(codeobj, DataContext(self.d), {'d' : self.d})
             return str(value)
 
-class DataContext(dict):
-    excluded = set([i for i in dir(builtins) if not i.startswith('_')] + ['oe'])
 
+class DataContext(dict):
     def __init__(self, metadata, **kwargs):
         self.metadata = metadata
         dict.__init__(self, **kwargs)
         self['d'] = metadata
-        self.context = set(bb.utils.get_context())
 
     def __missing__(self, key):
-        if key in self.excluded or key in self.context:
-            raise KeyError(key)
-
         value = self.metadata.getVar(key)
-        if value is None:
+        if value is None or self.metadata.getVarFlag(key, 'func', False):
             raise KeyError(key)
         else:
             return value
@@ -362,16 +336,6 @@ class VariableHistory(object):
                 lines.append(line)
         return lines
 
-    def get_variable_refs(self, var):
-        """Return a dict of file/line references"""
-        var_history = self.variable(var)
-        refs = {}
-        for event in var_history:
-            if event['file'] not in refs:
-                refs[event['file']] = []
-            refs[event['file']].append(event['line'])
-        return refs
-
     def get_variable_items_files(self, var):
         """
         Use variable history to map items added to a list variable and
@@ -406,23 +370,6 @@ class VariableHistory(object):
             else:
                 self.variables[var] = []
 
-def _print_rename_error(var, loginfo, renamedvars, fullvar=None):
-    info = ""
-    if "file" in loginfo:
-        info = " file: %s" % loginfo["file"]
-    if "line" in loginfo:
-        info += " line: %s" % loginfo["line"]
-    if fullvar and fullvar != var:
-        info += " referenced as: %s" % fullvar
-    if info:
-        info = " (%s)" % info.strip()
-    renameinfo = renamedvars[var]
-    if " " in renameinfo:
-        # A space signals a string to display instead of a rename
-        bb.erroronce('Variable %s %s%s' % (var, renameinfo, info))
-    else:
-        bb.erroronce('Variable %s has been renamed to %s%s' % (var, renameinfo, info))
-
 class DataSmart(MutableMapping):
     def __init__(self):
         self.dict = {}
@@ -430,8 +377,6 @@ class DataSmart(MutableMapping):
         self.inchistory = IncludeHistory()
         self.varhistory = VariableHistory(self)
         self._tracking = False
-        self._var_renames = {}
-        self._var_renames.update(bitbake_renamed_vars)
 
         self.expand_cache = {}
 
@@ -453,9 +398,9 @@ class DataSmart(MutableMapping):
     def expandWithRefs(self, s, varname):
 
         if not isinstance(s, str): # sanity check
-            return VariableParse(varname, self, s, s)
+            return VariableParse(varname, self, s)
 
-        varparse = VariableParse(varname, self, s)
+        varparse = VariableParse(varname, self)
 
         while s.find('${') != -1:
             olds = s
@@ -487,19 +432,24 @@ class DataSmart(MutableMapping):
     def expand(self, s, varname = None):
         return self.expandWithRefs(s, varname).value
 
+    def finalize(self, parent = False):
+        return
+
+    def internal_finalize(self, parent = False):
+        """Performs final steps upon the datastore, including application of overrides"""
+        self.overrides = None
+
     def need_overrides(self):
         if self.overrides is not None:
             return
         if self.inoverride:
             return
-        overrride_stack = []
         for count in range(5):
             self.inoverride = True
             # Can end up here recursively so setup dummy values
             self.overrides = []
             self.overridesset = set()
             self.overrides = (self.getVar("OVERRIDES") or "").split(":") or []
-            overrride_stack.append(self.overrides)
             self.overridesset = set(self.overrides)
             self.inoverride = False
             self.expand_cache = {}
@@ -509,7 +459,7 @@ class DataSmart(MutableMapping):
             self.overrides = newoverrides
             self.overridesset = set(self.overrides)
         else:
-            bb.fatal("Overrides could not be expanded into a stable state after 5 iterations, overrides must be being referenced by other overridden variables in some recursive fashion. Please provide your configuration to bitbake-devel so we can laugh, er, I mean try and understand how to make it work. The list of failing override expansions: %s" % "\n".join(str(s) for s in overrride_stack))
+            bb.fatal("Overrides could not be expanded into a stable state after 5 iterations, overrides must be being referenced by other overridden variables in some recursive fashion. Please provide your configuration to bitbake-devel so we can laugh, er, I mean try and understand how to make it work.")
 
     def initVar(self, var):
         self.expand_cache = {}
@@ -520,44 +470,36 @@ class DataSmart(MutableMapping):
         dest = self.dict
         while dest:
             if var in dest:
-                return dest[var]
+                return dest[var], self.overridedata.get(var, None)
 
             if "_data" not in dest:
                 break
             dest = dest["_data"]
-        return None
+        return None, self.overridedata.get(var, None)
 
     def _makeShadowCopy(self, var):
         if var in self.dict:
             return
 
-        local_var = self._findVar(var)
+        local_var, _ = self._findVar(var)
 
         if local_var:
             self.dict[var] = copy.copy(local_var)
         else:
             self.initVar(var)
 
-    def hasOverrides(self, var):
-        return var in self.overridedata
 
     def setVar(self, var, value, **loginfo):
         #print("var=" + str(var) + "  val=" + str(value))
 
-        if not var.startswith("__anon_") and ("_append" in var or "_prepend" in var or "_remove" in var):
+        if "_append" in var or "_prepend" in var or "_remove" in var:
             info = "%s" % var
-            if "file" in loginfo:
-                info += " file: %s" % loginfo["file"]
-            if "line" in loginfo:
-                info += " line: %s" % loginfo["line"]
+            if "filename" in loginfo:
+                info += " file: %s" % loginfo[filename]
+            if "lineno" in loginfo:
+                info += " line: %s" % loginfo[lineno]
             bb.fatal("Variable %s contains an operation using the old override syntax. Please convert this layer/metadata before attempting to use with a newer bitbake." % info)
 
-        shortvar = var.split(":", 1)[0]
-        if shortvar in self._var_renames:
-            _print_rename_error(shortvar, loginfo, self._var_renames, fullvar=var)
-            # Mark that we have seen a renamed variable
-            self.setVar("_FAILPARSINGERRORHANDLED", True)
-
         self.expand_cache = {}
         parsing=False
         if 'parsing' in loginfo:
@@ -639,7 +581,7 @@ class DataSmart(MutableMapping):
                 nextnew.update(vardata.references)
                 nextnew.update(vardata.contains.keys())
             new = nextnew
-        self.overrides = None
+        self.internal_finalize(True)
 
     def _setvar_update_overrides(self, var, **loginfo):
         # aka pay the cookie monster
@@ -679,11 +621,10 @@ class DataSmart(MutableMapping):
             self.varhistory.record(**loginfo)
             self.setVar(newkey, val, ignore=True, parsing=True)
 
-        srcflags = self.getVarFlags(key, False, True) or {}
-        for i in srcflags:
-            if i not in (__setvar_keyword__):
+        for i in (__setvar_keyword__):
+            src = self.getVarFlag(key, i, False)
+            if src is None:
                 continue
-            src = srcflags[i]
 
             dest = self.getVarFlag(newkey, i, False) or []
             dest.extend(src)
@@ -726,7 +667,7 @@ class DataSmart(MutableMapping):
         if ':' in var:
             override = var[var.rfind(':')+1:]
             shortvar = var[:var.rfind(':')]
-            while override and __override_regexp__.match(override):
+            while override and override.islower():
                 try:
                     if shortvar in self.overridedata:
                         # Force CoW by recreating the list first
@@ -744,14 +685,6 @@ class DataSmart(MutableMapping):
     def setVarFlag(self, var, flag, value, **loginfo):
         self.expand_cache = {}
 
-        if var == "BB_RENAMED_VARIABLES":
-            self._var_renames[flag] = value
-
-        if var in self._var_renames:
-            _print_rename_error(var, loginfo, self._var_renames)
-            # Mark that we have seen a renamed variable
-            self.setVar("_FAILPARSINGERRORHANDLED", True)
-
         if 'op' not in loginfo:
             loginfo['op'] = "set"
         loginfo['flag'] = flag
@@ -781,18 +714,13 @@ class DataSmart(MutableMapping):
                 return None
             cachename = var + "[" + flag + "]"
 
-        if not expand and retparser and cachename in self.expand_cache:
-            return self.expand_cache[cachename].unexpanded_value, self.expand_cache[cachename]
-
         if expand and cachename in self.expand_cache:
             return self.expand_cache[cachename].value
 
-        local_var = self._findVar(var)
+        local_var, overridedata = self._findVar(var)
         value = None
         removes = set()
-        if flag == "_content" and not parsing:
-            overridedata = self.overridedata.get(var, None)
-        if flag == "_content" and not parsing and overridedata is not None:
+        if flag == "_content" and overridedata is not None and not parsing:
             match = False
             active = {}
             self.need_overrides()
@@ -882,7 +810,7 @@ class DataSmart(MutableMapping):
                     expanded_removes[r] = self.expand(r).split()
 
                 parser.removes = set()
-                val = []
+                val = ""
                 for v in __whitespace_split__.split(parser.value):
                     skip = False
                     for r in removes:
@@ -891,8 +819,8 @@ class DataSmart(MutableMapping):
                             skip = True
                     if skip:
                         continue
-                    val.append(v)
-                parser.value = "".join(val)
+                    val = val + v
+                parser.value = val
                 if expand:
                     value = parser.value
 
@@ -907,7 +835,7 @@ class DataSmart(MutableMapping):
     def delVarFlag(self, var, flag, **loginfo):
         self.expand_cache = {}
 
-        local_var = self._findVar(var)
+        local_var, _ = self._findVar(var)
         if not local_var:
             return
         if not var in self.dict:
@@ -950,7 +878,7 @@ class DataSmart(MutableMapping):
             self.dict[var][i] = flags[i]
 
     def getVarFlags(self, var, expand = False, internalflags=False):
-        local_var = self._findVar(var)
+        local_var, _ = self._findVar(var)
         flags = {}
 
         if local_var:
@@ -996,7 +924,6 @@ class DataSmart(MutableMapping):
         data.inchistory = self.inchistory.copy()
 
         data._tracking = self._tracking
-        data._var_renames = self._var_renames
 
         data.overrides = None
         data.overridevars = copy.copy(self.overridevars)
@@ -1019,7 +946,7 @@ class DataSmart(MutableMapping):
         value = self.getVar(variable, False)
         for key in keys:
             referrervalue = self.getVar(key, False)
-            if referrervalue and isinstance(referrervalue, str) and ref in referrervalue:
+            if referrervalue and ref in referrervalue:
                 self.setVar(key, referrervalue.replace(ref, value))
 
     def localkeys(self):
@@ -1085,10 +1012,10 @@ class DataSmart(MutableMapping):
         d = self.createCopy()
         bb.data.expandKeys(d)
 
-        config_ignore_vars = set((d.getVar("BB_HASHCONFIG_IGNORE_VARS") or "").split())
+        config_whitelist = set((d.getVar("BB_HASHCONFIG_WHITELIST") or "").split())
         keys = set(key for key in iter(d) if not key.startswith("__"))
         for key in keys:
-            if key in config_ignore_vars:
+            if key in config_whitelist:
                 continue
 
             value = d.getVar(key, False) or ""

bitbake/lib/bb/event.py

@@ -40,7 +40,7 @@ class HeartbeatEvent(Event):
     """Triggered at regular time intervals of 10 seconds. Other events can fire much more often
        (runQueueTaskStarted when there are many short tasks) or not at all for long periods
        of time (again runQueueTaskStarted, when there is just one long-running task), so this
-       event is more suitable for doing some task-independent work occasionally."""
+       event is more suitable for doing some task-independent work occassionally."""
     def __init__(self, time):
         Event.__init__(self)
         self.time = time
@@ -68,39 +68,29 @@ _catchall_handlers = {}
 _eventfilter = None
 _uiready = False
 _thread_lock = threading.Lock()
-_heartbeat_enabled = False
-_should_exit = threading.Event()
+_thread_lock_enabled = False
+
+if hasattr(__builtins__, '__setitem__'):
+    builtins = __builtins__
+else:
+    builtins = __builtins__.__dict__
 
 def enable_threadlock():
-    # Always needed now
-    return
+    global _thread_lock_enabled
+    _thread_lock_enabled = True
 
 def disable_threadlock():
-    # Always needed now
-    return
-
-def enable_heartbeat():
-    global _heartbeat_enabled
-    _heartbeat_enabled = True
-
-def disable_heartbeat():
-    global _heartbeat_enabled
-    _heartbeat_enabled = False
-
-#
-# In long running code, this function should be called periodically
-# to check if we should exit due to an interuption (.e.g Ctrl+C from the UI)
-#
-def check_for_interrupts(d):
-    global _should_exit
-    if _should_exit.is_set():
-        bb.warn("Exiting due to interrupt.")
-        raise bb.BBHandledException()
+    global _thread_lock_enabled
+    _thread_lock_enabled = False
 
 def execute_handler(name, handler, event, d):
     event.data = d
+    addedd = False
+    if 'd' not in builtins:
+        builtins['d'] = d
+        addedd = True
     try:
-        ret = handler(event, d)
+        ret = handler(event)
     except (bb.parse.SkipRecipe, bb.BBHandledException):
         raise
     except Exception:
@@ -114,7 +104,8 @@ def execute_handler(name, handler, event, d):
         raise
     finally:
         del event.data
-
+        if addedd:
+            del builtins['d']
 
 def fire_class_handlers(event, d):
     if isinstance(event, logging.LogRecord):
@@ -141,14 +132,8 @@ def print_ui_queue():
     if not _uiready:
         from bb.msg import BBLogFormatter
         # Flush any existing buffered content
-        try:
-            sys.stdout.flush()
-        except:
-            pass
-        try:
-            sys.stderr.flush()
-        except:
-            pass
+        sys.stdout.flush()
+        sys.stderr.flush()
         stdout = logging.StreamHandler(sys.stdout)
         stderr = logging.StreamHandler(sys.stderr)
         formatter = BBLogFormatter("%(levelname)s: %(message)s")
@@ -189,30 +174,36 @@ def print_ui_queue():
 
 def fire_ui_handlers(event, d):
     global _thread_lock
+    global _thread_lock_enabled
 
     if not _uiready:
         # No UI handlers registered yet, queue up the messages
         ui_queue.append(event)
         return
 
-    with bb.utils.lock_timeout(_thread_lock):
-        errors = []
-        for h in _ui_handlers:
-            #print "Sending event %s" % event
-            try:
-                 if not _ui_logfilters[h].filter(event):
-                     continue
-                 # We use pickle here since it better handles object instances
-                 # which xmlrpc's marshaller does not. Events *must* be serializable
-                 # by pickle.
-                 if hasattr(_ui_handlers[h].event, "sendpickle"):
-                    _ui_handlers[h].event.sendpickle((pickle.dumps(event)))
-                 else:
-                    _ui_handlers[h].event.send(event)
-            except:
-                errors.append(h)
-        for h in errors:
-            del _ui_handlers[h]
+    if _thread_lock_enabled:
+        _thread_lock.acquire()
+
+    errors = []
+    for h in _ui_handlers:
+        #print "Sending event %s" % event
+        try:
+             if not _ui_logfilters[h].filter(event):
+                 continue
+             # We use pickle here since it better handles object instances
+             # which xmlrpc's marshaller does not. Events *must* be serializable
+             # by pickle.
+             if hasattr(_ui_handlers[h].event, "sendpickle"):
+                _ui_handlers[h].event.sendpickle((pickle.dumps(event)))
+             else:
+                _ui_handlers[h].event.send(event)
+        except:
+            errors.append(h)
+    for h in errors:
+        del _ui_handlers[h]
+
+    if _thread_lock_enabled:
+        _thread_lock.release()
 
 def fire(event, d):
     """Fire off an Event"""
@@ -256,12 +247,12 @@ def register(name, handler, mask=None, filename=None, lineno=None, data=None):
     if handler is not None:
         # handle string containing python code
         if isinstance(handler, str):
-            tmp = "def %s(e, d):\n%s" % (name, handler)
+            tmp = "def %s(e):\n%s" % (name, handler)
             try:
                 code = bb.methodpool.compile_cache(tmp)
                 if not code:
                     if filename is None:
-                        filename = "%s(e, d)" % name
+                        filename = "%s(e)" % name
                     code = compile(tmp, filename, "exec", ast.PyCF_ONLY_AST)
                     if lineno is not None:
                         ast.increment_lineno(code, lineno-1)
@@ -326,23 +317,21 @@ def set_eventfilter(func):
     _eventfilter = func
 
 def register_UIHhandler(handler, mainui=False):
-    with bb.utils.lock_timeout(_thread_lock):
-        bb.event._ui_handler_seq = bb.event._ui_handler_seq + 1
-        _ui_handlers[_ui_handler_seq] = handler
-        level, debug_domains = bb.msg.constructLogOptions()
-        _ui_logfilters[_ui_handler_seq] = UIEventFilter(level, debug_domains)
-        if mainui:
-            global _uiready
-            _uiready = _ui_handler_seq
-        return _ui_handler_seq
+    bb.event._ui_handler_seq = bb.event._ui_handler_seq + 1
+    _ui_handlers[_ui_handler_seq] = handler
+    level, debug_domains = bb.msg.constructLogOptions()
+    _ui_logfilters[_ui_handler_seq] = UIEventFilter(level, debug_domains)
+    if mainui:
+        global _uiready
+        _uiready = _ui_handler_seq
+    return _ui_handler_seq
 
 def unregister_UIHhandler(handlerNum, mainui=False):
     if mainui:
         global _uiready
         _uiready = False
-    with bb.utils.lock_timeout(_thread_lock):
-        if handlerNum in _ui_handlers:
-            del _ui_handlers[handlerNum]
+    if handlerNum in _ui_handlers:
+        del _ui_handlers[handlerNum]
     return
 
 def get_uihandler():
@@ -497,7 +486,7 @@ class BuildCompleted(BuildBase, OperationCompleted):
         BuildBase.__init__(self, n, p, failures)
 
 class DiskFull(Event):
-    """Disk full case build halted"""
+    """Disk full case build aborted"""
     def __init__(self, dev, type, freespace, mountpoint):
         Event.__init__(self)
         self._dev = dev
@@ -775,7 +764,7 @@ class LogHandler(logging.Handler):
 class MetadataEvent(Event):
     """
     Generic event that target for OE-Core classes
-    to report information during asynchronous execution
+    to report information during asynchrous execution
     """
     def __init__(self, eventtype, eventdata):
         Event.__init__(self)
@@ -856,11 +845,3 @@ class FindSigInfoResult(Event):
     def __init__(self, result):
         Event.__init__(self)
         self.result = result
-
-class ParseError(Event):
-    """
-    Event to indicate parse failed
-    """
-    def __init__(self, msg):
-        super().__init__()
-        self._msg = msg

bitbake/lib/bb/exceptions.py

@@ -1,6 +1,4 @@
 #
-# Copyright BitBake Contributors
-#
 # SPDX-License-Identifier: GPL-2.0-only
 #
 

bitbake/lib/bb/fetch2/README

@@ -1,57 +0,0 @@
-There are expectations of users of the fetcher code. This file attempts to document
-some of the constraints that are present. Some are obvious, some are less so. It is
-documented in the context of how OE uses it but the API calls are generic.
-
-a) network access for sources is only expected to happen in the do_fetch task.
-   This is not enforced or tested but is required so that we can:
-
-   i) audit the sources used (i.e. for license/manifest reasons) 
-   ii) support offline builds with a suitable cache
-   iii) allow work to continue even with downtime upstream
-   iv) allow for changes upstream in incompatible ways
-   v) allow rebuilding of the software in X years time
-
-b) network access is not expected in do_unpack task.
-
-c) you can take DL_DIR and use it as a mirror for offline builds.
-
-d) access to the network is only made when explicitly configured in recipes
-   (e.g. use of AUTOREV, or use of git tags which change revision).
-
-e) fetcher output is deterministic (i.e. if you fetch configuration XXX now it 
-   will match in future exactly in a clean build with a new DL_DIR).
-   One specific pain point example are git tags. They can be replaced and change
-   so the git fetcher has to resolve them with the network. We use git revisions
-   where possible to avoid this and ensure determinism.
-
-f) network access is expected to work with the standard linux proxy variables
-   so that access behind firewalls works (the fetcher sets these in the 
-   environment but only in the do_fetch tasks).
-
-g) access during parsing has to be minimal, a "git ls-remote" for an AUTOREV 
-   git recipe might be ok but you can't expect to checkout a git tree.
-
-h) we need to provide revision information during parsing such that a version
-   for the recipe can be constructed.
-
-i) versions are expected to be able to increase in a way which sorts allowing 
-   package feeds to operate (see PR server required for git revisions to sort).
-
-j) API to query for possible version upgrades of a url is highly desireable to 
-   allow our automated upgrage code to function (it is implied this does always 
-   have network access).
-
-k) Where fixes or changes to behaviour in the fetcher are made, we ask that 
-   test cases are added (run with "bitbake-selftest bb.tests.fetch"). We do 
-   have fairly extensive test coverage of the fetcher as it is the only way
-   to track all of its corner cases, it still doesn't give entire coverage 
-   though sadly.
-   
-l) If using tools during parse time, they will have to be in ASSUME_PROVIDED
-   in OE's context as we can't build git-native, then parse a recipe and use
-   git ls-remote.
-
-Not all fetchers support all features, autorev is optional and doesn't make
-sense for some. Upgrade detection means different things in different contexts
-too.
-

bitbake/lib/bb/fetch2/__init__.py

@@ -113,7 +113,7 @@ class MissingParameterError(BBFetchException):
         self.args = (missing, url)
 
 class ParameterError(BBFetchException):
-    """Exception raised when a url cannot be processed due to invalid parameters."""
+    """Exception raised when a url cannot be proccessed due to invalid parameters."""
     def __init__(self, message, url):
         msg = "URL: '%s' has invalid parameters. %s" % (url, message)
         self.url = url
@@ -182,7 +182,7 @@ class URI(object):
     Some notes about relative URIs: while it's specified that
     a URI beginning with <scheme>:// should either be directly
     followed by a hostname or a /, the old URI handling of the
-    fetch2 library did not conform to this. Therefore, this URI
+    fetch2 library did not comform to this. Therefore, this URI
     class has some kludges to make sure that URIs are parsed in
     a way comforming to bitbake's current usage. This URI class
     supports the following:
@@ -199,7 +199,7 @@ class URI(object):
      file://hostname/absolute/path.diff (would be IETF compliant)
 
     Note that the last case only applies to a list of
-    explicitly allowed schemes (currently only file://), that requires
+    "whitelisted" schemes (currently only file://), that requires
     its URIs to not have a network location.
     """
 
@@ -388,7 +388,7 @@ def decodeurl(url):
             if s:
                 if not '=' in s:
                     raise MalformedUrl(url, "The URL: '%s' is invalid: parameter %s does not specify a value (missing '=')" % (url, s))
-                s1, s2 = s.split('=', 1)
+                s1, s2 = s.split('=')
                 p[s1] = s2
 
     return type, host, urllib.parse.unquote(path), user, pswd, p
@@ -402,24 +402,24 @@ def encodeurl(decoded):
 
     if not type:
         raise MissingParameterError('type', "encoded from the data %s" % str(decoded))
-    url = ['%s://' % type]
+    url = '%s://' % type
     if user and type != "file":
-        url.append("%s" % user)
+        url += "%s" % user
         if pswd:
-            url.append(":%s" % pswd)
-        url.append("@")
+            url += ":%s" % pswd
+        url += "@"
     if host and type != "file":
-        url.append("%s" % host)
+        url += "%s" % host
     if path:
         # Standardise path to ensure comparisons work
         while '//' in path:
             path = path.replace("//", "/")
-        url.append("%s" % urllib.parse.quote(path))
+        url += "%s" % urllib.parse.quote(path)
     if p:
         for parm in p:
-            url.append(";%s=%s" % (parm, p[parm]))
+            url += ";%s=%s" % (parm, p[parm])
 
-    return "".join(url)
+    return url
 
 def uri_replace(ud, uri_find, uri_replace, replacements, d, mirrortarball=None):
     if not ud.url or not uri_find or not uri_replace:
@@ -469,18 +469,14 @@ def uri_replace(ud, uri_find, uri_replace, replacements, d, mirrortarball=None):
                     basename = os.path.basename(mirrortarball)
                     # Kill parameters, they make no sense for mirror tarballs
                     uri_decoded[5] = {}
-                    uri_find_decoded[5] = {}
                 elif ud.localpath and ud.method.supports_checksum(ud):
                     basename = os.path.basename(ud.localpath)
                 if basename:
                     uri_basename = os.path.basename(uri_decoded[loc])
-                    # Prefix with a slash as a sentinel in case
-                    # result_decoded[loc] does not contain one.
-                    path = "/" + result_decoded[loc]
-                    if uri_basename and basename != uri_basename and path.endswith("/" + uri_basename):
-                        result_decoded[loc] = path[1:-len(uri_basename)] + basename
-                    elif not path.endswith("/" + basename):
-                        result_decoded[loc] = os.path.join(path[1:], basename)
+                    if uri_basename and basename != uri_basename and result_decoded[loc].endswith(uri_basename):
+                        result_decoded[loc] = result_decoded[loc].replace(uri_basename, basename)
+                    elif not result_decoded[loc].endswith(basename):
+                        result_decoded[loc] = os.path.join(result_decoded[loc], basename)
         else:
             return None
     result = encodeurl(result_decoded)
@@ -518,7 +514,7 @@ def fetcher_init(d):
     else:
         raise FetchError("Invalid SRCREV cache policy of: %s" % srcrev_policy)
 
-    _checksum_cache.init_cache(d.getVar("BB_CACHEDIR"))
+    _checksum_cache.init_cache(d)
 
     for m in methods:
         if hasattr(m, "init"):
@@ -546,7 +542,7 @@ def mirror_from_string(data):
         bb.warn('Invalid mirror data %s, should have paired members.' % data)
     return list(zip(*[iter(mirrors)]*2))
 
-def verify_checksum(ud, d, precomputed={}, localpath=None, fatal_nochecksum=True):
+def verify_checksum(ud, d, precomputed={}):
     """
     verify the MD5 and SHA256 checksum for downloaded src
 
@@ -560,19 +556,17 @@ def verify_checksum(ud, d, precomputed={}, localpath=None, fatal_nochecksum=True
     file against those in the recipe each time, rather than only after
     downloading. See https://bugzilla.yoctoproject.org/show_bug.cgi?id=5571.
     """
+
     if ud.ignore_checksums or not ud.method.supports_checksum(ud):
         return {}
 
-    if localpath is None:
-        localpath = ud.localpath
-
     def compute_checksum_info(checksum_id):
         checksum_name = getattr(ud, "%s_name" % checksum_id)
 
         if checksum_id in precomputed:
             checksum_data = precomputed[checksum_id]
         else:
-            checksum_data = getattr(bb.utils, "%s_file" % checksum_id)(localpath)
+            checksum_data = getattr(bb.utils, "%s_file" % checksum_id)(ud.localpath)
 
         checksum_expected = getattr(ud, "%s_expected" % checksum_id)
 
@@ -598,13 +592,17 @@ def verify_checksum(ud, d, precomputed={}, localpath=None, fatal_nochecksum=True
             checksum_lines = ["SRC_URI[%s] = \"%s\"" % (ci["name"], ci["data"])]
 
     # If no checksum has been provided
-    if fatal_nochecksum and ud.method.recommends_checksum(ud) and all(ci["expected"] is None for ci in checksum_infos):
+    if ud.method.recommends_checksum(ud) and all(ci["expected"] is None for ci in checksum_infos):
         messages = []
         strict = d.getVar("BB_STRICT_CHECKSUM") or "0"
 
         # If strict checking enabled and neither sum defined, raise error
         if strict == "1":
-            raise NoChecksumError("\n".join(checksum_lines))
+            messages.append("No checksum specified for '%s', please add at " \
+                            "least one to the recipe:" % ud.localpath)
+            messages.extend(checksum_lines)
+            logger.error("\n".join(messages))
+            raise NoChecksumError("Missing SRC_URI checksum", ud.url)
 
         bb.event.fire(MissingChecksumEvent(ud.url, **checksum_event), d)
 
@@ -626,7 +624,7 @@ def verify_checksum(ud, d, precomputed={}, localpath=None, fatal_nochecksum=True
     for ci in checksum_infos:
         if ci["expected"] and ci["expected"] != ci["data"]:
             messages.append("File: '%s' has %s checksum '%s' when '%s' was " \
-                            "expected" % (localpath, ci["id"], ci["data"], ci["expected"]))
+                            "expected" % (ud.localpath, ci["id"], ci["data"], ci["expected"]))
             bad_checksum = ci["data"]
 
     if bad_checksum:
@@ -744,16 +742,13 @@ def subprocess_setup():
     # SIGPIPE errors are known issues with gzip/bash
     signal.signal(signal.SIGPIPE, signal.SIG_DFL)
 
-def mark_recipe_nocache(d):
+def get_autorev(d):
+    #  only not cache src rev in autorev case
     if d.getVar('BB_SRCREV_POLICY') != "cache":
         d.setVar('BB_DONT_CACHE', '1')
-
-def get_autorev(d):
-    mark_recipe_nocache(d)
-    d.setVar("__BBAUTOREV_SEEN", True)
     return "AUTOINC"
 
-def _get_srcrev(d, method_name='sortable_revision'):
+def get_srcrev(d, method_name='sortable_revision'):
     """
     Return the revision string, usually for use in the version string (PV) of the current package
     Most packages usually only have one SCM so we just pass on the call.
@@ -767,34 +762,29 @@ def _get_srcrev(d, method_name='sortable_revision'):
     that fetcher provides a method with the given name and the same signature as sortable_revision.
     """
 
-    d.setVar("__BBSRCREV_SEEN", "1")
     recursion = d.getVar("__BBINSRCREV")
     if recursion:
         raise FetchError("There are recursive references in fetcher variables, likely through SRC_URI")
     d.setVar("__BBINSRCREV", True)
 
     scms = []
-    revs = []
     fetcher = Fetch(d.getVar('SRC_URI').split(), d)
     urldata = fetcher.ud
     for u in urldata:
         if urldata[u].method.supports_srcrev():
             scms.append(u)
 
-    if not scms:
-        d.delVar("__BBINSRCREV")
-        return "", revs
-
+    if len(scms) == 0:
+        raise FetchError("SRCREV was used yet no valid SCM was found in SRC_URI")
 
     if len(scms) == 1 and len(urldata[scms[0]].names) == 1:
         autoinc, rev = getattr(urldata[scms[0]].method, method_name)(urldata[scms[0]], d, urldata[scms[0]].names[0])
-        revs.append(rev)
         if len(rev) > 10:
             rev = rev[:10]
         d.delVar("__BBINSRCREV")
         if autoinc:
-            return "AUTOINC+" + rev, revs
-        return rev, revs
+            return "AUTOINC+" + rev
+        return rev
 
     #
     # Mutiple SCMs are in SRC_URI so we resort to SRCREV_FORMAT
@@ -810,7 +800,6 @@ def _get_srcrev(d, method_name='sortable_revision'):
         ud = urldata[scm]
         for name in ud.names:
             autoinc, rev = getattr(ud.method, method_name)(ud, d, name)
-            revs.append(rev)
             seenautoinc = seenautoinc or autoinc
             if len(rev) > 10:
                 rev = rev[:10]
@@ -828,21 +817,7 @@ def _get_srcrev(d, method_name='sortable_revision'):
         format = "AUTOINC+" + format
 
     d.delVar("__BBINSRCREV")
-    return format, revs
-
-def get_hashvalue(d, method_name='sortable_revision'):
-    pkgv, revs = _get_srcrev(d, method_name=method_name)
-    return " ".join(revs)
-
-def get_pkgv_string(d, method_name='sortable_revision'):
-    pkgv, revs = _get_srcrev(d, method_name=method_name)
-    return pkgv
-
-def get_srcrev(d, method_name='sortable_revision'):
-    pkgv, revs = _get_srcrev(d, method_name=method_name)
-    if not pkgv:
-        raise FetchError("SRCREV was used yet no valid SCM was found in SRC_URI")
-    return pkgv
+    return format
 
 def localpath(url, d):
     fetcher = bb.fetch2.Fetch([url], d)
@@ -860,7 +835,6 @@ FETCH_EXPORT_VARS = ['HOME', 'PATH',
                      'ALL_PROXY', 'all_proxy',
                      'GIT_PROXY_COMMAND',
                      'GIT_SSH',
-                     'GIT_SSH_COMMAND',
                      'GIT_SSL_CAINFO',
                      'GIT_SMART_HTTP',
                      'SSH_AUTH_SOCK', 'SSH_AGENT_PID',
@@ -868,24 +842,10 @@ FETCH_EXPORT_VARS = ['HOME', 'PATH',
                      'DBUS_SESSION_BUS_ADDRESS',
                      'P4CONFIG',
                      'SSL_CERT_FILE',
-                     'NODE_EXTRA_CA_CERTS',
                      'AWS_PROFILE',
                      'AWS_ACCESS_KEY_ID',
                      'AWS_SECRET_ACCESS_KEY',
-                     'AWS_DEFAULT_REGION',
-                     'GIT_CACHE_PATH',
-                     'SSL_CERT_DIR']
-
-def get_fetcher_environment(d):
-    newenv = {}
-    origenv = d.getVar("BB_ORIGENV")
-    for name in bb.fetch2.FETCH_EXPORT_VARS:
-        value = d.getVar(name)
-        if not value and origenv:
-            value = origenv.getVar(name)
-        if value:
-            newenv[name] = value
-    return newenv
+                     'AWS_DEFAULT_REGION']
 
 def runfetchcmd(cmd, d, quiet=False, cleanup=None, log=None, workdir=None):
     """
@@ -1001,7 +961,6 @@ def build_mirroruris(origud, mirrors, ld):
 
                 try:
                     newud = FetchData(newuri, ld)
-                    newud.ignore_checksums = True
                     newud.setup_localpath(ld)
                 except bb.fetch2.BBFetchException as e:
                     logger.debug("Mirror fetch failure for url %s (original url: %s)" % (newuri, origud.url))
@@ -1122,8 +1081,6 @@ def try_mirror_url(fetch, origud, ud, ld, check = False):
 
 def ensure_symlink(target, link_name):
     if not os.path.exists(link_name):
-        dirname = os.path.dirname(link_name)
-        bb.utils.mkdirhier(dirname)
         if os.path.islink(link_name):
             # Broken symbolic link
             os.unlink(link_name)
@@ -1236,7 +1193,6 @@ def srcrev_internal_helper(ud, d, name):
     if srcrev == "INVALID" or not srcrev:
         raise FetchError("Please set a valid SRCREV for url %s (possible key names are %s, or use a ;rev=X URL parameter)" % (str(attempts), ud.url), ud.url)
     if srcrev == "AUTOINC":
-        d.setVar("__BBAUTOREV_ACTED_UPON", True)
         srcrev = ud.method.latest_revision(ud, d, name)
 
     return srcrev
@@ -1248,21 +1204,23 @@ def get_checksum_file_list(d):
     SRC_URI as a space-separated string
     """
     fetch = Fetch([], d, cache = False, localonly = True)
+
+    dl_dir = d.getVar('DL_DIR')
     filelist = []
     for u in fetch.urls:
         ud = fetch.ud[u]
+
         if ud and isinstance(ud.method, local.Local):
-            found = False
-            paths = ud.method.localfile_searchpaths(ud, d)
+            paths = ud.method.localpaths(ud, d)
             for f in paths:
                 pth = ud.decodedurl
-                if os.path.exists(f):
-                    found = True
+                if f.startswith(dl_dir):
+                    # The local fetcher's behaviour is to return a path under DL_DIR if it couldn't find the file anywhere else
+                    if os.path.exists(f):
+                        bb.warn("Getting checksum for %s SRC_URI entry %s: file not found except in DL_DIR" % (d.getVar('PN'), os.path.basename(f)))
+                    else:
+                        bb.warn("Unable to get checksum for %s SRC_URI entry %s: file could not be found" % (d.getVar('PN'), os.path.basename(f)))
                 filelist.append(f + ":" + str(os.path.exists(f)))
-            if not found:
-                bb.fatal(("Unable to get checksum for %s SRC_URI entry %s: file could not be found"
-                            "\nThe following paths were searched:"
-                            "\n%s") % (d.getVar('PN'), os.path.basename(f), '\n'.join(paths)))
 
     return " ".join(filelist)
 
@@ -1309,13 +1267,18 @@ class FetchData(object):
 
             if checksum_name in self.parm:
                 checksum_expected = self.parm[checksum_name]
-            elif self.type not in ["http", "https", "ftp", "ftps", "sftp", "s3", "az", "crate", "gs"]:
+            elif self.type not in ["http", "https", "ftp", "ftps", "sftp", "s3", "az"]:
                 checksum_expected = None
             else:
                 checksum_expected = d.getVarFlag("SRC_URI", checksum_name)
 
             setattr(self, "%s_expected" % checksum_id, checksum_expected)
 
+        for checksum_id in CHECKSUM_LIST:
+            configure_checksum(checksum_id)
+
+        self.ignore_checksums = False
+
         self.names = self.parm.get("name",'default').split(',')
 
         self.method = None
@@ -1337,11 +1300,6 @@ class FetchData(object):
         if hasattr(self.method, "urldata_init"):
             self.method.urldata_init(self, d)
 
-        for checksum_id in CHECKSUM_LIST:
-            configure_checksum(checksum_id)
-
-        self.ignore_checksums = False
-
         if "localpath" in self.parm:
             # if user sets localpath for file, use it instead.
             self.localpath = self.parm["localpath"]
@@ -1421,9 +1379,6 @@ class FetchMethod(object):
         Is localpath something that can be represented by a checksum?
         """
 
-        # We cannot compute checksums for None
-        if urldata.localpath is None:
-            return False
         # We cannot compute checksums for directories
         if os.path.isdir(urldata.localpath):
             return False
@@ -1503,33 +1458,30 @@ class FetchMethod(object):
         cmd = None
 
         if unpack:
-            tar_cmd = 'tar --extract --no-same-owner'
-            if 'striplevel' in urldata.parm:
-                tar_cmd += ' --strip-components=%s' %  urldata.parm['striplevel']
             if file.endswith('.tar'):
-                cmd = '%s -f %s' % (tar_cmd, file)
+                cmd = 'tar x --no-same-owner -f %s' % file
             elif file.endswith('.tgz') or file.endswith('.tar.gz') or file.endswith('.tar.Z'):
-                cmd = '%s -z -f %s' % (tar_cmd, file)
+                cmd = 'tar xz --no-same-owner -f %s' % file
             elif file.endswith('.tbz') or file.endswith('.tbz2') or file.endswith('.tar.bz2'):
-                cmd = 'bzip2 -dc %s | %s -f -' % (file, tar_cmd)
+                cmd = 'bzip2 -dc %s | tar x --no-same-owner -f -' % file
             elif file.endswith('.gz') or file.endswith('.Z') or file.endswith('.z'):
                 cmd = 'gzip -dc %s > %s' % (file, efile)
             elif file.endswith('.bz2'):
                 cmd = 'bzip2 -dc %s > %s' % (file, efile)
             elif file.endswith('.txz') or file.endswith('.tar.xz'):
-                cmd = 'xz -dc %s | %s -f -' % (file, tar_cmd)
+                cmd = 'xz -dc %s | tar x --no-same-owner -f -' % file
             elif file.endswith('.xz'):
                 cmd = 'xz -dc %s > %s' % (file, efile)
             elif file.endswith('.tar.lz'):
-                cmd = 'lzip -dc %s | %s -f -' % (file, tar_cmd)
+                cmd = 'lzip -dc %s | tar x --no-same-owner -f -' % file
             elif file.endswith('.lz'):
                 cmd = 'lzip -dc %s > %s' % (file, efile)
             elif file.endswith('.tar.7z'):
-                cmd = '7z x -so %s | %s -f -' % (file, tar_cmd)
+                cmd = '7z x -so %s | tar x --no-same-owner -f -' % file
             elif file.endswith('.7z'):
                 cmd = '7za x -y %s 1>/dev/null' % file
             elif file.endswith('.tzst') or file.endswith('.tar.zst'):
-                cmd = 'zstd --decompress --stdout %s | %s -f -' % (file, tar_cmd)
+                cmd = 'zstd --decompress --stdout %s | tar x --no-same-owner -f -' % file
             elif file.endswith('.zst'):
                 cmd = 'zstd --decompress --stdout %s > %s' % (file, efile)
             elif file.endswith('.zip') or file.endswith('.jar'):
@@ -1562,7 +1514,7 @@ class FetchMethod(object):
                         raise UnpackError("Unable to unpack deb/ipk package - does not contain data.tar.* file", urldata.url)
                 else:
                     raise UnpackError("Unable to unpack deb/ipk package - could not list contents", urldata.url)
-                cmd = 'ar x %s %s && %s -p -f %s && rm %s' % (file, datafile, tar_cmd, datafile, datafile)
+                cmd = 'ar x %s %s && tar --no-same-owner -xpf %s && rm %s' % (file, datafile, datafile, datafile)
 
         # If 'subdir' param exists, create a dir and use it as destination for unpack cmd
         if 'subdir' in urldata.parm:
@@ -1688,7 +1640,7 @@ class Fetch(object):
         if localonly and cache:
             raise Exception("bb.fetch2.Fetch.__init__: cannot set cache and localonly at same time")
 
-        if not urls:
+        if len(urls) == 0:
             urls = d.getVar("SRC_URI").split()
         self.urls = urls
         self.d = d
@@ -1745,7 +1697,6 @@ class Fetch(object):
         network = self.d.getVar("BB_NO_NETWORK")
         premirroronly = bb.utils.to_boolean(self.d.getVar("BB_FETCH_PREMIRRORONLY"))
 
-        checksum_missing_messages = []
         for u in urls:
             ud = self.ud[u]
             ud.setup_localpath(self.d)
@@ -1757,6 +1708,7 @@ class Fetch(object):
 
             try:
                 self.d.setVar("BB_NO_NETWORK", network)
+
                 if m.verify_donestamp(ud, self.d) and not m.need_update(ud, self.d):
                     done = True
                 elif m.try_premirror(ud, self.d):
@@ -1777,9 +1729,7 @@ class Fetch(object):
                     self.d.setVar("BB_NO_NETWORK", "1")
 
                 firsterr = None
-                verified_stamp = False
-                if done:
-                    verified_stamp = m.verify_donestamp(ud, self.d)
+                verified_stamp = m.verify_donestamp(ud, self.d)
                 if not done and (not verified_stamp or m.need_update(ud, self.d)):
                     try:
                         if not trusted_network(self.d, ud.url):
@@ -1828,28 +1778,17 @@ class Fetch(object):
                     raise ChecksumError("Stale Error Detected")
 
             except BBFetchException as e:
-                if isinstance(e, NoChecksumError):
-                    (message, _) = e.args
-                    checksum_missing_messages.append(message)
-                    continue
-                elif isinstance(e, ChecksumError):
+                if isinstance(e, ChecksumError):
                     logger.error("Checksum failure fetching %s" % u)
                 raise
 
             finally:
                 if ud.lockfile:
                     bb.utils.unlockfile(lf)
-        if checksum_missing_messages:
-            logger.error("Missing SRC_URI checksum, please add those to the recipe: \n%s", "\n".join(checksum_missing_messages))
-            raise BBFetchException("There was some missing checksums in the recipe")
 
     def checkstatus(self, urls=None):
         """
-        Check all URLs exist upstream.
-
-        Returns None if the URLs exist, raises FetchError if the check wasn't
-        successful but there wasn't an error (such as file not found), and
-        raises other exceptions in error cases.
+        Check all urls exist upstream
         """
 
         if not urls:
@@ -1994,8 +1933,6 @@ from . import clearcase
 from . import npm
 from . import npmsw
 from . import az
-from . import crate
-from . import gcp
 
 methods.append(local.Local())
 methods.append(wget.Wget())
@@ -2016,5 +1953,3 @@ methods.append(clearcase.ClearCase())
 methods.append(npm.Npm())
 methods.append(npmsw.NpmShrinkWrap())
 methods.append(az.Az())
-methods.append(crate.Crate())
-methods.append(gcp.GCP())

bitbake/lib/bb/fetch2/gcp.py

@@ -1,98 +0,0 @@
-"""
-BitBake 'Fetch' implementation for Google Cloup Platform Storage.
-
-Class for fetching files from Google Cloud Storage using the
-Google Cloud Storage Python Client. The GCS Python Client must
-be correctly installed, configured and authenticated prior to use.
-Additionally, gsutil must also be installed.
-
-"""
-
-# Copyright (C) 2023, Snap Inc.
-#
-# Based in part on bb.fetch2.s3:
-#    Copyright (C) 2017 Andre McCurdy
-#
-# SPDX-License-Identifier: GPL-2.0-only
-#
-# Based on functions from the base bb module, Copyright 2003 Holger Schurig
-
-import os
-import bb
-import urllib.parse, urllib.error
-from bb.fetch2 import FetchMethod
-from bb.fetch2 import FetchError
-from bb.fetch2 import logger
-
-class GCP(FetchMethod):
-    """
-    Class to fetch urls via GCP's Python API.
-    """
-    def __init__(self):
-        self.gcp_client = None
-
-    def supports(self, ud, d):
-        """
-        Check to see if a given url can be fetched with GCP.
-        """
-        return ud.type in ['gs']
-
-    def recommends_checksum(self, urldata):
-        return True
-
-    def urldata_init(self, ud, d):
-        if 'downloadfilename' in ud.parm:
-            ud.basename = ud.parm['downloadfilename']
-        else:
-            ud.basename = os.path.basename(ud.path)
-
-        ud.localfile = d.expand(urllib.parse.unquote(ud.basename))
-
-    def get_gcp_client(self):
-        from google.cloud import storage
-        self.gcp_client = storage.Client(project=None)
-
-    def download(self, ud, d):
-        """
-        Fetch urls using the GCP API.
-        Assumes localpath was called first.
-        """
-        logger.debug2(f"Trying to download gs://{ud.host}{ud.path} to {ud.localpath}")
-        if self.gcp_client is None:
-            self.get_gcp_client()
-
-        bb.fetch2.check_network_access(d, "gsutil stat", ud.url)
-
-        # Path sometimes has leading slash, so strip it
-        path = ud.path.lstrip("/")
-        blob = self.gcp_client.bucket(ud.host).blob(path)
-        blob.download_to_filename(ud.localpath)
-
-        # Additional sanity checks copied from the wget class (although there
-        # are no known issues which mean these are required, treat the GCP API
-        # tool with a little healthy suspicion).
-        if not os.path.exists(ud.localpath):
-            raise FetchError(f"The GCP API returned success for gs://{ud.host}{ud.path} but {ud.localpath} doesn't exist?!")
-
-        if os.path.getsize(ud.localpath) == 0:
-            os.remove(ud.localpath)
-            raise FetchError(f"The downloaded file for gs://{ud.host}{ud.path} resulted in a zero size file?! Deleting and failing since this isn't right.")
-
-        return True
-
-    def checkstatus(self, fetch, ud, d):
-        """
-        Check the status of a URL.
-        """
-        logger.debug2(f"Checking status of gs://{ud.host}{ud.path}")
-        if self.gcp_client is None:
-            self.get_gcp_client()
-
-        bb.fetch2.check_network_access(d, "gsutil stat", ud.url)
-
-        # Path sometimes has leading slash, so strip it
-        path = ud.path.lstrip("/")
-        if self.gcp_client.bucket(ud.host).blob(path).exists() == False:
-            raise FetchError(f"The GCP API reported that gs://{ud.host}{ud.path} does not exist")
-        else:
-            return True

bitbake/lib/bb/fetch2/git.py

@@ -44,8 +44,7 @@ Supported SRC_URI options are:
 
 - nobranch
    Don't check the SHA validation for branch. set this option for the recipe
-   referring to commit which is valid in any namespace (branch, tag, ...)
-   instead of branch.
+   referring to commit which is valid in tag instead of branch.
    The default is "0", set nobranch=1 if needed.
 
 - usehead
@@ -65,7 +64,6 @@ import fnmatch
 import os
 import re
 import shlex
-import shutil
 import subprocess
 import tempfile
 import bb
@@ -76,9 +74,6 @@ from   bb.fetch2 import runfetchcmd
 from   bb.fetch2 import logger
 
 
-sha1_re = re.compile(r'^[0-9a-f]{40}$')
-slash_re = re.compile(r"/+")
-
 class GitProgressHandler(bb.progress.LineFilterProgressHandler):
     """Extract progress information from git output"""
     def __init__(self, d):
@@ -151,7 +146,6 @@ class Git(FetchMethod):
             # github stopped supporting git protocol
             # https://github.blog/2021-09-01-improving-git-protocol-security-github/#no-more-unauthenticated-git
             ud.proto = "https"
-            bb.warn("URL: %s uses git protocol which is no longer supported by github. Please change to ;protocol=https in the url." % ud.url)
 
         if not ud.proto in ('git', 'file', 'ssh', 'http', 'https', 'rsync'):
             raise bb.fetch2.ParameterError("Invalid protocol type", ud.url)
@@ -175,10 +169,7 @@ class Git(FetchMethod):
             ud.nocheckout = 1
   
         ud.unresolvedrev = {}
-        branches = ud.parm.get("branch", "").split(',')
-        if branches == [""] and not ud.nobranch:
-            bb.warn("URL: %s does not set any branch parameter. The future default branch used by tools and repositories is uncertain and we will therefore soon require this is set in all git urls." % ud.url)
-            branches = ["master"]
+        branches = ud.parm.get("branch", "master").split(',')
         if len(branches) != len(ud.names):
             raise bb.fetch2.ParameterError("The number of name and branch parameters is not balanced", ud.url)
 
@@ -245,7 +236,7 @@ class Git(FetchMethod):
             for name in ud.names:
                 ud.unresolvedrev[name] = 'HEAD'
 
-        ud.basecmd = d.getVar("FETCHCMD_git") or "git -c gc.autoDetach=false -c core.pager=cat"
+        ud.basecmd = d.getVar("FETCHCMD_git") or "git -c core.fsyncobjectfiles=0 -c gc.autoDetach=false"
 
         write_tarballs = d.getVar("BB_GENERATE_MIRROR_TARBALLS") or "0"
         ud.write_tarballs = write_tarballs != "0" or ud.rebaseable
@@ -254,8 +245,8 @@ class Git(FetchMethod):
         ud.setup_revisions(d)
 
         for name in ud.names:
-            # Ensure any revision that doesn't look like a SHA-1 is translated into one
-            if not sha1_re.match(ud.revisions[name] or ''):
+            # Ensure anything that doesn't look like a sha256 checksum/revision is translated into one
+            if not ud.revisions[name] or len(ud.revisions[name]) != 40  or (False in [c in "abcdef0123456789" for c in ud.revisions[name]]):
                 if ud.revisions[name]:
                     ud.unresolvedrev[name] = ud.revisions[name]
                 ud.revisions[name] = self.latest_revision(ud, d, name)
@@ -264,10 +255,10 @@ class Git(FetchMethod):
         if gitsrcname.startswith('.'):
             gitsrcname = gitsrcname[1:]
 
-        # For a rebaseable git repo, it is necessary to keep a mirror tar ball
-        # per revision, so that even if the revision disappears from the
+        # for rebaseable git repo, it is necessary to keep mirror tar ball
+        # per revision, so that even the revision disappears from the
         # upstream repo in the future, the mirror will remain intact and still
-        # contain the revision
+        # contains the revision
         if ud.rebaseable:
             for name in ud.names:
                 gitsrcname = gitsrcname + '_' + ud.revisions[name]
@@ -355,53 +346,17 @@ class Git(FetchMethod):
         if ud.shallow and os.path.exists(ud.fullshallow) and self.need_update(ud, d):
             ud.localpath = ud.fullshallow
             return
-        elif os.path.exists(ud.fullmirror) and self.need_update(ud, d):
-            if not os.path.exists(ud.clonedir):
-                bb.utils.mkdirhier(ud.clonedir)
-                runfetchcmd("tar -xzf %s" % ud.fullmirror, d, workdir=ud.clonedir)
-            else:
-                tmpdir = tempfile.mkdtemp(dir=d.getVar('DL_DIR'))
-                runfetchcmd("tar -xzf %s" % ud.fullmirror, d, workdir=tmpdir)
-                fetch_cmd = "LANG=C %s fetch -f --progress %s " % (ud.basecmd, shlex.quote(tmpdir))
-                runfetchcmd(fetch_cmd, d, workdir=ud.clonedir)
+        elif os.path.exists(ud.fullmirror) and not os.path.exists(ud.clonedir):
+            bb.utils.mkdirhier(ud.clonedir)
+            runfetchcmd("tar -xzf %s" % ud.fullmirror, d, workdir=ud.clonedir)
+
         repourl = self._get_repo_url(ud)
 
-        needs_clone = False
-        if os.path.exists(ud.clonedir):
-            # The directory may exist, but not be the top level of a bare git
-            # repository in which case it needs to be deleted and re-cloned.
-            try:
-                # Since clones can be bare, use --absolute-git-dir instead of --show-toplevel
-                output = runfetchcmd("LANG=C %s rev-parse --absolute-git-dir" % ud.basecmd, d, workdir=ud.clonedir)
-
-                toplevel = os.path.abspath(output.rstrip())
-                abs_clonedir = os.path.abspath(ud.clonedir).rstrip('/')
-                # The top level Git directory must either be the clone directory
-                # or a child of the clone directory. Any ancestor directory of
-                # the clone directory is not valid as the Git directory (and
-                # probably belongs to some other unrelated repository), so a
-                # clone is required
-                if os.path.commonprefix([abs_clonedir, toplevel]) != abs_clonedir:
-                    logger.warning("Top level directory '%s' doesn't match expected '%s'. Re-cloning", toplevel, ud.clonedir)
-                    needs_clone = True
-            except bb.fetch2.FetchError as e:
-                logger.warning("Unable to get top level for %s (not a git directory?): %s", ud.clonedir, e)
-                needs_clone = True
-
-            if needs_clone:
-                shutil.rmtree(ud.clonedir)
-        else:
-            needs_clone = True
-
         # If the repo still doesn't exist, fallback to cloning it
-        if needs_clone:
-            # We do this since git will use a "-l" option automatically for local urls where possible,
-            # but it doesn't work when git/objects is a symlink, only works when it is a directory.
+        if not os.path.exists(ud.clonedir):
+            # We do this since git will use a "-l" option automatically for local urls where possible
             if repourl.startswith("file://"):
-                repourl_path = repourl[7:]
-                objects = os.path.join(repourl_path, 'objects')
-                if os.path.isdir(objects) and not os.path.islink(objects):
-                    repourl = repourl_path
+                repourl = repourl[7:]
             clone_cmd = "LANG=C %s clone --bare --mirror %s %s --progress" % (ud.basecmd, shlex.quote(repourl), ud.clonedir)
             if ud.proto.lower() != 'file':
                 bb.fetch2.check_network_access(d, clone_cmd, ud.url)
@@ -415,11 +370,7 @@ class Git(FetchMethod):
               runfetchcmd("%s remote rm origin" % ud.basecmd, d, workdir=ud.clonedir)
 
             runfetchcmd("%s remote add --mirror=fetch origin %s" % (ud.basecmd, shlex.quote(repourl)), d, workdir=ud.clonedir)
-
-            if ud.nobranch:
-                fetch_cmd = "LANG=C %s fetch -f --progress %s refs/*:refs/*" % (ud.basecmd, shlex.quote(repourl))
-            else:
-                fetch_cmd = "LANG=C %s fetch -f --progress %s refs/heads/*:refs/heads/* refs/tags/*:refs/tags/*" % (ud.basecmd, shlex.quote(repourl))
+            fetch_cmd = "LANG=C %s fetch -f --progress %s refs/*:refs/*" % (ud.basecmd, shlex.quote(repourl))
             if ud.proto.lower() != 'file':
                 bb.fetch2.check_network_access(d, fetch_cmd, ud.url)
             progresshandler = GitProgressHandler(d)
@@ -444,12 +395,13 @@ class Git(FetchMethod):
 
         if self._contains_lfs(ud, d, ud.clonedir) and self._need_lfs(ud):
             # Unpack temporary working copy, use it to run 'git checkout' to force pre-fetching
-            # of all LFS blobs needed at the srcrev.
+            # of all LFS blobs needed at the the srcrev.
             #
             # It would be nice to just do this inline here by running 'git-lfs fetch'
             # on the bare clonedir, but that operation requires a working copy on some
             # releases of Git LFS.
-            with tempfile.TemporaryDirectory(dir=d.getVar('DL_DIR')) as tmpdir:
+            tmpdir = tempfile.mkdtemp(dir=d.getVar('DL_DIR'))
+            try:
                 # Do the checkout. This implicitly involves a Git LFS fetch.
                 Git.unpack(self, ud, tmpdir, d)
 
@@ -467,6 +419,8 @@ class Git(FetchMethod):
                 # downloaded.
                 if os.path.exists(os.path.join(tmpdir, "git", ".git", "lfs")):
                     runfetchcmd("tar -cf - lfs | tar -xf - -C %s" % ud.clonedir, d, workdir="%s/git/.git" % tmpdir)
+            finally:
+                bb.utils.remove(tmpdir, recurse=True)
 
     def build_mirror_data(self, ud, d):
 
@@ -504,10 +458,7 @@ class Git(FetchMethod):
 
             logger.info("Creating tarball of git repository")
             with create_atomic(ud.fullmirror) as tfile:
-                mtime = runfetchcmd("git log --all -1 --format=%cD", d,
-                        quiet=True, workdir=ud.clonedir)
-                runfetchcmd("tar -czf %s --owner oe:0 --group oe:0 --mtime \"%s\" ."
-                        % (tfile, mtime), d, workdir=ud.clonedir)
+                runfetchcmd("tar -czf %s ." % tfile, d, workdir=ud.clonedir)
             runfetchcmd("touch %s.done" % ud.fullmirror, d)
 
     def clone_shallow_local(self, ud, dest, d):
@@ -569,24 +520,13 @@ class Git(FetchMethod):
     def unpack(self, ud, destdir, d):
         """ unpack the downloaded src to destdir"""
 
-        subdir = ud.parm.get("subdir")
-        subpath = ud.parm.get("subpath")
-        readpathspec = ""
-        def_destsuffix = "git/"
-
-        if subpath:
-            readpathspec = ":%s" % subpath
-            def_destsuffix = "%s/" % os.path.basename(subpath.rstrip('/'))
-
-        if subdir:
-            # If 'subdir' param exists, create a dir and use it as destination for unpack cmd
-            if os.path.isabs(subdir):
-                if not os.path.realpath(subdir).startswith(os.path.realpath(destdir)):
-                    raise bb.fetch2.UnpackError("subdir argument isn't a subdirectory of unpack root %s" % destdir, ud.url)
-                destdir = subdir
-            else:
-                destdir = os.path.join(destdir, subdir)
-            def_destsuffix = ""
+        subdir = ud.parm.get("subpath", "")
+        if subdir != "":
+            readpathspec = ":%s" % subdir
+            def_destsuffix = "%s/" % os.path.basename(subdir.rstrip('/'))
+        else:
+            readpathspec = ""
+            def_destsuffix = "git/"
 
         destsuffix = ud.parm.get("destsuffix", def_destsuffix)
         destdir = ud.destdir = os.path.join(destdir, destsuffix)
@@ -601,12 +541,13 @@ class Git(FetchMethod):
         source_found = False
         source_error = []
 
-        clonedir_is_up_to_date = not self.clonedir_need_update(ud, d)
-        if clonedir_is_up_to_date:
-            runfetchcmd("%s clone %s %s/ %s" % (ud.basecmd, ud.cloneflags, ud.clonedir, destdir), d)
-            source_found = True
-        else:
-            source_error.append("clone directory not available or not up to date: " + ud.clonedir)
+        if not source_found:
+            clonedir_is_up_to_date = not self.clonedir_need_update(ud, d)
+            if clonedir_is_up_to_date:
+                runfetchcmd("%s clone %s %s/ %s" % (ud.basecmd, ud.cloneflags, ud.clonedir, destdir), d)
+                source_found = True
+            else:
+                source_error.append("clone directory not available or not up to date: " + ud.clonedir)
 
         if not source_found:
             if ud.shallow:
@@ -632,7 +573,7 @@ class Git(FetchMethod):
                 bb.note("Repository %s has LFS content but it is not being fetched" % (repourl))
 
         if not ud.nocheckout:
-            if subpath:
+            if subdir != "":
                 runfetchcmd("%s read-tree %s%s" % (ud.basecmd, ud.revisions[ud.names[0]], readpathspec), d,
                             workdir=destdir)
                 runfetchcmd("%s checkout-index -q -f -a" % ud.basecmd, d, workdir=destdir)
@@ -689,6 +630,11 @@ class Git(FetchMethod):
         Check if the repository has 'lfs' (large file) content
         """
 
+        if not ud.nobranch:
+            branchname = ud.branches[ud.names[0]]
+        else:
+            branchname = "master"
+
         # The bare clonedir doesn't use the remote names; it has the branch immediately.
         if wd == ud.clonedir:
             refname = ud.branches[ud.names[0]]
@@ -733,6 +679,7 @@ class Git(FetchMethod):
         Return a unique key for the url
         """
         # Collapse adjacent slashes
+        slash_re = re.compile(r"/+")
         return "git:" + ud.host + slash_re.sub(".", ud.path) + ud.unresolvedrev[name]
 
     def _lsremote(self, ud, d, search):
@@ -765,12 +712,6 @@ class Git(FetchMethod):
         """
         Compute the HEAD revision for the url
         """
-        if not d.getVar("__BBSRCREV_SEEN"):
-            raise bb.fetch2.FetchError("Recipe uses a floating tag/branch '%s' for repo '%s' without a fixed SRCREV yet doesn't call bb.fetch2.get_srcrev() (use SRCPV in PV for OE)." % (ud.unresolvedrev[name], ud.host+ud.path))
-
-        # Ensure we mark as not cached
-        bb.fetch2.mark_recipe_nocache(d)
-
         output = self._lsremote(ud, d, "")
         # Tags of the form ^{} may not work, need to fallback to other form
         if ud.unresolvedrev[name][:5] == "refs/" or ud.usehead:

bitbake/lib/bb/fetch2/gitsm.py

@@ -88,9 +88,9 @@ class GitSM(Git):
                 subrevision[m] = module_hash.split()[2]
 
                 # Convert relative to absolute uri based on parent uri
-                if  uris[m].startswith('..') or uris[m].startswith('./'):
+                if uris[m].startswith('..'):
                     newud = copy.copy(ud)
-                    newud.path = os.path.normpath(os.path.join(newud.path, uris[m]))
+                    newud.path = os.path.realpath(os.path.join(newud.path, uris[m]))
                     uris[m] = Git._get_repo_url(self, newud)
 
         for module in submodules:
@@ -115,21 +115,10 @@ class GitSM(Git):
                     # This has to be a file reference
                     proto = "file"
                     url = "gitsm://" + uris[module]
-            if url.endswith("{}{}".format(ud.host, ud.path)):
-                raise bb.fetch2.FetchError("Submodule refers to the parent repository. This will cause deadlock situation in current version of Bitbake." \
-                                           "Consider using git fetcher instead.")
 
             url += ';protocol=%s' % proto
             url += ";name=%s" % module
             url += ";subpath=%s" % module
-            url += ";nobranch=1"
-            url += ";lfs=%s" % self._need_lfs(ud)
-            # Note that adding "user=" here to give credentials to the
-            # submodule is not supported. Since using SRC_URI to give git://
-            # URL a password is not supported, one have to use one of the
-            # recommended way (eg. ~/.netrc or SSH config) which does specify
-            # the user (See comment in git.py).
-            # So, we will not take patches adding "user=" support here.
 
             ld = d.createCopy()
             # Not necessary to set SRC_URI, since we're passing the URI to
@@ -174,7 +163,7 @@ class GitSM(Git):
         else:
             self.process_submodules(ud, ud.clonedir, need_update_submodule, d)
 
-        if need_update_list:
+        if len(need_update_list) > 0:
             logger.debug('gitsm: Submodules requiring update: %s' % (' '.join(need_update_list)))
             return True
 
@@ -243,12 +232,10 @@ class GitSM(Git):
         ret = self.process_submodules(ud, ud.destdir, unpack_submodules, d)
 
         if not ud.bareclone and ret:
-            # All submodules should already be downloaded and configured in the tree.  This simply
-            # sets up the configuration and checks out the files.  The main project config should
-            # remain unmodified, and no download from the internet should occur. As such, lfs smudge
-            # should also be skipped as these files were already smudged in the fetch stage if lfs
-            # was enabled.
-            runfetchcmd("GIT_LFS_SKIP_SMUDGE=1 %s submodule update --recursive --no-fetch" % (ud.basecmd), d, quiet=True, workdir=ud.destdir)
+            # All submodules should already be downloaded and configured in the tree.  This simply sets
+            # up the configuration and checks out the files.  The main project config should remain
+            # unmodified, and no download from the internet should occur.
+            runfetchcmd("%s submodule update --recursive --no-fetch" % (ud.basecmd), d, quiet=True, workdir=ud.destdir)
 
     def implicit_urldata(self, ud, d):
         import shutil, subprocess, tempfile

bitbake/lib/bb/fetch2/local.py

@@ -41,9 +41,9 @@ class Local(FetchMethod):
         """
         Return the local filename of a given url assuming a successful fetch.
         """
-        return self.localfile_searchpaths(urldata, d)[-1]
+        return self.localpaths(urldata, d)[-1]
 
-    def localfile_searchpaths(self, urldata, d):
+    def localpaths(self, urldata, d):
         """
         Return the local filename of a given url assuming a successful fetch.
         """
@@ -51,14 +51,18 @@ class Local(FetchMethod):
         path = urldata.decodedurl
         newpath = path
         if path[0] == "/":
-            logger.debug2("Using absolute %s" % (path))
             return [path]
         filespath = d.getVar('FILESPATH')
         if filespath:
             logger.debug2("Searching for %s in paths:\n    %s" % (path, "\n    ".join(filespath.split(":"))))
             newpath, hist = bb.utils.which(filespath, path, history=True)
-            logger.debug2("Using %s for %s" % (newpath, path))
             searched.extend(hist)
+        if not os.path.exists(newpath):
+            dldirfile = os.path.join(d.getVar("DL_DIR"), path)
+            logger.debug2("Defaulting to %s for %s" % (dldirfile, path))
+            bb.utils.mkdirhier(os.path.dirname(dldirfile))
+            searched.append(dldirfile)
+            return searched
         return searched
 
     def need_update(self, ud, d):
@@ -74,7 +78,9 @@ class Local(FetchMethod):
             filespath = d.getVar('FILESPATH')
             if filespath:
                 locations = filespath.split(":")
-            msg = "Unable to find file " + urldata.url + " anywhere to download to " + urldata.localpath + ". The paths that were searched were:\n    " + "\n    ".join(locations)
+            locations.append(d.getVar("DL_DIR"))
+
+            msg = "Unable to find file " + urldata.url + " anywhere. The paths that were searched were:\n    " + "\n    ".join(locations)
             raise FetchError(msg)
 
         return True

bitbake/lib/bb/fetch2/npm.py

@@ -44,24 +44,17 @@ def npm_package(package):
     """Convert the npm package name to remove unsupported character"""
     # Scoped package names (with the @) use the same naming convention
     # as the 'npm pack' command.
-    name = re.sub("/", "-", package)
-    name = name.lower()
-    name = re.sub(r"[^\-a-z0-9]", "", name)
-    name = name.strip("-")
-    return name
-
+    if package.startswith("@"):
+        return re.sub("/", "-", package[1:])
+    return package
 
 def npm_filename(package, version):
     """Get the filename of a npm package"""
     return npm_package(package) + "-" + version + ".tgz"
 
-def npm_localfile(package, version=None):
+def npm_localfile(package, version):
     """Get the local filename of a npm package"""
-    if version is not None:
-        filename = npm_filename(package, version)
-    else:
-        filename = package
-    return os.path.join("npm2", filename)
+    return os.path.join("npm2", npm_filename(package, version))
 
 def npm_integrity(integrity):
     """
@@ -79,19 +72,23 @@ def npm_unpack(tarball, destdir, d):
     cmd += " --delay-directory-restore"
     cmd += " --strip-components=1"
     runfetchcmd(cmd, d, workdir=destdir)
-    runfetchcmd("chmod -R +X '%s'" % (destdir), d, quiet=True, workdir=destdir)
+    runfetchcmd("chmod -R +X %s" % (destdir), d, quiet=True, workdir=destdir)
 
 class NpmEnvironment(object):
     """
     Using a npm config file seems more reliable than using cli arguments.
     This class allows to create a controlled environment for npm commands.
     """
-    def __init__(self, d, configs=[], npmrc=None):
+    def __init__(self, d, configs=None, npmrc=None):
         self.d = d
 
-        self.user_config = tempfile.NamedTemporaryFile(mode="w", buffering=1)
-        for key, value in configs:
-            self.user_config.write("%s=%s\n" % (key, value))
+        if configs:
+            self.user_config = tempfile.NamedTemporaryFile(mode="w", buffering=1)
+            self.user_config_name = self.user_config.name
+            for key, value in configs:
+                self.user_config.write("%s=%s\n" % (key, value))
+        else:
+            self.user_config_name = "/dev/null"
 
         if npmrc:
             self.global_config_name = npmrc
@@ -106,14 +103,13 @@ class NpmEnvironment(object):
         """Run npm command in a controlled environment"""
         with tempfile.TemporaryDirectory() as tmpdir:
             d = bb.data.createCopy(self.d)
-            d.setVar("PATH", d.getVar("PATH"))  # PATH might contain $HOME - evaluate it before patching
             d.setVar("HOME", tmpdir)
 
             if not workdir:
                 workdir = tmpdir
 
             def _run(cmd):
-                cmd = "NPM_CONFIG_USERCONFIG=%s " % (self.user_config.name) + cmd
+                cmd = "NPM_CONFIG_USERCONFIG=%s " % (self.user_config_name) + cmd
                 cmd = "NPM_CONFIG_GLOBALCONFIG=%s " % (self.global_config_name) + cmd
                 return runfetchcmd(cmd, d, workdir=workdir)
 
@@ -160,12 +156,12 @@ class Npm(FetchMethod):
             raise ParameterError("Invalid 'version' parameter", ud.url)
 
         # Extract the 'registry' part of the url
-        ud.registry = re.sub(r"^npm://", "https://", ud.url.split(";")[0])
+        ud.registry = re.sub(r"^npm://", "http://", ud.url.split(";")[0])
 
         # Using the 'downloadfilename' parameter as local filename
         # or the npm package name.
         if "downloadfilename" in ud.parm:
-            ud.localfile = npm_localfile(d.expand(ud.parm["downloadfilename"]))
+            ud.localfile = d.expand(ud.parm["downloadfilename"])
         else:
             ud.localfile = npm_localfile(ud.package, ud.version)
 

bitbake/lib/bb/fetch2/npmsw.py

@@ -30,8 +30,6 @@ from bb.fetch2.npm import npm_integrity
 from bb.fetch2.npm import npm_localfile
 from bb.fetch2.npm import npm_unpack
 from bb.utils import is_semver
-from bb.utils import lockfile
-from bb.utils import unlockfile
 
 def foreach_dependencies(shrinkwrap, callback=None, dev=False):
     """
@@ -41,9 +39,8 @@ def foreach_dependencies(shrinkwrap, callback=None, dev=False):
         with:
             name = the package name (string)
             params = the package parameters (dictionary)
-            destdir = the destination of the package (string)
+            deptree = the package dependency tree (array of strings)
     """
-    # For handling old style dependencies entries in shinkwrap files
     def _walk_deps(deps, deptree):
         for name in deps:
             subtree = [*deptree, name]
@@ -53,22 +50,9 @@ def foreach_dependencies(shrinkwrap, callback=None, dev=False):
                     continue
                 elif deps[name].get("bundled", False):
                     continue
-                destsubdirs = [os.path.join("node_modules", dep) for dep in subtree]
-                destsuffix = os.path.join(*destsubdirs)
-                callback(name, deps[name], destsuffix)
+                callback(name, deps[name], subtree)
 
-    # packages entry means new style shrinkwrap file, else use dependencies
-    packages = shrinkwrap.get("packages", None)
-    if packages is not None:
-        for package in packages:
-            if package != "":
-                name = package.split('node_modules/')[-1]
-                package_infos = packages.get(package, {})
-                if dev == False and package_infos.get("dev", False):
-                    continue
-                callback(name, package_infos, package)
-    else:
-        _walk_deps(shrinkwrap.get("dependencies", {}), [])
+    _walk_deps(shrinkwrap.get("dependencies", {}), [])
 
 class NpmShrinkWrap(FetchMethod):
     """Class to fetch all package from a shrinkwrap file"""
@@ -89,10 +73,12 @@ class NpmShrinkWrap(FetchMethod):
         # Resolve the dependencies
         ud.deps = []
 
-        def _resolve_dependency(name, params, destsuffix):
+        def _resolve_dependency(name, params, deptree):
             url = None
             localpath = None
             extrapaths = []
+            destsubdirs = [os.path.join("node_modules", dep) for dep in deptree]
+            destsuffix = os.path.join(*destsubdirs)
             unpack = True
 
             integrity = params.get("integrity", None)
@@ -100,11 +86,7 @@ class NpmShrinkWrap(FetchMethod):
             version = params.get("version", None)
 
             # Handle registry sources
-            if is_semver(version) and integrity:
-                # Handle duplicate dependencies without url
-                if not resolved:
-                    return
-
+            if is_semver(version) and resolved and integrity:
                 localfile = npm_localfile(name, version)
 
                 uri = URI(resolved)
@@ -129,7 +111,7 @@ class NpmShrinkWrap(FetchMethod):
 
             # Handle http tarball sources
             elif version.startswith("http") and integrity:
-                localfile = npm_localfile(os.path.basename(version))
+                localfile = os.path.join("npm2", os.path.basename(version))
 
                 uri = URI(version)
                 uri.params["downloadfilename"] = localfile
@@ -141,28 +123,8 @@ class NpmShrinkWrap(FetchMethod):
 
                 localpath = os.path.join(d.getVar("DL_DIR"), localfile)
 
-            # Handle local tarball and link sources
-            elif version.startswith("file"):
-                localpath = version[5:]
-                if not version.endswith(".tgz"):
-                    unpack = False
-
             # Handle git sources
-            elif version.startswith(("git", "bitbucket","gist")) or (
-                not version.endswith((".tgz", ".tar", ".tar.gz"))
-                and not version.startswith((".", "@", "/"))
-                and "/" in version
-            ):
-                if version.startswith("github:"):
-                    version = "git+https://github.com/" + version[len("github:"):]
-                elif version.startswith("gist:"):
-                    version = "git+https://gist.github.com/" + version[len("gist:"):]
-                elif version.startswith("bitbucket:"):
-                    version = "git+https://bitbucket.org/" + version[len("bitbucket:"):]
-                elif version.startswith("gitlab:"):
-                    version = "git+https://gitlab.com/" + version[len("gitlab:"):]
-                elif not version.startswith(("git+","git:")):
-                    version = "git+https://github.com/" + version
+            elif version.startswith("git"):
                 regex = re.compile(r"""
                     ^
                     git\+
@@ -188,6 +150,12 @@ class NpmShrinkWrap(FetchMethod):
 
                 url = str(uri)
 
+            # Handle local tarball and link sources
+            elif version.startswith("file"):
+                localpath = version[5:]
+                if not version.endswith(".tgz"):
+                    unpack = False
+
             else:
                 raise ParameterError("Unsupported dependency: %s" % name, ud.url)
 
@@ -217,23 +185,17 @@ class NpmShrinkWrap(FetchMethod):
         # This fetcher resolves multiple URIs from a shrinkwrap file and then
         # forwards it to a proxy fetcher. The management of the donestamp file,
         # the lockfile and the checksums are forwarded to the proxy fetcher.
-        shrinkwrap_urls = [dep["url"] for dep in ud.deps if dep["url"]]
-        if shrinkwrap_urls:
-            ud.proxy = Fetch(shrinkwrap_urls, data)
+        ud.proxy = Fetch([dep["url"] for dep in ud.deps if dep["url"]], data)
         ud.needdonestamp = False
 
     @staticmethod
     def _foreach_proxy_method(ud, handle):
         returns = []
-        #Check if there are dependencies before try to fetch them
-        if len(ud.deps) > 0:
-            for proxy_url in ud.proxy.urls:
-                proxy_ud = ud.proxy.ud[proxy_url]
-                proxy_d = ud.proxy.d
-                proxy_ud.setup_localpath(proxy_d)
-                lf = lockfile(proxy_ud.lockfile)
-                returns.append(handle(proxy_ud.method, proxy_ud, proxy_d))
-                unlockfile(lf)
+        for proxy_url in ud.proxy.urls:
+            proxy_ud = ud.proxy.ud[proxy_url]
+            proxy_d = ud.proxy.d
+            proxy_ud.setup_localpath(proxy_d)
+            returns.append(handle(proxy_ud.method, proxy_ud, proxy_d))
         return returns
 
     def verify_donestamp(self, ud, d):

bitbake/lib/bb/fetch2/osc.py

@@ -1,6 +1,4 @@
 #
-# Copyright BitBake Contributors
-#
 # SPDX-License-Identifier: GPL-2.0-only
 #
 """
@@ -11,7 +9,6 @@ Based on the svn "Fetch" implementation.
 
 import logging
 import os
-import re
 import  bb
 from    bb.fetch2 import FetchMethod
 from    bb.fetch2 import FetchError
@@ -39,7 +36,6 @@ class Osc(FetchMethod):
         # Create paths to osc checkouts
         oscdir = d.getVar("OSCDIR") or (d.getVar("DL_DIR") + "/osc")
         relpath = self._strip_leading_slashes(ud.path)
-        ud.oscdir = oscdir
         ud.pkgdir = os.path.join(oscdir, ud.host)
         ud.moddir = os.path.join(ud.pkgdir, relpath, ud.module)
 
@@ -47,13 +43,13 @@ class Osc(FetchMethod):
             ud.revision = ud.parm['rev']
         else:
             pv = d.getVar("PV", False)
-            rev = bb.fetch2.srcrev_internal_helper(ud, d, '')
+            rev = bb.fetch2.srcrev_internal_helper(ud, d)
             if rev:
                 ud.revision = rev
             else:
                 ud.revision = ""
 
-        ud.localfile = d.expand('%s_%s_%s.tar.gz' % (ud.module.replace('/', '.'), relpath.replace('/', '.'), ud.revision))
+        ud.localfile = d.expand('%s_%s_%s.tar.gz' % (ud.module.replace('/', '.'), ud.path.replace('/', '.'), ud.revision))
 
     def _buildosccommand(self, ud, d, command):
         """
@@ -63,49 +59,26 @@ class Osc(FetchMethod):
 
         basecmd = d.getVar("FETCHCMD_osc") or "/usr/bin/env osc"
 
-        proto = ud.parm.get('protocol', 'https')
+        proto = ud.parm.get('protocol', 'ocs')
 
         options = []
 
         config = "-c %s" % self.generate_config(ud, d)
 
-        if getattr(ud, 'revision', ''):
+        if ud.revision:
             options.append("-r %s" % ud.revision)
 
         coroot = self._strip_leading_slashes(ud.path)
 
         if command == "fetch":
-            osccmd = "%s %s -A %s://%s co %s/%s %s" % (basecmd, config, proto, ud.host, coroot, ud.module, " ".join(options))
+            osccmd = "%s %s co %s/%s %s" % (basecmd, config, coroot, ud.module, " ".join(options))
         elif command == "update":
-            osccmd = "%s %s -A %s://%s up %s" % (basecmd, config, proto, ud.host, " ".join(options))
-        elif command == "api_source":
-            osccmd = "%s %s -A %s://%s api source/%s/%s" % (basecmd, config, proto, ud.host, coroot, ud.module)
+            osccmd = "%s %s up %s" % (basecmd, config, " ".join(options))
         else:
             raise FetchError("Invalid osc command %s" % command, ud.url)
 
         return osccmd
 
-    def _latest_revision(self, ud, d, name):
-        """
-        Fetch latest revision for the given package
-        """
-        api_source_cmd = self._buildosccommand(ud, d, "api_source")
-
-        output = runfetchcmd(api_source_cmd, d)
-        match = re.match(r'<directory ?.* rev="(\d+)".*>', output)
-        if match is None:
-            raise FetchError("Unable to parse osc response", ud.url)
-        return match.groups()[0]
-
-    def _revision_key(self, ud, d, name):
-        """
-        Return a unique key for the url
-        """
-        # Collapse adjacent slashes
-        slash_re = re.compile(r"/+")
-        rev = getattr(ud, 'revision', "latest")
-        return "osc:%s%s.%s.%s" % (ud.host, slash_re.sub(".", ud.path), name, rev)
-
     def download(self, ud, d):
         """
         Fetch url
@@ -113,7 +86,7 @@ class Osc(FetchMethod):
 
         logger.debug2("Fetch: checking for module directory '" + ud.moddir + "'")
 
-        if os.access(ud.moddir, os.R_OK):
+        if os.access(os.path.join(d.getVar('OSCDIR'), ud.path, ud.module), os.R_OK):
             oscupdatecmd = self._buildosccommand(ud, d, "update")
             logger.info("Update "+ ud.url)
             # update sources there
@@ -141,23 +114,20 @@ class Osc(FetchMethod):
         Generate a .oscrc to be used for this run.
         """
 
-        config_path = os.path.join(ud.oscdir, "oscrc")
-        if not os.path.exists(ud.oscdir):
-            bb.utils.mkdirhier(ud.oscdir)
-
+        config_path = os.path.join(d.getVar('OSCDIR'), "oscrc")
         if (os.path.exists(config_path)):
             os.remove(config_path)
 
         f = open(config_path, 'w')
-        proto = ud.parm.get('protocol', 'https')
         f.write("[general]\n")
-        f.write("apiurl = %s://%s\n" % (proto, ud.host))
+        f.write("apisrv = %s\n" % ud.host)
+        f.write("scheme = http\n")
         f.write("su-wrapper = su -c\n")
         f.write("build-root = %s\n" % d.getVar('WORKDIR'))
         f.write("urllist = %s\n" % d.getVar("OSCURLLIST"))
         f.write("extra-pkgs = gzip\n")
         f.write("\n")
-        f.write("[%s://%s]\n" % (proto, ud.host))
+        f.write("[%s]\n" % ud.host)
         f.write("user = %s\n" % ud.parm["user"])
         f.write("pass = %s\n" % ud.parm["pswd"])
         f.close()

bitbake/lib/bb/fetch2/sftp.py

@@ -103,7 +103,7 @@ class SFTP(FetchMethod):
         if path[:3] == '/~/':
             path = path[3:]
 
-        remote = '"%s%s:%s"' % (user, urlo.hostname, path)
+        remote = '%s%s:%s' % (user, urlo.hostname, path)
 
         cmd = '%s %s %s %s' % (basecmd, port, remote, lpath)
 

bitbake/lib/bb/fetch2/ssh.py

@@ -32,7 +32,6 @@ IETF secsh internet draft:
 
 import re, os
 from bb.fetch2 import check_network_access, FetchMethod, ParameterError, runfetchcmd
-import urllib
 
 
 __pattern__ = re.compile(r'''
@@ -41,9 +40,9 @@ __pattern__ = re.compile(r'''
  (                   # Optional username/password block
   (?P<user>\S+)      # username
   (:(?P<pass>\S+))?  # colon followed by the password (optional)
+ )?
  (?P<cparam>(;[^;]+)*)?  # connection parameters block (optional)
  @
- )?
  (?P<host>\S+?)          # non-greedy match of the host
  (:(?P<port>[0-9]+))?    # colon followed by the port (optional)
  /
@@ -71,7 +70,6 @@ class SSH(FetchMethod):
                 "git:// prefix with protocol=ssh", urldata.url)
         m = __pattern__.match(urldata.url)
         path = m.group('path')
-        path = urllib.parse.unquote(path)
         host = m.group('host')
         urldata.localpath = os.path.join(d.getVar('DL_DIR'),
                 os.path.basename(os.path.normpath(path)))
@@ -98,11 +96,6 @@ class SSH(FetchMethod):
             fr += '@%s' % host
         else:
             fr = host
-
-        if path[0] != '~':
-            path = '/%s' % path
-        path = urllib.parse.unquote(path)
-
         fr += ':%s' % path
 
         cmd = 'scp -B -r %s %s %s/' % (
@@ -115,41 +108,3 @@ class SSH(FetchMethod):
 
         runfetchcmd(cmd, d)
 
-    def checkstatus(self, fetch, urldata, d):
-        """
-        Check the status of the url
-        """
-        m = __pattern__.match(urldata.url)
-        path = m.group('path')
-        host = m.group('host')
-        port = m.group('port')
-        user = m.group('user')
-        password = m.group('pass')
-
-        if port:
-            portarg = '-P %s' % port
-        else:
-            portarg = ''
-
-        if user:
-            fr = user
-            if password:
-                fr += ':%s' % password
-            fr += '@%s' % host
-        else:
-            fr = host
-
-        if path[0] != '~':
-            path = '/%s' % path
-        path = urllib.parse.unquote(path)
-
-        cmd = 'ssh -o BatchMode=true %s %s [ -f %s ]' % (
-            portarg,
-            fr,
-            path
-        )
-
-        check_network_access(d, cmd, urldata.url)
-        runfetchcmd(cmd, d)
-
-        return True

bitbake/lib/bb/fetch2/wget.py

@@ -26,6 +26,7 @@ from   bb.fetch2 import FetchMethod
 from   bb.fetch2 import FetchError
 from   bb.fetch2 import logger
 from   bb.fetch2 import runfetchcmd
+from   bb.utils import export_proxies
 from   bs4 import BeautifulSoup
 from   bs4 import SoupStrainer
 
@@ -105,22 +106,13 @@ class Wget(FetchMethod):
 
         fetchcmd = self.basecmd
 
-        localpath = os.path.join(d.getVar("DL_DIR"), ud.localfile) + ".tmp"
-        bb.utils.mkdirhier(os.path.dirname(localpath))
-        fetchcmd += " -O %s" % shlex.quote(localpath)
+        if 'downloadfilename' in ud.parm:
+            localpath = os.path.join(d.getVar("DL_DIR"), ud.localfile)
+            bb.utils.mkdirhier(os.path.dirname(localpath))
+            fetchcmd += " -O %s" % shlex.quote(localpath)
 
         if ud.user and ud.pswd:
-            fetchcmd += " --auth-no-challenge"
-            if ud.parm.get("redirectauth", "1") == "1":
-                # An undocumented feature of wget is that if the
-                # username/password are specified on the URI, wget will only
-                # send the Authorization header to the first host and not to
-                # any hosts that it is redirected to.  With the increasing
-                # usage of temporary AWS URLs, this difference now matters as
-                # AWS will reject any request that has authentication both in
-                # the query parameters (from the redirect) and in the
-                # Authorization header.
-                fetchcmd += " --user=%s --password=%s" % (ud.user, ud.pswd)
+            fetchcmd += " --user=%s --password=%s --auth-no-challenge" % (ud.user, ud.pswd)
 
         uri = ud.url.split(";")[0]
         if os.path.exists(ud.localpath):
@@ -131,15 +123,6 @@ class Wget(FetchMethod):
 
         self._runwget(ud, d, fetchcmd, False)
 
-        # Try and verify any checksum now, meaning if it isn't correct, we don't remove the
-        # original file, which might be a race (imagine two recipes referencing the same
-        # source, one with an incorrect checksum)
-        bb.fetch2.verify_checksum(ud, d, localpath=localpath, fatal_nochecksum=False)
-
-        # Remove the ".tmp" and move the file into position atomically
-        # Our lock prevents multiple writers but mirroring code may grab incomplete files
-        os.rename(localpath, localpath[:-4])
-
         # Sanity check since wget can pretend it succeed when it didn't
         # Also, this used to happen if sourceforge sent us to the mirror page
         if not os.path.exists(ud.localpath):
@@ -235,7 +218,7 @@ class Wget(FetchMethod):
                         # We let the request fail and expect it to be
                         # tried once more ("try_again" in check_status()),
                         # with the dead connection removed from the cache.
-                        # If it still fails, we give up, which can happen for bad
+                        # If it still fails, we give up, which can happend for bad
                         # HTTP proxy settings.
                         fetch.connection_cache.remove_connection(h.host, h.port)
                     raise urllib.error.URLError(err)
@@ -322,7 +305,15 @@ class Wget(FetchMethod):
         # Avoid tramping the environment too much by using bb.utils.environment
         # to scope the changes to the build_opener request, which is when the
         # environment lookups happen.
-        newenv = bb.fetch2.get_fetcher_environment(d)
+        newenv = {}
+        for name in bb.fetch2.FETCH_EXPORT_VARS:
+            value = d.getVar(name)
+            if not value:
+                origenv = d.getVar("BB_ORIGENV")
+                if origenv:
+                    value = origenv.getVar(name)
+            if value:
+                newenv[name] = value
 
         with bb.utils.environment(**newenv):
             import ssl
@@ -340,8 +331,7 @@ class Wget(FetchMethod):
             opener = urllib.request.build_opener(*handlers)
 
             try:
-                uri_base = ud.url.split(";")[0]
-                uri = "{}://{}{}".format(urllib.parse.urlparse(uri_base).scheme, ud.host, ud.path)
+                uri = ud.url.split(";")[0]
                 r = urllib.request.Request(uri)
                 r.get_method = lambda: "HEAD"
                 # Some servers (FusionForge, as used on Alioth) require that the
@@ -360,16 +350,23 @@ class Wget(FetchMethod):
 
                 try:
                     import netrc
-                    auth_data = netrc.netrc().authenticators(urllib.parse.urlparse(uri).hostname)
-                    if auth_data:
-                        login, _, password = auth_data
-                        add_basic_auth("%s:%s" % (login, password), r)
-                except (FileNotFoundError, netrc.NetrcParseError):
+                    n = netrc.netrc()
+                    login, unused, password = n.authenticators(urllib.parse.urlparse(uri).hostname)
+                    add_basic_auth("%s:%s" % (login, password), r)
+                except (TypeError, ImportError, IOError, netrc.NetrcParseError):
                     pass
 
                 with opener.open(r, timeout=30) as response:
                     pass
-            except (urllib.error.URLError, ConnectionResetError, TimeoutError) as e:
+            except urllib.error.URLError as e:
+                if try_again:
+                    logger.debug2("checkstatus: trying again")
+                    return self.checkstatus(fetch, ud, d, False)
+                else:
+                    # debug for now to avoid spamming the logs in e.g. remote sstate searches
+                    logger.debug2("checkstatus() urlopen failed: %s" % e)
+                    return False
+            except ConnectionResetError as e:
                 if try_again:
                     logger.debug2("checkstatus: trying again")
                     return self.checkstatus(fetch, ud, d, False)
@@ -586,7 +583,7 @@ class Wget(FetchMethod):
 
         # src.rpm extension was added only for rpm package. Can be removed if the rpm
         # packaged will always be considered as having to be manually upgraded
-        psuffix_regex = r"(tar\.\w+|tgz|zip|xz|rpm|bz2|orig\.tar\.\w+|src\.tar\.\w+|src\.tgz|svnr\d+\.tar\.\w+|stable\.tar\.\w+|src\.rpm)"
+        psuffix_regex = r"(tar\.gz|tgz|tar\.bz2|zip|xz|tar\.lz|rpm|bz2|orig\.tar\.gz|tar\.xz|src\.tar\.gz|src\.tgz|svnr\d+\.tar\.bz2|stable\.tar\.gz|src\.rpm)"
 
         # match name, version and archive type of a package
         package_regex_comp = re.compile(r"(?P<name>%s?\.?v?)(?P<pver>%s)(?P<arch>%s)?[\.-](?P<type>%s$)"
@@ -637,10 +634,10 @@ class Wget(FetchMethod):
             # search for version matches on folders inside the path, like:
             # "5.7" in http://download.gnome.org/sources/${PN}/5.7/${PN}-${PV}.tar.gz
             dirver_regex = re.compile(r"(?P<dirver>[^/]*(\d+\.)*\d+([-_]r\d+)*)/")
-            m = dirver_regex.findall(path)
+            m = dirver_regex.search(path)
             if m:
                 pn = d.getVar('PN')
-                dirver = m[-1][0]
+                dirver = m.group('dirver')
 
                 dirver_pn_regex = re.compile(r"%s\d?" % (re.escape(pn)))
                 if not dirver_pn_regex.search(dirver):

bitbake/lib/bb/main.py

@@ -12,12 +12,11 @@
 import os
 import sys
 import logging
-import argparse
+import optparse
 import warnings
 import fcntl
 import time
 import traceback
-import datetime
 
 import bb
 from bb import event
@@ -44,18 +43,18 @@ def present_options(optionlist):
     else:
         return optionlist[0]
 
-class BitbakeHelpFormatter(argparse.HelpFormatter):
-    def _get_help_string(self, action):
+class BitbakeHelpFormatter(optparse.IndentedHelpFormatter):
+    def format_option(self, option):
         # We need to do this here rather than in the text we supply to
         # add_option() because we don't want to call list_extension_modules()
         # on every execution (since it imports all of the modules)
         # Note also that we modify option.help rather than the returned text
         # - this is so that we don't have to re-format the text ourselves
-        if action.dest == 'ui':
+        if option.dest == 'ui':
             valid_uis = list_extension_modules(bb.ui, 'main')
-            return action.help.replace('@CHOICES@', present_options(valid_uis))
+            option.help = option.help.replace('@CHOICES@', present_options(valid_uis))
 
-        return action.help
+        return optparse.IndentedHelpFormatter.format_option(self, option)
 
 def list_extension_modules(pkg, checkattr):
     """
@@ -115,205 +114,180 @@ def _showwarning(message, category, filename, lineno, file=None, line=None):
 warnings.showwarning = _showwarning
 
 def create_bitbake_parser():
-    parser = argparse.ArgumentParser(
-        description="""\
-            It is assumed there is a conf/bblayers.conf available in cwd or in BBPATH which
-            will provide the layer, BBFILES and other configuration information.
-            """,
-        formatter_class=BitbakeHelpFormatter,
-        allow_abbrev=False,
-        add_help=False, # help is manually added below in a specific argument group
-    )
+    parser = optparse.OptionParser(
+        formatter=BitbakeHelpFormatter(),
+        version="BitBake Build Tool Core version %s" % bb.__version__,
+        usage="""%prog [options] [recipename/target recipe:do_task ...]
 
-    general_group  = parser.add_argument_group('General options')
-    task_group     = parser.add_argument_group('Task control options')
-    exec_group     = parser.add_argument_group('Execution control options')
-    logging_group  = parser.add_argument_group('Logging/output control options')
-    server_group   = parser.add_argument_group('Server options')
-    config_group   = parser.add_argument_group('Configuration options')
+    Executes the specified task (default is 'build') for a given set of target recipes (.bb files).
+    It is assumed there is a conf/bblayers.conf available in cwd or in BBPATH which
+    will provide the layer, BBFILES and other configuration information.""")
 
-    general_group.add_argument("targets", nargs="*", metavar="recipename/target",
-                        help="Execute the specified task (default is 'build') for these target "
-                             "recipes (.bb files).")
+    parser.add_option("-b", "--buildfile", action="store", dest="buildfile", default=None,
+                      help="Execute tasks from a specific .bb recipe directly. WARNING: Does "
+                           "not handle any dependencies from other recipes.")
 
-    general_group.add_argument("-s", "--show-versions", action="store_true",
-                        help="Show current and preferred versions of all recipes.")
+    parser.add_option("-k", "--continue", action="store_false", dest="abort", default=True,
+                      help="Continue as much as possible after an error. While the target that "
+                           "failed and anything depending on it cannot be built, as much as "
+                           "possible will be built before stopping.")
 
-    general_group.add_argument("-e", "--environment", action="store_true",
-                        dest="show_environment",
-                        help="Show the global or per-recipe environment complete with information"
-                             " about where variables were set/changed.")
+    parser.add_option("-f", "--force", action="store_true", dest="force", default=False,
+                      help="Force the specified targets/task to run (invalidating any "
+                           "existing stamp file).")
 
-    general_group.add_argument("-g", "--graphviz", action="store_true", dest="dot_graph",
-                        help="Save dependency tree information for the specified "
-                             "targets in the dot syntax.")
+    parser.add_option("-c", "--cmd", action="store", dest="cmd",
+                      help="Specify the task to execute. The exact options available "
+                           "depend on the metadata. Some examples might be 'compile'"
+                           " or 'populate_sysroot' or 'listtasks' may give a list of "
+                           "the tasks available.")
+
+    parser.add_option("-C", "--clear-stamp", action="store", dest="invalidate_stamp",
+                      help="Invalidate the stamp for the specified task such as 'compile' "
+                           "and then run the default task for the specified target(s).")
+
+    parser.add_option("-r", "--read", action="append", dest="prefile", default=[],
+                      help="Read the specified file before bitbake.conf.")
+
+    parser.add_option("-R", "--postread", action="append", dest="postfile", default=[],
+                      help="Read the specified file after bitbake.conf.")
+
+    parser.add_option("-v", "--verbose", action="store_true", dest="verbose", default=False,
+                      help="Enable tracing of shell tasks (with 'set -x'). "
+                           "Also print bb.note(...) messages to stdout (in "
+                           "addition to writing them to ${T}/log.do_<task>).")
+
+    parser.add_option("-D", "--debug", action="count", dest="debug", default=0,
+                      help="Increase the debug level. You can specify this "
+                           "more than once. -D sets the debug level to 1, "
+                           "where only bb.debug(1, ...) messages are printed "
+                           "to stdout; -DD sets the debug level to 2, where "
+                           "both bb.debug(1, ...) and bb.debug(2, ...) "
+                           "messages are printed; etc. Without -D, no debug "
+                           "messages are printed. Note that -D only affects "
+                           "output to stdout. All debug messages are written "
+                           "to ${T}/log.do_taskname, regardless of the debug "
+                           "level.")
+
+    parser.add_option("-q", "--quiet", action="count", dest="quiet", default=0,
+                      help="Output less log message data to the terminal. You can specify this more than once.")
+
+    parser.add_option("-n", "--dry-run", action="store_true", dest="dry_run", default=False,
+                      help="Don't execute, just go through the motions.")
+
+    parser.add_option("-S", "--dump-signatures", action="append", dest="dump_signatures",
+                      default=[], metavar="SIGNATURE_HANDLER",
+                      help="Dump out the signature construction information, with no task "
+                           "execution. The SIGNATURE_HANDLER parameter is passed to the "
+                           "handler. Two common values are none and printdiff but the handler "
+                           "may define more/less. none means only dump the signature, printdiff"
+                           " means compare the dumped signature with the cached one.")
+
+    parser.add_option("-p", "--parse-only", action="store_true",
+                      dest="parse_only", default=False,
+                      help="Quit after parsing the BB recipes.")
+
+    parser.add_option("-s", "--show-versions", action="store_true",
+                      dest="show_versions", default=False,
+                      help="Show current and preferred versions of all recipes.")
+
+    parser.add_option("-e", "--environment", action="store_true",
+                      dest="show_environment", default=False,
+                      help="Show the global or per-recipe environment complete with information"
+                           " about where variables were set/changed.")
+
+    parser.add_option("-g", "--graphviz", action="store_true", dest="dot_graph", default=False,
+                      help="Save dependency tree information for the specified "
+                           "targets in the dot syntax.")
+
+    parser.add_option("-I", "--ignore-deps", action="append",
+                      dest="extra_assume_provided", default=[],
+                      help="Assume these dependencies don't exist and are already provided "
+                           "(equivalent to ASSUME_PROVIDED). Useful to make dependency "
+                           "graphs more appealing")
+
+    parser.add_option("-l", "--log-domains", action="append", dest="debug_domains", default=[],
+                      help="Show debug logging for the specified logging domains")
+
+    parser.add_option("-P", "--profile", action="store_true", dest="profile", default=False,
+                      help="Profile the command and save reports.")
 
     # @CHOICES@ is substituted out by BitbakeHelpFormatter above
-    general_group.add_argument("-u", "--ui",
-                        default=os.environ.get('BITBAKE_UI', 'knotty'),
-                        help="The user interface to use (@CHOICES@ - default %(default)s).")
+    parser.add_option("-u", "--ui", action="store", dest="ui",
+                      default=os.environ.get('BITBAKE_UI', 'knotty'),
+                      help="The user interface to use (@CHOICES@ - default %default).")
 
-    general_group.add_argument("--version", action="store_true",
-                        help="Show programs version and exit.")
+    parser.add_option("", "--token", action="store", dest="xmlrpctoken",
+                      default=os.environ.get("BBTOKEN"),
+                      help="Specify the connection token to be used when connecting "
+                           "to a remote server.")
 
-    general_group.add_argument('-h', '--help', action='help',
-                        help='Show this help message and exit.')
+    parser.add_option("", "--revisions-changed", action="store_true",
+                      dest="revisions_changed", default=False,
+                      help="Set the exit code depending on whether upstream floating "
+                           "revisions have changed or not.")
 
+    parser.add_option("", "--server-only", action="store_true",
+                      dest="server_only", default=False,
+                      help="Run bitbake without a UI, only starting a server "
+                           "(cooker) process.")
 
-    task_group.add_argument("-f", "--force", action="store_true",
-                        help="Force the specified targets/task to run (invalidating any "
-                             "existing stamp file).")
+    parser.add_option("-B", "--bind", action="store", dest="bind", default=False,
+                      help="The name/address for the bitbake xmlrpc server to bind to.")
 
-    task_group.add_argument("-c", "--cmd",
-                        help="Specify the task to execute. The exact options available "
-                             "depend on the metadata. Some examples might be 'compile'"
-                             " or 'populate_sysroot' or 'listtasks' may give a list of "
-                             "the tasks available.")
+    parser.add_option("-T", "--idle-timeout", type=float, dest="server_timeout",
+                      default=os.getenv("BB_SERVER_TIMEOUT"),
+                      help="Set timeout to unload bitbake server due to inactivity, "
+                           "set to -1 means no unload, "
+                           "default: Environment variable BB_SERVER_TIMEOUT.")
 
-    task_group.add_argument("-C", "--clear-stamp", dest="invalidate_stamp",
-                        help="Invalidate the stamp for the specified task such as 'compile' "
-                             "and then run the default task for the specified target(s).")
+    parser.add_option("", "--no-setscene", action="store_true",
+                      dest="nosetscene", default=False,
+                      help="Do not run any setscene tasks. sstate will be ignored and "
+                           "everything needed, built.")
 
-    task_group.add_argument("--runall", action="append", default=[],
-                        help="Run the specified task for any recipe in the taskgraph of the "
-                             "specified target (even if it wouldn't otherwise have run).")
+    parser.add_option("", "--skip-setscene", action="store_true",
+                      dest="skipsetscene", default=False,
+                      help="Skip setscene tasks if they would be executed. Tasks previously "
+                           "restored from sstate will be kept, unlike --no-setscene")
 
-    task_group.add_argument("--runonly", action="append",
-                        help="Run only the specified task within the taskgraph of the "
-                             "specified targets (and any task dependencies those tasks may have).")
+    parser.add_option("", "--setscene-only", action="store_true",
+                      dest="setsceneonly", default=False,
+                      help="Only run setscene tasks, don't run any real tasks.")
 
-    task_group.add_argument("--no-setscene", action="store_true",
-                        dest="nosetscene",
-                        help="Do not run any setscene tasks. sstate will be ignored and "
-                            "everything needed, built.")
+    parser.add_option("", "--remote-server", action="store", dest="remote_server",
+                      default=os.environ.get("BBSERVER"),
+                      help="Connect to the specified server.")
 
-    task_group.add_argument("--skip-setscene", action="store_true",
-                        dest="skipsetscene",
-                        help="Skip setscene tasks if they would be executed. Tasks previously "
-                            "restored from sstate will be kept, unlike --no-setscene.")
+    parser.add_option("-m", "--kill-server", action="store_true",
+                      dest="kill_server", default=False,
+                      help="Terminate any running bitbake server.")
 
-    task_group.add_argument("--setscene-only", action="store_true",
-                        dest="setsceneonly",
-                        help="Only run setscene tasks, don't run any real tasks.")
+    parser.add_option("", "--observe-only", action="store_true",
+                      dest="observe_only", default=False,
+                      help="Connect to a server as an observing-only client.")
 
+    parser.add_option("", "--status-only", action="store_true",
+                      dest="status_only", default=False,
+                      help="Check the status of the remote bitbake server.")
 
-    exec_group.add_argument("-n", "--dry-run", action="store_true",
-                        help="Don't execute, just go through the motions.")
+    parser.add_option("-w", "--write-log", action="store", dest="writeeventlog",
+                      default=os.environ.get("BBEVENTLOG"),
+                      help="Writes the event log of the build to a bitbake event json file. "
+                           "Use '' (empty string) to assign the name automatically.")
 
-    exec_group.add_argument("-p", "--parse-only", action="store_true",
-                        help="Quit after parsing the BB recipes.")
-
-    exec_group.add_argument("-k", "--continue", action="store_false", dest="halt",
-                        help="Continue as much as possible after an error. While the target that "
-                             "failed and anything depending on it cannot be built, as much as "
-                             "possible will be built before stopping.")
-
-    exec_group.add_argument("-P", "--profile", action="store_true",
-                        help="Profile the command and save reports.")
-
-    exec_group.add_argument("-S", "--dump-signatures", action="append",
-                        default=[], metavar="SIGNATURE_HANDLER",
-                        help="Dump out the signature construction information, with no task "
-                             "execution. The SIGNATURE_HANDLER parameter is passed to the "
-                             "handler. Two common values are none and printdiff but the handler "
-                             "may define more/less. none means only dump the signature, printdiff"
-                             " means compare the dumped signature with the cached one.")
-
-    exec_group.add_argument("--revisions-changed", action="store_true",
-                        help="Set the exit code depending on whether upstream floating "
-                            "revisions have changed or not.")
-
-    exec_group.add_argument("-b", "--buildfile",
-                        help="Execute tasks from a specific .bb recipe directly. WARNING: Does "
-                             "not handle any dependencies from other recipes.")
-
-    logging_group.add_argument("-D", "--debug", action="count", default=0,
-                        help="Increase the debug level. You can specify this "
-                             "more than once. -D sets the debug level to 1, "
-                             "where only bb.debug(1, ...) messages are printed "
-                             "to stdout; -DD sets the debug level to 2, where "
-                             "both bb.debug(1, ...) and bb.debug(2, ...) "
-                             "messages are printed; etc. Without -D, no debug "
-                             "messages are printed. Note that -D only affects "
-                             "output to stdout. All debug messages are written "
-                             "to ${T}/log.do_taskname, regardless of the debug "
-                             "level.")
-
-    logging_group.add_argument("-l", "--log-domains", action="append", dest="debug_domains",
-                        default=[],
-                        help="Show debug logging for the specified logging domains.")
-
-    logging_group.add_argument("-v", "--verbose", action="store_true",
-                        help="Enable tracing of shell tasks (with 'set -x'). "
-                             "Also print bb.note(...) messages to stdout (in "
-                             "addition to writing them to ${T}/log.do_<task>).")
-
-    logging_group.add_argument("-q", "--quiet", action="count", default=0,
-                        help="Output less log message data to the terminal. You can specify this "
-                             "more than once.")
-
-    logging_group.add_argument("-w", "--write-log", dest="writeeventlog",
-                        default=os.environ.get("BBEVENTLOG"),
-                        help="Writes the event log of the build to a bitbake event json file. "
-                            "Use '' (empty string) to assign the name automatically.")
-
-
-    server_group.add_argument("-B", "--bind", default=False,
-                        help="The name/address for the bitbake xmlrpc server to bind to.")
-
-    server_group.add_argument("-T", "--idle-timeout", type=float, dest="server_timeout",
-                        default=os.getenv("BB_SERVER_TIMEOUT"),
-                        help="Set timeout to unload bitbake server due to inactivity, "
-                             "set to -1 means no unload, "
-                             "default: Environment variable BB_SERVER_TIMEOUT.")
-
-    server_group.add_argument("--remote-server",
-                        default=os.environ.get("BBSERVER"),
-                        help="Connect to the specified server.")
-
-    server_group.add_argument("-m", "--kill-server", action="store_true",
-                        help="Terminate any running bitbake server.")
-
-    server_group.add_argument("--token", dest="xmlrpctoken",
-                        default=os.environ.get("BBTOKEN"),
-                        help="Specify the connection token to be used when connecting "
-                             "to a remote server.")
-
-    server_group.add_argument("--observe-only", action="store_true",
-                        help="Connect to a server as an observing-only client.")
-
-    server_group.add_argument("--status-only", action="store_true",
-                        help="Check the status of the remote bitbake server.")
-
-    server_group.add_argument("--server-only", action="store_true",
-                        help="Run bitbake without a UI, only starting a server "
-                            "(cooker) process.")
-
-
-    config_group.add_argument("-r", "--read", action="append", dest="prefile", default=[],
-                        help="Read the specified file before bitbake.conf.")
-
-    config_group.add_argument("-R", "--postread", action="append", dest="postfile", default=[],
-                        help="Read the specified file after bitbake.conf.")
-
-
-    config_group.add_argument("-I", "--ignore-deps", action="append",
-                        dest="extra_assume_provided", default=[],
-                        help="Assume these dependencies don't exist and are already provided "
-                             "(equivalent to ASSUME_PROVIDED). Useful to make dependency "
-                             "graphs more appealing.")
+    parser.add_option("", "--runall", action="append", dest="runall",
+                      help="Run the specified task for any recipe in the taskgraph of the specified target (even if it wouldn't otherwise have run).")
 
+    parser.add_option("", "--runonly", action="append", dest="runonly",
+                      help="Run only the specified task within the taskgraph of the specified targets (and any task dependencies those tasks may have).")
     return parser
 
 
 class BitBakeConfigParameters(cookerdata.ConfigParameters):
     def parseCommandLine(self, argv=sys.argv):
         parser = create_bitbake_parser()
-        options = parser.parse_intermixed_args(argv[1:])
-
-        if options.version:
-            print("BitBake Build Tool Core version %s" % bb.__version__)
-            sys.exit(0)
+        options, targets = parser.parse_args(argv)
 
         if options.quiet and options.verbose:
             parser.error("options --quiet and --verbose are mutually exclusive")
@@ -345,7 +319,7 @@ class BitBakeConfigParameters(cookerdata.ConfigParameters):
         else:
             options.xmlrpcinterface = (None, 0)
 
-        return options, options.targets
+        return options, targets[1:]
 
 
 def bitbake_main(configParams, configuration):
@@ -410,9 +384,6 @@ def bitbake_main(configParams, configuration):
 
     return 1
 
-def timestamp():
-    return datetime.datetime.now().strftime('%H:%M:%S.%f')
-
 def setup_bitbake(configParams, extrafeatures=None):
     # Ensure logging messages get sent to the UI as events
     handler = bb.event.LogHandler()
@@ -420,11 +391,6 @@ def setup_bitbake(configParams, extrafeatures=None):
         # In status only mode there are no logs and no UI
         logger.addHandler(handler)
 
-    if configParams.dump_signatures:
-        if extrafeatures is None:
-            extrafeatures = []
-        extrafeatures.append(bb.cooker.CookerFeatures.RECIPE_SIGGEN_INFO)
-
     if configParams.server_only:
         featureset = []
         ui_module = None
@@ -452,7 +418,7 @@ def setup_bitbake(configParams, extrafeatures=None):
         retries = 8
         while retries:
             try:
-                topdir, lock, lockfile = lockBitbake()
+                topdir, lock = lockBitbake()
                 sockname = topdir + "/bitbake.sock"
                 if lock:
                     if configParams.status_only or configParams.kill_server:
@@ -463,22 +429,18 @@ def setup_bitbake(configParams, extrafeatures=None):
                     logger.info("Starting bitbake server...")
                     # Clear the event queue since we already displayed messages
                     bb.event.ui_queue = []
-                    server = bb.server.process.BitBakeServer(lock, sockname, featureset, configParams.server_timeout, configParams.xmlrpcinterface, configParams.profile)
+                    server = bb.server.process.BitBakeServer(lock, sockname, featureset, configParams.server_timeout, configParams.xmlrpcinterface)
 
                 else:
                     logger.info("Reconnecting to bitbake server...")
                     if not os.path.exists(sockname):
-                        logger.info("Previous bitbake instance shutting down?, waiting to retry... (%s)" % timestamp())
-                        procs = bb.server.process.get_lockfile_process_msg(lockfile)
-                        if procs:
-                            logger.info("Processes holding bitbake.lock (missing socket %s):\n%s" % (sockname, procs))
-                        logger.info("Directory listing: %s" % (str(os.listdir(topdir))))
+                        logger.info("Previous bitbake instance shutting down?, waiting to retry...")
                         i = 0
                         lock = None
                         # Wait for 5s or until we can get the lock
                         while not lock and i < 50:
                             time.sleep(0.1)
-                            _, lock, _ = lockBitbake()
+                            _, lock = lockBitbake()
                             i += 1
                         if lock:
                             bb.utils.unlockfile(lock)
@@ -497,9 +459,9 @@ def setup_bitbake(configParams, extrafeatures=None):
                 retries -= 1
                 tryno = 8 - retries
                 if isinstance(e, (bb.server.process.ProcessTimeout, BrokenPipeError, EOFError, SystemExit)):
-                    logger.info("Retrying server connection (#%d)... (%s)" % (tryno, timestamp()))
+                    logger.info("Retrying server connection (#%d)..." % tryno)
                 else:
-                    logger.info("Retrying server connection (#%d)... (%s, %s)" % (tryno, traceback.format_exc(), timestamp()))
+                    logger.info("Retrying server connection (#%d)... (%s)" % (tryno, traceback.format_exc()))
 
             if not retries:
                 bb.fatal("Unable to connect to bitbake server, or start one (server startup failures would be in bitbake-cookerdaemon.log).")
@@ -528,5 +490,5 @@ def lockBitbake():
         bb.error("Unable to find conf/bblayers.conf or conf/bitbake.conf. BBPATH is unset and/or not in a build directory?")
         raise BBMainFatal
     lockfile = topdir + "/bitbake.lock"
-    return topdir, bb.utils.lockfile(lockfile, False, False), lockfile
+    return topdir, bb.utils.lockfile(lockfile, False, False)
 

bitbake/lib/bb/monitordisk.py

@@ -76,12 +76,7 @@ def getDiskData(BBDirs):
             return None
 
         action = pathSpaceInodeRe.group(1)
-        if action == "ABORT":
-            # Emit a deprecation warning
-            logger.warnonce("The BB_DISKMON_DIRS \"ABORT\" action has been renamed to \"HALT\", update configuration")
-            action = "HALT"
-
-        if action not in ("HALT", "STOPTASKS", "WARN"):
+        if action not in ("ABORT", "STOPTASKS", "WARN"):
             printErr("Unknown disk space monitor action: %s" % action)
             return None
 
@@ -182,7 +177,7 @@ class diskMonitor:
                     # use them to avoid printing too many warning messages
                     self.preFreeS = {}
                     self.preFreeI = {}
-                    # This is for STOPTASKS and HALT, to avoid printing the message
+                    # This is for STOPTASKS and ABORT, to avoid printing the message
                     # repeatedly while waiting for the tasks to finish
                     self.checked = {}
                     for k in self.devDict:
@@ -224,8 +219,8 @@ class diskMonitor:
                         self.checked[k] = True
                         rq.finish_runqueue(False)
                         bb.event.fire(bb.event.DiskFull(dev, 'disk', freeSpace, path), self.configuration)
-                    elif action == "HALT" and not self.checked[k]:
-                        logger.error("Immediately halt since the disk space monitor action is \"HALT\"!")
+                    elif action == "ABORT" and not self.checked[k]:
+                        logger.error("Immediately abort since the disk space monitor action is \"ABORT\"!")
                         self.checked[k] = True
                         rq.finish_runqueue(True)
                         bb.event.fire(bb.event.DiskFull(dev, 'disk', freeSpace, path), self.configuration)
@@ -234,10 +229,9 @@ class diskMonitor:
                 freeInode = st.f_favail
 
                 if minInode and freeInode < minInode:
-                    # Some filesystems use dynamic inodes so can't run out.
-                    # This is reported by the inode count being 0 (btrfs) or the free
-                    # inode count being -1 (cephfs).
-                    if st.f_files == 0 or st.f_favail == -1:
+                    # Some filesystems use dynamic inodes so can't run out
+                    # (e.g. btrfs). This is reported by the inode count being 0.
+                    if st.f_files == 0:
                         self.devDict[k][2] = None
                         continue
                     # Always show warning, the self.checked would always be False if the action is WARN
@@ -251,8 +245,8 @@ class diskMonitor:
                         self.checked[k] = True
                         rq.finish_runqueue(False)
                         bb.event.fire(bb.event.DiskFull(dev, 'inode', freeInode, path), self.configuration)
-                    elif action  == "HALT" and not self.checked[k]:
-                        logger.error("Immediately halt since the disk space monitor action is \"HALT\"!")
+                    elif action  == "ABORT" and not self.checked[k]:
+                        logger.error("Immediately abort since the disk space monitor action is \"ABORT\"!")
                         self.checked[k] = True
                         rq.finish_runqueue(True)
                         bb.event.fire(bb.event.DiskFull(dev, 'inode', freeInode, path), self.configuration)

bitbake/lib/bb/msg.py

@@ -30,9 +30,7 @@ class BBLogFormatter(logging.Formatter):
     PLAIN = logging.INFO + 1
     VERBNOTE = logging.INFO + 2
     ERROR = logging.ERROR
-    ERRORONCE = logging.ERROR - 1
     WARNING = logging.WARNING
-    WARNONCE = logging.WARNING - 1
     CRITICAL = logging.CRITICAL
 
     levelnames = {
@@ -44,9 +42,7 @@ class BBLogFormatter(logging.Formatter):
         PLAIN  : '',
         VERBNOTE: 'NOTE',
         WARNING : 'WARNING',
-        WARNONCE : 'WARNING',
         ERROR   : 'ERROR',
-        ERRORONCE   : 'ERROR',
         CRITICAL: 'ERROR',
     }
 
@@ -62,9 +58,7 @@ class BBLogFormatter(logging.Formatter):
         PLAIN   : BASECOLOR,
         VERBNOTE: BASECOLOR,
         WARNING : YELLOW,
-        WARNONCE : YELLOW,
         ERROR   : RED,
-        ERRORONCE : RED,
         CRITICAL: RED,
     }
 
@@ -127,22 +121,6 @@ class BBLogFilter(object):
             return True
         return False
 
-class LogFilterShowOnce(logging.Filter):
-    def __init__(self):
-        self.seen_warnings = set()
-        self.seen_errors = set()
-
-    def filter(self, record):
-        if record.levelno == bb.msg.BBLogFormatter.WARNONCE:
-            if record.msg in self.seen_warnings:
-                return False
-            self.seen_warnings.add(record.msg)
-        if record.levelno == bb.msg.BBLogFormatter.ERRORONCE:
-            if record.msg in self.seen_errors:
-                return False
-            self.seen_errors.add(record.msg)
-        return True
-
 class LogFilterGEQLevel(logging.Filter):
     def __init__(self, level):
         self.strlevel = str(level)
@@ -228,7 +206,6 @@ def logger_create(name, output=sys.stderr, level=logging.INFO, preserve_handlers
     """Standalone logger creation function"""
     logger = logging.getLogger(name)
     console = logging.StreamHandler(output)
-    console.addFilter(bb.msg.LogFilterShowOnce())
     format = bb.msg.BBLogFormatter("%(levelname)s: %(message)s")
     if color == 'always' or (color == 'auto' and output.isatty()):
         format.enable_color()
@@ -316,17 +293,10 @@ def setLoggingConfig(defaultconfig, userconfigfile=None):
 
     # Convert all level parameters to integers in case users want to use the
     # bitbake defined level names
-    for name, h in logconfig["handlers"].items():
+    for h in logconfig["handlers"].values():
         if "level" in h:
             h["level"] = bb.msg.stringToLevel(h["level"])
 
-        # Every handler needs its own instance of the once filter.
-        once_filter_name = name + ".showonceFilter"
-        logconfig.setdefault("filters", {})[once_filter_name] = {
-            "()": "bb.msg.LogFilterShowOnce",
-        }
-        h.setdefault("filters", []).append(once_filter_name)
-
     for l in logconfig["loggers"].values():
         if "level" in l:
             l["level"] = bb.msg.stringToLevel(l["level"])

bitbake/lib/bb/parse/__init__.py

@@ -99,12 +99,12 @@ def supports(fn, data):
             return 1
     return 0
 
-def handle(fn, data, include=0, baseconfig=False):
+def handle(fn, data, include = 0):
     """Call the handler that is appropriate for this file"""
     for h in handlers:
         if h['supports'](fn, data):
             with data.inchistory.include(fn):
-                return h['handle'](fn, data, include, baseconfig)
+                return h['handle'](fn, data, include)
     raise ParseError("not a BitBake file", fn)
 
 def init(fn, data):
@@ -113,8 +113,6 @@ def init(fn, data):
             return h['init'](data)
 
 def init_parser(d):
-    if hasattr(bb.parse, "siggen"):
-        bb.parse.siggen.exit()
     bb.parse.siggen = bb.siggen.init(d)
 
 def resolve_file(fn, d):

bitbake/lib/bb/parse/ast.py

@@ -9,7 +9,6 @@
 # SPDX-License-Identifier: GPL-2.0-only
 #
 
-import sys
 import bb
 from bb import methodpool
 from bb.parse import logger
@@ -131,10 +130,6 @@ class DataNode(AstNode):
         else:
             val = groupd["value"]
 
-        if ":append" in key or ":remove" in key or ":prepend" in key:
-            if op in ["append", "prepend", "postdot", "predot", "ques"]:
-                bb.warn(key + " " + groupd[op] + " is not a recommended operator combination, please replace it.")
-
         flag = None
         if 'flag' in groupd and groupd['flag'] is not None:
             flag = groupd['flag']
@@ -224,7 +219,7 @@ class ExportFuncsNode(AstNode):
             for flag in [ "func", "python" ]:
                 if data.getVarFlag(calledfunc, flag, False):
                     data.setVarFlag(func, flag, data.getVarFlag(calledfunc, flag, False))
-            for flag in ["dirs", "cleandirs", "fakeroot"]:
+            for flag in [ "dirs" ]:
                 if data.getVarFlag(func, flag, False):
                     data.setVarFlag(calledfunc, flag, data.getVarFlag(func, flag, False))
             data.setVarFlag(func, "filename", "autogenerated")
@@ -270,41 +265,6 @@ class BBHandlerNode(AstNode):
             data.setVarFlag(h, "handler", 1)
         data.setVar('__BBHANDLERS', bbhands)
 
-class PyLibNode(AstNode):
-    def __init__(self, filename, lineno, libdir, namespace):
-        AstNode.__init__(self, filename, lineno)
-        self.libdir = libdir
-        self.namespace = namespace
-
-    def eval(self, data):
-        global_mods = (data.getVar("BB_GLOBAL_PYMODULES") or "").split()
-        for m in global_mods:
-            if m not in bb.utils._context:
-                bb.utils._context[m] = __import__(m)
-
-        libdir = data.expand(self.libdir)
-        if libdir not in sys.path:
-            sys.path.append(libdir)
-        try:
-            bb.utils._context[self.namespace] = __import__(self.namespace)
-            toimport = getattr(bb.utils._context[self.namespace], "BBIMPORTS", [])
-            for i in toimport:
-                bb.utils._context[self.namespace] = __import__(self.namespace + "." + i)
-                mod = getattr(bb.utils._context[self.namespace], i)
-                fn = getattr(mod, "__file__")
-                funcs = {}
-                for f in dir(mod):
-                    if f.startswith("_"):
-                        continue
-                    fcall = getattr(mod, f)
-                    if not callable(fcall):
-                        continue
-                    funcs[f] = fcall
-                bb.codeparser.add_module_functions(fn, funcs, "%s.%s" % (self.namespace, i))
-
-        except AttributeError as e:
-            bb.error("Error importing OE modules: %s" % str(e))
-
 class InheritNode(AstNode):
     def __init__(self, filename, lineno, classes):
         AstNode.__init__(self, filename, lineno)
@@ -356,9 +316,6 @@ def handleDelTask(statements, filename, lineno, m):
 def handleBBHandlers(statements, filename, lineno, m):
     statements.append(BBHandlerNode(filename, lineno, m.group(1)))
 
-def handlePyLib(statements, filename, lineno, m):
-    statements.append(PyLibNode(filename, lineno, m.group(1), m.group(2)))
-
 def handleInherit(statements, filename, lineno, m):
     classes = m.group(1)
     statements.append(InheritNode(filename, lineno, classes))
@@ -372,10 +329,6 @@ def runAnonFuncs(d):
 def finalize(fn, d, variant = None):
     saved_handlers = bb.event.get_handlers().copy()
     try:
-        # Found renamed variables. Exit immediately
-        if d.getVar("_FAILPARSINGERRORHANDLED", False) == True:
-            raise bb.BBHandledException()
-
         for var in d.getVar('__BBHANDLERS', False) or []:
             # try to add the handler
             handlerfn = d.getVarFlag(var, "filename", False)
@@ -400,9 +353,6 @@ def finalize(fn, d, variant = None):
 
         d.setVar('BBINCLUDED', bb.parse.get_file_depends(d))
 
-        if d.getVar('__BBAUTOREV_SEEN') and d.getVar('__BBSRCREV_SEEN') and not d.getVar("__BBAUTOREV_ACTED_UPON"):
-            bb.fatal("AUTOREV/SRCPV set too late for the fetcher to work properly, please set the variables earlier in parsing. Erroring instead of later obtuse build failures.")
-
         bb.event.fire(bb.event.RecipeParsed(fn), d)
     finally:
         bb.event.set_handlers(saved_handlers)

bitbake/lib/bb/parse/parse_py/BBHandler.py

@@ -44,36 +44,23 @@ def inherit(files, fn, lineno, d):
     __inherit_cache = d.getVar('__inherit_cache', False) or []
     files = d.expand(files).split()
     for file in files:
-        classtype = d.getVar("__bbclasstype", False)
-        origfile = file
-        for t in ["classes-" + classtype, "classes"]:
-            file = origfile
-            if not os.path.isabs(file) and not file.endswith(".bbclass"):
-                file = os.path.join(t, '%s.bbclass' % file)
+        if not os.path.isabs(file) and not file.endswith(".bbclass"):
+            file = os.path.join('classes', '%s.bbclass' % file)
 
-            if not os.path.isabs(file):
-                bbpath = d.getVar("BBPATH")
-                abs_fn, attempts = bb.utils.which(bbpath, file, history=True)
-                for af in attempts:
-                    if af != abs_fn:
-                        bb.parse.mark_dependency(d, af)
-                if abs_fn:
-                    file = abs_fn
-
-            if os.path.exists(file):
-                break
-
-        if not os.path.exists(file):
-            raise ParseError("Could not inherit file %s" % (file), fn, lineno)
+        if not os.path.isabs(file):
+            bbpath = d.getVar("BBPATH")
+            abs_fn, attempts = bb.utils.which(bbpath, file, history=True)
+            for af in attempts:
+                if af != abs_fn:
+                    bb.parse.mark_dependency(d, af)
+            if abs_fn:
+                file = abs_fn
 
         if not file in __inherit_cache:
             logger.debug("Inheriting %s (from %s:%d)" % (file, fn, lineno))
             __inherit_cache.append( file )
             d.setVar('__inherit_cache', __inherit_cache)
-            try:
-                bb.parse.handle(file, d, True)
-            except (IOError, OSError) as exc:
-                raise ParseError("Could not inherit file %s: %s" % (fn, exc.strerror), fn, lineno)
+            include(fn, file, lineno, d, "inherit")
             __inherit_cache = d.getVar('__inherit_cache', False) or []
 
 def get_statements(filename, absolute_filename, base_name):
@@ -101,8 +88,8 @@ def get_statements(filename, absolute_filename, base_name):
             cached_statements[absolute_filename] = statements
         return statements
 
-def handle(fn, d, include, baseconfig=False):
-    global __infunc__, __body__, __residue__, __classname__
+def handle(fn, d, include):
+    global __func_start_regexp__, __inherit_regexp__, __export_func_regexp__, __addtask_regexp__, __addhandler_regexp__, __infunc__, __body__, __residue__, __classname__
     __body__ = []
     __infunc__ = []
     __classname__ = ""
@@ -154,7 +141,7 @@ def handle(fn, d, include, baseconfig=False):
     return d
 
 def feeder(lineno, s, fn, root, statements, eof=False):
-    global __inpython__, __infunc__, __body__, __residue__, __classname__
+    global __func_start_regexp__, __inherit_regexp__, __export_func_regexp__, __addtask_regexp__, __addhandler_regexp__, __def_regexp__, __python_func_regexp__, __inpython__, __infunc__, __body__, bb, __residue__, __classname__
 
     # Check tabs in python functions:
     # - def py_funcname(): covered by __inpython__
@@ -191,10 +178,10 @@ def feeder(lineno, s, fn, root, statements, eof=False):
 
     if s and s[0] == '#':
         if len(__residue__) != 0 and __residue__[0][0] != "#":
-            bb.fatal("There is a comment on line %s of file %s:\n'''\n%s\n'''\nwhich is in the middle of a multiline expression. This syntax is invalid, please correct it." % (lineno, fn, s))
+            bb.fatal("There is a comment on line %s of file %s (%s) which is in the middle of a multiline expression.\nBitbake used to ignore these but no longer does so, please fix your metadata as errors are likely as a result of this change." % (lineno, fn, s))
 
     if len(__residue__) != 0 and __residue__[0][0] == "#" and (not s or s[0] != "#"):
-        bb.fatal("There is a confusing multiline partially commented expression on line %s of file %s:\n%s\nPlease clarify whether this is all a comment or should be parsed." % (lineno - len(__residue__), fn, "\n".join(__residue__)))
+        bb.fatal("There is a confusing multiline, partially commented expression on line %s of file %s (%s).\nPlease clarify whether this is all a comment or should be parsed." % (lineno, fn, s))
 
     if s and s[-1] == '\\':
         __residue__.append(s[:-1])
@@ -265,7 +252,7 @@ def feeder(lineno, s, fn, root, statements, eof=False):
         ast.handleInherit(statements, fn, lineno, m)
         return
 
-    return ConfHandler.feeder(lineno, s, fn, statements, conffile=False)
+    return ConfHandler.feeder(lineno, s, fn, statements)
 
 # Add us to the handlers list
 from .. import handlers

bitbake/lib/bb/parse/parse_py/ConfHandler.py

@@ -21,7 +21,7 @@ __config_regexp__  = re.compile( r"""
     ^
     (?P<exp>export\s+)?
     (?P<var>[a-zA-Z0-9\-_+.${}/~:]+?)
-    (\[(?P<flag>[a-zA-Z0-9\-_+.][a-zA-Z0-9\-_+.@]*)\])?
+    (\[(?P<flag>[a-zA-Z0-9\-_+.]+)\])?
 
     \s* (
         (?P<colon>:=) |
@@ -45,11 +45,13 @@ __include_regexp__ = re.compile( r"include\s+(.+)" )
 __require_regexp__ = re.compile( r"require\s+(.+)" )
 __export_regexp__ = re.compile( r"export\s+([a-zA-Z0-9\-_+.${}/~]+)$" )
 __unset_regexp__ = re.compile( r"unset\s+([a-zA-Z0-9\-_+.${}/~]+)$" )
-__unset_flag_regexp__ = re.compile( r"unset\s+([a-zA-Z0-9\-_+.${}/~]+)\[([a-zA-Z0-9\-_+.][a-zA-Z0-9\-_+.@]+)\]$" )
-__addpylib_regexp__      = re.compile(r"addpylib\s+(.+)\s+(.+)" )
+__unset_flag_regexp__ = re.compile( r"unset\s+([a-zA-Z0-9\-_+.${}/~]+)\[([a-zA-Z0-9\-_+.]+)\]$" )
 
 def init(data):
-    return
+    topdir = data.getVar('TOPDIR', False)
+    if not topdir:
+        data.setVar('TOPDIR', os.getcwd())
+
 
 def supports(fn, d):
     return fn[-5:] == ".conf"
@@ -103,12 +105,12 @@ def include_single_file(parentfn, fn, lineno, data, error_out):
 # We have an issue where a UI might want to enforce particular settings such as
 # an empty DISTRO variable. If configuration files do something like assigning
 # a weak default, it turns out to be very difficult to filter out these changes,
-# particularly when the weak default might appear half way though parsing a chain
+# particularly when the weak default might appear half way though parsing a chain 
 # of configuration files. We therefore let the UIs hook into configuration file
 # parsing. This turns out to be a hard problem to solve any other way.
 confFilters = []
 
-def handle(fn, data, include, baseconfig=False):
+def handle(fn, data, include):
     init(data)
 
     if include == 0:
@@ -126,26 +128,21 @@ def handle(fn, data, include, baseconfig=False):
             s = f.readline()
             if not s:
                 break
-            origlineno = lineno
-            origline = s
             w = s.strip()
             # skip empty lines
             if not w:
                 continue
             s = s.rstrip()
             while s[-1] == '\\':
-                line = f.readline()
-                origline += line
-                s2 = line.rstrip()
+                s2 = f.readline().rstrip()
                 lineno = lineno + 1
                 if (not s2 or s2 and s2[0] != "#") and s[0] == "#" :
-                    bb.fatal("There is a confusing multiline, partially commented expression starting on line %s of file %s:\n%s\nPlease clarify whether this is all a comment or should be parsed." % (origlineno, fn, origline))
-
+                    bb.fatal("There is a confusing multiline, partially commented expression on line %s of file %s (%s).\nPlease clarify whether this is all a comment or should be parsed." % (lineno, fn, s))
                 s = s[:-1] + s2
             # skip comments
             if s[0] == '#':
                 continue
-            feeder(lineno, s, abs_fn, statements, baseconfig=baseconfig)
+            feeder(lineno, s, abs_fn, statements)
 
     # DONE WITH PARSING... time to evaluate
     data.setVar('FILE', abs_fn)
@@ -153,14 +150,14 @@ def handle(fn, data, include, baseconfig=False):
     if oldfile:
         data.setVar('FILE', oldfile)
 
+    f.close()
+
     for f in confFilters:
         f(fn, data)
 
     return data
 
-# baseconfig is set for the bblayers/layer.conf cookerdata config parsing
-# The function is also used by BBHandler, conffile would be False
-def feeder(lineno, s, fn, statements, baseconfig=False, conffile=True):
+def feeder(lineno, s, fn, statements):
     m = __config_regexp__.match(s)
     if m:
         groupd = m.groupdict()
@@ -192,11 +189,6 @@ def feeder(lineno, s, fn, statements, baseconfig=False, conffile=True):
         ast.handleUnsetFlag(statements, fn, lineno, m)
         return
 
-    m = __addpylib_regexp__.match(s)
-    if baseconfig and conffile and m:
-        ast.handlePyLib(statements, fn, lineno, m)
-        return
-
     raise ParseError("unparsed line: '%s'" % s, fn, lineno);
 
 # Add us to the handlers list

bitbake/lib/bb/persist_data.py

@@ -63,7 +63,7 @@ class SQLTable(collections.abc.MutableMapping):
             """
             Decorator that starts a database transaction and creates a database
             cursor for performing queries. If no exception is thrown, the
-            database results are committed. If an exception occurs, the database
+            database results are commited. If an exception occurs, the database
             is rolled back. In all cases, the cursor is closed after the
             function ends.
 
@@ -208,7 +208,7 @@ class SQLTable(collections.abc.MutableMapping):
 
     def __lt__(self, other):
         if not isinstance(other, Mapping):
-            raise NotImplementedError()
+            raise NotImplemented
 
         return len(self) < len(other)
 
@@ -249,23 +249,4 @@ def persist(domain, d):
 
     bb.utils.mkdirhier(cachedir)
     cachefile = os.path.join(cachedir, "bb_persist_data.sqlite3")
-
-    try:
-        return SQLTable(cachefile, domain)
-    except sqlite3.OperationalError:
-        # Sqlite fails to open database when its path is too long.
-        # After testing, 504 is the biggest path length that can be opened by
-        # sqlite.
-        # Note: This code is called before sanity.bbclass and its path length
-        # check
-        max_len = 504
-        if len(cachefile) > max_len:
-            logger.critical("The path of the cache file is too long "
-                    "({0} chars > {1}) to be opened by sqlite! "
-                    "Your cache file is \"{2}\"".format(
-                        len(cachefile),
-                        max_len,
-                        cachefile))
-            sys.exit(1)
-        else:
-            raise
+    return SQLTable(cachefile, domain)

bitbake/lib/bb/process.py

@@ -1,6 +1,4 @@
 #
-# Copyright BitBake Contributors
-#
 # SPDX-License-Identifier: GPL-2.0-only
 #
 
@@ -62,7 +60,7 @@ class Popen(subprocess.Popen):
         "close_fds": True,
         "preexec_fn": subprocess_setup,
         "stdout": subprocess.PIPE,
-        "stderr": subprocess.PIPE,
+        "stderr": subprocess.STDOUT,
         "stdin": subprocess.PIPE,
         "shell": False,
     }
@@ -144,7 +142,7 @@ def _logged_communicate(pipe, log, input, extrafiles):
         while pipe.poll() is None:
             read_all_pipes(log, rin, outdata, errdata)
 
-        # Process closed, drain all pipes...
+        # Pocess closed, drain all pipes...
         read_all_pipes(log, rin, outdata, errdata)
     finally:
         log.flush()

bitbake/lib/bb/progress.py

@@ -148,7 +148,7 @@ class MultiStageProgressReporter:
     for tasks made up of python code spread across multiple
     classes / functions - the progress reporter object can
     be passed around or stored at the object level and calls
-    to next_stage() and update() made wherever needed.
+    to next_stage() and update() made whereever needed.
     """
     def __init__(self, d, stage_weights, debug=False):
         """

bitbake/lib/bb/providers.py

@@ -396,8 +396,8 @@ def getRuntimeProviders(dataCache, rdepend):
         return rproviders
 
     # Only search dynamic packages if we can't find anything in other variables
-    for pat_key in dataCache.packages_dynamic:
-        pattern = pat_key.replace(r'+', r"\+")
+    for pattern in dataCache.packages_dynamic:
+        pattern = pattern.replace(r'+', r"\+")
         if pattern in regexp_cache:
             regexp = regexp_cache[pattern]
         else:
@@ -408,7 +408,7 @@ def getRuntimeProviders(dataCache, rdepend):
                 raise
             regexp_cache[pattern] = regexp
         if regexp.match(rdepend):
-            rproviders += dataCache.packages_dynamic[pat_key]
+            rproviders += dataCache.packages_dynamic[pattern]
             logger.debug("Assuming %s is a dynamic package, but it may not exist" % rdepend)
 
     return rproviders

bitbake/lib/bb/server/process.py

@@ -27,8 +27,6 @@ import re
 import datetime
 import pickle
 import traceback
-import gc
-import stat
 import bb.server.xmlrpcserver
 from bb import daemonize
 from multiprocessing import queues
@@ -38,46 +36,10 @@ logger = logging.getLogger('BitBake')
 class ProcessTimeout(SystemExit):
     pass
 
-def currenttime():
-    return datetime.datetime.now().strftime('%H:%M:%S.%f')
-
 def serverlog(msg):
-    print(str(os.getpid()) + " " +  currenttime() + " " + msg)
+    print(str(os.getpid()) + " " +  datetime.datetime.now().strftime('%H:%M:%S.%f') + " " + msg)
     sys.stdout.flush()
 
-#
-# When we have lockfile issues, try and find infomation about which process is
-# using the lockfile
-#
-def get_lockfile_process_msg(lockfile):
-    # Some systems may not have lsof available
-    procs = None
-    try:
-        procs = subprocess.check_output(["lsof", '-w', lockfile], stderr=subprocess.STDOUT)
-    except subprocess.CalledProcessError:
-        # File was deleted?
-        pass
-    except OSError as e:
-        if e.errno != errno.ENOENT:
-            raise
-    if procs is None:
-        # Fall back to fuser if lsof is unavailable
-        try:
-            procs = subprocess.check_output(["fuser", '-v', lockfile], stderr=subprocess.STDOUT)
-        except subprocess.CalledProcessError:
-            # File was deleted?
-            pass
-        except OSError as e:
-            if e.errno != errno.ENOENT:
-                raise
-    if procs:
-        return procs.decode("utf-8")
-    return None
-
-class idleFinish():
-    def __init__(self, msg):
-         self.msg = msg
-
 class ProcessServer():
     profile_filename = "profile.log"
     profile_processed_filename = "profile.log.processed"
@@ -95,19 +57,12 @@ class ProcessServer():
         self.maxuiwait = 30
         self.xmlrpc = False
 
-        self.idle = None
-        # Need a lock for _idlefuns changes
         self._idlefuns = {}
-        self._idlefuncsLock = threading.Lock()
-        self.idle_cond = threading.Condition(self._idlefuncsLock)
 
         self.bitbake_lock = lock
         self.bitbake_lock_name = lockname
         self.sock = sock
         self.sockname = sockname
-        # It is possible the directory may be renamed. Cache the inode of the socket file
-        # so we can tell if things changed.
-        self.sockinode = os.stat(self.sockname)[stat.ST_INO]
 
         self.server_timeout = server_timeout
         self.timeout = self.server_timeout
@@ -116,9 +71,7 @@ class ProcessServer():
     def register_idle_function(self, function, data):
         """Register a function to be called while the server is idle"""
         assert hasattr(function, '__call__')
-        with bb.utils.lock_timeout(self._idlefuncsLock):
-            self._idlefuns[function] = data
-        serverlog("Registering idle function %s" % str(function))
+        self._idlefuns[function] = data
 
     def run(self):
 
@@ -157,31 +110,6 @@ class ProcessServer():
 
         return ret
 
-    def _idle_check(self):
-        return len(self._idlefuns) == 0 and self.cooker.command.currentAsyncCommand is None
-
-    def wait_for_idle(self, timeout=30):
-        # Wait for the idle loop to have cleared
-        with bb.utils.lock_timeout(self._idlefuncsLock):
-            return self.idle_cond.wait_for(self._idle_check, timeout) is not False
-
-    def set_async_cmd(self, cmd):
-        with bb.utils.lock_timeout(self._idlefuncsLock):
-            ret = self.idle_cond.wait_for(self._idle_check, 30)
-            if ret is False:
-                return False
-            self.cooker.command.currentAsyncCommand = cmd
-            return True
-
-    def clear_async_cmd(self):
-        with bb.utils.lock_timeout(self._idlefuncsLock):
-            self.cooker.command.currentAsyncCommand = None
-            self.idle_cond.notify_all()
-
-    def get_async_cmd(self):
-        with bb.utils.lock_timeout(self._idlefuncsLock):
-            return self.cooker.command.currentAsyncCommand
-
     def main(self):
         self.cooker.pre_serve()
 
@@ -196,19 +124,14 @@ class ProcessServer():
             fds.append(self.xmlrpc)
         seendata = False
         serverlog("Entering server connection loop")
-        serverlog("Lockfile is: %s\nSocket is %s (%s)" % (self.bitbake_lock_name, self.sockname, os.path.exists(self.sockname)))
 
         def disconnect_client(self, fds):
-            serverlog("Disconnecting Client (socket: %s)" % os.path.exists(self.sockname))
+            serverlog("Disconnecting Client")
             if self.controllersock:
                 fds.remove(self.controllersock)
                 self.controllersock.close()
                 self.controllersock = False
             if self.haveui:
-                # Wait for the idle loop to have cleared (30s max)
-                if not self.wait_for_idle(30):
-                    serverlog("Idle loop didn't finish queued commands after 30s, exiting.")
-                    self.quit = True
                 fds.remove(self.command_channel)
                 bb.event.unregister_UIHhandler(self.event_handle, True)
                 self.command_channel_reply.writer.close()
@@ -220,7 +143,7 @@ class ProcessServer():
                 self.cooker.clientComplete()
                 self.haveui = False
             ready = select.select(fds,[],[],0)[0]
-            if newconnections and not self.quit:
+            if newconnections:
                 serverlog("Starting new client")
                 conn = newconnections.pop(-1)
                 fds.append(conn)
@@ -292,10 +215,8 @@ class ProcessServer():
                     continue
                 try:
                     serverlog("Running command %s" % command)
-                    reply = self.cooker.command.runCommand(command, self)
-                    serverlog("Sending reply %s" % repr(reply))
-                    self.command_channel_reply.send(reply)
-                    serverlog("Command Completed (socket: %s)" % os.path.exists(self.sockname))
+                    self.command_channel_reply.send(self.cooker.command.runCommand(command))
+                    serverlog("Command Completed")
                 except Exception as e:
                    stack = traceback.format_exc()
                    serverlog('Exception in server main event loop running command %s (%s)' % (command, stack))
@@ -322,25 +243,19 @@ class ProcessServer():
 
             ready = self.idle_commands(.1, fds)
 
-        if self.idle:
-            self.idle.join()
+        if len(threading.enumerate()) != 1:
+            serverlog("More than one thread left?: " + str(threading.enumerate()))
 
-        serverlog("Exiting (socket: %s)" % os.path.exists(self.sockname))
+        serverlog("Exiting")
         # Remove the socket file so we don't get any more connections to avoid races
-        # The build directory could have been renamed so if the file isn't the one we created
-        # we shouldn't delete it.
         try:
-            sockinode = os.stat(self.sockname)[stat.ST_INO]
-            if sockinode == self.sockinode:
-                os.unlink(self.sockname)
-            else:
-                serverlog("bitbake.sock inode mismatch (%s vs %s), not deleting." % (sockinode, self.sockinode))
-        except Exception as err:
-            serverlog("Removing socket file '%s' failed (%s)" % (self.sockname, err))
+            os.unlink(self.sockname)
+        except:
+            pass
         self.sock.close()
 
         try:
-            self.cooker.shutdown(True, idle=False)
+            self.cooker.shutdown(True)
             self.cooker.notifier.stop()
             self.cooker.confignotifier.stop()
         except:
@@ -348,9 +263,6 @@ class ProcessServer():
 
         self.cooker.post_serve()
 
-        if len(threading.enumerate()) != 1:
-            serverlog("More than one thread left?: " + str(threading.enumerate()))
-
         # Flush logs before we release the lock
         sys.stdout.flush()
         sys.stderr.flush()
@@ -366,21 +278,20 @@ class ProcessServer():
             except FileNotFoundError:
                 return None
 
+        lockcontents = get_lock_contents(lockfile)
+        serverlog("Original lockfile contents: " + str(lockcontents))
+
         lock.close()
         lock = None
 
         while not lock:
             i = 0
             lock = None
-            if not os.path.exists(os.path.basename(lockfile)):
-                serverlog("Lockfile directory gone, exiting.")
-                return
-
             while not lock and i < 30:
                 lock = bb.utils.lockfile(lockfile, shared=False, retry=False, block=False)
                 if not lock:
                     newlockcontents = get_lock_contents(lockfile)
-                    if not newlockcontents[0].startswith([f"{os.getpid()}\n", f"{os.getpid()} "]):
+                    if newlockcontents != lockcontents:
                         # A new server was started, the lockfile contents changed, we can exit
                         serverlog("Lockfile now contains different contents, exiting: " + str(newlockcontents))
                         return
@@ -394,98 +305,80 @@ class ProcessServer():
                 return
 
             if not lock:
-                procs = get_lockfile_process_msg(lockfile)
-                msg = ["Delaying shutdown due to active processes which appear to be holding bitbake.lock"]
-                if procs:
-                    msg.append(":\n%s" % procs)
-                serverlog("".join(msg))
-
-    def idle_thread(self):
-        def remove_idle_func(function):
-            with bb.utils.lock_timeout(self._idlefuncsLock):
-                del self._idlefuns[function]
-                self.idle_cond.notify_all()
-
-        while not self.quit:
-            nextsleep = 0.1
-            fds = []
-
-            try:
-                self.cooker.process_inotify_updates()
-            except Exception as exc:
-                serverlog("Exception %s in inofify updates broke the idle_thread, exiting" % traceback.format_exc())
-                self.quit = True
-
-            with bb.utils.lock_timeout(self._idlefuncsLock):
-                items = list(self._idlefuns.items())
-
-            for function, data in items:
+                # Some systems may not have lsof available
+                procs = None
                 try:
-                    retval = function(self, data, False)
-                    if isinstance(retval, idleFinish):
-                        serverlog("Removing idle function %s at idleFinish" % str(function))
-                        remove_idle_func(function)
-                        self.cooker.command.finishAsyncCommand(retval.msg)
-                        nextsleep = None
-                    elif retval is False:
-                        serverlog("Removing idle function %s" % str(function))
-                        remove_idle_func(function)
-                        nextsleep = None
-                    elif retval is True:
-                        nextsleep = None
-                    elif isinstance(retval, float) and nextsleep:
-                        if (retval < nextsleep):
-                            nextsleep = retval
-                    elif nextsleep is None:
-                        continue
-                    else:
-                        fds = fds + retval
-                except SystemExit:
-                    raise
-                except Exception as exc:
-                    if not isinstance(exc, bb.BBHandledException):
-                        logger.exception('Running idle function')
-                    remove_idle_func(function)
-                    serverlog("Exception %s broke the idle_thread, exiting" % traceback.format_exc())
-                    self.quit = True
-
-            # Create new heartbeat event?
-            now = time.time()
-            if bb.event._heartbeat_enabled and now >= self.next_heartbeat:
-                # We might have missed heartbeats. Just trigger once in
-                # that case and continue after the usual delay.
-                self.next_heartbeat += self.heartbeat_seconds
-                if self.next_heartbeat <= now:
-                    self.next_heartbeat = now + self.heartbeat_seconds
-                if hasattr(self.cooker, "data"):
-                    heartbeat = bb.event.HeartbeatEvent(now)
+                    procs = subprocess.check_output(["lsof", '-w', lockfile], stderr=subprocess.STDOUT)
+                except subprocess.CalledProcessError:
+                    # File was deleted?
+                    continue
+                except OSError as e:
+                    if e.errno != errno.ENOENT:
+                        raise
+                if procs is None:
+                    # Fall back to fuser if lsof is unavailable
                     try:
-                        bb.event.fire(heartbeat, self.cooker.data)
-                    except Exception as exc:
-                        if not isinstance(exc, bb.BBHandledException):
-                            logger.exception('Running heartbeat function')
-                        serverlog("Exception %s broke in idle_thread, exiting" % traceback.format_exc())
-                        self.quit = True
-            if nextsleep and bb.event._heartbeat_enabled and now + nextsleep > self.next_heartbeat:
-                # Shorten timeout so that we we wake up in time for
-                # the heartbeat.
-                nextsleep = self.next_heartbeat - now
+                        procs = subprocess.check_output(["fuser", '-v', lockfile], stderr=subprocess.STDOUT)
+                    except subprocess.CalledProcessError:
+                        # File was deleted?
+                        continue
+                    except OSError as e:
+                        if e.errno != errno.ENOENT:
+                            raise
 
-            if nextsleep is not None:
-                select.select(fds,[],[],nextsleep)[0]
+                msg = "Delaying shutdown due to active processes which appear to be holding bitbake.lock"
+                if procs:
+                    msg += ":\n%s" % str(procs.decode("utf-8"))
+                serverlog(msg)
 
     def idle_commands(self, delay, fds=None):
         nextsleep = delay
         if not fds:
             fds = []
 
-        if not self.idle:
-            self.idle = threading.Thread(target=self.idle_thread)
-            self.idle.start()
-        elif self.idle and not self.idle.is_alive():
-            serverlog("Idle thread terminated, main thread exiting too")
-            bb.error("Idle thread terminated, main thread exiting too")
-            self.quit = True
+        for function, data in list(self._idlefuns.items()):
+            try:
+                retval = function(self, data, False)
+                if retval is False:
+                    del self._idlefuns[function]
+                    nextsleep = None
+                elif retval is True:
+                    nextsleep = None
+                elif isinstance(retval, float) and nextsleep:
+                    if (retval < nextsleep):
+                        nextsleep = retval
+                elif nextsleep is None:
+                    continue
+                else:
+                    fds = fds + retval
+            except SystemExit:
+                raise
+            except Exception as exc:
+                if not isinstance(exc, bb.BBHandledException):
+                    logger.exception('Running idle function')
+                del self._idlefuns[function]
+                self.quit = True
+
+        # Create new heartbeat event?
+        now = time.time()
+        if now >= self.next_heartbeat:
+            # We might have missed heartbeats. Just trigger once in
+            # that case and continue after the usual delay.
+            self.next_heartbeat += self.heartbeat_seconds
+            if self.next_heartbeat <= now:
+                self.next_heartbeat = now + self.heartbeat_seconds
+            if hasattr(self.cooker, "data"):
+                heartbeat = bb.event.HeartbeatEvent(now)
+                try:
+                    bb.event.fire(heartbeat, self.cooker.data)
+                except Exception as exc:
+                    if not isinstance(exc, bb.BBHandledException):
+                        logger.exception('Running heartbeat function')
+                    self.quit = True
+        if nextsleep and now + nextsleep > self.next_heartbeat:
+            # Shorten timeout so that we we wake up in time for
+            # the heartbeat.
+            nextsleep = self.next_heartbeat - now
 
         if nextsleep is not None:
             if self.xmlrpc:
@@ -507,9 +400,9 @@ class ServerCommunicator():
     def runCommand(self, command):
         self.connection.send(command)
         if not self.recv.poll(30):
-            logger.info("No reply from server in 30s (for command %s at %s)" % (command[0], currenttime()))
+            logger.info("No reply from server in 30s")
             if not self.recv.poll(30):
-                raise ProcessTimeout("Timeout while waiting for a reply from the bitbake server (60s at %s)" % currenttime())
+                raise ProcessTimeout("Timeout while waiting for a reply from the bitbake server (60s)")
         ret, exc = self.recv.get()
         # Should probably turn all exceptions in exc back into exceptions?
         # For now, at least handle BBHandledException
@@ -543,7 +436,6 @@ class BitBakeProcessServerConnection(object):
         self.socket_connection = sock
 
     def terminate(self):
-        self.events.close()
         self.socket_connection.close()
         self.connection.connection.close()
         self.connection.recv.close()
@@ -554,14 +446,13 @@ start_log_datetime_format = '%Y-%m-%d %H:%M:%S.%f'
 
 class BitBakeServer(object):
 
-    def __init__(self, lock, sockname, featureset, server_timeout, xmlrpcinterface, profile):
+    def __init__(self, lock, sockname, featureset, server_timeout, xmlrpcinterface):
 
         self.server_timeout = server_timeout
         self.xmlrpcinterface = xmlrpcinterface
         self.featureset = featureset
         self.sockname = sockname
         self.bitbake_lock = lock
-        self.profile = profile
         self.readypipe, self.readypipein = os.pipe()
 
         # Place the log in the builddirectory alongside the lock file
@@ -625,9 +516,9 @@ class BitBakeServer(object):
         os.set_inheritable(self.bitbake_lock.fileno(), True)
         os.set_inheritable(self.readypipein, True)
         serverscript = os.path.realpath(os.path.dirname(__file__) + "/../../../bin/bitbake-server")
-        os.execl(sys.executable, "bitbake-server", serverscript, "decafbad", str(self.bitbake_lock.fileno()), str(self.readypipein), self.logfile, self.bitbake_lock.name, self.sockname,  str(self.server_timeout or 0), str(int(self.profile)), str(self.xmlrpcinterface[0]), str(self.xmlrpcinterface[1]))
+        os.execl(sys.executable, "bitbake-server", serverscript, "decafbad", str(self.bitbake_lock.fileno()), str(self.readypipein), self.logfile, self.bitbake_lock.name, self.sockname,  str(self.server_timeout or 0), str(self.xmlrpcinterface[0]), str(self.xmlrpcinterface[1]))
 
-def execServer(lockfd, readypipeinfd, lockname, sockname, server_timeout, xmlrpcinterface, profile):
+def execServer(lockfd, readypipeinfd, lockname, sockname, server_timeout, xmlrpcinterface):
 
     import bb.cookerdata
     import bb.cooker
@@ -639,7 +530,6 @@ def execServer(lockfd, readypipeinfd, lockname, sockname, server_timeout, xmlrpc
 
         # Create server control socket
         if os.path.exists(sockname):
-            serverlog("WARNING: removing existing socket file '%s'" % sockname)
             os.unlink(sockname)
 
         sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
@@ -656,8 +546,7 @@ def execServer(lockfd, readypipeinfd, lockname, sockname, server_timeout, xmlrpc
         writer = ConnectionWriter(readypipeinfd)
         try:
             featureset = []
-            cooker = bb.cooker.BBCooker(featureset, server)
-            cooker.configuration.profile = profile
+            cooker = bb.cooker.BBCooker(featureset, server.register_idle_function)
         except bb.BBHandledException:
             return None
         writer.send("r")
@@ -667,7 +556,7 @@ def execServer(lockfd, readypipeinfd, lockname, sockname, server_timeout, xmlrpc
 
         server.run()
     finally:
-        # Flush any messages/errors to the logfile before exit
+        # Flush any ,essages/errors to the logfile before exit
         sys.stdout.flush()
         sys.stderr.flush()
 
@@ -772,18 +661,23 @@ class BBUIEventQueue:
         self.reader = ConnectionReader(readfd)
 
         self.t = threading.Thread()
+        self.t.daemon = True
         self.t.run = self.startCallbackHandler
         self.t.start()
 
     def getEvent(self):
-        with bb.utils.lock_timeout(self.eventQueueLock):
-            if len(self.eventQueue) == 0:
-                return None
+        self.eventQueueLock.acquire()
 
-            item = self.eventQueue.pop(0)
-            if len(self.eventQueue) == 0:
-                self.eventQueueNotify.clear()
+        if len(self.eventQueue) == 0:
+            self.eventQueueLock.release()
+            return None
 
+        item = self.eventQueue.pop(0)
+
+        if len(self.eventQueue) == 0:
+            self.eventQueueNotify.clear()
+
+        self.eventQueueLock.release()
         return item
 
     def waitEvent(self, delay):
@@ -791,9 +685,10 @@ class BBUIEventQueue:
         return self.getEvent()
 
     def queue_event(self, event):
-        with bb.utils.lock_timeout(self.eventQueueLock):
-            self.eventQueue.append(event)
-            self.eventQueueNotify.set()
+        self.eventQueueLock.acquire()
+        self.eventQueue.append(event)
+        self.eventQueueNotify.set()
+        self.eventQueueLock.release()
 
     def send_event(self, event):
         self.queue_event(pickle.loads(event))
@@ -802,17 +697,13 @@ class BBUIEventQueue:
         bb.utils.set_process_name("UIEventQueue")
         while True:
             try:
-                ready = self.reader.wait(0.25)
-                if ready:
-                    event = self.reader.get()
-                    self.queue_event(event)
-            except (EOFError, OSError, TypeError):
+                self.reader.wait()
+                event = self.reader.get()
+                self.queue_event(event)
+            except EOFError:
                 # Easiest way to exit is to close the file descriptor to cause an exit
                 break
-
-    def close(self):
         self.reader.close()
-        self.t.join()
 
 class ConnectionReader(object):
 
@@ -827,7 +718,7 @@ class ConnectionReader(object):
         return self.reader.poll(timeout)
 
     def get(self):
-        with bb.utils.lock_timeout(self.rlock):
+        with self.rlock:
             res = self.reader.recv_bytes()
         return multiprocessing.reduction.ForkingPickler.loads(res)
 
@@ -846,31 +737,10 @@ class ConnectionWriter(object):
         # Why bb.event needs this I have no idea
         self.event = self
 
-    def _send(self, obj):
-        gc.disable()
-        with bb.utils.lock_timeout(self.wlock):
-            self.writer.send_bytes(obj)
-        gc.enable()
-
     def send(self, obj):
         obj = multiprocessing.reduction.ForkingPickler.dumps(obj)
-        # See notes/code in CookerParser
-        # We must not terminate holding this lock else processes will hang.
-        # For SIGTERM, raising afterwards avoids this.
-        # For SIGINT, we don't want to have written partial data to the pipe.
-        # pthread_sigmask block/unblock would be nice but doesn't work, https://bugs.python.org/issue47139
-        process = multiprocessing.current_process()
-        if process and hasattr(process, "queue_signals"):
-            with bb.utils.lock_timeout(process.signal_threadlock):
-                process.queue_signals = True
-                self._send(obj)
-                process.queue_signals = False
-
-                while len(process.signal_received) > 0:
-                    sig = process.signal_received.pop()
-                    process.handle_sig(sig, None)
-        else:
-            self._send(obj)
+        with self.wlock:
+            self.writer.send_bytes(obj)
 
     def fileno(self):
         return self.writer.fileno()

bitbake/lib/bb/server/xmlrpcserver.py

@@ -11,7 +11,6 @@ import hashlib
 import time
 import inspect
 from xmlrpc.server import SimpleXMLRPCServer, SimpleXMLRPCRequestHandler
-import bb.server.xmlrpcclient
 
 import bb
 
@@ -118,7 +117,7 @@ class BitBakeXMLRPCServerCommands():
         """
         Run a cooker command on the server
         """
-        return self.server.cooker.command.runCommand(command, self.server.parent, self.server.readonly)
+        return self.server.cooker.command.runCommand(command, self.server.readonly)
 
     def getEventHandle(self):
         return self.event_handle

bitbake/lib/bb/siggen.py

@@ -1,6 +1,4 @@
 #
-# Copyright BitBake Contributors
-#
 # SPDX-License-Identifier: GPL-2.0-only
 #
 
@@ -13,9 +11,6 @@ import pickle
 import bb.data
 import difflib
 import simplediff
-import json
-import types
-import bb.compress.zstd
 from bb.checksum import FileChecksumCache
 from bb import runqueue
 import hashserv
@@ -24,17 +19,6 @@ import hashserv.client
 logger = logging.getLogger('BitBake.SigGen')
 hashequiv_logger = logging.getLogger('BitBake.SigGen.HashEquiv')
 
-class SetEncoder(json.JSONEncoder):
-    def default(self, obj):
-        if isinstance(obj, set) or isinstance(obj, frozenset):
-            return dict(_set_object=list(sorted(obj)))
-        return json.JSONEncoder.default(self, obj)
-
-def SetDecoder(dct):
-    if '_set_object' in dct:
-        return frozenset(dct['_set_object'])
-    return dct
-
 def init(d):
     siggens = [obj for obj in globals().values()
                       if type(obj) is type and issubclass(obj, SignatureGenerator)]
@@ -43,6 +27,7 @@ def init(d):
     for sg in siggens:
         if desired == sg.name:
             return sg(d)
+            break
     else:
         logger.error("Invalid signature generator '%s', using default 'noop'\n"
                      "Available generators: %s", desired,
@@ -54,6 +39,11 @@ class SignatureGenerator(object):
     """
     name = "noop"
 
+    # If the derived class supports multiconfig datacaches, set this to True
+    # The default is False for backward compatibility with derived signature
+    # generators that do not understand multiconfig caches
+    supports_multiconfig_datacaches = False
+
     def __init__(self, data):
         self.basehash = {}
         self.taskhash = {}
@@ -71,27 +61,6 @@ class SignatureGenerator(object):
     def postparsing_clean_cache(self):
         return
 
-    def setup_datacache(self, datacaches):
-        self.datacaches = datacaches
-
-    def setup_datacache_from_datastore(self, mcfn, d):
-        # In task context we have no cache so setup internal data structures
-        # from the fully parsed data store provided
-
-        mc = d.getVar("__BBMULTICONFIG", False) or ""
-        tasks = d.getVar('__BBTASKS', False)
-
-        self.datacaches = {}
-        self.datacaches[mc] = types.SimpleNamespace()
-        setattr(self.datacaches[mc], "stamp", {})
-        self.datacaches[mc].stamp[mcfn] = d.getVar('STAMP')
-        setattr(self.datacaches[mc], "stamp_extrainfo", {})
-        self.datacaches[mc].stamp_extrainfo[mcfn] = {}
-        for t in tasks:
-            flag = d.getVarFlag(t, "stamp-extra-info")
-            if flag:
-                self.datacaches[mc].stamp_extrainfo[mcfn][t] = flag
-
     def get_unihash(self, tid):
         return self.taskhash[tid]
 
@@ -106,51 +75,17 @@ class SignatureGenerator(object):
         """Write/update the file checksum cache onto disk"""
         return
 
-    def stampfile_base(self, mcfn):
-        mc = bb.runqueue.mc_from_tid(mcfn)
-        return self.datacaches[mc].stamp[mcfn]
-
-    def stampfile_mcfn(self, taskname, mcfn, extrainfo=True):
-        mc = bb.runqueue.mc_from_tid(mcfn)
-        stamp = self.datacaches[mc].stamp[mcfn]
-        if not stamp:
-            return
-
-        stamp_extrainfo = ""
-        if extrainfo:
-            taskflagname = taskname
-            if taskname.endswith("_setscene"):
-                taskflagname = taskname.replace("_setscene", "")
-            stamp_extrainfo = self.datacaches[mc].stamp_extrainfo[mcfn].get(taskflagname) or ""
-
-        return self.stampfile(stamp, mcfn, taskname, stamp_extrainfo)
-
     def stampfile(self, stampbase, file_name, taskname, extrainfo):
         return ("%s.%s.%s" % (stampbase, taskname, extrainfo)).rstrip('.')
 
-    def stampcleanmask_mcfn(self, taskname, mcfn):
-        mc = bb.runqueue.mc_from_tid(mcfn)
-        stamp = self.datacaches[mc].stamp[mcfn]
-        if not stamp:
-            return []
-
-        taskflagname = taskname
-        if taskname.endswith("_setscene"):
-            taskflagname = taskname.replace("_setscene", "")
-        stamp_extrainfo = self.datacaches[mc].stamp_extrainfo[mcfn].get(taskflagname) or ""
-
-        return self.stampcleanmask(stamp, mcfn, taskname, stamp_extrainfo)
-
     def stampcleanmask(self, stampbase, file_name, taskname, extrainfo):
         return ("%s.%s.%s" % (stampbase, taskname, extrainfo)).rstrip('.')
 
-    def dump_sigtask(self, mcfn, task, stampbase, runtime):
+    def dump_sigtask(self, fn, task, stampbase, runtime):
         return
 
-    def invalidate_task(self, task, mcfn):
-        mc = bb.runqueue.mc_from_tid(mcfn)
-        stamp = self.datacaches[mc].stamp[mcfn]
-        bb.utils.remove(stamp)
+    def invalidate_task(self, task, d, fn):
+        bb.build.del_stamp(task, d, fn)
 
     def dump_sigs(self, dataCache, options):
         return
@@ -173,19 +108,40 @@ class SignatureGenerator(object):
     def save_unitaskhashes(self):
         return
 
-    def copy_unitaskhashes(self, targetdir):
-        return
-
     def set_setscene_tasks(self, setscene_tasks):
         return
 
-    def exit(self):
-        return
+    @classmethod
+    def get_data_caches(cls, dataCaches, mc):
+        """
+        This function returns the datacaches that should be passed to signature
+        generator functions. If the signature generator supports multiconfig
+        caches, the entire dictionary of data caches is sent, otherwise a
+        special proxy is sent that support both index access to all
+        multiconfigs, and also direct access for the default multiconfig.
 
-def build_pnid(mc, pn, taskname):
-    if mc:
-        return "mc:" + mc + ":" + pn + ":" + taskname
-    return pn + ":" + taskname
+        The proxy class allows code in this class itself to always use
+        multiconfig aware code (to ease maintenance), but derived classes that
+        are unaware of multiconfig data caches can still access the default
+        multiconfig as expected.
+
+        Do not override this function in derived classes; it will be removed in
+        the future when support for multiconfig data caches is mandatory
+        """
+        class DataCacheProxy(object):
+            def __init__(self):
+                pass
+
+            def __getitem__(self, key):
+                return dataCaches[key]
+
+            def __getattr__(self, name):
+                return getattr(dataCaches[mc], name)
+
+        if cls.supports_multiconfig_datacaches:
+            return dataCaches
+
+        return DataCacheProxy()
 
 class SignatureGeneratorBasic(SignatureGenerator):
     """
@@ -196,12 +152,15 @@ class SignatureGeneratorBasic(SignatureGenerator):
         self.basehash = {}
         self.taskhash = {}
         self.unihash = {}
+        self.taskdeps = {}
         self.runtaskdeps = {}
         self.file_checksum_values = {}
         self.taints = {}
+        self.gendeps = {}
+        self.lookupcache = {}
         self.setscenetasks = set()
-        self.basehash_ignore_vars = set((data.getVar("BB_BASEHASH_IGNORE_VARS") or "").split())
-        self.taskhash_ignore_tasks = None
+        self.basewhitelist = set((data.getVar("BB_HASHBASE_WHITELIST") or "").split())
+        self.taskwhitelist = None
         self.init_rundepcheck(data)
         checksum_cache_file = data.getVar("BB_HASH_CHECKSUM_CACHE_FILE")
         if checksum_cache_file:
@@ -216,21 +175,21 @@ class SignatureGeneratorBasic(SignatureGenerator):
         self.tidtopn = {}
 
     def init_rundepcheck(self, data):
-        self.taskhash_ignore_tasks = data.getVar("BB_TASKHASH_IGNORE_TASKS") or None
-        if self.taskhash_ignore_tasks:
-            self.twl = re.compile(self.taskhash_ignore_tasks)
+        self.taskwhitelist = data.getVar("BB_HASHTASK_WHITELIST") or None
+        if self.taskwhitelist:
+            self.twl = re.compile(self.taskwhitelist)
         else:
             self.twl = None
 
-    def _build_data(self, mcfn, d):
+    def _build_data(self, fn, d):
 
         ignore_mismatch = ((d.getVar("BB_HASH_IGNORE_MISMATCH") or '') == '1')
-        tasklist, gendeps, lookupcache = bb.data.generate_dependencies(d, self.basehash_ignore_vars)
+        tasklist, gendeps, lookupcache = bb.data.generate_dependencies(d, self.basewhitelist)
 
-        taskdeps, basehash = bb.data.generate_dependency_hash(tasklist, gendeps, lookupcache, self.basehash_ignore_vars, mcfn)
+        taskdeps, basehash = bb.data.generate_dependency_hash(tasklist, gendeps, lookupcache, self.basewhitelist, fn)
 
         for task in tasklist:
-            tid = mcfn + ":" + task
+            tid = fn + ":" + task
             if not ignore_mismatch and tid in self.basehash and self.basehash[tid] != basehash[tid]:
                 bb.error("When reparsing %s, the basehash value changed from %s to %s. The metadata is not deterministic and this needs to be fixed." % (tid, self.basehash[tid], basehash[tid]))
                 bb.error("The following commands may help:")
@@ -241,7 +200,11 @@ class SignatureGeneratorBasic(SignatureGenerator):
                 bb.error("%s -Sprintdiff\n" % cmd)
             self.basehash[tid] = basehash[tid]
 
-        return taskdeps, gendeps, lookupcache
+        self.taskdeps[fn] = taskdeps
+        self.gendeps[fn] = gendeps
+        self.lookupcache[fn] = lookupcache
+
+        return taskdeps
 
     def set_setscene_tasks(self, setscene_tasks):
         self.setscenetasks = set(setscene_tasks)
@@ -249,47 +212,35 @@ class SignatureGeneratorBasic(SignatureGenerator):
     def finalise(self, fn, d, variant):
 
         mc = d.getVar("__BBMULTICONFIG", False) or ""
-        mcfn = fn
         if variant or mc:
-            mcfn = bb.cache.realfn2virtual(fn, variant, mc)
+            fn = bb.cache.realfn2virtual(fn, variant, mc)
 
         try:
-            taskdeps, gendeps, lookupcache = self._build_data(mcfn, d)
+            taskdeps = self._build_data(fn, d)
         except bb.parse.SkipRecipe:
             raise
         except:
-            bb.warn("Error during finalise of %s" % mcfn)
+            bb.warn("Error during finalise of %s" % fn)
             raise
 
-        basehashes = {}
-        for task in taskdeps:
-            basehashes[task] = self.basehash[mcfn + ":" + task]
-
-        d.setVar("__siggen_basehashes", basehashes)
-        d.setVar("__siggen_gendeps", gendeps)
-        d.setVar("__siggen_varvals", lookupcache)
-        d.setVar("__siggen_taskdeps", taskdeps)
-
         #Slow but can be useful for debugging mismatched basehashes
-        #self.setup_datacache_from_datastore(mcfn, d)
-        #for task in taskdeps:
-        #    self.dump_sigtask(mcfn, task, d.getVar("STAMP"), False)
+        #for task in self.taskdeps[fn]:
+        #    self.dump_sigtask(fn, task, d.getVar("STAMP"), False)
 
-    def setup_datacache_from_datastore(self, mcfn, d):
-        super().setup_datacache_from_datastore(mcfn, d)
+        for task in taskdeps:
+            d.setVar("BB_BASEHASH:task-%s" % task, self.basehash[fn + ":" + task])
 
-        mc = bb.runqueue.mc_from_tid(mcfn)
-        for attr in ["siggen_varvals", "siggen_taskdeps", "siggen_gendeps"]:
-            if not hasattr(self.datacaches[mc], attr):
-                setattr(self.datacaches[mc], attr, {})
-        self.datacaches[mc].siggen_varvals[mcfn] = d.getVar("__siggen_varvals")
-        self.datacaches[mc].siggen_taskdeps[mcfn] = d.getVar("__siggen_taskdeps")
-        self.datacaches[mc].siggen_gendeps[mcfn] = d.getVar("__siggen_gendeps")
+    def postparsing_clean_cache(self):
+        #
+        # After parsing we can remove some things from memory to reduce our memory footprint
+        #
+        self.gendeps = {}
+        self.lookupcache = {}
+        self.taskdeps = {}
 
     def rundep_check(self, fn, recipename, task, dep, depname, dataCaches):
         # Return True if we should keep the dependency, False to drop it
-        # We only manipulate the dependencies for packages not in the ignore
-        # list
+        # We only manipulate the dependencies for packages not in the whitelist
         if self.twl and not self.twl.search(recipename):
             # then process the actual dependencies
             if self.twl.search(depname):
@@ -307,37 +258,38 @@ class SignatureGeneratorBasic(SignatureGenerator):
 
     def prep_taskhash(self, tid, deps, dataCaches):
 
-        (mc, _, task, mcfn) = bb.runqueue.split_tid_mcfn(tid)
+        (mc, _, task, fn) = bb.runqueue.split_tid_mcfn(tid)
 
         self.basehash[tid] = dataCaches[mc].basetaskhash[tid]
         self.runtaskdeps[tid] = []
         self.file_checksum_values[tid] = []
-        recipename = dataCaches[mc].pkg_fn[mcfn]
+        recipename = dataCaches[mc].pkg_fn[fn]
 
         self.tidtopn[tid] = recipename
-        # save hashfn for deps into siginfo?
-        for dep in deps:
-            (depmc, _, deptask, depmcfn) = bb.runqueue.split_tid_mcfn(dep)
-            dep_pn = dataCaches[depmc].pkg_fn[depmcfn]
 
-            if not self.rundep_check(mcfn, recipename, task, dep, dep_pn, dataCaches):
+        for dep in sorted(deps, key=clean_basepath):
+            (depmc, _, _, depmcfn) = bb.runqueue.split_tid_mcfn(dep)
+            depname = dataCaches[depmc].pkg_fn[depmcfn]
+            if not self.supports_multiconfig_datacaches and mc != depmc:
+                # If the signature generator doesn't understand multiconfig
+                # data caches, any dependency not in the same multiconfig must
+                # be skipped for backward compatibility
+                continue
+            if not self.rundep_check(fn, recipename, task, dep, depname, dataCaches):
                 continue
-
             if dep not in self.taskhash:
                 bb.fatal("%s is not in taskhash, caller isn't calling in dependency order?" % dep)
+            self.runtaskdeps[tid].append(dep)
 
-            dep_pnid = build_pnid(depmc, dep_pn, deptask)
-            self.runtaskdeps[tid].append((dep_pnid, dep))
-
-        if task in dataCaches[mc].file_checksums[mcfn]:
+        if task in dataCaches[mc].file_checksums[fn]:
             if self.checksum_cache:
-                checksums = self.checksum_cache.get_checksums(dataCaches[mc].file_checksums[mcfn][task], recipename, self.localdirsexclude)
+                checksums = self.checksum_cache.get_checksums(dataCaches[mc].file_checksums[fn][task], recipename, self.localdirsexclude)
             else:
-                checksums = bb.fetch2.get_file_checksums(dataCaches[mc].file_checksums[mcfn][task], recipename, self.localdirsexclude)
+                checksums = bb.fetch2.get_file_checksums(dataCaches[mc].file_checksums[fn][task], recipename, self.localdirsexclude)
             for (f,cs) in checksums:
                 self.file_checksum_values[tid].append((f,cs))
 
-        taskdep = dataCaches[mc].task_deps[mcfn]
+        taskdep = dataCaches[mc].task_deps[fn]
         if 'nostamp' in taskdep and task in taskdep['nostamp']:
             # Nostamp tasks need an implicit taint so that they force any dependent tasks to run
             if tid in self.taints and self.taints[tid].startswith("nostamp:"):
@@ -348,7 +300,7 @@ class SignatureGeneratorBasic(SignatureGenerator):
                 taint = str(uuid.uuid4())
                 self.taints[tid] = "nostamp:" + taint
 
-        taint = self.read_taint(mcfn, task, dataCaches[mc].stamp[mcfn])
+        taint = self.read_taint(fn, task, dataCaches[mc].stamp[fn])
         if taint:
             self.taints[tid] = taint
             logger.warning("%s is tainted from a forced run" % tid)
@@ -358,20 +310,18 @@ class SignatureGeneratorBasic(SignatureGenerator):
     def get_taskhash(self, tid, deps, dataCaches):
 
         data = self.basehash[tid]
-        for dep in sorted(self.runtaskdeps[tid]):
-            data += self.get_unihash(dep[1])
+        for dep in self.runtaskdeps[tid]:
+            data = data + self.get_unihash(dep)
 
-        for (f, cs) in sorted(self.file_checksum_values[tid], key=clean_checksum_file_path):
+        for (f, cs) in self.file_checksum_values[tid]:
             if cs:
-                if "/./" in f:
-                    data += "./" + f.split("/./")[1]
-                data += cs
+                data = data + cs
 
         if tid in self.taints:
             if self.taints[tid].startswith("nostamp:"):
-                data += self.taints[tid][8:]
+                data = data + self.taints[tid][8:]
             else:
-                data += self.taints[tid]
+                data = data + self.taints[tid]
 
         h = hashlib.sha256(data.encode("utf-8")).hexdigest()
         self.taskhash[tid] = h
@@ -390,12 +340,9 @@ class SignatureGeneratorBasic(SignatureGenerator):
     def save_unitaskhashes(self):
         self.unihash_cache.save(self.unitaskhashes)
 
-    def copy_unitaskhashes(self, targetdir):
-        self.unihash_cache.copyfile(targetdir)
+    def dump_sigtask(self, fn, task, stampbase, runtime):
 
-    def dump_sigtask(self, mcfn, task, stampbase, runtime):
-        tid = mcfn + ":" + task
-        mc = bb.runqueue.mc_from_tid(mcfn)
+        tid = fn + ":" + task
         referencestamp = stampbase
         if isinstance(runtime, str) and runtime.startswith("customfile"):
             sigfile = stampbase
@@ -410,34 +357,29 @@ class SignatureGeneratorBasic(SignatureGenerator):
 
         data = {}
         data['task'] = task
-        data['basehash_ignore_vars'] = self.basehash_ignore_vars
-        data['taskhash_ignore_tasks'] = self.taskhash_ignore_tasks
-        data['taskdeps'] = self.datacaches[mc].siggen_taskdeps[mcfn][task]
+        data['basewhitelist'] = self.basewhitelist
+        data['taskwhitelist'] = self.taskwhitelist
+        data['taskdeps'] = self.taskdeps[fn][task]
         data['basehash'] = self.basehash[tid]
         data['gendeps'] = {}
         data['varvals'] = {}
-        data['varvals'][task] = self.datacaches[mc].siggen_varvals[mcfn][task]
-        for dep in self.datacaches[mc].siggen_taskdeps[mcfn][task]:
-            if dep in self.basehash_ignore_vars:
+        data['varvals'][task] = self.lookupcache[fn][task]
+        for dep in self.taskdeps[fn][task]:
+            if dep in self.basewhitelist:
                 continue
-            data['gendeps'][dep] = self.datacaches[mc].siggen_gendeps[mcfn][dep]
-            data['varvals'][dep] = self.datacaches[mc].siggen_varvals[mcfn][dep]
+            data['gendeps'][dep] = self.gendeps[fn][dep]
+            data['varvals'][dep] = self.lookupcache[fn][dep]
 
         if runtime and tid in self.taskhash:
-            data['runtaskdeps'] = [dep[0] for dep in sorted(self.runtaskdeps[tid])]
-            data['file_checksum_values'] = []
-            for f,cs in sorted(self.file_checksum_values[tid], key=clean_checksum_file_path):
-                if "/./" in f:
-                    data['file_checksum_values'].append(("./" + f.split("/./")[1], cs))
-                else:
-                    data['file_checksum_values'].append((os.path.basename(f), cs))
+            data['runtaskdeps'] = self.runtaskdeps[tid]
+            data['file_checksum_values'] = [(os.path.basename(f), cs) for f,cs in self.file_checksum_values[tid]]
             data['runtaskhashes'] = {}
-            for dep in self.runtaskdeps[tid]:
-                data['runtaskhashes'][dep[0]] = self.get_unihash(dep[1])
+            for dep in data['runtaskdeps']:
+                data['runtaskhashes'][dep] = self.get_unihash(dep)
             data['taskhash'] = self.taskhash[tid]
             data['unihash'] = self.get_unihash(tid)
 
-        taint = self.read_taint(mcfn, task, referencestamp)
+        taint = self.read_taint(fn, task, referencestamp)
         if taint:
             data['taint'] = taint
 
@@ -454,11 +396,11 @@ class SignatureGeneratorBasic(SignatureGenerator):
                 bb.error("Taskhash mismatch %s versus %s for %s" % (computed_taskhash, self.taskhash[tid], tid))
                 sigfile = sigfile.replace(self.taskhash[tid], computed_taskhash)
 
-        fd, tmpfile = bb.utils.mkstemp(dir=os.path.dirname(sigfile), prefix="sigtask.")
+        fd, tmpfile = tempfile.mkstemp(dir=os.path.dirname(sigfile), prefix="sigtask.")
         try:
-            with bb.compress.zstd.open(fd, "wt", encoding="utf-8", num_threads=1) as f:
-                json.dump(data, f, sort_keys=True, separators=(",", ":"), cls=SetEncoder)
-                f.flush()
+            with os.fdopen(fd, "wb") as stream:
+                p = pickle.dump(data, stream, -1)
+                stream.flush()
             os.chmod(tmpfile, 0o664)
             bb.utils.rename(tmpfile, sigfile)
         except (OSError, IOError) as err:
@@ -468,6 +410,18 @@ class SignatureGeneratorBasic(SignatureGenerator):
                 pass
             raise err
 
+    def dump_sigfn(self, fn, dataCaches, options):
+        if fn in self.taskdeps:
+            for task in self.taskdeps[fn]:
+                tid = fn + ":" + task
+                mc = bb.runqueue.mc_from_tid(tid)
+                if tid not in self.taskhash:
+                    continue
+                if dataCaches[mc].basetaskhash[tid] != self.basehash[tid]:
+                    bb.error("Bitbake's cached basehash does not match the one we just generated (%s)!" % tid)
+                    bb.error("The mismatched hashes were %s and %s" % (dataCaches[mc].basetaskhash[tid], self.basehash[tid]))
+                self.dump_sigtask(fn, task, dataCaches[mc].stamp[fn], True)
+
 class SignatureGeneratorBasicHash(SignatureGeneratorBasic):
     name = "basichash"
 
@@ -478,11 +432,11 @@ class SignatureGeneratorBasicHash(SignatureGeneratorBasic):
         # If task is not in basehash, then error
         return self.basehash[tid]
 
-    def stampfile(self, stampbase, mcfn, taskname, extrainfo, clean=False):
-        if taskname.endswith("_setscene"):
-            tid = mcfn + ":" + taskname[:-9]
+    def stampfile(self, stampbase, fn, taskname, extrainfo, clean=False):
+        if taskname != "do_setscene" and taskname.endswith("_setscene"):
+            tid = fn + ":" + taskname[:-9]
         else:
-            tid = mcfn + ":" + taskname
+            tid = fn + ":" + taskname
         if clean:
             h = "*"
         else:
@@ -490,23 +444,12 @@ class SignatureGeneratorBasicHash(SignatureGeneratorBasic):
 
         return ("%s.%s.%s.%s" % (stampbase, taskname, h, extrainfo)).rstrip('.')
 
-    def stampcleanmask(self, stampbase, mcfn, taskname, extrainfo):
-        return self.stampfile(stampbase, mcfn, taskname, extrainfo, clean=True)
+    def stampcleanmask(self, stampbase, fn, taskname, extrainfo):
+        return self.stampfile(stampbase, fn, taskname, extrainfo, clean=True)
 
-    def invalidate_task(self, task, mcfn):
-        bb.note("Tainting hash to force rebuild of task %s, %s" % (mcfn, task))
-
-        mc = bb.runqueue.mc_from_tid(mcfn)
-        stamp = self.datacaches[mc].stamp[mcfn]
-
-        taintfn = stamp + '.' + task + '.taint'
-
-        import uuid
-        bb.utils.mkdirhier(os.path.dirname(taintfn))
-        # The specific content of the taint file is not really important,
-        # we just need it to be random, so a random UUID is used
-        with open(taintfn, 'w') as taintf:
-            taintf.write(str(uuid.uuid4()))
+    def invalidate_task(self, task, d, fn):
+        bb.note("Tainting hash to force rebuild of task %s, %s" % (fn, task))
+        bb.build.write_taint(task, d, fn)
 
 class SignatureGeneratorUniHashMixIn(object):
     def __init__(self, data):
@@ -525,18 +468,6 @@ class SignatureGeneratorUniHashMixIn(object):
             self._client = hashserv.create_client(self.server)
         return self._client
 
-    def reset(self, data):
-        if getattr(self, '_client', None) is not None:
-            self._client.close()
-            self._client = None 
-        return super().reset(data)
-
-    def exit(self):
-        if getattr(self, '_client', None) is not None:
-            self._client.close()
-            self._client = None
-        return super().exit()
-
     def get_stampfile_hash(self, tid):
         if tid in self.taskhash:
             # If a unique hash is reported, use it as the stampfile hash. This
@@ -608,7 +539,7 @@ class SignatureGeneratorUniHashMixIn(object):
                 # A unique hash equal to the taskhash is not very interesting,
                 # so it is reported it at debug level 2. If they differ, that
                 # is much more interesting, so it is reported at debug level 1
-                hashequiv_logger.bbdebug((1, 2)[unihash == taskhash], 'Found unihash %s in place of %s for %s from %s' % (unihash, taskhash, tid, self.server))
+                hashequiv_logger.debug((1, 2)[unihash == taskhash], 'Found unihash %s in place of %s for %s from %s' % (unihash, taskhash, tid, self.server))
             else:
                 hashequiv_logger.debug2('No reported unihash for %s:%s from %s' % (tid, taskhash, self.server))
         except ConnectionError as e:
@@ -625,14 +556,14 @@ class SignatureGeneratorUniHashMixIn(object):
         unihash = d.getVar('BB_UNIHASH')
         report_taskdata = d.getVar('SSTATE_HASHEQUIV_REPORT_TASKDATA') == '1'
         tempdir = d.getVar('T')
-        mcfn = d.getVar('BB_FILENAME')
-        tid = mcfn + ':do_' + task
+        fn = d.getVar('BB_FILENAME')
+        tid = fn + ':do_' + task
         key = tid + ':' + taskhash
 
         if self.setscenetasks and tid not in self.setscenetasks:
             return
 
-        # This can happen if locked sigs are in action. Detect and just exit
+        # This can happen if locked sigs are in action. Detect and just abort
         if taskhash != self.taskhash[tid]:
             return
 
@@ -685,7 +616,7 @@ class SignatureGeneratorUniHashMixIn(object):
 
                 if new_unihash != unihash:
                     hashequiv_logger.debug('Task %s unihash changed %s -> %s by server %s' % (taskhash, unihash, new_unihash, self.server))
-                    bb.event.fire(bb.runqueue.taskUniHashUpdate(mcfn + ':do_' + task, new_unihash), d)
+                    bb.event.fire(bb.runqueue.taskUniHashUpdate(fn + ':do_' + task, new_unihash), d)
                     self.set_unihash(tid, new_unihash)
                     d.setVar('BB_UNIHASH', new_unihash)
                 else:
@@ -745,18 +676,19 @@ class SignatureGeneratorTestEquivHash(SignatureGeneratorUniHashMixIn, SignatureG
         self.server = data.getVar('BB_HASHSERVE')
         self.method = "sstate_output_hash"
 
-def clean_checksum_file_path(file_checksum_tuple):
-    f, cs = file_checksum_tuple
-    if "/./" in f:
-        return "./" + f.split("/./")[1]
-    return f
+#
+# Dummy class used for bitbake-selftest
+#
+class SignatureGeneratorTestMulticonfigDepends(SignatureGeneratorBasicHash):
+    name = "TestMulticonfigDepends"
+    supports_multiconfig_datacaches = True
 
 def dump_this_task(outfile, d):
     import bb.parse
-    mcfn = d.getVar("BB_FILENAME")
+    fn = d.getVar("BB_FILENAME")
     task = "do_" + d.getVar("BB_CURRENTTASK")
-    referencestamp = bb.parse.siggen.stampfile_base(mcfn)
-    bb.parse.siggen.dump_sigtask(mcfn, task, outfile, "customfile:" + referencestamp)
+    referencestamp = bb.build.stamp_internal(task, d, None, True)
+    bb.parse.siggen.dump_sigtask(fn, task, outfile, "customfile:" + referencestamp)
 
 def init_colors(enable_color):
     """Initialise colour dict for passing to compare_sigfiles()"""
@@ -809,15 +741,38 @@ def list_inline_diff(oldlist, newlist, colors=None):
             ret.append(item)
     return '[%s]' % (', '.join(ret))
 
-# Handled renamed fields
-def handle_renames(data):
-    if 'basewhitelist' in data:
-        data['basehash_ignore_vars'] = data['basewhitelist']
-        del data['basewhitelist']
-    if 'taskwhitelist' in data:
-        data['taskhash_ignore_tasks'] = data['taskwhitelist']
-        del data['taskwhitelist']
+def clean_basepath(basepath):
+    basepath, dir, recipe_task = basepath.rsplit("/", 2)
+    cleaned = dir + '/' + recipe_task
 
+    if basepath[0] == '/':
+        return cleaned
+
+    if basepath.startswith("mc:") and basepath.count(':') >= 2:
+        mc, mc_name, basepath = basepath.split(":", 2)
+        mc_suffix = ':mc:' + mc_name
+    else:
+        mc_suffix = ''
+
+    # mc stuff now removed from basepath. Whatever was next, if present will be the first
+    # suffix. ':/', recipe path start, marks the end of this. Something like
+    # 'virtual:a[:b[:c]]:/path...' (b and c being optional)
+    if basepath[0] != '/':
+        cleaned += ':' + basepath.split(':/', 1)[0]
+
+    return cleaned + mc_suffix
+
+def clean_basepaths(a):
+    b = {}
+    for x in a:
+        b[clean_basepath(x)] = a[x]
+    return b
+
+def clean_basepaths_list(a):
+    b = []
+    for x in a:
+        b.append(clean_basepath(x))
+    return b
 
 def compare_sigfiles(a, b, recursecb=None, color=False, collapsed=False):
     output = []
@@ -839,21 +794,20 @@ def compare_sigfiles(a, b, recursecb=None, color=False, collapsed=False):
         formatparams.update(values)
         return formatstr.format(**formatparams)
 
-    with bb.compress.zstd.open(a, "rt", encoding="utf-8", num_threads=1) as f:
-        a_data = json.load(f, object_hook=SetDecoder)
-    with bb.compress.zstd.open(b, "rt", encoding="utf-8", num_threads=1) as f:
-        b_data = json.load(f, object_hook=SetDecoder)
+    with open(a, 'rb') as f:
+        p1 = pickle.Unpickler(f)
+        a_data = p1.load()
+    with open(b, 'rb') as f:
+        p2 = pickle.Unpickler(f)
+        b_data = p2.load()
 
-    for data in [a_data, b_data]:
-        handle_renames(data)
-
-    def dict_diff(a, b, ignored_vars=set()):
+    def dict_diff(a, b, whitelist=set()):
         sa = set(a.keys())
         sb = set(b.keys())
         common = sa & sb
         changed = set()
         for i in common:
-            if a[i] != b[i] and i not in ignored_vars:
+            if a[i] != b[i] and i not in whitelist:
                 changed.add(i)
         added = sb - sa
         removed = sa - sb
@@ -861,11 +815,11 @@ def compare_sigfiles(a, b, recursecb=None, color=False, collapsed=False):
 
     def file_checksums_diff(a, b):
         from collections import Counter
-
-        # Convert lists back to tuples
-        a = [(f[0], f[1]) for f in a]
-        b = [(f[0], f[1]) for f in b]
-
+        # Handle old siginfo format
+        if isinstance(a, dict):
+            a = [(os.path.basename(f), cs) for f, cs in a.items()]
+        if isinstance(b, dict):
+            b = [(os.path.basename(f), cs) for f, cs in b.items()]
         # Compare lists, ensuring we can handle duplicate filenames if they exist
         removedcount = Counter(a)
         removedcount.subtract(b)
@@ -892,15 +846,15 @@ def compare_sigfiles(a, b, recursecb=None, color=False, collapsed=False):
         removed = [x[0] for x in removed]
         return changed, added, removed
 
-    if 'basehash_ignore_vars' in a_data and a_data['basehash_ignore_vars'] != b_data['basehash_ignore_vars']:
-        output.append(color_format("{color_title}basehash_ignore_vars changed{color_default} from '%s' to '%s'") % (a_data['basehash_ignore_vars'], b_data['basehash_ignore_vars']))
-        if a_data['basehash_ignore_vars'] and b_data['basehash_ignore_vars']:
-            output.append("changed items: %s" % a_data['basehash_ignore_vars'].symmetric_difference(b_data['basehash_ignore_vars']))
+    if 'basewhitelist' in a_data and a_data['basewhitelist'] != b_data['basewhitelist']:
+        output.append(color_format("{color_title}basewhitelist changed{color_default} from '%s' to '%s'") % (a_data['basewhitelist'], b_data['basewhitelist']))
+        if a_data['basewhitelist'] and b_data['basewhitelist']:
+            output.append("changed items: %s" % a_data['basewhitelist'].symmetric_difference(b_data['basewhitelist']))
 
-    if 'taskhash_ignore_tasks' in a_data and a_data['taskhash_ignore_tasks'] != b_data['taskhash_ignore_tasks']:
-        output.append(color_format("{color_title}taskhash_ignore_tasks changed{color_default} from '%s' to '%s'") % (a_data['taskhash_ignore_tasks'], b_data['taskhash_ignore_tasks']))
-        if a_data['taskhash_ignore_tasks'] and b_data['taskhash_ignore_tasks']:
-            output.append("changed items: %s" % a_data['taskhash_ignore_tasks'].symmetric_difference(b_data['taskhash_ignore_tasks']))
+    if 'taskwhitelist' in a_data and a_data['taskwhitelist'] != b_data['taskwhitelist']:
+        output.append(color_format("{color_title}taskwhitelist changed{color_default} from '%s' to '%s'") % (a_data['taskwhitelist'], b_data['taskwhitelist']))
+        if a_data['taskwhitelist'] and b_data['taskwhitelist']:
+            output.append("changed items: %s" % a_data['taskwhitelist'].symmetric_difference(b_data['taskwhitelist']))
 
     if a_data['taskdeps'] != b_data['taskdeps']:
         output.append(color_format("{color_title}Task dependencies changed{color_default} from:\n%s\nto:\n%s") % (sorted(a_data['taskdeps']), sorted(b_data['taskdeps'])))
@@ -908,7 +862,7 @@ def compare_sigfiles(a, b, recursecb=None, color=False, collapsed=False):
     if a_data['basehash'] != b_data['basehash'] and not collapsed:
         output.append(color_format("{color_title}basehash changed{color_default} from %s to %s") % (a_data['basehash'], b_data['basehash']))
 
-    changed, added, removed = dict_diff(a_data['gendeps'], b_data['gendeps'], a_data['basehash_ignore_vars'] & b_data['basehash_ignore_vars'])
+    changed, added, removed = dict_diff(a_data['gendeps'], b_data['gendeps'], a_data['basewhitelist'] & b_data['basewhitelist'])
     if changed:
         for dep in sorted(changed):
             output.append(color_format("{color_title}List of dependencies for variable %s changed from '{color_default}%s{color_title}' to '{color_default}%s{color_title}'") % (dep, a_data['gendeps'][dep], b_data['gendeps'][dep]))
@@ -948,9 +902,9 @@ def compare_sigfiles(a, b, recursecb=None, color=False, collapsed=False):
                 output.append(color_format("{color_title}Variable {var} value changed from '{color_default}{oldval}{color_title}' to '{color_default}{newval}{color_title}'{color_default}", var=dep, oldval=oldval, newval=newval))
 
     if not 'file_checksum_values' in a_data:
-         a_data['file_checksum_values'] = []
+         a_data['file_checksum_values'] = {}
     if not 'file_checksum_values' in b_data:
-         b_data['file_checksum_values'] = []
+         b_data['file_checksum_values'] = {}
 
     changed, added, removed = file_checksums_diff(a_data['file_checksum_values'], b_data['file_checksum_values'])
     if changed:
@@ -977,11 +931,11 @@ def compare_sigfiles(a, b, recursecb=None, color=False, collapsed=False):
                 a = a_data['runtaskdeps'][idx]
                 b = b_data['runtaskdeps'][idx]
                 if a_data['runtaskhashes'][a] != b_data['runtaskhashes'][b] and not collapsed:
-                    changed.append("%s with hash %s\n changed to\n%s with hash %s" % (a, a_data['runtaskhashes'][a], b, b_data['runtaskhashes'][b]))
+                    changed.append("%s with hash %s\n changed to\n%s with hash %s" % (clean_basepath(a), a_data['runtaskhashes'][a], clean_basepath(b), b_data['runtaskhashes'][b]))
 
         if changed:
-            clean_a = a_data['runtaskdeps']
-            clean_b = b_data['runtaskdeps']
+            clean_a = clean_basepaths_list(a_data['runtaskdeps'])
+            clean_b = clean_basepaths_list(b_data['runtaskdeps'])
             if clean_a != clean_b:
                 output.append(color_format("{color_title}runtaskdeps changed:{color_default}\n%s") % list_inline_diff(clean_a, clean_b, colors))
             else:
@@ -1002,7 +956,7 @@ def compare_sigfiles(a, b, recursecb=None, color=False, collapsed=False):
                             #output.append("Dependency on task %s was replaced by %s with same hash" % (dep, bdep))
                             bdep_found = True
                 if not bdep_found:
-                    output.append(color_format("{color_title}Dependency on task %s was added{color_default} with hash %s") % (dep, b[dep]))
+                    output.append(color_format("{color_title}Dependency on task %s was added{color_default} with hash %s") % (clean_basepath(dep), b[dep]))
         if removed:
             for dep in sorted(removed):
                 adep_found = False
@@ -1012,11 +966,11 @@ def compare_sigfiles(a, b, recursecb=None, color=False, collapsed=False):
                             #output.append("Dependency on task %s was replaced by %s with same hash" % (adep, dep))
                             adep_found = True
                 if not adep_found:
-                    output.append(color_format("{color_title}Dependency on task %s was removed{color_default} with hash %s") % (dep, a[dep]))
+                    output.append(color_format("{color_title}Dependency on task %s was removed{color_default} with hash %s") % (clean_basepath(dep), a[dep]))
         if changed:
             for dep in sorted(changed):
                 if not collapsed:
-                    output.append(color_format("{color_title}Hash for task dependency %s changed{color_default} from %s to %s") % (dep, a[dep], b[dep]))
+                    output.append(color_format("{color_title}Hash for dependent task %s changed{color_default} from %s to %s") % (clean_basepath(dep), a[dep], b[dep]))
                 if callable(recursecb):
                     recout = recursecb(dep, a[dep], b[dep])
                     if recout:
@@ -1026,7 +980,6 @@ def compare_sigfiles(a, b, recursecb=None, color=False, collapsed=False):
                             # If a dependent hash changed, might as well print the line above and then defer to the changes in
                             # that hash since in all likelyhood, they're the same changes this task also saw.
                             output = [output[-1]] + recout
-                            break
 
     a_taint = a_data.get('taint', None)
     b_taint = b_data.get('taint', None)
@@ -1048,7 +1001,7 @@ def calc_basehash(sigdata):
         basedata = ''
 
     alldeps = sigdata['taskdeps']
-    for dep in sorted(alldeps):
+    for dep in alldeps:
         basedata = basedata + dep
         val = sigdata['varvals'][dep]
         if val is not None:
@@ -1064,8 +1017,6 @@ def calc_taskhash(sigdata):
 
     for c in sigdata['file_checksum_values']:
         if c[1]:
-            if "./" in c[0]:
-                data = data + c[0]
             data = data + c[1]
 
     if 'taint' in sigdata:
@@ -1080,33 +1031,32 @@ def calc_taskhash(sigdata):
 def dump_sigfile(a):
     output = []
 
-    with bb.compress.zstd.open(a, "rt", encoding="utf-8", num_threads=1) as f:
-        a_data = json.load(f, object_hook=SetDecoder)
+    with open(a, 'rb') as f:
+        p1 = pickle.Unpickler(f)
+        a_data = p1.load()
 
-    handle_renames(a_data)
+    output.append("basewhitelist: %s" % (a_data['basewhitelist']))
 
-    output.append("basehash_ignore_vars: %s" % (sorted(a_data['basehash_ignore_vars'])))
-
-    output.append("taskhash_ignore_tasks: %s" % (sorted(a_data['taskhash_ignore_tasks'] or [])))
+    output.append("taskwhitelist: %s" % (a_data['taskwhitelist']))
 
     output.append("Task dependencies: %s" % (sorted(a_data['taskdeps'])))
 
     output.append("basehash: %s" % (a_data['basehash']))
 
-    for dep in sorted(a_data['gendeps']):
-        output.append("List of dependencies for variable %s is %s" % (dep, sorted(a_data['gendeps'][dep])))
+    for dep in a_data['gendeps']:
+        output.append("List of dependencies for variable %s is %s" % (dep, a_data['gendeps'][dep]))
 
-    for dep in sorted(a_data['varvals']):
+    for dep in a_data['varvals']:
         output.append("Variable %s value is %s" % (dep, a_data['varvals'][dep]))
 
     if 'runtaskdeps' in a_data:
-        output.append("Tasks this task depends on: %s" % (sorted(a_data['runtaskdeps'])))
+        output.append("Tasks this task depends on: %s" % (a_data['runtaskdeps']))
 
     if 'file_checksum_values' in a_data:
-        output.append("This task depends on the checksums of files: %s" % (sorted(a_data['file_checksum_values'])))
+        output.append("This task depends on the checksums of files: %s" % (a_data['file_checksum_values']))
 
     if 'runtaskhashes' in a_data:
-        for dep in sorted(a_data['runtaskhashes']):
+        for dep in a_data['runtaskhashes']:
             output.append("Hash for dependent task %s is %s" % (dep, a_data['runtaskhashes'][dep]))
 
     if 'taint' in a_data:

bitbake/lib/bb/taskdata.py

@@ -39,7 +39,7 @@ class TaskData:
     """
     BitBake Task Data implementation
     """
-    def __init__(self, halt = True, skiplist = None, allowincomplete = False):
+    def __init__(self, abort = True, skiplist = None, allowincomplete = False):
         self.build_targets = {}
         self.run_targets = {}
 
@@ -57,7 +57,7 @@ class TaskData:
         self.failed_rdeps = []
         self.failed_fns = []
 
-        self.halt = halt
+        self.abort = abort
         self.allowincomplete = allowincomplete
 
         self.skiplist = skiplist
@@ -328,7 +328,7 @@ class TaskData:
         try:
             self.add_provider_internal(cfgData, dataCache, item)
         except bb.providers.NoProvider:
-            if self.halt:
+            if self.abort:
                 raise
             self.remove_buildtarget(item)
 
@@ -451,12 +451,12 @@ class TaskData:
         for target in self.build_targets:
             if fn in self.build_targets[target]:
                 self.build_targets[target].remove(fn)
-                if not self.build_targets[target]:
+                if len(self.build_targets[target]) == 0:
                     self.remove_buildtarget(target, missing_list)
         for target in self.run_targets:
             if fn in self.run_targets[target]:
                 self.run_targets[target].remove(fn)
-                if not self.run_targets[target]:
+                if len(self.run_targets[target]) == 0:
                     self.remove_runtarget(target, missing_list)
 
     def remove_buildtarget(self, target, missing_list=None):
@@ -479,7 +479,7 @@ class TaskData:
                     fn = tid.rsplit(":",1)[0]
                     self.fail_fn(fn, missing_list)
 
-        if self.halt and target in self.external_targets:
+        if self.abort and target in self.external_targets:
             logger.error("Required build target '%s' has no buildable providers.\nMissing or unbuildable dependency chain was: %s", target, missing_list)
             raise bb.providers.NoProvider(target)
 
@@ -516,7 +516,7 @@ class TaskData:
                     self.add_provider_internal(cfgData, dataCache, target)
                     added = added + 1
                 except bb.providers.NoProvider:
-                    if self.halt and target in self.external_targets and not self.allowincomplete:
+                    if self.abort and target in self.external_targets and not self.allowincomplete:
                         raise
                     if not self.allowincomplete:
                         self.remove_buildtarget(target)

bitbake/lib/bb/tests/codeparser.py

@@ -44,7 +44,6 @@ class VariableReferenceTest(ReferenceTest):
     def parseExpression(self, exp):
         parsedvar = self.d.expandWithRefs(exp, None)
         self.references = parsedvar.references
-        self.execs = parsedvar.execs
 
     def test_simple_reference(self):
         self.setEmptyVars(["FOO"])
@@ -62,11 +61,6 @@ class VariableReferenceTest(ReferenceTest):
         self.parseExpression("${@d.getVar('BAR') + 'foo'}")
         self.assertReferences(set(["BAR"]))
 
-    def test_python_exec_reference(self):
-        self.parseExpression("${@eval('3 * 5')}")
-        self.assertReferences(set())
-        self.assertExecs(set(["eval"]))
-
 class ShellReferenceTest(ReferenceTest):
 
     def parseExpression(self, exp):
@@ -324,7 +318,7 @@ d.getVar(a(), False)
             "filename": "example.bb",
         })
 
-        deps, values = bb.data.build_dependencies("FOO", set(self.d.keys()), set(), set(), set(), set(), self.d, self.d)
+        deps, values = bb.data.build_dependencies("FOO", set(self.d.keys()), set(), set(), self.d)
 
         self.assertEqual(deps, set(["somevar", "bar", "something", "inexpand", "test", "test2", "a"]))
 
@@ -371,7 +365,7 @@ esac
         self.d.setVarFlags("FOO", {"func": True})
         self.setEmptyVars(execs)
 
-        deps, values = bb.data.build_dependencies("FOO", set(self.d.keys()), set(), set(), set(), set(), self.d, self.d)
+        deps, values = bb.data.build_dependencies("FOO", set(self.d.keys()), set(), set(), self.d)
 
         self.assertEqual(deps, set(["somevar", "inverted"] + execs))
 
@@ -381,7 +375,7 @@ esac
         self.d.setVar("FOO", "foo=oe_libinstall; eval $foo")
         self.d.setVarFlag("FOO", "vardeps", "oe_libinstall")
 
-        deps, values = bb.data.build_dependencies("FOO", set(self.d.keys()), set(), set(), set(), set(), self.d, self.d)
+        deps, values = bb.data.build_dependencies("FOO", set(self.d.keys()), set(), set(), self.d)
 
         self.assertEqual(deps, set(["oe_libinstall"]))
 
@@ -390,7 +384,7 @@ esac
         self.d.setVar("FOO", "foo=oe_libinstall; eval $foo")
         self.d.setVarFlag("FOO", "vardeps", "${@'oe_libinstall'}")
 
-        deps, values = bb.data.build_dependencies("FOO", set(self.d.keys()), set(), set(), set(), set(), self.d, self.d)
+        deps, values = bb.data.build_dependencies("FOO", set(self.d.keys()), set(), set(), self.d)
 
         self.assertEqual(deps, set(["oe_libinstall"]))
 
@@ -405,7 +399,7 @@ esac
         # Check dependencies
         self.d.setVar('ANOTHERVAR', expr)
         self.d.setVar('TESTVAR', 'anothervalue testval testval2')
-        deps, values = bb.data.build_dependencies("ANOTHERVAR", set(self.d.keys()), set(), set(), set(), set(), self.d, self.d)
+        deps, values = bb.data.build_dependencies("ANOTHERVAR", set(self.d.keys()), set(), set(), self.d)
         self.assertEqual(sorted(values.splitlines()),
                          sorted([expr,
                           'TESTVAR{anothervalue} = Set',
@@ -418,24 +412,6 @@ esac
         # Check final value
         self.assertEqual(self.d.getVar('ANOTHERVAR').split(), ['anothervalue', 'yetanothervalue', 'lastone'])
 
-    def test_contains_vardeps_excluded(self):
-        # Check the ignored_vars option to build_dependencies is handled by contains functionality
-        varval = '${TESTVAR2} ${@bb.utils.filter("TESTVAR", "somevalue anothervalue", d)}'
-        self.d.setVar('ANOTHERVAR', varval)
-        self.d.setVar('TESTVAR', 'anothervalue testval testval2')
-        self.d.setVar('TESTVAR2', 'testval3')
-        deps, values = bb.data.build_dependencies("ANOTHERVAR", set(self.d.keys()), set(), set(), set(), set(["TESTVAR"]), self.d, self.d)
-        self.assertEqual(sorted(values.splitlines()), sorted([varval]))
-        self.assertEqual(deps, set(["TESTVAR2"]))
-        self.assertEqual(self.d.getVar('ANOTHERVAR').split(), ['testval3', 'anothervalue'])
-
-        # Check the vardepsexclude flag is handled by contains functionality
-        self.d.setVarFlag('ANOTHERVAR', 'vardepsexclude', 'TESTVAR')
-        deps, values = bb.data.build_dependencies("ANOTHERVAR", set(self.d.keys()), set(), set(), set(), set(), self.d, self.d)
-        self.assertEqual(sorted(values.splitlines()), sorted([varval]))
-        self.assertEqual(deps, set(["TESTVAR2"]))
-        self.assertEqual(self.d.getVar('ANOTHERVAR').split(), ['testval3', 'anothervalue'])
-
     #Currently no wildcard support
     #def test_vardeps_wildcards(self):
     #    self.d.setVar("oe_libinstall", "echo test")

bitbake/lib/bb/tests/color.py

@@ -20,7 +20,7 @@ class ProgressWatcher:
     def __init__(self):
         self._reports = []
 
-    def handle_event(self, event, d):
+    def handle_event(self, event):
         self._reports.append((event.progress, event.rate))
 
     def reports(self):

bitbake/lib/bb/tests/compression.py

@@ -1,6 +1,4 @@
 #
-# Copyright BitBake Contributors
-#
 # SPDX-License-Identifier: GPL-2.0-only
 #
 

bitbake/lib/bb/tests/cooker.py

@@ -1,8 +1,6 @@
 #
 # BitBake Tests for cooker.py
 #
-# Copyright BitBake Contributors
-#
 # SPDX-License-Identifier: GPL-2.0-only
 #
 

bitbake/lib/bb/tests/data.py

@@ -60,15 +60,6 @@ class DataExpansions(unittest.TestCase):
         val = self.d.expand("${@5*12}")
         self.assertEqual(str(val), "60")
 
-    def test_python_snippet_w_dict(self):
-        val = self.d.expand("${@{ 'green': 1, 'blue': 2 }['green']}")
-        self.assertEqual(str(val), "1")
-
-    def test_python_unexpanded_multi(self):
-        self.d.setVar("bar", "${unsetvar}")
-        val = self.d.expand("${@2*2},${foo},${@d.getVar('foo') + ' ${bar}'},${foo}")
-        self.assertEqual(str(val), "4,value_of_foo,${@d.getVar('foo') + ' ${unsetvar}'},value_of_foo")
-
     def test_expand_in_python_snippet(self):
         val = self.d.expand("${@'boo ' + '${foo}'}")
         self.assertEqual(str(val), "boo value_of_foo")
@@ -77,18 +68,6 @@ class DataExpansions(unittest.TestCase):
         val = self.d.expand("${@d.getVar('foo') + ' ${bar}'}")
         self.assertEqual(str(val), "value_of_foo value_of_bar")
 
-    def test_python_snippet_function_reference(self):
-        self.d.setVar("TESTVAL", "testvalue")
-        self.d.setVar("testfunc", 'd.getVar("TESTVAL")')
-        context = bb.utils.get_context()
-        context["testfunc"] = lambda d: d.getVar("TESTVAL")
-        val = self.d.expand("${@testfunc(d)}")
-        self.assertEqual(str(val), "testvalue")
-
-    def test_python_snippet_builtin_metadata(self):
-        self.d.setVar("eval", "INVALID")
-        self.d.expand("${@eval('3')}")
-
     def test_python_unexpanded(self):
         self.d.setVar("bar", "${unsetvar}")
         val = self.d.expand("${@d.getVar('foo') + ' ${bar}'}")

bitbake/lib/bb/tests/event.py

@@ -157,7 +157,7 @@ class EventHandlingTest(unittest.TestCase):
                                  self._test_process.event_handler,
                                  event,
                                  None)
-        self._test_process.event_handler.assert_called_once_with(event, None)
+        self._test_process.event_handler.assert_called_once_with(event)
 
     def test_fire_class_handlers(self):
         """ Test fire_class_handlers method """
@@ -175,10 +175,10 @@ class EventHandlingTest(unittest.TestCase):
         bb.event.fire_class_handlers(event1, None)
         bb.event.fire_class_handlers(event2, None)
         bb.event.fire_class_handlers(event2, None)
-        expected_event_handler1 = [call(event1, None)]
-        expected_event_handler2 = [call(event1, None),
-                                   call(event2, None),
-                                   call(event2, None)]
+        expected_event_handler1 = [call(event1)]
+        expected_event_handler2 = [call(event1),
+                                   call(event2),
+                                   call(event2)]
         self.assertEqual(self._test_process.event_handler1.call_args_list,
                          expected_event_handler1)
         self.assertEqual(self._test_process.event_handler2.call_args_list,
@@ -205,7 +205,7 @@ class EventHandlingTest(unittest.TestCase):
         bb.event.fire_class_handlers(event2, None)
         bb.event.fire_class_handlers(event2, None)
         expected_event_handler1 = []
-        expected_event_handler2 = [call(event1, None)]
+        expected_event_handler2 = [call(event1)]
         self.assertEqual(self._test_process.event_handler1.call_args_list,
                          expected_event_handler1)
         self.assertEqual(self._test_process.event_handler2.call_args_list,
@@ -223,7 +223,7 @@ class EventHandlingTest(unittest.TestCase):
         self.assertEqual(result, bb.event.Registered)
         bb.event.fire_class_handlers(event1, None)
         bb.event.fire_class_handlers(event2, None)
-        expected = [call(event1, None), call(event2, None)]
+        expected = [call(event1), call(event2)]
         self.assertEqual(self._test_process.event_handler1.call_args_list,
                          expected)
 
@@ -237,7 +237,7 @@ class EventHandlingTest(unittest.TestCase):
         self.assertEqual(result, bb.event.Registered)
         bb.event.fire_class_handlers(event1, None)
         bb.event.fire_class_handlers(event2, None)
-        expected = [call(event1, None), call(event2, None), call(event1, None)]
+        expected = [call(event1), call(event2), call(event1)]
         self.assertEqual(self._test_process.event_handler1.call_args_list,
                          expected)
 
@@ -251,7 +251,7 @@ class EventHandlingTest(unittest.TestCase):
         self.assertEqual(result, bb.event.Registered)
         bb.event.fire_class_handlers(event1, None)
         bb.event.fire_class_handlers(event2, None)
-        expected = [call(event1,None), call(event2, None), call(event1, None), call(event2, None)]
+        expected = [call(event1), call(event2), call(event1), call(event2)]
         self.assertEqual(self._test_process.event_handler1.call_args_list,
                          expected)
 
@@ -359,10 +359,9 @@ class EventHandlingTest(unittest.TestCase):
 
         event1 = bb.event.ConfigParsed()
         bb.event.fire(event1, None)
-        expected = [call(event1, None)]
+        expected = [call(event1)]
         self.assertEqual(self._test_process.event_handler1.call_args_list,
                          expected)
-        expected = [call(event1)]
         self.assertEqual(self._test_ui1.event.send.call_args_list,
                          expected)
 
@@ -451,9 +450,10 @@ class EventHandlingTest(unittest.TestCase):
             and disable threadlocks tests """
         bb.event.fire(bb.event.OperationStarted(), None)
 
-    def test_event_threadlock(self):
+    def test_enable_threadlock(self):
         """ Test enable_threadlock method """
         self._set_threadlock_test_mockups()
+        bb.event.enable_threadlock()
         self._set_and_run_threadlock_test_workers()
         # Calls to UI handlers should be in order as all the registered
         # handlers for the event coming from the first worker should be
@@ -461,6 +461,20 @@ class EventHandlingTest(unittest.TestCase):
         self.assertEqual(self._threadlock_test_calls,
                          ["w1_ui1", "w1_ui2", "w2_ui1", "w2_ui2"])
 
+
+    def test_disable_threadlock(self):
+        """ Test disable_threadlock method """
+        self._set_threadlock_test_mockups()
+        bb.event.disable_threadlock()
+        self._set_and_run_threadlock_test_workers()
+        # Calls to UI handlers should be intertwined together. Thanks to the
+        # delay in the registered handlers for the event coming from the first
+        # worker, the event coming from the second worker starts being
+        # processed before finishing handling the first worker event.
+        self.assertEqual(self._threadlock_test_calls,
+                         ["w1_ui1", "w2_ui1", "w1_ui2", "w2_ui2"])
+
+
 class EventClassesTest(unittest.TestCase):
     """ Event classes test class """
 

bitbake/lib/bb/tests/fetch-testdata/debian/pool/main/m/minicom/index.html

@@ -1,59 +0,0 @@
-<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 3.2 Final//EN">
-<html>
- <head>
-  <title>Index of /debian/pool/main/m/minicom</title>
- </head>
- <body>
-<h1>Index of /debian/pool/main/m/minicom</h1>
-  <table>
-   <tr><th valign="top"><img src="/icons/blank.gif" alt="[ICO]"></th><th><a href="?C=N;O=D">Name</a></th><th><a href="?C=M;O=A">Last modified</a></th><th><a href="?C=S;O=A">Size</a></th></tr>
-   <tr><th colspan="4"><hr></th></tr>
-<tr><td valign="top"><img src="/icons/back.gif" alt="[PARENTDIR]"></td><td><a href="/debian/pool/main/m/">Parent Directory</a></td><td>&nbsp;</td><td align="right">  - </td></tr>
-<tr><td valign="top"><img src="/icons/unknown.gif" alt="[   ]"></td><td><a href="minicom_2.7-1+deb8u1.debian.tar.xz">minicom_2.7-1+deb8u1.debian.tar.xz</a></td><td align="right">2017-04-24 08:22  </td><td align="right"> 14K</td></tr>
-<tr><td valign="top"><img src="/icons/unknown.gif" alt="[   ]"></td><td><a href="minicom_2.7-1+deb8u1.dsc">minicom_2.7-1+deb8u1.dsc</a></td><td align="right">2017-04-24 08:22  </td><td align="right">1.9K</td></tr>
-<tr><td valign="top"><img src="/icons/unknown.gif" alt="[   ]"></td><td><a href="minicom_2.7-1+deb8u1_amd64.deb">minicom_2.7-1+deb8u1_amd64.deb</a></td><td align="right">2017-04-25 21:10  </td><td align="right">257K</td></tr>
-<tr><td valign="top"><img src="/icons/unknown.gif" alt="[   ]"></td><td><a href="minicom_2.7-1+deb8u1_armel.deb">minicom_2.7-1+deb8u1_armel.deb</a></td><td align="right">2017-04-26 00:58  </td><td align="right">246K</td></tr>
-<tr><td valign="top"><img src="/icons/unknown.gif" alt="[   ]"></td><td><a href="minicom_2.7-1+deb8u1_armhf.deb">minicom_2.7-1+deb8u1_armhf.deb</a></td><td align="right">2017-04-26 00:58  </td><td align="right">245K</td></tr>
-<tr><td valign="top"><img src="/icons/unknown.gif" alt="[   ]"></td><td><a href="minicom_2.7-1+deb8u1_i386.deb">minicom_2.7-1+deb8u1_i386.deb</a></td><td align="right">2017-04-25 21:41  </td><td align="right">258K</td></tr>
-<tr><td valign="top"><img src="/icons/unknown.gif" alt="[   ]"></td><td><a href="minicom_2.7-1.1.debian.tar.xz">minicom_2.7-1.1.debian.tar.xz</a></td><td align="right">2017-04-22 09:34  </td><td align="right"> 14K</td></tr>
-<tr><td valign="top"><img src="/icons/unknown.gif" alt="[   ]"></td><td><a href="minicom_2.7-1.1.dsc">minicom_2.7-1.1.dsc</a></td><td align="right">2017-04-22 09:34  </td><td align="right">1.9K</td></tr>
-<tr><td valign="top"><img src="/icons/unknown.gif" alt="[   ]"></td><td><a href="minicom_2.7-1.1_amd64.deb">minicom_2.7-1.1_amd64.deb</a></td><td align="right">2017-04-22 15:29  </td><td align="right">261K</td></tr>
-<tr><td valign="top"><img src="/icons/unknown.gif" alt="[   ]"></td><td><a href="minicom_2.7-1.1_arm64.deb">minicom_2.7-1.1_arm64.deb</a></td><td align="right">2017-04-22 15:29  </td><td align="right">250K</td></tr>
-<tr><td valign="top"><img src="/icons/unknown.gif" alt="[   ]"></td><td><a href="minicom_2.7-1.1_armel.deb">minicom_2.7-1.1_armel.deb</a></td><td align="right">2017-04-22 15:29  </td><td align="right">255K</td></tr>
-<tr><td valign="top"><img src="/icons/unknown.gif" alt="[   ]"></td><td><a href="minicom_2.7-1.1_armhf.deb">minicom_2.7-1.1_armhf.deb</a></td><td align="right">2017-04-22 15:29  </td><td align="right">254K</td></tr>
-<tr><td valign="top"><img src="/icons/unknown.gif" alt="[   ]"></td><td><a href="minicom_2.7-1.1_i386.deb">minicom_2.7-1.1_i386.deb</a></td><td align="right">2017-04-22 15:29  </td><td align="right">266K</td></tr>
-<tr><td valign="top"><img src="/icons/unknown.gif" alt="[   ]"></td><td><a href="minicom_2.7-1.1_mips.deb">minicom_2.7-1.1_mips.deb</a></td><td align="right">2017-04-22 15:29  </td><td align="right">258K</td></tr>
-<tr><td valign="top"><img src="/icons/unknown.gif" alt="[   ]"></td><td><a href="minicom_2.7-1.1_mips64el.deb">minicom_2.7-1.1_mips64el.deb</a></td><td align="right">2017-04-22 15:29  </td><td align="right">259K</td></tr>
-<tr><td valign="top"><img src="/icons/unknown.gif" alt="[   ]"></td><td><a href="minicom_2.7-1.1_mipsel.deb">minicom_2.7-1.1_mipsel.deb</a></td><td align="right">2017-04-22 15:29  </td><td align="right">259K</td></tr>
-<tr><td valign="top"><img src="/icons/unknown.gif" alt="[   ]"></td><td><a href="minicom_2.7-1.1_ppc64el.deb">minicom_2.7-1.1_ppc64el.deb</a></td><td align="right">2017-04-22 15:29  </td><td align="right">253K</td></tr>
-<tr><td valign="top"><img src="/icons/unknown.gif" alt="[   ]"></td><td><a href="minicom_2.7-1.1_s390x.deb">minicom_2.7-1.1_s390x.deb</a></td><td align="right">2017-04-22 15:29  </td><td align="right">261K</td></tr>
-<tr><td valign="top"><img src="/icons/unknown.gif" alt="[   ]"></td><td><a href="minicom_2.7.1-1+b1_amd64.deb">minicom_2.7.1-1+b1_amd64.deb</a></td><td align="right">2018-05-06 08:14  </td><td align="right">262K</td></tr>
-<tr><td valign="top"><img src="/icons/unknown.gif" alt="[   ]"></td><td><a href="minicom_2.7.1-1+b1_arm64.deb">minicom_2.7.1-1+b1_arm64.deb</a></td><td align="right">2018-05-06 07:58  </td><td align="right">250K</td></tr>
-<tr><td valign="top"><img src="/icons/unknown.gif" alt="[   ]"></td><td><a href="minicom_2.7.1-1+b1_armel.deb">minicom_2.7.1-1+b1_armel.deb</a></td><td align="right">2018-05-06 08:45  </td><td align="right">253K</td></tr>
-<tr><td valign="top"><img src="/icons/unknown.gif" alt="[   ]"></td><td><a href="minicom_2.7.1-1+b1_armhf.deb">minicom_2.7.1-1+b1_armhf.deb</a></td><td align="right">2018-05-06 10:42  </td><td align="right">253K</td></tr>
-<tr><td valign="top"><img src="/icons/unknown.gif" alt="[   ]"></td><td><a href="minicom_2.7.1-1+b1_i386.deb">minicom_2.7.1-1+b1_i386.deb</a></td><td align="right">2018-05-06 08:55  </td><td align="right">266K</td></tr>
-<tr><td valign="top"><img src="/icons/unknown.gif" alt="[   ]"></td><td><a href="minicom_2.7.1-1+b1_mips.deb">minicom_2.7.1-1+b1_mips.deb</a></td><td align="right">2018-05-06 08:14  </td><td align="right">258K</td></tr>
-<tr><td valign="top"><img src="/icons/unknown.gif" alt="[   ]"></td><td><a href="minicom_2.7.1-1+b1_mipsel.deb">minicom_2.7.1-1+b1_mipsel.deb</a></td><td align="right">2018-05-06 12:13  </td><td align="right">259K</td></tr>
-<tr><td valign="top"><img src="/icons/unknown.gif" alt="[   ]"></td><td><a href="minicom_2.7.1-1+b1_ppc64el.deb">minicom_2.7.1-1+b1_ppc64el.deb</a></td><td align="right">2018-05-06 09:10  </td><td align="right">260K</td></tr>
-<tr><td valign="top"><img src="/icons/unknown.gif" alt="[   ]"></td><td><a href="minicom_2.7.1-1+b1_s390x.deb">minicom_2.7.1-1+b1_s390x.deb</a></td><td align="right">2018-05-06 08:14  </td><td align="right">257K</td></tr>
-<tr><td valign="top"><img src="/icons/unknown.gif" alt="[   ]"></td><td><a href="minicom_2.7.1-1+b2_mips64el.deb">minicom_2.7.1-1+b2_mips64el.deb</a></td><td align="right">2018-05-06 09:41  </td><td align="right">260K</td></tr>
-<tr><td valign="top"><img src="/icons/unknown.gif" alt="[   ]"></td><td><a href="minicom_2.7.1-1.debian.tar.xz">minicom_2.7.1-1.debian.tar.xz</a></td><td align="right">2017-08-13 15:40  </td><td align="right"> 14K</td></tr>
-<tr><td valign="top"><img src="/icons/unknown.gif" alt="[   ]"></td><td><a href="minicom_2.7.1-1.dsc">minicom_2.7.1-1.dsc</a></td><td align="right">2017-08-13 15:40  </td><td align="right">1.8K</td></tr>
-<tr><td valign="top"><img src="/icons/compressed.gif" alt="[   ]"></td><td><a href="minicom_2.7.1.orig.tar.gz">minicom_2.7.1.orig.tar.gz</a></td><td align="right">2017-08-13 15:40  </td><td align="right">855K</td></tr>
-<tr><td valign="top"><img src="/icons/compressed.gif" alt="[   ]"></td><td><a href="minicom_2.7.orig.tar.gz">minicom_2.7.orig.tar.gz</a></td><td align="right">2014-01-01 09:36  </td><td align="right">843K</td></tr>
-<tr><td valign="top"><img src="/icons/unknown.gif" alt="[   ]"></td><td><a href="minicom_2.8-2.debian.tar.xz">minicom_2.8-2.debian.tar.xz</a></td><td align="right">2021-06-15 03:47  </td><td align="right"> 14K</td></tr>
-<tr><td valign="top"><img src="/icons/unknown.gif" alt="[   ]"></td><td><a href="minicom_2.8-2.dsc">minicom_2.8-2.dsc</a></td><td align="right">2021-06-15 03:47  </td><td align="right">1.8K</td></tr>
-<tr><td valign="top"><img src="/icons/unknown.gif" alt="[   ]"></td><td><a href="minicom_2.8-2_amd64.deb">minicom_2.8-2_amd64.deb</a></td><td align="right">2021-06-15 03:58  </td><td align="right">280K</td></tr>
-<tr><td valign="top"><img src="/icons/unknown.gif" alt="[   ]"></td><td><a href="minicom_2.8-2_arm64.deb">minicom_2.8-2_arm64.deb</a></td><td align="right">2021-06-15 04:13  </td><td align="right">275K</td></tr>
-<tr><td valign="top"><img src="/icons/unknown.gif" alt="[   ]"></td><td><a href="minicom_2.8-2_armel.deb">minicom_2.8-2_armel.deb</a></td><td align="right">2021-06-15 04:13  </td><td align="right">271K</td></tr>
-<tr><td valign="top"><img src="/icons/unknown.gif" alt="[   ]"></td><td><a href="minicom_2.8-2_armhf.deb">minicom_2.8-2_armhf.deb</a></td><td align="right">2021-06-15 04:13  </td><td align="right">272K</td></tr>
-<tr><td valign="top"><img src="/icons/unknown.gif" alt="[   ]"></td><td><a href="minicom_2.8-2_i386.deb">minicom_2.8-2_i386.deb</a></td><td align="right">2021-06-15 04:13  </td><td align="right">285K</td></tr>
-<tr><td valign="top"><img src="/icons/unknown.gif" alt="[   ]"></td><td><a href="minicom_2.8-2_mips64el.deb">minicom_2.8-2_mips64el.deb</a></td><td align="right">2021-06-15 04:13  </td><td align="right">277K</td></tr>
-<tr><td valign="top"><img src="/icons/unknown.gif" alt="[   ]"></td><td><a href="minicom_2.8-2_mipsel.deb">minicom_2.8-2_mipsel.deb</a></td><td align="right">2021-06-15 04:13  </td><td align="right">278K</td></tr>
-<tr><td valign="top"><img src="/icons/unknown.gif" alt="[   ]"></td><td><a href="minicom_2.8-2_ppc64el.deb">minicom_2.8-2_ppc64el.deb</a></td><td align="right">2021-06-15 04:13  </td><td align="right">286K</td></tr>
-<tr><td valign="top"><img src="/icons/unknown.gif" alt="[   ]"></td><td><a href="minicom_2.8-2_s390x.deb">minicom_2.8-2_s390x.deb</a></td><td align="right">2021-06-15 03:58  </td><td align="right">275K</td></tr>
-<tr><td valign="top"><img src="/icons/unknown.gif" alt="[   ]"></td><td><a href="minicom_2.8.orig.tar.bz2">minicom_2.8.orig.tar.bz2</a></td><td align="right">2021-01-03 12:44  </td><td align="right">598K</td></tr>
-   <tr><th colspan="4"><hr></th></tr>
-</table>
-<address>Apache Server at ftp.debian.org Port 80</address>
-</body></html>

bitbake/lib/bb/tests/fetch-testdata/software/libxml2/2.10/index.html

@@ -1,20 +0,0 @@
-<!DOCTYPE html><html><head><meta http-equiv="content-type" content="text/html; charset=utf-8"><meta name="viewport" content="width=device-width"><style type="text/css">body,html {background:#fff;font-family:"Bitstream Vera Sans","Lucida Grande","Lucida Sans Unicode",Lucidux,Verdana,Lucida,sans-serif;}tr:nth-child(even) {background:#f4f4f4;}th,td {padding:0.1em 0.5em;}th {text-align:left;font-weight:bold;background:#eee;border-bottom:1px solid #aaa;}#list {border:1px solid #aaa;width:100%;}a {color:#a33;}a:hover {color:#e33;}</style>
-
-<title>Index of /sources/libxml2/2.10/</title>
-</head><body><h1>Index of /sources/libxml2/2.10/</h1>
-<table id="list"><thead><tr><th style="width:55%"><a href="?C=N&amp;O=A">File Name</a>&nbsp;<a href="?C=N&amp;O=D">&nbsp;&darr;&nbsp;</a></th><th style="width:20%"><a href="?C=S&amp;O=A">File Size</a>&nbsp;<a href="?C=S&amp;O=D">&nbsp;&darr;&nbsp;</a></th><th style="width:25%"><a href="?C=M&amp;O=A">Date</a>&nbsp;<a href="?C=M&amp;O=D">&nbsp;&darr;&nbsp;</a></th></tr></thead>
-<tbody><tr><td class="link"><a href="../">Parent directory/</a></td><td class="size">-</td><td class="date">-</td></tr>
-<tr><td class="link"><a href="LATEST-IS-2.10.3" title="LATEST-IS-2.10.3">LATEST-IS-2.10.3</a></td><td class="size">2.5 MiB</td><td class="date">2022-Oct-14 12:55</td></tr>
-<tr><td class="link"><a href="libxml2-2.10.0.news" title="libxml2-2.10.0.news">libxml2-2.10.0.news</a></td><td class="size">7.1 KiB</td><td class="date">2022-Aug-17 11:55</td></tr>
-<tr><td class="link"><a href="libxml2-2.10.0.sha256sum" title="libxml2-2.10.0.sha256sum">libxml2-2.10.0.sha256sum</a></td><td class="size">174 B</td><td class="date">2022-Aug-17 11:55</td></tr>
-<tr><td class="link"><a href="libxml2-2.10.0.tar.xz" title="libxml2-2.10.0.tar.xz">libxml2-2.10.0.tar.xz</a></td><td class="size">2.6 MiB</td><td class="date">2022-Aug-17 11:55</td></tr>
-<tr><td class="link"><a href="libxml2-2.10.1.news" title="libxml2-2.10.1.news">libxml2-2.10.1.news</a></td><td class="size">455 B</td><td class="date">2022-Aug-25 11:33</td></tr>
-<tr><td class="link"><a href="libxml2-2.10.1.sha256sum" title="libxml2-2.10.1.sha256sum">libxml2-2.10.1.sha256sum</a></td><td class="size">174 B</td><td class="date">2022-Aug-25 11:33</td></tr>
-<tr><td class="link"><a href="libxml2-2.10.1.tar.xz" title="libxml2-2.10.1.tar.xz">libxml2-2.10.1.tar.xz</a></td><td class="size">2.6 MiB</td><td class="date">2022-Aug-25 11:33</td></tr>
-<tr><td class="link"><a href="libxml2-2.10.2.news" title="libxml2-2.10.2.news">libxml2-2.10.2.news</a></td><td class="size">309 B</td><td class="date">2022-Aug-29 14:56</td></tr>
-<tr><td class="link"><a href="libxml2-2.10.2.sha256sum" title="libxml2-2.10.2.sha256sum">libxml2-2.10.2.sha256sum</a></td><td class="size">174 B</td><td class="date">2022-Aug-29 14:56</td></tr>
-<tr><td class="link"><a href="libxml2-2.10.2.tar.xz" title="libxml2-2.10.2.tar.xz">libxml2-2.10.2.tar.xz</a></td><td class="size">2.5 MiB</td><td class="date">2022-Aug-29 14:56</td></tr>
-<tr><td class="link"><a href="libxml2-2.10.3.news" title="libxml2-2.10.3.news">libxml2-2.10.3.news</a></td><td class="size">294 B</td><td class="date">2022-Oct-14 12:55</td></tr>
-<tr><td class="link"><a href="libxml2-2.10.3.sha256sum" title="libxml2-2.10.3.sha256sum">libxml2-2.10.3.sha256sum</a></td><td class="size">174 B</td><td class="date">2022-Oct-14 12:55</td></tr>
-<tr><td class="link"><a href="libxml2-2.10.3.tar.xz" title="libxml2-2.10.3.tar.xz">libxml2-2.10.3.tar.xz</a></td><td class="size">2.5 MiB</td><td class="date">2022-Oct-14 12:55</td></tr>
-</tbody></table></body></html>

bitbake/lib/bb/tests/fetch-testdata/software/libxml2/2.9/index.html

@@ -1,40 +0,0 @@
-<!DOCTYPE html><html><head><meta http-equiv="content-type" content="text/html; charset=utf-8"><meta name="viewport" content="width=device-width"><style type="text/css">body,html {background:#fff;font-family:"Bitstream Vera Sans","Lucida Grande","Lucida Sans Unicode",Lucidux,Verdana,Lucida,sans-serif;}tr:nth-child(even) {background:#f4f4f4;}th,td {padding:0.1em 0.5em;}th {text-align:left;font-weight:bold;background:#eee;border-bottom:1px solid #aaa;}#list {border:1px solid #aaa;width:100%;}a {color:#a33;}a:hover {color:#e33;}</style>
-
-<title>Index of /sources/libxml2/2.9/</title>
-</head><body><h1>Index of /sources/libxml2/2.9/</h1>
-<table id="list"><thead><tr><th style="width:55%"><a href="?C=N&amp;O=A">File Name</a>&nbsp;<a href="?C=N&amp;O=D">&nbsp;&darr;&nbsp;</a></th><th style="width:20%"><a href="?C=S&amp;O=A">File Size</a>&nbsp;<a href="?C=S&amp;O=D">&nbsp;&darr;&nbsp;</a></th><th style="width:25%"><a href="?C=M&amp;O=A">Date</a>&nbsp;<a href="?C=M&amp;O=D">&nbsp;&darr;&nbsp;</a></th></tr></thead>
-<tbody><tr><td class="link"><a href="../">Parent directory/</a></td><td class="size">-</td><td class="date">-</td></tr>
-<tr><td class="link"><a href="LATEST-IS-2.9.14" title="LATEST-IS-2.9.14">LATEST-IS-2.9.14</a></td><td class="size">3.0 MiB</td><td class="date">2022-May-02 12:03</td></tr>
-<tr><td class="link"><a href="libxml2-2.9.0.sha256sum" title="libxml2-2.9.0.sha256sum">libxml2-2.9.0.sha256sum</a></td><td class="size">87 B</td><td class="date">2022-Feb-14 18:27</td></tr>
-<tr><td class="link"><a href="libxml2-2.9.0.tar.xz" title="libxml2-2.9.0.tar.xz">libxml2-2.9.0.tar.xz</a></td><td class="size">3.0 MiB</td><td class="date">2022-Feb-14 18:27</td></tr>
-<tr><td class="link"><a href="libxml2-2.9.1.sha256sum" title="libxml2-2.9.1.sha256sum">libxml2-2.9.1.sha256sum</a></td><td class="size">87 B</td><td class="date">2022-Feb-14 18:28</td></tr>
-<tr><td class="link"><a href="libxml2-2.9.1.tar.xz" title="libxml2-2.9.1.tar.xz">libxml2-2.9.1.tar.xz</a></td><td class="size">3.0 MiB</td><td class="date">2022-Feb-14 18:28</td></tr>
-<tr><td class="link"><a href="libxml2-2.9.10.sha256sum" title="libxml2-2.9.10.sha256sum">libxml2-2.9.10.sha256sum</a></td><td class="size">88 B</td><td class="date">2022-Feb-14 18:42</td></tr>
-<tr><td class="link"><a href="libxml2-2.9.10.tar.xz" title="libxml2-2.9.10.tar.xz">libxml2-2.9.10.tar.xz</a></td><td class="size">3.2 MiB</td><td class="date">2022-Feb-14 18:42</td></tr>
-<tr><td class="link"><a href="libxml2-2.9.11.sha256sum" title="libxml2-2.9.11.sha256sum">libxml2-2.9.11.sha256sum</a></td><td class="size">88 B</td><td class="date">2022-Feb-14 18:43</td></tr>
-<tr><td class="link"><a href="libxml2-2.9.11.tar.xz" title="libxml2-2.9.11.tar.xz">libxml2-2.9.11.tar.xz</a></td><td class="size">3.2 MiB</td><td class="date">2022-Feb-14 18:43</td></tr>
-<tr><td class="link"><a href="libxml2-2.9.12.sha256sum" title="libxml2-2.9.12.sha256sum">libxml2-2.9.12.sha256sum</a></td><td class="size">88 B</td><td class="date">2022-Feb-14 18:45</td></tr>
-<tr><td class="link"><a href="libxml2-2.9.12.tar.xz" title="libxml2-2.9.12.tar.xz">libxml2-2.9.12.tar.xz</a></td><td class="size">3.2 MiB</td><td class="date">2022-Feb-14 18:45</td></tr>
-<tr><td class="link"><a href="libxml2-2.9.13.news" title="libxml2-2.9.13.news">libxml2-2.9.13.news</a></td><td class="size">26.6 KiB</td><td class="date">2022-Feb-20 12:42</td></tr>
-<tr><td class="link"><a href="libxml2-2.9.13.sha256sum" title="libxml2-2.9.13.sha256sum">libxml2-2.9.13.sha256sum</a></td><td class="size">174 B</td><td class="date">2022-Feb-20 12:42</td></tr>
-<tr><td class="link"><a href="libxml2-2.9.13.tar.xz" title="libxml2-2.9.13.tar.xz">libxml2-2.9.13.tar.xz</a></td><td class="size">3.1 MiB</td><td class="date">2022-Feb-20 12:42</td></tr>
-<tr><td class="link"><a href="libxml2-2.9.14.news" title="libxml2-2.9.14.news">libxml2-2.9.14.news</a></td><td class="size">1.0 KiB</td><td class="date">2022-May-02 12:03</td></tr>
-<tr><td class="link"><a href="libxml2-2.9.14.sha256sum" title="libxml2-2.9.14.sha256sum">libxml2-2.9.14.sha256sum</a></td><td class="size">174 B</td><td class="date">2022-May-02 12:03</td></tr>
-<tr><td class="link"><a href="libxml2-2.9.14.tar.xz" title="libxml2-2.9.14.tar.xz">libxml2-2.9.14.tar.xz</a></td><td class="size">3.0 MiB</td><td class="date">2022-May-02 12:03</td></tr>
-<tr><td class="link"><a href="libxml2-2.9.2.sha256sum" title="libxml2-2.9.2.sha256sum">libxml2-2.9.2.sha256sum</a></td><td class="size">87 B</td><td class="date">2022-Feb-14 18:30</td></tr>
-<tr><td class="link"><a href="libxml2-2.9.2.tar.xz" title="libxml2-2.9.2.tar.xz">libxml2-2.9.2.tar.xz</a></td><td class="size">3.2 MiB</td><td class="date">2022-Feb-14 18:30</td></tr>
-<tr><td class="link"><a href="libxml2-2.9.3.sha256sum" title="libxml2-2.9.3.sha256sum">libxml2-2.9.3.sha256sum</a></td><td class="size">87 B</td><td class="date">2022-Feb-14 18:31</td></tr>
-<tr><td class="link"><a href="libxml2-2.9.3.tar.xz" title="libxml2-2.9.3.tar.xz">libxml2-2.9.3.tar.xz</a></td><td class="size">3.2 MiB</td><td class="date">2022-Feb-14 18:31</td></tr>
-<tr><td class="link"><a href="libxml2-2.9.4.sha256sum" title="libxml2-2.9.4.sha256sum">libxml2-2.9.4.sha256sum</a></td><td class="size">87 B</td><td class="date">2022-Feb-14 18:33</td></tr>
-<tr><td class="link"><a href="libxml2-2.9.4.tar.xz" title="libxml2-2.9.4.tar.xz">libxml2-2.9.4.tar.xz</a></td><td class="size">2.9 MiB</td><td class="date">2022-Feb-14 18:33</td></tr>
-<tr><td class="link"><a href="libxml2-2.9.5.sha256sum" title="libxml2-2.9.5.sha256sum">libxml2-2.9.5.sha256sum</a></td><td class="size">87 B</td><td class="date">2022-Feb-14 18:35</td></tr>
-<tr><td class="link"><a href="libxml2-2.9.5.tar.xz" title="libxml2-2.9.5.tar.xz">libxml2-2.9.5.tar.xz</a></td><td class="size">3.0 MiB</td><td class="date">2022-Feb-14 18:35</td></tr>
-<tr><td class="link"><a href="libxml2-2.9.6.sha256sum" title="libxml2-2.9.6.sha256sum">libxml2-2.9.6.sha256sum</a></td><td class="size">87 B</td><td class="date">2022-Feb-14 18:36</td></tr>
-<tr><td class="link"><a href="libxml2-2.9.6.tar.xz" title="libxml2-2.9.6.tar.xz">libxml2-2.9.6.tar.xz</a></td><td class="size">3.0 MiB</td><td class="date">2022-Feb-14 18:36</td></tr>
-<tr><td class="link"><a href="libxml2-2.9.7.sha256sum" title="libxml2-2.9.7.sha256sum">libxml2-2.9.7.sha256sum</a></td><td class="size">87 B</td><td class="date">2022-Feb-14 18:37</td></tr>
-<tr><td class="link"><a href="libxml2-2.9.7.tar.xz" title="libxml2-2.9.7.tar.xz">libxml2-2.9.7.tar.xz</a></td><td class="size">3.0 MiB</td><td class="date">2022-Feb-14 18:37</td></tr>
-<tr><td class="link"><a href="libxml2-2.9.8.sha256sum" title="libxml2-2.9.8.sha256sum">libxml2-2.9.8.sha256sum</a></td><td class="size">87 B</td><td class="date">2022-Feb-14 18:39</td></tr>
-<tr><td class="link"><a href="libxml2-2.9.8.tar.xz" title="libxml2-2.9.8.tar.xz">libxml2-2.9.8.tar.xz</a></td><td class="size">3.0 MiB</td><td class="date">2022-Feb-14 18:39</td></tr>
-<tr><td class="link"><a href="libxml2-2.9.9.sha256sum" title="libxml2-2.9.9.sha256sum">libxml2-2.9.9.sha256sum</a></td><td class="size">87 B</td><td class="date">2022-Feb-14 18:40</td></tr>
-<tr><td class="link"><a href="libxml2-2.9.9.tar.xz" title="libxml2-2.9.9.tar.xz">libxml2-2.9.9.tar.xz</a></td><td class="size">3.0 MiB</td><td class="date">2022-Feb-14 18:40</td></tr>
-</tbody></table></body></html>

bitbake/lib/bb/tests/fetch-testdata/software/libxml2/index.html

@@ -1,19 +0,0 @@
-<!DOCTYPE html><html><head><meta http-equiv="content-type" content="text/html; charset=utf-8"><meta name="viewport" content="width=device-width"><style type="text/css">body,html {background:#fff;font-family:"Bitstream Vera Sans","Lucida Grande","Lucida Sans Unicode",Lucidux,Verdana,Lucida,sans-serif;}tr:nth-child(even) {background:#f4f4f4;}th,td {padding:0.1em 0.5em;}th {text-align:left;font-weight:bold;background:#eee;border-bottom:1px solid #aaa;}#list {border:1px solid #aaa;width:100%;}a {color:#a33;}a:hover {color:#e33;}</style>
-
-<title>Index of /sources/libxml2/</title>
-</head><body><h1>Index of /sources/libxml2/</h1>
-<table id="list"><thead><tr><th style="width:55%"><a href="?C=N&amp;O=A">File Name</a>&nbsp;<a href="?C=N&amp;O=D">&nbsp;&darr;&nbsp;</a></th><th style="width:20%"><a href="?C=S&amp;O=A">File Size</a>&nbsp;<a href="?C=S&amp;O=D">&nbsp;&darr;&nbsp;</a></th><th style="width:25%"><a href="?C=M&amp;O=A">Date</a>&nbsp;<a href="?C=M&amp;O=D">&nbsp;&darr;&nbsp;</a></th></tr></thead>
-<tbody><tr><td class="link"><a href="../">Parent directory/</a></td><td class="size">-</td><td class="date">-</td></tr>
-<tr><td class="link"><a href="2.0/" title="2.0">2.0/</a></td><td class="size">-</td><td class="date">2009-Jul-14 13:04</td></tr>
-<tr><td class="link"><a href="2.1/" title="2.1">2.1/</a></td><td class="size">-</td><td class="date">2009-Jul-14 13:04</td></tr>
-<tr><td class="link"><a href="2.10/" title="2.10">2.10/</a></td><td class="size">-</td><td class="date">2022-Oct-14 12:55</td></tr>
-<tr><td class="link"><a href="2.2/" title="2.2">2.2/</a></td><td class="size">-</td><td class="date">2009-Jul-14 13:04</td></tr>
-<tr><td class="link"><a href="2.3/" title="2.3">2.3/</a></td><td class="size">-</td><td class="date">2009-Jul-14 13:05</td></tr>
-<tr><td class="link"><a href="2.4/" title="2.4">2.4/</a></td><td class="size">-</td><td class="date">2009-Jul-14 13:05</td></tr>
-<tr><td class="link"><a href="2.5/" title="2.5">2.5/</a></td><td class="size">-</td><td class="date">2009-Jul-14 13:05</td></tr>
-<tr><td class="link"><a href="2.6/" title="2.6">2.6/</a></td><td class="size">-</td><td class="date">2009-Jul-14 13:05</td></tr>
-<tr><td class="link"><a href="2.7/" title="2.7">2.7/</a></td><td class="size">-</td><td class="date">2022-Feb-14 18:24</td></tr>
-<tr><td class="link"><a href="2.8/" title="2.8">2.8/</a></td><td class="size">-</td><td class="date">2022-Feb-14 18:26</td></tr>
-<tr><td class="link"><a href="2.9/" title="2.9">2.9/</a></td><td class="size">-</td><td class="date">2022-May-02 12:04</td></tr>
-<tr><td class="link"><a href="cache.json" title="cache.json">cache.json</a></td><td class="size">22.8 KiB</td><td class="date">2022-Oct-14 12:55</td></tr>
-</tbody></table></body></html>

bitbake/lib/bb/tests/parse.py

@@ -119,7 +119,7 @@ EXTRA_OECONF:class-target = "b"
 EXTRA_OECONF:append = " c"
 """
 
-    def test_parse_overrides2(self):
+    def test_parse_overrides(self):
         f = self.parsehelper(self.overridetest2)
         d = bb.parse.handle(f.name, self.d)['']
         d.appendVar("EXTRA_OECONF", " d")
@@ -164,7 +164,6 @@ python () {
     # become unset/disappear.
     #
     def test_parse_classextend_contamination(self):
-        self.d.setVar("__bbclasstype", "recipe")
         cls = self.parsehelper(self.classextend_bbclass, suffix=".bbclass")
         #clsname = os.path.basename(cls.name).replace(".bbclass", "")
         self.classextend = self.classextend.replace("###CLASS###", cls.name)
@@ -186,60 +185,12 @@ deltask ${EMPTYVAR}
 """
     def test_parse_addtask_deltask(self):
         import sys
-
-        with self.assertLogs() as logs:
-            f = self.parsehelper(self.addtask_deltask)
-            d = bb.parse.handle(f.name, self.d)['']
-
-        output = "".join(logs.output)
-        self.assertTrue("addtask contained multiple 'before' keywords" in output)
-        self.assertTrue("addtask contained multiple 'after' keywords" in output)
-        self.assertTrue('addtask ignored: " do_patch"' in output)
-        #self.assertTrue('dependent task do_foo for do_patch does not exist' in output)
-
-    broken_multiline_comment = """
-# First line of comment \\
-# Second line of comment \\
-
-"""
-    def test_parse_broken_multiline_comment(self):
-        f = self.parsehelper(self.broken_multiline_comment)
-        with self.assertRaises(bb.BBHandledException):
-            d = bb.parse.handle(f.name, self.d)['']
-
-
-    comment_in_var = """
-VAR = " \\
-    SOMEVAL \\
-#   some comment \\
-    SOMEOTHERVAL \\
-"
-"""
-    def test_parse_comment_in_var(self):
-        f = self.parsehelper(self.comment_in_var)
-        with self.assertRaises(bb.BBHandledException):
-            d = bb.parse.handle(f.name, self.d)['']
-
-
-    at_sign_in_var_flag = """
-A[flag@.service] = "nonet"
-B[flag@.target] = "ntb"
-C[f] = "flag"
-
-unset A[flag@.service]
-"""
-    def test_parse_at_sign_in_var_flag(self):
-        f = self.parsehelper(self.at_sign_in_var_flag)
+        f = self.parsehelper(self.addtask_deltask)
         d = bb.parse.handle(f.name, self.d)['']
-        self.assertEqual(d.getVar("A"), None)
-        self.assertEqual(d.getVar("B"), None)
-        self.assertEqual(d.getVarFlag("A","flag@.service"), None)
-        self.assertEqual(d.getVarFlag("B","flag@.target"), "ntb")
-        self.assertEqual(d.getVarFlag("C","f"), "flag")
 
-    def test_parse_invalid_at_sign_in_var_flag(self):
-        invalid_at_sign = self.at_sign_in_var_flag.replace("B[f", "B[@f")
-        f = self.parsehelper(invalid_at_sign)
-        with self.assertRaises(bb.parse.ParseError):
-            d = bb.parse.handle(f.name, self.d)['']
+        stdout = sys.stdout.getvalue()
+        self.assertTrue("addtask contained multiple 'before' keywords" in stdout)
+        self.assertTrue("addtask contained multiple 'after' keywords" in stdout)
+        self.assertTrue('addtask ignored: " do_patch"' in stdout)
+        #self.assertTrue('dependent task do_foo for do_patch does not exist' in stdout)
 

bitbake/lib/bb/tests/runqueue-tests/conf/bitbake.conf

@@ -12,6 +12,6 @@ STAMP = "${TMPDIR}/stamps/${PN}"
 T = "${TMPDIR}/workdir/${PN}/temp"
 BB_NUMBER_THREADS = "4"
 
-BB_BASEHASH_IGNORE_VARS = "BB_CURRENT_MC BB_HASHSERVE TMPDIR TOPDIR SLOWTASKS SSTATEVALID FILE BB_CURRENTTASK"
+BB_HASHBASE_WHITELIST = "BB_CURRENT_MC BB_HASHSERVE TMPDIR TOPDIR SLOWTASKS SSTATEVALID FILE"
 
 include conf/multiconfig/${BB_CURRENT_MC}.conf

bitbake/lib/bb/tests/runqueue.py

@@ -29,14 +29,13 @@ class RunQueueTests(unittest.TestCase):
     def run_bitbakecmd(self, cmd, builddir, sstatevalid="", slowtasks="", extraenv=None, cleanup=False):
         env = os.environ.copy()
         env["BBPATH"] = os.path.realpath(os.path.join(os.path.dirname(__file__), "runqueue-tests"))
-        env["BB_ENV_PASSTHROUGH_ADDITIONS"] = "SSTATEVALID SLOWTASKS TOPDIR"
+        env["BB_ENV_EXTRAWHITE"] = "SSTATEVALID SLOWTASKS"
         env["SSTATEVALID"] = sstatevalid
         env["SLOWTASKS"] = slowtasks
-        env["TOPDIR"] = builddir
         if extraenv:
             for k in extraenv:
                 env[k] = extraenv[k]
-                env["BB_ENV_PASSTHROUGH_ADDITIONS"] = env["BB_ENV_PASSTHROUGH_ADDITIONS"] + " " + k
+                env["BB_ENV_EXTRAWHITE"] = env["BB_ENV_EXTRAWHITE"] + " " + k
         try:
             output = subprocess.check_output(cmd, env=env, stderr=subprocess.STDOUT,universal_newlines=True, cwd=builddir)
             print(output)
@@ -59,8 +58,6 @@ class RunQueueTests(unittest.TestCase):
             expected = ['a1:' + x for x in self.alltasks]
             self.assertEqual(set(tasks), set(expected))
 
-            self.shutdown(tempdir)
-
     def test_single_setscenevalid(self):
         with tempfile.TemporaryDirectory(prefix="runqueuetest") as tempdir:
             cmd = ["bitbake", "a1"]
@@ -71,8 +68,6 @@ class RunQueueTests(unittest.TestCase):
                         'a1:populate_sysroot', 'a1:build']
             self.assertEqual(set(tasks), set(expected))
 
-            self.shutdown(tempdir)
-
     def test_intermediate_setscenevalid(self):
         with tempfile.TemporaryDirectory(prefix="runqueuetest") as tempdir:
             cmd = ["bitbake", "a1"]
@@ -82,8 +77,6 @@ class RunQueueTests(unittest.TestCase):
                         'a1:populate_sysroot_setscene', 'a1:build']
             self.assertEqual(set(tasks), set(expected))
 
-            self.shutdown(tempdir)
-
     def test_intermediate_notcovered(self):
         with tempfile.TemporaryDirectory(prefix="runqueuetest") as tempdir:
             cmd = ["bitbake", "a1"]
@@ -93,8 +86,6 @@ class RunQueueTests(unittest.TestCase):
                         'a1:package_qa_setscene', 'a1:build', 'a1:populate_sysroot_setscene']
             self.assertEqual(set(tasks), set(expected))
 
-            self.shutdown(tempdir)
-
     def test_all_setscenevalid(self):
         with tempfile.TemporaryDirectory(prefix="runqueuetest") as tempdir:
             cmd = ["bitbake", "a1"]
@@ -104,8 +95,6 @@ class RunQueueTests(unittest.TestCase):
                         'a1:package_qa_setscene', 'a1:build', 'a1:populate_sysroot_setscene']
             self.assertEqual(set(tasks), set(expected))
 
-            self.shutdown(tempdir)
-
     def test_no_settasks(self):
         with tempfile.TemporaryDirectory(prefix="runqueuetest") as tempdir:
             cmd = ["bitbake", "a1", "-c", "patch"]
@@ -114,8 +103,6 @@ class RunQueueTests(unittest.TestCase):
             expected = ['a1:fetch', 'a1:unpack', 'a1:patch']
             self.assertEqual(set(tasks), set(expected))
 
-            self.shutdown(tempdir)
-
     def test_mix_covered_notcovered(self):
         with tempfile.TemporaryDirectory(prefix="runqueuetest") as tempdir:
             cmd = ["bitbake", "a1:do_patch", "a1:do_populate_sysroot"]
@@ -124,7 +111,6 @@ class RunQueueTests(unittest.TestCase):
             expected = ['a1:fetch', 'a1:unpack', 'a1:patch', 'a1:populate_sysroot_setscene']
             self.assertEqual(set(tasks), set(expected))
 
-            self.shutdown(tempdir)
 
     # Test targets with intermediate setscene tasks alongside a target with no intermediate setscene tasks
     def test_mixed_direct_tasks_setscene_tasks(self):
@@ -136,8 +122,6 @@ class RunQueueTests(unittest.TestCase):
                         'a1:package_qa_setscene', 'a1:build', 'a1:populate_sysroot_setscene']
             self.assertEqual(set(tasks), set(expected))
 
-            self.shutdown(tempdir)
-
     # This test slows down the execution of do_package_setscene until after other real tasks have
     # started running which tests for a bug where tasks were being lost from the buildable list of real
     # tasks if they weren't in tasks_covered or tasks_notcovered
@@ -152,14 +136,12 @@ class RunQueueTests(unittest.TestCase):
                         'a1:populate_sysroot', 'a1:build']
             self.assertEqual(set(tasks), set(expected))
 
-            self.shutdown(tempdir)
-
-    def test_setscene_ignore_tasks(self):
+    def test_setscenewhitelist(self):
         with tempfile.TemporaryDirectory(prefix="runqueuetest") as tempdir:
             cmd = ["bitbake", "a1"]
             extraenv = {
                 "BB_SETSCENE_ENFORCE" : "1",
-                "BB_SETSCENE_ENFORCE_IGNORE_TASKS" : "a1:do_package_write_rpm a1:do_build"
+                "BB_SETSCENE_ENFORCE_WHITELIST" : "a1:do_package_write_rpm a1:do_build"
             }
             sstatevalid = "a1:do_package a1:do_package_qa a1:do_packagedata a1:do_package_write_ipk a1:do_populate_lic a1:do_populate_sysroot"
             tasks = self.run_bitbakecmd(cmd, tempdir, sstatevalid, extraenv=extraenv)
@@ -167,8 +149,6 @@ class RunQueueTests(unittest.TestCase):
                         'a1:populate_sysroot_setscene', 'a1:package_setscene']
             self.assertEqual(set(tasks), set(expected))
 
-            self.shutdown(tempdir)
-
     # Tests for problems with dependencies between setscene tasks
     def test_no_setscenevalid_harddeps(self):
         with tempfile.TemporaryDirectory(prefix="runqueuetest") as tempdir:
@@ -182,8 +162,6 @@ class RunQueueTests(unittest.TestCase):
                         'd1:populate_sysroot', 'd1:build']
             self.assertEqual(set(tasks), set(expected))
 
-            self.shutdown(tempdir)
-
     def test_no_setscenevalid_withdeps(self):
         with tempfile.TemporaryDirectory(prefix="runqueuetest") as tempdir:
             cmd = ["bitbake", "b1"]
@@ -194,8 +172,6 @@ class RunQueueTests(unittest.TestCase):
             expected.remove('a1:package_qa')
             self.assertEqual(set(tasks), set(expected))
 
-            self.shutdown(tempdir)
-
     def test_single_a1_setscenevalid_withdeps(self):
         with tempfile.TemporaryDirectory(prefix="runqueuetest") as tempdir:
             cmd = ["bitbake", "b1"]
@@ -206,8 +182,6 @@ class RunQueueTests(unittest.TestCase):
                         'a1:populate_sysroot'] + ['b1:' + x for x in self.alltasks]
             self.assertEqual(set(tasks), set(expected))
 
-            self.shutdown(tempdir)
-
     def test_single_b1_setscenevalid_withdeps(self):
         with tempfile.TemporaryDirectory(prefix="runqueuetest") as tempdir:
             cmd = ["bitbake", "b1"]
@@ -219,8 +193,6 @@ class RunQueueTests(unittest.TestCase):
             expected.remove('b1:package')
             self.assertEqual(set(tasks), set(expected))
 
-            self.shutdown(tempdir)
-
     def test_intermediate_setscenevalid_withdeps(self):
         with tempfile.TemporaryDirectory(prefix="runqueuetest") as tempdir:
             cmd = ["bitbake", "b1"]
@@ -231,8 +203,6 @@ class RunQueueTests(unittest.TestCase):
             expected.remove('b1:package')
             self.assertEqual(set(tasks), set(expected))
 
-            self.shutdown(tempdir)
-
     def test_all_setscenevalid_withdeps(self):
         with tempfile.TemporaryDirectory(prefix="runqueuetest") as tempdir:
             cmd = ["bitbake", "b1"]
@@ -243,8 +213,6 @@ class RunQueueTests(unittest.TestCase):
                         'b1:packagedata_setscene', 'b1:package_qa_setscene', 'b1:populate_sysroot_setscene']
             self.assertEqual(set(tasks), set(expected))
 
-            self.shutdown(tempdir)
-
     def test_multiconfig_setscene_optimise(self):
         with tempfile.TemporaryDirectory(prefix="runqueuetest") as tempdir:
             extraenv = {
@@ -264,8 +232,6 @@ class RunQueueTests(unittest.TestCase):
                 expected.remove(x)
             self.assertEqual(set(tasks), set(expected))
 
-            self.shutdown(tempdir)
-
     def test_multiconfig_bbmask(self):
         # This test validates that multiconfigs can independently mask off
         # recipes they do not want with BBMASK. It works by having recipes
@@ -282,13 +248,11 @@ class RunQueueTests(unittest.TestCase):
             cmd = ["bitbake", "mc:mc-1:fails-mc2", "mc:mc_2:fails-mc1"]
             self.run_bitbakecmd(cmd, tempdir, "", extraenv=extraenv)
 
-            self.shutdown(tempdir)
-
     def test_multiconfig_mcdepends(self):
         with tempfile.TemporaryDirectory(prefix="runqueuetest") as tempdir:
             extraenv = {
                 "BBMULTICONFIG" : "mc-1 mc_2",
-                "BB_SIGNATURE_HANDLER" : "basichash",
+                "BB_SIGNATURE_HANDLER" : "TestMulticonfigDepends",
                 "EXTRA_BBFILES": "${COREBASE}/recipes/fails-mc/*.bb",
             }
             tasks = self.run_bitbakecmd(["bitbake", "mc:mc-1:f1"], tempdir, "", extraenv=extraenv, cleanup=True)
@@ -314,8 +278,7 @@ class RunQueueTests(unittest.TestCase):
                        ["mc_2:a1:%s" % t for t in rerun_tasks]
             self.assertEqual(set(tasks), set(expected))
 
-            self.shutdown(tempdir)
-
+    @unittest.skipIf(sys.version_info < (3, 5, 0), 'Python 3.5 or later required')
     def test_hashserv_single(self):
         with tempfile.TemporaryDirectory(prefix="runqueuetest") as tempdir:
             extraenv = {
@@ -341,6 +304,7 @@ class RunQueueTests(unittest.TestCase):
 
             self.shutdown(tempdir)
 
+    @unittest.skipIf(sys.version_info < (3, 5, 0), 'Python 3.5 or later required')
     def test_hashserv_double(self):
         with tempfile.TemporaryDirectory(prefix="runqueuetest") as tempdir:
             extraenv = {
@@ -365,6 +329,7 @@ class RunQueueTests(unittest.TestCase):
 
             self.shutdown(tempdir)
 
+    @unittest.skipIf(sys.version_info < (3, 5, 0), 'Python 3.5 or later required')
     def test_hashserv_multiple_setscene(self):
         # Runs e1:do_package_setscene twice
         with tempfile.TemporaryDirectory(prefix="runqueuetest") as tempdir:
@@ -396,6 +361,7 @@ class RunQueueTests(unittest.TestCase):
 
     def shutdown(self, tempdir):
         # Wait for the hashserve socket to disappear else we'll see races with the tempdir cleanup
-        while (os.path.exists(tempdir + "/hashserve.sock") or os.path.exists(tempdir + "cache/hashserv.db-wal") or os.path.exists(tempdir + "/bitbake.lock")):
+        while (os.path.exists(tempdir + "/hashserve.sock") or os.path.exists(tempdir + "cache/hashserv.db-wal")):
             time.sleep(0.5)
 
+

bitbake/lib/bb/tests/siggen.py

@@ -17,12 +17,75 @@ import bb.siggen
 
 class SiggenTest(unittest.TestCase):
 
-    def test_build_pnid(self):
-        tests = {
-            ('', 'helloworld', 'do_sometask') : 'helloworld:do_sometask',
-            ('XX', 'helloworld', 'do_sometask') : 'mc:XX:helloworld:do_sometask',
-        }
+    def test_clean_basepath_simple_target_basepath(self):
+        basepath = '/full/path/to/poky/meta/recipes-whatever/helloworld/helloworld_1.2.3.bb:do_sometask'
+        expected_cleaned = 'helloworld/helloworld_1.2.3.bb:do_sometask'
 
-        for t in tests:
-            self.assertEqual(bb.siggen.build_pnid(*t), tests[t])
+        actual_cleaned = bb.siggen.clean_basepath(basepath)
 
+        self.assertEqual(actual_cleaned, expected_cleaned)
+
+    def test_clean_basepath_basic_virtual_basepath(self):
+        basepath = 'virtual:something:/full/path/to/poky/meta/recipes-whatever/helloworld/helloworld_1.2.3.bb:do_sometask'
+        expected_cleaned = 'helloworld/helloworld_1.2.3.bb:do_sometask:virtual:something'
+
+        actual_cleaned = bb.siggen.clean_basepath(basepath)
+
+        self.assertEqual(actual_cleaned, expected_cleaned)
+
+    def test_clean_basepath_mc_basepath(self):
+        basepath = 'mc:somemachine:/full/path/to/poky/meta/recipes-whatever/helloworld/helloworld_1.2.3.bb:do_sometask'
+        expected_cleaned = 'helloworld/helloworld_1.2.3.bb:do_sometask:mc:somemachine'
+
+        actual_cleaned = bb.siggen.clean_basepath(basepath)
+
+        self.assertEqual(actual_cleaned, expected_cleaned)
+
+    def test_clean_basepath_virtual_long_prefix_basepath(self):
+        basepath = 'virtual:something:A:B:C:/full/path/to/poky/meta/recipes-whatever/helloworld/helloworld_1.2.3.bb:do_sometask'
+        expected_cleaned = 'helloworld/helloworld_1.2.3.bb:do_sometask:virtual:something:A:B:C'
+
+        actual_cleaned = bb.siggen.clean_basepath(basepath)
+
+        self.assertEqual(actual_cleaned, expected_cleaned)
+
+    def test_clean_basepath_mc_virtual_basepath(self):
+        basepath = 'mc:somemachine:virtual:something:/full/path/to/poky/meta/recipes-whatever/helloworld/helloworld_1.2.3.bb:do_sometask'
+        expected_cleaned = 'helloworld/helloworld_1.2.3.bb:do_sometask:virtual:something:mc:somemachine'
+
+        actual_cleaned = bb.siggen.clean_basepath(basepath)
+
+        self.assertEqual(actual_cleaned, expected_cleaned)
+
+    def test_clean_basepath_mc_virtual_long_prefix_basepath(self):
+        basepath = 'mc:X:virtual:something:C:B:A:/full/path/to/poky/meta/recipes-whatever/helloworld/helloworld_1.2.3.bb:do_sometask'
+        expected_cleaned = 'helloworld/helloworld_1.2.3.bb:do_sometask:virtual:something:C:B:A:mc:X'
+
+        actual_cleaned = bb.siggen.clean_basepath(basepath)
+
+        self.assertEqual(actual_cleaned, expected_cleaned)
+
+
+    # def test_clean_basepath_performance(self):
+    #     input_basepaths = [
+    #         'mc:X:/full/path/to/poky/meta/recipes-whatever/helloworld/helloworld_1.2.3.bb:do_sometask',
+    #         'mc:X:virtual:something:C:B:A:/full/path/to/poky/meta/recipes-whatever/helloworld/helloworld_1.2.3.bb:do_sometask',
+    #         'virtual:something:C:B:A:/different/path/to/poky/meta/recipes-whatever/helloworld/helloworld_1.2.3.bb:do_sometask',
+    #         'virtual:something:A:/full/path/to/poky/meta/recipes-whatever/helloworld/helloworld_1.2.3.bb:do_sometask',
+    #         '/this/is/most/common/input/recipes-whatever/helloworld/helloworld_1.2.3.bb:do_sometask',
+    #         '/and/should/be/tested/with/recipes-whatever/helloworld/helloworld_1.2.3.bb:do_sometask',
+    #         '/more/weight/recipes-whatever/helloworld/helloworld_1.2.3.bb:do_sometask',
+    #     ]
+
+    #     time_start = time.time()
+
+    #     i = 2000000
+    #     while i >= 0:
+    #         for basepath in input_basepaths:
+    #             bb.siggen.clean_basepath(basepath)
+    #         i -= 1
+
+    #     elapsed = time.time() - time_start
+    #     print('{} ({}s)'.format(self.id(), round(elapsed, 3)))
+
+    #     self.assertTrue(False)

bitbake/lib/bb/tinfoil.py

@@ -10,7 +10,6 @@
 import logging
 import os
 import sys
-import time
 import atexit
 import re
 from collections import OrderedDict, defaultdict
@@ -449,7 +448,7 @@ class Tinfoil:
         self.run_actions(config_params)
         self.recipes_parsed = True
 
-    def run_command(self, command, *params, handle_events=True):
+    def run_command(self, command, *params):
         """
         Run a command on the server (as implemented in bb.command).
         Note that there are two types of command - synchronous and
@@ -469,7 +468,7 @@ class Tinfoil:
         try:
             result = self.server_connection.connection.runCommand(commandline)
         finally:
-            while handle_events:
+            while True:
                 event = self.wait_event()
                 if not event:
                     break
@@ -494,7 +493,7 @@ class Tinfoil:
         Wait for an event from the server for the specified time.
         A timeout of 0 means don't wait if there are no events in the queue.
         Returns the next event in the queue or None if the timeout was
-        reached. Note that in order to receive any events you will
+        reached. Note that in order to recieve any events you will
         first need to set the internal event mask using set_event_mask()
         (otherwise whatever event mask the UI set up will be in effect).
         """
@@ -730,7 +729,6 @@ class Tinfoil:
 
         ret = self.run_command('buildTargets', targets, task)
         if handle_events:
-            lastevent = time.time()
             result = False
             # Borrowed from knotty, instead somewhat hackily we use the helper
             # as the object to store "shutdown" on
@@ -743,7 +741,6 @@ class Tinfoil:
                     try:
                         event = self.wait_event(0.25)
                         if event:
-                            lastevent = time.time()
                             if event_callback and event_callback(event):
                                 continue
                             if helper.eventHandler(event):
@@ -764,7 +761,7 @@ class Tinfoil:
                                 if parseprogress:
                                     parseprogress.update(event.progress)
                                 else:
-                                    bb.warn("Got ProcessProgress event for something that never started?")
+                                    bb.warn("Got ProcessProgress event for someting that never started?")
                                 continue
                             if isinstance(event, bb.event.ProcessFinished):
                                 if self.quiet > 1:
@@ -776,7 +773,7 @@ class Tinfoil:
                             if isinstance(event, bb.command.CommandCompleted):
                                 result = True
                                 break
-                            if isinstance(event, (bb.command.CommandFailed, bb.command.CommandExit)):
+                            if isinstance(event, bb.command.CommandFailed):
                                 self.logger.error(str(event))
                                 result = False
                                 break
@@ -788,13 +785,10 @@ class Tinfoil:
                                 self.logger.error(str(event))
                                 result = False
                                 break
+
                         elif helper.shutdown > 1:
                             break
                         termfilter.updateFooter()
-                        if time.time() > (lastevent + (3*60)):
-                            if not self.run_command('ping', handle_events=False):
-                                print("\nUnable to ping server and no events, closing down...\n")
-                                return False
                     except KeyboardInterrupt:
                         termfilter.clearFooter()
                         if helper.shutdown == 1:

bitbake/lib/bb/ui/buildinfohelper.py

@@ -45,7 +45,7 @@ from pprint import pformat
 import logging
 from datetime import datetime, timedelta
 
-from django.db import transaction
+from django.db import transaction, connection
 
 
 # pylint: disable=invalid-name
@@ -227,12 +227,6 @@ class ORMWrapper(object):
         build.completed_on = timezone.now()
         build.outcome = outcome
         build.save()
-
-        # We force a sync point here to force the outcome status commit,
-        # which resolves a race condition with the build completion takedown
-        transaction.set_autocommit(True)
-        transaction.set_autocommit(False)
-
         signal_runbuilds()
 
     def update_target_set_license_manifest(self, target, license_manifest_path):
@@ -489,14 +483,14 @@ class ORMWrapper(object):
 
             # we already created the root directory, so ignore any
             # entry for it
-            if not path:
+            if len(path) == 0:
                 continue
 
             parent_path = "/".join(path.split("/")[:len(path.split("/")) - 1])
-            if not parent_path:
+            if len(parent_path) == 0:
                 parent_path = "/"
             parent_obj = self._cached_get(Target_File, target = target_obj, path = parent_path, inodetype = Target_File.ITYPE_DIRECTORY)
-            Target_File.objects.create(
+            tf_obj = Target_File.objects.create(
                         target = target_obj,
                         path = path,
                         size = size,
@@ -561,7 +555,7 @@ class ORMWrapper(object):
 
             parent_obj = Target_File.objects.get(target = target_obj, path = parent_path, inodetype = Target_File.ITYPE_DIRECTORY)
 
-            Target_File.objects.create(
+            tf_obj = Target_File.objects.create(
                         target = target_obj,
                         path = path,
                         size = size,
@@ -577,7 +571,7 @@ class ORMWrapper(object):
         assert isinstance(build_obj, Build)
         assert isinstance(target_obj, Target)
 
-        errormsg = []
+        errormsg = ""
         for p in packagedict:
             # Search name swtiches round the installed name vs package name
             # by default installed name == package name
@@ -639,10 +633,10 @@ class ORMWrapper(object):
                         packagefile_objects.append(Package_File( package = packagedict[p]['object'],
                             path = targetpath,
                             size = targetfilesize))
-                    if packagefile_objects:
+                    if len(packagefile_objects):
                         Package_File.objects.bulk_create(packagefile_objects)
                 except KeyError as e:
-                    errormsg.append("  stpi: Key error, package %s key %s \n" % (p, e))
+                    errormsg += "  stpi: Key error, package %s key %s \n" % ( p, e )
 
             # save disk installed size
             packagedict[p]['object'].installed_size = packagedict[p]['size']
@@ -679,13 +673,13 @@ class ORMWrapper(object):
                     logger.warning("Could not add dependency to the package %s "
                                    "because %s is an unknown package", p, px)
 
-        if packagedeps_objs:
+        if len(packagedeps_objs) > 0:
             Package_Dependency.objects.bulk_create(packagedeps_objs)
         else:
             logger.info("No package dependencies created")
 
-        if errormsg:
-            logger.warning("buildinfohelper: target_package_info could not identify recipes: \n%s", "".join(errormsg))
+        if len(errormsg) > 0:
+            logger.warning("buildinfohelper: target_package_info could not identify recipes: \n%s", errormsg)
 
     def save_target_image_file_information(self, target_obj, file_name, file_size):
         Target_Image_File.objects.create(target=target_obj,
@@ -773,7 +767,7 @@ class ORMWrapper(object):
             packagefile_objects.append(Package_File( package = bp_object,
                                         path = path,
                                         size = package_info['FILES_INFO'][path] ))
-        if packagefile_objects:
+        if len(packagefile_objects):
             Package_File.objects.bulk_create(packagefile_objects)
 
         def _po_byname(p):
@@ -815,7 +809,7 @@ class ORMWrapper(object):
                 packagedeps_objs.append(Package_Dependency(  package = bp_object,
                     depends_on = _po_byname(p), dep_type = Package_Dependency.TYPE_RCONFLICTS))
 
-        if packagedeps_objs:
+        if len(packagedeps_objs) > 0:
             Package_Dependency.objects.bulk_create(packagedeps_objs)
 
         return bp_object
@@ -832,7 +826,7 @@ class ORMWrapper(object):
                     desc = vardump[root_var]['doc']
             if desc is None:
                 desc = ''
-            if desc:
+            if len(desc):
                 HelpText.objects.get_or_create(build=build_obj,
                                                area=HelpText.VARIABLE,
                                                key=k, text=desc)
@@ -852,7 +846,7 @@ class ORMWrapper(object):
                                 file_name = vh['file'],
                                 line_number = vh['line'],
                                 operation = vh['op']))
-                if varhist_objects:
+                if len(varhist_objects):
                     VariableHistory.objects.bulk_create(varhist_objects)
 
 
@@ -899,6 +893,9 @@ class BuildInfoHelper(object):
         self.task_order = 0
         self.autocommit_step = 1
         self.server = server
+        # we use manual transactions if the database doesn't autocommit on us
+        if not connection.features.autocommits_when_autocommit_is_off:
+            transaction.set_autocommit(False)
         self.orm_wrapper = ORMWrapper()
         self.has_build_history = has_build_history
         self.tmp_dir = self.server.runCommand(["getVariable", "TMPDIR"])[0]
@@ -1062,6 +1059,27 @@ class BuildInfoHelper(object):
 
         return recipe_info
 
+    def _get_path_information(self, task_object):
+        self._ensure_build()
+
+        assert isinstance(task_object, Task)
+        build_stats_format = "{tmpdir}/buildstats/{buildname}/{package}/"
+        build_stats_path = []
+
+        for t in self.internal_state['targets']:
+            buildname = self.internal_state['build'].build_name
+            pe, pv = task_object.recipe.version.split(":",1)
+            if len(pe) > 0:
+                package = task_object.recipe.name + "-" + pe + "_" + pv
+            else:
+                package = task_object.recipe.name + "-" + pv
+
+            build_stats_path.append(build_stats_format.format(tmpdir=self.tmp_dir,
+                                                     buildname=buildname,
+                                                     package=package))
+
+        return build_stats_path
+
 
     ################################
     ## external available methods to store information
@@ -1295,11 +1313,12 @@ class BuildInfoHelper(object):
                 task_information['outcome'] = Task.OUTCOME_FAILED
                 del self.internal_state['taskdata'][identifier]
 
-        # we force a sync point here, to get the progress bar to show
-        if self.autocommit_step % 3 == 0:
-            transaction.set_autocommit(True)
-            transaction.set_autocommit(False)
-        self.autocommit_step += 1
+        if not connection.features.autocommits_when_autocommit_is_off:
+            # we force a sync point here, to get the progress bar to show
+            if self.autocommit_step % 3 == 0:
+                transaction.set_autocommit(True)
+                transaction.set_autocommit(False)
+            self.autocommit_step += 1
 
         self.orm_wrapper.get_update_task_object(task_information, True) # must exist
 
@@ -1385,7 +1404,7 @@ class BuildInfoHelper(object):
         assert 'pn' in event._depgraph
         assert 'tdepends' in event._depgraph
 
-        errormsg = []
+        errormsg = ""
 
         # save layer version priorities
         if 'layer-priorities' in event._depgraph.keys():
@@ -1477,7 +1496,7 @@ class BuildInfoHelper(object):
                 elif dep in self.internal_state['recipes']:
                     dependency = self.internal_state['recipes'][dep]
                 else:
-                    errormsg.append("  stpd: KeyError saving recipe dependency for %s, %s \n" % (recipe, dep))
+                    errormsg += "  stpd: KeyError saving recipe dependency for %s, %s \n" % (recipe, dep)
                     continue
                 recipe_dep = Recipe_Dependency(recipe=target,
                                                depends_on=dependency,
@@ -1518,8 +1537,8 @@ class BuildInfoHelper(object):
                 taskdeps_objects.append(Task_Dependency( task = target, depends_on = dep ))
         Task_Dependency.objects.bulk_create(taskdeps_objects)
 
-        if errormsg:
-            logger.warning("buildinfohelper: dependency info not identify recipes: \n%s", "".join(errormsg))
+        if len(errormsg) > 0:
+            logger.warning("buildinfohelper: dependency info not identify recipes: \n%s", errormsg)
 
 
     def store_build_package_information(self, event):
@@ -1599,7 +1618,7 @@ class BuildInfoHelper(object):
 
         if 'backlog' in self.internal_state:
             # if we have a backlog of events, do our best to save them here
-            if self.internal_state['backlog']:
+            if len(self.internal_state['backlog']):
                 tempevent = self.internal_state['backlog'].pop()
                 logger.debug("buildinfohelper: Saving stored event %s "
                              % tempevent)
@@ -1746,6 +1765,7 @@ class BuildInfoHelper(object):
 
         buildname = self.server.runCommand(['getVariable', 'BUILDNAME'])[0]
         machine = self.server.runCommand(['getVariable', 'MACHINE'])[0]
+        image_name = self.server.runCommand(['getVariable', 'IMAGE_NAME'])[0]
 
         # location of the manifest files for this build;
         # note that this file is only produced if an image is produced
@@ -1766,18 +1786,6 @@ class BuildInfoHelper(object):
         # filter out anything which isn't an image target
         image_targets = [target for target in targets if target.is_image]
 
-        if len(image_targets) > 0:
-            #if there are image targets retrieve image_name
-            image_name = self.server.runCommand(['getVariable', 'IMAGE_NAME'])[0]
-            if not image_name:
-                #When build target is an image and image_name is not found as an environment variable
-                logger.info("IMAGE_NAME not found, extracting from bitbake command")
-                cmd = self.server.runCommand(['getVariable','BB_CMDLINE'])[0]
-                #filter out tokens that are command line options
-                cmd = [token for token in cmd if not token.startswith('-')]
-                image_name = cmd[1].split(':', 1)[0] # remove everything after : in image name
-                logger.info("IMAGE_NAME found as : %s " % image_name)
-
         for image_target in image_targets:
             # this is set to True if we find at least one file relating to
             # this target; if this remains False after the scan, we copy the
@@ -1982,6 +1990,8 @@ class BuildInfoHelper(object):
             # Do not skip command line build events
             self.store_log_event(tempevent,False)
 
+        if not connection.features.autocommits_when_autocommit_is_off:
+            transaction.set_autocommit(True)
 
         # unset the brbe; this is to prevent subsequent command-line builds
         # being incorrectly attached to the previous Toaster-triggered build;

bitbake/lib/bb/ui/knotty.py

@@ -25,7 +25,7 @@ from itertools import groupby
 
 from bb.ui import uihelper
 
-featureSet = [bb.cooker.CookerFeatures.SEND_SANITYEVENTS, bb.cooker.CookerFeatures.BASEDATASTORE_TRACKING]
+featureSet = [bb.cooker.CookerFeatures.SEND_SANITYEVENTS]
 
 logger = logging.getLogger("BitBake")
 interactive = sys.stdout.isatty()
@@ -228,9 +228,7 @@ class TerminalFilter(object):
 
     def keepAlive(self, t):
         if not self.cuu:
-            print("Bitbake still alive (no events for %ds). Active tasks:" % t)
-            for t in self.helper.running_tasks:
-                print(t)
+            print("Bitbake still alive (%ds)" % t)
             sys.stdout.flush()
 
     def updateFooter(self):
@@ -252,68 +250,58 @@ class TerminalFilter(object):
             return
         tasks = []
         for t in runningpids:
-            start_time = activetasks[t].get("starttime", None)
-            if start_time:
-                msg = "%s - %s (pid %s)" % (activetasks[t]["title"], self.elapsed(currenttime - start_time), activetasks[t]["pid"])
-            else:
-                msg = "%s (pid %s)" % (activetasks[t]["title"], activetasks[t]["pid"])
             progress = activetasks[t].get("progress", None)
             if progress is not None:
                 pbar = activetasks[t].get("progressbar", None)
                 rate = activetasks[t].get("rate", None)
+                start_time = activetasks[t].get("starttime", None)
                 if not pbar or pbar.bouncing != (progress < 0):
                     if progress < 0:
-                        pbar = BBProgress("0: %s" % msg, 100, widgets=[' ', progressbar.BouncingSlider(), ''], extrapos=3, resize_handler=self.sigwinch_handle)
+                        pbar = BBProgress("0: %s (pid %s)" % (activetasks[t]["title"], activetasks[t]["pid"]), 100, widgets=[' ', progressbar.BouncingSlider(), ''], extrapos=3, resize_handler=self.sigwinch_handle)
                         pbar.bouncing = True
                     else:
-                        pbar = BBProgress("0: %s" % msg, 100, widgets=[' ', progressbar.Percentage(), ' ', progressbar.Bar(), ''], extrapos=5, resize_handler=self.sigwinch_handle)
+                        pbar = BBProgress("0: %s (pid %s)" % (activetasks[t]["title"], activetasks[t]["pid"]), 100, widgets=[' ', progressbar.Percentage(), ' ', progressbar.Bar(), ''], extrapos=5, resize_handler=self.sigwinch_handle)
                         pbar.bouncing = False
                     activetasks[t]["progressbar"] = pbar
-                tasks.append((pbar, msg, progress, rate, start_time))
+                tasks.append((pbar, progress, rate, start_time))
             else:
-                tasks.append(msg)
+                start_time = activetasks[t].get("starttime", None)
+                if start_time:
+                    tasks.append("%s - %s (pid %s)" % (activetasks[t]["title"], self.elapsed(currenttime - start_time), activetasks[t]["pid"]))
+                else:
+                    tasks.append("%s (pid %s)" % (activetasks[t]["title"], activetasks[t]["pid"]))
 
         if self.main.shutdown:
-            content = pluralise("Waiting for %s running task to finish",
-                                "Waiting for %s running tasks to finish", len(activetasks))
-            if not self.quiet:
-                content += ':'
+            content = "Waiting for %s running tasks to finish:" % len(activetasks)
             print(content)
         else:
-            scene_tasks = "%s of %s" % (self.helper.setscene_current, self.helper.setscene_total)
-            cur_tasks = "%s of %s" % (self.helper.tasknumber_current, self.helper.tasknumber_total)
-
-            content = ''
-            if not self.quiet:
-                msg = "Setscene tasks: %s" % scene_tasks
-                content += msg + "\n"
-                print(msg)
-
             if self.quiet:
-                msg = "Running tasks (%s, %s)" % (scene_tasks, cur_tasks)
+                content = "Running tasks (%s of %s/%s of %s)" % (self.helper.setscene_current, self.helper.setscene_total, self.helper.tasknumber_current, self.helper.tasknumber_total)
             elif not len(activetasks):
-                msg = "No currently running tasks (%s)" % cur_tasks
+                content = "No currently running tasks (%s of %s/%s of %s)" % (self.helper.setscene_current, self.helper.setscene_total, self.helper.tasknumber_current, self.helper.tasknumber_total)
             else:
-                msg = "Currently %2s running tasks (%s)" % (len(activetasks), cur_tasks)
+                content = "Currently %2s running tasks (%s of %s/%s of %s)" % (len(activetasks), self.helper.setscene_current, self.helper.setscene_total, self.helper.tasknumber_current, self.helper.tasknumber_total)
             maxtask = self.helper.tasknumber_total
             if not self.main_progress or self.main_progress.maxval != maxtask:
                 widgets = [' ', progressbar.Percentage(), ' ', progressbar.Bar()]
                 self.main_progress = BBProgress("Running tasks", maxtask, widgets=widgets, resize_handler=self.sigwinch_handle)
                 self.main_progress.start(False)
-            self.main_progress.setmessage(msg)
-            progress = max(0, self.helper.tasknumber_current - 1)
-            content += self.main_progress.update(progress)
+            self.main_progress.setmessage(content)
+            progress = self.helper.tasknumber_current - 1
+            if progress < 0:
+                progress = 0
+            content = self.main_progress.update(progress)
             print('')
-        lines = self.getlines(content)
-        if not self.quiet:
-            for tasknum, task in enumerate(tasks[:(self.rows - 1 - lines)]):
+        lines = 1 + int(len(content) / (self.columns + 1))
+        if self.quiet == 0:
+            for tasknum, task in enumerate(tasks[:(self.rows - 2)]):
                 if isinstance(task, tuple):
-                    pbar, msg, progress, rate, start_time = task
+                    pbar, progress, rate, start_time = task
                     if not pbar.start_time:
                         pbar.start(False)
                         if start_time:
                             pbar.start_time = start_time
-                    pbar.setmessage('%s: %s' % (tasknum, msg))
+                    pbar.setmessage('%s:%s' % (tasknum, pbar.msg.split(':', 1)[1]))
                     pbar.setextra(rate)
                     if progress > -1:
                         content = pbar.update(progress)
@@ -323,17 +311,11 @@ class TerminalFilter(object):
                 else:
                     content = "%s: %s" % (tasknum, task)
                     print(content)
-                lines = lines + self.getlines(content)
+                lines = lines + 1 + int(len(content) / (self.columns + 1))
         self.footer_present = lines
         self.lastpids = runningpids[:]
         self.lastcount = self.helper.tasknumber_current
 
-    def getlines(self, content):
-        lines = 0
-        for line in content.split("\n"):
-            lines = lines + 1 + int(len(line) / (self.columns + 1))
-        return lines
-
     def finish(self):
         if self.stdinbackup:
             fd = sys.stdin.fileno()
@@ -623,40 +605,26 @@ def main(server, eventHandler, params, tf = TerminalFilter):
     warnings = 0
     taskfailures = []
 
-    printintervaldelta = 10 * 60 # 10 minutes
-    printinterval = printintervaldelta
-    pinginterval = 1 * 60 # 1 minute
-    lastevent = lastprint = time.time()
+    printinterval = 5000
+    lastprint = time.time()
 
     termfilter = tf(main, helper, console_handlers, params.options.quiet)
     atexit.register(termfilter.finish)
 
-    # shutdown levels
-    # 0 - normal operation
-    # 1 - no new task execution, let current running tasks finish
-    # 2 - interrupting currently executing tasks
-    # 3 - we're done, exit
-    while main.shutdown < 3:
+    while True:
         try:
             if (lastprint + printinterval) <= time.time():
                 termfilter.keepAlive(printinterval)
-                printinterval += printintervaldelta
+                printinterval += 5000
             event = eventHandler.waitEvent(0)
             if event is None:
-                if (lastevent + pinginterval) <= time.time():
-                    ret, error = server.runCommand(["ping"])
-                    if error or not ret:
-                        termfilter.clearFooter()
-                        print("No reply after pinging server (%s, %s), exiting." % (str(error), str(ret)))
-                        return_value = 3
-                        main.shutdown = 3
-                    lastevent = time.time()
+                if main.shutdown > 1:
+                    break
                 if not parseprogress:
                     termfilter.updateFooter()
                 event = eventHandler.waitEvent(0.25)
                 if event is None:
                     continue
-            lastevent = time.time()
             helper.eventHandler(event)
             if isinstance(event, bb.runqueue.runQueueExitWait):
                 if not main.shutdown:
@@ -678,8 +646,8 @@ def main(server, eventHandler, params, tf = TerminalFilter):
 
             if isinstance(event, logging.LogRecord):
                 lastprint = time.time()
-                printinterval = printintervaldelta
-                if event.levelno >= bb.msg.BBLogFormatter.ERRORONCE:
+                printinterval = 5000
+                if event.levelno >= bb.msg.BBLogFormatter.ERROR:
                     errors = errors + 1
                     return_value = 1
                 elif event.levelno == bb.msg.BBLogFormatter.WARNING:
@@ -693,10 +661,10 @@ def main(server, eventHandler, params, tf = TerminalFilter):
                         continue
 
                     # Prefix task messages with recipe/task
-                    if event.taskpid in helper.pidmap and event.levelno not in [bb.msg.BBLogFormatter.PLAIN, bb.msg.BBLogFormatter.WARNONCE, bb.msg.BBLogFormatter.ERRORONCE]:
+                    if event.taskpid in helper.pidmap and event.levelno != bb.msg.BBLogFormatter.PLAIN:
                         taskinfo = helper.running_tasks[helper.pidmap[event.taskpid]]
                         event.msg = taskinfo['title'] + ': ' + event.msg
-                if hasattr(event, 'fn') and event.levelno not in [bb.msg.BBLogFormatter.WARNONCE, bb.msg.BBLogFormatter.ERRORONCE]:
+                if hasattr(event, 'fn'):
                     event.msg = event.fn + ': ' + event.msg
                 logging.getLogger(event.name).handle(event)
                 continue
@@ -761,15 +729,15 @@ def main(server, eventHandler, params, tf = TerminalFilter):
                 if event.error:
                     errors = errors + 1
                     logger.error(str(event))
-                main.shutdown = 3
+                main.shutdown = 2
                 continue
             if isinstance(event, bb.command.CommandExit):
                 if not return_value:
                     return_value = event.exitcode
-                main.shutdown = 3
+                main.shutdown = 2
                 continue
             if isinstance(event, (bb.command.CommandCompleted, bb.cooker.CookerExit)):
-                main.shutdown = 3
+                main.shutdown = 2
                 continue
             if isinstance(event, bb.event.MultipleProviders):
                 logger.info(str(event))
@@ -890,6 +858,7 @@ def main(server, eventHandler, params, tf = TerminalFilter):
                     state_force_shutdown()
 
             main.shutdown = main.shutdown + 1
+            pass
         except Exception as e:
             import traceback
             sys.stderr.write(traceback.format_exc())
@@ -906,11 +875,11 @@ def main(server, eventHandler, params, tf = TerminalFilter):
             for failure in taskfailures:
                 summary += "\n  %s" % failure
         if warnings:
-            summary += pluralise("\nSummary: There was %s WARNING message.",
-                                 "\nSummary: There were %s WARNING messages.", warnings)
+            summary += pluralise("\nSummary: There was %s WARNING message shown.",
+                                 "\nSummary: There were %s WARNING messages shown.", warnings)
         if return_value and errors:
-            summary += pluralise("\nSummary: There was %s ERROR message, returning a non-zero exit code.",
-                                 "\nSummary: There were %s ERROR messages, returning a non-zero exit code.", errors)
+            summary += pluralise("\nSummary: There was %s ERROR message shown, returning a non-zero exit code.",
+                                 "\nSummary: There were %s ERROR messages shown, returning a non-zero exit code.", errors)
         if summary and params.options.quiet == 0:
             print(summary)
 

bitbake/lib/bb/ui/taskexp.py

@@ -177,7 +177,7 @@ class gtkthread(threading.Thread):
     quit = threading.Event()
     def __init__(self, shutdown):
         threading.Thread.__init__(self)
-        self.daemon = True
+        self.setDaemon(True)
         self.shutdown = shutdown
         if not Gtk.init_check()[0]:
             sys.stderr.write("Gtk+ init failed. Make sure DISPLAY variable is set.\n")

bitbake/lib/bb/ui/uievent.py

@@ -44,7 +44,7 @@ class BBUIEventQueue:
         for count_tries in range(5):
             ret = self.BBServer.registerEventHandler(self.host, self.port)
 
-            if isinstance(ret, collections.abc.Iterable):
+            if isinstance(ret, collections.Iterable):
                 self.EventHandle, error = ret
             else:
                 self.EventHandle = ret
@@ -65,27 +65,35 @@ class BBUIEventQueue:
         self.server = server
 
         self.t = threading.Thread()
-        self.t.daemon = True
+        self.t.setDaemon(True)
         self.t.run = self.startCallbackHandler
         self.t.start()
 
     def getEvent(self):
-        with bb.utils.lock_timeout(self.eventQueueLock):
-            if not self.eventQueue:
-                return None
-            item = self.eventQueue.pop(0)
-            if not self.eventQueue:
-                self.eventQueueNotify.clear()
-            return item
+
+        self.eventQueueLock.acquire()
+
+        if len(self.eventQueue) == 0:
+            self.eventQueueLock.release()
+            return None
+
+        item = self.eventQueue.pop(0)
+
+        if len(self.eventQueue) == 0:
+            self.eventQueueNotify.clear()
+
+        self.eventQueueLock.release()
+        return item
 
     def waitEvent(self, delay):
         self.eventQueueNotify.wait(delay)
         return self.getEvent()
 
     def queue_event(self, event):
-        with bb.utils.lock_timeout(self.eventQueueLock):
-            self.eventQueue.append(event)
-            self.eventQueueNotify.set()
+        self.eventQueueLock.acquire()
+        self.eventQueue.append(event)
+        self.eventQueueNotify.set()
+        self.eventQueueLock.release()
 
     def send_event(self, event):
         self.queue_event(pickle.loads(event))

bitbake/lib/bb/utils.py

@@ -13,12 +13,10 @@ import errno
 import logging
 import bb
 import bb.msg
-import locale
 import multiprocessing
 import fcntl
 import importlib
-import importlib.machinery
-import importlib.util
+from importlib import machinery
 import itertools
 import subprocess
 import glob
@@ -28,11 +26,6 @@ import errno
 import signal
 import collections
 import copy
-import ctypes
-import random
-import socket
-import struct
-import tempfile
 from subprocess import getstatusoutput
 from contextlib import contextmanager
 from ctypes import cdll
@@ -258,7 +251,7 @@ def explode_dep_versions(s):
     """
     Take an RDEPENDS style string of format:
     "DEPEND1 (optional version) DEPEND2 (optional version) ..."
-    skip null value and items appeared in dependency string multiple times
+    skip null value and items appeared in dependancy string multiple times
     and return a dictionary of dependencies and versions.
     """
     r = explode_dep_versions2(s)
@@ -386,7 +379,7 @@ def _print_exception(t, value, tb, realfile, text, context):
 
         error.append("Exception: %s" % ''.join(exception))
 
-        # If the exception is from spawning a task, let's be helpful and display
+        # If the exception is from spwaning a task, let's be helpful and display
         # the output (which hopefully includes stderr).
         if isinstance(value, subprocess.CalledProcessError) and value.output:
             error.append("Subprocess output:")
@@ -407,7 +400,7 @@ def better_exec(code, context, text = None, realfile = "<code>", pythonexception
         code = better_compile(code, realfile, realfile)
     try:
         exec(code, get_context(), context)
-    except (bb.BBHandledException, bb.parse.SkipRecipe, bb.data_smart.ExpansionError, bb.process.ExecutionError):
+    except (bb.BBHandledException, bb.parse.SkipRecipe, bb.data_smart.ExpansionError):
         # Error already shown so passthrough, no need for traceback
         raise
     except Exception as e:
@@ -434,14 +427,12 @@ def better_eval(source, locals, extraglobals = None):
     return eval(source, ctx, locals)
 
 @contextmanager
-def fileslocked(files, *args, **kwargs):
+def fileslocked(files):
     """Context manager for locking and unlocking file locks."""
     locks = []
     if files:
         for lockfile in files:
-            l = bb.utils.lockfile(lockfile, *args, **kwargs)
-            if l is not None:
-                locks.append(l)
+            locks.append(bb.utils.lockfile(lockfile))
 
     try:
         yield
@@ -460,16 +451,13 @@ def lockfile(name, shared=False, retry=True, block=False):
     consider the possibility of sending a signal to the process to break
     out - at which point you want block=True rather than retry=True.
     """
-    basename = os.path.basename(name)
-    if len(basename) > 255:
-        root, ext = os.path.splitext(basename)
-        basename = root[:255 - len(ext)] + ext
+    if len(name) > 255:
+        root, ext = os.path.splitext(name)
+        name = root[:255 - len(ext)] + ext
 
     dirname = os.path.dirname(name)
     mkdirhier(dirname)
 
-    name = os.path.join(dirname, basename)
-
     if not os.access(dirname, os.W_OK):
         logger.error("Unable to acquire lock '%s', directory is not writable",
                      name)
@@ -548,12 +536,7 @@ def md5_file(filename):
     Return the hex string representation of the MD5 checksum of filename.
     """
     import hashlib
-    try:
-        sig = hashlib.new('MD5', usedforsecurity=False)
-    except TypeError:
-        # Some configurations don't appear to support two arguments
-        sig = hashlib.new('MD5')
-    return _hasher(sig, filename)
+    return _hasher(hashlib.md5(), filename)
 
 def sha256_file(filename):
     """
@@ -604,26 +587,11 @@ def preserved_envvars():
     v = [
         'BBPATH',
         'BB_PRESERVE_ENV',
-        'BB_ENV_PASSTHROUGH',
-        'BB_ENV_PASSTHROUGH_ADDITIONS',
+        'BB_ENV_WHITELIST',
+        'BB_ENV_EXTRAWHITE',
     ]
     return v + preserved_envvars_exported()
 
-def check_system_locale():
-    """Make sure the required system locale are available and configured"""
-    default_locale = locale.getlocale(locale.LC_CTYPE)
-
-    try:
-        locale.setlocale(locale.LC_CTYPE, ("en_US", "UTF-8"))
-    except:
-        sys.exit("Please make sure locale 'en_US.UTF-8' is available on your system")
-    else:
-        locale.setlocale(locale.LC_CTYPE, default_locale)
-
-    if sys.getfilesystemencoding() != "utf-8":
-        sys.exit("Please use a locale setting which supports UTF-8 (such as LANG=en_US.UTF-8).\n"
-                 "Python can't change the filesystem locale after loading so we need a UTF-8 when Python starts or things won't work.")
-
 def filter_environment(good_vars):
     """
     Create a pristine environment for bitbake. This will remove variables that
@@ -651,21 +619,21 @@ def filter_environment(good_vars):
 
 def approved_variables():
     """
-    Determine and return the list of variables which are approved
+    Determine and return the list of whitelisted variables which are approved
     to remain in the environment.
     """
     if 'BB_PRESERVE_ENV' in os.environ:
         return os.environ.keys()
     approved = []
-    if 'BB_ENV_PASSTHROUGH' in os.environ:
-        approved = os.environ['BB_ENV_PASSTHROUGH'].split()
-        approved.extend(['BB_ENV_PASSTHROUGH'])
+    if 'BB_ENV_WHITELIST' in os.environ:
+        approved = os.environ['BB_ENV_WHITELIST'].split()
+        approved.extend(['BB_ENV_WHITELIST'])
     else:
         approved = preserved_envvars()
-    if 'BB_ENV_PASSTHROUGH_ADDITIONS' in os.environ:
-        approved.extend(os.environ['BB_ENV_PASSTHROUGH_ADDITIONS'].split())
-        if 'BB_ENV_PASSTHROUGH_ADDITIONS' not in approved:
-            approved.extend(['BB_ENV_PASSTHROUGH_ADDITIONS'])
+    if 'BB_ENV_EXTRAWHITE' in os.environ:
+        approved.extend(os.environ['BB_ENV_EXTRAWHITE'].split())
+        if 'BB_ENV_EXTRAWHITE' not in approved:
+            approved.extend(['BB_ENV_EXTRAWHITE'])
     return approved
 
 def clean_environment():
@@ -719,8 +687,8 @@ def remove(path, recurse=False, ionice=False):
         return
     if recurse:
         for name in glob.glob(path):
-            if _check_unsafe_delete_path(name):
-                raise Exception('bb.utils.remove: called with dangerous path "%s" and recurse=True, refusing to delete!' % name)
+            if _check_unsafe_delete_path(path):
+                raise Exception('bb.utils.remove: called with dangerous path "%s" and recurse=True, refusing to delete!' % path)
         # shutil.rmtree(name) would be ideal but its too slow
         cmd = []
         if ionice:
@@ -778,7 +746,7 @@ def movefile(src, dest, newmtime = None, sstat = None):
         if not sstat:
             sstat = os.lstat(src)
     except Exception as e:
-        logger.warning("movefile: Stating source file failed...", e)
+        print("movefile: Stating source file failed...", e)
         return None
 
     destexists = 1
@@ -806,7 +774,7 @@ def movefile(src, dest, newmtime = None, sstat = None):
             os.unlink(src)
             return os.lstat(dest)
         except Exception as e:
-            logger.warning("movefile: failed to properly create symlink:", dest, "->", target, e)
+            print("movefile: failed to properly create symlink:", dest, "->", target, e)
             return None
 
     renamefailed = 1
@@ -823,7 +791,7 @@ def movefile(src, dest, newmtime = None, sstat = None):
         except Exception as e:
             if e.errno != errno.EXDEV:
                 # Some random error.
-                logger.warning("movefile: Failed to move", src, "to", dest, e)
+                print("movefile: Failed to move", src, "to", dest, e)
                 return None
             # Invalid cross-device-link 'bind' mounted or actually Cross-Device
 
@@ -835,13 +803,13 @@ def movefile(src, dest, newmtime = None, sstat = None):
                 bb.utils.rename(destpath + "#new", destpath)
                 didcopy = 1
             except Exception as e:
-                logger.warning('movefile: copy', src, '->', dest, 'failed.', e)
+                print('movefile: copy', src, '->', dest, 'failed.', e)
                 return None
         else:
             #we don't yet handle special, so we need to fall back to /bin/mv
             a = getstatusoutput("/bin/mv -f " + "'" + src + "' '" + dest + "'")
             if a[0] != 0:
-                logger.warning("movefile: Failed to move special file:" + src + "' to '" + dest + "'", a)
+                print("movefile: Failed to move special file:" + src + "' to '" + dest + "'", a)
                 return None # failure
         try:
             if didcopy:
@@ -849,7 +817,7 @@ def movefile(src, dest, newmtime = None, sstat = None):
                 os.chmod(destpath, stat.S_IMODE(sstat[stat.ST_MODE])) # Sticky is reset on chown
                 os.unlink(src)
         except Exception as e:
-            logger.warning("movefile: Failed to chown/chmod/unlink", dest, e)
+            print("movefile: Failed to chown/chmod/unlink", dest, e)
             return None
 
     if newmtime:
@@ -1008,9 +976,6 @@ def to_boolean(string, default=None):
     if not string:
         return default
 
-    if isinstance(string, int):
-        return string != 0
-
     normalized = string.lower()
     if normalized in ("y", "yes", "1", "true"):
         return True
@@ -1629,89 +1594,33 @@ def set_process_name(name):
     except:
         pass
 
-def enable_loopback_networking():
-    # From bits/ioctls.h
-    SIOCGIFFLAGS = 0x8913
-    SIOCSIFFLAGS = 0x8914
-    SIOCSIFADDR = 0x8916
-    SIOCSIFNETMASK = 0x891C
-
-    # if.h
-    IFF_UP = 0x1
-    IFF_RUNNING = 0x40
-
-    # bits/socket.h
-    AF_INET = 2
-
-    # char ifr_name[IFNAMSIZ=16]
-    ifr_name = struct.pack("@16s", b"lo")
-    def netdev_req(fd, req, data = b""):
-        # Pad and add interface name
-        data = ifr_name + data + (b'\x00' * (16 - len(data)))
-        # Return all data after interface name
-        return fcntl.ioctl(fd, req, data)[16:]
-
-    with socket.socket(socket.AF_INET, socket.SOCK_DGRAM, socket.IPPROTO_IP) as sock:
-        fd = sock.fileno()
-
-        # struct sockaddr_in ifr_addr { unsigned short family; uint16_t sin_port ; uint32_t in_addr; }
-        req = struct.pack("@H", AF_INET) + struct.pack("=H4B", 0, 127, 0, 0, 1)
-        netdev_req(fd, SIOCSIFADDR, req)
-
-        # short ifr_flags
-        flags = struct.unpack_from('@h', netdev_req(fd, SIOCGIFFLAGS))[0]
-        flags |= IFF_UP | IFF_RUNNING
-        netdev_req(fd, SIOCSIFFLAGS, struct.pack('@h', flags))
-
-        # struct sockaddr_in ifr_netmask
-        req = struct.pack("@H", AF_INET) + struct.pack("=H4B", 0, 255, 0, 0, 0)
-        netdev_req(fd, SIOCSIFNETMASK, req)
-
-def disable_network(uid=None, gid=None):
-    """
-    Disable networking in the current process if the kernel supports it, else
-    just return after logging to debug. To do this we need to create a new user
-    namespace, then map back to the original uid/gid.
-    """
-    libc = ctypes.CDLL('libc.so.6')
-
-    # From sched.h
-    # New user namespace
-    CLONE_NEWUSER = 0x10000000
-    # New network namespace
-    CLONE_NEWNET = 0x40000000
-
-    if uid is None:
-        uid = os.getuid()
-    if gid is None:
-        gid = os.getgid()
-
-    ret = libc.unshare(CLONE_NEWNET | CLONE_NEWUSER)
-    if ret != 0:
-        logger.debug("System doesn't support disabling network without admin privs")
-        return
-    with open("/proc/self/uid_map", "w") as f:
-        f.write("%s %s 1" % (uid, uid))
-    with open("/proc/self/setgroups", "w") as f:
-        f.write("deny")
-    with open("/proc/self/gid_map", "w") as f:
-        f.write("%s %s 1" % (gid, gid))
-
 def export_proxies(d):
-    from bb.fetch2 import get_fetcher_environment
     """ export common proxies variables from datastore to environment """
-    newenv = get_fetcher_environment(d)
-    for v in newenv:
-        os.environ[v] = newenv[v]
+    import os
+
+    variables = ['http_proxy', 'HTTP_PROXY', 'https_proxy', 'HTTPS_PROXY',
+                    'ftp_proxy', 'FTP_PROXY', 'no_proxy', 'NO_PROXY',
+                    'GIT_PROXY_COMMAND']
+    exported = False
+
+    for v in variables:
+        if v in os.environ.keys():
+            exported = True
+        else:
+            v_proxy = d.getVar(v)
+            if v_proxy is not None:
+                os.environ[v] = v_proxy
+                exported = True
+
+    return exported
+
 
 def load_plugins(logger, plugins, pluginpath):
     def load_plugin(name):
         logger.debug('Loading plugin %s' % name)
         spec = importlib.machinery.PathFinder.find_spec(name, path=[pluginpath] )
         if spec:
-            mod = importlib.util.module_from_spec(spec)
-            spec.loader.exec_module(mod)
-            return mod
+            return spec.loader.load_module()
 
     logger.debug('Loading plugins from %s...' % pluginpath)
 
@@ -1790,53 +1699,5 @@ def environment(**envvars):
         for var in envvars:
             if var in backup:
                 os.environ[var] = backup[var]
-            elif var in os.environ:
+            else:
                 del os.environ[var]
-
-def is_local_uid(uid=''):
-    """
-    Check whether uid is a local one or not.
-    Can't use pwd module since it gets all UIDs, not local ones only.
-    """
-    if not uid:
-        uid = os.getuid()
-    with open('/etc/passwd', 'r') as f:
-        for line in f:
-            line_split = line.split(':')
-            if len(line_split) < 3:
-                continue
-            if str(uid) == line_split[2]:
-                return True
-    return False
-
-def mkstemp(suffix=None, prefix=None, dir=None, text=False):
-    """
-    Generates a unique filename, independent of time.
-
-    mkstemp() in glibc (at least) generates unique file names based on the
-    current system time. When combined with highly parallel builds, and
-    operating over NFS (e.g. shared sstate/downloads) this can result in
-    conflicts and race conditions.
-
-    This function adds additional entropy to the file name so that a collision
-    is independent of time and thus extremely unlikely.
-    """
-    entropy = "".join(random.choices("abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890", k=20))
-    if prefix:
-        prefix = prefix + entropy
-    else:
-        prefix = tempfile.gettempprefix() + entropy
-    return tempfile.mkstemp(suffix=suffix, prefix=prefix, dir=dir, text=text)
-
-# If we don't have a timeout of some kind and a process/thread exits badly (for example
-# OOM killed) and held a lock, we'd just hang in the lock futex forever. It is better
-# we exit at some point than hang. 5 minutes with no progress means we're probably deadlocked.
-@contextmanager
-def lock_timeout(lock):
-    held = lock.acquire(timeout=5*60)
-    try:
-        if not held:
-            os._exit(1)
-        yield held
-    finally:
-        lock.release()

bitbake/lib/bb/xattr.py

@@ -1,126 +0,0 @@
-#! /usr/bin/env python3
-#
-# Copyright 2023 by Garmin Ltd. or its subsidiaries
-#
-# SPDX-License-Identifier: MIT
-
-import sys
-import ctypes
-import os
-import errno
-
-libc = ctypes.CDLL("libc.so.6", use_errno=True)
-fsencoding = sys.getfilesystemencoding()
-
-
-libc.listxattr.argtypes = [ctypes.c_char_p, ctypes.c_char_p, ctypes.c_size_t]
-libc.llistxattr.argtypes = [ctypes.c_char_p, ctypes.c_char_p, ctypes.c_size_t]
-
-
-def listxattr(path, follow=True):
-    func = libc.listxattr if follow else libc.llistxattr
-
-    os_path = os.fsencode(path)
-
-    while True:
-        length = func(os_path, None, 0)
-
-        if length < 0:
-            err = ctypes.get_errno()
-            raise OSError(err, os.strerror(err), str(path))
-
-        if length == 0:
-            return []
-
-        arr = ctypes.create_string_buffer(length)
-
-        read_length = func(os_path, arr, length)
-        if read_length != length:
-            # Race!
-            continue
-
-        return [a.decode(fsencoding) for a in arr.raw.split(b"\x00") if a]
-
-
-libc.getxattr.argtypes = [
-    ctypes.c_char_p,
-    ctypes.c_char_p,
-    ctypes.c_char_p,
-    ctypes.c_size_t,
-]
-libc.lgetxattr.argtypes = [
-    ctypes.c_char_p,
-    ctypes.c_char_p,
-    ctypes.c_char_p,
-    ctypes.c_size_t,
-]
-
-
-def getxattr(path, name, follow=True):
-    func = libc.getxattr if follow else libc.lgetxattr
-
-    os_path = os.fsencode(path)
-    os_name = os.fsencode(name)
-
-    while True:
-        length = func(os_path, os_name, None, 0)
-
-        if length < 0:
-            err = ctypes.get_errno()
-            if err == errno.ENODATA:
-                return None
-            raise OSError(err, os.strerror(err), str(path))
-
-        if length == 0:
-            return ""
-
-        arr = ctypes.create_string_buffer(length)
-
-        read_length = func(os_path, os_name, arr, length)
-        if read_length != length:
-            # Race!
-            continue
-
-        return arr.raw
-
-
-def get_all_xattr(path, follow=True):
-    attrs = {}
-
-    names = listxattr(path, follow)
-
-    for name in names:
-        value = getxattr(path, name, follow)
-        if value is None:
-            # This can happen if a value is erased after listxattr is called,
-            # so ignore it
-            continue
-        attrs[name] = value
-
-    return attrs
-
-
-def main():
-    import argparse
-    from pathlib import Path
-
-    parser = argparse.ArgumentParser()
-    parser.add_argument("path", help="File Path", type=Path)
-
-    args = parser.parse_args()
-
-    attrs = get_all_xattr(args.path)
-
-    for name, value in attrs.items():
-        try:
-            value = value.decode(fsencoding)
-        except UnicodeDecodeError:
-            pass
-
-        print(f"{name} = {value}")
-
-    return 0
-
-
-if __name__ == "__main__":
-    sys.exit(main())

bitbake/lib/bblayers/__init__.py

@@ -1,6 +1,4 @@
 #
-# Copyright BitBake Contributors
-#
 # SPDX-License-Identifier: GPL-2.0-only
 #
 

bitbake/lib/bblayers/action.py

@@ -1,6 +1,4 @@
 #
-# Copyright BitBake Contributors
-#
 # SPDX-License-Identifier: GPL-2.0-only
 #
 
@@ -11,7 +9,6 @@ import shutil
 import sys
 import tempfile
 
-from bb.cookerdata import findTopdir
 import bb.utils
 
 from bblayers.common import LayerPlugin
@@ -38,7 +35,7 @@ class ActionPlugin(LayerPlugin):
                 sys.stderr.write("Specified layer directory %s doesn't contain a conf/layer.conf file\n" % layerdir)
                 return 1
 
-        bblayers_conf = os.path.join(findTopdir(),'conf', 'bblayers.conf')
+        bblayers_conf = os.path.join('conf', 'bblayers.conf')
         if not os.path.exists(bblayers_conf):
             sys.stderr.write("Unable to find bblayers.conf\n")
             return 1
@@ -56,7 +53,7 @@ class ActionPlugin(LayerPlugin):
                 except (bb.tinfoil.TinfoilUIException, bb.BBHandledException):
                     # Restore the back up copy of bblayers.conf
                     shutil.copy2(backup, bblayers_conf)
-                    bb.fatal("Parse failure with the specified layer added, exiting.")
+                    bb.fatal("Parse failure with the specified layer added, aborting.")
                 else:
                     for item in notadded:
                         sys.stderr.write("Specified layer %s is already in BBLAYERS\n" % item)
@@ -66,7 +63,7 @@ class ActionPlugin(LayerPlugin):
 
     def do_remove_layer(self, args):
         """Remove one or more layers from bblayers.conf."""
-        bblayers_conf = os.path.join(findTopdir() ,'conf', 'bblayers.conf')
+        bblayers_conf = os.path.join('conf', 'bblayers.conf')
         if not os.path.exists(bblayers_conf):
             sys.stderr.write("Unable to find bblayers.conf\n")
             return 1