build-appliance-image: Update to walnascar head revision

(From OE-Core rev: 347cb0861dde58613541ce692778f907943a60ea)

Signed-off-by: Steve Sakoman <steve@sakoman.com>

.b4-config

@@ -0,0 +1,4 @@
+[b4]
+  prep-perpatch-check-cmd = ./scripts/b4-wrapper-poky.py prep-perpatch-check-cmd
+  send-auto-cc-cmd = ./scripts/b4-wrapper-poky.py send-auto-cc-cmd
+  send-auto-to-cmd = ./scripts/b4-wrapper-poky.py send-auto-to-cmd

MAINTAINERS.md

@@ -67,5 +67,3 @@ Shadow maintainers/development needed
 
 * toaster
 * bitbake
-
-

README.OE-Core.md

@@ -6,28 +6,27 @@ of OpenEmbedded. It is distro-less (can build a functional image with
 DISTRO = "nodistro") and contains only emulated machine support.
 
 For information about OpenEmbedded, see the OpenEmbedded website:
-    https://www.openembedded.org/
+<https://www.openembedded.org/>
 
 The Yocto Project has extensive documentation about OE including a reference manual
 which can be found at:
-    https://docs.yoctoproject.org/
-
+<https://docs.yoctoproject.org/>
 
 Contributing
 ------------
 
-Please refer to our contributor guide here: https://docs.yoctoproject.org/dev/contributor-guide/
+Please refer to our contributor guide here: <https://docs.yoctoproject.org/dev/contributor-guide/>
 for full details on how to submit changes.
 
 As a quick guide, patches should be sent to openembedded-core@lists.openembedded.org
 The git command to do that would be:
 
-     git send-email -M -1 --to openembedded-core@lists.openembedded.org
+```
+git send-email -M -1 --to openembedded-core@lists.openembedded.org
+```
 
 Mailing list:
-
-    https://lists.openembedded.org/g/openembedded-core
+<https://lists.openembedded.org/g/openembedded-core>
 
 Source code:
-
-    https://git.openembedded.org/openembedded-core/
+<https://git.openembedded.org/openembedded-core/>

README.qemu.md

@@ -5,10 +5,10 @@ To simplify development, the build system supports building images to
 work with the QEMU emulator in system emulation mode. Several architectures
 are currently supported in 32 and 64 bit variants:
 
-  * ARM (qemuarm + qemuarm64)
-  * x86 (qemux86 + qemux86-64)
-  * PowerPC (qemuppc only)
-  * MIPS (qemumips + qemumips64)
+* ARM (qemuarm + qemuarm64)
+* x86 (qemux86 + qemux86-64)
+* PowerPC (qemuppc only)
+* MIPS (qemumips + qemumips64)
 
 Use of the QEMU images is covered in the Yocto Project Reference Manual.
 The appropriate MACHINE variable value corresponding to the target is given

SECURITY.md

@@ -1,9 +1,9 @@
-How to Report a Potential Vulnerability?
-========================================
+How to Report a Potential Vulnerability
+=======================================
 
 If you would like to report a public issue (for example, one with a released
 CVE number), please report it using the
-[https://bugzilla.yoctoproject.org/enter_bug.cgi?product=Security Security Bugzilla]
+[Security Bugzilla](https://bugzilla.yoctoproject.org/enter_bug.cgi?product=Security)
 
 If you are dealing with a not-yet released or urgent issue, please send a
 message to security AT yoctoproject DOT org, including as many details as
@@ -13,10 +13,10 @@ and any example code, if available.
 Branches maintained with security fixes
 ---------------------------------------
 
-See [https://wiki.yoctoproject.org/wiki/Stable_Release_and_LTS Stable release and LTS]
+See [Stable release and LTS](https://wiki.yoctoproject.org/wiki/Stable_Release_and_LTS)
 for detailed info regarding the policies and maintenance of Stable branches.
 
-The [https://wiki.yoctoproject.org/wiki/Releases Release page] contains a list of all
-releases of the Yocto Project. Versions in grey are no longer actively maintained with
-security patches, but well-tested patches may still be accepted for them for
-significant issues.
+The [Release page](https://wiki.yoctoproject.org/wiki/Releases) contains
+a list of all releases of the Yocto Project. Versions in grey are no longer
+actively maintained with security patches, but well-tested patches may still
+be accepted for them for significant issues.

bitbake/.b4-config

@@ -0,0 +1,4 @@
+[b4]
+  send-series-to = bitbake-devel@lists.openembedded.org
+  send-auto-cc-cmd = ./contrib/b4-wrapper-bitbake.py send-auto-cc-cmd
+  prep-pre-flight-checks = disable-needs-checking

bitbake/bin/bitbake

@@ -27,7 +27,7 @@ from bb.main import bitbake_main, BitBakeConfigParameters, BBMainException
 
 bb.utils.check_system_locale()
 
-__version__ = "2.9.1"
+__version__ = "2.12.0"
 
 if __name__ == "__main__":
     if __version__ != bb.__version__:

bitbake/bin/bitbake-config-build

@@ -0,0 +1 @@
+bitbake-layers

bitbake/bin/bitbake-getvar

@@ -16,6 +16,7 @@ bindir = os.path.dirname(__file__)
 topdir = os.path.dirname(bindir)
 sys.path[0:0] = [os.path.join(topdir, 'lib')]
 
+import bb.providers
 import bb.tinfoil
 
 if __name__ == "__main__":
@@ -40,10 +41,15 @@ if __name__ == "__main__":
     with bb.tinfoil.Tinfoil(tracking=True, setup_logging=not quiet) as tinfoil:
         if args.recipe:
             tinfoil.prepare(quiet=3 if quiet else 2)
-            d = tinfoil.parse_recipe(args.recipe)
+            try:
+                d = tinfoil.parse_recipe(args.recipe)
+            except bb.providers.NoProvider as e:
+                sys.exit(str(e))
         else:
             tinfoil.prepare(quiet=2, config_only=True)
-            d = tinfoil.config_data
+            # Expand keys and run anonymous functions to get identical result to
+            # "bitbake -e"
+            d = tinfoil.finalizeData()
 
         value = None
         if args.flag:

bitbake/bin/bitbake-hashclient

@@ -17,6 +17,7 @@ import warnings
 import netrc
 import json
 import statistics
+import textwrap
 warnings.simplefilter("default")
 
 try:
@@ -226,6 +227,27 @@ def main():
         print("New hashes marked: %d" % result["count"])
         return 0
 
+    def handle_gc_mark_stream(args, client):
+        stdin = (l.strip() for l in sys.stdin)
+        marked_hashes = 0
+
+        try:
+            result = client.gc_mark_stream(args.mark, stdin)
+            marked_hashes = result["count"]
+        except ConnectionError:
+            logger.warning(
+                "Server doesn't seem to support `gc-mark-stream`. Sending "
+                "hashes sequentially using `gc-mark` API."
+            )
+            for line in stdin:
+                pairs = line.split()
+                condition = dict(zip(pairs[::2], pairs[1::2]))
+                result = client.gc_mark(args.mark, condition)
+                marked_hashes += result["count"]
+
+        print("New hashes marked: %d" % marked_hashes)
+        return 0
+
     def handle_gc_sweep(args, client):
         result = client.gc_sweep(args.mark)
         print("Removed %d rows" % result["count"])
@@ -265,7 +287,19 @@ def main():
         print(f"Max time is:             {max(times):.3f}s")
         return 0
 
-    parser = argparse.ArgumentParser(description='Hash Equivalence Client')
+    parser = argparse.ArgumentParser(
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+        description='Hash Equivalence Client',
+        epilog=textwrap.dedent(
+            """
+            Possible ADDRESS options are:
+                unix://PATH         Connect to UNIX domain socket at PATH
+                ws://HOST[:PORT]    Connect to websocket at HOST:PORT (default port is 80)
+                wss://HOST[:PORT]   Connect to secure websocket at HOST:PORT (default port is 443)
+                HOST:PORT           Connect to TCP server at HOST:PORT
+            """
+        ),
+    )
     parser.add_argument('--address', default=DEFAULT_ADDRESS, help='Server address (default "%(default)s")')
     parser.add_argument('--log', default='WARNING', help='Set logging level')
     parser.add_argument('--login', '-l', metavar="USERNAME", help="Authenticate as USERNAME")
@@ -353,6 +387,16 @@ def main():
                              help="Keep entries in table where KEY == VALUE")
     gc_mark_parser.set_defaults(func=handle_gc_mark)
 
+    gc_mark_parser_stream = subparsers.add_parser(
+        'gc-mark-stream',
+        help=(
+            "Mark multiple hashes to be retained for garbage collection. Input should be provided via stdin, "
+            "with each line formatted as key-value pairs separated by spaces, for example 'column1 foo column2 bar'."
+        )
+    )
+    gc_mark_parser_stream.add_argument("mark", help="Mark for this garbage collection operation")
+    gc_mark_parser_stream.set_defaults(func=handle_gc_mark_stream)
+
     gc_sweep_parser = subparsers.add_parser('gc-sweep', help="Perform garbage collection and delete any entries that are not marked")
     gc_sweep_parser.add_argument("mark", help="Mark for this garbage collection operation")
     gc_sweep_parser.set_defaults(func=handle_gc_sweep)

bitbake/bin/bitbake-layers

@@ -18,13 +18,14 @@ import warnings
 warnings.simplefilter("default")
 
 bindir = os.path.dirname(__file__)
+toolname = os.path.basename(__file__).split(".")[0]
 topdir = os.path.dirname(bindir)
 sys.path[0:0] = [os.path.join(topdir, 'lib')]
 
 import bb.tinfoil
 import bb.msg
 
-logger = bb.msg.logger_create('bitbake-layers', sys.stdout)
+logger = bb.msg.logger_create(toolname, sys.stdout)
 
 def main():
     parser = argparse.ArgumentParser(
@@ -57,17 +58,18 @@ def main():
                          level=logger.getEffectiveLevel())
 
     plugins = []
-    tinfoil = bb.tinfoil.Tinfoil(tracking=True)
-    tinfoil.logger.setLevel(logger.getEffectiveLevel())
-    if global_args.force > 1:
-        bbpaths = []
-    else:
-        tinfoil.prepare(True)
-        bbpaths = tinfoil.config_data.getVar('BBPATH').split(':')
-    
-    try: 
+    with bb.tinfoil.Tinfoil(tracking=True) as tinfoil:
+        tinfoil.logger.setLevel(logger.getEffectiveLevel())
+
+        if global_args.force > 1:
+            bbpaths = []
+        else:
+            tinfoil.prepare(True)
+            bbpaths = tinfoil.config_data.getVar('BBPATH').split(':')
+
         for path in ([topdir] + bbpaths):
-            pluginpath = os.path.join(path, 'lib', 'bblayers')
+            pluginbasepath = {"bitbake-layers":'bblayers', 'bitbake-config-build':'bbconfigbuild'}[toolname]
+            pluginpath = os.path.join(path, 'lib', pluginbasepath)
             bb.utils.load_plugins(logger, plugins, pluginpath)
 
         registered = False
@@ -90,8 +92,6 @@ def main():
             tinfoil.config_data.enableTracking()
 
         return args.func(args)
-    finally:
-        tinfoil.shutdown()
 
 
 if __name__ == "__main__":

bitbake/bin/bitbake-prserv

@@ -80,7 +80,7 @@ def main():
     parser.add_argument(
         "-u",
         "--upstream",
-        default=os.environ.get("PRSERVER_UPSTREAM", None),
+        default=os.environ.get("PRSERV_UPSTREAM", None),
         help="Upstream PR service (host:port)",
     )
 

bitbake/bin/bitbake-selftest

@@ -28,7 +28,6 @@ tests = ["bb.tests.codeparser",
          "bb.tests.event",
          "bb.tests.fetch",
          "bb.tests.parse",
-         "bb.tests.persist_data",
          "bb.tests.runqueue",
          "bb.tests.siggen",
          "bb.tests.utils",

bitbake/bin/bitbake-server

@@ -9,6 +9,7 @@ import os
 import sys
 import warnings
 warnings.simplefilter("default")
+warnings.filterwarnings("ignore", category=DeprecationWarning, message=".*use.of.fork.*may.lead.to.deadlocks.in.the.child.*")
 import logging
 sys.path.insert(0, os.path.join(os.path.dirname(os.path.dirname(sys.argv[0])), 'lib'))
 
@@ -38,9 +39,9 @@ if xmlrpcinterface[0] == "None":
 with open('/dev/null', 'r') as si:
     os.dup2(si.fileno(), sys.stdin.fileno())
 
-so = open(logfile, 'a+')
-os.dup2(so.fileno(), sys.stdout.fileno())
-os.dup2(so.fileno(), sys.stderr.fileno())
+with open(logfile, 'a+') as so:
+    os.dup2(so.fileno(), sys.stdout.fileno())
+    os.dup2(so.fileno(), sys.stderr.fileno())
 
 # Have stdout and stderr be the same so log output matches chronologically
 # and there aren't two seperate buffers

bitbake/bin/bitbake-worker

@@ -9,6 +9,7 @@ import os
 import sys
 import warnings
 warnings.simplefilter("default")
+warnings.filterwarnings("ignore", category=DeprecationWarning, message=".*use.of.fork.*may.lead.to.deadlocks.in.the.child.*")
 sys.path.insert(0, os.path.join(os.path.dirname(os.path.dirname(sys.argv[0])), 'lib'))
 from bb import fetch2
 import logging
@@ -21,9 +22,14 @@ import traceback
 import queue
 import shlex
 import subprocess
+import fcntl
 from multiprocessing import Lock
 from threading import Thread
 
+# Remove when we have a minimum of python 3.10
+if not hasattr(fcntl, 'F_SETPIPE_SZ'):
+    fcntl.F_SETPIPE_SZ = 1031
+
 bb.utils.check_system_locale()
 
 # Users shouldn't be running this code directly
@@ -44,7 +50,6 @@ if sys.argv[1].startswith("decafbadbad"):
 # updates to log files for use with tail
 try:
     if sys.stdout.name == '<stdout>':
-        import fcntl
         fl = fcntl.fcntl(sys.stdout.fileno(), fcntl.F_GETFL)
         fl |= os.O_SYNC 
         fcntl.fcntl(sys.stdout.fileno(), fcntl.F_SETFL, fl)
@@ -56,6 +61,12 @@ logger = logging.getLogger("BitBake")
 
 worker_pipe = sys.stdout.fileno()
 bb.utils.nonblockingfd(worker_pipe)
+# Try to make the pipe buffers larger as it is much more efficient. If we can't
+# e.g. out of buffer space (/proc/sys/fs/pipe-user-pages-soft) then just pass over.
+try:
+    fcntl.fcntl(worker_pipe, fcntl.F_SETPIPE_SZ, 512 * 1024)
+except:
+    pass
 # Need to guard against multiprocessing being used in child processes
 # and multiple processes trying to write to the parent at the same time
 worker_pipe_lock = None
@@ -105,7 +116,7 @@ def worker_flush(worker_queue):
                 if not worker_queue.empty():
                     worker_queue_int.extend(worker_queue.get())
                 written = os.write(worker_pipe, worker_queue_int)
-                worker_queue_int = worker_queue_int[written:]
+                del worker_queue_int[0:written]
             except (IOError, OSError) as e:
                 if e.errno != errno.EAGAIN and e.errno != errno.EPIPE:
                     raise
@@ -357,7 +368,7 @@ class runQueueWorkerPipe():
     def read(self):
         start = len(self.queue)
         try:
-            self.queue.extend(self.input.read(102400) or b"")
+            self.queue.extend(self.input.read(512*1024) or b"")
         except (OSError, IOError) as e:
             if e.errno != errno.EAGAIN:
                 raise

bitbake/bin/git-make-shallow

@@ -115,8 +115,8 @@ def filter_refs(refs):
     all_refs = get_all_refs()
     to_remove = set(all_refs) - set(refs)
     if to_remove:
-        check_output(['xargs', '-0', '-n', '1'] + git_cmd + ['update-ref', '-d', '--no-deref'],
-                     input=''.join(l + '\0' for l in to_remove))
+        check_output(git_cmd + ['update-ref', '--no-deref', '--stdin', '-z'],
+                     input=''.join('delete ' + l + '\0\0' for l in to_remove))
 
 
 def follow_history_intersections(revisions, refs):

bitbake/contrib/b4-wrapper-bitbake.py

@@ -0,0 +1,40 @@
+#!/usr/bin/env python3
+#
+# Copyright OpenEmbedded Contributors
+#
+# SPDX-License-Identifier: MIT
+#
+# This script is to be called by b4:
+# - through b4.send-auto-cc-cmd with "send-auto-cc-cmd" as first argument,
+#
+# When send-auto-cc-cmd is passed:
+#
+#  This returns the list of Cc recipients for a patch.
+#
+# This script takes as stdin a patch.
+
+import subprocess
+import sys
+
+cmd = sys.argv[1]
+if cmd != "send-auto-cc-cmd":
+    sys.exit(-1)
+
+patch = sys.stdin.read()
+
+if subprocess.call(["which", "lsdiff"], stdout=subprocess.DEVNULL) != 0:
+    print("lsdiff missing from host, please install patchutils")
+    sys.exit(-1)
+
+files = subprocess.check_output(["lsdiff", "--strip-match=1", "--strip=1", "--include=doc/*"],
+                                input=patch, text=True)
+if len(files):
+    print("docs@lists.yoctoproject.org")
+else:
+# Handle patches made with --no-prefix
+    files = subprocess.check_output(["lsdiff", "--include=doc/*"],
+                                    input=patch, text=True)
+    if len(files):
+        print("docs@lists.yoctoproject.org")
+
+sys.exit(0)

bitbake/contrib/hashserv/Dockerfile

@@ -11,7 +11,7 @@
 
 FROM alpine:3.13.1
 
-RUN apk add --no-cache python3
+RUN apk add --no-cache python3 libgcc
 
 COPY bin/bitbake-hashserv /opt/bbhashserv/bin/
 COPY lib/hashserv /opt/bbhashserv/lib/hashserv/

bitbake/contrib/vim/ftdetect/bitbake.vim

@@ -11,10 +11,18 @@ if &compatible || version < 600 || exists("b:loaded_bitbake_plugin")
 endif
 
 " .bb, .bbappend and .bbclass
-au BufNewFile,BufRead *.{bb,bbappend,bbclass}  set filetype=bitbake
+au BufNewFile,BufRead *.{bb,bbappend,bbclass}  setfiletype bitbake
 
-" .inc
-au BufNewFile,BufRead *.inc		set filetype=bitbake
+" .inc -- meanwhile included upstream
+if !has("patch-9.0.0055")
+    au BufNewFile,BufRead *.inc                call s:BBIncDetect()
+    def s:BBIncDetect()
+        l:lines = getline(1) .. getline(2) .. getline(3)
+        if l:lines =~# '\<\%(require\|inherit\)\>' || lines =~# '[A-Z][A-Za-z0-9_:${}]*\s\+\%(??\|[?:+]\)\?= '
+            set filetype bitbake
+        endif
+    enddef
+endif
 
 " .conf
 au BufNewFile,BufRead *.conf

bitbake/doc/bitbake-user-manual/bitbake-user-manual-intro.rst

@@ -349,40 +349,84 @@ Usage and syntax
 Following is the usage and syntax for BitBake::
 
    $ bitbake -h
-   Usage: bitbake [options] [recipename/target recipe:do_task ...]
+   usage: bitbake [-s] [-e] [-g] [-u UI] [--version] [-h] [-f] [-c CMD]
+                  [-C INVALIDATE_STAMP] [--runall RUNALL] [--runonly RUNONLY]
+                  [--no-setscene] [--skip-setscene] [--setscene-only] [-n] [-p]
+                  [-k] [-P] [-S SIGNATURE_HANDLER] [--revisions-changed]
+                  [-b BUILDFILE] [-D] [-l DEBUG_DOMAINS] [-v] [-q]
+                  [-w WRITEEVENTLOG] [-B BIND] [-T SERVER_TIMEOUT]
+                  [--remote-server REMOTE_SERVER] [-m] [--token XMLRPCTOKEN]
+                  [--observe-only] [--status-only] [--server-only] [-r PREFILE]
+                  [-R POSTFILE] [-I EXTRA_ASSUME_PROVIDED]
+                  [recipename/target ...]
 
-       Executes the specified task (default is 'build') for a given set of target recipes (.bb files).
-       It is assumed there is a conf/bblayers.conf available in cwd or in BBPATH which
-       will provide the layer, BBFILES and other configuration information.
+   It is assumed there is a conf/bblayers.conf available in cwd or in BBPATH
+   which will provide the layer, BBFILES and other configuration information.
 
-   Options:
-     --version             show program's version number and exit
-     -h, --help            show this help message and exit
-     -b BUILDFILE, --buildfile=BUILDFILE
-                           Execute tasks from a specific .bb recipe directly.
-                           WARNING: Does not handle any dependencies from other
-                           recipes.
+   General options:
+     recipename/target     Execute the specified task (default is 'build') for
+                           these target recipes (.bb files).
+     -s, --show-versions   Show current and preferred versions of all recipes.
+     -e, --environment     Show the global or per-recipe environment complete
+                           with information about where variables were
+                           set/changed.
+     -g, --graphviz        Save dependency tree information for the specified
+                           targets in the dot syntax.
+     -u UI, --ui UI        The user interface to use (knotty, ncurses, taskexp,
+                           taskexp_ncurses or teamcity - default knotty).
+     --version             Show programs version and exit.
+     -h, --help            Show this help message and exit.
+
+   Task control options:
+     -f, --force           Force the specified targets/task to run (invalidating
+                           any existing stamp file).
+     -c CMD, --cmd CMD     Specify the task to execute. The exact options
+                           available depend on the metadata. Some examples might
+                           be 'compile' or 'populate_sysroot' or 'listtasks' may
+                           give a list of the tasks available.
+     -C INVALIDATE_STAMP, --clear-stamp INVALIDATE_STAMP
+                           Invalidate the stamp for the specified task such as
+                           'compile' and then run the default task for the
+                           specified target(s).
+     --runall RUNALL       Run the specified task for any recipe in the taskgraph
+                           of the specified target (even if it wouldn't otherwise
+                           have run).
+     --runonly RUNONLY     Run only the specified task within the taskgraph of
+                           the specified targets (and any task dependencies those
+                           tasks may have).
+     --no-setscene         Do not run any setscene tasks. sstate will be ignored
+                           and everything needed, built.
+     --skip-setscene       Skip setscene tasks if they would be executed. Tasks
+                           previously restored from sstate will be kept, unlike
+                           --no-setscene.
+     --setscene-only       Only run setscene tasks, don't run any real tasks.
+
+   Execution control options:
+     -n, --dry-run         Don't execute, just go through the motions.
+     -p, --parse-only      Quit after parsing the BB recipes.
      -k, --continue        Continue as much as possible after an error. While the
                            target that failed and anything depending on it cannot
                            be built, as much as possible will be built before
                            stopping.
-     -f, --force           Force the specified targets/task to run (invalidating
-                           any existing stamp file).
-     -c CMD, --cmd=CMD     Specify the task to execute. The exact options
-                           available depend on the metadata. Some examples might
-                           be 'compile' or 'populate_sysroot' or 'listtasks' may
-                           give a list of the tasks available.
-     -C INVALIDATE_STAMP, --clear-stamp=INVALIDATE_STAMP
-                           Invalidate the stamp for the specified task such as
-                           'compile' and then run the default task for the
-                           specified target(s).
-     -r PREFILE, --read=PREFILE
-                           Read the specified file before bitbake.conf.
-     -R POSTFILE, --postread=POSTFILE
-                           Read the specified file after bitbake.conf.
-     -v, --verbose         Enable tracing of shell tasks (with 'set -x'). Also
-                           print bb.note(...) messages to stdout (in addition to
-                           writing them to ${T}/log.do_<task>).
+     -P, --profile         Profile the command and save reports.
+     -S SIGNATURE_HANDLER, --dump-signatures SIGNATURE_HANDLER
+                           Dump out the signature construction information, with
+                           no task execution. The SIGNATURE_HANDLER parameter is
+                           passed to the handler. Two common values are none and
+                           printdiff but the handler may define more/less. none
+                           means only dump the signature, printdiff means
+                           recursively compare the dumped signature with the most
+                           recent one in a local build or sstate cache (can be
+                           used to find out why tasks re-run when that is not
+                           expected)
+     --revisions-changed   Set the exit code depending on whether upstream
+                           floating revisions have changed or not.
+     -b BUILDFILE, --buildfile BUILDFILE
+                           Execute tasks from a specific .bb recipe directly.
+                           WARNING: Does not handle any dependencies from other
+                           recipes.
+
+   Logging/output control options:
      -D, --debug           Increase the debug level. You can specify this more
                            than once. -D sets the debug level to 1, where only
                            bb.debug(1, ...) messages are printed to stdout; -DD
@@ -392,65 +436,47 @@ Following is the usage and syntax for BitBake::
                            -D only affects output to stdout. All debug messages
                            are written to ${T}/log.do_taskname, regardless of the
                            debug level.
+     -l DEBUG_DOMAINS, --log-domains DEBUG_DOMAINS
+                           Show debug logging for the specified logging domains.
+     -v, --verbose         Enable tracing of shell tasks (with 'set -x'). Also
+                           print bb.note(...) messages to stdout (in addition to
+                           writing them to ${T}/log.do_<task>).
      -q, --quiet           Output less log message data to the terminal. You can
                            specify this more than once.
-     -n, --dry-run         Don't execute, just go through the motions.
-     -S SIGNATURE_HANDLER, --dump-signatures=SIGNATURE_HANDLER
-                           Dump out the signature construction information, with
-                           no task execution. The SIGNATURE_HANDLER parameter is
-                           passed to the handler. Two common values are none and
-                           printdiff but the handler may define more/less. none
-                           means only dump the signature, printdiff means compare
-                           the dumped signature with the cached one.
-     -p, --parse-only      Quit after parsing the BB recipes.
-     -s, --show-versions   Show current and preferred versions of all recipes.
-     -e, --environment     Show the global or per-recipe environment complete
-                           with information about where variables were
-                           set/changed.
-     -g, --graphviz        Save dependency tree information for the specified
-                           targets in the dot syntax.
-     -I EXTRA_ASSUME_PROVIDED, --ignore-deps=EXTRA_ASSUME_PROVIDED
-                           Assume these dependencies don't exist and are already
-                           provided (equivalent to ASSUME_PROVIDED). Useful to
-                           make dependency graphs more appealing
-     -l DEBUG_DOMAINS, --log-domains=DEBUG_DOMAINS
-                           Show debug logging for the specified logging domains
-     -P, --profile         Profile the command and save reports.
-     -u UI, --ui=UI        The user interface to use (knotty, ncurses, taskexp or
-                           teamcity - default knotty).
-     --token=XMLRPCTOKEN   Specify the connection token to be used when
-                           connecting to a remote server.
-     --revisions-changed   Set the exit code depending on whether upstream
-                           floating revisions have changed or not.
-     --server-only         Run bitbake without a UI, only starting a server
-                           (cooker) process.
-     -B BIND, --bind=BIND  The name/address for the bitbake xmlrpc server to bind
-                           to.
-     -T SERVER_TIMEOUT, --idle-timeout=SERVER_TIMEOUT
-                           Set timeout to unload bitbake server due to
-                           inactivity, set to -1 means no unload, default:
-                           Environment variable BB_SERVER_TIMEOUT.
-     --no-setscene         Do not run any setscene tasks. sstate will be ignored
-                           and everything needed, built.
-     --skip-setscene       Skip setscene tasks if they would be executed. Tasks
-                           previously restored from sstate will be kept, unlike
-                           --no-setscene
-     --setscene-only       Only run setscene tasks, don't run any real tasks.
-     --remote-server=REMOTE_SERVER
-                           Connect to the specified server.
-     -m, --kill-server     Terminate any running bitbake server.
-     --observe-only        Connect to a server as an observing-only client.
-     --status-only         Check the status of the remote bitbake server.
-     -w WRITEEVENTLOG, --write-log=WRITEEVENTLOG
+     -w WRITEEVENTLOG, --write-log WRITEEVENTLOG
                            Writes the event log of the build to a bitbake event
                            json file. Use '' (empty string) to assign the name
                            automatically.
-     --runall=RUNALL       Run the specified task for any recipe in the taskgraph
-                           of the specified target (even if it wouldn't otherwise
-                           have run).
-     --runonly=RUNONLY     Run only the specified task within the taskgraph of
-                           the specified targets (and any task dependencies those
-                           tasks may have).
+
+   Server options:
+     -B BIND, --bind BIND  The name/address for the bitbake xmlrpc server to bind
+                           to.
+     -T SERVER_TIMEOUT, --idle-timeout SERVER_TIMEOUT
+                           Set timeout to unload bitbake server due to
+                           inactivity, set to -1 means no unload, default:
+                           Environment variable BB_SERVER_TIMEOUT.
+     --remote-server REMOTE_SERVER
+                           Connect to the specified server.
+     -m, --kill-server     Terminate any running bitbake server.
+     --token XMLRPCTOKEN   Specify the connection token to be used when
+                           connecting to a remote server.
+     --observe-only        Connect to a server as an observing-only client.
+     --status-only         Check the status of the remote bitbake server.
+     --server-only         Run bitbake without a UI, only starting a server
+                           (cooker) process.
+
+   Configuration options:
+     -r PREFILE, --read PREFILE
+                           Read the specified file before bitbake.conf.
+     -R POSTFILE, --postread POSTFILE
+                           Read the specified file after bitbake.conf.
+     -I EXTRA_ASSUME_PROVIDED, --ignore-deps EXTRA_ASSUME_PROVIDED
+                           Assume these dependencies don't exist and are already
+                           provided (equivalent to ASSUME_PROVIDED). Useful to
+                           make dependency graphs more appealing.
+
+..
+    Bitbake help output generated with "stty columns 80; bin/bitbake -h"
 
 .. _bitbake-examples:
 

bitbake/doc/bitbake-user-manual/bitbake-user-manual-metadata.rst

@@ -754,7 +754,9 @@ share the task.
 This section presents the mechanisms BitBake provides to allow you to
 share functionality between recipes. Specifically, the mechanisms
 include ``include``, ``inherit``, :term:`INHERIT`, and ``require``
-directives.
+directives. There is also a higher-level abstraction called
+``configuration fragments`` that is enabled with ``addfragments``
+directive.
 
 Locating Include and Class Files
 --------------------------------
@@ -771,6 +773,8 @@ In order for include and class files to be found by BitBake, they need
 to be located in a "classes" subdirectory that can be found in
 :term:`BBPATH`.
 
+.. _ref-bitbake-user-manual-metadata-inherit:
+
 ``inherit`` Directive
 ---------------------
 
@@ -809,19 +813,43 @@ An advantage with the inherit directive as compared to both the
 :ref:`include <bitbake-user-manual/bitbake-user-manual-metadata:\`\`include\`\` directive>` and :ref:`require <bitbake-user-manual/bitbake-user-manual-metadata:\`\`require\`\` directive>`
 directives is that you can inherit class files conditionally. You can
 accomplish this by using a variable expression after the ``inherit``
-statement. Here is an example::
+statement.
 
-   inherit ${VARNAME}
+For inheriting classes conditionally, using the :ref:`inherit_defer
+<ref-bitbake-user-manual-metadata-inherit-defer>` directive is advised as
+:ref:`inherit_defer <ref-bitbake-user-manual-metadata-inherit-defer>` is
+evaluated at the end of parsing.
+
+.. _ref-bitbake-user-manual-metadata-inherit-defer:
+
+``inherit_defer`` Directive
+~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+The :ref:`inherit_defer <ref-bitbake-user-manual-metadata-inherit-defer>`
+directive works like the :ref:`inherit
+<ref-bitbake-user-manual-metadata-inherit>` directive, except that it is only
+evaluated at the end of parsing. Its usage is recommended when a conditional
+expression is used.
+
+This allows conditional expressions to be evaluated "late", meaning changes to
+the variable after the line is parsed will take effect. With the :ref:`inherit
+<ref-bitbake-user-manual-metadata-inherit>` directive this is not the case.
+
+Here is an example::
+
+   inherit_defer ${VARNAME}
 
 If ``VARNAME`` is
-going to be set, it needs to be set before the ``inherit`` statement is
+going to be set, it needs to be set before the ``inherit_defer`` statement is
 parsed. One way to achieve a conditional inherit in this case is to use
 overrides::
 
    VARIABLE = ""
    VARIABLE:someoverride = "myclass"
 
-Another method is by using anonymous Python. Here is an example::
+Another method is by using :ref:`anonymous Python
+<bitbake-user-manual/bitbake-user-manual-metadata:Anonymous Python Functions>`.
+Here is an example::
 
    python () {
        if condition == value:
@@ -830,11 +858,14 @@ Another method is by using anonymous Python. Here is an example::
            d.setVar('VARIABLE', '')
    }
 
-Alternatively, you could use an in-line Python expression in the
+Alternatively, you could use an inline Python expression in the
 following form::
 
-   inherit ${@'classname' if condition else ''}
-   inherit ${@functionname(params)}
+   inherit_defer ${@'classname' if condition else ''}
+
+Or::
+
+   inherit_defer ${@bb.utils.contains('VARIABLE', 'something', 'classname', '', d)}
 
 In all cases, if the expression evaluates to an
 empty string, the statement does not trigger a syntax error because it
@@ -869,6 +900,33 @@ definitions::
    of include . Doing so makes sure that an error is produced if the file cannot
    be found.
 
+``include_all`` Directive
+-------------------------
+
+The ``include_all`` directive works like the :ref:`include
+<bitbake-user-manual/bitbake-user-manual-metadata:\`\`include\`\` directive>`
+directive but will include all of the files that match the specified path in
+the enabled layers (layers part of :term:`BBLAYERS`).
+
+For example, let's say a ``maintainers.inc`` file is present in different layers
+and is conventionally placed in the ``conf/distro/include`` directory of each
+layer. In that case the ``include_all`` directive can be used to include
+the ``maintainers.inc`` file for all of these layers::
+
+   include_all conf/distro/include/maintainers.inc
+
+In other words, the ``maintainers.inc`` file for each layer is included through
+the :ref:`include <bitbake-user-manual/bitbake-user-manual-metadata:\`\`include\`\` directive>`
+directive.
+
+BitBake will iterate through the colon-separated :term:`BBPATH` list to look for
+matching files to include, from left to right. As a consequence, matching files
+are included in that order.
+
+As the ``include_all`` directive uses the :ref:`include
+<bitbake-user-manual/bitbake-user-manual-metadata:\`\`include\`\` directive>`
+directive in the background, no error is produced if no files are matched.
+
 .. _require-inclusion:
 
 ``require`` Directive
@@ -933,6 +991,50 @@ the ``autotools`` and ``pkgconfig`` classes::
 
    INHERIT += "autotools pkgconfig"
 
+``addfragments`` Directive
+--------------------------
+
+This directive allows fine-tuning local configurations with configuration
+snippets contained in layers in a structured, controlled way. Typically it would
+go into ``bitbake.conf``, for example::
+
+   addfragments conf/fragments OE_FRAGMENTS OE_FRAGMENTS_METADATA_VARS
+
+``addfragments`` takes three parameters:
+
+-  path prefix for fragment files inside the layer file tree that bitbake
+   uses to construct full paths to the fragment files
+
+-  name of variable that holds the list of enabled fragments in an
+   active build
+
+-  name of variable that contains a list of variable names containing
+   fragment-specific metadata (such as descriptions)
+
+This allows listing enabled configuration fragments in ``OE_FRAGMENTS``
+variable like this::
+
+   OE_FRAGMENTS = "core/domain/somefragment core/someotherfragment anotherlayer/anotherdomain/anotherfragment"
+
+Fragment names listed in this variable must be prefixed by the layer name
+where a fragment file is located, defined by :term:`BBFILE_COLLECTIONS` in ``layer.conf``.
+
+The implementation then expands this list into
+:ref:`require <bitbake-user-manual/bitbake-user-manual-metadata:\`\`require\`\` directive>`
+directives with full paths to respective layers::
+
+   require /path/to/core-layer/conf/fragments/domain/somefragment.conf
+   require /path/to/core-layer/conf/fragments/someotherfragment.conf
+   require /path/to/another-layer/conf/fragments/anotherdomain/anotherfragment.conf
+
+The variable containing a list of fragment metadata variables could look like this::
+
+   OE_FRAGMENTS_METADATA_VARS = "BB_CONF_FRAGMENT_SUMMARY BB_CONF_FRAGMENT_DESCRIPTION"
+
+The implementation will add a flag containing the fragment name to each of those variables
+when parsing fragments, so that the variables are namespaced by fragment name, and do not override
+each other when several fragments are enabled.
+
 Functions
 =========
 

bitbake/doc/bitbake-user-manual/bitbake-user-manual-ref-variables.rst

@@ -127,17 +127,10 @@ overview of their function and contents.
       Contains the name of the currently running task. The name does not
       include the ``do_`` prefix.
 
-   :term:`BB_DANGLINGAPPENDS_WARNONLY`
-      Defines how BitBake handles situations where an append file
-      (``.bbappend``) has no corresponding recipe file (``.bb``). This
-      condition often occurs when layers get out of sync (e.g. ``oe-core``
-      bumps a recipe version and the old recipe no longer exists and the
-      other layer has not been updated to the new version of the recipe
-      yet).
-
-      The default fatal behavior is safest because it is the sane reaction
-      given something is out of sync. It is important to realize when your
-      changes are no longer being applied.
+   :term:`BB_CURRENT_MC`
+      Contains the name of the current multiconfig a task is being run under.
+      The name is taken from the multiconfig configuration file (a file
+      ``mc1.conf`` would make this variable equal to ``mc1``).
 
    :term:`BB_DEFAULT_TASK`
       The default task to use when none is specified (e.g. with the ``-c``
@@ -327,11 +320,26 @@ overview of their function and contents.
       mirror tarball. If the shallow mirror tarball cannot be fetched, it will
       try to fetch the full mirror tarball and use that.
 
-      When a mirror tarball is not available, a full git clone will be performed
-      regardless of whether this variable is set or not. Support for shallow
-      clones is not currently implemented as git does not directly support
-      shallow cloning a particular git commit hash (it only supports cloning
-      from a tag or branch reference).
+      This setting causes an initial shallow clone instead of an initial full bare clone.
+      The amount of data transferred during the initial clone will be significantly reduced.
+
+      However, every time the source revision (referenced in :term:`SRCREV`)
+      changes, regardless of whether the cache within the download directory
+      (defined by :term:`DL_DIR`) has been cleaned up or not,
+      the data transfer may be significantly higher because entirely
+      new shallow clones are required for each source revision change.
+
+      Over time, numerous shallow clones may cumulatively transfer
+      the same amount of data as an initial full bare clone.
+      This is especially the case with very large repositories.
+
+      Existing initial full bare clones, created without this setting,
+      will still be utilized.
+
+      If the Git error "Server does not allow request for unadvertised object"
+      occurs, an initial full bare clone is fetched automatically.
+      This may happen if the Git server does not allow the request
+      or if the Git client has issues with this functionality.
 
       See also :term:`BB_GIT_SHALLOW_DEPTH` and
       :term:`BB_GENERATE_SHALLOW_TARBALLS`.
@@ -699,6 +707,12 @@ overview of their function and contents.
       Within an executing task, this variable holds the hash of the task as
       returned by the currently enabled signature generator.
 
+   :term:`BB_USE_HOME_NPMRC`
+      Controls whether or not BitBake uses the user's .npmrc file within their
+      home directory within the npm fetcher. This can be used for authentication
+      of private NPM registries, among other uses. This is turned off by default
+      and requires the user to explicitly set it to "1" to enable.
+
    :term:`BB_VERBOSE_LOGS`
       Controls how verbose BitBake is during builds. If set, shell scripts
       echo commands and shell script output appears on standard out
@@ -766,6 +780,10 @@ overview of their function and contents.
    :term:`BBFILE_PRIORITY`
       Assigns the priority for recipe files in each layer.
 
+      This variable is used in the ``conf/layer.conf`` file and must be
+      suffixed with a `_` followed by the name of the specific layer (e.g.
+      ``BBFILE_PRIORITY_emenlow``). Colon as separator is not supported.
+
       This variable is useful in situations where the same recipe appears
       in more than one layer. Setting this variable allows you to
       prioritize a layer against other layers that contain the same recipe
@@ -780,7 +798,7 @@ overview of their function and contents.
       higher precedence. For example, the value 6 has a higher precedence
       than the value 5. If not specified, the :term:`BBFILE_PRIORITY` variable
       is set based on layer dependencies (see the :term:`LAYERDEPENDS` variable
-      for more information. The default priority, if unspecified for a
+      for more information). The default priority, if unspecified for a
       layer with no dependencies, is the lowest defined priority + 1 (or 1
       if no priorities are defined).
 

bitbake/doc/releases.rst

@@ -4,6 +4,12 @@
 BitBake Supported Release Manuals
 =================================
 
+****************************
+Release Series 5.1 (styhead)
+****************************
+
+- :yocto_docs:`BitBake 2.10 User Manual </bitbake/2.10/>`
+
 *******************************
 Release Series 5.0 (scarthgap)
 *******************************
@@ -16,12 +22,6 @@ Release Series 4.0 (kirkstone)
 
 - :yocto_docs:`BitBake 2.0 User Manual </bitbake/2.0/>`
 
-****************************
-Release Series 3.1 (dunfell)
-****************************
-
-- :yocto_docs:`BitBake 1.46 User Manual </bitbake/1.46/>`
-
 ================================
 BitBake Outdated Release Manuals
 ================================
@@ -62,10 +62,11 @@ Release Series 3.2 (gatesgarth)
 
 - :yocto_docs:`BitBake 1.48 User Manual </bitbake/1.48/>`
 
-*******************************************
-Release Series 3.1 (dunfell first versions)
-*******************************************
+****************************
+Release Series 3.1 (dunfell)
+****************************
 
+- :yocto_docs:`BitBake 1.46 User Manual </bitbake/1.46/>`
 - :yocto_docs:`3.1 BitBake User Manual </3.1/bitbake-user-manual/bitbake-user-manual.html>`
 - :yocto_docs:`3.1.1 BitBake User Manual </3.1.1/bitbake-user-manual/bitbake-user-manual.html>`
 - :yocto_docs:`3.1.2 BitBake User Manual </3.1.2/bitbake-user-manual/bitbake-user-manual.html>`

bitbake/lib/bb/COW.py

@@ -36,8 +36,9 @@ class COWDictMeta(COWMeta):
     __marker__ = tuple()
 
     def __str__(cls):
-        # FIXME: I have magic numbers!
-        return "<COWDict Level: %i Current Keys: %i>" % (cls.__count__, len(cls.__dict__) - 3)
+        ignored_keys = set(["__count__", "__doc__", "__module__", "__firstlineno__", "__static_attributes__"])
+        keys = set(cls.__dict__.keys()) - ignored_keys
+        return "<COWDict Level: %i Current Keys: %i>" % (cls.__count__, len(keys))
 
     __repr__ = __str__
 
@@ -161,8 +162,9 @@ class COWDictMeta(COWMeta):
 
 class COWSetMeta(COWDictMeta):
     def __str__(cls):
-        # FIXME: I have magic numbers!
-        return "<COWSet Level: %i Current Keys: %i>" % (cls.__count__, len(cls.__dict__) - 3)
+        ignored_keys = set(["__count__", "__doc__", "__module__", "__firstlineno__", "__static_attributes__"])
+        keys = set(cls.__dict__.keys()) - ignored_keys
+        return "<COWSet Level: %i Current Keys: %i>" % (cls.__count__, len(keys))
 
     __repr__ = __str__
 

bitbake/lib/bb/__init__.py

@@ -9,11 +9,11 @@
 # SPDX-License-Identifier: GPL-2.0-only
 #
 
-__version__ = "2.9.1"
+__version__ = "2.12.0"
 
 import sys
-if sys.version_info < (3, 8, 0):
-    raise RuntimeError("Sorry, python 3.8.0 or later is required for this version of bitbake")
+if sys.version_info < (3, 9, 0):
+    raise RuntimeError("Sorry, python 3.9.0 or later is required for this version of bitbake")
 
 if sys.version_info < (3, 10, 0):
     # With python 3.8 and 3.9, we see errors of "libgcc_s.so.1 must be installed for pthread_cancel to work"
@@ -194,7 +194,6 @@ def deprecated(func, name=None, advice=""):
 # For compatibility
 def deprecate_import(current, modulename, fromlist, renames = None):
     """Import objects from one module into another, wrapping them with a DeprecationWarning"""
-    import sys
 
     module = __import__(modulename, fromlist = fromlist)
     for position, objname in enumerate(fromlist):

bitbake/lib/bb/acl.py

@@ -195,8 +195,6 @@ class ACL(object):
 
 def main():
     import argparse
-    import pwd
-    import grp
     from pathlib import Path
 
     parser = argparse.ArgumentParser()

bitbake/lib/bb/asyncrpc/client.py

@@ -112,11 +112,16 @@ class AsyncClient(object):
             )
 
         async def connect_sock():
-            websocket = await websockets.connect(
-                uri,
-                ping_interval=None,
-                open_timeout=self.timeout,
-            )
+            try:
+                websocket = await websockets.connect(
+                    uri,
+                    ping_interval=None,
+                    open_timeout=self.timeout,
+                )
+            except asyncio.exceptions.TimeoutError:
+                raise ConnectionError("Timeout while connecting to websocket")
+            except (OSError, websockets.InvalidHandshake, websockets.InvalidURI) as exc:
+                raise ConnectionError(f"Could not connect to websocket: {exc}") from exc
             return WebsocketConnection(websocket, self.timeout)
 
         self._connect_sock = connect_sock

bitbake/lib/bb/build.py

@@ -397,7 +397,7 @@ def create_progress_handler(func, progress, logfile, d):
         # Use specified regex
         return bb.progress.OutOfProgressHandler(d, regex=progress.split(':', 1)[1], outfile=logfile)
     elif progress.startswith("custom:"):
-        # Use a custom progress handler that was injected via OE_EXTRA_IMPORTS or __builtins__
+        # Use a custom progress handler that was injected via other means
         import functools
         from types import ModuleType
 
@@ -1028,3 +1028,9 @@ def tasksbetween(task_start, task_end, d):
         chain.pop()
     follow_chain(task_start, task_end)
     return outtasks
+
+def listtasks(d):
+    """
+    Return the list of tasks in the current recipe.
+    """
+    return tuple(d.getVar('__BBTASKS', False) or ())

bitbake/lib/bb/cache.py

@@ -395,7 +395,7 @@ class Cache(object):
         # It will be used later for deciding whether we
         # need extra cache file dump/load support
         self.mc = mc
-        self.logger = PrefixLoggerAdapter("Cache: %s: " % (mc if mc else "default"), logger)
+        self.logger = PrefixLoggerAdapter("Cache: %s: " % (mc if mc else ''), logger)
         self.caches_array = caches_array
         self.cachedir = self.data.getVar("CACHE")
         self.clean = set()
@@ -847,6 +847,16 @@ class MultiProcessCache(object):
         data = [{}]
         return data
 
+    def clear_cache(self):
+        if not self.cachefile:
+            bb.fatal("Can't clear invalid cachefile")
+
+        self.cachedata = self.create_cachedata()
+        self.cachedata_extras = self.create_cachedata()
+        with bb.utils.fileslocked([self.cachefile + ".lock"]):
+            bb.utils.remove(self.cachefile)
+            bb.utils.remove(self.cachefile + "-*")
+
     def save_extras(self):
         if not self.cachefile:
             return

bitbake/lib/bb/checksum.py

@@ -142,3 +142,28 @@ class FileChecksumCache(MultiProcessCache):
 
         checksums.sort(key=operator.itemgetter(1))
         return checksums
+
+class RevisionsCache(MultiProcessCache):
+    cache_file_name = "local_srcrevisions.dat"
+    CACHE_VERSION = 1
+
+    def __init__(self):
+        MultiProcessCache.__init__(self)
+
+    def get_revs(self):
+        return self.cachedata[0]
+
+    def get_rev(self, k):
+        if k in self.cachedata_extras[0]:
+            return self.cachedata_extras[0][k]
+        if k in self.cachedata[0]:
+            return self.cachedata[0][k]
+        return None
+
+    def set_rev(self, k, v):
+        self.cachedata[0][k] = v
+        self.cachedata_extras[0][k] = v
+
+    def merge_data(self, source, dest):
+        for h in source[0]:
+            dest[0][h] = source[0][h]

bitbake/lib/bb/codeparser.py

@@ -72,12 +72,20 @@ def add_module_functions(fn, functions, namespace):
             parser.parse_python(None, filename=fn, lineno=1, fixedhash=fixedhash+f)
             #bb.warn("Cached %s" % f)
         except KeyError:
-            targetfn = inspect.getsourcefile(functions[f])
+            try:
+                targetfn = inspect.getsourcefile(functions[f])
+            except TypeError:
+                # Builtin
+                continue
             if fn != targetfn:
                 # Skip references to other modules outside this file
                 #bb.warn("Skipping %s" % name)
                 continue
-            lines, lineno = inspect.getsourcelines(functions[f])
+            try:
+                lines, lineno = inspect.getsourcelines(functions[f])
+            except TypeError:
+                # Builtin
+                continue
             src = "".join(lines)
             parser.parse_python(src, filename=fn, lineno=lineno, fixedhash=fixedhash+f)
             #bb.warn("Not cached %s" % f)

bitbake/lib/bb/command.py

@@ -24,6 +24,7 @@ import io
 import bb.event
 import bb.cooker
 import bb.remotedata
+import bb.parse
 
 class DataStoreConnectionHandle(object):
     def __init__(self, dsindex=0):
@@ -108,7 +109,7 @@ class Command:
 
     def runAsyncCommand(self, _, process_server, halt):
         try:
-            if self.cooker.state in (bb.cooker.state.error, bb.cooker.state.shutdown, bb.cooker.state.forceshutdown):
+            if self.cooker.state in (bb.cooker.State.ERROR, bb.cooker.State.SHUTDOWN, bb.cooker.State.FORCE_SHUTDOWN):
                 # updateCache will trigger a shutdown of the parser
                 # and then raise BBHandledException triggering an exit
                 self.cooker.updateCache()
@@ -118,7 +119,7 @@ class Command:
                 (command, options) = cmd
                 commandmethod = getattr(CommandsAsync, command)
                 needcache = getattr( commandmethod, "needcache" )
-                if needcache and self.cooker.state != bb.cooker.state.running:
+                if needcache and self.cooker.state != bb.cooker.State.RUNNING:
                     self.cooker.updateCache()
                     return True
                 else:
@@ -142,14 +143,14 @@ class Command:
                 return bb.server.process.idleFinish(traceback.format_exc())
 
     def finishAsyncCommand(self, msg=None, code=None):
+        self.cooker.finishcommand()
+        self.process_server.clear_async_cmd()
         if msg or msg == "":
             bb.event.fire(CommandFailed(msg), self.cooker.data)
         elif code:
             bb.event.fire(CommandExit(code), self.cooker.data)
         else:
             bb.event.fire(CommandCompleted(), self.cooker.data)
-        self.cooker.finishcommand()
-        self.process_server.clear_async_cmd()
 
     def reset(self):
         if self.remotedatastores:
@@ -310,7 +311,7 @@ class CommandsSync:
     def revalidateCaches(self, command, params):
         """Called by UI clients when metadata may have changed"""
         command.cooker.revalidateCaches()
-    parseConfiguration.needconfig = False
+    revalidateCaches.needconfig = False
 
     def getRecipes(self, command, params):
         try:
@@ -420,15 +421,30 @@ class CommandsSync:
         return command.cooker.recipecaches[mc].pkg_dp
     getDefaultPreference.readonly = True
 
+
     def getSkippedRecipes(self, command, params):
+        """
+        Get the map of skipped recipes for the specified multiconfig/mc name (`params[0]`).
+
+        Invoked by `bb.tinfoil.Tinfoil.get_skipped_recipes`
+
+        :param command: Internally used parameter.
+        :param params: Parameter array. params[0] is multiconfig/mc name. If not given, then default mc '' is assumed.
+        :return: Dict whose keys are virtualfns and values are `bb.cooker.SkippedPackage`
+        """
+        try:
+            mc = params[0]
+        except IndexError:
+            mc = ''
+
         # Return list sorted by reverse priority order
         import bb.cache
         def sortkey(x):
             vfn, _ = x
-            realfn, _, mc = bb.cache.virtualfn2realfn(vfn)
-            return (-command.cooker.collections[mc].calc_bbfile_priority(realfn)[0], vfn)
+            realfn, _, item_mc = bb.cache.virtualfn2realfn(vfn)
+            return -command.cooker.collections[item_mc].calc_bbfile_priority(realfn)[0], vfn
 
-        skipdict = OrderedDict(sorted(command.cooker.skiplist.items(), key=sortkey))
+        skipdict = OrderedDict(sorted(command.cooker.skiplist_by_mc[mc].items(), key=sortkey))
         return list(skipdict.items())
     getSkippedRecipes.readonly = True
 
@@ -582,6 +598,13 @@ class CommandsSync:
         return DataStoreConnectionHandle(idx)
     parseRecipeFile.readonly = True
 
+    def finalizeData(self, command, params):
+        newdata = command.cooker.data.createCopy()
+        bb.data.expandKeys(newdata)
+        bb.parse.ast.runAnonFuncs(newdata)
+        idx = command.remotedatastores.store(newdata)
+        return DataStoreConnectionHandle(idx)
+
 class CommandsAsync:
     """
     A class of asynchronous commands

bitbake/lib/bb/compress/lz4.py

@@ -13,7 +13,7 @@ def open(*args, **kwargs):
 
 class LZ4File(bb.compress._pipecompress.PipeFile):
     def get_compress(self):
-        return ["lz4c", "-z", "-c"]
+        return ["lz4", "-z", "-c"]
 
     def get_decompress(self):
-        return ["lz4c", "-d", "-c"]
+        return ["lz4", "-d", "-c"]

bitbake/lib/bb/cooker.py

@@ -8,7 +8,7 @@
 #
 # SPDX-License-Identifier: GPL-2.0-only
 #
-
+import enum
 import sys, os, glob, os.path, re, time
 import itertools
 import logging
@@ -48,16 +48,15 @@ class CollectionError(bb.BBHandledException):
     Exception raised when layer configuration is incorrect
     """
 
-class state:
-    initial, parsing, running, shutdown, forceshutdown, stopped, error = list(range(7))
 
-    @classmethod
-    def get_name(cls, code):
-        for name in dir(cls):
-            value = getattr(cls, name)
-            if type(value) == type(cls.initial) and value == code:
-                return name
-        raise ValueError("Invalid status code: %s" % code)
+class State(enum.Enum):
+    INITIAL = 0,
+    PARSING = 1,
+    RUNNING = 2,
+    SHUTDOWN = 3,
+    FORCE_SHUTDOWN = 4,
+    STOPPED = 5,
+    ERROR = 6
 
 
 class SkippedPackage:
@@ -134,7 +133,8 @@ class BBCooker:
         self.baseconfig_valid = False
         self.parsecache_valid = False
         self.eventlog = None
-        self.skiplist = {}
+        # The skiplists, one per multiconfig
+        self.skiplist_by_mc = defaultdict(dict)
         self.featureset = CookerFeatures()
         if featureSet:
             for f in featureSet:
@@ -180,7 +180,7 @@ class BBCooker:
             pass
 
         self.command = bb.command.Command(self, self.process_server)
-        self.state = state.initial
+        self.state = State.INITIAL
 
         self.parser = None
 
@@ -226,23 +226,22 @@ class BBCooker:
             bb.warn("Cooker received SIGTERM, shutting down...")
         elif signum == signal.SIGHUP:
             bb.warn("Cooker received SIGHUP, shutting down...")
-        self.state = state.forceshutdown
+        self.state = State.FORCE_SHUTDOWN
         bb.event._should_exit.set()
 
     def setFeatures(self, features):
         # we only accept a new feature set if we're in state initial, so we can reset without problems
-        if not self.state in [state.initial, state.shutdown, state.forceshutdown, state.stopped, state.error]:
+        if not self.state in [State.INITIAL, State.SHUTDOWN, State.FORCE_SHUTDOWN, State.STOPPED, State.ERROR]:
             raise Exception("Illegal state for feature set change")
         original_featureset = list(self.featureset)
         for feature in features:
             self.featureset.setFeature(feature)
         bb.debug(1, "Features set %s (was %s)" % (original_featureset, list(self.featureset)))
-        if (original_featureset != list(self.featureset)) and self.state != state.error and hasattr(self, "data"):
+        if (original_featureset != list(self.featureset)) and self.state != State.ERROR and hasattr(self, "data"):
             self.reset()
 
     def initConfigurationData(self):
-
-        self.state = state.initial
+        self.state = State.INITIAL
         self.caches_array = []
 
         sys.path = self.orig_syspath.copy()
@@ -317,8 +316,14 @@ class BBCooker:
                     try:
                         with hashserv.create_client(upstream) as client:
                             client.ping()
-                    except (ConnectionError, ImportError) as e:
-                        bb.warn("BB_HASHSERVE_UPSTREAM is not valid, unable to connect hash equivalence server at '%s': %s"
+                    except ImportError as e:
+                        bb.fatal(""""Unable to use hash equivalence server at '%s' due to missing or incorrect python module:
+%s
+Please install the needed module on the build host, or use an environment containing it (e.g a pip venv or OpenEmbedded's buildtools tarball).
+You can also remove the BB_HASHSERVE_UPSTREAM setting, but this may result in significantly longer build times as bitbake will be unable to reuse prebuilt sstate artefacts."""
+                                 % (upstream, repr(e)))
+                    except ConnectionError as e:
+                        bb.warn("Unable to connect to hash equivalence server at '%s', please correct or remove BB_HASHSERVE_UPSTREAM:\n%s"
                                  % (upstream, repr(e)))
                         upstream = None
 
@@ -616,8 +621,8 @@ class BBCooker:
         localdata = {}
 
         for mc in self.multiconfigs:
-            taskdata[mc] = bb.taskdata.TaskData(halt, skiplist=self.skiplist, allowincomplete=allowincomplete)
-            localdata[mc] = data.createCopy(self.databuilder.mcdata[mc])
+            taskdata[mc] = bb.taskdata.TaskData(halt, skiplist=self.skiplist_by_mc[mc], allowincomplete=allowincomplete)
+            localdata[mc] = bb.data.createCopy(self.databuilder.mcdata[mc])
             bb.data.expandKeys(localdata[mc])
 
         current = 0
@@ -900,10 +905,11 @@ class BBCooker:
 
         depgraph = self.generateTaskDepTreeData(pkgs_to_build, task)
 
-        with open('pn-buildlist', 'w') as f:
-            for pn in depgraph["pn"]:
-                f.write(pn + "\n")
-        logger.info("PN build list saved to 'pn-buildlist'")
+        pns = depgraph["pn"].keys()
+        if pns:
+            with open('pn-buildlist', 'w') as f:
+                f.write("%s\n" % "\n".join(sorted(pns)))
+            logger.info("PN build list saved to 'pn-buildlist'")
 
         # Remove old format output files to ensure no confusion with stale data
         try:
@@ -937,7 +943,7 @@ class BBCooker:
         for mc in self.multiconfigs:
             # First get list of recipes, including skipped
             recipefns = list(self.recipecaches[mc].pkg_fn.keys())
-            recipefns.extend(self.skiplist.keys())
+            recipefns.extend(self.skiplist_by_mc[mc].keys())
 
             # Work out list of bbappends that have been applied
             applied_appends = []
@@ -956,13 +962,7 @@ class BBCooker:
                                                                         '\n  '.join(appends_without_recipes[mc])))
 
         if msgs:
-            msg = "\n".join(msgs)
-            warn_only = self.databuilder.mcdata[mc].getVar("BB_DANGLINGAPPENDS_WARNONLY", \
-                False) or "no"
-            if warn_only.lower() in ("1", "yes", "true"):
-                bb.warn(msg)
-            else:
-                bb.fatal(msg)
+            bb.fatal("\n".join(msgs))
 
     def handlePrefProviders(self):
 
@@ -1403,11 +1403,11 @@ class BBCooker:
 
             msg = None
             interrupted = 0
-            if halt or self.state == state.forceshutdown:
+            if halt or self.state == State.FORCE_SHUTDOWN:
                 rq.finish_runqueue(True)
                 msg = "Forced shutdown"
                 interrupted = 2
-            elif self.state == state.shutdown:
+            elif self.state == State.SHUTDOWN:
                 rq.finish_runqueue(False)
                 msg = "Stopped build"
                 interrupted = 1
@@ -1477,12 +1477,12 @@ class BBCooker:
         def buildTargetsIdle(server, rq, halt):
             msg = None
             interrupted = 0
-            if halt or self.state == state.forceshutdown:
+            if halt or self.state == State.FORCE_SHUTDOWN:
                 bb.event._should_exit.set()
                 rq.finish_runqueue(True)
                 msg = "Forced shutdown"
                 interrupted = 2
-            elif self.state == state.shutdown:
+            elif self.state == State.SHUTDOWN:
                 rq.finish_runqueue(False)
                 msg = "Stopped build"
                 interrupted = 1
@@ -1577,7 +1577,7 @@ class BBCooker:
 
 
     def updateCacheSync(self):
-        if self.state == state.running:
+        if self.state == State.RUNNING:
             return
 
         if not self.baseconfig_valid:
@@ -1587,19 +1587,19 @@ class BBCooker:
 
     # This is called for all async commands when self.state != running
     def updateCache(self):
-        if self.state == state.running:
+        if self.state == State.RUNNING:
             return
 
-        if self.state in (state.shutdown, state.forceshutdown, state.error):
+        if self.state in (State.SHUTDOWN, State.FORCE_SHUTDOWN, State.ERROR):
             if hasattr(self.parser, 'shutdown'):
                 self.parser.shutdown(clean=False)
                 self.parser.final_cleanup()
             raise bb.BBHandledException()
 
-        if self.state != state.parsing:
+        if self.state != State.PARSING:
             self.updateCacheSync()
 
-        if self.state != state.parsing and not self.parsecache_valid:
+        if self.state != State.PARSING and not self.parsecache_valid:
             bb.server.process.serverlog("Parsing started")
             self.parsewatched = {}
 
@@ -1633,9 +1633,10 @@ class BBCooker:
             self.parser = CookerParser(self, mcfilelist, total_masked)
             self._parsecache_set(True)
 
-        self.state = state.parsing
+        self.state = State.PARSING
 
         if not self.parser.parse_next():
+            bb.server.process.serverlog("Parsing completed")
             collectlog.debug("parsing complete")
             if self.parser.error:
                 raise bb.BBHandledException()
@@ -1643,7 +1644,7 @@ class BBCooker:
             self.handlePrefProviders()
             for mc in self.multiconfigs:
                 self.recipecaches[mc].bbfile_priority = self.collections[mc].collection_priorities(self.recipecaches[mc].pkg_fn, self.parser.mcfilelist[mc], self.data)
-            self.state = state.running
+            self.state = State.RUNNING
 
             # Send an event listing all stamps reachable after parsing
             # which the metadata may use to clean up stale data
@@ -1716,10 +1717,10 @@ class BBCooker:
 
     def shutdown(self, force=False):
         if force:
-            self.state = state.forceshutdown
+            self.state = State.FORCE_SHUTDOWN
             bb.event._should_exit.set()
         else:
-            self.state = state.shutdown
+            self.state = State.SHUTDOWN
 
         if self.parser:
             self.parser.shutdown(clean=False)
@@ -1729,7 +1730,7 @@ class BBCooker:
         if hasattr(self.parser, 'shutdown'):
             self.parser.shutdown(clean=False)
             self.parser.final_cleanup()
-        self.state = state.initial
+        self.state = State.INITIAL
         bb.event._should_exit.clear()
 
     def reset(self):
@@ -2362,7 +2363,7 @@ class CookerParser(object):
         for virtualfn, info_array in result:
             if info_array[0].skipped:
                 self.skipped += 1
-                self.cooker.skiplist[virtualfn] = SkippedPackage(info_array[0])
+                self.cooker.skiplist_by_mc[mc][virtualfn] = SkippedPackage(info_array[0])
             self.bb_caches[mc].add_info(virtualfn, info_array, self.cooker.recipecaches[mc],
                                         parsed=parsed, watcher = self.cooker.add_filewatch)
         return True

bitbake/lib/bb/cookerdata.py

@@ -1,3 +1,4 @@
+
 #
 # Copyright (C) 2003, 2004  Chris Larson
 # Copyright (C) 2003, 2004  Phil Blundell
@@ -267,8 +268,8 @@ class CookerDataBuilder(object):
         try:
             self.data = self.parseConfigurationFiles(self.prefiles, self.postfiles)
 
-            if self.data.getVar("BB_WORKERCONTEXT", False) is None and not worker:
-                bb.fetch.fetcher_init(self.data)
+            servercontext = self.data.getVar("BB_WORKERCONTEXT", False) is None and not worker
+            bb.fetch.fetcher_init(self.data, servercontext)
             bb.parse.init_parser(self.data)
 
             bb.event.fire(bb.event.ConfigParsed(), self.data)
@@ -345,7 +346,7 @@ class CookerDataBuilder(object):
     def _findLayerConf(self, data):
         return findConfigFile("bblayers.conf", data)
 
-    def parseConfigurationFiles(self, prefiles, postfiles, mc = "default"):
+    def parseConfigurationFiles(self, prefiles, postfiles, mc = ""):
         data = bb.data.createCopy(self.basedata)
         data.setVar("BB_CURRENT_MC", mc)
 

bitbake/lib/bb/data_smart.py

@@ -31,7 +31,7 @@ logger = logging.getLogger("BitBake.Data")
 
 __setvar_keyword__ = [":append", ":prepend", ":remove"]
 __setvar_regexp__ = re.compile(r'(?P<base>.*?)(?P<keyword>:append|:prepend|:remove)(:(?P<add>[^A-Z]*))?$')
-__expand_var_regexp__ = re.compile(r"\${[a-zA-Z0-9\-_+./~:]+?}")
+__expand_var_regexp__ = re.compile(r"\${[a-zA-Z0-9\-_+./~:]+}")
 __expand_python_regexp__ = re.compile(r"\${@(?:{.*?}|.)+?}")
 __whitespace_split__ = re.compile(r'(\s)')
 __override_regexp__ = re.compile(r'[a-z0-9]+')
@@ -106,52 +106,52 @@ class VariableParse:
         self.contains = {}
 
     def var_sub(self, match):
-            key = match.group()[2:-1]
-            if self.varname and key:
-                if self.varname == key:
-                    raise Exception("variable %s references itself!" % self.varname)
-            var = self.d.getVarFlag(key, "_content")
-            self.references.add(key)
-            if var is not None:
-                return var
-            else:
-                return match.group()
+        key = match.group()[2:-1]
+        if self.varname and key:
+            if self.varname == key:
+                raise Exception("variable %s references itself!" % self.varname)
+        var = self.d.getVarFlag(key, "_content")
+        self.references.add(key)
+        if var is not None:
+            return var
+        else:
+            return match.group()
 
     def python_sub(self, match):
-            if isinstance(match, str):
-                code = match
-            else:
-                code = match.group()[3:-1]
+        if isinstance(match, str):
+            code = match
+        else:
+            code = match.group()[3:-1]
 
-            # Do not run code that contains one or more unexpanded variables
-            # instead return the code with the characters we removed put back
-            if __expand_var_regexp__.findall(code):
-                return "${@" + code + "}"
+        # Do not run code that contains one or more unexpanded variables
+        # instead return the code with the characters we removed put back
+        if __expand_var_regexp__.findall(code):
+            return "${@" + code + "}"
 
-            if self.varname:
-                varname = 'Var <%s>' % self.varname
-            else:
-                varname = '<expansion>'
-            codeobj = compile(code.strip(), varname, "eval")
+        if self.varname:
+            varname = 'Var <%s>' % self.varname
+        else:
+            varname = '<expansion>'
+        codeobj = compile(code.strip(), varname, "eval")
 
-            parser = bb.codeparser.PythonParser(self.varname, logger)
-            parser.parse_python(code)
-            if self.varname:
-                vardeps = self.d.getVarFlag(self.varname, "vardeps")
-                if vardeps is None:
-                    parser.log.flush()
-            else:
+        parser = bb.codeparser.PythonParser(self.varname, logger)
+        parser.parse_python(code)
+        if self.varname:
+            vardeps = self.d.getVarFlag(self.varname, "vardeps")
+            if vardeps is None:
                 parser.log.flush()
-            self.references |= parser.references
-            self.execs |= parser.execs
+        else:
+            parser.log.flush()
+        self.references |= parser.references
+        self.execs |= parser.execs
 
-            for k in parser.contains:
-                if k not in self.contains:
-                    self.contains[k] = parser.contains[k].copy()
-                else:
-                    self.contains[k].update(parser.contains[k])
-            value = utils.better_eval(codeobj, DataContext(self.d), {'d' : self.d})
-            return str(value)
+        for k in parser.contains:
+            if k not in self.contains:
+                self.contains[k] = parser.contains[k].copy()
+            else:
+                self.contains[k].update(parser.contains[k])
+        value = utils.better_eval(codeobj, DataContext(self.d), {'d' : self.d})
+        return str(value)
 
 class DataContext(dict):
     excluded = set([i for i in dir(builtins) if not i.startswith('_')] + ['oe'])
@@ -580,12 +580,10 @@ class DataSmart(MutableMapping):
             else:
                 loginfo['op'] = keyword
             self.varhistory.record(**loginfo)
-            # todo make sure keyword is not __doc__ or __module__
             # pay the cookie monster
 
             # more cookies for the cookie monster
-            if ':' in var:
-                self._setvar_update_overrides(base, **loginfo)
+            self._setvar_update_overrides(base, **loginfo)
 
             if base in self.overridevars:
                 self._setvar_update_overridevars(var, value)
@@ -638,6 +636,7 @@ class DataSmart(MutableMapping):
                 nextnew.update(vardata.contains.keys())
             new = nextnew
         self.overrides = None
+        self.expand_cache = {}
 
     def _setvar_update_overrides(self, var, **loginfo):
         # aka pay the cookie monster
@@ -827,6 +826,8 @@ class DataSmart(MutableMapping):
                 value = copy.copy(local_var[flag])
             elif flag == "_content" and "_defaultval" in local_var and not noweakdefault:
                 value = copy.copy(local_var["_defaultval"])
+            elif "_defaultval_flag_"+flag in local_var and not noweakdefault:
+                value = copy.copy(local_var["_defaultval_flag_"+flag])
 
 
         if flag == "_content" and local_var is not None and ":append" in local_var and not parsing:
@@ -918,6 +919,8 @@ class DataSmart(MutableMapping):
             self.varhistory.record(**loginfo)
 
             del self.dict[var][flag]
+            if ("_defaultval_flag_" + flag) in self.dict[var]:
+                del self.dict[var]["_defaultval_flag_" + flag]
 
     def appendVarFlag(self, var, flag, value, **loginfo):
         loginfo['op'] = 'append'
@@ -952,17 +955,22 @@ class DataSmart(MutableMapping):
         flags = {}
 
         if local_var:
-            for i in local_var:
-                if i.startswith(("_", ":")) and not internalflags:
+            for i, val in local_var.items():
+                if i.startswith("_defaultval_flag_") and not internalflags:
+                    i = i[len("_defaultval_flag_"):]
+                    if i not in local_var:
+                        flags[i] = val
+                elif i.startswith(("_", ":")) and not internalflags:
                     continue
-                flags[i] = local_var[i]
+                else:
+                    flags[i] = val
+
                 if expand and i in expand:
                     flags[i] = self.expand(flags[i], var + "[" + i + "]")
         if len(flags) == 0:
             return None
         return flags
 
-
     def delVarFlags(self, var, **loginfo):
         self.expand_cache = {}
         if not var in self.dict:
@@ -1112,5 +1120,10 @@ class DataSmart(MutableMapping):
                     value = d.getVar(i, False) or ""
                     data.update({i:value})
 
+        moddeps = bb.codeparser.modulecode_deps
+        for dep in sorted(moddeps):
+            # Ignore visitor code, sort sets
+            data.update({'moddep[%s]' % dep : [sorted(moddeps[dep][0]), sorted(moddeps[dep][1]), sorted(moddeps[dep][2]), sorted(moddeps[dep][3]), moddeps[dep][4]]})
+
         data_str = str([(k, data[k]) for k in sorted(data.keys())])
         return hashlib.sha256(data_str.encode("utf-8")).hexdigest()

bitbake/lib/bb/event.py

@@ -194,7 +194,12 @@ def fire_ui_handlers(event, d):
         ui_queue.append(event)
         return
 
-    with bb.utils.lock_timeout(_thread_lock):
+    with bb.utils.lock_timeout_nocheck(_thread_lock) as lock:
+        if not lock:
+            # If we can't get the lock, we may be recursively called, queue and return
+            ui_queue.append(event)
+            return
+
         errors = []
         for h in _ui_handlers:
             #print "Sending event %s" % event
@@ -213,6 +218,9 @@ def fire_ui_handlers(event, d):
         for h in errors:
             del _ui_handlers[h]
 
+    while ui_queue:
+        fire_ui_handlers(ui_queue.pop(), d)
+
 def fire(event, d):
     """Fire off an Event"""
 

bitbake/lib/bb/fetch2/__init__.py

@@ -23,17 +23,18 @@ import collections
 import subprocess
 import pickle
 import errno
-import bb.persist_data, bb.utils
+import bb.utils
 import bb.checksum
 import bb.process
 import bb.event
 
 __version__ = "2"
 _checksum_cache = bb.checksum.FileChecksumCache()
+_revisions_cache = bb.checksum.RevisionsCache()
 
 logger = logging.getLogger("BitBake.Fetcher")
 
-CHECKSUM_LIST = [ "md5", "sha256", "sha1", "sha384", "sha512" ]
+CHECKSUM_LIST = [ "goh1", "md5", "sha256", "sha1", "sha384", "sha512" ]
 SHOWN_CHECKSUM_LIST = ["sha256"]
 
 class BBFetchException(Exception):
@@ -237,7 +238,7 @@ class URI(object):
         # to RFC compliant URL format. E.g.:
         #   file://foo.diff -> file:foo.diff
         if urlp.scheme in self._netloc_forbidden:
-            uri = re.sub("(?<=:)//(?!/)", "", uri, 1)
+            uri = re.sub(r"(?<=:)//(?!/)", "", uri, count=1)
             reparse = 1
 
         if reparse:
@@ -352,6 +353,14 @@ def decodeurl(url):
     user, password, parameters).
     """
 
+    uri = URI(url)
+    path = uri.path if uri.path else "/"
+    return uri.scheme, uri.hostport, path, uri.username, uri.password, uri.params
+
+def decodemirrorurl(url):
+    """Decodes a mirror URL into the tokens (scheme, network location, path,
+    user, password, parameters).
+    """
     m = re.compile('(?P<type>[^:]*)://((?P<user>[^/;]+)@)?(?P<location>[^;]+)(;(?P<parm>.*))?').match(url)
     if not m:
         raise MalformedUrl(url)
@@ -370,6 +379,9 @@ def decodeurl(url):
     elif type.lower() == 'file':
         host = ""
         path = location
+        if user:
+            path = user + '@' + path
+            user = ""
     else:
         host = location
         path = "/"
@@ -402,32 +414,34 @@ def encodeurl(decoded):
 
     if not type:
         raise MissingParameterError('type', "encoded from the data %s" % str(decoded))
-    url = ['%s://' % type]
+    uri = URI()
+    uri.scheme = type
     if user and type != "file":
-        url.append("%s" % user)
+        uri.username = user
         if pswd:
-            url.append(":%s" % pswd)
-        url.append("@")
+            uri.password = pswd
     if host and type != "file":
-        url.append("%s" % host)
+        uri.hostname = host
     if path:
         # Standardise path to ensure comparisons work
         while '//' in path:
             path = path.replace("//", "/")
-        url.append("%s" % urllib.parse.quote(path))
+        uri.path = path
+        if type == "file":
+            # Use old not IETF compliant style
+            uri.relative = False
     if p:
-        for parm in p:
-            url.append(";%s=%s" % (parm, p[parm]))
+        uri.params = p
 
-    return "".join(url)
+    return str(uri)
 
 def uri_replace(ud, uri_find, uri_replace, replacements, d, mirrortarball=None):
     if not ud.url or not uri_find or not uri_replace:
         logger.error("uri_replace: passed an undefined value, not replacing")
         return None
-    uri_decoded = list(decodeurl(ud.url))
-    uri_find_decoded = list(decodeurl(uri_find))
-    uri_replace_decoded = list(decodeurl(uri_replace))
+    uri_decoded = list(decodemirrorurl(ud.url))
+    uri_find_decoded = list(decodemirrorurl(uri_find))
+    uri_replace_decoded = list(decodemirrorurl(uri_replace))
     logger.debug2("For url %s comparing %s to %s" % (uri_decoded, uri_find_decoded, uri_replace_decoded))
     result_decoded = ['', '', '', '', '', {}]
     # 0 - type, 1 - host, 2 - path, 3 - user,  4- pswd, 5 - params
@@ -460,7 +474,7 @@ def uri_replace(ud, uri_find, uri_replace, replacements, d, mirrortarball=None):
                 for k in replacements:
                     uri_replace_decoded[loc] = uri_replace_decoded[loc].replace(k, replacements[k])
                 #bb.note("%s %s %s" % (regexp, uri_replace_decoded[loc], uri_decoded[loc]))
-                result_decoded[loc] = re.sub(regexp, uri_replace_decoded[loc], uri_decoded[loc], 1)
+                result_decoded[loc] = re.sub(regexp, uri_replace_decoded[loc], uri_decoded[loc], count=1)
             if loc == 2:
                 # Handle path manipulations
                 basename = None
@@ -493,42 +507,48 @@ methods = []
 urldata_cache = {}
 saved_headrevs = {}
 
-def fetcher_init(d):
+def fetcher_init(d, servercontext=True):
     """
     Called to initialize the fetchers once the configuration data is known.
     Calls before this must not hit the cache.
     """
 
-    with bb.persist_data.persist('BB_URI_HEADREVS', d) as revs:
-        try:
-            # fetcher_init is called multiple times, so make sure we only save the
-            # revs the first time it is called.
-            if not bb.fetch2.saved_headrevs:
-                bb.fetch2.saved_headrevs = dict(revs)
-        except:
-            pass
+    _checksum_cache.init_cache(d.getVar("BB_CACHEDIR"))
+    _revisions_cache.init_cache(d.getVar("BB_CACHEDIR"))
 
-        # When to drop SCM head revisions controlled by user policy
-        srcrev_policy = d.getVar('BB_SRCREV_POLICY') or "clear"
-        if srcrev_policy == "cache":
-            logger.debug("Keeping SRCREV cache due to cache policy of: %s", srcrev_policy)
-        elif srcrev_policy == "clear":
-            logger.debug("Clearing SRCREV cache due to cache policy of: %s", srcrev_policy)
-            revs.clear()
-        else:
-            raise FetchError("Invalid SRCREV cache policy of: %s" % srcrev_policy)
+    if not servercontext:
+        return
 
-        _checksum_cache.init_cache(d.getVar("BB_CACHEDIR"))
+    try:
+        # fetcher_init is called multiple times, so make sure we only save the
+        # revs the first time it is called.
+        if not bb.fetch2.saved_headrevs:
+            bb.fetch2.saved_headrevs = _revisions_cache.get_revs()
+    except:
+        pass
 
-        for m in methods:
-            if hasattr(m, "init"):
-                m.init(d)
+    # When to drop SCM head revisions controlled by user policy
+    srcrev_policy = d.getVar('BB_SRCREV_POLICY') or "clear"
+    if srcrev_policy == "cache":
+        logger.debug("Keeping SRCREV cache due to cache policy of: %s", srcrev_policy)
+    elif srcrev_policy == "clear":
+        logger.debug("Clearing SRCREV cache due to cache policy of: %s", srcrev_policy)
+        _revisions_cache.clear_cache()
+    else:
+        raise FetchError("Invalid SRCREV cache policy of: %s" % srcrev_policy)
+
+
+    for m in methods:
+        if hasattr(m, "init"):
+            m.init(d)
 
 def fetcher_parse_save():
     _checksum_cache.save_extras()
+    _revisions_cache.save_extras()
 
 def fetcher_parse_done():
     _checksum_cache.save_merge()
+    _revisions_cache.save_merge()
 
 def fetcher_compare_revisions(d):
     """
@@ -536,8 +556,8 @@ def fetcher_compare_revisions(d):
     when bitbake was started and return true if they have changed.
     """
 
-    with dict(bb.persist_data.persist('BB_URI_HEADREVS', d)) as headrevs:
-        return headrevs != bb.fetch2.saved_headrevs
+    headrevs = _revisions_cache.get_revs()
+    return headrevs != bb.fetch2.saved_headrevs
 
 def mirror_from_string(data):
     mirrors = (data or "").replace('\\n',' ').split()
@@ -786,8 +806,8 @@ def _get_srcrev(d, method_name='sortable_revision'):
         return "", revs
 
 
-    if len(scms) == 1 and len(urldata[scms[0]].names) == 1:
-        autoinc, rev = getattr(urldata[scms[0]].method, method_name)(urldata[scms[0]], d, urldata[scms[0]].names[0])
+    if len(scms) == 1:
+        autoinc, rev = getattr(urldata[scms[0]].method, method_name)(urldata[scms[0]], d, urldata[scms[0]].name)
         revs.append(rev)
         if len(rev) > 10:
             rev = rev[:10]
@@ -808,13 +828,12 @@ def _get_srcrev(d, method_name='sortable_revision'):
     seenautoinc = False
     for scm in scms:
         ud = urldata[scm]
-        for name in ud.names:
-            autoinc, rev = getattr(ud.method, method_name)(ud, d, name)
-            revs.append(rev)
-            seenautoinc = seenautoinc or autoinc
-            if len(rev) > 10:
-                rev = rev[:10]
-            name_to_rev[name] = rev
+        autoinc, rev = getattr(ud.method, method_name)(ud, d, ud.name)
+        revs.append(rev)
+        seenautoinc = seenautoinc or autoinc
+        if len(rev) > 10:
+            rev = rev[:10]
+        name_to_rev[ud.name] = rev
     # Replace names by revisions in the SRCREV_FORMAT string. The approach used
     # here can handle names being prefixes of other names and names appearing
     # as substrings in revisions (in which case the name should not be
@@ -878,6 +897,7 @@ FETCH_EXPORT_VARS = ['HOME', 'PATH',
                      'AWS_SESSION_TOKEN',
                      'GIT_CACHE_PATH',
                      'REMOTE_CONTAINERS_IPC',
+                     'GITHUB_TOKEN',
                      'SSL_CERT_DIR']
 
 def get_fetcher_environment(d):
@@ -1174,7 +1194,7 @@ def trusted_network(d, url):
     if bb.utils.to_boolean(d.getVar("BB_NO_NETWORK")):
         return True
 
-    pkgname = d.expand(d.getVar('PN', False))
+    pkgname = d.getVar('PN')
     trusted_hosts = None
     if pkgname:
         trusted_hosts = d.getVarFlag('BB_ALLOWED_NETWORKS', pkgname, False)
@@ -1227,20 +1247,17 @@ def srcrev_internal_helper(ud, d, name):
         if srcrev and srcrev != "INVALID":
             break
 
-    if 'rev' in ud.parm and 'tag' in ud.parm:
-        raise FetchError("Please specify a ;rev= parameter or a ;tag= parameter in the url %s but not both." % (ud.url))
-
-    if 'rev' in ud.parm or 'tag' in ud.parm:
-        if 'rev' in ud.parm:
-            parmrev = ud.parm['rev']
-        else:
-            parmrev = ud.parm['tag']
+    if 'rev' in ud.parm:
+        parmrev = ud.parm['rev']
         if srcrev == "INVALID" or not srcrev:
             return parmrev
         if srcrev != parmrev:
             raise FetchError("Conflicting revisions (%s from SRCREV and %s from the url) found, please specify one valid value" % (srcrev, parmrev))
         return parmrev
 
+    if 'tag' in ud.parm and (srcrev == "INVALID" or not srcrev):
+        return ud.parm['tag']
+
     if srcrev == "INVALID" or not srcrev:
         raise FetchError("Please set a valid SRCREV for url %s (possible key names are %s, or use a ;rev=X URL parameter)" % (str(attempts), ud.url), ud.url)
     if srcrev == "AUTOINC":
@@ -1263,7 +1280,7 @@ def get_checksum_file_list(d):
             found = False
             paths = ud.method.localfile_searchpaths(ud, d)
             for f in paths:
-                pth = ud.decodedurl
+                pth = ud.path
                 if os.path.exists(f):
                     found = True
                 filelist.append(f + ":" + str(os.path.exists(f)))
@@ -1308,23 +1325,28 @@ class FetchData(object):
         self.setup = False
 
         def configure_checksum(checksum_id):
+            checksum_plain_name = "%ssum" % checksum_id
             if "name" in self.parm:
                 checksum_name = "%s.%ssum" % (self.parm["name"], checksum_id)
             else:
-                checksum_name = "%ssum" % checksum_id
-
-            setattr(self, "%s_name" % checksum_id, checksum_name)
+                checksum_name = checksum_plain_name
 
             if checksum_name in self.parm:
                 checksum_expected = self.parm[checksum_name]
-            elif self.type not in ["http", "https", "ftp", "ftps", "sftp", "s3", "az", "crate", "gs", "gomod"]:
+            elif checksum_plain_name in self.parm:
+                checksum_expected = self.parm[checksum_plain_name]
+                checksum_name = checksum_plain_name
+            elif self.type not in ["http", "https", "ftp", "ftps", "sftp", "s3", "az", "crate", "gs", "gomod", "npm"]:
                 checksum_expected = None
             else:
                 checksum_expected = d.getVarFlag("SRC_URI", checksum_name)
 
+            setattr(self, "%s_name" % checksum_id, checksum_name)
             setattr(self, "%s_expected" % checksum_id, checksum_expected)
 
-        self.names = self.parm.get("name",'default').split(',')
+        self.name = self.parm.get("name",'default')
+        if "," in self.name:
+            raise ParameterError("The fetcher no longer supports multiple name parameters in a single url", self.url)
 
         self.method = None
         for m in methods:
@@ -1376,13 +1398,7 @@ class FetchData(object):
         self.lockfile = basepath + '.lock'
 
     def setup_revisions(self, d):
-        self.revisions = {}
-        for name in self.names:
-            self.revisions[name] = srcrev_internal_helper(self, d, name)
-
-        # add compatibility code for non name specified case
-        if len(self.names) == 1:
-            self.revision = self.revisions[self.names[0]]
+        self.revision = srcrev_internal_helper(self, d, self.name)
 
     def setup_localpath(self, d):
         if not self.localpath:
@@ -1510,7 +1526,7 @@ class FetchMethod(object):
                      (file, urldata.parm.get('unpack')))
 
         base, ext = os.path.splitext(file)
-        if ext in ['.gz', '.bz2', '.Z', '.xz', '.lz']:
+        if ext in ['.gz', '.bz2', '.Z', '.xz', '.lz', '.zst']:
             efile = os.path.join(rootdir, os.path.basename(base))
         else:
             efile = file
@@ -1662,13 +1678,13 @@ class FetchMethod(object):
         if not hasattr(self, "_latest_revision"):
             raise ParameterError("The fetcher for this URL does not support _latest_revision", ud.url)
 
-        with bb.persist_data.persist('BB_URI_HEADREVS', d) as revs:
-            key = self.generate_revision_key(ud, d, name)
-            try:
-                return revs[key]
-            except KeyError:
-                revs[key] = rev = self._latest_revision(ud, d, name)
-                return rev
+        key = self.generate_revision_key(ud, d, name)
+
+        rev = _revisions_cache.get_rev(key)
+        if rev is None:
+            rev = self._latest_revision(ud, d, name)
+            _revisions_cache.set_rev(key, rev)
+        return rev
 
     def sortable_revision(self, ud, d, name):
         latest_rev = self._build_revision(ud, d, name)
@@ -1806,7 +1822,7 @@ class Fetch(object):
             self.ud[url] = FetchData(url, self.d)
 
         self.ud[url].setup_localpath(self.d)
-        return self.d.expand(self.ud[url].localpath)
+        return self.ud[url].localpath
 
     def localpaths(self):
         """
@@ -1859,25 +1875,28 @@ class Fetch(object):
                             logger.debug(str(e))
                             done = False
 
+                d = self.d
                 if premirroronly:
-                    self.d.setVar("BB_NO_NETWORK", "1")
+                    # Only disable the network in a copy
+                    d = bb.data.createCopy(self.d)
+                    d.setVar("BB_NO_NETWORK", "1")
 
                 firsterr = None
                 verified_stamp = False
                 if done:
-                    verified_stamp = m.verify_donestamp(ud, self.d)
-                if not done and (not verified_stamp or m.need_update(ud, self.d)):
+                    verified_stamp = m.verify_donestamp(ud, d)
+                if not done and (not verified_stamp or m.need_update(ud, d)):
                     try:
-                        if not trusted_network(self.d, ud.url):
+                        if not trusted_network(d, ud.url):
                             raise UntrustedUrl(ud.url)
                         logger.debug("Trying Upstream")
-                        m.download(ud, self.d)
+                        m.download(ud, d)
                         if hasattr(m, "build_mirror_data"):
-                            m.build_mirror_data(ud, self.d)
+                            m.build_mirror_data(ud, d)
                         done = True
                         # early checksum verify, so that if checksum mismatched,
                         # fetcher still have chance to fetch from mirror
-                        m.update_donestamp(ud, self.d)
+                        m.update_donestamp(ud, d)
 
                     except bb.fetch2.NetworkAccess:
                         raise
@@ -1896,17 +1915,17 @@ class Fetch(object):
                         firsterr = e
                         # Remove any incomplete fetch
                         if not verified_stamp and m.cleanup_upon_failure():
-                            m.clean(ud, self.d)
+                            m.clean(ud, d)
                         logger.debug("Trying MIRRORS")
-                        mirrors = mirror_from_string(self.d.getVar('MIRRORS'))
-                        done = m.try_mirrors(self, ud, self.d, mirrors)
+                        mirrors = mirror_from_string(d.getVar('MIRRORS'))
+                        done = m.try_mirrors(self, ud, d, mirrors)
 
-                if not done or not m.done(ud, self.d):
+                if not done or not m.done(ud, d):
                     if firsterr:
                         logger.error(str(firsterr))
                     raise FetchError("Unable to fetch URL from any source.", u)
 
-                m.update_donestamp(ud, self.d)
+                m.update_donestamp(ud, d)
 
             except IOError as e:
                 if e.errno in [errno.ESTALE]:

bitbake/lib/bb/fetch2/az.py

@@ -66,11 +66,12 @@ class Az(Wget):
         else:
             azuri = '%s%s%s' % ('https://', ud.host, ud.path)
 
+        dldir = d.getVar("DL_DIR")
         if os.path.exists(ud.localpath):
             # file exists, but we didnt complete it.. trying again.
-            fetchcmd += d.expand(" -c -P ${DL_DIR} '%s'" % azuri)
+            fetchcmd += " -c -P %s '%s'" % (dldir, azuri)
         else:
-            fetchcmd += d.expand(" -P ${DL_DIR} '%s'" % azuri)
+            fetchcmd += " -P %s '%s'" % (dldir, azuri)
 
         try:
             self._runwget(ud, d, fetchcmd, False)

bitbake/lib/bb/fetch2/clearcase.py

@@ -130,8 +130,6 @@ class ClearCase(FetchMethod):
         self.debug("configspecfile  = %s" % ud.configspecfile)
         self.debug("localfile       = %s" % ud.localfile)
 
-        ud.localfile = os.path.join(d.getVar("DL_DIR"), ud.localfile)
-
     def _build_ccase_command(self, ud, command):
         """
         Build up a commandline based on ud

bitbake/lib/bb/fetch2/gcp.py

@@ -46,7 +46,7 @@ class GCP(FetchMethod):
         else:
             ud.basename = os.path.basename(ud.path)
 
-        ud.localfile = d.expand(urllib.parse.unquote(ud.basename))
+        ud.localfile = ud.basename
 
     def get_gcp_client(self):
         from google.cloud import storage

bitbake/lib/bb/fetch2/git.py

@@ -9,15 +9,6 @@ Supported SRC_URI options are:
 - branch
    The git branch to retrieve from. The default is "master"
 
-   This option also supports multiple branch fetching, with branches
-   separated by commas.  In multiple branches case, the name option
-   must have the same number of names to match the branches, which is
-   used to specify the SRC_REV for the branch
-   e.g:
-   SRC_URI="git://some.host/somepath;branch=branchX,branchY;name=nameX,nameY"
-   SRCREV_nameX = "xxxxxxxxxxxxxxxxxxxx"
-   SRCREV_nameY = "YYYYYYYYYYYYYYYYYYYY"
-
 - tag
     The git tag to retrieve. The default is "master"
 
@@ -81,6 +72,7 @@ import shlex
 import shutil
 import subprocess
 import tempfile
+import urllib
 import bb
 import bb.progress
 from contextlib import contextmanager
@@ -191,13 +183,10 @@ class Git(FetchMethod):
         if ud.bareclone:
             ud.nocheckout = 1
   
-        ud.unresolvedrev = {}
-        branches = ud.parm.get("branch", "").split(',')
-        if branches == [""] and not ud.nobranch:
-            bb.warn("URL: %s does not set any branch parameter. The future default branch used by tools and repositories is uncertain and we will therefore soon require this is set in all git urls." % ud.url)
-            branches = ["master"]
-        if len(branches) != len(ud.names):
-            raise bb.fetch2.ParameterError("The number of name and branch parameters is not balanced", ud.url)
+        ud.unresolvedrev = ""
+        ud.branch = ud.parm.get("branch", "")
+        if not ud.branch and not ud.nobranch:
+            raise bb.fetch2.ParameterError("The url does not set any branch parameter or set nobranch=1.", ud.url)
 
         ud.noshared = d.getVar("BB_GIT_NOSHARED") == "1"
 
@@ -207,6 +196,7 @@ class Git(FetchMethod):
         if ud.bareclone:
             ud.cloneflags += " --mirror"
 
+        ud.shallow_skip_fast = False
         ud.shallow = d.getVar("BB_GIT_SHALLOW") == "1"
         ud.shallow_extra_refs = (d.getVar("BB_GIT_SHALLOW_EXTRA_REFS") or "").split()
 
@@ -225,32 +215,27 @@ class Git(FetchMethod):
 
         revs_default = d.getVar("BB_GIT_SHALLOW_REVS")
         ud.shallow_revs = []
-        ud.branches = {}
-        for pos, name in enumerate(ud.names):
-            branch = branches[pos]
-            ud.branches[name] = branch
-            ud.unresolvedrev[name] = branch
 
-            shallow_depth = d.getVar("BB_GIT_SHALLOW_DEPTH_%s" % name)
-            if shallow_depth is not None:
-                try:
-                    shallow_depth = int(shallow_depth or 0)
-                except ValueError:
-                    raise bb.fetch2.FetchError("Invalid depth for BB_GIT_SHALLOW_DEPTH_%s: %s" % (name, shallow_depth))
-                else:
-                    if shallow_depth < 0:
-                        raise bb.fetch2.FetchError("Invalid depth for BB_GIT_SHALLOW_DEPTH_%s: %s" % (name, shallow_depth))
-                    ud.shallow_depths[name] = shallow_depth
+        ud.unresolvedrev = ud.branch
 
-            revs = d.getVar("BB_GIT_SHALLOW_REVS_%s" % name)
-            if revs is not None:
-                ud.shallow_revs.extend(revs.split())
-            elif revs_default is not None:
-                ud.shallow_revs.extend(revs_default.split())
+        shallow_depth = d.getVar("BB_GIT_SHALLOW_DEPTH_%s" % ud.name)
+        if shallow_depth is not None:
+            try:
+                shallow_depth = int(shallow_depth or 0)
+            except ValueError:
+                raise bb.fetch2.FetchError("Invalid depth for BB_GIT_SHALLOW_DEPTH_%s: %s" % (ud.name, shallow_depth))
+            else:
+                if shallow_depth < 0:
+                    raise bb.fetch2.FetchError("Invalid depth for BB_GIT_SHALLOW_DEPTH_%s: %s" % (ud.name, shallow_depth))
+                ud.shallow_depths[ud.name] = shallow_depth
 
-        if (ud.shallow and
-                not ud.shallow_revs and
-                all(ud.shallow_depths[n] == 0 for n in ud.names)):
+        revs = d.getVar("BB_GIT_SHALLOW_REVS_%s" % ud.name)
+        if revs is not None:
+            ud.shallow_revs.extend(revs.split())
+        elif revs_default is not None:
+            ud.shallow_revs.extend(revs_default.split())
+
+        if ud.shallow and not ud.shallow_revs and ud.shallow_depths[ud.name] == 0:
             # Shallow disabled for this URL
             ud.shallow = False
 
@@ -259,8 +244,7 @@ class Git(FetchMethod):
             # rev of this repository. This will get resolved into a revision
             # later. If an actual revision happens to have also been provided
             # then this setting will be overridden.
-            for name in ud.names:
-                ud.unresolvedrev[name] = 'HEAD'
+            ud.unresolvedrev = 'HEAD'
 
         ud.basecmd = d.getVar("FETCHCMD_git") or "git -c gc.autoDetach=false -c core.pager=cat -c safe.bareRepository=all -c clone.defaultRemoteName=origin"
 
@@ -270,12 +254,11 @@ class Git(FetchMethod):
 
         ud.setup_revisions(d)
 
-        for name in ud.names:
-            # Ensure any revision that doesn't look like a SHA-1 is translated into one
-            if not sha1_re.match(ud.revisions[name] or ''):
-                if ud.revisions[name]:
-                    ud.unresolvedrev[name] = ud.revisions[name]
-                ud.revisions[name] = self.latest_revision(ud, d, name)
+        # Ensure any revision that doesn't look like a SHA-1 is translated into one
+        if not sha1_re.match(ud.revision or ''):
+            if ud.revision:
+                ud.unresolvedrev = ud.revision
+            ud.revision = self.latest_revision(ud, d, ud.name)
 
         gitsrcname = '%s%s' % (ud.host.replace(':', '.'), ud.path.replace('/', '.').replace('*', '.').replace(' ','_').replace('(', '_').replace(')', '_'))
         if gitsrcname.startswith('.'):
@@ -286,8 +269,7 @@ class Git(FetchMethod):
         # upstream repo in the future, the mirror will remain intact and still
         # contain the revision
         if ud.rebaseable:
-            for name in ud.names:
-                gitsrcname = gitsrcname + '_' + ud.revisions[name]
+            gitsrcname = gitsrcname + '_' + ud.revision
 
         dl_dir = d.getVar("DL_DIR")
         gitdir = d.getVar("GITDIR") or (dl_dir + "/git2")
@@ -305,15 +287,14 @@ class Git(FetchMethod):
             if ud.shallow_revs:
                 tarballname = "%s_%s" % (tarballname, "_".join(sorted(ud.shallow_revs)))
 
-            for name, revision in sorted(ud.revisions.items()):
-                tarballname = "%s_%s" % (tarballname, ud.revisions[name][:7])
-                depth = ud.shallow_depths[name]
-                if depth:
-                    tarballname = "%s-%s" % (tarballname, depth)
+            tarballname = "%s_%s" % (tarballname, ud.revision[:7])
+            depth = ud.shallow_depths[ud.name]
+            if depth:
+                tarballname = "%s-%s" % (tarballname, depth)
 
             shallow_refs = []
             if not ud.nobranch:
-                shallow_refs.extend(ud.branches.values())
+                shallow_refs.append(ud.branch)
             if ud.shallow_extra_refs:
                 shallow_refs.extend(r.replace('refs/heads/', '').replace('*', 'ALL') for r in ud.shallow_extra_refs)
             if shallow_refs:
@@ -338,18 +319,16 @@ class Git(FetchMethod):
             return True
         if ud.shallow and ud.write_shallow_tarballs and self.clonedir_need_shallow_revs(ud, d):
             return True
-        for name in ud.names:
-            if not self._contains_ref(ud, d, name, ud.clonedir):
-                return True
+        if not self._contains_ref(ud, d, ud.name, ud.clonedir):
+            return True
         return False
 
     def lfs_need_update(self, ud, d):
         if self.clonedir_need_update(ud, d):
             return True
 
-        for name in ud.names:
-            if not self._lfs_objects_downloaded(ud, d, name, ud.clonedir):
-                return True
+        if not self._lfs_objects_downloaded(ud, d, ud.name, ud.clonedir):
+            return True
         return False
 
     def clonedir_need_shallow_revs(self, ud, d):
@@ -446,6 +425,24 @@ class Git(FetchMethod):
             if ud.proto.lower() != 'file':
                 bb.fetch2.check_network_access(d, clone_cmd, ud.url)
             progresshandler = GitProgressHandler(d)
+
+            # Try creating a fast initial shallow clone
+            # Enabling ud.shallow_skip_fast will skip this
+            # If the Git error "Server does not allow request for unadvertised object"
+            # occurs, shallow_skip_fast is enabled automatically.
+            # This may happen if the Git server does not allow the request
+            # or if the Git client has issues with this functionality.
+            if ud.shallow and not ud.shallow_skip_fast:
+                try:
+                    self.clone_shallow_with_tarball(ud, d)
+                    # When the shallow clone has succeeded, use the shallow tarball
+                    ud.localpath = ud.fullshallow
+                    return
+                except:
+                    logger.warning("Creating fast initial shallow clone failed, try initial regular clone now.")
+
+            # When skipping fast initial shallow or the fast inital shallow clone failed:
+            # Try again with an initial regular clone
             runfetchcmd(clone_cmd, d, log=progresshandler)
 
         # Update the checkout if needed
@@ -473,9 +470,8 @@ class Git(FetchMethod):
                 if exc.errno != errno.ENOENT:
                     raise
 
-        for name in ud.names:
-            if not self._contains_ref(ud, d, name, ud.clonedir):
-                raise bb.fetch2.FetchError("Unable to find revision %s in branch %s even from upstream" % (ud.revisions[name], ud.branches[name]))
+        if not self._contains_ref(ud, d, ud.name, ud.clonedir):
+            raise bb.fetch2.FetchError("Unable to find revision %s in branch %s even from upstream" % (ud.revision, ud.branch))
 
         if ud.shallow and ud.write_shallow_tarballs:
             missing_rev = self.clonedir_need_shallow_revs(ud, d)
@@ -508,48 +504,74 @@ class Git(FetchMethod):
                 if os.path.exists(os.path.join(ud.destdir, ".git", "lfs")):
                     runfetchcmd("tar -cf - lfs | tar -xf - -C %s" % ud.clonedir, d, workdir="%s/.git" % ud.destdir)
 
+    def lfs_fetch(self, ud, d, clonedir, revision, fetchall=False, progresshandler=None):
+        """Helper method for fetching Git LFS data"""
+        try:
+            if self._need_lfs(ud) and self._contains_lfs(ud, d, clonedir) and self._find_git_lfs(d) and len(revision):
+                # Using worktree with the revision because .lfsconfig may exists
+                worktree_add_cmd = "%s worktree add wt %s" % (ud.basecmd, revision)
+                runfetchcmd(worktree_add_cmd, d, log=progresshandler, workdir=clonedir)
+                lfs_fetch_cmd = "%s lfs fetch %s" % (ud.basecmd, "--all" if fetchall else "")
+                runfetchcmd(lfs_fetch_cmd, d, log=progresshandler, workdir=(clonedir + "/wt"))
+                worktree_rem_cmd = "%s worktree remove -f wt" % ud.basecmd
+                runfetchcmd(worktree_rem_cmd, d, log=progresshandler, workdir=clonedir)
+        except:
+            logger.warning("Fetching LFS did not succeed.")
+
+    @contextmanager
+    def create_atomic(self, filename):
+        """Create as a temp file and move atomically into position to avoid races"""
+        fd, tfile = tempfile.mkstemp(dir=os.path.dirname(filename))
+        try:
+            yield tfile
+            umask = os.umask(0o666)
+            os.umask(umask)
+            os.chmod(tfile, (0o666 & ~umask))
+            os.rename(tfile, filename)
+        finally:
+            os.close(fd)
+
     def build_mirror_data(self, ud, d):
-
-        # Create as a temp file and move atomically into position to avoid races
-        @contextmanager
-        def create_atomic(filename):
-            fd, tfile = tempfile.mkstemp(dir=os.path.dirname(filename))
-            try:
-                yield tfile
-                umask = os.umask(0o666)
-                os.umask(umask)
-                os.chmod(tfile, (0o666 & ~umask))
-                os.rename(tfile, filename)
-            finally:
-                os.close(fd)
-
         if ud.shallow and ud.write_shallow_tarballs:
             if not os.path.exists(ud.fullshallow):
                 if os.path.islink(ud.fullshallow):
                     os.unlink(ud.fullshallow)
-                tempdir = tempfile.mkdtemp(dir=d.getVar('DL_DIR'))
-                shallowclone = os.path.join(tempdir, 'git')
-                try:
-                    self.clone_shallow_local(ud, shallowclone, d)
-
-                    logger.info("Creating tarball of git repository")
-                    with create_atomic(ud.fullshallow) as tfile:
-                        runfetchcmd("tar -czf %s ." % tfile, d, workdir=shallowclone)
-                    runfetchcmd("touch %s.done" % ud.fullshallow, d)
-                finally:
-                    bb.utils.remove(tempdir, recurse=True)
+                self.clone_shallow_with_tarball(ud, d)
         elif ud.write_tarballs and not os.path.exists(ud.fullmirror):
             if os.path.islink(ud.fullmirror):
                 os.unlink(ud.fullmirror)
 
             logger.info("Creating tarball of git repository")
-            with create_atomic(ud.fullmirror) as tfile:
+            with self.create_atomic(ud.fullmirror) as tfile:
                 mtime = runfetchcmd("{} log --all -1 --format=%cD".format(ud.basecmd), d,
                         quiet=True, workdir=ud.clonedir)
                 runfetchcmd("tar -czf %s --owner oe:0 --group oe:0 --mtime \"%s\" ."
                         % (tfile, mtime), d, workdir=ud.clonedir)
             runfetchcmd("touch %s.done" % ud.fullmirror, d)
 
+    def clone_shallow_with_tarball(self, ud, d):
+        ret = False
+        tempdir = tempfile.mkdtemp(dir=d.getVar('DL_DIR'))
+        shallowclone = os.path.join(tempdir, 'git')
+        try:
+            try:
+                self.clone_shallow_local(ud, shallowclone, d)
+            except:
+                logger.warning("Fash shallow clone failed, try to skip fast mode now.")
+                bb.utils.remove(tempdir, recurse=True)
+                os.mkdir(tempdir)
+                ud.shallow_skip_fast = True
+                self.clone_shallow_local(ud, shallowclone, d)
+            logger.info("Creating tarball of git repository")
+            with self.create_atomic(ud.fullshallow) as tfile:
+                runfetchcmd("tar -czf %s ." % tfile, d, workdir=shallowclone)
+            runfetchcmd("touch %s.done" % ud.fullshallow, d)
+            ret = True
+        finally:
+            bb.utils.remove(tempdir, recurse=True)
+
+        return ret
+
     def clone_shallow_local(self, ud, dest, d):
         """
         Shallow fetch from ud.clonedir (${DL_DIR}/git2/<gitrepo> by default):
@@ -557,53 +579,64 @@ class Git(FetchMethod):
         - For BB_GIT_SHALLOW_REVS: git fetch --shallow-exclude=<revs> rev
         """
 
+        progresshandler = GitProgressHandler(d)
+        repourl = self._get_repo_url(ud)
         bb.utils.mkdirhier(dest)
         init_cmd = "%s init -q" % ud.basecmd
         if ud.bareclone:
             init_cmd += " --bare"
         runfetchcmd(init_cmd, d, workdir=dest)
-        runfetchcmd("%s remote add origin %s" % (ud.basecmd, ud.clonedir), d, workdir=dest)
+        # Use repourl when creating a fast initial shallow clone
+        # Prefer already existing full bare clones if available
+        if not ud.shallow_skip_fast and not os.path.exists(ud.clonedir):
+            remote = shlex.quote(repourl)
+        else:
+            remote = ud.clonedir
+        runfetchcmd("%s remote add origin %s" % (ud.basecmd, remote), d, workdir=dest)
 
         # Check the histories which should be excluded
         shallow_exclude = ''
         for revision in ud.shallow_revs:
             shallow_exclude += " --shallow-exclude=%s" % revision
 
-        for name in ud.names:
-            revision = ud.revisions[name]
-            depth = ud.shallow_depths[name]
+        revision = ud.revision
+        depth = ud.shallow_depths[ud.name]
 
-            # The --depth and --shallow-exclude can't be used together
-            if depth and shallow_exclude:
-                raise bb.fetch2.FetchError("BB_GIT_SHALLOW_REVS is set, but BB_GIT_SHALLOW_DEPTH is not 0.")
+        # The --depth and --shallow-exclude can't be used together
+        if depth and shallow_exclude:
+            raise bb.fetch2.FetchError("BB_GIT_SHALLOW_REVS is set, but BB_GIT_SHALLOW_DEPTH is not 0.")
 
-            # For nobranch, we need a ref, otherwise the commits will be
-            # removed, and for non-nobranch, we truncate the branch to our
-            # srcrev, to avoid keeping unnecessary history beyond that.
-            branch = ud.branches[name]
-            if ud.nobranch:
-                ref = "refs/shallow/%s" % name
-            elif ud.bareclone:
-                ref = "refs/heads/%s" % branch
-            else:
-                ref = "refs/remotes/origin/%s" % branch
+        # For nobranch, we need a ref, otherwise the commits will be
+        # removed, and for non-nobranch, we truncate the branch to our
+        # srcrev, to avoid keeping unnecessary history beyond that.
+        branch = ud.branch
+        if ud.nobranch:
+            ref = "refs/shallow/%s" % ud.name
+        elif ud.bareclone:
+            ref = "refs/heads/%s" % branch
+        else:
+            ref = "refs/remotes/origin/%s" % branch
 
-            fetch_cmd = "%s fetch origin %s" % (ud.basecmd, revision)
-            if depth:
-                fetch_cmd += " --depth %s" % depth
+        fetch_cmd = "%s fetch origin %s" % (ud.basecmd, revision)
+        if depth:
+            fetch_cmd += " --depth %s" % depth
 
-            if shallow_exclude:
-                fetch_cmd += shallow_exclude
+        if shallow_exclude:
+            fetch_cmd += shallow_exclude
 
-            # Advertise the revision for lower version git such as 2.25.1:
-            # error: Server does not allow request for unadvertised object.
-            # The ud.clonedir is a local temporary dir, will be removed when
-            # fetch is done, so we can do anything on it.
-            adv_cmd = 'git branch -f advertise-%s %s' % (revision, revision)
+        # Advertise the revision for lower version git such as 2.25.1:
+        # error: Server does not allow request for unadvertised object.
+        # The ud.clonedir is a local temporary dir, will be removed when
+        # fetch is done, so we can do anything on it.
+        adv_cmd = 'git branch -f advertise-%s %s' % (revision, revision)
+        if ud.shallow_skip_fast:
             runfetchcmd(adv_cmd, d, workdir=ud.clonedir)
 
-            runfetchcmd(fetch_cmd, d, workdir=dest)
-            runfetchcmd("%s update-ref %s %s" % (ud.basecmd, ref, revision), d, workdir=dest)
+        runfetchcmd(fetch_cmd, d, workdir=dest)
+        runfetchcmd("%s update-ref %s %s" % (ud.basecmd, ref, revision), d, workdir=dest)
+        # Fetch Git LFS data for fast shallow clones
+        if not ud.shallow_skip_fast:
+            self.lfs_fetch(ud, d, dest, ud.revision)
 
         # Apply extra ref wildcards
         all_refs_remote = runfetchcmd("%s ls-remote origin 'refs/*'" % ud.basecmd, \
@@ -612,6 +645,8 @@ class Git(FetchMethod):
         for line in all_refs_remote:
             all_refs.append(line.split()[-1])
         extra_refs = []
+        if 'tag' in ud.parm:
+            extra_refs.append(ud.parm['tag'])
         for r in ud.shallow_extra_refs:
             if not ud.bareclone:
                 r = r.replace('refs/heads/', 'refs/remotes/origin/')
@@ -629,7 +664,6 @@ class Git(FetchMethod):
             runfetchcmd("%s update-ref %s %s" % (ud.basecmd, ref, revision), d, workdir=dest)
 
         # The url is local ud.clonedir, set it to upstream one
-        repourl = self._get_repo_url(ud)
         runfetchcmd("%s remote set-url origin %s" % (ud.basecmd, shlex.quote(repourl)), d, workdir=dest)
 
     def unpack(self, ud, destdir, d):
@@ -690,6 +724,14 @@ class Git(FetchMethod):
         if not source_found:
             raise bb.fetch2.UnpackError("No up to date source found: " + "; ".join(source_error), ud.url)
 
+        # If there is a tag parameter in the url and we also have a fixed srcrev, check the tag
+        # matches the revision
+        if 'tag' in ud.parm and sha1_re.match(ud.revision):
+            output = runfetchcmd("%s rev-list -n 1 %s" % (ud.basecmd, ud.parm['tag']), d, workdir=destdir)
+            output = output.strip()
+            if output != ud.revision:
+                raise bb.fetch2.FetchError("The revision the git tag '%s' resolved to didn't match the SRCREV in use (%s vs %s)" % (ud.parm['tag'], output, ud.revision), ud.url)
+
         repourl = self._get_repo_url(ud)
         runfetchcmd("%s remote set-url origin %s" % (ud.basecmd, shlex.quote(repourl)), d, workdir=destdir)
 
@@ -703,17 +745,17 @@ class Git(FetchMethod):
 
         if not ud.nocheckout:
             if subpath:
-                runfetchcmd("%s read-tree %s%s" % (ud.basecmd, ud.revisions[ud.names[0]], readpathspec), d,
+                runfetchcmd("%s read-tree %s%s" % (ud.basecmd, ud.revision, readpathspec), d,
                             workdir=destdir)
                 runfetchcmd("%s checkout-index -q -f -a" % ud.basecmd, d, workdir=destdir)
             elif not ud.nobranch:
-                branchname =  ud.branches[ud.names[0]]
+                branchname =  ud.branch
                 runfetchcmd("%s checkout -B %s %s" % (ud.basecmd, branchname, \
-                            ud.revisions[ud.names[0]]), d, workdir=destdir)
+                            ud.revision), d, workdir=destdir)
                 runfetchcmd("%s branch %s --set-upstream-to origin/%s" % (ud.basecmd, branchname, \
                             branchname), d, workdir=destdir)
             else:
-                runfetchcmd("%s checkout %s" % (ud.basecmd, ud.revisions[ud.names[0]]), d, workdir=destdir)
+                runfetchcmd("%s checkout %s" % (ud.basecmd, ud.revision), d, workdir=destdir)
 
         return True
 
@@ -727,8 +769,13 @@ class Git(FetchMethod):
             clonedir = os.path.realpath(ud.localpath)
             to_remove.append(clonedir)
 
+        # Remove shallow mirror tarball
+        if ud.shallow:
+            to_remove.append(ud.fullshallow)
+            to_remove.append(ud.fullshallow + ".done")
+
         for r in to_remove:
-            if os.path.exists(r):
+            if os.path.exists(r) or os.path.islink(r):
                 bb.note('Removing %s' % r)
                 bb.utils.remove(r, True)
 
@@ -739,10 +786,10 @@ class Git(FetchMethod):
         cmd = ""
         if ud.nobranch:
             cmd = "%s log --pretty=oneline -n 1 %s -- 2> /dev/null | wc -l" % (
-                ud.basecmd, ud.revisions[name])
+                ud.basecmd, ud.revision)
         else:
             cmd =  "%s branch --contains %s --list %s 2> /dev/null | wc -l" % (
-                ud.basecmd, ud.revisions[name], ud.branches[name])
+                ud.basecmd, ud.revision, ud.branch)
         try:
             output = runfetchcmd(cmd, d, quiet=True, workdir=wd)
         except bb.fetch2.FetchError:
@@ -763,7 +810,7 @@ class Git(FetchMethod):
         # existence.
         # [1] https://github.com/git-lfs/git-lfs/blob/main/docs/spec.md#intercepting-git
         cmd = "%s lfs ls-files -l %s" \
-                % (ud.basecmd, ud.revisions[name])
+                % (ud.basecmd, ud.revision)
         output = runfetchcmd(cmd, d, quiet=True, workdir=wd).rstrip()
         # Do not do any further matching if no objects are managed by LFS
         if not output:
@@ -790,12 +837,12 @@ class Git(FetchMethod):
 
         if ud.nobranch:
             # If no branch is specified, use the current git commit
-            refname = self._build_revision(ud, d, ud.names[0])
+            refname = self._build_revision(ud, d, ud.name)
         elif wd == ud.clonedir:
             # The bare clonedir doesn't use the remote names; it has the branch immediately.
-            refname = ud.branches[ud.names[0]]
+            refname = ud.branch
         else:
-            refname = "origin/%s" % ud.branches[ud.names[0]]
+            refname = "origin/%s" % ud.branch
 
         cmd = "%s grep lfs %s:.gitattributes | wc -l" % (
             ud.basecmd, refname)
@@ -812,7 +859,6 @@ class Git(FetchMethod):
         """
         Return True if git-lfs can be found, False otherwise.
         """
-        import shutil
         return shutil.which("git-lfs", path=d.getVar('PATH')) is not None
 
     def _get_repo_url(self, ud):
@@ -828,14 +874,14 @@ class Git(FetchMethod):
             username = ud.user + '@'
         else:
             username = ""
-        return "%s://%s%s%s" % (ud.proto, username, ud.host, ud.path)
+        return "%s://%s%s%s" % (ud.proto, username, ud.host, urllib.parse.quote(ud.path))
 
     def _revision_key(self, ud, d, name):
         """
         Return a unique key for the url
         """
         # Collapse adjacent slashes
-        return "git:" + ud.host + slash_re.sub(".", ud.path) + ud.unresolvedrev[name]
+        return "git:" + ud.host + slash_re.sub(".", ud.path) + ud.unresolvedrev
 
     def _lsremote(self, ud, d, search):
         """
@@ -868,26 +914,26 @@ class Git(FetchMethod):
         Compute the HEAD revision for the url
         """
         if not d.getVar("__BBSRCREV_SEEN"):
-            raise bb.fetch2.FetchError("Recipe uses a floating tag/branch '%s' for repo '%s' without a fixed SRCREV yet doesn't call bb.fetch2.get_srcrev() (use SRCPV in PV for OE)." % (ud.unresolvedrev[name], ud.host+ud.path))
+            raise bb.fetch2.FetchError("Recipe uses a floating tag/branch '%s' for repo '%s' without a fixed SRCREV yet doesn't call bb.fetch2.get_srcrev() (use SRCPV in PV for OE)." % (ud.unresolvedrev, ud.host+ud.path))
 
         # Ensure we mark as not cached
         bb.fetch2.mark_recipe_nocache(d)
 
         output = self._lsremote(ud, d, "")
         # Tags of the form ^{} may not work, need to fallback to other form
-        if ud.unresolvedrev[name][:5] == "refs/" or ud.usehead:
-            head = ud.unresolvedrev[name]
-            tag = ud.unresolvedrev[name]
+        if ud.unresolvedrev[:5] == "refs/" or ud.usehead:
+            head = ud.unresolvedrev
+            tag = ud.unresolvedrev
         else:
-            head = "refs/heads/%s" % ud.unresolvedrev[name]
-            tag = "refs/tags/%s" % ud.unresolvedrev[name]
+            head = "refs/heads/%s" % ud.unresolvedrev
+            tag = "refs/tags/%s" % ud.unresolvedrev
         for s in [head, tag + "^{}", tag]:
             for l in output.strip().split('\n'):
                 sha1, ref = l.split()
                 if s == ref:
                     return sha1
         raise bb.fetch2.FetchError("Unable to resolve '%s' in upstream git repository in git ls-remote output for %s" % \
-            (ud.unresolvedrev[name], ud.host+ud.path))
+            (ud.unresolvedrev, ud.host+ud.path))
 
     def latest_versionstring(self, ud, d):
         """
@@ -938,7 +984,7 @@ class Git(FetchMethod):
         return pupver
 
     def _build_revision(self, ud, d, name):
-        return ud.revisions[name]
+        return ud.revision
 
     def gitpkgv_revision(self, ud, d, name):
         """

bitbake/lib/bb/fetch2/gitsm.py

@@ -62,36 +62,35 @@ class GitSM(Git):
             return modules
 
         # Collect the defined submodules, and their attributes
-        for name in ud.names:
+        try:
+            gitmodules = runfetchcmd("%s show %s:.gitmodules" % (ud.basecmd, ud.revision), d, quiet=True, workdir=workdir)
+        except:
+            # No submodules to update
+            gitmodules = ""
+
+        for m, md in parse_gitmodules(gitmodules).items():
             try:
-                gitmodules = runfetchcmd("%s show %s:.gitmodules" % (ud.basecmd, ud.revisions[name]), d, quiet=True, workdir=workdir)
+                module_hash = runfetchcmd("%s ls-tree -z -d %s %s" % (ud.basecmd, ud.revision, md['path']), d, quiet=True, workdir=workdir)
             except:
-                # No submodules to update
+                # If the command fails, we don't have a valid file to check.  If it doesn't
+                # fail -- it still might be a failure, see next check...
+                module_hash = ""
+
+            if not module_hash:
+                logger.debug("submodule %s is defined, but is not initialized in the repository. Skipping", m)
                 continue
 
-            for m, md in parse_gitmodules(gitmodules).items():
-                try:
-                    module_hash = runfetchcmd("%s ls-tree -z -d %s %s" % (ud.basecmd, ud.revisions[name], md['path']), d, quiet=True, workdir=workdir)
-                except:
-                    # If the command fails, we don't have a valid file to check.  If it doesn't
-                    # fail -- it still might be a failure, see next check...
-                    module_hash = ""
+            submodules.append(m)
+            paths[m] = md['path']
+            revision[m] = ud.revision
+            uris[m] = md['url']
+            subrevision[m] = module_hash.split()[2]
 
-                if not module_hash:
-                    logger.debug("submodule %s is defined, but is not initialized in the repository. Skipping", m)
-                    continue
-
-                submodules.append(m)
-                paths[m] = md['path']
-                revision[m] = ud.revisions[name]
-                uris[m] = md['url']
-                subrevision[m] = module_hash.split()[2]
-
-                # Convert relative to absolute uri based on parent uri
-                if  uris[m].startswith('..') or uris[m].startswith('./'):
-                    newud = copy.copy(ud)
-                    newud.path = os.path.normpath(os.path.join(newud.path, uris[m]))
-                    uris[m] = Git._get_repo_url(self, newud)
+            # Convert relative to absolute uri based on parent uri
+            if  uris[m].startswith('..') or uris[m].startswith('./'):
+                newud = copy.copy(ud)
+                newud.path = os.path.normpath(os.path.join(newud.path, uris[m]))
+                uris[m] = Git._get_repo_url(self, newud)
 
         for module in submodules:
             # Translate the module url into a SRC_URI
@@ -150,7 +149,10 @@ class GitSM(Git):
     def call_process_submodules(self, ud, d, extra_check, subfunc):
         # If we're using a shallow mirror tarball it needs to be
         # unpacked temporarily so that we can examine the .gitmodules file
-        if ud.shallow and os.path.exists(ud.fullshallow) and extra_check:
+        # Unpack even when ud.clonedir is not available,
+        # which may occur during a fast shallow clone
+        unpack = extra_check or not os.path.exists(ud.clonedir)
+        if ud.shallow and os.path.exists(ud.fullshallow) and unpack:
             tmpdir = tempfile.mkdtemp(dir=d.getVar("DL_DIR"))
             try:
                 runfetchcmd("tar -xzf %s" % ud.fullshallow, d, workdir=tmpdir)
@@ -249,9 +251,22 @@ class GitSM(Git):
             # should also be skipped as these files were already smudged in the fetch stage if lfs
             # was enabled.
             runfetchcmd("GIT_LFS_SKIP_SMUDGE=1 %s submodule update --recursive --no-fetch" % (ud.basecmd), d, quiet=True, workdir=ud.destdir)
+    def clean(self, ud, d):
+        def clean_submodule(ud, url, module, modpath, workdir, d):
+            url += ";bareclone=1;nobranch=1"
+            try:
+                newfetch = Fetch([url], d, cache=False)
+                newfetch.clean()
+            except Exception as e:
+                logger.warning('gitsm: submodule clean failed: %s %s' % (type(e).__name__, str(e)))
+
+        self.call_process_submodules(ud, d, True, clean_submodule)
+
+        # Clean top git dir
+        Git.clean(self, ud, d)
 
     def implicit_urldata(self, ud, d):
-        import shutil, subprocess, tempfile
+        import subprocess
 
         urldata = []
         def add_submodule(ud, url, module, modpath, workdir, d):

bitbake/lib/bb/fetch2/gomod.py

@@ -107,26 +107,23 @@ class GoMod(Wget):
         if ud.path != '/':
             module += ud.path
         ud.parm['module'] = module
+        version = ud.parm['version']
 
         # Set URL and filename for wget download
-        path = escape(module + '/@v/' + ud.parm['version'])
         if ud.parm.get('mod', '0') == '1':
-            path += '.mod'
+            ext = '.mod'
         else:
-            path += '.zip'
-            ud.parm['unpack'] = '0'
+            ext = '.zip'
+        path = escape(f"{module}/@v/{version}{ext}")
         ud.url = bb.fetch2.encodeurl(
             ('https', proxy, '/' + path, None, None, None))
-        ud.parm['downloadfilename'] = path
+        ud.parm['downloadfilename'] =  f"{module.replace('/', '.')}@{version}{ext}"
 
-        # Set name parameter if sha256sum is set in recipe
-        name = f"{module}@{ud.parm['version']}"
-        if d.getVarFlag('SRC_URI', name + '.sha256sum'):
-            ud.parm['name'] = name
+        # Set name for checksum verification
+        ud.parm['name'] = f"{module}@{version}"
 
-        # Set subdir for unpack
-        ud.parm['subdir'] = os.path.join(moddir, 'cache/download',
-                                         os.path.dirname(path))
+        # Set path for unpack
+        ud.parm['unpackpath'] = os.path.join(moddir, 'cache/download', path)
 
         super().urldata_init(ud, d)
 
@@ -134,13 +131,22 @@ class GoMod(Wget):
         """Unpack the module in the module cache."""
 
         # Unpack the module zip file or go.mod file
-        super().unpack(ud, rootdir, d)
+        unpackpath = os.path.join(rootdir, ud.parm['unpackpath'])
+        unpackdir = os.path.dirname(unpackpath)
+        bb.utils.mkdirhier(unpackdir)
+        ud.unpack_tracer.unpack("file-copy", unpackdir)
+        cmd = f"cp {ud.localpath} {unpackpath}"
+        path = d.getVar('PATH')
+        if path:
+            cmd = f"PATH={path} {cmd}"
+        name = os.path.basename(unpackpath)
+        bb.note(f"Unpacking {name} to {unpackdir}/")
+        subprocess.check_call(cmd, shell=True, preexec_fn=subprocess_setup)
 
-        if ud.localpath.endswith('.zip'):
+        if name.endswith('.zip'):
             # Unpack the go.mod file from the zip file
             module = ud.parm['module']
-            unpackdir = os.path.join(rootdir, ud.parm['subdir'])
-            name = os.path.basename(ud.localpath).rsplit('.', 1)[0] + '.mod'
+            name = name.rsplit('.', 1)[0] + '.mod'
             bb.note(f"Unpacking {name} to {unpackdir}/")
             with zipfile.ZipFile(ud.localpath) as zf:
                 with open(os.path.join(unpackdir, name), mode='wb') as mf:
@@ -193,15 +199,14 @@ class GoModGit(Git):
                 ud.path = ''
         if 'protocol' not in ud.parm:
             ud.parm['protocol'] = 'https'
-        name = f"{module}@{ud.parm['version']}"
-        ud.names = [name]
-        srcrev = d.getVar('SRCREV_' + name)
+        ud.name = f"{module}@{ud.parm['version']}"
+        srcrev = d.getVar('SRCREV_' + ud.name)
         if srcrev:
             if 'srcrev' not in ud.parm:
                 ud.parm['srcrev'] = srcrev
         else:
             if 'srcrev' in ud.parm:
-                d.setVar('SRCREV_' + name, ud.parm['srcrev'])
+                d.setVar('SRCREV_' + ud.name, ud.parm['srcrev'])
         if 'branch' not in ud.parm:
             ud.parm['nobranch'] = '1'
 

bitbake/lib/bb/fetch2/local.py

@@ -29,11 +29,10 @@ class Local(FetchMethod):
 
     def urldata_init(self, ud, d):
         # We don't set localfile as for this fetcher the file is already local!
-        ud.decodedurl = urllib.parse.unquote(ud.url.split("://")[1].split(";")[0])
-        ud.basename = os.path.basename(ud.decodedurl)
-        ud.basepath = ud.decodedurl
+        ud.basename = os.path.basename(ud.path)
+        ud.basepath = ud.path
         ud.needdonestamp = False
-        if "*" in ud.decodedurl:
+        if "*" in ud.path:
             raise bb.fetch2.ParameterError("file:// urls using globbing are no longer supported. Please place the files in a directory and reference that instead.", ud.url)
         return
 
@@ -48,7 +47,7 @@ class Local(FetchMethod):
         Return the local filename of a given url assuming a successful fetch.
         """
         searched = []
-        path = urldata.decodedurl
+        path = urldata.path
         newpath = path
         if path[0] == "/":
             logger.debug2("Using absolute %s" % (path))

bitbake/lib/bb/fetch2/npm.py

@@ -91,6 +91,12 @@ class NpmEnvironment(object):
         self.d = d
 
         self.user_config = tempfile.NamedTemporaryFile(mode="w", buffering=1)
+
+        hn = self._home_npmrc(d)
+        if hn is not None:
+            with open(hn, 'r') as hnf:
+                self.user_config.write(hnf.read())
+
         for key, value in configs:
             self.user_config.write("%s=%s\n" % (key, value))
 
@@ -103,6 +109,15 @@ class NpmEnvironment(object):
         if self.user_config:
             self.user_config.close()
 
+    def _home_npmrc(self, d):
+        """Function to return user's HOME .npmrc file (or None if it doesn't exist)"""
+        home_npmrc_file = os.path.join(os.environ.get("HOME"), ".npmrc")
+        if d.getVar("BB_USE_HOME_NPMRC") == "1" and os.path.exists(home_npmrc_file):
+            bb.warn(f"BB_USE_HOME_NPMRC flag set and valid .npmrc detected - "\
+                    f"npm fetcher will use {home_npmrc_file}")
+            return home_npmrc_file
+        return None
+
     def run(self, cmd, args=None, configs=None, workdir=None):
         """Run npm command in a controlled environment"""
         with tempfile.TemporaryDirectory() as tmpdir:
@@ -166,7 +181,7 @@ class Npm(FetchMethod):
         # Using the 'downloadfilename' parameter as local filename
         # or the npm package name.
         if "downloadfilename" in ud.parm:
-            ud.localfile = npm_localfile(d.expand(ud.parm["downloadfilename"]))
+            ud.localfile = npm_localfile(ud.parm["downloadfilename"])
         else:
             ud.localfile = npm_localfile(ud.package, ud.version)
 

bitbake/lib/bb/fetch2/npmsw.py

@@ -37,38 +37,26 @@ def foreach_dependencies(shrinkwrap, callback=None, dev=False):
     """
         Run a callback for each dependencies of a shrinkwrap file.
         The callback is using the format:
-            callback(name, params, deptree)
+            callback(name, data, location)
         with:
             name = the package name (string)
-            params = the package parameters (dictionary)
-            destdir = the destination of the package (string)
+            data = the package data (dictionary)
+            location = the location of the package (string)
     """
-    # For handling old style dependencies entries in shinkwrap files
-    def _walk_deps(deps, deptree):
-        for name in deps:
-            subtree = [*deptree, name]
-            _walk_deps(deps[name].get("dependencies", {}), subtree)
-            if callback is not None:
-                if deps[name].get("dev", False) and not dev:
-                    continue
-                elif deps[name].get("bundled", False):
-                    continue
-                destsubdirs = [os.path.join("node_modules", dep) for dep in subtree]
-                destsuffix = os.path.join(*destsubdirs)
-                callback(name, deps[name], destsuffix)
+    packages = shrinkwrap.get("packages")
+    if not packages:
+        raise FetchError("Invalid shrinkwrap file format")
 
-    # packages entry means new style shrinkwrap file, else use dependencies
-    packages = shrinkwrap.get("packages", None)
-    if packages is not None:
-        for package in packages:
-            if package != "":
-                name = package.split('node_modules/')[-1]
-                package_infos = packages.get(package, {})
-                if dev == False and package_infos.get("dev", False):
-                    continue
-                callback(name, package_infos, package)
-    else:
-        _walk_deps(shrinkwrap.get("dependencies", {}), [])
+    for location, data in packages.items():
+        # Skip empty main and local link target packages
+        if not location.startswith('node_modules/'):
+            continue
+        elif not dev and data.get("dev", False):
+            continue
+        elif data.get("inBundle", False):
+            continue
+        name = location.split('node_modules/')[-1]
+        callback(name, data, location)
 
 class NpmShrinkWrap(FetchMethod):
     """Class to fetch all package from a shrinkwrap file"""
@@ -95,12 +83,18 @@ class NpmShrinkWrap(FetchMethod):
             extrapaths = []
             unpack = True
 
-            integrity = params.get("integrity", None)
-            resolved = params.get("resolved", None)
-            version = params.get("version", resolved)
+            integrity = params.get("integrity")
+            resolved = params.get("resolved")
+            version = params.get("version")
+            link = params.get("link", False)
+
+            # Handle link sources
+            if link:
+                localpath = resolved
+                unpack = False
 
             # Handle registry sources
-            if is_semver(version) and integrity:
+            elif version and is_semver(version) and integrity:
                 # Handle duplicate dependencies without url
                 if not resolved:
                     return
@@ -128,10 +122,10 @@ class NpmShrinkWrap(FetchMethod):
                 extrapaths.append(resolvefile)
 
             # Handle http tarball sources
-            elif version.startswith("http") and integrity:
-                localfile = npm_localfile(os.path.basename(version))
+            elif resolved.startswith("http") and integrity:
+                localfile = npm_localfile(os.path.basename(resolved))
 
-                uri = URI(version)
+                uri = URI(resolved)
                 uri.params["downloadfilename"] = localfile
 
                 checksum_name, checksum_expected = npm_integrity(integrity)
@@ -141,28 +135,12 @@ class NpmShrinkWrap(FetchMethod):
 
                 localpath = os.path.join(d.getVar("DL_DIR"), localfile)
 
-            # Handle local tarball and link sources
-            elif version.startswith("file"):
-                localpath = version[5:]
-                if not version.endswith(".tgz"):
-                    unpack = False
+            # Handle local tarball sources
+            elif resolved.startswith("file"):
+                localpath = resolved[5:]
 
             # Handle git sources
-            elif version.startswith(("git", "bitbucket","gist")) or (
-                not version.endswith((".tgz", ".tar", ".tar.gz"))
-                and not version.startswith((".", "@", "/"))
-                and "/" in version
-            ):
-                if version.startswith("github:"):
-                    version = "git+https://github.com/" + version[len("github:"):]
-                elif version.startswith("gist:"):
-                    version = "git+https://gist.github.com/" + version[len("gist:"):]
-                elif version.startswith("bitbucket:"):
-                    version = "git+https://bitbucket.org/" + version[len("bitbucket:"):]
-                elif version.startswith("gitlab:"):
-                    version = "git+https://gitlab.com/" + version[len("gitlab:"):]
-                elif not version.startswith(("git+","git:")):
-                    version = "git+https://github.com/" + version
+            elif resolved.startswith("git"):
                 regex = re.compile(r"""
                     ^
                     git\+
@@ -174,10 +152,9 @@ class NpmShrinkWrap(FetchMethod):
                     $
                     """, re.VERBOSE)
 
-                match = regex.match(version)
-
+                match = regex.match(resolved)
                 if not match:
-                    raise ParameterError("Invalid git url: %s" % version, ud.url)
+                    raise ParameterError("Invalid git url: %s" % resolved, ud.url)
 
                 groups = match.groupdict()
 

bitbake/lib/bb/fetch2/s3.py

@@ -77,7 +77,7 @@ class S3(FetchMethod):
         else:
             ud.basename = os.path.basename(ud.path)
 
-        ud.localfile = d.expand(urllib.parse.unquote(ud.basename))
+        ud.localfile = ud.basename
 
         ud.basecmd = d.getVar("FETCHCMD_s3") or "/usr/bin/env aws s3"
 

bitbake/lib/bb/fetch2/sftp.py

@@ -77,7 +77,7 @@ class SFTP(FetchMethod):
         else:
             ud.basename = os.path.basename(ud.path)
 
-        ud.localfile = d.expand(urllib.parse.unquote(ud.basename))
+        ud.localfile = ud.basename
 
     def download(self, ud, d):
         """Fetch urls"""

bitbake/lib/bb/fetch2/ssh.py

@@ -73,8 +73,7 @@ class SSH(FetchMethod):
         path = m.group('path')
         path = urllib.parse.unquote(path)
         host = m.group('host')
-        urldata.localpath = os.path.join(d.getVar('DL_DIR'),
-                os.path.basename(os.path.normpath(path)))
+        urldata.localfile = os.path.basename(os.path.normpath(path))
 
     def download(self, urldata, d):
         dldir = d.getVar('DL_DIR')

bitbake/lib/bb/fetch2/wget.py

@@ -53,11 +53,6 @@ class WgetProgressHandler(bb.progress.LineFilterProgressHandler):
 class Wget(FetchMethod):
     """Class to fetch urls via 'wget'"""
 
-    # CDNs like CloudFlare may do a 'browser integrity test' which can fail
-    # with the standard wget/urllib User-Agent, so pretend to be a modern
-    # browser.
-    user_agent = "Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:84.0) Gecko/20100101 Firefox/84.0"
-
     def check_certs(self, d):
         """
         Should certificates be checked?
@@ -83,11 +78,11 @@ class Wget(FetchMethod):
         else:
             ud.basename = os.path.basename(ud.path)
 
-        ud.localfile = d.expand(urllib.parse.unquote(ud.basename))
+        ud.localfile = ud.basename
         if not ud.localfile:
-            ud.localfile = d.expand(urllib.parse.unquote(ud.host + ud.path).replace("/", "."))
+            ud.localfile = ud.host + ud.path.replace("/", ".")
 
-        self.basecmd = d.getVar("FETCHCMD_wget") or "/usr/bin/env wget -t 2 -T 100"
+        self.basecmd = d.getVar("FETCHCMD_wget") or "/usr/bin/env wget --tries=2 --timeout=100"
 
         if ud.type == 'ftp' or ud.type == 'ftps':
             self.basecmd += " --passive-ftp"
@@ -101,7 +96,7 @@ class Wget(FetchMethod):
 
         logger.debug2("Fetching %s using command '%s'" % (ud.url, command))
         bb.fetch2.check_network_access(d, command, ud.url)
-        runfetchcmd(command + ' --progress=dot -v', d, quiet, log=progresshandler, workdir=workdir)
+        runfetchcmd(command + ' --progress=dot --verbose', d, quiet, log=progresshandler, workdir=workdir)
 
     def download(self, ud, d):
         """Fetch urls"""
@@ -111,7 +106,7 @@ class Wget(FetchMethod):
         dldir = os.path.realpath(d.getVar("DL_DIR"))
         localpath = os.path.join(dldir, ud.localfile) + ".tmp"
         bb.utils.mkdirhier(os.path.dirname(localpath))
-        fetchcmd += " -O %s" % shlex.quote(localpath)
+        fetchcmd += " --output-document=%s" % shlex.quote(localpath)
 
         if ud.user and ud.pswd:
             fetchcmd += " --auth-no-challenge"
@@ -127,12 +122,7 @@ class Wget(FetchMethod):
                 fetchcmd += " --user=%s --password=%s" % (ud.user, ud.pswd)
 
         uri = ud.url.split(";")[0]
-        if os.path.exists(ud.localpath):
-            # file exists, but we didnt complete it.. trying again..
-            fetchcmd += " -c -P " + dldir + " '" + uri + "'"
-        else:
-            fetchcmd += " -P " + dldir + " '" + uri + "'"
-
+        fetchcmd += " --continue --directory-prefix=%s '%s'" % (dldir, uri)
         self._runwget(ud, d, fetchcmd, False)
 
         # Sanity check since wget can pretend it succeed when it didn't
@@ -310,13 +300,45 @@ class Wget(FetchMethod):
 
         class FixedHTTPRedirectHandler(urllib.request.HTTPRedirectHandler):
             """
-            urllib2.HTTPRedirectHandler resets the method to GET on redirect,
-            when we want to follow redirects using the original method.
+            urllib2.HTTPRedirectHandler before 3.13 has two flaws:
+            
+            It resets the method to GET on redirect when we want to follow
+            redirects using the original method (typically HEAD). This was fixed
+            in 759e8e7.
+
+            It also doesn't handle 308 (Permanent Redirect). This was fixed in
+            c379bc5.
+
+            Until we depend on Python 3.13 onwards, copy the redirect_request
+            method to fix these issues.
             """
             def redirect_request(self, req, fp, code, msg, headers, newurl):
-                newreq = urllib.request.HTTPRedirectHandler.redirect_request(self, req, fp, code, msg, headers, newurl)
-                newreq.get_method = req.get_method
-                return newreq
+                m = req.get_method()
+                if (not (code in (301, 302, 303, 307, 308) and m in ("GET", "HEAD")
+                    or code in (301, 302, 303) and m == "POST")):
+                    raise urllib.HTTPError(req.full_url, code, msg, headers, fp)
+
+                # Strictly (according to RFC 2616), 301 or 302 in response to
+                # a POST MUST NOT cause a redirection without confirmation
+                # from the user (of urllib.request, in this case).  In practice,
+                # essentially all clients do redirect in this case, so we do
+                # the same.
+
+                # Be conciliant with URIs containing a space.  This is mainly
+                # redundant with the more complete encoding done in http_error_302(),
+                # but it is kept for compatibility with other callers.
+                newurl = newurl.replace(' ', '%20')
+
+                CONTENT_HEADERS = ("content-length", "content-type")
+                newheaders = {k: v for k, v in req.headers.items()
+                            if k.lower() not in CONTENT_HEADERS}
+                return urllib.request.Request(newurl,
+                            method="HEAD" if m == "HEAD" else "GET",
+                            headers=newheaders,
+                            origin_req_host=req.origin_req_host,
+                            unverifiable=True)
+
+            http_error_308 = urllib.request.HTTPRedirectHandler.http_error_302
 
         # We need to update the environment here as both the proxy and HTTPS
         # handlers need variables set. The proxy needs http_proxy and friends to
@@ -349,14 +371,14 @@ class Wget(FetchMethod):
             opener = urllib.request.build_opener(*handlers)
 
             try:
-                uri_base = ud.url.split(";")[0]
-                uri = "{}://{}{}".format(urllib.parse.urlparse(uri_base).scheme, ud.host, ud.path)
+                parts = urllib.parse.urlparse(ud.url.split(";")[0])
+                uri = "{}://{}{}".format(parts.scheme, parts.netloc, parts.path)
                 r = urllib.request.Request(uri)
                 r.get_method = lambda: "HEAD"
                 # Some servers (FusionForge, as used on Alioth) require that the
                 # optional Accept header is set.
                 r.add_header("Accept", "*/*")
-                r.add_header("User-Agent", self.user_agent)
+                r.add_header("User-Agent", "bitbake/{}".format(bb.__version__))
                 def add_basic_auth(login_str, request):
                     '''Adds Basic auth to http request, pass in login:password as string'''
                     import base64
@@ -463,7 +485,7 @@ class Wget(FetchMethod):
         f = tempfile.NamedTemporaryFile()
         with tempfile.TemporaryDirectory(prefix="wget-index-") as workdir, tempfile.NamedTemporaryFile(dir=workdir, prefix="wget-listing-") as f:
             fetchcmd = self.basecmd
-            fetchcmd += " -O " + f.name + " --user-agent='" + self.user_agent + "' '" + uri + "'"
+            fetchcmd += " --output-document=%s '%s'" % (f.name, uri)
             try:
                 self._runwget(ud, d, fetchcmd, True, workdir=workdir)
                 fetchresult = f.read()
@@ -623,13 +645,17 @@ class Wget(FetchMethod):
 
         sanity check to ensure same name and type.
         """
-        package = ud.path.split("/")[-1]
+        if 'downloadfilename' in ud.parm:
+            package = ud.parm['downloadfilename']
+        else:
+            package = ud.path.split("/")[-1]
         current_version = ['', d.getVar('PV'), '']
 
         """possible to have no version in pkg name, such as spectrum-fw"""
         if not re.search(r"\d+", package):
             current_version[1] = re.sub('_', '.', current_version[1])
             current_version[1] = re.sub('-', '.', current_version[1])
+            bb.debug(3, "latest_versionstring: no version found in %s" % package)
             return (current_version[1], '')
 
         package_regex = self._init_regexes(package, ud, d)

bitbake/lib/bb/parse/ast.py

@@ -43,6 +43,21 @@ class IncludeNode(AstNode):
         else:
             bb.parse.ConfHandler.include(self.filename, s, self.lineno, data, False)
 
+class IncludeAllNode(AstNode):
+    def __init__(self, filename, lineno, what_file):
+        AstNode.__init__(self, filename, lineno)
+        self.what_file = what_file
+
+    def eval(self, data):
+        """
+        Include the file and evaluate the statements
+        """
+        s = data.expand(self.what_file)
+        logger.debug2("CONF %s:%s: including %s", self.filename, self.lineno, s)
+
+        for path in data.getVar("BBPATH").split(":"):
+            bb.parse.ConfHandler.include(self.filename, os.path.join(path, s), self.lineno, data, False)
+
 class ExportNode(AstNode):
     def __init__(self, filename, lineno, var):
         AstNode.__init__(self, filename, lineno)
@@ -137,7 +152,10 @@ class DataNode(AstNode):
 
         flag = None
         if 'flag' in groupd and groupd['flag'] is not None:
-            flag = groupd['flag']
+            if groupd["lazyques"]:
+                flag = "_defaultval_flag_"+groupd['flag']
+            else:
+                flag = groupd['flag']
         elif groupd["lazyques"]:
             flag = "_defaultval"
 
@@ -326,9 +344,49 @@ class InheritDeferredNode(AstNode):
         inherits.append(self.inherit)
         data.setVar('__BBDEFINHERITS', inherits)
 
+class AddFragmentsNode(AstNode):
+    def __init__(self, filename, lineno, fragments_path_prefix, fragments_variable, flagged_variables_list_variable):
+        AstNode.__init__(self, filename, lineno)
+        self.fragments_path_prefix = fragments_path_prefix
+        self.fragments_variable = fragments_variable
+        self.flagged_variables_list_variable = flagged_variables_list_variable
+
+    def eval(self, data):
+        # No need to use mark_dependency since we would only match a fragment
+        # from a specific layer and there can only be a single layer with a
+        # given namespace.
+        def find_fragment(layers, layerid, full_fragment_name):
+           for layerpath in layers.split():
+               candidate_fragment_path = os.path.join(layerpath, full_fragment_name)
+               if os.path.exists(candidate_fragment_path) and bb.utils.get_file_layer(candidate_fragment_path, data) == layerid:
+                   return candidate_fragment_path
+           return None
+
+        fragments = data.getVar(self.fragments_variable)
+        layers = data.getVar('BBLAYERS')
+        flagged_variables = data.getVar(self.flagged_variables_list_variable).split()
+
+        if not fragments:
+            return
+        for f in fragments.split():
+            layerid, fragment_name = f.split('/', 1)
+            full_fragment_name = data.expand("{}/{}.conf".format(self.fragments_path_prefix, fragment_name))
+            fragment_path = find_fragment(layers, layerid, full_fragment_name)
+            if fragment_path:
+                bb.parse.ConfHandler.include(self.filename, fragment_path, self.lineno, data, "include fragment")
+                for flagged_var in flagged_variables:
+                    val = data.getVar(flagged_var)
+                    data.setVarFlag(flagged_var, f, val)
+                    data.setVar(flagged_var, None)
+            else:
+                bb.error("Could not find fragment {} in enabled layers: {}".format(f, layers))
+
 def handleInclude(statements, filename, lineno, m, force):
     statements.append(IncludeNode(filename, lineno, m.group(1), force))
 
+def handleIncludeAll(statements, filename, lineno, m):
+    statements.append(IncludeAllNode(filename, lineno, m.group(1)))
+
 def handleExport(statements, filename, lineno, m):
     statements.append(ExportNode(filename, lineno, m.group(1)))
 
@@ -370,12 +428,42 @@ def handleInheritDeferred(statements, filename, lineno, m):
     classes = m.group(1)
     statements.append(InheritDeferredNode(filename, lineno, classes))
 
+def handleAddFragments(statements, filename, lineno, m):
+    fragments_path_prefix = m.group(1)
+    fragments_variable = m.group(2)
+    flagged_variables_list_variable = m.group(3)
+    statements.append(AddFragmentsNode(filename, lineno, fragments_path_prefix, fragments_variable, flagged_variables_list_variable))
+
 def runAnonFuncs(d):
     code = []
     for funcname in d.getVar("__BBANONFUNCS", False) or []:
         code.append("%s(d)" % funcname)
     bb.utils.better_exec("\n".join(code), {"d": d})
 
+# Handle recipe level PREFERRED_PROVIDERs
+def handleVirtRecipeProviders(tasklist, d):
+    depends = (d.getVar("DEPENDS") or "").split()
+    virtprovs = (d.getVar("BB_RECIPE_VIRTUAL_PROVIDERS") or "").split()
+    newdeps = []
+    for dep in depends:
+        if dep in virtprovs:
+            newdep = d.getVar("PREFERRED_PROVIDER_" + dep)
+            if not newdep:
+                 bb.fatal("Error, recipe virtual provider PREFERRED_PROVIDER_%s not set" % dep)
+            newdeps.append(newdep)
+        else:
+            newdeps.append(dep)
+    d.setVar("DEPENDS", " ".join(newdeps))
+    for task in tasklist:
+        taskdeps = (d.getVarFlag(task, "depends") or "").split()
+        remapped = []
+        for entry in taskdeps:
+            r, t = entry.split(":")
+            if r in virtprovs:
+                r = d.getVar("PREFERRED_PROVIDER_" + r)
+            remapped.append("%s:%s" % (r, t))
+        d.setVarFlag(task, "depends", " ".join(remapped))
+
 def finalize(fn, d, variant = None):
     saved_handlers = bb.event.get_handlers().copy()
     try:
@@ -401,6 +489,7 @@ def finalize(fn, d, variant = None):
 
         tasklist = d.getVar('__BBTASKS', False) or []
         bb.event.fire(bb.event.RecipeTaskPreProcess(fn, list(tasklist)), d)
+        handleVirtRecipeProviders(tasklist, d)
         bb.build.add_tasks(tasklist, d)
 
         bb.parse.siggen.finalise(fn, d, variant)

bitbake/lib/bb/parse/parse_py/ConfHandler.py

@@ -20,8 +20,8 @@ from bb.parse import ParseError, resolve_file, ast, logger, handle
 __config_regexp__  = re.compile( r"""
     ^
     (?P<exp>export\s+)?
-    (?P<var>[a-zA-Z0-9\-_+.${}/~:]+?)
-    (\[(?P<flag>[a-zA-Z0-9\-_+.][a-zA-Z0-9\-_+.@]*)\])?
+    (?P<var>[a-zA-Z0-9\-_+.${}/~:]*?)
+    (\[(?P<flag>[a-zA-Z0-9\-_+.][a-zA-Z0-9\-_+.@/]*)\])?
 
     \s* (
         (?P<colon>:=) |
@@ -43,10 +43,12 @@ __config_regexp__  = re.compile( r"""
     """, re.X)
 __include_regexp__ = re.compile( r"include\s+(.+)" )
 __require_regexp__ = re.compile( r"require\s+(.+)" )
+__includeall_regexp__ = re.compile( r"include_all\s+(.+)" )
 __export_regexp__ = re.compile( r"export\s+([a-zA-Z0-9\-_+.${}/~]+)$" )
 __unset_regexp__ = re.compile( r"unset\s+([a-zA-Z0-9\-_+.${}/~]+)$" )
 __unset_flag_regexp__ = re.compile( r"unset\s+([a-zA-Z0-9\-_+.${}/~]+)\[([a-zA-Z0-9\-_+.][a-zA-Z0-9\-_+.@]+)\]$" )
 __addpylib_regexp__      = re.compile(r"addpylib\s+(.+)\s+(.+)" )
+__addfragments_regexp__  = re.compile(r"addfragments\s+(.+)\s+(.+)\s+(.+)" )
 
 def init(data):
     return
@@ -164,6 +166,8 @@ def feeder(lineno, s, fn, statements, baseconfig=False, conffile=True):
     m = __config_regexp__.match(s)
     if m:
         groupd = m.groupdict()
+        if groupd['var'] == "":
+            raise ParseError("Empty variable name in assignment: '%s'" % s, fn, lineno);
         ast.handleData(statements, fn, lineno, groupd)
         return
 
@@ -177,6 +181,11 @@ def feeder(lineno, s, fn, statements, baseconfig=False, conffile=True):
         ast.handleInclude(statements, fn, lineno, m, True)
         return
 
+    m = __includeall_regexp__.match(s)
+    if m:
+        ast.handleIncludeAll(statements, fn, lineno, m)
+        return
+
     m = __export_regexp__.match(s)
     if m:
         ast.handleExport(statements, fn, lineno, m)
@@ -197,6 +206,11 @@ def feeder(lineno, s, fn, statements, baseconfig=False, conffile=True):
         ast.handlePyLib(statements, fn, lineno, m)
         return
 
+    m = __addfragments_regexp__.match(s)
+    if m:
+        ast.handleAddFragments(statements, fn, lineno, m)
+        return
+
     raise ParseError("unparsed line: '%s'" % s, fn, lineno);
 
 # Add us to the handlers list

bitbake/lib/bb/persist_data.py

@@ -1,272 +0,0 @@
-"""BitBake Persistent Data Store
-
-Used to store data in a central location such that other threads/tasks can
-access them at some future date.  Acts as a convenience wrapper around sqlite,
-currently, providing a key/value store accessed by 'domain'.
-"""
-
-# Copyright (C) 2007        Richard Purdie
-# Copyright (C) 2010        Chris Larson <chris_larson@mentor.com>
-#
-# SPDX-License-Identifier: GPL-2.0-only
-#
-
-import collections
-import collections.abc
-import contextlib
-import functools
-import logging
-import os.path
-import sqlite3
-import sys
-from collections.abc import Mapping
-
-sqlversion = sqlite3.sqlite_version_info
-if sqlversion[0] < 3 or (sqlversion[0] == 3 and sqlversion[1] < 3):
-    raise Exception("sqlite3 version 3.3.0 or later is required.")
-
-
-logger = logging.getLogger("BitBake.PersistData")
-
-@functools.total_ordering
-class SQLTable(collections.abc.MutableMapping):
-    class _Decorators(object):
-        @staticmethod
-        def retry(*, reconnect=True):
-            """
-            Decorator that restarts a function if a database locked sqlite
-            exception occurs. If reconnect is True, the database connection
-            will be closed and reopened each time a failure occurs
-            """
-            def retry_wrapper(f):
-                def wrap_func(self, *args, **kwargs):
-                    # Reconnect if necessary
-                    if self.connection is None and reconnect:
-                        self.reconnect()
-
-                    count = 0
-                    while True:
-                        try:
-                            return f(self, *args, **kwargs)
-                        except sqlite3.OperationalError as exc:
-                            if count < 500 and ('is locked' in str(exc) or 'locking protocol' in str(exc)):
-                                count = count + 1
-                                if reconnect:
-                                    self.reconnect()
-                                continue
-                            raise
-                return wrap_func
-            return retry_wrapper
-
-        @staticmethod
-        def transaction(f):
-            """
-            Decorator that starts a database transaction and creates a database
-            cursor for performing queries. If no exception is thrown, the
-            database results are committed. If an exception occurs, the database
-            is rolled back. In all cases, the cursor is closed after the
-            function ends.
-
-            Note that the cursor is passed as an extra argument to the function
-            after `self` and before any of the normal arguments
-            """
-            def wrap_func(self, *args, **kwargs):
-                # Context manager will COMMIT the database on success,
-                # or ROLLBACK on an exception
-                with self.connection:
-                    # Automatically close the cursor when done
-                    with contextlib.closing(self.connection.cursor()) as cursor:
-                        return f(self, cursor, *args, **kwargs)
-            return wrap_func
-
-    """Object representing a table/domain in the database"""
-    def __init__(self, cachefile, table):
-        self.cachefile = cachefile
-        self.table = table
-
-        self.connection = None
-        self._execute_single("CREATE TABLE IF NOT EXISTS %s(key TEXT PRIMARY KEY NOT NULL, value TEXT);" % table)
-
-    @_Decorators.retry(reconnect=False)
-    @_Decorators.transaction
-    def _setup_database(self, cursor):
-        cursor.execute("pragma synchronous = off;")
-        # Enable WAL and keep the autocheckpoint length small (the default is
-        # usually 1000). Persistent caches are usually read-mostly, so keeping
-        # this short will keep readers running quickly
-        cursor.execute("pragma journal_mode = WAL;")
-        cursor.execute("pragma wal_autocheckpoint = 100;")
-
-    def reconnect(self):
-        if self.connection is not None:
-            self.connection.close()
-        self.connection = sqlite3.connect(self.cachefile, timeout=5)
-        self.connection.text_factory = str
-        self._setup_database()
-
-    @_Decorators.retry()
-    @_Decorators.transaction
-    def _execute_single(self, cursor, *query):
-        """
-        Executes a single query and discards the results. This correctly closes
-        the database cursor when finished
-        """
-        cursor.execute(*query)
-
-    @_Decorators.retry()
-    def _row_iter(self, f, *query):
-        """
-        Helper function that returns a row iterator. Each time __next__ is
-        called on the iterator, the provided function is evaluated to determine
-        the return value
-        """
-        class CursorIter(object):
-            def __init__(self, cursor):
-                self.cursor = cursor
-
-            def __iter__(self):
-                return self
-
-            def __next__(self):
-                row = self.cursor.fetchone()
-                if row is None:
-                    self.cursor.close()
-                    raise StopIteration
-                return f(row)
-
-            def __enter__(self):
-                return self
-
-            def __exit__(self, typ, value, traceback):
-                self.cursor.close()
-                return False
-
-        cursor = self.connection.cursor()
-        try:
-            cursor.execute(*query)
-            return CursorIter(cursor)
-        except:
-            cursor.close()
-
-    def __enter__(self):
-        self.connection.__enter__()
-        return self
-
-    def __exit__(self, *excinfo):
-        self.connection.__exit__(*excinfo)
-        self.connection.close()
-
-    @_Decorators.retry()
-    @_Decorators.transaction
-    def __getitem__(self, cursor, key):
-        cursor.execute("SELECT * from %s where key=?;" % self.table, [key])
-        row = cursor.fetchone()
-        if row is not None:
-            return row[1]
-        raise KeyError(key)
-
-    @_Decorators.retry()
-    @_Decorators.transaction
-    def __delitem__(self, cursor, key):
-        if key not in self:
-            raise KeyError(key)
-        cursor.execute("DELETE from %s where key=?;" % self.table, [key])
-
-    @_Decorators.retry()
-    @_Decorators.transaction
-    def __setitem__(self, cursor, key, value):
-        if not isinstance(key, str):
-            raise TypeError('Only string keys are supported')
-        elif not isinstance(value, str):
-            raise TypeError('Only string values are supported')
-
-        # Ensure the entire transaction (including SELECT) executes under write lock
-        cursor.execute("BEGIN EXCLUSIVE")
-
-        cursor.execute("SELECT * from %s where key=?;" % self.table, [key])
-        row = cursor.fetchone()
-        if row is not None:
-            cursor.execute("UPDATE %s SET value=? WHERE key=?;" % self.table, [value, key])
-        else:
-            cursor.execute("INSERT into %s(key, value) values (?, ?);" % self.table, [key, value])
-
-    @_Decorators.retry()
-    @_Decorators.transaction
-    def __contains__(self, cursor, key):
-        cursor.execute('SELECT * from %s where key=?;' % self.table, [key])
-        return cursor.fetchone() is not None
-
-    @_Decorators.retry()
-    @_Decorators.transaction
-    def __len__(self, cursor):
-        cursor.execute("SELECT COUNT(key) FROM %s;" % self.table)
-        row = cursor.fetchone()
-        if row is not None:
-            return row[0]
-
-    def __iter__(self):
-        return self._row_iter(lambda row: row[0], "SELECT key from %s;" % self.table)
-
-    def __lt__(self, other):
-        if not isinstance(other, Mapping):
-            raise NotImplementedError()
-
-        return len(self) < len(other)
-
-    def get_by_pattern(self, pattern):
-        return self._row_iter(lambda row: row[1], "SELECT * FROM %s WHERE key LIKE ?;" %
-                              self.table, [pattern])
-
-    def values(self):
-        return list(self.itervalues())
-
-    def itervalues(self):
-        return self._row_iter(lambda row: row[0], "SELECT value FROM %s;" %
-                              self.table)
-
-    def items(self):
-        return list(self.iteritems())
-
-    def iteritems(self):
-        return self._row_iter(lambda row: (row[0], row[1]), "SELECT * FROM %s;" %
-                              self.table)
-
-    @_Decorators.retry()
-    @_Decorators.transaction
-    def clear(self, cursor):
-        cursor.execute("DELETE FROM %s;" % self.table)
-
-    def has_key(self, key):
-        return key in self
-
-def persist(domain, d):
-    """Convenience factory for SQLTable objects based upon metadata"""
-    import bb.utils
-    cachedir = (d.getVar("PERSISTENT_DIR") or
-                d.getVar("CACHE"))
-    if not cachedir:
-        logger.critical("Please set the 'PERSISTENT_DIR' or 'CACHE' variable")
-        sys.exit(1)
-
-    bb.utils.mkdirhier(cachedir)
-    cachefile = os.path.join(cachedir, "bb_persist_data.sqlite3")
-
-    try:
-        return SQLTable(cachefile, domain)
-    except sqlite3.OperationalError:
-        # Sqlite fails to open database when its path is too long.
-        # After testing, 504 is the biggest path length that can be opened by
-        # sqlite.
-        # Note: This code is called before sanity.bbclass and its path length
-        # check
-        max_len = 504
-        if len(cachefile) > max_len:
-            logger.critical("The path of the cache file is too long "
-                    "({0} chars > {1}) to be opened by sqlite! "
-                    "Your cache file is \"{2}\"".format(
-                        len(cachefile),
-                        max_len,
-                        cachefile))
-            sys.exit(1)
-        else:
-            raise

bitbake/lib/bb/runqueue.py

@@ -14,6 +14,7 @@ import os
 import sys
 import stat
 import errno
+import itertools
 import logging
 import re
 import bb
@@ -128,6 +129,7 @@ class RunQueueStats:
 # runQueue state machine
 runQueuePrepare = 2
 runQueueSceneInit = 3
+runQueueDumpSigs = 4
 runQueueRunning = 6
 runQueueFailed = 7
 runQueueCleanUp = 8
@@ -475,7 +477,6 @@ class RunQueueData:
         self.runtaskentries = {}
 
     def runq_depends_names(self, ids):
-        import re
         ret = []
         for id in ids:
             nam = os.path.basename(id)
@@ -728,6 +729,8 @@ class RunQueueData:
                 if mc == frommc:
                     fn = taskData[mcdep].build_targets[pn][0]
                     newdep = '%s:%s' % (fn,deptask)
+                    if newdep not in taskData[mcdep].taskentries:
+                        bb.fatal("Task mcdepends on non-existent task %s" % (newdep))
                     taskData[mc].taskentries[tid].tdepends.append(newdep)
 
         for mc in taskData:
@@ -1588,14 +1591,19 @@ class RunQueue:
             self.rqdata.init_progress_reporter.next_stage()
             self.rqexe = RunQueueExecute(self)
 
-            dump = self.cooker.configuration.dump_signatures
-            if dump:
+            dumpsigs = self.cooker.configuration.dump_signatures
+            if dumpsigs:
                 self.rqdata.init_progress_reporter.finish()
-                if 'printdiff' in dump:
-                    invalidtasks = self.print_diffscenetasks()
-                self.dump_signatures(dump)
-                if 'printdiff' in dump:
-                    self.write_diffscenetasks(invalidtasks)
+                if 'printdiff' in dumpsigs:
+                    self.invalidtasks_dump = self.print_diffscenetasks()
+                self.state = runQueueDumpSigs
+
+        if self.state is runQueueDumpSigs:
+            dumpsigs = self.cooker.configuration.dump_signatures
+            retval = self.dump_signatures(dumpsigs)
+            if retval is False:
+                if 'printdiff' in dumpsigs:
+                    self.write_diffscenetasks(self.invalidtasks_dump)
                 self.state = runQueueComplete
 
         if self.state is runQueueSceneInit:
@@ -1686,33 +1694,42 @@ class RunQueue:
             bb.parse.siggen.dump_sigtask(taskfn, taskname, dataCaches[mc].stamp[taskfn], True)
 
     def dump_signatures(self, options):
-        if bb.cooker.CookerFeatures.RECIPE_SIGGEN_INFO not in self.cooker.featureset:
-            bb.fatal("The dump signatures functionality needs the RECIPE_SIGGEN_INFO feature enabled")
+        if not hasattr(self, "dumpsigs_launched"):
+            if bb.cooker.CookerFeatures.RECIPE_SIGGEN_INFO not in self.cooker.featureset:
+                bb.fatal("The dump signatures functionality needs the RECIPE_SIGGEN_INFO feature enabled")
 
-        bb.note("Writing task signature files")
+            bb.note("Writing task signature files")
 
-        max_process = int(self.cfgData.getVar("BB_NUMBER_PARSE_THREADS") or os.cpu_count() or 1)
-        def chunkify(l, n):
-            return [l[i::n] for i in range(n)]
-        tids = chunkify(list(self.rqdata.runtaskentries), max_process)
-        # We cannot use the real multiprocessing.Pool easily due to some local data
-        # that can't be pickled. This is a cheap multi-process solution.
-        launched = []
-        while tids:
-            if len(launched) < max_process:
-                p = Process(target=self._rq_dump_sigtid, args=(tids.pop(), ))
+            max_process = int(self.cfgData.getVar("BB_NUMBER_PARSE_THREADS") or os.cpu_count() or 1)
+            def chunkify(l, n):
+                return [l[i::n] for i in range(n)]
+            dumpsigs_tids = chunkify(list(self.rqdata.runtaskentries), max_process)
+
+            # We cannot use the real multiprocessing.Pool easily due to some local data
+            # that can't be pickled. This is a cheap multi-process solution.
+            self.dumpsigs_launched = []
+
+            for tids in dumpsigs_tids:
+                p = Process(target=self._rq_dump_sigtid, args=(tids, ))
                 p.start()
-                launched.append(p)
-            for q in launched:
-                # The finished processes are joined when calling is_alive()
-                if not q.is_alive():
-                    launched.remove(q)
-        for p in launched:
+                self.dumpsigs_launched.append(p)
+
+            return 1.0
+
+        for q in self.dumpsigs_launched:
+            # The finished processes are joined when calling is_alive()
+            if not q.is_alive():
+                self.dumpsigs_launched.remove(q)
+
+        if self.dumpsigs_launched:
+            return 1.0
+
+        for p in self.dumpsigs_launched:
                 p.join()
 
         bb.parse.siggen.dump_sigs(self.rqdata.dataCaches, options)
 
-        return
+        return False
 
     def print_diffscenetasks(self):
         def get_root_invalid_tasks(task, taskdepends, valid, noexec, visited_invalid):
@@ -2189,12 +2206,20 @@ class RunQueueExecute:
         if not hasattr(self, "sorted_setscene_tids"):
             # Don't want to sort this set every execution
             self.sorted_setscene_tids = sorted(self.rqdata.runq_setscene_tids)
+            # Resume looping where we left off when we returned to feed the mainloop
+            self.setscene_tids_generator = itertools.cycle(self.rqdata.runq_setscene_tids)
 
         task = None
         if not self.sqdone and self.can_start_task():
-            # Find the next setscene to run
-            for nexttask in self.sorted_setscene_tids:
+            loopcount = 0
+            # Find the next setscene to run, exit the loop when we've processed all tids or found something to execute
+            while loopcount < len(self.rqdata.runq_setscene_tids):
+                loopcount += 1
+                nexttask = next(self.setscene_tids_generator)
                 if nexttask in self.sq_buildable and nexttask not in self.sq_running and self.sqdata.stamps[nexttask] not in self.build_stamps.values() and nexttask not in self.sq_harddep_deferred:
+                    if nexttask in self.sq_deferred and self.sq_deferred[nexttask] not in self.runq_complete:
+                        # Skip deferred tasks quickly before the 'expensive' tests below - this is key to performant multiconfig builds
+                        continue
                     if nexttask not in self.sqdata.unskippable and self.sqdata.sq_revdeps[nexttask] and \
                             nexttask not in self.sq_needed_harddeps and \
                             self.sqdata.sq_revdeps[nexttask].issubset(self.scenequeue_covered) and \
@@ -2224,8 +2249,7 @@ class RunQueueExecute:
                         if t in self.runq_running and t not in self.runq_complete:
                             continue
                     if nexttask in self.sq_deferred:
-                        if self.sq_deferred[nexttask] not in self.runq_complete:
-                            continue
+                        # Deferred tasks that were still deferred were skipped above so we now need to process
                         logger.debug("Task %s no longer deferred" % nexttask)
                         del self.sq_deferred[nexttask]
                         valid = self.rq.validate_hashes(set([nexttask]), self.cooker.data, 0, False, summary=False)
@@ -2748,8 +2772,12 @@ class RunQueueExecute:
                 logger.debug2("%s was unavailable and is a hard dependency of %s so skipping" % (task, dep))
                 self.sq_task_failoutright(dep)
                 continue
+
+        # For performance, only compute allcovered once if needed
+        if self.sqdata.sq_deps[task]:
+            allcovered = self.scenequeue_covered | self.scenequeue_notcovered
         for dep in sorted(self.sqdata.sq_deps[task]):
-            if self.sqdata.sq_revdeps[dep].issubset(self.scenequeue_covered | self.scenequeue_notcovered):
+            if self.sqdata.sq_revdeps[dep].issubset(allcovered):
                 if dep not in self.sq_buildable:
                     self.sq_buildable.add(dep)
 
@@ -3303,7 +3331,7 @@ class runQueuePipe():
 
         start = len(self.queue)
         try:
-            self.queue.extend(self.input.read(102400) or b"")
+            self.queue.extend(self.input.read(512 * 1024) or b"")
         except (OSError, IOError) as e:
             if e.errno != errno.EAGAIN:
                 raise

bitbake/lib/bb/server/process.py

@@ -321,7 +321,22 @@ class ProcessServer():
                     bb.warn('Ignoring invalid BB_SERVER_TIMEOUT=%s, must be a float specifying seconds.' % self.timeout)
                 seendata = True
 
-            ready = self.idle_commands(.1, fds)
+            if not self.idle:
+                self.idle = threading.Thread(target=self.idle_thread)
+                self.idle.start()
+            elif self.idle and not self.idle.is_alive():
+                serverlog("Idle thread terminated, main thread exiting too")
+                bb.error("Idle thread terminated, main thread exiting too")
+                self.quit = True
+
+            nextsleep = 1.0
+            if self.xmlrpc:
+                nextsleep = self.xmlrpc.get_timeout(nextsleep)
+            try:
+                ready = select.select(fds,[],[],nextsleep)[0]
+            except InterruptedError:
+                # Ignore EINTR
+                ready = []
 
         if self.idle:
             self.idle.join()
@@ -424,7 +439,7 @@ class ProcessServer():
                 self.idle_cond.notify_all()
 
         while not self.quit:
-            nextsleep = 0.1
+            nextsleep = 1.0
             fds = []
 
             with bb.utils.lock_timeout(self._idlefuncsLock):
@@ -462,7 +477,7 @@ class ProcessServer():
 
             # Create new heartbeat event?
             now = time.time()
-            if bb.event._heartbeat_enabled and now >= self.next_heartbeat:
+            if items and bb.event._heartbeat_enabled and now >= self.next_heartbeat:
                 # We might have missed heartbeats. Just trigger once in
                 # that case and continue after the usual delay.
                 self.next_heartbeat += self.heartbeat_seconds
@@ -485,31 +500,6 @@ class ProcessServer():
             if nextsleep is not None:
                 select.select(fds,[],[],nextsleep)[0]
 
-    def idle_commands(self, delay, fds=None):
-        nextsleep = delay
-        if not fds:
-            fds = []
-
-        if not self.idle:
-            self.idle = threading.Thread(target=self.idle_thread)
-            self.idle.start()
-        elif self.idle and not self.idle.is_alive():
-            serverlog("Idle thread terminated, main thread exiting too")
-            bb.error("Idle thread terminated, main thread exiting too")
-            self.quit = True
-
-        if nextsleep is not None:
-            if self.xmlrpc:
-                nextsleep = self.xmlrpc.get_timeout(nextsleep)
-            try:
-                return select.select(fds,[],[],nextsleep)[0]
-            except InterruptedError:
-                # Ignore EINTR
-                return []
-        else:
-            return select.select(fds,[],[],0)[0]
-
-
 class ServerCommunicator():
     def __init__(self, connection, recv):
         self.connection = connection

bitbake/lib/bb/server/xmlrpcserver.py

@@ -14,6 +14,8 @@ from xmlrpc.server import SimpleXMLRPCServer, SimpleXMLRPCRequestHandler
 import bb.server.xmlrpcclient
 
 import bb
+import bb.cooker
+import bb.event
 
 # This request handler checks if the request has a "Bitbake-token" header
 # field (this comes from the client side) and compares it with its internal
@@ -54,7 +56,7 @@ class BitBakeXMLRPCServer(SimpleXMLRPCServer):
 
     def __init__(self, interface, cooker, parent):
         # Use auto port configuration
-        if (interface[1] == -1):
+        if interface[1] == -1:
             interface = (interface[0], 0)
         SimpleXMLRPCServer.__init__(self, interface,
                                     requestHandler=BitBakeXMLRPCRequestHandler,
@@ -87,11 +89,12 @@ class BitBakeXMLRPCServer(SimpleXMLRPCServer):
     def handle_requests(self):
         self._handle_request_noblock()
 
-class BitBakeXMLRPCServerCommands():
+class BitBakeXMLRPCServerCommands:
 
     def __init__(self, server):
         self.server = server
         self.has_client = False
+        self.event_handle = None
 
     def registerEventHandler(self, host, port):
         """
@@ -100,8 +103,8 @@ class BitBakeXMLRPCServerCommands():
         s, t = bb.server.xmlrpcclient._create_server(host, port)
 
         # we don't allow connections if the cooker is running
-        if (self.server.cooker.state in [bb.cooker.state.parsing, bb.cooker.state.running]):
-            return None, "Cooker is busy: %s" % bb.cooker.state.get_name(self.server.cooker.state)
+        if self.server.cooker.state in [bb.cooker.State.PARSING, bb.cooker.State.RUNNING]:
+            return None, f"Cooker is busy: {self.server.cooker.state.name}"
 
         self.event_handle = bb.event.register_UIHhandler(s, True)
         return self.event_handle, 'OK'

bitbake/lib/bb/tests/compression.py

@@ -66,8 +66,8 @@ class CompressionTests(object):
 
 class LZ4Tests(CompressionTests, unittest.TestCase):
     def setUp(self):
-        if shutil.which("lz4c") is None:
-            self.skipTest("'lz4c' not found")
+        if shutil.which("lz4") is None:
+            self.skipTest("'lz4' not found")
         super().setUp()
 
     @contextlib.contextmanager

bitbake/lib/bb/tests/data.py

@@ -450,17 +450,64 @@ class TestFlags(unittest.TestCase):
         self.d = bb.data.init()
         self.d.setVar("foo", "value of foo")
         self.d.setVarFlag("foo", "flag1", "value of flag1")
+        self.d.setVarFlag("foo", "_defaultval_flag_flag1", "default of flag1")
         self.d.setVarFlag("foo", "flag2", "value of flag2")
+        self.d.setVarFlag("foo", "_defaultval_flag_flag2", "default of flag2")
+        self.d.setVarFlag("foo", "flag3", "value of flag3")
+        self.d.setVarFlag("foo", "_defaultval_flag_flagnovalue", "default of flagnovalue")
 
     def test_setflag(self):
         self.assertEqual(self.d.getVarFlag("foo", "flag1", False), "value of flag1")
         self.assertEqual(self.d.getVarFlag("foo", "flag2", False), "value of flag2")
+        self.assertDictEqual(
+            self.d.getVarFlags("foo"),
+            {
+                "flag1": "value of flag1",
+                "flag2": "value of flag2",
+                "flag3": "value of flag3",
+                "flagnovalue": "default of flagnovalue",
+            }
+        )
+        self.assertDictEqual(
+            self.d.getVarFlags("foo", internalflags=True),
+            {
+                "_content": "value of foo",
+                "flag1": "value of flag1",
+                "flag2": "value of flag2",
+                "flag3": "value of flag3",
+                "_defaultval_flag_flag1": "default of flag1",
+                "_defaultval_flag_flag2": "default of flag2",
+                "_defaultval_flag_flagnovalue": "default of flagnovalue",
+            }
+        )
 
     def test_delflag(self):
         self.d.delVarFlag("foo", "flag2")
+        self.d.delVarFlag("foo", "flag3")
         self.assertEqual(self.d.getVarFlag("foo", "flag1", False), "value of flag1")
         self.assertEqual(self.d.getVarFlag("foo", "flag2", False), None)
+        self.assertDictEqual(
+            self.d.getVarFlags("foo"),
+            {
+                "flag1": "value of flag1",
+                "flagnovalue": "default of flagnovalue",
+            }
+        )
+        self.assertDictEqual(
+            self.d.getVarFlags("foo", internalflags=True),
+            {
+                "_content": "value of foo",
+                "flag1": "value of flag1",
+                "_defaultval_flag_flag1": "default of flag1",
+                "_defaultval_flag_flagnovalue": "default of flagnovalue",
+            }
+        )
 
+    def test_delvar(self):
+        self.d.delVar("foo")
+        self.assertEqual(self.d.getVarFlag("foo", "flag1", False), None)
+        self.assertEqual(self.d.getVarFlag("foo", "flag2", False), None)
+        self.assertEqual(self.d.getVarFlags("foo", internalflags=True), None)
 
 class Contains(unittest.TestCase):
     def setUp(self):

bitbake/lib/bb/tests/parse.py

@@ -75,6 +75,59 @@ unset B[flag]
         self.assertEqual(d.getVarFlag("A","flag"), None)
         self.assertEqual(d.getVar("B"), "2")
 
+    defaulttest = """
+A = "set value"
+A ??= "default value"
+
+A[flag_set_vs_question] = "set flag"
+A[flag_set_vs_question] ?= "question flag"
+
+A[flag_set_vs_default] = "set flag"
+A[flag_set_vs_default] ??= "default flag"
+
+A[flag_question] ?= "question flag"
+
+A[flag_default] ??= "default flag"
+
+A[flag_question_vs_default] ?= "question flag"
+A[flag_question_vs_default] ??= "default flag"
+
+A[flag_default_vs_question] ??= "default flag"
+A[flag_default_vs_question] ?= "question flag"
+
+A[flag_set_question_default] = "set flag"
+A[flag_set_question_default] ?= "question flag"
+A[flag_set_question_default] ??= "default flag"
+
+A[flag_set_default_question] = "set flag"
+A[flag_set_default_question] ??= "default flag"
+A[flag_set_default_question] ?= "question flag"
+
+A[flag_set_twice] = "set flag first"
+A[flag_set_twice] = "set flag second"
+
+A[flag_question_twice] ?= "question flag first"
+A[flag_question_twice] ?= "question flag second"
+
+A[flag_default_twice] ??= "default flag first"
+A[flag_default_twice] ??= "default flag second"
+"""
+    def test_parse_defaulttest(self):
+        f = self.parsehelper(self.defaulttest)
+        d = bb.parse.handle(f.name, self.d)['']
+        self.assertEqual(d.getVar("A"), "set value")
+        self.assertEqual(d.getVarFlag("A","flag_set_vs_question"), "set flag")
+        self.assertEqual(d.getVarFlag("A","flag_set_vs_default"), "set flag")
+        self.assertEqual(d.getVarFlag("A","flag_question"), "question flag")
+        self.assertEqual(d.getVarFlag("A","flag_default"), "default flag")
+        self.assertEqual(d.getVarFlag("A","flag_question_vs_default"), "question flag")
+        self.assertEqual(d.getVarFlag("A","flag_default_vs_question"), "question flag")
+        self.assertEqual(d.getVarFlag("A","flag_set_question_default"), "set flag")
+        self.assertEqual(d.getVarFlag("A","flag_set_default_question"), "set flag")
+        self.assertEqual(d.getVarFlag("A","flag_set_twice"), "set flag second")
+        self.assertEqual(d.getVarFlag("A","flag_question_twice"), "question flag first")
+        self.assertEqual(d.getVarFlag("A","flag_default_twice"), "default flag second")
+
     exporttest = """
 A = "a"
 export B = "b"
@@ -201,7 +254,7 @@ deltask ${EMPTYVAR}
         f = self.parsehelper(self.addtask_deltask)
         d = bb.parse.handle(f.name, self.d)['']
 
-        self.assertEqual(['do_fetch2', 'do_patch2', 'do_myplaintask', 'do_mytask', 'do_mytask2', 'do_mytask5'], d.getVar("__BBTASKS"))
+        self.assertSequenceEqual(['do_fetch2', 'do_patch2', 'do_myplaintask', 'do_mytask', 'do_mytask2', 'do_mytask5'], bb.build.listtasks(d))
         self.assertEqual(['do_mytask'], d.getVarFlag("do_mytask5", "deps"))
 
     broken_multiline_comment = """
@@ -348,3 +401,65 @@ EXPORT_FUNCTIONS do_compile do_compilepython
             self.assertIn("else", d.getVar("do_compilepython"))
             check_function_flags(d)
 
+    export_function_unclosed_tab = """
+do_compile () {
+       bb.note("Something")
+\t}
+"""
+    export_function_unclosed_space = """
+do_compile () {
+       bb.note("Something")
+ }
+"""
+    export_function_residue = """
+do_compile () {
+       bb.note("Something")
+}
+
+include \\
+"""
+
+    def test_unclosed_functions(self):
+        def test_helper(content, expected_error):
+            with tempfile.TemporaryDirectory() as tempdir:
+                recipename = tempdir + "/recipe_unclosed.bb"
+                with open(recipename, "w") as f:
+                    f.write(content)
+                    f.flush()
+                os.chdir(tempdir)
+                with self.assertRaises(bb.parse.ParseError) as error:
+                    bb.parse.handle(recipename, bb.data.createCopy(self.d))
+                self.assertIn(expected_error, str(error.exception))
+
+        with tempfile.TemporaryDirectory() as tempdir:
+            test_helper(self.export_function_unclosed_tab, "Unparsed lines from unclosed function")
+            test_helper(self.export_function_unclosed_space, "Unparsed lines from unclosed function")
+            test_helper(self.export_function_residue, "Unparsed lines")
+
+            recipename_closed = tempdir + "/recipe_closed.bb"
+            with open(recipename_closed, "w") as in_file:
+                lines = self.export_function_unclosed_tab.split("\n")
+                lines[3] = "}"
+                in_file.write("\n".join(lines))
+                in_file.flush()
+            bb.parse.handle(recipename_closed, bb.data.createCopy(self.d))
+
+    special_character_assignment = """
+A+="a"
+A+ = "b"
++ = "c"
+"""
+    ambigous_assignment = """
++= "d"
+"""
+    def test_parse_special_character_assignment(self):
+        f = self.parsehelper(self.special_character_assignment)
+        d = bb.parse.handle(f.name, self.d)['']
+        self.assertEqual(d.getVar("A"), " a")
+        self.assertEqual(d.getVar("A+"), "b")
+        self.assertEqual(d.getVar("+"), "c")
+
+        f = self.parsehelper(self.ambigous_assignment)
+        with self.assertRaises(bb.parse.ParseError) as error:
+            bb.parse.handle(f.name, self.d)
+        self.assertIn("Empty variable name in assignment", str(error.exception))

bitbake/lib/bb/tests/persist_data.py

@@ -1,129 +0,0 @@
-#
-# BitBake Test for lib/bb/persist_data/
-#
-# Copyright (C) 2018 Garmin Ltd.
-#
-# SPDX-License-Identifier: GPL-2.0-only
-#
-
-import unittest
-import bb.data
-import bb.persist_data
-import tempfile
-import threading
-
-class PersistDataTest(unittest.TestCase):
-    def _create_data(self):
-        return bb.persist_data.persist('TEST_PERSIST_DATA', self.d)
-
-    def setUp(self):
-        self.d = bb.data.init()
-        self.tempdir = tempfile.TemporaryDirectory()
-        self.d['PERSISTENT_DIR'] = self.tempdir.name
-        self.data = self._create_data()
-        self.items = {
-                'A1': '1',
-                'B1': '2',
-                'C2': '3'
-                }
-        self.stress_count = 10000
-        self.thread_count = 5
-
-        for k,v in self.items.items():
-            self.data[k] = v
-
-    def tearDown(self):
-        self.tempdir.cleanup()
-
-    def _iter_helper(self, seen, iterator):
-        with iter(iterator):
-            for v in iterator:
-                self.assertTrue(v in seen)
-                seen.remove(v)
-        self.assertEqual(len(seen), 0, '%s not seen' % seen)
-
-    def test_get(self):
-        for k, v in self.items.items():
-            self.assertEqual(self.data[k], v)
-
-        self.assertIsNone(self.data.get('D'))
-        with self.assertRaises(KeyError):
-            self.data['D']
-
-    def test_set(self):
-        for k, v in self.items.items():
-            self.data[k] += '-foo'
-
-        for k, v in self.items.items():
-            self.assertEqual(self.data[k], v + '-foo')
-
-    def test_delete(self):
-        self.data['D'] = '4'
-        self.assertEqual(self.data['D'], '4')
-        del self.data['D']
-        self.assertIsNone(self.data.get('D'))
-        with self.assertRaises(KeyError):
-            self.data['D']
-
-    def test_contains(self):
-        for k in self.items:
-            self.assertTrue(k in self.data)
-            self.assertTrue(self.data.has_key(k))
-        self.assertFalse('NotFound' in self.data)
-        self.assertFalse(self.data.has_key('NotFound'))
-
-    def test_len(self):
-        self.assertEqual(len(self.data), len(self.items))
-
-    def test_iter(self):
-        self._iter_helper(set(self.items.keys()), self.data)
-
-    def test_itervalues(self):
-        self._iter_helper(set(self.items.values()), self.data.itervalues())
-
-    def test_iteritems(self):
-        self._iter_helper(set(self.items.items()), self.data.iteritems())
-
-    def test_get_by_pattern(self):
-        self._iter_helper({'1', '2'}, self.data.get_by_pattern('_1'))
-
-    def _stress_read(self, data):
-        for i in range(self.stress_count):
-            for k in self.items:
-                data[k]
-
-    def _stress_write(self, data):
-        for i in range(self.stress_count):
-            for k, v in self.items.items():
-                data[k] = v + str(i)
-
-    def _validate_stress(self):
-        for k, v in self.items.items():
-            self.assertEqual(self.data[k], v + str(self.stress_count - 1))
-
-    def test_stress(self):
-        self._stress_read(self.data)
-        self._stress_write(self.data)
-        self._validate_stress()
-
-    def test_stress_threads(self):
-        def read_thread():
-            data = self._create_data()
-            self._stress_read(data)
-
-        def write_thread():
-            data = self._create_data()
-            self._stress_write(data)
-
-        threads = []
-        for i in range(self.thread_count):
-            threads.append(threading.Thread(target=read_thread))
-            threads.append(threading.Thread(target=write_thread))
-
-        for t in threads:
-            t.start()
-        self._stress_read(self.data)
-        for t in threads:
-            t.join()
-        self._validate_stress()
-

bitbake/lib/bb/tests/runqueue-tests/classes/base.bbclass

@@ -9,7 +9,7 @@ def stamptask(d):
     with open(stampname, "a+") as f:
         f.write(d.getVar("BB_UNIHASH") + "\n")
 
-    if d.getVar("BB_CURRENT_MC") != "default":
+    if d.getVar("BB_CURRENT_MC") != "":
         thistask = d.expand("${BB_CURRENT_MC}:${PN}:${BB_CURRENTTASK}")
     if thistask in d.getVar("SLOWTASKS").split():
         bb.note("Slowing task %s" % thistask)

bitbake/lib/bb/tests/runqueue-tests/recipes/g1.bb

@@ -0,0 +1,2 @@
+do_build[mcdepends] = "mc::mc-1:h1:do_invalid"
+

bitbake/lib/bb/tests/runqueue.py

@@ -26,7 +26,7 @@ class RunQueueTests(unittest.TestCase):
     a1_sstatevalid = "a1:do_package a1:do_package_qa a1:do_packagedata a1:do_package_write_ipk a1:do_package_write_rpm a1:do_populate_lic a1:do_populate_sysroot"
     b1_sstatevalid = "b1:do_package b1:do_package_qa b1:do_packagedata b1:do_package_write_ipk b1:do_package_write_rpm b1:do_populate_lic b1:do_populate_sysroot"
 
-    def run_bitbakecmd(self, cmd, builddir, sstatevalid="", slowtasks="", extraenv=None, cleanup=False):
+    def run_bitbakecmd(self, cmd, builddir, sstatevalid="", slowtasks="", extraenv=None, cleanup=False, allowfailure=False):
         env = os.environ.copy()
         env["BBPATH"] = os.path.realpath(os.path.join(os.path.dirname(__file__), "runqueue-tests"))
         env["BB_ENV_PASSTHROUGH_ADDITIONS"] = "SSTATEVALID SLOWTASKS TOPDIR"
@@ -41,6 +41,8 @@ class RunQueueTests(unittest.TestCase):
             output = subprocess.check_output(cmd, env=env, stderr=subprocess.STDOUT,universal_newlines=True, cwd=builddir)
             print(output)
         except subprocess.CalledProcessError as e:
+            if allowfailure:
+                return e.output
             self.fail("Command %s failed with %s" % (cmd, e.output))
         tasks = []
         tasklog = builddir + "/task.log"
@@ -314,6 +316,13 @@ class RunQueueTests(unittest.TestCase):
                        ["mc_2:a1:%s" % t for t in rerun_tasks]
             self.assertEqual(set(tasks), set(expected))
 
+            # Check that a multiconfig that doesn't exist rasies a correct error message
+            error_output = self.run_bitbakecmd(["bitbake", "g1"], tempdir, "", extraenv=extraenv, cleanup=True, allowfailure=True)
+            self.assertIn("non-existent task", error_output)
+            # If the word 'Traceback' or 'KeyError' is in the output we've regressed
+            self.assertNotIn("Traceback", error_output)
+            self.assertNotIn("KeyError", error_output)
+
             self.shutdown(tempdir)
 
     def test_hashserv_single(self):

bitbake/lib/bb/tests/utils.py

@@ -130,6 +130,14 @@ class Checksum(unittest.TestCase):
             checksum = bb.utils.sha256_file(f.name)
             self.assertEqual(checksum, "fcfbae8bf6b721dbb9d2dc6a9334a58f2031a9a9b302999243f99da4d7f12d0f")
 
+    def test_goh1(self):
+        import hashlib
+        with tempfile.NamedTemporaryFile() as f:
+            f.write(self.filler)
+            f.flush()
+            checksum = bb.utils.goh1_file(f.name)
+            self.assertEqual(checksum, "81191f04d4abf413e5badd234814e4202d9efa73e6f9437e9ddd6b8165b569ef")
+
 class EditMetadataFile(unittest.TestCase):
     _origfile = """
 # A comment

bitbake/lib/bb/tinfoil.py

@@ -15,6 +15,7 @@ import atexit
 import re
 from collections import OrderedDict, defaultdict
 from functools import partial
+from contextlib import contextmanager
 
 import bb.cache
 import bb.cooker
@@ -188,11 +189,19 @@ class TinfoilCookerAdapter:
             self._cache[name] = attrvalue
             return attrvalue
 
+    class TinfoilSkiplistByMcAdapter:
+        def __init__(self, tinfoil):
+            self.tinfoil = tinfoil
+
+        def __getitem__(self, mc):
+            return self.tinfoil.get_skipped_recipes(mc)
+
     def __init__(self, tinfoil):
         self.tinfoil = tinfoil
         self.multiconfigs = [''] + (tinfoil.config_data.getVar('BBMULTICONFIG') or '').split()
         self.collections = {}
         self.recipecaches = {}
+        self.skiplist_by_mc = self.TinfoilSkiplistByMcAdapter(tinfoil)
         for mc in self.multiconfigs:
             self.collections[mc] = self.TinfoilCookerCollectionAdapter(tinfoil, mc)
             self.recipecaches[mc] = self.TinfoilRecipeCacheAdapter(tinfoil, mc)
@@ -201,8 +210,6 @@ class TinfoilCookerAdapter:
         # Grab these only when they are requested since they aren't always used
         if name in self._cache:
             return self._cache[name]
-        elif name == 'skiplist':
-            attrvalue = self.tinfoil.get_skipped_recipes()
         elif name == 'bbfile_config_priorities':
             ret = self.tinfoil.run_command('getLayerPriorities')
             bbfile_config_priorities = []
@@ -514,12 +521,12 @@ class Tinfoil:
         """
         return defaultdict(list, self.run_command('getOverlayedRecipes', mc))
 
-    def get_skipped_recipes(self):
+    def get_skipped_recipes(self, mc=''):
         """
         Find recipes which were skipped (i.e. SkipRecipe was raised
         during parsing).
         """
-        return OrderedDict(self.run_command('getSkippedRecipes'))
+        return OrderedDict(self.run_command('getSkippedRecipes', mc))
 
     def get_all_providers(self, mc=''):
         return defaultdict(list, self.run_command('allProviders', mc))
@@ -533,6 +540,7 @@ class Tinfoil:
     def get_runtime_providers(self, rdep):
         return self.run_command('getRuntimeProviders', rdep)
 
+    # TODO: teach this method about mc
     def get_recipe_file(self, pn):
         """
         Get the file name for the specified recipe/target. Raises
@@ -541,6 +549,7 @@ class Tinfoil:
         """
         best = self.find_best_provider(pn)
         if not best or (len(best) > 3 and not best[3]):
+            # TODO: pass down mc
             skiplist = self.get_skipped_recipes()
             taskdata = bb.taskdata.TaskData(None, skiplist=skiplist)
             skipreasons = taskdata.get_reasons(pn)
@@ -633,6 +642,29 @@ class Tinfoil:
         fn = self.get_recipe_file(pn)
         return self.parse_recipe_file(fn)
 
+    @contextmanager
+    def _data_tracked_if_enabled(self):
+        """
+        A context manager to enable data tracking for a code segment if data
+        tracking was enabled for this tinfoil instance.
+        """
+        if self.tracking:
+            # Enable history tracking just for the operation
+            self.run_command('enableDataTracking')
+
+        # Here goes the operation with the optional data tracking
+        yield
+
+        if self.tracking:
+            self.run_command('disableDataTracking')
+
+    def finalizeData(self):
+        """
+        Run anonymous functions and expand keys
+        """
+        with self._data_tracked_if_enabled():
+            return self._reconvert_type(self.run_command('finalizeData'), 'DataStoreConnectionHandle')
+
     def parse_recipe_file(self, fn, appends=True, appendlist=None, config_data=None):
         """
         Parse the specified recipe file (with or without bbappends)
@@ -645,10 +677,7 @@ class Tinfoil:
             appendlist: optional list of bbappend files to apply, if you
                         want to filter them
         """
-        if self.tracking:
-            # Enable history tracking just for the parse operation
-            self.run_command('enableDataTracking')
-        try:
+        with self._data_tracked_if_enabled():
             if appends and appendlist == []:
                 appends = False
             if config_data:
@@ -660,9 +689,6 @@ class Tinfoil:
                 return self._reconvert_type(dscon, 'DataStoreConnectionHandle')
             else:
                 return None
-        finally:
-            if self.tracking:
-                self.run_command('disableDataTracking')
 
     def build_file(self, buildfile, task, internal=True):
         """

bitbake/lib/bb/ui/knotty.py

@@ -24,6 +24,12 @@ import atexit
 from itertools import groupby
 
 from bb.ui import uihelper
+import bb.build
+import bb.command
+import bb.cooker
+import bb.event
+import bb.runqueue
+import bb.utils
 
 featureSet = [bb.cooker.CookerFeatures.SEND_SANITYEVENTS, bb.cooker.CookerFeatures.BASEDATASTORE_TRACKING]
 
@@ -103,7 +109,7 @@ def new_progress(msg, maxval):
         return NonInteractiveProgress(msg, maxval)
 
 def pluralise(singular, plural, qty):
-    if(qty == 1):
+    if qty == 1:
         return singular % qty
     else:
         return plural % qty
@@ -112,6 +118,7 @@ def pluralise(singular, plural, qty):
 class InteractConsoleLogFilter(logging.Filter):
     def __init__(self, tf):
         self.tf = tf
+        super().__init__()
 
     def filter(self, record):
         if record.levelno == bb.msg.BBLogFormatter.NOTE and (record.msg.startswith("Running") or record.msg.startswith("recipe ")):
@@ -555,13 +562,23 @@ def main(server, eventHandler, params, tf = TerminalFilter):
                 }
             })
 
-        bb.utils.mkdirhier(os.path.dirname(consolelogfile))
-        loglink = os.path.join(os.path.dirname(consolelogfile), 'console-latest.log')
+        consolelogdirname = os.path.dirname(consolelogfile)
+        # `bb.utils.mkdirhier` has this check, but it reports failure using bb.fatal, which logs
+        # to the very logger we are trying to set up.
+        if '${' in str(consolelogdirname):
+            print(
+                "FATAL: Directory name {} contains unexpanded bitbake variable. This may cause build failures and WORKDIR pollution.".format(
+                    consolelogdirname))
+            if '${MACHINE}' in consolelogdirname:
+                print("HINT: It looks like you forgot to set MACHINE in local.conf.")
+
+        bb.utils.mkdirhier(consolelogdirname)
+        loglink = os.path.join(consolelogdirname, 'console-latest.log')
         bb.utils.remove(loglink)
         try:
-           os.symlink(os.path.basename(consolelogfile), loglink)
+            os.symlink(os.path.basename(consolelogfile), loglink)
         except OSError:
-           pass
+            pass
 
     # Add the logging domains specified by the user on the command line
     for (domainarg, iterator) in groupby(params.debug_domains):

bitbake/lib/bb/ui/uihelper.py

@@ -31,7 +31,7 @@ class BBUIHelper:
 
         if isinstance(event, bb.build.TaskStarted):
             tid = event._fn + ":" + event._task
-            if event._mc != "default":
+            if event._mc != "":
                 self.running_tasks[tid] = { 'title' : "mc:%s:%s %s" % (event._mc, event._package, event._task), 'starttime' : time.time(), 'pid' : event.pid }
             else:
                 self.running_tasks[tid] = { 'title' : "%s %s" % (event._package, event._task), 'starttime' : time.time(), 'pid' : event.pid }

bitbake/lib/bb/utils.py

@@ -11,11 +11,8 @@ import re, fcntl, os, string, stat, shutil, time
 import sys
 import errno
 import logging
-import bb
-import bb.msg
 import locale
 import multiprocessing
-import fcntl
 import importlib
 import importlib.machinery
 import importlib.util
@@ -24,7 +21,6 @@ import subprocess
 import glob
 import fnmatch
 import traceback
-import errno
 import signal
 import collections
 import copy
@@ -36,6 +32,8 @@ import tempfile
 from subprocess import getstatusoutput
 from contextlib import contextmanager
 from ctypes import cdll
+import bb
+import bb.msg
 
 logger = logging.getLogger("BitBake.Util")
 python_extensions = importlib.machinery.all_suffixes()
@@ -446,6 +444,7 @@ def fileslocked(files, *args, **kwargs):
     try:
         yield
     finally:
+        locks.reverse()
         for lock in locks:
             bb.utils.unlockfile(lock)
 
@@ -584,6 +583,31 @@ def sha512_file(filename):
     import hashlib
     return _hasher(hashlib.sha512(), filename)
 
+def goh1_file(filename):
+    """
+    Return the hex string representation of the Go mod h1 checksum of the
+    filename. The Go mod h1 checksum uses the Go dirhash package. The package
+    defines hashes over directory trees and is used by go mod for mod files and
+    zip archives.
+    """
+    import hashlib
+    import zipfile
+
+    lines = []
+    if zipfile.is_zipfile(filename):
+        with zipfile.ZipFile(filename) as archive:
+            for fn in sorted(archive.namelist()):
+                method = hashlib.sha256()
+                method.update(archive.read(fn))
+                hash = method.hexdigest()
+                lines.append("%s  %s\n" % (hash, fn))
+    else:
+        hash = _hasher(hashlib.sha256(), filename)
+        lines.append("%s  go.mod\n" % hash)
+    method = hashlib.sha256()
+    method.update("".join(lines).encode('utf-8'))
+    return method.hexdigest()
+
 def preserved_envvars_exported():
     """Variables which are taken from the environment and placed in and exported
     from the metadata"""
@@ -1431,8 +1455,6 @@ def edit_bblayers_conf(bblayers_conf, add, remove, edit_cb=None):
             but weren't (because they weren't in the list)
     """
 
-    import fnmatch
-
     def remove_trailing_sep(pth):
         if pth and pth[-1] == os.sep:
             pth = pth[:-1]
@@ -1623,7 +1645,7 @@ def ioprio_set(who, cls, value):
         bb.warn("Unable to set IO Prio for arch %s" % _unamearch)
 
 def set_process_name(name):
-    from ctypes import cdll, byref, create_string_buffer
+    from ctypes import byref, create_string_buffer
     # This is nice to have for debugging, not essential
     try:
         libc = cdll.LoadLibrary('libc.so.6')
@@ -1854,15 +1876,42 @@ def path_is_descendant(descendant, ancestor):
 
     return False
 
+# Recomputing the sets in signal.py is expensive (bitbake -pP idle)
+# so try and use _signal directly to avoid it
+valid_signals = signal.valid_signals()
+try:
+    import _signal
+    sigmask = _signal.pthread_sigmask
+except ImportError:
+    sigmask = signal.pthread_sigmask
+
 # If we don't have a timeout of some kind and a process/thread exits badly (for example
 # OOM killed) and held a lock, we'd just hang in the lock futex forever. It is better
 # we exit at some point than hang. 5 minutes with no progress means we're probably deadlocked.
+# This function can still deadlock python since it can't signal the other threads to exit
+# (signals are handled in the main thread) and even os._exit() will wait on non-daemon threads
+# to exit.
 @contextmanager
 def lock_timeout(lock):
-    held = lock.acquire(timeout=5*60)
     try:
+        s = sigmask(signal.SIG_BLOCK, valid_signals)
+        held = lock.acquire(timeout=5*60)
         if not held:
+            bb.server.process.serverlog("Couldn't get the lock for 5 mins, timed out, exiting.\n%s" % traceback.format_stack())
             os._exit(1)
         yield held
     finally:
         lock.release()
+        sigmask(signal.SIG_SETMASK, s)
+
+# A version of lock_timeout without the check that the lock was locked and a shorter timeout
+@contextmanager
+def lock_timeout_nocheck(lock):
+    try:
+        s = sigmask(signal.SIG_BLOCK, valid_signals)
+        l = lock.acquire(timeout=10)
+        yield l
+    finally:
+        if l:
+            lock.release()
+        sigmask(signal.SIG_SETMASK, s)

bitbake/lib/bblayers/query.py

@@ -142,10 +142,10 @@ skipped recipes will also be listed, with a " (skipped)" suffix.
         # Ensure we list skipped recipes
         # We are largely guessing about PN, PV and the preferred version here,
         # but we have no choice since skipped recipes are not fully parsed
-        skiplist = list(self.tinfoil.cooker.skiplist.keys())
-        mcspec = 'mc:%s:' % mc
+        skiplist = list(self.tinfoil.cooker.skiplist_by_mc[mc].keys())
+
         if mc:
-            skiplist = [s[len(mcspec):] for s in skiplist if s.startswith(mcspec)]
+            skiplist = [s.removeprefix(f'mc:{mc}:') for s in skiplist]
 
         for fn in skiplist:
             recipe_parts = os.path.splitext(os.path.basename(fn))[0].split('_')
@@ -162,7 +162,7 @@ skipped recipes will also be listed, with a " (skipped)" suffix.
         def print_item(f, pn, ver, layer, ispref):
             if not selected_layer or layer == selected_layer:
                 if not bare and f in skiplist:
-                    skipped = ' (skipped: %s)' % self.tinfoil.cooker.skiplist[f].skipreason
+                    skipped = ' (skipped: %s)' % self.tinfoil.cooker.skiplist_by_mc[mc][f].skipreason
                 else:
                     skipped = ''
                 if show_filenames:
@@ -301,7 +301,7 @@ Lists recipes with the bbappends that apply to them as subitems.
             if self.show_appends_for_pn(pn, cooker_data, args.mc):
                 appends = True
 
-        if not args.pnspec and self.show_appends_for_skipped():
+        if not args.pnspec and self.show_appends_for_skipped(args.mc):
             appends = True
 
         if not appends:
@@ -317,9 +317,9 @@ Lists recipes with the bbappends that apply to them as subitems.
 
         return self.show_appends_output(filenames, best_filename)
 
-    def show_appends_for_skipped(self):
+    def show_appends_for_skipped(self, mc):
         filenames = [os.path.basename(f)
-                    for f in self.tinfoil.cooker.skiplist.keys()]
+                    for f in self.tinfoil.cooker.skiplist_by_mc[mc].keys()]
         return self.show_appends_output(filenames, None, " (skipped)")
 
     def show_appends_output(self, filenames, best_filename, name_suffix = ''):

bitbake/lib/bs4/__init__.py

@@ -835,6 +835,5 @@ class FeatureNotFound(ValueError):
 
 #If this file is run as a script, act as an HTML pretty-printer.
 if __name__ == '__main__':
-    import sys
     soup = BeautifulSoup(sys.stdin)
     print((soup.prettify()))

bitbake/lib/bs4/diagnose.py

@@ -17,7 +17,6 @@ import tempfile
 import time
 import traceback
 import sys
-import cProfile
 
 def diagnose(data):
     """Diagnostic suite for isolating common problems.

bitbake/lib/hashserv/client.py

@@ -78,6 +78,7 @@ class AsyncClient(bb.asyncrpc.AsyncClient):
     MODE_NORMAL = 0
     MODE_GET_STREAM = 1
     MODE_EXIST_STREAM = 2
+    MODE_MARK_STREAM = 3
 
     def __init__(self, username=None, password=None):
         super().__init__("OEHASHEQUIV", "1.1", logger)
@@ -164,6 +165,8 @@ class AsyncClient(bb.asyncrpc.AsyncClient):
             await normal_to_stream("get-stream")
         elif new_mode == self.MODE_EXIST_STREAM:
             await normal_to_stream("exists-stream")
+        elif new_mode == self.MODE_MARK_STREAM:
+            await normal_to_stream("gc-mark-stream")
         elif new_mode != self.MODE_NORMAL:
             raise Exception("Undefined mode transition {self.mode!r} -> {new_mode!r}")
 
@@ -306,6 +309,24 @@ class AsyncClient(bb.asyncrpc.AsyncClient):
         """
         return await self.invoke({"gc-mark": {"mark": mark, "where": where}})
 
+    async def gc_mark_stream(self, mark, rows):
+        """
+        Similar to `gc-mark`, but accepts a list of "where" key-value pair
+        conditions. It utilizes stream mode to mark hashes, which helps reduce
+        the impact of latency when communicating with the hash equivalence
+        server.
+        """
+        def row_to_dict(row):
+            pairs = row.split()
+            return dict(zip(pairs[::2], pairs[1::2]))
+
+        responses = await self.send_stream_batch(
+            self.MODE_MARK_STREAM,
+            (json.dumps({"mark": mark, "where": row_to_dict(row)}) for row in rows),
+        )
+
+        return {"count": sum(int(json.loads(r)["count"]) for r in responses)}
+
     async def gc_sweep(self, mark):
         """
         Finishes garbage collection for "mark". All unihash entries that have
@@ -351,6 +372,7 @@ class Client(bb.asyncrpc.Client):
             "get_db_query_columns",
             "gc_status",
             "gc_mark",
+            "gc_mark_stream",
             "gc_sweep",
         )
 

bitbake/lib/hashserv/server.py

@@ -10,6 +10,7 @@ import math
 import time
 import os
 import base64
+import json
 import hashlib
 from . import create_async_client
 import bb.asyncrpc
@@ -256,6 +257,7 @@ class ServerClient(bb.asyncrpc.AsyncServerConnection):
                     "backfill-wait": self.handle_backfill_wait,
                     "remove": self.handle_remove,
                     "gc-mark": self.handle_gc_mark,
+                    "gc-mark-stream": self.handle_gc_mark_stream,
                     "gc-sweep": self.handle_gc_sweep,
                     "gc-status": self.handle_gc_status,
                     "clean-unused": self.handle_clean_unused,
@@ -583,6 +585,33 @@ class ServerClient(bb.asyncrpc.AsyncServerConnection):
 
         return {"count": await self.db.gc_mark(mark, condition)}
 
+    @permissions(DB_ADMIN_PERM)
+    async def handle_gc_mark_stream(self, request):
+        async def handler(line):
+            try:
+                decoded_line = json.loads(line)
+            except json.JSONDecodeError as exc:
+                raise bb.asyncrpc.InvokeError(
+                    "Could not decode JSONL input '%s'" % line
+                ) from exc
+
+            try:
+                mark = decoded_line["mark"]
+                condition = decoded_line["where"]
+                if not isinstance(mark, str):
+                    raise TypeError("Bad mark type %s" % type(mark))
+
+                if not isinstance(condition, dict):
+                    raise TypeError("Bad condition type %s" % type(condition))
+            except KeyError as exc:
+                raise bb.asyncrpc.InvokeError(
+                    "Input line is missing key '%s' " % exc
+                ) from exc
+
+            return json.dumps({"count": await self.db.gc_mark(mark, condition)})
+
+        return await self._stream_handler(handler)
+
     @permissions(DB_ADMIN_PERM)
     async def handle_gc_sweep(self, request):
         mark = request["mark"]

bitbake/lib/hashserv/tests.py

@@ -969,6 +969,48 @@ class HashEquivalenceCommonTests(object):
         # First hash is still present
         self.assertClientGetHash(self.client, taskhash, unihash)
 
+    def test_gc_stream(self):
+        taskhash = '53b8dce672cb6d0c73170be43f540460bfc347b4'
+        outhash = '5a9cb1649625f0bf41fc7791b635cd9c2d7118c7f021ba87dcd03f72b67ce7a8'
+        unihash = 'f37918cc02eb5a520b1aff86faacbc0a38124646'
+
+        result = self.client.report_unihash(taskhash, self.METHOD, outhash, unihash)
+        self.assertEqual(result['unihash'], unihash, 'Server returned bad unihash')
+
+        taskhash2 = '3bf6f1e89d26205aec90da04854fbdbf73afe6b4'
+        outhash2 = '77623a549b5b1a31e3732dfa8fe61d7ce5d44b3370f253c5360e136b852967b4'
+        unihash2 = 'af36b199320e611fbb16f1f277d3ee1d619ca58b'
+
+        result = self.client.report_unihash(taskhash2, self.METHOD, outhash2, unihash2)
+        self.assertClientGetHash(self.client, taskhash2, unihash2)
+
+        taskhash3 = 'a1117c1f5a7c9ab2f5a39cc6fe5e6152169d09c0'
+        outhash3 = '7289c414905303700a1117c1f5a7c9ab2f5a39cc6fe5e6152169d09c04f9a53c'
+        unihash3 = '905303700a1117c1f5a7c9ab2f5a39cc6fe5e615'
+
+        result = self.client.report_unihash(taskhash3, self.METHOD, outhash3, unihash3)
+        self.assertClientGetHash(self.client, taskhash3, unihash3)
+
+        # Mark the first unihash to be kept
+        ret = self.client.gc_mark_stream("ABC", (f"unihash {h}" for h in [unihash, unihash2]))
+        self.assertEqual(ret, {"count": 2})
+
+        ret = self.client.gc_status()
+        self.assertEqual(ret, {"mark": "ABC", "keep": 2, "remove": 1})
+
+        # Third hash is still there; mark doesn't delete hashes
+        self.assertClientGetHash(self.client, taskhash3, unihash3)
+
+        ret = self.client.gc_sweep("ABC")
+        self.assertEqual(ret, {"count": 1})
+
+        # Hash is gone. Taskhash is returned for second hash
+        self.assertClientGetHash(self.client, taskhash3, None)
+        # First hash is still present
+        self.assertClientGetHash(self.client, taskhash, unihash)
+        # Second hash is still present
+        self.assertClientGetHash(self.client, taskhash2, unihash2)
+
     def test_gc_switch_mark(self):
         taskhash = '53b8dce672cb6d0c73170be43f540460bfc347b4'
         outhash = '5a9cb1649625f0bf41fc7791b635cd9c2d7118c7f021ba87dcd03f72b67ce7a8'

bitbake/lib/ply/yacc.py

@@ -1122,7 +1122,6 @@ class LRParser:
 # manipulate the rules that make up a grammar. 
 # -----------------------------------------------------------------------------
 
-import re
 
 # regex matching identifiers
 _is_identifier = re.compile(r'^[a-zA-Z0-9_-]+$')

bitbake/lib/toaster/orm/fixtures/check_fixtures.py

@@ -0,0 +1,38 @@
+#!/usr/bin/env python3
+#
+# Copyright (C) 2025 Linux Foundation
+# SPDX-License-Identifier: GPL-2.0-only
+#
+
+import json
+import urllib.request
+
+import gen_fixtures as fixtures
+
+RELEASE_URL = "https://dashboard.yoctoproject.org/releases.json"
+
+with urllib.request.urlopen(RELEASE_URL) as response:
+    if response.getcode() == 200:
+        data = response.read().decode("utf-8")
+        releases = json.loads(data)
+    else:
+        print("Couldn't access %s: %s" % (RELEASE_URL, reponse.getcode()))
+        exit(1)
+
+
+# grab the recent release branches and add master, so we can ignore old branches
+active_releases = [
+    e["release_codename"].lower() for e in releases if e["series"] == "current"
+]
+active_releases.append("master")
+active_releases.append("head")
+
+fixtures_releases = [x[0].lower() for x in fixtures.current_releases]
+
+if set(active_releases) != set(fixtures_releases):
+    print("WARNING: Active releases don't match toaster configured releases, the difference is: %s" % set(active_releases).difference(set(fixtures_releases)))
+    print("Active releases: %s" % sorted(active_releases))
+    print("Toaster configured releases: %s" % sorted(fixtures_releases))
+else:
+    print("Success, configuration matches")
+

bitbake/lib/toaster/orm/fixtures/gen_fixtures.py

@@ -41,15 +41,15 @@ current_releases = [
     # Release slot #3 'master'
     ['Master','master','','Yocto Project master','master','','master'],
     # Release slot #4
+    ['Walnascar','5.2','April 2025','5.2.0 (April 2024)','Support for 7 months (until October 2025)','','2.12'],
     ['Styhead','5.1','November 2024','5.1.0 (November 2024)','Support for 7 months (until May 2025)','','2.10'],
-#   ['Nanbield','4.3','November 2023','4.3.0 (November 2023)','Support for 7 months (until May 2024)','','2.6'],
-#   ['Mickledore','4.2','April 2023','4.2.0 (April 2023)','Support for 7 months (until October 2023)','','2.4'],
-#   ['Langdale','4.1','October 2022','4.1.2 (January 2023)','Support for 7 months (until May 2023)','','2.2'],
+    #['Nanbield','4.3','November 2023','4.3.0 (November 2023)','Support for 7 months (until May 2024)','','2.6'],
+    #['Mickledore','4.2','April 2023','4.2.0 (April 2023)','Support for 7 months (until October 2023)','','2.4'],
+    #['Langdale','4.1','October 2022','4.1.2 (January 2023)','Support for 7 months (until May 2023)','','2.2'],
     ['Kirkstone','4.0','April 2022','4.0.8 (March 2023)','Stable - Long Term Support (until Apr. 2024)','','2.0'],
-#   ['Honister','3.4','October 2021','3.4.2 (February 2022)','Support for 7 months (until May 2022)','26.0','1.52'],
-#   ['Hardknott','3.3','April 2021','3.3.5 (March 2022)','Stable - Support for 13 months (until Apr. 2022)','25.0','1.50'],
-#   ['Gatesgarth','3.2','Oct 2020','3.2.4 (May 2021)','EOL','24.0','1.48'],
-    # Optional Release slot #5
+    #['Honister','3.4','October 2021','3.4.2 (February 2022)','Support for 7 months (until May 2022)','26.0','1.52'],
+    #['Hardknott','3.3','April 2021','3.3.5 (March 2022)','Stable - Support for 13 months (until Apr. 2022)','25.0','1.50'],
+    #['Gatesgarth','3.2','Oct 2020','3.2.4 (May 2021)','EOL','24.0','1.48'],
     #['Dunfell','3.1','April 2020','3.1.23 (February 2023)','Stable - Long Term Support (until Apr. 2024)','23.0','1.46'],
 ]
 

bitbake/lib/toaster/orm/fixtures/oe-core.xml

@@ -23,11 +23,16 @@
     <field type="CharField" name="branch">master</field>
   </object>
   <object model="orm.bitbakeversion" pk="4">
+    <field type="CharField" name="name">walnascar</field>
+    <field type="CharField" name="giturl">git://git.openembedded.org/bitbake</field>
+    <field type="CharField" name="branch">2.12</field>
+  </object>
+  <object model="orm.bitbakeversion" pk="5">
     <field type="CharField" name="name">styhead</field>
     <field type="CharField" name="giturl">git://git.openembedded.org/bitbake</field>
     <field type="CharField" name="branch">2.10</field>
   </object>
-  <object model="orm.bitbakeversion" pk="5">
+  <object model="orm.bitbakeversion" pk="6">
     <field type="CharField" name="name">kirkstone</field>
     <field type="CharField" name="giturl">git://git.openembedded.org/bitbake</field>
     <field type="CharField" name="branch">2.0</field>
@@ -56,16 +61,23 @@
     <field type="TextField" name="helptext">Toaster will run your builds using the tip of the &lt;a href=\"https://cgit.openembedded.org/openembedded-core/log/\"&gt;OpenEmbedded master&lt;/a&gt; branch.</field>
   </object>
   <object model="orm.release" pk="4">
+    <field type="CharField" name="name">walnascar</field>
+    <field type="CharField" name="description">Openembedded Walnascar</field>
+    <field rel="ManyToOneRel" to="orm.bitbakeversion" name="bitbake_version">4</field>
+    <field type="CharField" name="branch_name">walnascar</field>
+    <field type="TextField" name="helptext">Toaster will run your builds using the tip of the &lt;a href=\"https://cgit.openembedded.org/openembedded-core/log/?h=walnascar\"&gt;OpenEmbedded Walnascar&lt;/a&gt; branch.</field>
+  </object>
+  <object model="orm.release" pk="5">
     <field type="CharField" name="name">styhead</field>
     <field type="CharField" name="description">Openembedded Styhead</field>
-    <field rel="ManyToOneRel" to="orm.bitbakeversion" name="bitbake_version">4</field>
+    <field rel="ManyToOneRel" to="orm.bitbakeversion" name="bitbake_version">5</field>
     <field type="CharField" name="branch_name">styhead</field>
     <field type="TextField" name="helptext">Toaster will run your builds using the tip of the &lt;a href=\"https://cgit.openembedded.org/openembedded-core/log/?h=styhead\"&gt;OpenEmbedded Styhead&lt;/a&gt; branch.</field>
   </object>
-  <object model="orm.release" pk="5">
+  <object model="orm.release" pk="6">
     <field type="CharField" name="name">kirkstone</field>
     <field type="CharField" name="description">Openembedded Kirkstone</field>
-    <field rel="ManyToOneRel" to="orm.bitbakeversion" name="bitbake_version">5</field>
+    <field rel="ManyToOneRel" to="orm.bitbakeversion" name="bitbake_version">6</field>
     <field type="CharField" name="branch_name">kirkstone</field>
     <field type="TextField" name="helptext">Toaster will run your builds using the tip of the &lt;a href=\"https://cgit.openembedded.org/openembedded-core/log/?h=kirkstone\"&gt;OpenEmbedded Kirkstone&lt;/a&gt; branch.</field>
   </object>
@@ -91,6 +103,10 @@
     <field rel="ManyToOneRel" to="orm.release" name="release">5</field>
     <field type="CharField" name="layer_name">openembedded-core</field>
   </object>
+  <object model="orm.releasedefaultlayer" pk="6">
+    <field rel="ManyToOneRel" to="orm.release" name="release">6</field>
+    <field type="CharField" name="layer_name">openembedded-core</field>
+  </object>
 
 
   <!-- Layer for the Local release -->

bitbake/lib/toaster/orm/fixtures/poky.xml

@@ -26,12 +26,18 @@
     <field type="CharField" name="dirpath">bitbake</field>
   </object>
   <object model="orm.bitbakeversion" pk="4">
+    <field type="CharField" name="name">walnascar</field>
+    <field type="CharField" name="giturl">git://git.yoctoproject.org/poky</field>
+    <field type="CharField" name="branch">walnascar</field>
+    <field type="CharField" name="dirpath">bitbake</field>
+  </object>
+  <object model="orm.bitbakeversion" pk="5">
     <field type="CharField" name="name">styhead</field>
     <field type="CharField" name="giturl">git://git.yoctoproject.org/poky</field>
     <field type="CharField" name="branch">styhead</field>
     <field type="CharField" name="dirpath">bitbake</field>
   </object>
-  <object model="orm.bitbakeversion" pk="5">
+  <object model="orm.bitbakeversion" pk="6">
     <field type="CharField" name="name">kirkstone</field>
     <field type="CharField" name="giturl">git://git.yoctoproject.org/poky</field>
     <field type="CharField" name="branch">kirkstone</field>
@@ -62,16 +68,23 @@
     <field type="TextField" name="helptext">Toaster will run your builds using the tip of the &lt;a href="https://git.yoctoproject.org/cgit/cgit.cgi/poky/log/"&gt;Yocto Project Master branch&lt;/a&gt;.</field>
   </object>
   <object model="orm.release" pk="4">
+    <field type="CharField" name="name">walnascar</field>
+    <field type="CharField" name="description">Yocto Project 5.2 "Walnascar"</field>
+    <field rel="ManyToOneRel" to="orm.bitbakeversion" name="bitbake_version">4</field>
+    <field type="CharField" name="branch_name">walnascar</field>
+    <field type="TextField" name="helptext">Toaster will run your builds using the tip of the &lt;a href="https://git.yoctoproject.org/cgit/cgit.cgi/poky/log/?h=walnascar"&gt;Yocto Project Walnascar branch&lt;/a&gt;.</field>
+  </object>
+  <object model="orm.release" pk="5">
     <field type="CharField" name="name">styhead</field>
     <field type="CharField" name="description">Yocto Project 5.1 "Styhead"</field>
-    <field rel="ManyToOneRel" to="orm.bitbakeversion" name="bitbake_version">4</field>
+    <field rel="ManyToOneRel" to="orm.bitbakeversion" name="bitbake_version">5</field>
     <field type="CharField" name="branch_name">styhead</field>
     <field type="TextField" name="helptext">Toaster will run your builds using the tip of the &lt;a href="https://git.yoctoproject.org/cgit/cgit.cgi/poky/log/?h=styhead"&gt;Yocto Project Styhead branch&lt;/a&gt;.</field>
   </object>
-  <object model="orm.release" pk="5">
+  <object model="orm.release" pk="6">
     <field type="CharField" name="name">kirkstone</field>
     <field type="CharField" name="description">Yocto Project 4.0 "Kirkstone"</field>
-    <field rel="ManyToOneRel" to="orm.bitbakeversion" name="bitbake_version">5</field>
+    <field rel="ManyToOneRel" to="orm.bitbakeversion" name="bitbake_version">6</field>
     <field type="CharField" name="branch_name">kirkstone</field>
     <field type="TextField" name="helptext">Toaster will run your builds using the tip of the &lt;a href="https://git.yoctoproject.org/cgit/cgit.cgi/poky/log/?h=kirkstone"&gt;Yocto Project Kirkstone branch&lt;/a&gt;.</field>
   </object>
@@ -137,6 +150,18 @@
     <field rel="ManyToOneRel" to="orm.release" name="release">5</field>
     <field type="CharField" name="layer_name">meta-yocto-bsp</field>
   </object>
+  <object model="orm.releasedefaultlayer" pk="16">
+    <field rel="ManyToOneRel" to="orm.release" name="release">6</field>
+    <field type="CharField" name="layer_name">openembedded-core</field>
+  </object>
+  <object model="orm.releasedefaultlayer" pk="17">
+    <field rel="ManyToOneRel" to="orm.release" name="release">6</field>
+    <field type="CharField" name="layer_name">meta-poky</field>
+  </object>
+  <object model="orm.releasedefaultlayer" pk="18">
+    <field rel="ManyToOneRel" to="orm.release" name="release">6</field>
+    <field type="CharField" name="layer_name">meta-yocto-bsp</field>
+  </object>
 
   <!-- Default layers provided by poky
        openembedded-core
@@ -177,13 +202,20 @@
     <field rel="ManyToOneRel" to="orm.layer" name="layer">1</field>
     <field type="IntegerField" name="layer_source">0</field>
     <field rel="ManyToOneRel" to="orm.release" name="release">4</field>
-    <field type="CharField" name="branch">styhead</field>
+    <field type="CharField" name="branch">walnascar</field>
     <field type="CharField" name="dirpath">meta</field>
   </object>
   <object model="orm.layer_version" pk="5">
     <field rel="ManyToOneRel" to="orm.layer" name="layer">1</field>
     <field type="IntegerField" name="layer_source">0</field>
     <field rel="ManyToOneRel" to="orm.release" name="release">5</field>
+    <field type="CharField" name="branch">styhead</field>
+    <field type="CharField" name="dirpath">meta</field>
+  </object>
+  <object model="orm.layer_version" pk="6">
+    <field rel="ManyToOneRel" to="orm.layer" name="layer">1</field>
+    <field type="IntegerField" name="layer_source">0</field>
+    <field rel="ManyToOneRel" to="orm.release" name="release">6</field>
     <field type="CharField" name="branch">kirkstone</field>
     <field type="CharField" name="dirpath">meta</field>
   </object>
@@ -196,14 +228,14 @@
     <field type="CharField" name="vcs_web_tree_base_url">https://git.yoctoproject.org/cgit/cgit.cgi/poky/tree/%path%?h=%branch%</field>
     <field type="CharField" name="vcs_web_file_base_url">https://git.yoctoproject.org/cgit/cgit.cgi/poky/tree/%path%?h=%branch%</field>
   </object>
-  <object model="orm.layer_version" pk="6">
+  <object model="orm.layer_version" pk="7">
     <field rel="ManyToOneRel" to="orm.layer" name="layer">2</field>
     <field type="IntegerField" name="layer_source">0</field>
     <field rel="ManyToOneRel" to="orm.release" name="release">1</field>
     <field type="CharField" name="branch">scarthgap</field>
     <field type="CharField" name="dirpath">meta-poky</field>
   </object>
-  <object model="orm.layer_version" pk="7">
+  <object model="orm.layer_version" pk="8">
     <field rel="ManyToOneRel" to="orm.layer" name="layer">2</field>
     <field type="IntegerField" name="layer_source">0</field>
     <field rel="ManyToOneRel" to="orm.release" name="release">2</field>
@@ -211,24 +243,31 @@
     <field type="CharField" name="commit">HEAD</field>
     <field type="CharField" name="dirpath">meta-poky</field>
   </object>
-  <object model="orm.layer_version" pk="8">
+  <object model="orm.layer_version" pk="9">
     <field rel="ManyToOneRel" to="orm.layer" name="layer">2</field>
     <field type="IntegerField" name="layer_source">0</field>
     <field rel="ManyToOneRel" to="orm.release" name="release">3</field>
     <field type="CharField" name="branch">master</field>
     <field type="CharField" name="dirpath">meta-poky</field>
   </object>
-  <object model="orm.layer_version" pk="9">
-    <field rel="ManyToOneRel" to="orm.layer" name="layer">2</field>
-    <field type="IntegerField" name="layer_source">0</field>
-    <field rel="ManyToOneRel" to="orm.release" name="release">4</field>
-    <field type="CharField" name="branch">styhead</field>
-    <field type="CharField" name="dirpath">meta-poky</field>
-  </object>
   <object model="orm.layer_version" pk="10">
+    <field rel="ManyToOneRel" to="orm.layer" name="layer">2</field>
+    <field type="IntegerField" name="layer_source">0</field>
+    <field rel="ManyToOneRel" to="orm.release" name="release">4</field>
+    <field type="CharField" name="branch">walnascar</field>
+    <field type="CharField" name="dirpath">meta-poky</field>
+  </object>
+  <object model="orm.layer_version" pk="11">
     <field rel="ManyToOneRel" to="orm.layer" name="layer">2</field>
     <field type="IntegerField" name="layer_source">0</field>
     <field rel="ManyToOneRel" to="orm.release" name="release">5</field>
+    <field type="CharField" name="branch">styhead</field>
+    <field type="CharField" name="dirpath">meta-poky</field>
+  </object>
+  <object model="orm.layer_version" pk="12">
+    <field rel="ManyToOneRel" to="orm.layer" name="layer">2</field>
+    <field type="IntegerField" name="layer_source">0</field>
+    <field rel="ManyToOneRel" to="orm.release" name="release">6</field>
     <field type="CharField" name="branch">kirkstone</field>
     <field type="CharField" name="dirpath">meta-poky</field>
   </object>
@@ -241,14 +280,14 @@
     <field type="CharField" name="vcs_web_tree_base_url">https://git.yoctoproject.org/cgit/cgit.cgi/poky/tree/%path%?h=%branch%</field>
     <field type="CharField" name="vcs_web_file_base_url">https://git.yoctoproject.org/cgit/cgit.cgi/poky/tree/%path%?h=%branch%</field>
   </object>
-  <object model="orm.layer_version" pk="11">
+  <object model="orm.layer_version" pk="13">
     <field rel="ManyToOneRel" to="orm.layer" name="layer">3</field>
     <field type="IntegerField" name="layer_source">0</field>
     <field rel="ManyToOneRel" to="orm.release" name="release">1</field>
     <field type="CharField" name="branch">scarthgap</field>
     <field type="CharField" name="dirpath">meta-yocto-bsp</field>
   </object>
-  <object model="orm.layer_version" pk="12">
+  <object model="orm.layer_version" pk="14">
     <field rel="ManyToOneRel" to="orm.layer" name="layer">3</field>
     <field type="IntegerField" name="layer_source">0</field>
     <field rel="ManyToOneRel" to="orm.release" name="release">2</field>
@@ -256,24 +295,31 @@
     <field type="CharField" name="commit">HEAD</field>
     <field type="CharField" name="dirpath">meta-yocto-bsp</field>
   </object>
-  <object model="orm.layer_version" pk="13">
+  <object model="orm.layer_version" pk="15">
     <field rel="ManyToOneRel" to="orm.layer" name="layer">3</field>
     <field type="IntegerField" name="layer_source">0</field>
     <field rel="ManyToOneRel" to="orm.release" name="release">3</field>
     <field type="CharField" name="branch">master</field>
     <field type="CharField" name="dirpath">meta-yocto-bsp</field>
   </object>
-  <object model="orm.layer_version" pk="14">
+  <object model="orm.layer_version" pk="16">
     <field rel="ManyToOneRel" to="orm.layer" name="layer">3</field>
     <field type="IntegerField" name="layer_source">0</field>
     <field rel="ManyToOneRel" to="orm.release" name="release">4</field>
-    <field type="CharField" name="branch">styhead</field>
+    <field type="CharField" name="branch">walnascar</field>
     <field type="CharField" name="dirpath">meta-yocto-bsp</field>
   </object>
-  <object model="orm.layer_version" pk="15">
+  <object model="orm.layer_version" pk="17">
     <field rel="ManyToOneRel" to="orm.layer" name="layer">3</field>
     <field type="IntegerField" name="layer_source">0</field>
     <field rel="ManyToOneRel" to="orm.release" name="release">5</field>
+    <field type="CharField" name="branch">styhead</field>
+    <field type="CharField" name="dirpath">meta-yocto-bsp</field>
+  </object>
+  <object model="orm.layer_version" pk="18">
+    <field rel="ManyToOneRel" to="orm.layer" name="layer">3</field>
+    <field type="IntegerField" name="layer_source">0</field>
+    <field rel="ManyToOneRel" to="orm.release" name="release">6</field>
     <field type="CharField" name="branch">kirkstone</field>
     <field type="CharField" name="dirpath">meta-yocto-bsp</field>
   </object>

bitbake/lib/toaster/orm/models.py

@@ -79,7 +79,6 @@ if 'sqlite' in settings.DATABASES['default']['ENGINE']:
     # end of HACK
 
 class GitURLValidator(validators.URLValidator):
-    import re
     regex = re.compile(
         r'^(?:ssh|git|http|ftp)s?://'  # http:// or https://
         r'(?:(?:[A-Z0-9](?:[A-Z0-9-]{0,61}[A-Z0-9])?\.)+(?:[A-Z]{2,6}\.?|[A-Z0-9-]{2,}\.?)|'  # domain...
@@ -1500,7 +1499,7 @@ class Layer_Version(models.Model):
     # code lifted, with adaptations, from the layerindex-web application
     # https://git.yoctoproject.org/cgit/cgit.cgi/layerindex-web/
     def _handle_url_path(self, base_url, path):
-        import re, posixpath
+        import posixpath
         if base_url:
             if self.dirpath:
                 if path:

bitbake/lib/toaster/tests/browser/selenium_helpers_base.py

@@ -27,7 +27,7 @@ from selenium.webdriver.common.by import By
 from selenium.webdriver.common.desired_capabilities import DesiredCapabilities
 from selenium.common.exceptions import NoSuchElementException, \
         StaleElementReferenceException, TimeoutException, \
-        SessionNotCreatedException
+        SessionNotCreatedException, WebDriverException
 
 def create_selenium_driver(cls,browser='chrome'):
     # set default browser string based on env (if available)
@@ -90,7 +90,7 @@ class Wait(WebDriverWait):
     Subclass of WebDriverWait with predetermined timeout and poll
     frequency. Also deals with a wider variety of exceptions.
     """
-    _TIMEOUT = 10
+    _TIMEOUT = 20
     _POLL_FREQUENCY = 0.5
 
     def __init__(self, driver, timeout=_TIMEOUT, poll=_POLL_FREQUENCY):
@@ -114,6 +114,9 @@ class Wait(WebDriverWait):
                 pass
             except StaleElementReferenceException:
                 pass
+            except WebDriverException:
+                # selenium.common.exceptions.WebDriverException: Message: unknown error: unhandled inspector error: {"code":-32000,"message":"Node with given id does not belong to the document"}
+                pass
 
             time.sleep(self._poll)
             if time.time() > end_time:
@@ -183,7 +186,7 @@ class SeleniumTestCaseBase(unittest.TestCase):
         self.driver.get(abs_url)
 
         try:  # Ensure page is loaded before proceeding
-            self.wait_until_visible("#global-nav", poll=3)
+            self.wait_until_visible("#global-nav")
         except NoSuchElementException:
             self.driver.implicitly_wait(3)
         except TimeoutException:
@@ -208,36 +211,43 @@ class SeleniumTestCaseBase(unittest.TestCase):
         """ Return the element which currently has focus on the page """
         return self.driver.switch_to.active_element
 
-    def wait_until_present(self, selector, poll=0.5):
+    def wait_until_present(self, selector, timeout=Wait._TIMEOUT):
         """ Wait until element matching CSS selector is on the page """
         is_present = lambda driver: self.find(selector)
         msg = 'An element matching "%s" should be on the page' % selector
-        element = Wait(self.driver, poll=poll).until(is_present, msg)
-        if poll > 2:
-            time.sleep(poll)  # element need more delay to be present
+        element = Wait(self.driver, timeout=timeout).until(is_present, msg)
         return element
 
-    def wait_until_visible(self, selector, poll=1):
+    def wait_until_visible(self, selector, timeout=Wait._TIMEOUT):
         """ Wait until element matching CSS selector is visible on the page """
         is_visible = lambda driver: self.find(selector).is_displayed()
         msg = 'An element matching "%s" should be visible' % selector
-        Wait(self.driver, poll=poll).until(is_visible, msg)
-        time.sleep(poll)  # wait for visibility to settle
+        Wait(self.driver, timeout=timeout).until(is_visible, msg)
         return self.find(selector)
 
-    def wait_until_clickable(self, selector, poll=1):
-        """ Wait until element matching CSS selector is visible on the page """
-        WebDriverWait(
-            self.driver,
-            Wait._TIMEOUT,
-            poll_frequency=poll
-        ).until(
-            EC.element_to_be_clickable((By.ID, selector.removeprefix('#')
-                                        )
-                                       )
-        )
+    def wait_until_not_visible(self, selector, timeout=Wait._TIMEOUT):
+        """ Wait until element matching CSS selector is not visible on the page """
+        is_visible = lambda driver: self.find(selector).is_displayed()
+        msg = 'An element matching "%s" should be visible' % selector
+        Wait(self.driver, timeout=timeout).until_not(is_visible, msg)
         return self.find(selector)
 
+    def wait_until_clickable(self, selector, timeout=Wait._TIMEOUT):
+        """ Wait until element matching CSS selector is visible on the page """
+        WebDriverWait(self.driver, timeout=timeout).until(lambda driver: self.driver.execute_script("return jQuery.active == 0"))
+        is_clickable = lambda driver: (self.find(selector).is_displayed() and self.find(selector).is_enabled())
+        msg = 'An element matching "%s" should be clickable' % selector
+        Wait(self.driver, timeout=timeout).until(is_clickable, msg)
+        return self.find(selector)
+
+    def wait_until_element_clickable(self, finder, timeout=Wait._TIMEOUT):
+        """ Wait until element is clickable """
+        WebDriverWait(self.driver, timeout=timeout).until(lambda driver: self.driver.execute_script("return jQuery.active == 0"))
+        is_clickable = lambda driver: (finder(driver).is_displayed() and finder(driver).is_enabled())
+        msg = 'A matching element never became be clickable'
+        Wait(self.driver, timeout=timeout).until(is_clickable, msg)
+        return finder(self.driver)
+
     def wait_until_focused(self, selector):
         """ Wait until element matching CSS selector has focus """
         is_focused = \

bitbake/lib/toaster/tests/browser/test_all_builds_page.py

@@ -200,6 +200,7 @@ class TestAllBuildsPage(SeleniumTestCase):
 
         # should see a rebuild button for non-command-line builds
         self.wait_until_visible('#allbuildstable tbody tr')
+        self.wait_until_visible('.rebuild-btn')
         selector = 'div[data-latest-build-result="%s"] .rebuild-btn' % build1.id
         run_again_button = self.find_all(selector)
         self.assertEqual(len(run_again_button), 1,
@@ -224,7 +225,7 @@ class TestAllBuildsPage(SeleniumTestCase):
 
         url = reverse('all-builds')
         self.get(url)
-        self.wait_until_visible('#allbuildstable', poll=3)
+        self.wait_until_visible('#allbuildstable')
 
         # get the project name cells from the table
         cells = self.find_all('#allbuildstable td[class="project"]')
@@ -257,7 +258,7 @@ class TestAllBuildsPage(SeleniumTestCase):
 
         url = reverse('all-builds')
         self.get(url)
-        self.wait_until_visible('#allbuildstable', poll=3)
+        self.wait_until_visible('#allbuildstable')
 
         # test recent builds area for successful build
         element = self._get_build_time_element(build1)
@@ -452,7 +453,7 @@ class TestAllBuildsPage(SeleniumTestCase):
         def test_show_rows(row_to_show, show_row_link):
             # Check that we can show rows == row_to_show
             show_row_link.select_by_value(str(row_to_show))
-            self.wait_until_visible('#allbuildstable tbody tr', poll=3)
+            self.wait_until_visible('#allbuildstable tbody tr')
             # check at least some rows are visible
             self.assertTrue(
                 len(self.find_all('#allbuildstable tbody tr')) > 0

bitbake/lib/toaster/tests/browser/test_all_projects_page.py

@@ -81,7 +81,7 @@ class TestAllProjectsPage(SeleniumTestCase):
 
     def _get_row_for_project(self, project_name):
         """ Get the HTML row for a project, or None if not found """
-        self.wait_until_visible('#projectstable tbody tr', poll=3)
+        self.wait_until_visible('#projectstable tbody tr')
         rows = self.find_all('#projectstable tbody tr')
 
         # find the row with a project name matching the one supplied
@@ -236,7 +236,7 @@ class TestAllProjectsPage(SeleniumTestCase):
         self.get(url)
 
         # Chseck search box is present and works
-        self.wait_until_visible('#projectstable tbody tr', poll=3)
+        self.wait_until_visible('#projectstable tbody tr')
         search_box = self.find('#search-input-projectstable')
         self.assertTrue(search_box.is_displayed())
 
@@ -244,7 +244,7 @@ class TestAllProjectsPage(SeleniumTestCase):
         search_box.send_keys('test project 10')
         search_btn = self.find('#search-submit-projectstable')
         search_btn.click()
-        self.wait_until_visible('#projectstable tbody tr', poll=3)
+        self.wait_until_visible('#projectstable tbody tr')
         rows = self.find_all('#projectstable tbody tr')
         self.assertTrue(len(rows) == 1)
 
@@ -290,7 +290,7 @@ class TestAllProjectsPage(SeleniumTestCase):
                 )
         url = reverse('all-projects')
         self.get(url)
-        self.wait_until_visible('#projectstable tbody tr', poll=3)
+        self.wait_until_visible('#projectstable tbody tr')
 
         # Check edit column
         edit_column = self.find('#edit-columns-button')
@@ -313,7 +313,7 @@ class TestAllProjectsPage(SeleniumTestCase):
         def test_show_rows(row_to_show, show_row_link):
             # Check that we can show rows == row_to_show
             show_row_link.select_by_value(str(row_to_show))
-            self.wait_until_visible('#projectstable tbody tr', poll=3)
+            self.wait_until_visible('#projectstable tbody tr')
             # check at least some rows are visible
             self.assertTrue(
                 len(self.find_all('#projectstable tbody tr')) > 0
@@ -321,7 +321,7 @@ class TestAllProjectsPage(SeleniumTestCase):
 
         url = reverse('all-projects')
         self.get(url)
-        self.wait_until_visible('#projectstable tbody tr', poll=3)
+        self.wait_until_visible('#projectstable tbody tr')
 
         show_rows = self.driver.find_elements(
             By.XPATH,

bitbake/lib/toaster/tests/browser/test_builddashboard_page.py

@@ -162,7 +162,7 @@ class TestBuildDashboardPage(SeleniumTestCase):
         """
         url = reverse('builddashboard', args=(build.id,))
         self.get(url)
-        self.wait_until_visible('#global-nav', poll=3)
+        self.wait_until_visible('#global-nav')
 
     def _get_build_dashboard_errors(self, build):
         """

bitbake/lib/toaster/tests/browser/test_landing_page.py

@@ -34,6 +34,7 @@ class TestLandingPage(SeleniumTestCase):
     def test_icon_info_visible_and_clickable(self):
         """ Test that the information icon is visible and clickable """
         self.get(reverse('landing'))
+        self.wait_until_visible('#toaster-version-info-sign')
         info_sign = self.find('#toaster-version-info-sign')
 
         # check that the info sign is visible
@@ -43,6 +44,7 @@ class TestLandingPage(SeleniumTestCase):
         # and info modal is appearing when clicking on the info sign
         info_sign.click()  # click on the info sign make attribute 'aria-describedby' visible
         info_model_id = info_sign.get_attribute('aria-describedby')
+        self.wait_until_visible(f'#{info_model_id}')
         info_modal = self.find(f'#{info_model_id}')
         self.assertTrue(info_modal.is_displayed())
         self.assertTrue("Toaster version information" in info_modal.text)
@@ -50,6 +52,7 @@ class TestLandingPage(SeleniumTestCase):
     def test_documentation_link_displayed(self):
         """ Test that the documentation link is displayed """
         self.get(reverse('landing'))
+        self.wait_until_visible('#navbar-docs')
         documentation_link = self.find('#navbar-docs > a')
 
         # check that the documentation link is visible
@@ -65,6 +68,7 @@ class TestLandingPage(SeleniumTestCase):
     def test_openembedded_jumbotron_link_visible_and_clickable(self):
         """ Test OpenEmbedded link jumbotron is visible and clickable: """
         self.get(reverse('landing'))
+        self.wait_until_visible('.jumbotron')
         jumbotron = self.find('.jumbotron')
 
         # check OpenEmbedded
@@ -76,6 +80,7 @@ class TestLandingPage(SeleniumTestCase):
     def test_bitbake_jumbotron_link_visible_and_clickable(self):
         """ Test BitBake link jumbotron is visible and clickable: """
         self.get(reverse('landing'))
+        self.wait_until_visible('.jumbotron')
         jumbotron = self.find('.jumbotron')
 
         # check BitBake
@@ -88,6 +93,7 @@ class TestLandingPage(SeleniumTestCase):
     def test_yoctoproject_jumbotron_link_visible_and_clickable(self):
         """ Test Yocto Project link jumbotron is visible and clickable: """
         self.get(reverse('landing'))
+        self.wait_until_visible('.jumbotron')
         jumbotron = self.find('.jumbotron')
 
         # check Yocto Project
@@ -101,6 +107,7 @@ class TestLandingPage(SeleniumTestCase):
             if visible and clickable
         """
         self.get(reverse('landing'))
+        self.wait_until_visible('.jumbotron')
         jumbotron = self.find('.jumbotron')
 
         # check Big magenta button
@@ -119,6 +126,7 @@ class TestLandingPage(SeleniumTestCase):
         Layer_Version.objects.create(layer=layer)
 
         self.get(reverse('landing'))
+        self.wait_until_visible('.jumbotron')
         jumbotron = self.find('.jumbotron')
 
         # check Big Blue button
@@ -132,6 +140,7 @@ class TestLandingPage(SeleniumTestCase):
     def test_toaster_manual_link_visible_and_clickable(self):
         """ Test Read the Toaster manual link jumbotron is visible and clickable: """
         self.get(reverse('landing'))
+        self.wait_until_visible('.jumbotron')
         jumbotron = self.find('.jumbotron')
 
         # check Read the Toaster manual
@@ -145,6 +154,7 @@ class TestLandingPage(SeleniumTestCase):
     def test_contrib_to_toaster_link_visible_and_clickable(self):
         """ Test Contribute to Toaster link jumbotron is visible and clickable: """
         self.get(reverse('landing'))
+        self.wait_until_visible('.jumbotron')
         jumbotron = self.find('.jumbotron')
 
         # check Contribute to Toaster
@@ -161,6 +171,7 @@ class TestLandingPage(SeleniumTestCase):
         => should see the landing page
         """
         self.get(reverse('landing'))
+        self.wait_until_visible('.jumbotron')
         self.assertTrue(self.LANDING_PAGE_TITLE in self.get_page_source())
 
     def test_default_project_has_build(self):
@@ -193,6 +204,7 @@ class TestLandingPage(SeleniumTestCase):
         user_project.save()
 
         self.get(reverse('landing'))
+        self.wait_until_visible('#projectstable')
 
         elements = self.find_all('#projectstable')
         self.assertEqual(len(elements), 1, 'should redirect to projects')
@@ -213,7 +225,7 @@ class TestLandingPage(SeleniumTestCase):
 
         self.get(reverse('landing'))
 
-        self.wait_until_visible("#latest-builds", poll=3)
+        self.wait_until_visible("#latest-builds")
         elements = self.find_all('#allbuildstable')
         self.assertEqual(len(elements), 1, 'should redirect to builds')
         content = self.get_page_source()

bitbake/lib/toaster/tests/browser/test_layerdetails_page.py

@@ -64,7 +64,7 @@ class TestLayerDetailsPage(SeleniumTestCase):
                            args=(self.project.pk,
                                  self.imported_layer_version.pk))
 
-    def _edit_layerdetails(self):
+    def test_edit_layerdetails_page(self):
         """ Edit all the editable fields for the layer refresh the page and
         check that the new values exist"""
 
@@ -100,24 +100,17 @@ class TestLayerDetailsPage(SeleniumTestCase):
                             (self.initial_values, value))
 
             # Make sure the input visible beofre sending keys
-            self.wait_until_visible("#layer-git input[type=text]")
+            self.wait_until_clickable("#layer-git input[type=text]")
             inputs.send_keys("-edited")
 
         # Save the new values
         for save_btn in self.find_all(".change-btn"):
             save_btn.click()
 
-        try:
-            self.wait_until_visible("#save-changes-for-switch", poll=3)
-            btn_save_chg_for_switch = self.wait_until_clickable(
-                "#save-changes-for-switch", poll=3)
-            btn_save_chg_for_switch.click()
-        except ElementClickInterceptedException:
-            self.skipTest(
-                "save-changes-for-switch click intercepted. Element not visible or maybe covered by another element.")
-        except TimeoutException:
-            self.skipTest(
-                "save-changes-for-switch is not clickable within the specified timeout.")
+        self.wait_until_visible("#save-changes-for-switch")
+        btn_save_chg_for_switch = self.wait_until_clickable(
+            "#save-changes-for-switch")
+        btn_save_chg_for_switch.click()
 
         self.wait_until_visible("#edit-layer-source")
 
@@ -147,17 +140,10 @@ class TestLayerDetailsPage(SeleniumTestCase):
         new_dir = "/home/test/my-meta-dir"
         dir_input.send_keys(new_dir)
 
-        try:
-            self.wait_until_visible("#save-changes-for-switch", poll=3)
-            btn_save_chg_for_switch = self.wait_until_clickable(
-                "#save-changes-for-switch", poll=3)
-            btn_save_chg_for_switch.click()
-        except ElementClickInterceptedException:
-            self.skipTest(
-                "save-changes-for-switch click intercepted. Element not properly visible or maybe behind another element.")
-        except TimeoutException:
-            self.skipTest(
-                "save-changes-for-switch is not clickable within the specified timeout.")
+        self.wait_until_visible("#save-changes-for-switch")
+        btn_save_chg_for_switch = self.wait_until_clickable(
+            "#save-changes-for-switch")
+        btn_save_chg_for_switch.click()
 
         self.wait_until_visible("#edit-layer-source")
 
@@ -168,12 +154,6 @@ class TestLayerDetailsPage(SeleniumTestCase):
                         "Expected %s in the dir value for layer directory" %
                         new_dir)
 
-    def test_edit_layerdetails_page(self):
-        try:
-            self._edit_layerdetails()
-        except ElementClickInterceptedException:
-            self.skipTest(
-                "ElementClickInterceptedException occured. Element not visible or maybe covered by another element.")
 
     def test_delete_layer(self):
         """ Delete the layer """
@@ -211,6 +191,7 @@ class TestLayerDetailsPage(SeleniumTestCase):
         self.get(self.url)
 
         # Add the layer
+        self.wait_until_clickable("#add-remove-layer-btn")
         self.click("#add-remove-layer-btn")
 
         notification = self.wait_until_visible("#change-notification-msg")
@@ -218,12 +199,17 @@ class TestLayerDetailsPage(SeleniumTestCase):
         expected_text = "You have added 1 layer to your project: %s" % \
             self.imported_layer_version.layer.name
 
-        self.assertTrue(expected_text in notification.text,
+        self.assertIn(expected_text, notification.text,
                         "Expected notification text %s not found was "
                         " \"%s\" instead" %
                         (expected_text, notification.text))
 
+        hide_button = self.find('#hide-alert')
+        hide_button.click()
+        self.wait_until_not_visible('#change-notification')
+
         # Remove the layer
+        self.wait_until_clickable("#add-remove-layer-btn")
         self.click("#add-remove-layer-btn")
 
         notification = self.wait_until_visible("#change-notification-msg")
@@ -231,7 +217,7 @@ class TestLayerDetailsPage(SeleniumTestCase):
         expected_text = "You have removed 1 layer from your project: %s" % \
             self.imported_layer_version.layer.name
 
-        self.assertTrue(expected_text in notification.text,
+        self.assertIn(expected_text, notification.text,
                         "Expected notification text %s not found was "
                         " \"%s\" instead" %
                         (expected_text, notification.text))

bitbake/lib/toaster/tests/browser/test_new_custom_image_page.py

@@ -90,7 +90,7 @@ class TestNewCustomImagePage(SeleniumTestCase):
         """
         url = reverse('newcustomimage', args=(self.project.id,))
         self.get(url)
-        self.wait_until_visible('#global-nav', poll=3)
+        self.wait_until_visible('#global-nav')
 
         self.click('button[data-recipe="%s"]' % self.recipe.id)
 

bitbake/lib/toaster/tests/browser/test_new_project_page.py

@@ -47,7 +47,7 @@ class TestNewProjectPage(SeleniumTestCase):
 
         url = reverse('newproject')
         self.get(url)
-        self.wait_until_visible('#new-project-name', poll=3)
+        self.wait_until_visible('#new-project-name')
         self.enter_text('#new-project-name', project_name)
 
         select = Select(self.find('#projectversion'))
@@ -58,7 +58,7 @@ class TestNewProjectPage(SeleniumTestCase):
         # We should get redirected to the new project's page with the
         # notification at the top
         element = self.wait_until_visible(
-            '#project-created-notification', poll=3)
+            '#project-created-notification')
 
         self.assertTrue(project_name in element.text,
                         "New project name not in new project notification")
@@ -79,7 +79,7 @@ class TestNewProjectPage(SeleniumTestCase):
 
         url = reverse('newproject')
         self.get(url)
-        self.wait_until_visible('#new-project-name', poll=3)
+        self.wait_until_visible('#new-project-name')
 
         self.enter_text('#new-project-name', project_name)
 
@@ -89,12 +89,10 @@ class TestNewProjectPage(SeleniumTestCase):
         radio = self.driver.find_element(By.ID, 'type-new')
         radio.click()
 
-        self.click("#create-project-button")
-
-        self.wait_until_present('#hint-error-project-name', poll=3)
+        self.wait_until_visible('#hint-error-project-name')
         element = self.find('#hint-error-project-name')
 
-        self.assertTrue(("Project names must be unique" in element.text),
+        self.assertIn("Project names must be unique", element.text,
                         "Did not find unique project name error message")
 
         # Try and click it anyway, if it submits we'll have a new project in

bitbake/lib/toaster/tests/builds/buildtest.py

@@ -128,7 +128,7 @@ class BuildTest(unittest.TestCase):
         if os.environ.get("TOASTER_TEST_USE_SSTATE_MIRROR"):
             ProjectVariable.objects.get_or_create(
                 name="SSTATE_MIRRORS",
-                value="file://.* http://cdn.jsdelivr.net/yocto/sstate/all/PATH;downloadfilename=PATH",
+                value="file://.* http://sstate.yoctoproject.org/all/PATH;downloadfilename=PATH",
                 project=project)
 
         ProjectTarget.objects.create(project=project,

bitbake/lib/toaster/tests/functional/functional_helpers.py

@@ -12,9 +12,12 @@ import logging
 import subprocess
 import signal
 import re
+import requests
 
+from django.urls import reverse
 from tests.browser.selenium_helpers_base import SeleniumTestCaseBase
 from selenium.webdriver.common.by import By
+from selenium.webdriver.support.select import Select
 from selenium.common.exceptions import NoSuchElementException
 
 logger = logging.getLogger("toaster")
@@ -136,3 +139,86 @@ class SeleniumFunctionalTestCase(SeleniumTestCaseBase):
         except NoSuchElementException:
             return False
         return element
+
+    def create_new_project(
+        self,
+        project_name,
+        release,
+        release_title,
+        merge_toaster_settings,
+    ):
+        """ Create/Test new project using:
+          - Project Name: Any string
+          - Release: Any string
+          - Merge Toaster settings: True or False
+        """
+
+        # Obtain a CSRF token from a suitable URL
+        projs = requests.get(self.live_server_url + reverse('newproject'))
+        csrftoken = projs.cookies.get('csrftoken')
+
+        # Use the projects typeahead to find out if the project already exists
+        req = requests.get(self.live_server_url + reverse('xhr_projectstypeahead'), {'search': project_name, 'format' : 'json'})
+        data = req.json()
+        # Delete any existing projects
+        for result in data['results']:
+            del_url = reverse('xhr_project', args=(result['id'],))
+            del_response = requests.delete(self.live_server_url + del_url, cookies={'csrftoken': csrftoken}, headers={'X-CSRFToken': csrftoken})
+            self.assertEqual(del_response.status_code, 200)
+
+        self.get(reverse('newproject'))
+        self.wait_until_visible('#new-project-name')
+        self.driver.find_element(By.ID,
+                                 "new-project-name").send_keys(project_name)
+
+        select = Select(self.find('#projectversion'))
+        select.select_by_value(release)
+
+        # check merge toaster settings
+        checkbox = self.find('.checkbox-mergeattr')
+        if merge_toaster_settings:
+            if not checkbox.is_selected():
+                checkbox.click()
+        else:
+            if checkbox.is_selected():
+                checkbox.click()
+
+        self.wait_until_clickable('#create-project-button')
+
+        self.driver.find_element(By.ID, "create-project-button").click()
+
+        element = self.wait_until_visible('#project-created-notification')
+        self.assertTrue(
+            self.element_exists('#project-created-notification'),
+            f"Project:{project_name} creation notification not shown"
+        )
+        self.assertTrue(
+            project_name in element.text,
+            f"New project name:{project_name} not in new project notification"
+        )
+
+        # Use the projects typeahead again to check the project now exists
+        req = requests.get(self.live_server_url + reverse('xhr_projectstypeahead'), {'search': project_name, 'format' : 'json'})
+        data = req.json()
+        self.assertGreater(len(data['results']), 0, f"New project:{project_name} not found in database")
+
+        project_id = data['results'][0]['id']
+
+        self.wait_until_visible('#project-release-title')
+
+        # check release
+        if release_title is not None:
+            self.assertTrue(re.search(
+                release_title,
+                self.driver.find_element(By.XPATH,
+                                     "//span[@id='project-release-title']"
+                                     ).text),
+                            'The project release is not defined')
+
+        return project_id
+        
+    def load_projects_page_helper(self):        
+        self.wait_until_present('#projectstable')
+        # Need to wait for some data in the table too
+        self.wait_until_present('td[class="updated"]')
+

bitbake/lib/toaster/tests/functional/test_create_new_project.py

@@ -11,67 +11,10 @@ import pytest
 from django.urls import reverse
 from selenium.webdriver.support.select import Select
 from tests.functional.functional_helpers import SeleniumFunctionalTestCase
-from orm.models import Project
 from selenium.webdriver.common.by import By
 
-
-@pytest.mark.django_db
-@pytest.mark.order("last")
 class TestCreateNewProject(SeleniumFunctionalTestCase):
 
-    def _create_test_new_project(
-        self,
-        project_name,
-        release,
-        release_title,
-        merge_toaster_settings,
-    ):
-        """ Create/Test new project using:
-          - Project Name: Any string
-          - Release: Any string
-          - Merge Toaster settings: True or False
-        """
-        self.get(reverse('newproject'))
-        self.wait_until_visible('#new-project-name', poll=3)
-        self.driver.find_element(By.ID,
-                                 "new-project-name").send_keys(project_name)
-
-        select = Select(self.find('#projectversion'))
-        select.select_by_value(release)
-
-        # check merge toaster settings
-        checkbox = self.find('.checkbox-mergeattr')
-        if merge_toaster_settings:
-            if not checkbox.is_selected():
-                checkbox.click()
-        else:
-            if checkbox.is_selected():
-                checkbox.click()
-
-        self.driver.find_element(By.ID, "create-project-button").click()
-
-        element = self.wait_until_visible('#project-created-notification', poll=3)
-        self.assertTrue(
-            self.element_exists('#project-created-notification'),
-            f"Project:{project_name} creation notification not shown"
-        )
-        self.assertTrue(
-            project_name in element.text,
-            f"New project name:{project_name} not in new project notification"
-        )
-        self.assertTrue(
-            Project.objects.filter(name=project_name).count(),
-            f"New project:{project_name} not found in database"
-        )
-
-        # check release
-        self.assertTrue(re.search(
-            release_title,
-            self.driver.find_element(By.XPATH,
-                                     "//span[@id='project-release-title']"
-                                     ).text),
-                        'The project release is not defined')
-
     def test_create_new_project_master(self):
         """ Test create new project using:
           - Project Name: Any string
@@ -81,7 +24,7 @@ class TestCreateNewProject(SeleniumFunctionalTestCase):
         release = '3'
         release_title = 'Yocto Project master'
         project_name = 'projectmaster'
-        self._create_test_new_project(
+        self.create_new_project(
             project_name,
             release,
             release_title,
@@ -97,7 +40,7 @@ class TestCreateNewProject(SeleniumFunctionalTestCase):
         release = '1'
         release_title = 'Yocto Project 5.0 "Scarthgap"'
         project_name = 'projectscarthgap'
-        self._create_test_new_project(
+        self.create_new_project(
             project_name,
             release,
             release_title,
@@ -107,13 +50,13 @@ class TestCreateNewProject(SeleniumFunctionalTestCase):
     def test_create_new_project_kirkstone(self):
         """ Test create new project using:
           - Project Name: Any string
-          - Release: Yocto Project 4.0 "Kirkstone" (option value: 4)
+          - Release: Yocto Project 4.0 "Kirkstone" (option value: 6)
           - Merge Toaster settings: True
         """
-        release = '4'
+        release = '6'
         release_title = 'Yocto Project 4.0 "Kirkstone"'
         project_name = 'projectkirkstone'
-        self._create_test_new_project(
+        self.create_new_project(
             project_name,
             release,
             release_title,
@@ -129,7 +72,7 @@ class TestCreateNewProject(SeleniumFunctionalTestCase):
         release = '2'
         release_title = 'Local Yocto Project'
         project_name = 'projectlocal'
-        self._create_test_new_project(
+        self.create_new_project(
             project_name,
             release,
             release_title,
@@ -172,8 +115,10 @@ class TestCreateNewProject(SeleniumFunctionalTestCase):
                                  "import-project-dir").send_keys(wrong_path)
         self.driver.find_element(By.ID, "create-project-button").click()
 
+        self.wait_until_visible('.alert-danger')
+
         # check error message
         self.assertTrue(self.element_exists('.alert-danger'),
-                        'Allert message not shown')
+                        'Alert message not shown')
         self.assertTrue(wrong_path in self.find('.alert-danger').text,
                         "Wrong path not in alert message")

bitbake/lib/toaster/tests/functional/test_functional_basic.py

@@ -17,145 +17,132 @@ from selenium.webdriver.common.by import By
 from tests.functional.utils import get_projectId_from_url
 
 
-@pytest.mark.django_db
-@pytest.mark.order("second_to_last")
 class FuntionalTestBasic(SeleniumFunctionalTestCase):
     """Basic functional tests for Toaster"""
     project_id = None
+    project_url = None
 
     def setUp(self):
         super(FuntionalTestBasic, self).setUp()
         if not FuntionalTestBasic.project_id:
-            self._create_slenium_project()
-            current_url = self.driver.current_url
-            FuntionalTestBasic.project_id = get_projectId_from_url(current_url)
-
-#   testcase (1514)
-    def _create_slenium_project(self):
-        project_name = 'selenium-project'
-        self.get(reverse('newproject'))
-        self.wait_until_visible('#new-project-name', poll=3)
-        self.driver.find_element(By.ID, "new-project-name").send_keys(project_name)
-        self.driver.find_element(By.ID, 'projectversion').click()
-        self.driver.find_element(By.ID, "create-project-button").click()
-        element = self.wait_until_visible('#project-created-notification', poll=10)
-        self.assertTrue(self.element_exists('#project-created-notification'),'Project creation notification not shown')
-        self.assertTrue(project_name in element.text,
-                        "New project name not in new project notification")
-        self.assertTrue(Project.objects.filter(name=project_name).count(),
-                        "New project not found in database")
-        return Project.objects.last().id
+            FuntionalTestBasic.project_id = self.create_new_project('selenium-project', '3', None, False)
 
  #  testcase (1515)
     def test_verify_left_bar_menu(self):
         self.get(reverse('all-projects'))
-        self.wait_until_present('#projectstable', poll=10)
+        self.load_projects_page_helper()
         self.find_element_by_link_text_in_table('projectstable', 'selenium-project').click()
-        self.wait_until_present('#config-nav', poll=10)
+        self.wait_until_present('#config-nav')
         self.assertTrue(self.element_exists('#config-nav'),'Configuration Tab does not exist')
         project_URL=self.get_URL()
         self.driver.find_element(By.XPATH, '//a[@href="'+project_URL+'"]').click()
-        self.wait_until_present('#config-nav', poll=10)
 
         try:
+            self.wait_until_present('#config-nav')
             self.driver.find_element(By.XPATH, "//*[@id='config-nav']/ul/li/a[@href="+'"'+project_URL+'customimages/"'+"]").click()
-            self.wait_until_present('#config-nav', poll=10)
-            self.assertTrue(re.search("Custom images",self.driver.find_element(By.XPATH, "//div[@class='col-md-10']").text),'Custom images information is not loading properly')
+            self.wait_until_present('#filter-modal-customimagestable')
         except:
             self.fail(msg='No Custom images tab available')
+        self.assertTrue(re.search("Custom images",self.driver.find_element(By.XPATH, "//div[@class='col-md-10']").text),'Custom images information is not loading properly')
 
         try:
             self.driver.find_element(By.XPATH, "//*[@id='config-nav']/ul/li/a[@href="+'"'+project_URL+'images/"'+"]").click()
-            self.wait_until_present('#config-nav', poll=10)
-            self.assertTrue(re.search("Compatible image recipes",self.driver.find_element(By.XPATH, "//div[@class='col-md-10']").text),'The Compatible image recipes information is not loading properly')
+            self.wait_until_present('#filter-modal-imagerecipestable')
         except:
             self.fail(msg='No Compatible image tab available')
+        self.assertTrue(re.search("Compatible image recipes",self.driver.find_element(By.XPATH, "//div[@class='col-md-10']").text),'The Compatible image recipes information is not loading properly')
 
         try:
             self.driver.find_element(By.XPATH, "//*[@id='config-nav']/ul/li/a[@href="+'"'+project_URL+'softwarerecipes/"'+"]").click()
-            self.wait_until_present('#config-nav', poll=10)
-            self.assertTrue(re.search("Compatible software recipes",self.driver.find_element(By.XPATH, "//div[@class='col-md-10']").text),'The Compatible software recipe information is not loading properly')
+            self.wait_until_present('#filter-modal-softwarerecipestable')
         except:
             self.fail(msg='No Compatible software recipe tab available')
+        self.assertTrue(re.search("Compatible software recipes",self.driver.find_element(By.XPATH, "//div[@class='col-md-10']").text),'The Compatible software recipe information is not loading properly')
 
         try:
             self.driver.find_element(By.XPATH, "//*[@id='config-nav']/ul/li/a[@href="+'"'+project_URL+'machines/"'+"]").click()
-            self.wait_until_present('#config-nav', poll=10)
-            self.assertTrue(re.search("Compatible machines",self.driver.find_element(By.XPATH, "//div[@class='col-md-10']").text),'The Compatible machine information is not loading properly')
+            self.wait_until_present('#filter-modal-machinestable')
         except:
             self.fail(msg='No Compatible machines tab available')
+        self.assertTrue(re.search("Compatible machines",self.driver.find_element(By.XPATH, "//div[@class='col-md-10']").text),'The Compatible machine information is not loading properly')
 
         try:
             self.driver.find_element(By.XPATH, "//*[@id='config-nav']/ul/li/a[@href="+'"'+project_URL+'layers/"'+"]").click()
-            self.wait_until_present('#config-nav', poll=10)
-            self.assertTrue(re.search("Compatible layers",self.driver.find_element(By.XPATH, "//div[@class='col-md-10']").text),'The Compatible layer information is not loading properly')
+            self.wait_until_present('#filter-modal-layerstable')
         except:
             self.fail(msg='No Compatible layers tab available')
+        self.assertTrue(re.search("Compatible layers",self.driver.find_element(By.XPATH, "//div[@class='col-md-10']").text),'The Compatible layer information is not loading properly')
 
         try:
             self.driver.find_element(By.XPATH, "//*[@id='config-nav']/ul/li/a[@href="+'"'+project_URL+'configuration"'+"]").click()
-            self.wait_until_present('#config-nav', poll=10)
-            self.assertTrue(re.search("Bitbake variables",self.driver.find_element(By.XPATH, "//div[@class='col-md-10']").text),'The Bitbake variables information is not loading properly')
+            self.wait_until_present('#configvar-list')
         except:
             self.fail(msg='No Bitbake variables tab available')
+        self.assertTrue(re.search("Bitbake variables",self.driver.find_element(By.XPATH, "//div[@class='col-md-10']").text),'The Bitbake variables information is not loading properly')
 
 #   testcase (1516)
     def test_review_configuration_information(self):
         self.get(reverse('all-projects'))
-        self.wait_until_present('#projectstable', poll=10)
+        self.load_projects_page_helper()
         self.find_element_by_link_text_in_table('projectstable', 'selenium-project').click()
         project_URL=self.get_URL()
-        self.wait_until_present('#config-nav', poll=10)
+
+        # Machine section of page
+        self.wait_until_visible('#machine-section')
+        self.assertTrue(self.element_exists('#machine-section'),'Machine section for the project configuration page does not exist')
+        self.assertTrue(re.search("qemux86-64",self.driver.find_element(By.XPATH, "//span[@id='project-machine-name']").text),'The machine type is not assigned')
         try:
-           self.assertTrue(self.element_exists('#machine-section'),'Machine section for the project configuration page does not exist')
-           self.assertTrue(re.search("qemux86-64",self.driver.find_element(By.XPATH, "//span[@id='project-machine-name']").text),'The machine type is not assigned')
            self.driver.find_element(By.XPATH, "//span[@id='change-machine-toggle']").click()
-           self.wait_until_visible('#select-machine-form', poll=10)
-           self.wait_until_visible('#cancel-machine-change', poll=10)
+           self.wait_until_visible('#select-machine-form')
+           self.wait_until_visible('#cancel-machine-change')
            self.driver.find_element(By.XPATH, "//form[@id='select-machine-form']/a[@id='cancel-machine-change']").click()
         except:
            self.fail(msg='The machine information is wrong in the configuration page')
 
+        # Most built recipes section
+        self.wait_until_visible('#no-most-built')
         try:
            self.driver.find_element(By.ID, 'no-most-built')
         except:
            self.fail(msg='No Most built information in project detail page')
 
-        try:
-           self.assertTrue(re.search("Yocto Project master",self.driver.find_element(By.XPATH, "//span[@id='project-release-title']").text),'The project release is not defined')
-        except:
-           self.fail(msg='No project release title information in project detail page')
+        # Project Release title
+        self.assertTrue(re.search("Yocto Project master",self.driver.find_element(By.XPATH, "//span[@id='project-release-title']").text), 'The project release is not defined in the project detail page')
 
+        # List of layers in project
+        self.wait_until_visible('#layer-container')
+        self.driver.find_element(By.XPATH, "//div[@id='layer-container']")
+        self.assertTrue(re.search("3",self.driver.find_element(By.ID, "project-layers-count").text),'There should be 3 layers listed in the layer count')
         try:
-           self.driver.find_element(By.XPATH, "//div[@id='layer-container']")
-           self.assertTrue(re.search("3",self.driver.find_element(By.ID, "project-layers-count").text),'There should be 3 layers listed in the layer count')
            layer_list = self.driver.find_element(By.ID, "layers-in-project-list")
            layers = layer_list.find_elements(By.TAG_NAME, "li")
-           for layer in layers:
-               if re.match ("openembedded-core",layer.text):
-                    print ("openembedded-core layer is a default layer in the project configuration")
-               elif re.match ("meta-poky",layer.text):
-                  print ("meta-poky layer is a default layer in the project configuration")
-               elif re.match ("meta-yocto-bsp",layer.text):
-                  print ("meta-yocto-bsp is a default layer in the project configuratoin")
-               else:
-                  self.fail(msg='default layers are missing from the project configuration')
         except:
            self.fail(msg='No Layer information in project detail page')
 
+        for layer in layers:
+            if re.match ("openembedded-core", layer.text):
+                print ("openembedded-core layer is a default layer in the project configuration")
+            elif re.match ("meta-poky", layer.text):
+                 print ("meta-poky layer is a default layer in the project configuration")
+            elif re.match ("meta-yocto-bsp", layer.text):
+                 print ("meta-yocto-bsp is a default layer in the project configuratoin")
+            else:
+                 self.fail(msg='default layers are missing from the project configuration')
+
 #   testcase (1517)
     def test_verify_machine_information(self):
         self.get(reverse('all-projects'))
-        self.wait_until_present('#projectstable', poll=10)
+        self.load_projects_page_helper()
         self.find_element_by_link_text_in_table('projectstable', 'selenium-project').click()
-        self.wait_until_present('#config-nav', poll=10)
 
+        self.wait_until_visible('#machine-section')
+        self.assertTrue(self.element_exists('#machine-section'),'Machine section for the project configuration page does not exist')
+        self.wait_until_visible('#project-machine-name')
+        self.assertTrue(re.search("qemux86-64",self.driver.find_element(By.ID, "project-machine-name").text),'The machine type is not assigned')
         try:
-            self.assertTrue(self.element_exists('#machine-section'),'Machine section for the project configuration page does not exist')
-            self.assertTrue(re.search("qemux86-64",self.driver.find_element(By.ID, "project-machine-name").text),'The machine type is not assigned')
             self.driver.find_element(By.ID, "change-machine-toggle").click()
-            self.wait_until_visible('#select-machine-form', poll=10)
-            self.wait_until_visible('#cancel-machine-change', poll=10)
+            self.wait_until_visible('#select-machine-form')
+            self.wait_until_visible('#cancel-machine-change')
             self.driver.find_element(By.ID, "cancel-machine-change").click()
         except:
             self.fail(msg='The machine information is wrong in the configuration page')
@@ -163,83 +150,95 @@ class FuntionalTestBasic(SeleniumFunctionalTestCase):
 #   testcase (1518)
     def test_verify_most_built_recipes_information(self):
         self.get(reverse('all-projects'))
-        self.wait_until_present('#projectstable', poll=10)
+        self.load_projects_page_helper()
         self.find_element_by_link_text_in_table('projectstable', 'selenium-project').click()
-        self.wait_until_present('#config-nav', poll=10)
+        self.wait_until_present('#config-nav')
         project_URL=self.get_URL()
+
+        self.wait_until_visible('#no-most-built')
+        self.assertTrue(re.search("You haven't built any recipes yet",self.driver.find_element(By.ID, "no-most-built").text),'Default message of no builds is not present')
         try:
-            self.assertTrue(re.search("You haven't built any recipes yet",self.driver.find_element(By.ID, "no-most-built").text),'Default message of no builds is not present')
             self.driver.find_element(By.XPATH, "//div[@id='no-most-built']/p/a[@href="+'"'+project_URL+'images/"'+"]").click()
-            self.wait_until_present('#config-nav', poll=10)
-            self.assertTrue(re.search("Compatible image recipes",self.driver.find_element(By.XPATH, "//div[@class='col-md-10']").text),'The Choose a recipe to build link  is not working  properly')
         except:
             self.fail(msg='No Most built information in project detail page')
+        self.wait_until_visible('#config-nav')
+        self.assertTrue(re.search("Compatible image recipes",self.driver.find_element(By.XPATH, "//div[@class='col-md-10']").text),'The Choose a recipe to build link  is not working  properly')
 
 #   testcase (1519)
     def test_verify_project_release_information(self):
         self.get(reverse('all-projects'))
-        self.wait_until_present('#projectstable', poll=10)
+        self.load_projects_page_helper()
         self.find_element_by_link_text_in_table('projectstable', 'selenium-project').click()
-        self.wait_until_present('#config-nav', poll=10)
-
-        try:
-            self.assertTrue(re.search("Yocto Project master",self.driver.find_element(By.ID, "project-release-title").text),'The project release is not defined')
-        except:
-            self.fail(msg='No project release title information in project detail page')
+        self.wait_until_visible('#project-release-title')
+        self.assertTrue(re.search("Yocto Project master",self.driver.find_element(By.ID, "project-release-title").text), 'No project release title information in project detail page')
 
 #   testcase (1520)
     def test_verify_layer_information(self):
         self.get(reverse('all-projects'))
-        self.wait_until_present('#projectstable', poll=10)
+        self.load_projects_page_helper()
         self.find_element_by_link_text_in_table('projectstable', 'selenium-project').click()
-        self.wait_until_present('#config-nav', poll=10)
+        self.wait_until_present('#config-nav')
         project_URL=self.get_URL()
+        self.wait_until_visible('#layer-container')
+        self.driver.find_element(By.XPATH, "//div[@id='layer-container']")
+        self.wait_until_visible('#project-layers-count')
+        self.assertTrue(re.search("3",self.driver.find_element(By.ID, "project-layers-count").text),'There should be 3 layers listed in the layer count')
+
         try:
-           self.driver.find_element(By.XPATH, "//div[@id='layer-container']")
-           self.assertTrue(re.search("3",self.driver.find_element(By.ID, "project-layers-count").text),'There should be 3 layers listed in the layer count')
            layer_list = self.driver.find_element(By.ID, "layers-in-project-list")
            layers = layer_list.find_elements(By.TAG_NAME, "li")
+        except:
+            self.fail(msg='No Layer information in project detail page')
 
-           for layer in layers:
-               if re.match ("openembedded-core",layer.text):
-                    print ("openembedded-core layer is a default layer in the project configuration")
-               elif re.match ("meta-poky",layer.text):
-                  print ("meta-poky layer is a default layer in the project configuration")
-               elif re.match ("meta-yocto-bsp",layer.text):
-                  print ("meta-yocto-bsp is a default layer in the project configuratoin")
-               else:
-                  self.fail(msg='default layers are missing from the project configuration')
+        for layer in layers:
+            if re.match ("openembedded-core",layer.text):
+                print ("openembedded-core layer is a default layer in the project configuration")
+            elif re.match ("meta-poky",layer.text):
+                print ("meta-poky layer is a default layer in the project configuration")
+            elif re.match ("meta-yocto-bsp",layer.text):
+                print ("meta-yocto-bsp is a default layer in the project configuratoin")
+            else:
+                self.fail(msg='default layers are missing from the project configuration')
 
+        try:
            self.driver.find_element(By.XPATH, "//input[@id='layer-add-input']")
            self.driver.find_element(By.XPATH, "//button[@id='add-layer-btn']")
            self.driver.find_element(By.XPATH, "//div[@id='layer-container']/form[@class='form-inline']/p/a[@id='view-compatible-layers']")
            self.driver.find_element(By.XPATH, "//div[@id='layer-container']/form[@class='form-inline']/p/a[@href="+'"'+project_URL+'importlayer"'+"]")
         except:
-            self.fail(msg='No Layer information in project detail page')
+            self.fail(msg='Layer configuration controls missing')
 
 #   testcase (1521)
     def test_verify_project_detail_links(self):
         self.get(reverse('all-projects'))
-        self.wait_until_present('#projectstable', poll=10)
+        self.load_projects_page_helper()
         self.find_element_by_link_text_in_table('projectstable', 'selenium-project').click()
-        self.wait_until_present('#config-nav', poll=10)
+        self.wait_until_present('#config-nav')
         project_URL=self.get_URL()
         self.driver.find_element(By.XPATH, "//div[@id='project-topbar']/ul[@class='nav nav-tabs']/li[@id='topbar-configuration-tab']/a[@href="+'"'+project_URL+'"'+"]").click()
-        self.wait_until_present('#config-nav', poll=10)
+        self.wait_until_visible('#topbar-configuration-tab')
         self.assertTrue(re.search("Configuration",self.driver.find_element(By.XPATH, "//div[@id='project-topbar']/ul[@class='nav nav-tabs']/li[@id='topbar-configuration-tab']/a[@href="+'"'+project_URL+'"'+"]").text), 'Configuration tab in project topbar is misspelled')
 
         try:
             self.driver.find_element(By.XPATH, "//div[@id='project-topbar']/ul[@class='nav nav-tabs']/li/a[@href="+'"'+project_URL+'builds/"'+"]").click()
-            self.wait_until_visible('#project-topbar', poll=10)
-            self.assertTrue(re.search("Builds",self.driver.find_element(By.XPATH, "//div[@id='project-topbar']/ul[@class='nav nav-tabs']/li/a[@href="+'"'+project_URL+'builds/"'+"]").text), 'Builds tab in project topbar is misspelled')
+        except:
+            self.fail(msg='Builds tab information is not present')
+
+        self.wait_until_visible('#project-topbar')
+        self.assertTrue(re.search("Builds",self.driver.find_element(By.XPATH, "//div[@id='project-topbar']/ul[@class='nav nav-tabs']/li/a[@href="+'"'+project_URL+'builds/"'+"]").text), 'Builds tab in project topbar is misspelled')
+        try:
             self.driver.find_element(By.XPATH, "//div[@id='empty-state-projectbuildstable']")
         except:
             self.fail(msg='Builds tab information is not present')
 
         try:
             self.driver.find_element(By.XPATH, "//div[@id='project-topbar']/ul[@class='nav nav-tabs']/li/a[@href="+'"'+project_URL+'importlayer"'+"]").click()
-            self.wait_until_visible('#project-topbar', poll=10)
-            self.assertTrue(re.search("Import layer",self.driver.find_element(By.XPATH, "//div[@id='project-topbar']/ul[@class='nav nav-tabs']/li/a[@href="+'"'+project_URL+'importlayer"'+"]").text), 'Import layer tab in project topbar is misspelled')
+        except:
+            self.fail(msg='Import layer tab not loading properly')
+
+        self.wait_until_visible('#project-topbar')
+        self.assertTrue(re.search("Import layer",self.driver.find_element(By.XPATH, "//div[@id='project-topbar']/ul[@class='nav nav-tabs']/li/a[@href="+'"'+project_URL+'importlayer"'+"]").text), 'Import layer tab in project topbar is misspelled')
+        try:
             self.driver.find_element(By.XPATH, "//fieldset[@id='repo-select']")
             self.driver.find_element(By.XPATH, "//fieldset[@id='git-repo']")
         except:
@@ -247,11 +246,12 @@ class FuntionalTestBasic(SeleniumFunctionalTestCase):
 
         try:
             self.driver.find_element(By.XPATH, "//div[@id='project-topbar']/ul[@class='nav nav-tabs']/li/a[@href="+'"'+project_URL+'newcustomimage/"'+"]").click()
-            self.wait_until_visible('#project-topbar', poll=10)
-            self.assertTrue(re.search("New custom image",self.driver.find_element(By.XPATH, "//div[@id='project-topbar']/ul[@class='nav nav-tabs']/li/a[@href="+'"'+project_URL+'newcustomimage/"'+"]").text), 'New custom image tab in project topbar is misspelled')
-            self.assertTrue(re.search("Select the image recipe you want to customise",self.driver.find_element(By.XPATH, "//div[@class='col-md-12']/h2").text),'The new custom image tab is not loading correctly')
         except:
             self.fail(msg='New custom image tab not loading properly')
 
+        self.wait_until_visible('#project-topbar')
+        self.assertTrue(re.search("New custom image",self.driver.find_element(By.XPATH, "//div[@id='project-topbar']/ul[@class='nav nav-tabs']/li/a[@href="+'"'+project_URL+'newcustomimage/"'+"]").text), 'New custom image tab in project topbar is misspelled')
+        self.assertTrue(re.search("Select the image recipe you want to customise",self.driver.find_element(By.XPATH, "//div[@class='col-md-12']/h2").text),'The new custom image tab is not loading correctly')
+
 
 

bitbake/lib/toaster/tests/functional/test_project_config.py

@@ -7,7 +7,6 @@
 #
 
 import string
-import random
 import pytest
 from django.urls import reverse
 from selenium.webdriver import Keys
@@ -18,9 +17,6 @@ from selenium.webdriver.common.by import By
 
 from .utils import get_projectId_from_url
 
-
-@pytest.mark.django_db
-@pytest.mark.order("last")
 class TestProjectConfig(SeleniumFunctionalTestCase):
     project_id = None
     PROJECT_NAME = 'TestProjectConfig'
@@ -28,42 +24,6 @@ class TestProjectConfig(SeleniumFunctionalTestCase):
     INVALID_PATH_CHAR_TEXT = 'The directory path cannot include spaces or ' \
         'any of these characters'
 
-    def _create_project(self, project_name):
-        """ Create/Test new project using:
-          - Project Name: Any string
-          - Release: Any string
-          - Merge Toaster settings: True or False
-        """
-        self.get(reverse('newproject'))
-        self.wait_until_visible('#new-project-name', poll=2)
-        self.find("#new-project-name").send_keys(project_name)
-        select = Select(self.find("#projectversion"))
-        select.select_by_value('3')
-
-        # check merge toaster settings
-        checkbox = self.find('.checkbox-mergeattr')
-        if not checkbox.is_selected():
-            checkbox.click()
-
-        if self.PROJECT_NAME != 'TestProjectConfig':
-            # Reset project name if it's not the default one
-            self.PROJECT_NAME = 'TestProjectConfig'
-
-        self.find("#create-project-button").click()
-
-        try:
-            self.wait_until_visible('#hint-error-project-name', poll=2)
-            url = reverse('project', args=(TestProjectConfig.project_id, ))
-            self.get(url)
-            self.wait_until_visible('#config-nav', poll=3)
-        except TimeoutException:
-            self.wait_until_visible('#config-nav', poll=3)
-
-    def _random_string(self, length):
-        return ''.join(
-            random.choice(string.ascii_letters) for _ in range(length)
-        )
-
     def _get_config_nav_item(self, index):
         config_nav = self.find('#config-nav')
         return config_nav.find_elements(By.TAG_NAME, 'li')[index]
@@ -72,16 +32,14 @@ class TestProjectConfig(SeleniumFunctionalTestCase):
         """ Navigate to project BitBake variables page """
         # check if the menu is displayed
         if TestProjectConfig.project_id is None:
-            self._create_project(project_name=self._random_string(10))
-            current_url = self.driver.current_url
-            TestProjectConfig.project_id = get_projectId_from_url(current_url)
-        else:
-            url = reverse('projectconf', args=(TestProjectConfig.project_id,))
-            self.get(url)
-        self.wait_until_visible('#config-nav', poll=3)
+            TestProjectConfig.project_id = self.create_new_project(self.PROJECT_NAME, '3', None, True)
+
+        url = reverse('projectconf', args=(TestProjectConfig.project_id,))
+        self.get(url)
+        self.wait_until_visible('#config-nav')
         bbv_page_link = self._get_config_nav_item(9)
         bbv_page_link.click()
-        self.wait_until_visible('#config-nav', poll=3)
+        self.wait_until_visible('#config-nav')
 
     def test_no_underscore_iamgefs_type(self):
         """
@@ -90,13 +48,13 @@ class TestProjectConfig(SeleniumFunctionalTestCase):
         self._navigate_bbv_page()
         imagefs_type = "foo_bar"
 
-        self.wait_until_visible('#change-image_fstypes-icon', poll=2)
+        self.wait_until_visible('#change-image_fstypes-icon')
 
         self.click('#change-image_fstypes-icon')
 
         self.enter_text('#new-imagefs_types', imagefs_type)
 
-        element = self.wait_until_visible('#hintError-image-fs_type', poll=2)
+        element = self.wait_until_visible('#hintError-image-fs_type')
 
         self.assertTrue(("A valid image type cannot include underscores" in element.text),
                         "Did not find underscore error message")
@@ -110,7 +68,7 @@ class TestProjectConfig(SeleniumFunctionalTestCase):
 
         imagefs_type = "btrfs"
 
-        self.wait_until_visible('#change-image_fstypes-icon', poll=2)
+        self.wait_until_visible('#change-image_fstypes-icon')
 
         self.click('#change-image_fstypes-icon')
 
@@ -129,22 +87,20 @@ class TestProjectConfig(SeleniumFunctionalTestCase):
         """
         self._navigate_bbv_page()
 
-        self.wait_until_visible('#change-image_fstypes-icon', poll=2)
-
+        self.wait_until_visible('#change-image_fstypes-icon')
         self.click('#change-image_fstypes-icon')
 
         checkboxes_selector = '.fs-checkbox-fstypes'
 
-        self.wait_until_visible(checkboxes_selector, poll=2)
+        self.wait_until_visible(checkboxes_selector)
         checkboxes = self.find_all(checkboxes_selector)
 
         for checkbox in checkboxes:
             if checkbox.get_attribute("value") == "cpio":
                checkbox.click()
+               self.wait_until_visible('#new-imagefs_types')
                element = self.driver.find_element(By.ID, 'new-imagefs_types')
 
-               self.wait_until_visible('#new-imagefs_types', poll=2)
-
                self.assertTrue(("cpio" in element.get_attribute('value'),
                                "Imagefs not added into the textbox"))
                checkbox.click()
@@ -160,20 +116,19 @@ class TestProjectConfig(SeleniumFunctionalTestCase):
 
         # activate the input to edit download dir
         try:
-            change_dl_dir_btn = self.wait_until_visible('#change-dl_dir-icon', poll=2)
+            change_dl_dir_btn = self.wait_until_visible('#change-dl_dir-icon')
         except TimeoutException:
             # If download dir is not displayed, test is skipped
             change_dl_dir_btn = None
 
         if change_dl_dir_btn:
-            change_dl_dir_btn = self.wait_until_visible('#change-dl_dir-icon', poll=2)
             change_dl_dir_btn.click()
 
             # downloads dir path doesn't start with / or ${...}
-            input_field = self.wait_until_visible('#new-dl_dir', poll=2)
+            input_field = self.wait_until_visible('#new-dl_dir')
             input_field.clear()
             self.enter_text('#new-dl_dir', 'home/foo')
-            element = self.wait_until_visible('#hintError-initialChar-dl_dir', poll=2)
+            element = self.wait_until_visible('#hintError-initialChar-dl_dir')
 
             msg = 'downloads directory path starts with invalid character but ' \
                 'treated as valid'
@@ -183,7 +138,7 @@ class TestProjectConfig(SeleniumFunctionalTestCase):
             self.driver.find_element(By.ID, 'new-dl_dir').clear()
             self.enter_text('#new-dl_dir', '/foo/bar a')
 
-            element = self.wait_until_visible('#hintError-dl_dir', poll=2)
+            element = self.wait_until_visible('#hintError-dl_dir')
             msg = 'downloads directory path characters invalid but treated as valid'
             self.assertTrue((self.INVALID_PATH_CHAR_TEXT in element.text), msg)
 
@@ -191,7 +146,7 @@ class TestProjectConfig(SeleniumFunctionalTestCase):
             self.driver.find_element(By.ID,'new-dl_dir').clear()
             self.enter_text('#new-dl_dir', '${TOPDIR}/down foo')
 
-            element = self.wait_until_visible('#hintError-dl_dir', poll=2)
+            element = self.wait_until_visible('#hintError-dl_dir')
             msg = 'downloads directory path characters invalid but treated as valid'
             self.assertTrue((self.INVALID_PATH_CHAR_TEXT in element.text), msg)
 
@@ -219,10 +174,7 @@ class TestProjectConfig(SeleniumFunctionalTestCase):
         self._navigate_bbv_page()
 
         try:
-            btn_chg_sstate_dir = self.wait_until_visible(
-                '#change-sstate_dir-icon',
-                poll=2
-            )
+            btn_chg_sstate_dir = self.wait_until_visible('#change-sstate_dir-icon')
             self.click('#change-sstate_dir-icon')
         except TimeoutException:
             # If sstate_dir is not displayed, test is skipped
@@ -230,10 +182,10 @@ class TestProjectConfig(SeleniumFunctionalTestCase):
 
         if btn_chg_sstate_dir:  # Skip continuation if sstate_dir is not displayed
             # path doesn't start with / or ${...}
-            input_field = self.wait_until_visible('#new-sstate_dir', poll=2)
+            input_field = self.wait_until_visible('#new-sstate_dir')
             input_field.clear()
             self.enter_text('#new-sstate_dir', 'home/foo')
-            element = self.wait_until_visible('#hintError-initialChar-sstate_dir', poll=2)
+            element = self.wait_until_visible('#hintError-initialChar-sstate_dir')
 
             msg = 'sstate directory path starts with invalid character but ' \
                 'treated as valid'
@@ -243,7 +195,7 @@ class TestProjectConfig(SeleniumFunctionalTestCase):
             self.driver.find_element(By.ID, 'new-sstate_dir').clear()
             self.enter_text('#new-sstate_dir', '/foo/bar a')
 
-            element = self.wait_until_visible('#hintError-sstate_dir', poll=2)
+            element = self.wait_until_visible('#hintError-sstate_dir')
             msg = 'sstate directory path characters invalid but treated as valid'
             self.assertTrue((self.INVALID_PATH_CHAR_TEXT in element.text), msg)
 
@@ -251,7 +203,7 @@ class TestProjectConfig(SeleniumFunctionalTestCase):
             self.driver.find_element(By.ID,'new-sstate_dir').clear()
             self.enter_text('#new-sstate_dir', '${TOPDIR}/down foo')
 
-            element = self.wait_until_visible('#hintError-sstate_dir', poll=2)
+            element = self.wait_until_visible('#hintError-sstate_dir')
             msg = 'sstate directory path characters invalid but treated as valid'
             self.assertTrue((self.INVALID_PATH_CHAR_TEXT in element.text), msg)
 
@@ -275,13 +227,14 @@ class TestProjectConfig(SeleniumFunctionalTestCase):
         var_name, field, btn_id, input_id, value, save_btn, *_ = kwargs.values()
         """ Change bitbake variable value """
         self._navigate_bbv_page()
-        self.wait_until_visible(f'#{btn_id}', poll=2)
+        self.wait_until_visible(f'#{btn_id}')
         if kwargs.get('new_variable'):
             self.find(f"#{btn_id}").clear()
             self.enter_text(f"#{btn_id}", f"{var_name}")
         else:
             self.click(f'#{btn_id}')
-            self.wait_until_visible(f'#{input_id}', poll=2)
+
+        self.wait_until_visible(f'#{input_id}')
 
         if kwargs.get('is_select'):
             select = Select(self.find(f'#{input_id}'))

bitbake/lib/toaster/tests/functional/test_project_page.py

@@ -7,8 +7,8 @@
 #
 
 import os
-import random
 import string
+import time
 from unittest import skip
 import pytest
 from django.urls import reverse
@@ -22,58 +22,17 @@ from selenium.webdriver.common.by import By
 
 from .utils import get_projectId_from_url, wait_until_build, wait_until_build_cancelled
 
-
-@pytest.mark.django_db
-@pytest.mark.order("last")
-class TestProjectPage(SeleniumFunctionalTestCase):
+class TestProjectPageBase(SeleniumFunctionalTestCase):
     project_id = None
     PROJECT_NAME = 'TestProjectPage'
 
-    def _create_project(self, project_name):
-        """ Create/Test new project using:
-          - Project Name: Any string
-          - Release: Any string
-          - Merge Toaster settings: True or False
-        """
-        self.get(reverse('newproject'))
-        self.wait_until_visible('#new-project-name')
-        self.find("#new-project-name").send_keys(project_name)
-        select = Select(self.find("#projectversion"))
-        select.select_by_value('3')
-
-        # check merge toaster settings
-        checkbox = self.find('.checkbox-mergeattr')
-        if not checkbox.is_selected():
-            checkbox.click()
-
-        if self.PROJECT_NAME != 'TestProjectPage':
-            # Reset project name if it's not the default one
-            self.PROJECT_NAME = 'TestProjectPage'
-
-        self.find("#create-project-button").click()
-
-        try:
-            self.wait_until_visible('#hint-error-project-name')
-            url = reverse('project', args=(TestProjectPage.project_id, ))
-            self.get(url)
-            self.wait_until_visible('#config-nav', poll=3)
-        except TimeoutException:
-            self.wait_until_visible('#config-nav', poll=3)
-
-    def _random_string(self, length):
-        return ''.join(
-            random.choice(string.ascii_letters) for _ in range(length)
-        )
-
     def _navigate_to_project_page(self):
         # Navigate to project page
-        if TestProjectPage.project_id is None:
-            self._create_project(project_name=self._random_string(10))
-            current_url = self.driver.current_url
-            TestProjectPage.project_id = get_projectId_from_url(current_url)
-        else:
-            url = reverse('project', args=(TestProjectPage.project_id,))
-            self.get(url)
+        if TestProjectPageBase.project_id is None:
+            TestProjectPageBase.project_id = self.create_new_project(self.PROJECT_NAME, '3', None, True)
+
+        url = reverse('project', args=(TestProjectPageBase.project_id,))
+        self.get(url)
         self.wait_until_visible('#config-nav')
 
     def _get_create_builds(self, **kwargs):
@@ -81,14 +40,14 @@ class TestProjectPage(SeleniumFunctionalTestCase):
         # parameters for builds to associate with the projects
         now = timezone.now()
         self.project1_build_success = {
-            'project': Project.objects.get(id=TestProjectPage.project_id),
+            'project': Project.objects.get(id=TestProjectPageBase.project_id),
             'started_on': now,
             'completed_on': now,
             'outcome': Build.SUCCEEDED
         }
 
         self.project1_build_failure = {
-            'project': Project.objects.get(id=TestProjectPage.project_id),
+            'project': Project.objects.get(id=TestProjectPageBase.project_id),
             'started_on': now,
             'completed_on': now,
             'outcome': Build.FAILED
@@ -133,7 +92,8 @@ class TestProjectPage(SeleniumFunctionalTestCase):
             list_check_box_id: list
     ):
         # Check edit column
-        edit_column = self.find(f'#{edit_btn_id}')
+        finder = lambda driver: self.find(f'#{edit_btn_id}')
+        edit_column = self.wait_until_element_clickable(finder)
         self.assertTrue(edit_column.is_displayed())
         edit_column.click()
         # Check dropdown is visible
@@ -192,7 +152,7 @@ class TestProjectPage(SeleniumFunctionalTestCase):
         def test_show_rows(row_to_show, show_row_link):
             # Check that we can show rows == row_to_show
             show_row_link.select_by_value(str(row_to_show))
-            self.wait_until_visible(f'#{table_selector} tbody tr', poll=3)
+            self.wait_until_visible(f'#{table_selector} tbody tr')
             # check at least some rows are visible
             self.assertTrue(
                 len(self.find_all(f'#{table_selector} tbody tr')) > 0
@@ -222,34 +182,7 @@ class TestProjectPage(SeleniumFunctionalTestCase):
         rows = self.find_all(f'#{table_selector} tbody tr')
         self.assertTrue(len(rows) > 0)
 
-    def test_create_project(self):
-        """ Create/Test new project using:
-          - Project Name: Any string
-          - Release: Any string
-          - Merge Toaster settings: True or False
-        """
-        self._create_project(project_name=self.PROJECT_NAME)
-
-    def test_image_recipe_editColumn(self):
-        """ Test the edit column feature in image recipe table on project page """
-        self._get_create_builds(success=10, failure=10)
-
-        url = reverse('projectimagerecipes', args=(TestProjectPage.project_id,))
-        self.get(url)
-        self.wait_until_present('#imagerecipestable tbody tr')
-
-        column_list = [
-            'get_description_or_summary', 'layer_version__get_vcs_reference',
-            'layer_version__layer__name', 'license', 'recipe-file', 'section',
-            'version'
-        ]
-
-        # Check that we can hide the edit column
-        self._mixin_test_table_edit_column(
-            'imagerecipestable',
-            'edit-columns-button',
-            [f'checkbox-{column}' for column in column_list]
-        )
+class TestProjectPage(TestProjectPageBase):
 
     def test_page_header_on_project_page(self):
         """ Check page header in project page:
@@ -272,8 +205,8 @@ class TestProjectPage(SeleniumFunctionalTestCase):
         logo_img = logo.find_element(By.TAG_NAME, 'img')
         self.assertTrue(logo_img.is_displayed(),
                         'Logo of Yocto project not found')
-        self.assertTrue(
-            '/static/img/logo.png' in str(logo_img.get_attribute('src')),
+        self.assertIn(
+            '/static/img/logo.png', str(logo_img.get_attribute('src')),
             'Logo of Yocto project not found'
         )
         # "Toaster"+" Information icon", clickable
@@ -282,34 +215,34 @@ class TestProjectPage(SeleniumFunctionalTestCase):
             "//div[@class='toaster-navbar-brand']//a[@class='brand']",
         )
         self.assertTrue(toaster.is_displayed(), 'Toaster not found')
-        self.assertTrue(toaster.text == 'Toaster')
+        self.assertEqual(toaster.text, 'Toaster')
         info_sign = self.find('.glyphicon-info-sign')
         self.assertTrue(info_sign.is_displayed())
 
         # "Server Icon" + "All builds"
         all_builds = self.find('#navbar-all-builds')
         all_builds_link = all_builds.find_element(By.TAG_NAME, 'a')
-        self.assertTrue("All builds" in all_builds_link.text)
-        self.assertTrue(
-            '/toastergui/builds/' in str(all_builds_link.get_attribute('href'))
+        self.assertIn("All builds", all_builds_link.text)
+        self.assertIn(
+            '/toastergui/builds/', str(all_builds_link.get_attribute('href'))
         )
         server_icon = all_builds.find_element(By.TAG_NAME, 'i')
-        self.assertTrue(
-            server_icon.get_attribute('class') == 'glyphicon glyphicon-tasks'
+        self.assertEqual(
+            server_icon.get_attribute('class'), 'glyphicon glyphicon-tasks'
         )
         self.assertTrue(server_icon.is_displayed())
 
         # "Directory Icon" + "All projects"
         all_projects = self.find('#navbar-all-projects')
         all_projects_link = all_projects.find_element(By.TAG_NAME, 'a')
-        self.assertTrue("All projects" in all_projects_link.text)
-        self.assertTrue(
-            '/toastergui/projects/' in str(all_projects_link.get_attribute(
+        self.assertIn("All projects", all_projects_link.text)
+        self.assertIn(
+            '/toastergui/projects/', str(all_projects_link.get_attribute(
                 'href'))
         )
         dir_icon = all_projects.find_element(By.TAG_NAME, 'i')
-        self.assertTrue(
-            dir_icon.get_attribute('class') == 'icon-folder-open'
+        self.assertEqual(
+            dir_icon.get_attribute('class'), 'icon-folder-open'
         )
         self.assertTrue(dir_icon.is_displayed())
 
@@ -317,23 +250,23 @@ class TestProjectPage(SeleniumFunctionalTestCase):
         toaster_docs_link = self.find('#navbar-docs')
         toaster_docs_link_link = toaster_docs_link.find_element(By.TAG_NAME,
                                                                 'a')
-        self.assertTrue("Documentation" in toaster_docs_link_link.text)
-        self.assertTrue(
-            toaster_docs_link_link.get_attribute('href') == 'http://docs.yoctoproject.org/toaster-manual/index.html#toaster-user-manual'
+        self.assertIn("Documentation", toaster_docs_link_link.text)
+        self.assertEqual(
+            toaster_docs_link_link.get_attribute('href'), 'http://docs.yoctoproject.org/toaster-manual/index.html#toaster-user-manual'
         )
         book_icon = toaster_docs_link.find_element(By.TAG_NAME, 'i')
-        self.assertTrue(
-            book_icon.get_attribute('class') == 'glyphicon glyphicon-book'
+        self.assertEqual(
+            book_icon.get_attribute('class'), 'glyphicon glyphicon-book'
         )
         self.assertTrue(book_icon.is_displayed())
 
         # AT RIGHT -> button "New project"
         new_project_button = self.find('#new-project-button')
         self.assertTrue(new_project_button.is_displayed())
-        self.assertTrue(new_project_button.text == 'New project')
+        self.assertEqual(new_project_button.text, 'New project')
         new_project_button.click()
-        self.assertTrue(
-            '/toastergui/newproject/' in str(self.driver.current_url)
+        self.assertIn(
+            '/toastergui/newproject/', str(self.driver.current_url)
         )
 
     def test_edit_project_name(self):
@@ -348,7 +281,8 @@ class TestProjectPage(SeleniumFunctionalTestCase):
 
         # click on "Edit" icon button
         self.wait_until_visible('#project-name-container')
-        edit_button = self.find('#project-change-form-toggle')
+        finder = lambda driver: self.find('#project-change-form-toggle')
+        edit_button = self.wait_until_element_clickable(finder)
         edit_button.click()
         project_name_input = self.find('#project-name-change-input')
         self.assertTrue(project_name_input.is_displayed())
@@ -358,8 +292,8 @@ class TestProjectPage(SeleniumFunctionalTestCase):
 
         # check project name is changed
         self.wait_until_visible('#project-name-container')
-        self.assertTrue(
-            'New Name' in str(self.find('#project-name-container').text)
+        self.assertIn(
+            'New Name', str(self.find('#project-name-container').text)
         )
 
     def test_project_page_tabs(self):
@@ -376,10 +310,10 @@ class TestProjectPage(SeleniumFunctionalTestCase):
         # check "configuration" tab
         self.wait_until_visible('#topbar-configuration-tab')
         config_tab = self.find('#topbar-configuration-tab')
-        self.assertTrue(config_tab.get_attribute('class') == 'active')
-        self.assertTrue('Configuration' in str(config_tab.text))
-        self.assertTrue(
-            f"/toastergui/project/{TestProjectPage.project_id}" in str(self.driver.current_url)
+        self.assertEqual(config_tab.get_attribute('class'), 'active')
+        self.assertIn('Configuration', str(config_tab.text))
+        self.assertIn(
+            f"/toastergui/project/{TestProjectPageBase.project_id}", str(self.driver.current_url)
         )
 
         def get_tabs():
@@ -392,9 +326,9 @@ class TestProjectPage(SeleniumFunctionalTestCase):
         def check_tab_link(tab_index, tab_name, url):
             tab = get_tabs()[tab_index]
             tab_link = tab.find_element(By.TAG_NAME, 'a')
-            self.assertTrue(url in tab_link.get_attribute('href'))
-            self.assertTrue(tab_name in tab_link.text)
-            self.assertTrue(tab.get_attribute('class') == 'active')
+            self.assertIn(url, tab_link.get_attribute('href'))
+            self.assertIn(tab_name, tab_link.text)
+            self.assertEqual(tab.get_attribute('class'), 'active')
 
         # check "Builds" tab
         builds_tab = get_tabs()[1]
@@ -402,7 +336,7 @@ class TestProjectPage(SeleniumFunctionalTestCase):
         check_tab_link(
             1,
             'Builds',
-            f"/toastergui/project/{TestProjectPage.project_id}/builds"
+            f"/toastergui/project/{TestProjectPageBase.project_id}/builds"
         )
 
         # check "Import layers" tab
@@ -411,7 +345,7 @@ class TestProjectPage(SeleniumFunctionalTestCase):
         check_tab_link(
             2,
             'Import layer',
-            f"/toastergui/project/{TestProjectPage.project_id}/importlayer"
+            f"/toastergui/project/{TestProjectPageBase.project_id}/importlayer"
         )
 
         # check "New custom image" tab
@@ -420,7 +354,7 @@ class TestProjectPage(SeleniumFunctionalTestCase):
         check_tab_link(
             3,
             'New custom image',
-            f"/toastergui/project/{TestProjectPage.project_id}/newcustomimage"
+            f"/toastergui/project/{TestProjectPageBase.project_id}/newcustomimage"
         )
 
         # check search box can be use to build recipes
@@ -428,13 +362,17 @@ class TestProjectPage(SeleniumFunctionalTestCase):
         search_box.send_keys('core-image-minimal')
         self.find('#build-button').click()
         self.wait_until_visible('#latest-builds')
-        lastest_builds = self.driver.find_elements(
-            By.XPATH,
-            '//div[@id="latest-builds"]',
-        )
-        last_build = lastest_builds[0]
-        self.assertTrue(
-            'core-image-minimal' in str(last_build.text)
+        buildtext = "Loading"
+        while "Loading" in buildtext:
+            time.sleep(1)
+            lastest_builds = self.driver.find_elements(
+                By.XPATH,
+                '//div[@id="latest-builds"]',
+            )
+            last_build = lastest_builds[0]
+            buildtext = last_build.text
+        self.assertIn(
+            'core-image-minimal', str(last_build.text)
         )
 
     def test_softwareRecipe_page(self):
@@ -446,7 +384,7 @@ class TestProjectPage(SeleniumFunctionalTestCase):
         """
         self._navigate_to_config_nav('softwarerecipestable', 4)
         # check title "Compatible software recipes" is displayed
-        self.assertTrue("Compatible software recipes" in self.get_page_source())
+        self.assertIn("Compatible software recipes", self.get_page_source())
         # Test search input
         self._mixin_test_table_search_input(
             input_selector='search-input-softwarerecipestable',
@@ -455,12 +393,8 @@ class TestProjectPage(SeleniumFunctionalTestCase):
             table_selector='softwarerecipestable'
         )
         # check "build recipe" button works
-        rows = self.find_all('#softwarerecipestable tbody tr')
-        image_to_build = rows[0]
-        build_btn = image_to_build.find_element(
-            By.XPATH,
-            '//td[@class="add-del-layers"]//a[1]'
-        )
+        finder = lambda driver: self.find_all('#softwarerecipestable tbody tr')[0].find_element(By.XPATH, '//td[@class="add-del-layers"]/a')
+        build_btn = self.wait_until_element_clickable(finder)
         build_btn.click()
         build_state = wait_until_build(self, 'queued cloning starting parsing failed')
         lastest_builds = self.driver.find_elements(
@@ -468,11 +402,10 @@ class TestProjectPage(SeleniumFunctionalTestCase):
             '//div[@id="latest-builds"]/div'
         )
         self.assertTrue(len(lastest_builds) > 0)
-        last_build = lastest_builds[0]
-        cancel_button = last_build.find_element(
-            By.XPATH,
-            '//span[@class="cancel-build-btn pull-right alert-link"]',
-        )
+        # Find the latest builds, the last build and then the cancel button
+ 
+        finder = lambda driver: driver.find_elements(By.XPATH, '//div[@id="latest-builds"]/div')[0].find_element(By.XPATH, '//span[@class="cancel-build-btn pull-right alert-link"]')
+        cancel_button = self.wait_until_element_clickable(finder)
         cancel_button.click()
         if 'starting' not in build_state:  # change build state when cancelled in starting state
             wait_until_build_cancelled(self)
@@ -510,7 +443,7 @@ class TestProjectPage(SeleniumFunctionalTestCase):
         """
         self._navigate_to_config_nav('machinestable', 5)
         # check title "Compatible software recipes" is displayed
-        self.assertTrue("Compatible machines" in self.get_page_source())
+        self.assertIn("Compatible machines", self.get_page_source())
         # Test search input
         self._mixin_test_table_search_input(
             input_selector='search-input-machinestable',
@@ -519,17 +452,13 @@ class TestProjectPage(SeleniumFunctionalTestCase):
             table_selector='machinestable'
         )
         # check "Select machine" button works
-        rows = self.find_all('#machinestable tbody tr')
-        machine_to_select = rows[0]
-        select_btn = machine_to_select.find_element(
-            By.XPATH,
-            '//td[@class="add-del-layers"]//a[1]'
-        )
-        select_btn.send_keys(Keys.RETURN)
-        self.wait_until_visible('#config-nav')
+        finder = lambda driver: self.find_all('#machinestable tbody tr')[0].find_element(By.XPATH, '//td[@class="add-del-layers"]')
+        select_btn = self.wait_until_element_clickable(finder)
+        select_btn.click()
+        self.wait_until_visible('#project-machine-name')
         project_machine_name = self.find('#project-machine-name')
-        self.assertTrue(
-            'qemux86-64' in project_machine_name.text
+        self.assertIn(
+            'qemux86-64', project_machine_name.text
         )
         # check "Add layer" button works
         self._navigate_to_config_nav('machinestable', 5)
@@ -540,16 +469,23 @@ class TestProjectPage(SeleniumFunctionalTestCase):
             searchBtn_selector='search-submit-machinestable',
             table_selector='machinestable'
         )
-        self.wait_until_visible('#machinestable tbody tr', poll=3)
-        rows = self.find_all('#machinestable tbody tr')
-        machine_to_add = rows[0]
-        add_btn = machine_to_add.find_element(By.XPATH, '//td[@class="add-del-layers"]')
+
+        self.wait_until_visible('#machinestable tbody tr')
+        # Locate a machine to add button
+        finder = lambda driver: self.find_all('#machinestable tbody tr')[0].find_element(By.XPATH, '//td[@class="add-del-layers"]')
+        add_btn = self.wait_until_element_clickable(finder)
         add_btn.click()
         self.wait_until_visible('#change-notification')
         change_notification = self.find('#change-notification')
-        self.assertTrue(
-            f'You have added 1 layer to your project' in str(change_notification.text)
+        self.assertIn(
+            f'You have added 1 layer to your project', str(change_notification.text)
         )
+
+        finder = lambda driver: self.find('#hide-alert')
+        hide_button = self.wait_until_element_clickable(finder)
+        hide_button.click()
+        self.wait_until_not_visible('#change-notification')
+
         # check Machine table feature(show/hide column, pagination)
         self._navigate_to_config_nav('machinestable', 5)
         column_list = [
@@ -580,7 +516,7 @@ class TestProjectPage(SeleniumFunctionalTestCase):
         """
         self._navigate_to_config_nav('layerstable', 6)
         # check title "Compatible layers" is displayed
-        self.assertTrue("Compatible layers" in self.get_page_source())
+        self.assertIn("Compatible layers", self.get_page_source())
         # Test search input
         input_text='meta-tanowrt'
         self._mixin_test_table_search_input(
@@ -590,42 +526,44 @@ class TestProjectPage(SeleniumFunctionalTestCase):
             table_selector='layerstable'
         )
         # check "Add layer" button works
-        self.wait_until_visible('#layerstable tbody tr', poll=3)
-        rows = self.find_all('#layerstable tbody tr')
-        layer_to_add = rows[0]
-        add_btn = layer_to_add.find_element(
-            By.XPATH,
-            '//td[@class="add-del-layers"]'
-        )
+        self.wait_until_visible('#layerstable tbody tr')
+        finder = lambda driver: self.find_all('#layerstable tbody tr')[0].find_element(By.XPATH, '//td[@class="add-del-layers"]/a[@data-directive="add"]')
+        add_btn = self.wait_until_element_clickable(finder)
         add_btn.click()
         # check modal is displayed
-        self.wait_until_visible('#dependencies-modal', poll=3)
+        self.wait_until_visible('#dependencies-modal')
         list_dependencies = self.find_all('#dependencies-list li')
         # click on add-layers button
-        add_layers_btn = self.driver.find_element(
-            By.XPATH,
-            '//form[@id="dependencies-modal-form"]//button[@class="btn btn-primary"]'
-        )
+        finder = lambda driver: self.driver.find_element(By.XPATH, '//form[@id="dependencies-modal-form"]//button[@class="btn btn-primary"]')
+        add_layers_btn = self.wait_until_element_clickable(finder)
         add_layers_btn.click()
         self.wait_until_visible('#change-notification')
         change_notification = self.find('#change-notification')
-        self.assertTrue(
-            f'You have added {len(list_dependencies)+1} layers to your project: {input_text} and its dependencies' in str(change_notification.text)
+        self.assertIn(
+            f'You have added {len(list_dependencies)+1} layers to your project: {input_text} and its dependencies', str(change_notification.text)
         )
+
+        finder = lambda driver: self.find('#hide-alert')
+        hide_button = self.wait_until_element_clickable(finder)
+        hide_button.click()
+        self.wait_until_not_visible('#change-notification')
+
         # check "Remove layer" button works
-        self.wait_until_visible('#layerstable tbody tr', poll=3)
-        rows = self.find_all('#layerstable tbody tr')
-        layer_to_remove = rows[0]
-        remove_btn = layer_to_remove.find_element(
-            By.XPATH,
-            '//td[@class="add-del-layers"]'
-        )
+        self.wait_until_visible('#layerstable tbody tr')
+        finder = lambda driver: self.find_all('#layerstable tbody tr')[0].find_element(By.XPATH, '//td[@class="add-del-layers"]/a[@data-directive="remove"]')
+        remove_btn = self.wait_until_element_clickable(finder)
         remove_btn.click()
-        self.wait_until_visible('#change-notification', poll=2)
+        self.wait_until_visible('#change-notification')
         change_notification = self.find('#change-notification')
-        self.assertTrue(
-            f'You have removed 1 layer from your project: {input_text}' in str(change_notification.text)
+        self.assertIn(
+            f'You have removed 1 layer from your project: {input_text}', str(change_notification.text)
         )
+
+        finder = lambda driver: self.find('#hide-alert')
+        hide_button = self.wait_until_element_clickable(finder)
+        hide_button.click()
+        self.wait_until_not_visible('#change-notification')
+
         # check layers table feature(show/hide column, pagination)
         self._navigate_to_config_nav('layerstable', 6)
         column_list = [
@@ -656,7 +594,7 @@ class TestProjectPage(SeleniumFunctionalTestCase):
         """
         self._navigate_to_config_nav('distrostable', 7)
         # check title "Compatible distros" is displayed
-        self.assertTrue("Compatible Distros" in self.get_page_source())
+        self.assertIn("Compatible Distros", self.get_page_source())
         # Test search input
         input_text='poky-altcfg'
         self._mixin_test_table_search_input(
@@ -666,17 +604,14 @@ class TestProjectPage(SeleniumFunctionalTestCase):
             table_selector='distrostable'
         )
         # check "Add distro" button works
-        rows = self.find_all('#distrostable tbody tr')
-        distro_to_add = rows[0]
-        add_btn = distro_to_add.find_element(
-            By.XPATH,
-            '//td[@class="add-del-layers"]//a[1]'
-        )
+        self.wait_until_visible(".add-del-layers")
+        finder = lambda driver: self.find_all('#distrostable tbody tr')[0].find_element(By.XPATH, '//td[@class="add-del-layers"]')
+        add_btn = self.wait_until_element_clickable(finder)
         add_btn.click()
-        self.wait_until_visible('#change-notification', poll=2)
+        self.wait_until_visible('#change-notification')
         change_notification = self.find('#change-notification')
-        self.assertTrue(
-            f'You have changed the distro to: {input_text}' in str(change_notification.text)
+        self.assertIn(
+            f'You have changed the distro to: {input_text}', str(change_notification.text)
         )
         # check distro table feature(show/hide column, pagination)
         self._navigate_to_config_nav('distrostable', 7)
@@ -699,7 +634,7 @@ class TestProjectPage(SeleniumFunctionalTestCase):
         )
 
     def test_single_layer_page(self):
-        """ Test layer page
+        """ Test layer details page using meta-poky as an example (assumes is added to start with)
             - Check if title is displayed
             - Check add/remove layer button works
             - Check tabs(layers, recipes, machines) are displayed
@@ -708,45 +643,62 @@ class TestProjectPage(SeleniumFunctionalTestCase):
                 - Check layer summary
                 - Check layer description
         """
-        url = reverse("layerdetails", args=(TestProjectPage.project_id, 7))
-        self.get(url)
+        self._navigate_to_config_nav('layerstable', 6)
+        layer_link = self.driver.find_element(By.XPATH, '//tr/td[@class="layer__name"]/a[contains(text(),"meta-poky")]')
+        layer_link.click()
         self.wait_until_visible('.page-header')
         # check title is displayed
         self.assertTrue(self.find('.page-header h1').is_displayed())
 
-        # check add layer button works
-        remove_layer_btn = self.find('#add-remove-layer-btn')
+        # check remove layer button works
+        finder = lambda driver: self.find('#add-remove-layer-btn')
+        remove_layer_btn = self.wait_until_element_clickable(finder)
         remove_layer_btn.click()
-        self.wait_until_visible('#change-notification', poll=2)
+        self.wait_until_visible('#change-notification')
         change_notification = self.find('#change-notification')
-        self.assertTrue(
-            f'You have removed 1 layer from your project' in str(change_notification.text)
+        self.assertIn(
+            f'You have removed 1 layer from your project', str(change_notification.text)
         )
-        # check add layer button works, 18 is the random layer id
-        add_layer_btn = self.find('#add-remove-layer-btn')
+        finder = lambda driver: self.find('#hide-alert')
+        hide_button = self.wait_until_element_clickable(finder)
+        hide_button.click()
+        # check add layer button works
+        self.wait_until_not_visible('#change-notification')
+        finder = lambda driver: self.find('#add-remove-layer-btn')
+        add_layer_btn = self.wait_until_element_clickable(finder)
         add_layer_btn.click()
         self.wait_until_visible('#change-notification')
         change_notification = self.find('#change-notification')
-        self.assertTrue(
-            f'You have added 1 layer to your project' in str(change_notification.text)
+        self.assertIn(
+            f'You have added 1 layer to your project', str(change_notification.text)
         )
+        finder = lambda driver: self.find('#hide-alert')
+        hide_button = self.wait_until_element_clickable(finder)
+        hide_button.click()
+        self.wait_until_not_visible('#change-notification')
         # check tabs(layers, recipes, machines) are displayed
         tabs = self.find_all('.nav-tabs li')
         self.assertEqual(len(tabs), 3)
         # Check first tab
         tabs[0].click()
-        self.assertTrue(
-            'active' in str(self.find('#information').get_attribute('class'))
+        self.assertIn(
+            'active', str(self.find('#information').get_attribute('class'))
         )
-        # Check second tab
+        # Check second tab (recipes)
+        # Ensure page is scrolled to the top
+        self.driver.find_element(By.XPATH, '//body').send_keys(Keys.CONTROL + Keys.HOME)
+        self.wait_until_visible('.nav-tabs')
         tabs[1].click()
-        self.assertTrue(
-            'active' in str(self.find('#recipes').get_attribute('class'))
+        self.assertIn(
+            'active', str(self.find('#recipes').get_attribute('class'))
         )
-        # Check third tab
+        # Check third tab (machines)
+        # Ensure page is scrolled to the top
+        self.driver.find_element(By.XPATH, '//body').send_keys(Keys.CONTROL + Keys.HOME)
+        self.wait_until_visible('.nav-tabs')
         tabs[2].click()
-        self.assertTrue(
-            'active' in str(self.find('#machines').get_attribute('class'))
+        self.assertIn(
+            'active', str(self.find('#machines').get_attribute('class'))
         )
         # Check left section is displayed
         section = self.find('.well')
@@ -755,9 +707,13 @@ class TestProjectPage(SeleniumFunctionalTestCase):
             section.find_element(By.XPATH, '//h2[1]').is_displayed()
         )
         # Check layer summary
-        self.assertTrue("Summary" in section.text)
+        self.assertIn("Summary", section.text)
         # Check layer description
-        self.assertTrue("Description" in section.text)
+        self.assertIn("Description", section.text)
+
+@pytest.mark.django_db
+@pytest.mark.order("last")
+class TestProjectPageRecipes(TestProjectPageBase):
 
     def test_single_recipe_page(self):
         """ Test recipe page
@@ -767,7 +723,12 @@ class TestProjectPage(SeleniumFunctionalTestCase):
                 - Check recipe: name, summary, description, Version, Section,
                 License, Approx. packages included, Approx. size, Recipe file
         """
-        url = reverse("recipedetails", args=(TestProjectPage.project_id, 53428))
+        # Use a recipe which is likely to exist in the layer index but not enabled
+        # in poky out the box - xen-image-minimal from meta-virtualization
+        self._navigate_to_project_page()
+        prj = Project.objects.get(pk=TestProjectPageBase.project_id)
+        recipe_id = prj.get_all_compatible_recipes().get(name="xen-image-minimal").pk
+        url = reverse("recipedetails", args=(TestProjectPageBase.project_id, recipe_id))
         self.get(url)
         self.wait_until_visible('.page-header')
         # check title is displayed
@@ -782,11 +743,33 @@ class TestProjectPage(SeleniumFunctionalTestCase):
             section.find_element(By.XPATH, '//h2[1]').is_displayed()
         )
         # Check recipe sections details info are displayed
-        self.assertTrue("Summary" in section.text)
-        self.assertTrue("Description" in section.text)
-        self.assertTrue("Version" in section.text)
-        self.assertTrue("Section" in section.text)
-        self.assertTrue("License" in section.text)
-        self.assertTrue("Approx. packages included" in section.text)
-        self.assertTrue("Approx. package size" in section.text)
-        self.assertTrue("Recipe file" in section.text)
+        self.assertIn("Summary", section.text)
+        self.assertIn("Description", section.text)
+        self.assertIn("Version", section.text)
+        self.assertIn("Section", section.text)
+        self.assertIn("License", section.text)
+        self.assertIn("Approx. packages included", section.text)
+        self.assertIn("Approx. package size", section.text)
+        self.assertIn("Recipe file", section.text)
+
+    def test_image_recipe_editColumn(self):
+        """ Test the edit column feature in image recipe table on project page """
+        self._get_create_builds(success=10, failure=10)
+
+        url = reverse('projectimagerecipes', args=(TestProjectPageBase.project_id,))
+        self.get(url)
+        self.wait_until_present('#imagerecipestable tbody tr')
+
+        column_list = [
+            'get_description_or_summary', 'layer_version__get_vcs_reference',
+            'layer_version__layer__name', 'license', 'recipe-file', 'section',
+            'version'
+        ]
+
+        # Check that we can hide the edit column
+        self._mixin_test_table_edit_column(
+            'imagerecipestable',
+            'edit-columns-button',
+            [f'checkbox-{column}' for column in column_list]
+        )
+

bitbake/lib/toaster/tests/functional/test_project_page_tab_config.py

@@ -7,72 +7,27 @@
 #
 
 import string
-import random
+import time
 import pytest
 from django.urls import reverse
 from selenium.webdriver import Keys
 from selenium.webdriver.support.select import Select
 from selenium.common.exceptions import ElementClickInterceptedException, NoSuchElementException, TimeoutException
-from orm.models import Project
 from tests.functional.functional_helpers import SeleniumFunctionalTestCase
 from selenium.webdriver.common.by import By
 
 from .utils import get_projectId_from_url, wait_until_build, wait_until_build_cancelled
 
-
-@pytest.mark.django_db
-@pytest.mark.order("last")
-class TestProjectConfigTab(SeleniumFunctionalTestCase):
+class TestProjectConfigTabBase(SeleniumFunctionalTestCase):
     PROJECT_NAME = 'TestProjectConfigTab'
     project_id = None
 
-    def _create_project(self, project_name, **kwargs):
-        """ Create/Test new project using:
-          - Project Name: Any string
-          - Release: Any string
-          - Merge Toaster settings: True or False
-        """
-        release = kwargs.get('release', '3')
-        self.get(reverse('newproject'))
-        self.wait_until_visible('#new-project-name')
-        self.find("#new-project-name").send_keys(project_name)
-        select = Select(self.find("#projectversion"))
-        select.select_by_value(release)
-
-        # check merge toaster settings
-        checkbox = self.find('.checkbox-mergeattr')
-        if not checkbox.is_selected():
-            checkbox.click()
-
-        if self.PROJECT_NAME != 'TestProjectConfigTab':
-            # Reset project name if it's not the default one
-            self.PROJECT_NAME = 'TestProjectConfigTab'
-
-        self.find("#create-project-button").click()
-
-        try:
-            self.wait_until_visible('#hint-error-project-name', poll=3)
-            url = reverse('project', args=(TestProjectConfigTab.project_id, ))
-            self.get(url)
-            self.wait_until_visible('#config-nav', poll=3)
-        except TimeoutException:
-            self.wait_until_visible('#config-nav', poll=3)
-
-    def _random_string(self, length):
-        return ''.join(
-            random.choice(string.ascii_letters) for _ in range(length)
-        )
-
     def _navigate_to_project_page(self):
         # Navigate to project page
-        if TestProjectConfigTab.project_id is None:
-            self._create_project(project_name=self._random_string(10))
-            current_url = self.driver.current_url
-            TestProjectConfigTab.project_id = get_projectId_from_url(
-                current_url)
-        else:
-            url = reverse('project', args=(TestProjectConfigTab.project_id,))
-            self.get(url)
+        if TestProjectConfigTabBase.project_id is None:
+            TestProjectConfigTabBase.project_id = self.create_new_project(self.PROJECT_NAME, '3', None, True)
+        url = reverse('project', args=(TestProjectConfigTabBase.project_id,))
+        self.get(url)
         self.wait_until_visible('#config-nav')
 
     def _create_builds(self):
@@ -88,8 +43,8 @@ class TestProjectConfigTab(SeleniumFunctionalTestCase):
             '//div[@id="latest-builds"]/div',
         )
         last_build = lastest_builds[0]
-        self.assertTrue(
-            'foo' in str(last_build.text)
+        self.assertIn(
+            'foo', str(last_build.text)
         )
         last_build = lastest_builds[0]
         try:
@@ -114,6 +69,8 @@ class TestProjectConfigTab(SeleniumFunctionalTestCase):
         config_nav = self.find('#config-nav')
         return config_nav.find_elements(By.TAG_NAME, 'li')[index]
 
+class TestProjectConfigTab(TestProjectConfigTabBase):
+
     def test_project_config_nav(self):
         """ Test project config tab navigation:
         - Check if the menu is displayed and contains the right elements:
@@ -138,48 +95,48 @@ class TestProjectConfigTab(SeleniumFunctionalTestCase):
 
         def check_config_nav_item(index, item_name, url):
             item = _get_config_nav_item(index)
-            self.assertTrue(item_name in item.text)
-            self.assertTrue(item.get_attribute('class') == 'active')
-            self.assertTrue(url in self.driver.current_url)
+            self.assertIn(item_name, item.text)
+            self.assertEqual(item.get_attribute('class'), 'active')
+            self.assertIn(url, self.driver.current_url)
 
         # check if the menu contains the right elements
         # COMPATIBLE METADATA
         compatible_metadata = _get_config_nav_item(1)
-        self.assertTrue(
-            "compatible metadata" in compatible_metadata.text.lower()
+        self.assertIn(
+            "compatible metadata", compatible_metadata.text.lower()
         )
         # EXTRA CONFIGURATION
         extra_configuration = _get_config_nav_item(8)
-        self.assertTrue(
-            "extra configuration" in extra_configuration.text.lower()
+        self.assertIn(
+            "extra configuration", extra_configuration.text.lower()
         )
         # Actions
         actions = _get_config_nav_item(10)
-        self.assertTrue("actions" in str(actions.text).lower())
+        self.assertIn("actions", str(actions.text).lower())
 
         conf_nav_list = [
             # config
             [0, 'Configuration',
-                f"/toastergui/project/{TestProjectConfigTab.project_id}"],
+                f"/toastergui/project/{TestProjectConfigTabBase.project_id}"],
             # custom images
             [2, 'Custom images',
-                f"/toastergui/project/{TestProjectConfigTab.project_id}/customimages"],
+                f"/toastergui/project/{TestProjectConfigTabBase.project_id}/customimages"],
             # image recipes
             [3, 'Image recipes',
-                f"/toastergui/project/{TestProjectConfigTab.project_id}/images"],
+                f"/toastergui/project/{TestProjectConfigTabBase.project_id}/images"],
             # software recipes
             [4, 'Software recipes',
-                f"/toastergui/project/{TestProjectConfigTab.project_id}/softwarerecipes"],
+                f"/toastergui/project/{TestProjectConfigTabBase.project_id}/softwarerecipes"],
             # machines
             [5, 'Machines',
-                f"/toastergui/project/{TestProjectConfigTab.project_id}/machines"],
+                f"/toastergui/project/{TestProjectConfigTabBase.project_id}/machines"],
             # layers
             [6, 'Layers',
-                f"/toastergui/project/{TestProjectConfigTab.project_id}/layers"],
+                f"/toastergui/project/{TestProjectConfigTabBase.project_id}/layers"],
             # distro
             [7, 'Distros',
-                f"/toastergui/project/{TestProjectConfigTab.project_id}/distros"],
-            #  [9, 'BitBake variables', f"/toastergui/project/{TestProjectConfigTab.project_id}/configuration"],  # bitbake variables
+                f"/toastergui/project/{TestProjectConfigTabBase.project_id}/distros"],
+            #  [9, 'BitBake variables', f"/toastergui/project/{TestProjectConfigTabBase.project_id}/configuration"],  # bitbake variables
         ]
         for index, item_name, url in conf_nav_list:
             item = _get_config_nav_item(index)
@@ -253,7 +210,7 @@ class TestProjectConfigTab(SeleniumFunctionalTestCase):
         def test_show_rows(row_to_show, show_row_link):
             # Check that we can show rows == row_to_show
             show_row_link.select_by_value(str(row_to_show))
-            self.wait_until_visible('#imagerecipestable tbody tr', poll=3)
+            self.wait_until_visible('#imagerecipestable tbody tr')
             # check at least some rows are visible
             self.assertTrue(
                 len(self.find_all('#imagerecipestable tbody tr'))  > 0
@@ -299,9 +256,11 @@ class TestProjectConfigTab(SeleniumFunctionalTestCase):
                     - meta-poky
                     - meta-yocto-bsp
         """
-        # Create a new project for this test
-        project_name = self._random_string(10)
-        self._create_project(project_name=project_name)
+        project_id = self.create_new_project(self.PROJECT_NAME + "-ST", '3', None, True)
+        url = reverse('project', args=(project_id,))
+        self.get(url)
+        self.wait_until_visible('#config-nav')
+
         # check if the menu is displayed
         self.wait_until_visible('#project-page')
         block_l = self.driver.find_element(
@@ -313,7 +272,7 @@ class TestProjectConfigTab(SeleniumFunctionalTestCase):
         def check_machine_distro(self, item_name, new_item_name, block_id):
             block = self.find(f'#{block_id}')
             title = block.find_element(By.TAG_NAME, 'h3')
-            self.assertTrue(item_name.capitalize() in title.text)
+            self.assertIn(item_name.capitalize(), title.text)
             edit_btn = self.find(f'#change-{item_name}-toggle')
             edit_btn.click()
             self.wait_until_visible(f'#{item_name}-change-input')
@@ -324,12 +283,15 @@ class TestProjectConfigTab(SeleniumFunctionalTestCase):
             change_btn.click()
             self.wait_until_visible(f'#project-{item_name}-name')
             project_name = self.find(f'#project-{item_name}-name')
-            self.assertTrue(new_item_name in project_name.text)
+            self.assertIn(new_item_name, project_name.text)
             # check change notificaiton is displayed
             change_notification = self.find('#change-notification')
-            self.assertTrue(
-                f'You have changed the {item_name} to: {new_item_name}' in change_notification.text
+            self.assertIn(
+                f'You have changed the {item_name} to: {new_item_name}', change_notification.text
             )
+            hide_button = self.find('#hide-alert')
+            hide_button.click()
+            self.wait_until_not_visible('#change-notification')
 
         # Machine
         check_machine_distro(self, 'machine', 'qemux86-64', 'machine-section')
@@ -338,97 +300,51 @@ class TestProjectConfigTab(SeleniumFunctionalTestCase):
 
         # Project release
         title = project_release.find_element(By.TAG_NAME, 'h3')
-        self.assertTrue("Project release" in title.text)
-        self.assertTrue(
-            "Yocto Project master" in self.find('#project-release-title').text
+        self.assertIn("Project release", title.text)
+        self.assertIn(
+            "Yocto Project master", self.find('#project-release-title').text
         )
         # Layers
         title = layers.find_element(By.TAG_NAME, 'h3')
-        self.assertTrue("Layers" in title.text)
+        self.assertIn("Layers", title.text)
+        self.wait_until_clickable('#layer-add-input')
         # check at least three layers are displayed
         # openembedded-core
         # meta-poky
         # meta-yocto-bsp
-        layers_list = layers.find_element(By.ID, 'layers-in-project-list')
-        layers_list_items = layers_list.find_elements(By.TAG_NAME, 'li')
+        layer_list_items = []
+        starttime = time.time()
+        while len(layer_list_items) < 3:
+            layers_list = self.driver.find_element(By.ID, 'layers-in-project-list')
+            layer_list_items = layers_list.find_elements(By.TAG_NAME, 'li')
+            if time.time() > (starttime + 30):
+                self.fail("Layer list didn't contain at least 3 items within 30s (contained %d)" % len(layer_list_items))
+
         # remove all layers except the first three layers
-        for i in range(3, len(layers_list_items)):
-            layers_list_items[i].find_element(By.TAG_NAME, 'span').click()
+        for i in range(3, len(layer_list_items)):
+            layer_list_items[i].find_element(By.TAG_NAME, 'span').click()
+
         # check can add a layer if exists
         add_layer_input = layers.find_element(By.ID, 'layer-add-input')
         add_layer_input.send_keys('meta-oe')
         self.wait_until_visible('#layer-container > form > div > span > div')
-        dropdown_item = self.driver.find_element(
-            By.XPATH,
-            '//*[@id="layer-container"]/form/div/span/div'
-        )
-        try:
-            dropdown_item.click()
-        except ElementClickInterceptedException:
-            self.skipTest(
-                "layer-container dropdown item click intercepted. Element not properly visible.")
+        self.wait_until_visible('.dropdown-menu')
+        finder = lambda driver: driver.find_element(By.XPATH, '//*[@id="layer-container"]/form/div/span/div/div/div')
+        dropdown_item = self.wait_until_element_clickable(finder)
+        dropdown_item.click()
+        self.wait_until_clickable('#add-layer-btn')
         add_layer_btn = layers.find_element(By.ID, 'add-layer-btn')
         add_layer_btn.click()
         self.wait_until_visible('#layers-in-project-list')
+
         # check layer is added
-        layers_list_items = layers_list.find_elements(By.TAG_NAME, 'li')
-        self.assertTrue(len(layers_list_items) == 4)
-
-    def test_most_build_recipes(self):
-        """ Test most build recipes block contains"""
-        def rebuild_from_most_build_recipes(recipe_list_items):
-            checkbox = recipe_list_items[0].find_element(By.TAG_NAME, 'input')
-            checkbox.click()
-            build_btn = self.find('#freq-build-btn')
-            build_btn.click()
-            self.wait_until_visible('#latest-builds')
-            wait_until_build(self, 'queued cloning starting parsing failed')
-            lastest_builds = self.driver.find_elements(
-                By.XPATH,
-                '//div[@id="latest-builds"]/div'
-            )
-            self.assertTrue(len(lastest_builds) >= 2)
-            last_build = lastest_builds[0]
-            try:
-                cancel_button = last_build.find_element(
-                    By.XPATH,
-                    '//span[@class="cancel-build-btn pull-right alert-link"]',
-                )
-                cancel_button.click()
-            except NoSuchElementException:
-                # Skip if the build is already cancelled
-                pass
-            wait_until_build_cancelled(self)
-        # Create a new project for remaining asserts
-        project_name = self._random_string(10)
-        self._create_project(project_name=project_name, release='2')
-        current_url = self.driver.current_url
-        TestProjectConfigTab.project_id = get_projectId_from_url(current_url)
-        url = current_url.split('?')[0]
-
-        # Create a new builds
-        self._create_builds()
-
-        # back to project page
-        self.driver.get(url)
-
-        self.wait_until_visible('#project-page', poll=3)
-
-        # Most built recipes
-        most_built_recipes = self.driver.find_element(
-            By.XPATH, '//*[@id="project-page"]/div[1]/div[3]')
-        title = most_built_recipes.find_element(By.TAG_NAME, 'h3')
-        self.assertTrue("Most built recipes" in title.text)
-        # check can select a recipe and build it
-        self.wait_until_visible('#freq-build-list', poll=3)
-        recipe_list = self.find('#freq-build-list')
-        recipe_list_items = recipe_list.find_elements(By.TAG_NAME, 'li')
-        self.assertTrue(
-            len(recipe_list_items) > 0,
-            msg="Any recipes found in the most built recipes list",
-        )
-        rebuild_from_most_build_recipes(recipe_list_items)
-        TestProjectConfigTab.project_id = None  # reset project id
+        layer_list_items = []
+        starttime = time.time()
+        while len(layer_list_items) < 4:
+            layers_list = self.driver.find_element(By.ID, 'layers-in-project-list')
+            layer_list_items = layers_list.find_elements(By.TAG_NAME, 'li')
+            if time.time() > (starttime + 30):
+                self.fail("Layer list didn't contain at least 4 items within 30s (contained %d)" % len(layer_list_items))
 
     def test_project_page_tab_importlayer(self):
         """ Test project page tab import layer """
@@ -466,42 +382,42 @@ class TestProjectConfigTab(SeleniumFunctionalTestCase):
         layers = block_l.find_element(By.ID, 'layer-container')
         layers_list = layers.find_element(By.ID, 'layers-in-project-list')
         layers_list_items = layers_list.find_elements(By.TAG_NAME, 'li')
-        self.assertTrue(
-            'meta-fake' in str(layers_list_items[-1].text)
+        self.assertIn(
+            'meta-fake', str(layers_list_items[-1].text)
         )
 
     def test_project_page_custom_image_no_image(self):
         """ Test project page tab "New custom image" when no custom image """
-        project_name = self._random_string(10)
-        self._create_project(project_name=project_name)
-        current_url = self.driver.current_url
-        TestProjectConfigTab.project_id = get_projectId_from_url(current_url)
+        project_id = self.create_new_project(self.PROJECT_NAME + "-CustomImage", '3', None, True)
+        url = reverse('project', args=(project_id,))
+        self.get(url)
+        self.wait_until_visible('#config-nav')
+
         # navigate to "Custom image" tab
         custom_image_section = self._get_config_nav_item(2)
         custom_image_section.click()
         self.wait_until_visible('#empty-state-customimagestable')
 
         # Check message when no custom image
-        self.assertTrue(
-            "You have not created any custom images yet." in str(
+        self.assertIn(
+            "You have not created any custom images yet.", str(
                 self.find('#empty-state-customimagestable').text
             )
         )
         div_empty_msg = self.find('#empty-state-customimagestable')
         link_create_custom_image = div_empty_msg.find_element(
             By.TAG_NAME, 'a')
-        self.assertTrue(TestProjectConfigTab.project_id is not None)
-        self.assertTrue(
-            f"/toastergui/project/{TestProjectConfigTab.project_id}/newcustomimage" in str(
+        self.assertTrue(TestProjectConfigTabBase.project_id is not None)
+        self.assertIn(
+            f"/toastergui/project/{project_id}/newcustomimage", str(
                 link_create_custom_image.get_attribute('href')
             )
         )
-        self.assertTrue(
-            "Create your first custom image" in str(
+        self.assertIn(
+            "Create your first custom image", str(
                 link_create_custom_image.text
             )
         )
-        TestProjectConfigTab.project_id = None  # reset project id
 
     def test_project_page_image_recipe(self):
         """ Test project page section images
@@ -526,3 +442,66 @@ class TestProjectConfigTab(SeleniumFunctionalTestCase):
         self.wait_until_visible('#imagerecipestable tbody tr')
         rows = self.find_all('#imagerecipestable tbody tr')
         self.assertTrue(len(rows) > 0)
+
+@pytest.mark.django_db
+@pytest.mark.order("last") 
+class TestProjectConfigTabDB(TestProjectConfigTabBase):
+
+    def test_most_build_recipes(self):
+        """ Test most build recipes block contains"""
+        def rebuild_from_most_build_recipes(recipe_list_items):
+            checkbox = recipe_list_items[0].find_element(By.TAG_NAME, 'input')
+            checkbox.click()
+            build_btn = self.find('#freq-build-btn')
+            build_btn.click()
+            self.wait_until_visible('#latest-builds')
+            wait_until_build(self, 'queued cloning starting parsing failed')
+            lastest_builds = self.driver.find_elements(
+                By.XPATH,
+                '//div[@id="latest-builds"]/div'
+            )
+            self.assertTrue(len(lastest_builds) >= 2)
+            last_build = lastest_builds[0]
+            try:
+                cancel_button = last_build.find_element(
+                    By.XPATH,
+                    '//span[@class="cancel-build-btn pull-right alert-link"]',
+                )
+                cancel_button.click()
+            except NoSuchElementException:
+                # Skip if the build is already cancelled
+                pass
+            wait_until_build_cancelled(self)
+
+        # Create a new project for remaining asserts
+        project_id = self.create_new_project(self.PROJECT_NAME + "-MostBuilt", '2', None, True)
+        url = reverse('project', args=(project_id,))
+        self.get(url)
+        self.wait_until_visible('#config-nav')
+
+        current_url = self.driver.current_url
+        url = current_url.split('?')[0]
+
+        # Create a new builds
+        self._create_builds()
+
+        # back to project page
+        self.driver.get(url)
+
+        self.wait_until_visible('#project-page')
+
+        # Most built recipes
+        most_built_recipes = self.driver.find_element(
+            By.XPATH, '//*[@id="project-page"]/div[1]/div[3]')
+        title = most_built_recipes.find_element(By.TAG_NAME, 'h3')
+        self.assertIn("Most built recipes", title.text)
+        # check can select a recipe and build it
+        self.wait_until_visible('#freq-build-list')
+        recipe_list = self.find('#freq-build-list')
+        recipe_list_items = recipe_list.find_elements(By.TAG_NAME, 'li')
+        self.assertTrue(
+            len(recipe_list_items) > 0,
+            msg="No recipes found in the most built recipes list",
+        )
+        rebuild_from_most_build_recipes(recipe_list_items)
+

bitbake/lib/toaster/tests/functional/utils.py

@@ -8,7 +8,7 @@
 
 
 from time import sleep
-from selenium.common.exceptions import NoSuchElementException, StaleElementReferenceException, TimeoutException
+from selenium.common.exceptions import NoSuchElementException, StaleElementReferenceException, TimeoutException, WebDriverException
 from selenium.webdriver.common.by import By
 
 from orm.models import Build
@@ -36,7 +36,7 @@ def wait_until_build(test_instance, state):
             if 'failed' in str(build_state).lower():
                 break
         except NoSuchElementException:
-            continue
+            pass
         except TimeoutException:
             break
         start_time += 1
@@ -48,7 +48,6 @@ def wait_until_build_cancelled(test_instance):
     """
     timeout = 30
     start_time = 0
-    build = None
     while True:
         try:
             if start_time > timeout:
@@ -64,19 +63,17 @@ def wait_until_build_cancelled(test_instance):
             if 'failed' in str(build_state).lower():
                 break
             if 'cancelling' in str(build_state).lower():
-                # Change build state to cancelled
-                if not build:  # get build object only once
-                    build = Build.objects.last()
-                    build.outcome = Build.CANCELLED
-                    build.save()
+                pass
             if 'cancelled' in str(build_state).lower():
                 break
-        except NoSuchElementException:
-            continue
-        except StaleElementReferenceException:
-            continue
         except TimeoutException:
             break
+        except NoSuchElementException:
+            pass
+        except StaleElementReferenceException:
+            pass
+        except WebDriverException:
+            pass
         start_time += 1
         sleep(1) # take a breath and try again