build-appliance-image: Update to nanbield head revision

(From OE-Core rev: d0e68072d138ccc1fb5957fdc46a91871eb6a3e1)

Signed-off-by: Steve Sakoman <steve@sakoman.com>

Makefile

@@ -1,35 +0,0 @@
-# Minimal makefile for Sphinx documentation
-#
-
-# You can set these variables from the command line, and also
-# from the environment for the first two.
-SPHINXOPTS    ?=
-SPHINXBUILD   ?= sphinx-build
-SOURCEDIR     = .
-BUILDDIR      = _build
-DESTDIR       = final
-
-ifeq ($(shell if which $(SPHINXBUILD) >/dev/null 2>&1; then echo 1; else echo 0; fi),0)
-$(error "The '$(SPHINXBUILD)' command was not found. Make sure you have Sphinx installed")
-endif
-
-# Put it first so that "make" without argument is like "make help".
-help:
-	@$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)
-
-.PHONY: help Makefile.sphinx clean publish
-
-publish: Makefile.sphinx html singlehtml
-	rm -rf $(BUILDDIR)/$(DESTDIR)/
-	mkdir -p $(BUILDDIR)/$(DESTDIR)/
-	cp -r $(BUILDDIR)/html/* $(BUILDDIR)/$(DESTDIR)/
-	cp $(BUILDDIR)/singlehtml/index.html $(BUILDDIR)/$(DESTDIR)/singleindex.html
-	sed -i -e 's@index.html#@singleindex.html#@g' $(BUILDDIR)/$(DESTDIR)/singleindex.html
-
-clean:
-	@rm -rf $(BUILDDIR)
-
-# Catch-all target: route all unknown targets to Sphinx using the new
-# "make mode" option.  $(O) is meant as a shortcut for $(SPHINXOPTS).
-%: Makefile.sphinx
-	@$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)

README.OE-Core.md

@@ -16,9 +16,13 @@ which can be found at:
 Contributing
 ------------
 
-Please refer to
-https://www.openembedded.org/wiki/How_to_submit_a_patch_to_OpenEmbedded
-for guidelines on how to submit patches.
+Please refer to our contributor guide here: https://docs.yoctoproject.org/dev/contributor-guide/
+for full details on how to submit changes.
+
+As a quick guide, patches should be sent to openembedded-core@lists.openembedded.org
+The git command to do that would be:
+
+     git send-email -M -1 --to openembedded-core@lists.openembedded.org
 
 Mailing list:
 

SECURITY.md

@@ -0,0 +1,22 @@
+How to Report a Potential Vulnerability?
+========================================
+
+If you would like to report a public issue (for example, one with a released
+CVE number), please report it using the
+[https://bugzilla.yoctoproject.org/enter_bug.cgi?product=Security Security Bugzilla]
+
+If you are dealing with a not-yet released or urgent issue, please send a
+message to security AT yoctoproject DOT org, including as many details as
+possible: the layer or software module affected, the recipe and its version,
+and any example code, if available.
+
+Branches maintained with security fixes
+---------------------------------------
+
+See [https://wiki.yoctoproject.org/wiki/Stable_Release_and_LTS Stable release and LTS]
+for detailed info regarding the policies and maintenance of Stable branches.
+
+The [https://wiki.yoctoproject.org/wiki/Releases Release page] contains a list of all
+releases of the Yocto Project. Versions in grey are no longer actively maintained with
+security patches, but well-tested patches may still be accepted for them for
+significant issues.

bitbake/README

@@ -18,16 +18,19 @@ Bitbake requires Python version 3.8 or newer.
 Contributing
 ------------
 
-Please refer to
-https://www.openembedded.org/wiki/How_to_submit_a_patch_to_OpenEmbedded
-for guidelines on how to submit patches, just note that the latter documentation is intended
-for OpenEmbedded (and its core) not bitbake patches (bitbake-devel@lists.openembedded.org)
-but in general main guidelines apply. Once the commit(s) have been created, the way to send
-the patch is through git-send-email. For example, to send the last commit (HEAD) on current
-branch, type:
+Please refer to our contributor guide here: https://docs.yoctoproject.org/dev/contributor-guide/
+for full details on how to submit changes.
+
+As a quick guide, patches should be sent to bitbake-devel@lists.openembedded.org
+The git command to do that would be:
 
     git send-email -M -1 --to bitbake-devel@lists.openembedded.org
 
+If you're sending a patch related to the BitBake manual, make sure you copy
+the Yocto Project documentation mailing list:
+
+    git send-email -M -1 --to bitbake-devel@lists.openembedded.org --cc docs@lists.yoctoproject.org
+
 Mailing list:
 
     https://lists.openembedded.org/g/bitbake-devel

bitbake/SECURITY.md

@@ -0,0 +1,24 @@
+How to Report a Potential Vulnerability?
+========================================
+
+If you would like to report a public issue (for example, one with a released
+CVE number), please report it using the
+[https://bugzilla.yoctoproject.org/enter_bug.cgi?product=Security Security Bugzilla].
+If you have a patch ready, submit it following the same procedure as any other
+patch as described in README.md.
+
+If you are dealing with a not-yet released or urgent issue, please send a
+message to security AT yoctoproject DOT org, including as many details as
+possible: the layer or software module affected, the recipe and its version,
+and any example code, if available.
+
+Branches maintained with security fixes
+---------------------------------------
+
+See [https://wiki.yoctoproject.org/wiki/Stable_Release_and_LTS Stable release and LTS]
+for detailed info regarding the policies and maintenance of Stable branches.
+
+The [https://wiki.yoctoproject.org/wiki/Releases Release page] contains a list of all
+releases of the Yocto Project. Versions in grey are no longer actively maintained with
+security patches, but well-tested patches may still be accepted for them for
+significant issues.

bitbake/bin/bitbake

@@ -27,7 +27,7 @@ from bb.main import bitbake_main, BitBakeConfigParameters, BBMainException
 
 bb.utils.check_system_locale()
 
-__version__ = "2.4.0"
+__version__ = "2.6.0"
 
 if __name__ == "__main__":
     if __version__ != bb.__version__:

bitbake/bin/bitbake-getvar

@@ -26,26 +26,35 @@ if __name__ == "__main__":
     parser.add_argument('-f', '--flag', help='Specify a variable flag to query (with --value)', default=None)
     parser.add_argument('--value', help='Only report the value, no history and no variable name', action="store_true")
     parser.add_argument('-q', '--quiet', help='Silence bitbake server logging', action="store_true")
+    parser.add_argument('--ignore-undefined', help='Suppress any errors related to undefined variables', action="store_true")
     args = parser.parse_args()
 
-    if args.unexpand and not args.value:
-        print("--unexpand only makes sense with --value")
-        sys.exit(1)
+    if not args.value:
+        if args.unexpand:
+            sys.exit("--unexpand only makes sense with --value")
 
-    if args.flag and not args.value:
-        print("--flag only makes sense with --value")
-        sys.exit(1)
+        if args.flag:
+            sys.exit("--flag only makes sense with --value")
 
-    with bb.tinfoil.Tinfoil(tracking=True, setup_logging=not args.quiet) as tinfoil:
+    quiet = args.quiet or args.value
+    with bb.tinfoil.Tinfoil(tracking=True, setup_logging=not quiet) as tinfoil:
         if args.recipe:
-            tinfoil.prepare(quiet=2)
+            tinfoil.prepare(quiet=3 if quiet else 2)
             d = tinfoil.parse_recipe(args.recipe)
         else:
             tinfoil.prepare(quiet=2, config_only=True)
             d = tinfoil.config_data
+
+        value = None
         if args.flag:
-            print(str(d.getVarFlag(args.variable, args.flag, expand=(not args.unexpand))))
-        elif args.value:
-            print(str(d.getVar(args.variable, expand=(not args.unexpand))))
+            value = d.getVarFlag(args.variable, args.flag, expand=not args.unexpand)
+            if value is None and not args.ignore_undefined:
+                sys.exit(f"The flag '{args.flag}' is not defined for variable '{args.variable}'")
+        else:
+            value = d.getVar(args.variable, expand=not args.unexpand)
+            if value is None and not args.ignore_undefined:
+                sys.exit(f"The variable '{args.variable}' is not defined")
+        if args.value:
+            print(str(value if value is not None else ""))
         else:
             bb.data.emit_var(args.variable, d=d, all=True)

bitbake/bin/bitbake-hashclient

@@ -56,25 +56,24 @@ def main():
             nonlocal missed_hashes
             nonlocal max_time
 
-            client = hashserv.create_client(args.address)
+            with hashserv.create_client(args.address) as client:
+                for i in range(args.requests):
+                    taskhash = hashlib.sha256()
+                    taskhash.update(args.taskhash_seed.encode('utf-8'))
+                    taskhash.update(str(i).encode('utf-8'))
 
-            for i in range(args.requests):
-                taskhash = hashlib.sha256()
-                taskhash.update(args.taskhash_seed.encode('utf-8'))
-                taskhash.update(str(i).encode('utf-8'))
+                    start_time = time.perf_counter()
+                    l = client.get_unihash(METHOD, taskhash.hexdigest())
+                    elapsed = time.perf_counter() - start_time
 
-                start_time = time.perf_counter()
-                l = client.get_unihash(METHOD, taskhash.hexdigest())
-                elapsed = time.perf_counter() - start_time
+                    with lock:
+                        if l:
+                            found_hashes += 1
+                        else:
+                            missed_hashes += 1
 
-                with lock:
-                    if l:
-                        found_hashes += 1
-                    else:
-                        missed_hashes += 1
-
-                    max_time = max(elapsed, max_time)
-                    pbar.update()
+                        max_time = max(elapsed, max_time)
+                        pbar.update()
 
         max_time = 0
         found_hashes = 0
@@ -113,6 +112,19 @@ def main():
                     with lock:
                         pbar.update()
 
+    def handle_remove(args, client):
+        where = {k: v for k, v in args.where}
+        if where:
+            result = client.remove(where)
+            print("Removed %d row(s)" % (result["count"]))
+        else:
+            print("No query specified")
+
+    def handle_clean_unused(args, client):
+        result = client.clean_unused(args.max_age)
+        print("Removed %d rows" % (result["count"]))
+        return 0
+
     parser = argparse.ArgumentParser(description='Hash Equivalence Client')
     parser.add_argument('--address', default=DEFAULT_ADDRESS, help='Server address (default "%(default)s")')
     parser.add_argument('--log', default='WARNING', help='Set logging level')
@@ -137,6 +149,15 @@ def main():
                                help='Include string in outhash')
     stress_parser.set_defaults(func=handle_stress)
 
+    remove_parser = subparsers.add_parser('remove', help="Remove hash entries")
+    remove_parser.add_argument("--where", "-w", metavar="KEY VALUE", nargs=2, action="append", default=[],
+                               help="Remove entries from table where KEY == VALUE")
+    remove_parser.set_defaults(func=handle_remove)
+
+    clean_unused_parser = subparsers.add_parser('clean-unused', help="Remove unused database entries")
+    clean_unused_parser.add_argument("max_age", metavar="SECONDS", type=int, help="Remove unused entries older than SECONDS old")
+    clean_unused_parser.set_defaults(func=handle_clean_unused)
+
     args = parser.parse_args()
 
     logger = logging.getLogger('hashserv')
@@ -152,9 +173,8 @@ def main():
 
     func = getattr(args, 'func', None)
     if func:
-        client = hashserv.create_client(args.address)
-
-        return func(args, client)
+        with hashserv.create_client(args.address) as client:
+            return func(args, client)
 
     return 0
 

bitbake/bin/bitbake-worker

@@ -91,19 +91,19 @@ def worker_fire_prepickled(event):
 worker_thread_exit = False
 
 def worker_flush(worker_queue):
-    worker_queue_int = b""
+    worker_queue_int = bytearray()
     global worker_pipe, worker_thread_exit
 
     while True:
         try:
-            worker_queue_int = worker_queue_int + worker_queue.get(True, 1)
+            worker_queue_int.extend(worker_queue.get(True, 1))
         except queue.Empty:
             pass
         while (worker_queue_int or not worker_queue.empty()):
             try:
                 (_, ready, _) = select.select([], [worker_pipe], [], 1)
                 if not worker_queue.empty():
-                    worker_queue_int = worker_queue_int + worker_queue.get()
+                    worker_queue_int.extend(worker_queue.get())
                 written = os.write(worker_pipe, worker_queue_int)
                 worker_queue_int = worker_queue_int[written:]
             except (IOError, OSError) as e:
@@ -346,12 +346,12 @@ class runQueueWorkerPipe():
         if pipeout:
             pipeout.close()
         bb.utils.nonblockingfd(self.input)
-        self.queue = b""
+        self.queue = bytearray()
 
     def read(self):
         start = len(self.queue)
         try:
-            self.queue = self.queue + (self.input.read(102400) or b"")
+            self.queue.extend(self.input.read(102400) or b"")
         except (OSError, IOError) as e:
             if e.errno != errno.EAGAIN:
                 raise
@@ -379,7 +379,7 @@ class BitbakeWorker(object):
     def __init__(self, din):
         self.input = din
         bb.utils.nonblockingfd(self.input)
-        self.queue = b""
+        self.queue = bytearray()
         self.cookercfg = None
         self.databuilder = None
         self.data = None
@@ -413,7 +413,7 @@ class BitbakeWorker(object):
                     if len(r) == 0:
                         # EOF on pipe, server must have terminated
                         self.sigterm_exception(signal.SIGTERM, None)
-                    self.queue = self.queue + r
+                    self.queue.extend(r)
                 except (OSError, IOError):
                     pass
             if len(self.queue):

bitbake/contrib/vim/indent/bitbake.vim

@@ -40,7 +40,7 @@ set cpo&vim
 
 let s:maxoff = 50	" maximum number of lines to look backwards for ()
 
-function GetPythonIndent(lnum)
+function! GetBBPythonIndent(lnum)
 
   " If this line is explicitly joined: If the previous line was also joined,
   " line it up with that one, otherwise add two 'shiftwidth'
@@ -257,7 +257,7 @@ let b:did_indent = 1
 setlocal indentkeys+=0\"
 
 
-function BitbakeIndent(lnum)
+function! BitbakeIndent(lnum)
     if !has('syntax_items')
         return -1
     endif
@@ -315,7 +315,7 @@ function BitbakeIndent(lnum)
     endif
 
     if index(["bbPyDefRegion", "bbPyFuncRegion"], name) != -1
-        let ret = GetPythonIndent(a:lnum)
+        let ret = GetBBPythonIndent(a:lnum)
         " Should normally always be indented by at least one shiftwidth; but allow
         " return of -1 (defer to autoindent) or -2 (force indent to 0)
         if ret == 0

bitbake/doc/README

@@ -47,8 +47,8 @@ To install all required packages run:
 
 To build the documentation locally, run:
 
- $ cd documentation
- $ make -f Makefile.sphinx html
+ $ cd doc
+ $ make html
 
 The resulting HTML index page will be _build/html/index.html, and you
 can browse your own copy of the locally generated documentation with

bitbake/doc/bitbake-user-manual/bitbake-user-manual-fetching.rst

@@ -476,6 +476,14 @@ Here are some example URLs::
    easy to share metadata without removing passwords. SSH keys, ``~/.netrc``
    and ``~/.ssh/config`` files can be used as alternatives.
 
+Using tags with the git fetcher may cause surprising behaviour. Bitbake needs to
+resolve the tag to a specific revision and to do that, it has to connect to and use
+the upstream repository. This is because the revision the tags point at can change and
+we've seen cases of this happening in well known public repositories. This can mean
+many more network connections than expected and recipes may be reparsed at every build.
+Source mirrors will also be bypassed as the upstream repository is the only source
+of truth to resolve the revision accurately. For these reasons, whilst the fetcher
+can support tags, we recommend being specific about revisions in recipes.
 
 .. _gitsm-fetcher:
 
@@ -688,6 +696,41 @@ Here is an example URL::
 
 It can also be used when setting mirrors definitions using the :term:`PREMIRRORS` variable.
 
+.. _gcp-fetcher:
+
+GCP Fetcher (``gs://``)
+--------------------------
+
+This submodule fetches data from a
+`Google Cloud Storage Bucket <https://cloud.google.com/storage/docs/buckets>`__.
+It uses the `Google Cloud Storage Python Client <https://cloud.google.com/python/docs/reference/storage/latest>`__
+to check the status of objects in the bucket and download them.
+The use of the Python client makes it substantially faster than using command
+line tools such as gsutil.
+
+The fetcher requires the Google Cloud Storage Python Client to be installed, along
+with the gsutil tool.
+
+The fetcher requires that the machine has valid credentials for accessing the
+chosen bucket. Instructions for authentication can be found in the
+`Google Cloud documentation <https://cloud.google.com/docs/authentication/provide-credentials-adc#local-dev>`__.
+
+If it used from the OpenEmbedded build system, the fetcher can be used for
+fetching sstate artifacts from a GCS bucket by specifying the
+``SSTATE_MIRRORS`` variable as shown below::
+
+   SSTATE_MIRRORS ?= "\
+       file://.* gs://<bucket name>/PATH \
+   "
+
+The fetcher can also be used in recipes::
+
+   SRC_URI = "gs://<bucket name>/<foo_container>/<bar_file>"
+
+However, the checksum of the file should be also be provided::
+
+   SRC_URI[sha256sum] = "<sha256 string>"
+
 .. _crate-fetcher:
 
 Crate Fetcher (``crate://``)
@@ -791,6 +834,8 @@ Fetch submodules also exist for the following:
 
 -  OSC (``osc://``)
 
+-  S3 (``s3://``)
+
 -  Secure FTP (``sftp://``)
 
 -  Secure Shell (``ssh://``)

bitbake/lib/bb/__init__.py

@@ -9,12 +9,19 @@
 # SPDX-License-Identifier: GPL-2.0-only
 #
 
-__version__ = "2.4.0"
+__version__ = "2.6.0"
 
 import sys
 if sys.version_info < (3, 8, 0):
     raise RuntimeError("Sorry, python 3.8.0 or later is required for this version of bitbake")
 
+if sys.version_info < (3, 10, 0):
+    # With python 3.8 and 3.9, we see errors of "libgcc_s.so.1 must be installed for pthread_cancel to work"
+    # https://stackoverflow.com/questions/64797838/libgcc-s-so-1-must-be-installed-for-pthread-cancel-to-work
+    # https://bugs.ams1.psf.io/issue42888
+    # so ensure libgcc_s is loaded early on
+    import ctypes
+    libgcc_s = ctypes.CDLL('libgcc_s.so.1')
 
 class BBHandledException(Exception):
     """

bitbake/lib/bb/acl.py

@@ -0,0 +1,215 @@
+#! /usr/bin/env python3
+#
+# Copyright 2023 by Garmin Ltd. or its subsidiaries
+#
+# SPDX-License-Identifier: MIT
+
+
+import sys
+import ctypes
+import os
+import errno
+import pwd
+import grp
+
+libacl = ctypes.CDLL("libacl.so.1", use_errno=True)
+
+
+ACL_TYPE_ACCESS = 0x8000
+ACL_TYPE_DEFAULT = 0x4000
+
+ACL_FIRST_ENTRY = 0
+ACL_NEXT_ENTRY = 1
+
+ACL_UNDEFINED_TAG = 0x00
+ACL_USER_OBJ = 0x01
+ACL_USER = 0x02
+ACL_GROUP_OBJ = 0x04
+ACL_GROUP = 0x08
+ACL_MASK = 0x10
+ACL_OTHER = 0x20
+
+ACL_READ = 0x04
+ACL_WRITE = 0x02
+ACL_EXECUTE = 0x01
+
+acl_t = ctypes.c_void_p
+acl_entry_t = ctypes.c_void_p
+acl_permset_t = ctypes.c_void_p
+acl_perm_t = ctypes.c_uint
+
+acl_tag_t = ctypes.c_int
+
+libacl.acl_free.argtypes = [acl_t]
+
+
+def acl_free(acl):
+    libacl.acl_free(acl)
+
+
+libacl.acl_get_file.restype = acl_t
+libacl.acl_get_file.argtypes = [ctypes.c_char_p, ctypes.c_uint]
+
+
+def acl_get_file(path, typ):
+    acl = libacl.acl_get_file(os.fsencode(path), typ)
+    if acl is None:
+        err = ctypes.get_errno()
+        raise OSError(err, os.strerror(err), str(path))
+
+    return acl
+
+
+libacl.acl_get_entry.argtypes = [acl_t, ctypes.c_int, ctypes.c_void_p]
+
+
+def acl_get_entry(acl, entry_id):
+    entry = acl_entry_t()
+    ret = libacl.acl_get_entry(acl, entry_id, ctypes.byref(entry))
+    if ret < 0:
+        err = ctypes.get_errno()
+        raise OSError(err, os.strerror(err))
+
+    if ret == 0:
+        return None
+
+    return entry
+
+
+libacl.acl_get_tag_type.argtypes = [acl_entry_t, ctypes.c_void_p]
+
+
+def acl_get_tag_type(entry_d):
+    tag = acl_tag_t()
+    ret = libacl.acl_get_tag_type(entry_d, ctypes.byref(tag))
+    if ret < 0:
+        err = ctypes.get_errno()
+        raise OSError(err, os.strerror(err))
+    return tag.value
+
+
+libacl.acl_get_qualifier.restype = ctypes.c_void_p
+libacl.acl_get_qualifier.argtypes = [acl_entry_t]
+
+
+def acl_get_qualifier(entry_d):
+    ret = libacl.acl_get_qualifier(entry_d)
+    if ret is None:
+        err = ctypes.get_errno()
+        raise OSError(err, os.strerror(err))
+    return ctypes.c_void_p(ret)
+
+
+libacl.acl_get_permset.argtypes = [acl_entry_t, ctypes.c_void_p]
+
+
+def acl_get_permset(entry_d):
+    permset = acl_permset_t()
+    ret = libacl.acl_get_permset(entry_d, ctypes.byref(permset))
+    if ret < 0:
+        err = ctypes.get_errno()
+        raise OSError(err, os.strerror(err))
+
+    return permset
+
+
+libacl.acl_get_perm.argtypes = [acl_permset_t, acl_perm_t]
+
+
+def acl_get_perm(permset_d, perm):
+    ret = libacl.acl_get_perm(permset_d, perm)
+    if ret < 0:
+        err = ctypes.get_errno()
+        raise OSError(err, os.strerror(err))
+    return bool(ret)
+
+
+class Entry(object):
+    def __init__(self, tag, qualifier, mode):
+        self.tag = tag
+        self.qualifier = qualifier
+        self.mode = mode
+
+    def __str__(self):
+        typ = ""
+        qual = ""
+        if self.tag == ACL_USER:
+            typ = "user"
+            qual = pwd.getpwuid(self.qualifier).pw_name
+        elif self.tag == ACL_GROUP:
+            typ = "group"
+            qual = grp.getgrgid(self.qualifier).gr_name
+        elif self.tag == ACL_USER_OBJ:
+            typ = "user"
+        elif self.tag == ACL_GROUP_OBJ:
+            typ = "group"
+        elif self.tag == ACL_MASK:
+            typ = "mask"
+        elif self.tag == ACL_OTHER:
+            typ = "other"
+
+        r = "r" if self.mode & ACL_READ else "-"
+        w = "w" if self.mode & ACL_WRITE else "-"
+        x = "x" if self.mode & ACL_EXECUTE else "-"
+
+        return f"{typ}:{qual}:{r}{w}{x}"
+
+
+class ACL(object):
+    def __init__(self, acl):
+        self.acl = acl
+
+    def __del__(self):
+        acl_free(self.acl)
+
+    def entries(self):
+        entry_id = ACL_FIRST_ENTRY
+        while True:
+            entry = acl_get_entry(self.acl, entry_id)
+            if entry is None:
+                break
+
+            permset = acl_get_permset(entry)
+
+            mode = 0
+            for m in (ACL_READ, ACL_WRITE, ACL_EXECUTE):
+                if acl_get_perm(permset, m):
+                    mode |= m
+
+            qualifier = None
+            tag = acl_get_tag_type(entry)
+
+            if tag == ACL_USER or tag == ACL_GROUP:
+                qual = acl_get_qualifier(entry)
+                qualifier = ctypes.cast(qual, ctypes.POINTER(ctypes.c_int))[0]
+
+            yield Entry(tag, qualifier, mode)
+
+            entry_id = ACL_NEXT_ENTRY
+
+    @classmethod
+    def from_path(cls, path, typ):
+        acl = acl_get_file(path, typ)
+        return cls(acl)
+
+
+def main():
+    import argparse
+    import pwd
+    import grp
+    from pathlib import Path
+
+    parser = argparse.ArgumentParser()
+    parser.add_argument("path", help="File Path", type=Path)
+
+    args = parser.parse_args()
+
+    acl = ACL.from_path(args.path, ACL_TYPE_ACCESS)
+    for entry in acl.entries():
+        print(str(entry))
+
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())

bitbake/lib/bb/asyncrpc/client.py

@@ -126,6 +126,12 @@ class AsyncClient(object):
             {'ping': {}}
         )
 
+    async def __aenter__(self):
+        return self
+
+    async def __aexit__(self, exc_type, exc_value, traceback):
+        await self.close()
+
 
 class Client(object):
     def __init__(self):
@@ -176,3 +182,10 @@ class Client(object):
         if sys.version_info >= (3, 6):
             self.loop.run_until_complete(self.loop.shutdown_asyncgens())
         self.loop.close()
+
+    def __enter__(self):
+        return self
+
+    def __exit__(self, exc_type, exc_value, traceback):
+        self.close()
+        return False

bitbake/lib/bb/codeparser.py

@@ -82,7 +82,7 @@ def add_module_functions(fn, functions, namespace):
                 execs.remove(e)
                 execs.add(namespace + "." + e)
         modulecode_deps[name] = [parser.references.copy(), execs, parser.var_execs.copy(), parser.contains.copy()]
-        #bb.warn("%s: %s\nRefs:%s Execs: %s %s %s" % (name, src, parser.references, parser.execs, parser.var_execs, parser.contains))
+        #bb.warn("%s: %s\nRefs:%s Execs: %s %s %s" % (name, fn, parser.references, parser.execs, parser.var_execs, parser.contains))
 
 def update_module_dependencies(d):
     for mod in modulecode_deps:
@@ -255,8 +255,8 @@ class PythonParser():
     def visit_Call(self, node):
         name = self.called_node_name(node.func)
         if name and (name.endswith(self.getvars) or name.endswith(self.getvarflags) or name in self.containsfuncs or name in self.containsanyfuncs):
-            if isinstance(node.args[0], ast.Str):
-                varname = node.args[0].s
+            if isinstance(node.args[0], ast.Constant) and isinstance(node.args[0].value, str):
+                varname = node.args[0].value
                 if name in self.containsfuncs and isinstance(node.args[1], ast.Str):
                     if varname not in self.contains:
                         self.contains[varname] = set()

bitbake/lib/bb/command.py

@@ -65,7 +65,7 @@ class Command:
         command = commandline.pop(0)
 
         # Ensure cooker is ready for commands
-        if command != "updateConfig" and command != "setFeatures":
+        if command not in ["updateConfig", "setFeatures", "ping"]:
             try:
                 self.cooker.init_configdata()
                 if not self.remotedatastores:
@@ -85,7 +85,6 @@ class Command:
                 if not hasattr(command_method, 'readonly') or not getattr(command_method, 'readonly'):
                     return None, "Not able to execute not readonly commands in readonly mode"
             try:
-                self.cooker.process_inotify_updates_apply()
                 if getattr(command_method, 'needconfig', True):
                     self.cooker.updateCacheSync()
                 result = command_method(self, commandline)
@@ -109,7 +108,6 @@ class Command:
 
     def runAsyncCommand(self, _, process_server, halt):
         try:
-            self.cooker.process_inotify_updates_apply()
             if self.cooker.state in (bb.cooker.state.error, bb.cooker.state.shutdown, bb.cooker.state.forceshutdown):
                 # updateCache will trigger a shutdown of the parser
                 # and then raise BBHandledException triggering an exit
@@ -169,6 +167,8 @@ class CommandsSync:
         Allow a UI to check the server is still alive
         """
         return "Still alive!"
+    ping.needconfig = False
+    ping.readonly = True
 
     def stateShutdown(self, command, params):
         """
@@ -307,6 +307,11 @@ class CommandsSync:
         return ret
     getLayerPriorities.readonly = True
 
+    def revalidateCaches(self, command, params):
+        """Called by UI clients when metadata may have changed"""
+        command.cooker.revalidateCaches()
+    parseConfiguration.needconfig = False
+
     def getRecipes(self, command, params):
         try:
             mc = params[0]
@@ -545,8 +550,8 @@ class CommandsSync:
         and return a datastore object representing the environment
         for the recipe.
         """
-        fn = params[0]
-        mc = bb.runqueue.mc_from_tid(fn)
+        virtualfn = params[0]
+        (fn, cls, mc) = bb.cache.virtualfn2realfn(virtualfn)
         appends = params[1]
         appendlist = params[2]
         if len(params) > 3:
@@ -569,10 +574,10 @@ class CommandsSync:
         if config_data:
             # We have to use a different function here if we're passing in a datastore
             # NOTE: we took a copy above, so we don't do it here again
-            envdata = command.cooker.databuilder._parse_recipe(config_data, fn, appendfiles, mc, layername)['']
+            envdata = command.cooker.databuilder._parse_recipe(config_data, fn, appendfiles, mc, layername)[cls]
         else:
             # Use the standard path
-            envdata = command.cooker.databuilder.parseRecipe(fn, appendfiles, layername)
+            envdata = command.cooker.databuilder.parseRecipe(virtualfn, appendfiles, layername)
         idx = command.remotedatastores.store(envdata)
         return DataStoreConnectionHandle(idx)
     parseRecipeFile.readonly = True
@@ -776,3 +781,9 @@ class CommandsAsync:
         bb.event.fire(bb.event.FindSigInfoResult(res), command.cooker.databuilder.mcdata[mc])
         command.finishAsyncCommand()
     findSigInfo.needcache = False
+
+    def getTaskSignatures(self, command, params):
+        res = command.cooker.getTaskSignatures(params[0], params[1])
+        bb.event.fire(bb.event.GetTaskSignatureResult(res), command.cooker.data)
+        command.finishAsyncCommand()
+    getTaskSignatures.needcache = True

bitbake/lib/bb/cooker.py

@@ -22,7 +22,6 @@ from bb import utils, data, parse, event, cache, providers, taskdata, runqueue,
 import queue
 import signal
 import prserv.serv
-import pyinotify
 import json
 import pickle
 import codecs
@@ -173,17 +172,9 @@ class BBCooker:
             self.waitIdle = server.wait_for_idle
 
         bb.debug(1, "BBCooker starting %s" % time.time())
-        sys.stdout.flush()
 
-        self.configwatcher = None
-        self.confignotifier = None
-
-        self.watchmask = pyinotify.IN_CLOSE_WRITE | pyinotify.IN_CREATE | pyinotify.IN_DELETE | \
-                         pyinotify.IN_DELETE_SELF | pyinotify.IN_MODIFY | pyinotify.IN_MOVE_SELF | \
-                         pyinotify.IN_MOVED_FROM | pyinotify.IN_MOVED_TO
-
-        self.watcher = None
-        self.notifier = None
+        self.configwatched = {}
+        self.parsewatched = {}
 
         # If being called by something like tinfoil, we need to clean cached data
         # which may now be invalid
@@ -194,8 +185,6 @@ class BBCooker:
         self.hashserv = None
         self.hashservaddr = None
 
-        self.inotify_modified_files = []
-
         # TOSTOP must not be set or our children will hang when they output
         try:
             fd = sys.stdout.fileno()
@@ -219,53 +208,13 @@ class BBCooker:
         signal.signal(signal.SIGHUP, self.sigterm_exception)
 
         bb.debug(1, "BBCooker startup complete %s" % time.time())
-        sys.stdout.flush()
-
-        self.inotify_threadlock = threading.Lock()
 
     def init_configdata(self):
         if not hasattr(self, "data"):
             self.initConfigurationData()
             bb.debug(1, "BBCooker parsed base configuration %s" % time.time())
-            sys.stdout.flush()
             self.handlePRServ()
 
-    def setupConfigWatcher(self):
-        with bb.utils.lock_timeout(self.inotify_threadlock):
-            if self.configwatcher:
-                self.configwatcher.close()
-                self.confignotifier = None
-                self.configwatcher = None
-            self.configwatcher = pyinotify.WatchManager()
-            self.configwatcher.bbseen = set()
-            self.configwatcher.bbwatchedfiles = set()
-            self.confignotifier = pyinotify.Notifier(self.configwatcher, self.config_notifications)
-
-    def setupParserWatcher(self):
-        with bb.utils.lock_timeout(self.inotify_threadlock):
-            if self.watcher:
-                self.watcher.close()
-                self.notifier = None
-                self.watcher = None
-            self.watcher = pyinotify.WatchManager()
-            self.watcher.bbseen = set()
-            self.watcher.bbwatchedfiles = set()
-            self.notifier = pyinotify.Notifier(self.watcher, self.notifications)
-
-    def process_inotify_updates(self):
-        with bb.utils.lock_timeout(self.inotify_threadlock):
-            for n in [self.confignotifier, self.notifier]:
-                if n and n.check_events(timeout=0):
-                    # read notified events and enqueue them
-                    n.read_events()
-
-    def process_inotify_updates_apply(self):
-        with bb.utils.lock_timeout(self.inotify_threadlock):
-            for n in [self.confignotifier, self.notifier]:
-                if n and n.check_events(timeout=0):
-                    n.read_events()
-                    n.process_events()
-
     def _baseconfig_set(self, value):
         if value and not self.baseconfig_valid:
             bb.server.process.serverlog("Base config valid")
@@ -280,88 +229,16 @@ class BBCooker:
             bb.server.process.serverlog("Parse cache invalidated")
         self.parsecache_valid = value
 
-    def config_notifications(self, event):
-        if event.maskname == "IN_Q_OVERFLOW":
-            bb.warn("inotify event queue overflowed, invalidating caches.")
-            self._parsecache_set(False)
-            self._baseconfig_set(False)
-            bb.parse.clear_cache()
-            return
-        if not event.pathname in self.configwatcher.bbwatchedfiles:
-            return
-        if "IN_ISDIR" in event.maskname:
-            if "IN_CREATE" in event.maskname or "IN_DELETE" in event.maskname:
-                if event.pathname in self.configwatcher.bbseen:
-                    self.configwatcher.bbseen.remove(event.pathname)
-                # Could remove all entries starting with the directory but for now...
-                bb.parse.clear_cache()
-        if not event.pathname in self.inotify_modified_files:
-            self.inotify_modified_files.append(event.pathname)
-        self._baseconfig_set(False)
+    def add_filewatch(self, deps, configwatcher=False):
+        if configwatcher:
+            watcher = self.configwatched
+        else:
+            watcher = self.parsewatched
 
-    def notifications(self, event):
-        if event.maskname == "IN_Q_OVERFLOW":
-            bb.warn("inotify event queue overflowed, invalidating caches.")
-            self._parsecache_set(False)
-            bb.parse.clear_cache()
-            return
-        if event.pathname.endswith("bitbake-cookerdaemon.log") \
-                or event.pathname.endswith("bitbake.lock"):
-            return
-        if "IN_ISDIR" in event.maskname:
-            if "IN_CREATE" in event.maskname or "IN_DELETE" in event.maskname:
-                if event.pathname in self.watcher.bbseen:
-                    self.watcher.bbseen.remove(event.pathname)
-                # Could remove all entries starting with the directory but for now...
-                bb.parse.clear_cache()
-        if not event.pathname in self.inotify_modified_files:
-            self.inotify_modified_files.append(event.pathname)
-        self._parsecache_set(False)
-
-    def add_filewatch(self, deps, watcher=None, dirs=False):
-        if not watcher:
-            watcher = self.watcher
         for i in deps:
-            watcher.bbwatchedfiles.add(i[0])
-            if dirs:
-                f = i[0]
-            else:
-                f = os.path.dirname(i[0])
-            if f in watcher.bbseen:
-                continue
-            watcher.bbseen.add(f)
-            watchtarget = None
-            while True:
-                # We try and add watches for files that don't exist but if they did, would influence
-                # the parser. The parent directory of these files may not exist, in which case we need
-                # to watch any parent that does exist for changes.
-                try:
-                    watcher.add_watch(f, self.watchmask, quiet=False)
-                    if watchtarget:
-                        watcher.bbwatchedfiles.add(watchtarget)
-                    break
-                except pyinotify.WatchManagerError as e:
-                    if 'ENOENT' in str(e):
-                        watchtarget = f
-                        f = os.path.dirname(f)
-                        if f in watcher.bbseen:
-                            break
-                        watcher.bbseen.add(f)
-                        continue
-                    if 'ENOSPC' in str(e):
-                        providerlog.error("No space left on device or exceeds fs.inotify.max_user_watches?")
-                        providerlog.error("To check max_user_watches: sysctl -n fs.inotify.max_user_watches.")
-                        providerlog.error("To modify max_user_watches: sysctl -n -w fs.inotify.max_user_watches=<value>.")
-                        providerlog.error("Root privilege is required to modify max_user_watches.")
-                    raise
-
-    def handle_inotify_updates(self):
-        # reload files for which we got notifications
-        for p in self.inotify_modified_files:
-            bb.parse.update_cache(p)
-            if p in bb.parse.BBHandler.cached_statements:
-                del bb.parse.BBHandler.cached_statements[p]
-        self.inotify_modified_files = []
+            f = i[0]
+            mtime = i[1]
+            watcher[f] = mtime
 
     def sigterm_exception(self, signum, stackframe):
         if signum == signal.SIGTERM:
@@ -392,8 +269,7 @@ class BBCooker:
             if mod not in self.orig_sysmodules:
                 del sys.modules[mod]
 
-        self.handle_inotify_updates()
-        self.setupConfigWatcher()
+        self.configwatched = {}
 
         # Need to preserve BB_CONSOLELOG over resets
         consolelog = None
@@ -436,7 +312,7 @@ class BBCooker:
             self.disableDataTracking()
 
         for mc in self.databuilder.mcdata.values():
-            self.add_filewatch(mc.getVar("__base_depends", False), self.configwatcher)
+            self.add_filewatch(mc.getVar("__base_depends", False), configwatcher=True)
 
         self._baseconfig_set(True)
         self._parsecache_set(False)
@@ -486,6 +362,29 @@ class BBCooker:
         if hasattr(self, "data"):
             self.data.disableTracking()
 
+    def revalidateCaches(self):
+        bb.parse.clear_cache()
+
+        clean = True
+        for f in self.configwatched:
+            if not bb.parse.check_mtime(f, self.configwatched[f]):
+                bb.server.process.serverlog("Found %s changed, invalid cache" % f)
+                self._baseconfig_set(False)
+                self._parsecache_set(False)
+                clean = False
+                break
+
+        if clean:
+            for f in self.parsewatched:
+                if not bb.parse.check_mtime(f, self.parsewatched[f]):
+                    bb.server.process.serverlog("Found %s changed, invalid cache" % f)
+                    self._parsecache_set(False)
+                    clean = False
+                    break
+
+        if not clean:
+            bb.parse.BBHandler.cached_statements = {}
+
     def parseConfiguration(self):
         self.updateCacheSync()
 
@@ -566,6 +465,7 @@ class BBCooker:
         # Now update all the variables not in the datastore to match
         self.configuration.env = environment
 
+        self.revalidateCaches()
         if not clean:
             logger.debug("Base environment change, triggering reparse")
             self.reset()
@@ -650,7 +550,7 @@ class BBCooker:
                 raise
         else:
             if not mc in self.databuilder.mcdata:
-                bb.fatal('Not multiconfig named "%s" found' % mc)
+                bb.fatal('No multiconfig named "%s" found' % mc)
             envdata = self.databuilder.mcdata[mc]
             data.expandKeys(envdata)
             parse.ast.runAnonFuncs(envdata)
@@ -1542,6 +1442,37 @@ class BBCooker:
 
         self.idleCallBackRegister(buildFileIdle, rq)
 
+    def getTaskSignatures(self, target, tasks):
+        sig = []
+        getAllTaskSignatures = False
+
+        if not tasks:
+            tasks = ["do_build"]
+            getAllTaskSignatures = True
+
+        for task in tasks:
+            taskdata, runlist = self.buildTaskData(target, task, self.configuration.halt)
+            rq = bb.runqueue.RunQueue(self, self.data, self.recipecaches, taskdata, runlist)
+            rq.rqdata.prepare()
+
+            for l in runlist:
+                mc, pn, taskname, fn = l
+
+                taskdep = rq.rqdata.dataCaches[mc].task_deps[fn]
+                for t in taskdep['tasks']:
+                    if t in taskdep['nostamp'] or "setscene" in t:
+                        continue
+                    tid = bb.runqueue.build_tid(mc, fn, t)
+
+                    if t in task or getAllTaskSignatures:
+                        try:
+                            rq.rqdata.prepare_task_hash(tid)
+                            sig.append([pn, t, rq.rqdata.get_task_unihash(tid)])
+                        except KeyError:
+                            sig.append(self.getTaskSignatures(target, [t])[0])
+
+        return sig
+
     def buildTargets(self, targets, task):
         """
         Attempt to build the targets specified
@@ -1644,8 +1575,6 @@ class BBCooker:
         if self.state == state.running:
             return
 
-        self.handle_inotify_updates()
-
         if not self.baseconfig_valid:
             logger.debug("Reloading base configuration data")
             self.initConfigurationData()
@@ -1667,7 +1596,7 @@ class BBCooker:
 
         if self.state != state.parsing and not self.parsecache_valid:
             bb.server.process.serverlog("Parsing started")
-            self.setupParserWatcher()
+            self.parsewatched = {}
 
             bb.parse.siggen.reset(self.data)
             self.parseConfiguration ()
@@ -1692,9 +1621,9 @@ class BBCooker:
                 total_masked += masked
                 searchdirs |= set(search)
 
-            # Add inotify watches for directories searched for bb/bbappend files
+            # Add mtimes for directories searched for bb/bbappend files
             for dirent in searchdirs:
-                self.add_filewatch([[dirent]], dirs=True)
+                self.add_filewatch([(dirent, bb.parse.cached_mtime_noerror(dirent))])
 
             self.parser = CookerParser(self, mcfilelist, total_masked)
             self._parsecache_set(True)
@@ -1881,7 +1810,7 @@ class CookerCollectFiles(object):
             collectlog.error("no recipe files to build, check your BBPATH and BBFILES?")
             bb.event.fire(CookerExit(), eventdata)
 
-        # We need to track where we look so that we can add inotify watches. There
+        # We need to track where we look so that we can know when the cache is invalid. There
         # is no nice way to do this, this is horrid. We intercept the os.listdir()
         # (or os.scandir() for python 3.6+) calls while we run glob().
         origlistdir = os.listdir

bitbake/lib/bb/data.py

@@ -285,6 +285,7 @@ def build_dependencies(key, keys, mod_funcs, shelldeps, varflagsexcl, ignored_va
             value += "\n_remove of %s" % r
             deps |= r2.references
             deps = deps | (keys & r2.execs)
+            value = handle_contains(value, r2.contains, exclusions, d)
         return value
 
     deps = set()

bitbake/lib/bb/data_smart.py

@@ -16,7 +16,10 @@ BitBake build tools.
 #
 # Based on functions from the base bb module, Copyright 2003 Holger Schurig
 
-import copy, re, sys, traceback
+import builtins
+import copy
+import re
+import sys
 from collections.abc import MutableMapping
 import logging
 import hashlib
@@ -150,19 +153,21 @@ class VariableParse:
             value = utils.better_eval(codeobj, DataContext(self.d), {'d' : self.d})
             return str(value)
 
-
 class DataContext(dict):
+    excluded = set([i for i in dir(builtins) if not i.startswith('_')] + ['oe'])
+
     def __init__(self, metadata, **kwargs):
         self.metadata = metadata
         dict.__init__(self, **kwargs)
         self['d'] = metadata
+        self.context = set(bb.utils.get_context())
 
     def __missing__(self, key):
-        # Skip commonly accessed invalid variables
-        if key in ['bb', 'oe', 'int', 'bool', 'time', 'str', 'os']:
+        if key in self.excluded or key in self.context:
             raise KeyError(key)
+
         value = self.metadata.getVar(key)
-        if value is None or self.metadata.getVarFlag(key, 'func', False):
+        if value is None:
             raise KeyError(key)
         else:
             return value

bitbake/lib/bb/event.py

@@ -857,6 +857,14 @@ class FindSigInfoResult(Event):
         Event.__init__(self)
         self.result = result
 
+class GetTaskSignatureResult(Event):
+    """
+    Event to return results from GetTaskSignatures command
+    """
+    def __init__(self, sig):
+        Event.__init__(self)
+        self.sig = sig
+
 class ParseError(Event):
     """
     Event to indicate parse failed

bitbake/lib/bb/fetch2/__init__.py

@@ -388,7 +388,7 @@ def decodeurl(url):
             if s:
                 if not '=' in s:
                     raise MalformedUrl(url, "The URL: '%s' is invalid: parameter %s does not specify a value (missing '=')" % (url, s))
-                s1, s2 = s.split('=')
+                s1, s2 = s.split('=', 1)
                 p[s1] = s2
 
     return type, host, urllib.parse.unquote(path), user, pswd, p
@@ -753,7 +753,7 @@ def get_autorev(d):
     d.setVar("__BBAUTOREV_SEEN", True)
     return "AUTOINC"
 
-def get_srcrev(d, method_name='sortable_revision'):
+def _get_srcrev(d, method_name='sortable_revision'):
     """
     Return the revision string, usually for use in the version string (PV) of the current package
     Most packages usually only have one SCM so we just pass on the call.
@@ -774,6 +774,7 @@ def get_srcrev(d, method_name='sortable_revision'):
     d.setVar("__BBINSRCREV", True)
 
     scms = []
+    revs = []
     fetcher = Fetch(d.getVar('SRC_URI').split(), d)
     urldata = fetcher.ud
     for u in urldata:
@@ -781,16 +782,19 @@ def get_srcrev(d, method_name='sortable_revision'):
             scms.append(u)
 
     if not scms:
-        raise FetchError("SRCREV was used yet no valid SCM was found in SRC_URI")
+        d.delVar("__BBINSRCREV")
+        return "", revs
+
 
     if len(scms) == 1 and len(urldata[scms[0]].names) == 1:
         autoinc, rev = getattr(urldata[scms[0]].method, method_name)(urldata[scms[0]], d, urldata[scms[0]].names[0])
+        revs.append(rev)
         if len(rev) > 10:
             rev = rev[:10]
         d.delVar("__BBINSRCREV")
         if autoinc:
-            return "AUTOINC+" + rev
-        return rev
+            return "AUTOINC+" + rev, revs
+        return rev, revs
 
     #
     # Mutiple SCMs are in SRC_URI so we resort to SRCREV_FORMAT
@@ -806,6 +810,7 @@ def get_srcrev(d, method_name='sortable_revision'):
         ud = urldata[scm]
         for name in ud.names:
             autoinc, rev = getattr(ud.method, method_name)(ud, d, name)
+            revs.append(rev)
             seenautoinc = seenautoinc or autoinc
             if len(rev) > 10:
                 rev = rev[:10]
@@ -823,7 +828,21 @@ def get_srcrev(d, method_name='sortable_revision'):
         format = "AUTOINC+" + format
 
     d.delVar("__BBINSRCREV")
-    return format
+    return format, revs
+
+def get_hashvalue(d, method_name='sortable_revision'):
+    pkgv, revs = _get_srcrev(d, method_name=method_name)
+    return " ".join(revs)
+
+def get_pkgv_string(d, method_name='sortable_revision'):
+    pkgv, revs = _get_srcrev(d, method_name=method_name)
+    return pkgv
+
+def get_srcrev(d, method_name='sortable_revision'):
+    pkgv, revs = _get_srcrev(d, method_name=method_name)
+    if not pkgv:
+        raise FetchError("SRCREV was used yet no valid SCM was found in SRC_URI")
+    return pkgv
 
 def localpath(url, d):
     fetcher = bb.fetch2.Fetch([url], d)
@@ -855,6 +874,7 @@ FETCH_EXPORT_VARS = ['HOME', 'PATH',
                      'AWS_SECRET_ACCESS_KEY',
                      'AWS_DEFAULT_REGION',
                      'GIT_CACHE_PATH',
+                     'REMOTE_CONTAINERS_IPC',
                      'SSL_CERT_DIR']
 
 def get_fetcher_environment(d):
@@ -1290,7 +1310,7 @@ class FetchData(object):
 
             if checksum_name in self.parm:
                 checksum_expected = self.parm[checksum_name]
-            elif self.type not in ["http", "https", "ftp", "ftps", "sftp", "s3", "az", "crate"]:
+            elif self.type not in ["http", "https", "ftp", "ftps", "sftp", "s3", "az", "crate", "gs"]:
                 checksum_expected = None
             else:
                 checksum_expected = d.getVarFlag("SRC_URI", checksum_name)
@@ -1402,6 +1422,9 @@ class FetchMethod(object):
         Is localpath something that can be represented by a checksum?
         """
 
+        # We cannot compute checksums for None
+        if urldata.localpath is None:
+            return False
         # We cannot compute checksums for directories
         if os.path.isdir(urldata.localpath):
             return False
@@ -1556,6 +1579,7 @@ class FetchMethod(object):
             unpackdir = rootdir
 
         if not unpack or not cmd:
+            urldata.unpack_tracer.unpack("file-copy", unpackdir)
             # If file == dest, then avoid any copies, as we already put the file into dest!
             dest = os.path.join(unpackdir, os.path.basename(file))
             if file != dest and not (os.path.exists(dest) and os.path.samefile(file, dest)):
@@ -1570,6 +1594,8 @@ class FetchMethod(object):
                         destdir = urlpath.rsplit("/", 1)[0] + '/'
                         bb.utils.mkdirhier("%s/%s" % (unpackdir, destdir))
                 cmd = 'cp -fpPRH "%s" "%s"' % (file, destdir)
+        else:
+            urldata.unpack_tracer.unpack("archive-extract", unpackdir)
 
         if not cmd:
             return
@@ -1661,6 +1687,55 @@ class FetchMethod(object):
         """
         return []
 
+
+class DummyUnpackTracer(object):
+    """
+    Abstract API definition for a class that traces unpacked source files back
+    to their respective upstream SRC_URI entries, for software composition
+    analysis, license compliance and detailed SBOM generation purposes.
+    User may load their own unpack tracer class (instead of the dummy
+    one) by setting the BB_UNPACK_TRACER_CLASS config parameter.
+    """
+    def start(self, unpackdir, urldata_dict, d):
+        """
+        Start tracing the core Fetch.unpack process, using an index to map
+        unpacked files to each SRC_URI entry.
+        This method is called by Fetch.unpack and it may receive nested calls by
+        gitsm and npmsw fetchers, that expand SRC_URI entries by adding implicit
+        URLs and by recursively calling Fetch.unpack from new (nested) Fetch
+        instances.
+        """
+        return
+    def start_url(self, url):
+        """Start tracing url unpack process.
+        This method is called by Fetch.unpack before the fetcher-specific unpack
+        method starts, and it may receive nested calls by gitsm and npmsw
+        fetchers.
+        """
+        return
+    def unpack(self, unpack_type, destdir):
+        """
+        Set unpack_type and destdir for current url.
+        This method is called by the fetcher-specific unpack method after url
+        tracing started.
+        """
+        return
+    def finish_url(self, url):
+        """Finish tracing url unpack process and update the file index.
+        This method is called by Fetch.unpack after the fetcher-specific unpack
+        method finished its job, and it may receive nested calls by gitsm
+        and npmsw fetchers.
+        """
+        return
+    def complete(self):
+        """
+        Finish tracing the Fetch.unpack process, and check if all nested
+        Fecth.unpack calls (if any) have been completed; if so, save collected
+        metadata.
+        """
+        return
+
+
 class Fetch(object):
     def __init__(self, urls, d, cache = True, localonly = False, connection_cache = None):
         if localonly and cache:
@@ -1681,10 +1756,30 @@ class Fetch(object):
         if key in urldata_cache:
             self.ud = urldata_cache[key]
 
+        # the unpack_tracer object needs to be made available to possible nested
+        # Fetch instances (when those are created by gitsm and npmsw fetchers)
+        # so we set it as a global variable
+        global unpack_tracer
+        try:
+            unpack_tracer
+        except NameError:
+            class_path = d.getVar("BB_UNPACK_TRACER_CLASS")
+            if class_path:
+                # use user-defined unpack tracer class
+                import importlib
+                module_name, _, class_name = class_path.rpartition(".")
+                module = importlib.import_module(module_name)
+                class_ = getattr(module, class_name)
+                unpack_tracer = class_()
+            else:
+                # fall back to the dummy/abstract class
+                unpack_tracer = DummyUnpackTracer()
+
         for url in urls:
             if url not in self.ud:
                 try:
                     self.ud[url] = FetchData(url, d, localonly)
+                    self.ud[url].unpack_tracer = unpack_tracer
                 except NonLocalMethod:
                     if localonly:
                         self.ud[url] = None
@@ -1860,6 +1955,8 @@ class Fetch(object):
         if not urls:
             urls = self.urls
 
+        unpack_tracer.start(root, self.ud, self.d)
+
         for u in urls:
             ud = self.ud[u]
             ud.setup_localpath(self.d)
@@ -1867,11 +1964,15 @@ class Fetch(object):
             if ud.lockfile:
                 lf = bb.utils.lockfile(ud.lockfile)
 
+            unpack_tracer.start_url(u)
             ud.method.unpack(ud, root, self.d)
+            unpack_tracer.finish_url(u)
 
             if ud.lockfile:
                 bb.utils.unlockfile(lf)
 
+        unpack_tracer.complete()
+
     def clean(self, urls=None):
         """
         Clean files that the fetcher gets or places
@@ -1973,6 +2074,7 @@ from . import npm
 from . import npmsw
 from . import az
 from . import crate
+from . import gcp
 
 methods.append(local.Local())
 methods.append(wget.Wget())
@@ -1994,3 +2096,4 @@ methods.append(npm.Npm())
 methods.append(npmsw.NpmShrinkWrap())
 methods.append(az.Az())
 methods.append(crate.Crate())
+methods.append(gcp.GCP())

bitbake/lib/bb/fetch2/crate.py

@@ -101,8 +101,10 @@ class Crate(Wget):
         bp = d.getVar('BP')
         if bp == ud.parm.get('name'):
             cmd = "tar -xz --no-same-owner -f %s" % thefile
+            ud.unpack_tracer.unpack("crate-extract", rootdir)
         else:
             cargo_bitbake = self._cargo_bitbake_path(rootdir)
+            ud.unpack_tracer.unpack("cargo-extract", cargo_bitbake)
 
             cmd = "tar -xz --no-same-owner -f %s -C %s" % (thefile, cargo_bitbake)
 

bitbake/lib/bb/fetch2/gcp.py

@@ -0,0 +1,98 @@
+"""
+BitBake 'Fetch' implementation for Google Cloup Platform Storage.
+
+Class for fetching files from Google Cloud Storage using the
+Google Cloud Storage Python Client. The GCS Python Client must
+be correctly installed, configured and authenticated prior to use.
+Additionally, gsutil must also be installed.
+
+"""
+
+# Copyright (C) 2023, Snap Inc.
+#
+# Based in part on bb.fetch2.s3:
+#    Copyright (C) 2017 Andre McCurdy
+#
+# SPDX-License-Identifier: GPL-2.0-only
+#
+# Based on functions from the base bb module, Copyright 2003 Holger Schurig
+
+import os
+import bb
+import urllib.parse, urllib.error
+from bb.fetch2 import FetchMethod
+from bb.fetch2 import FetchError
+from bb.fetch2 import logger
+
+class GCP(FetchMethod):
+    """
+    Class to fetch urls via GCP's Python API.
+    """
+    def __init__(self):
+        self.gcp_client = None
+
+    def supports(self, ud, d):
+        """
+        Check to see if a given url can be fetched with GCP.
+        """
+        return ud.type in ['gs']
+
+    def recommends_checksum(self, urldata):
+        return True
+
+    def urldata_init(self, ud, d):
+        if 'downloadfilename' in ud.parm:
+            ud.basename = ud.parm['downloadfilename']
+        else:
+            ud.basename = os.path.basename(ud.path)
+
+        ud.localfile = d.expand(urllib.parse.unquote(ud.basename))
+
+    def get_gcp_client(self):
+        from google.cloud import storage
+        self.gcp_client = storage.Client(project=None)
+
+    def download(self, ud, d):
+        """
+        Fetch urls using the GCP API.
+        Assumes localpath was called first.
+        """
+        logger.debug2(f"Trying to download gs://{ud.host}{ud.path} to {ud.localpath}")
+        if self.gcp_client is None:
+            self.get_gcp_client()
+
+        bb.fetch2.check_network_access(d, "gsutil stat", ud.url)
+
+        # Path sometimes has leading slash, so strip it
+        path = ud.path.lstrip("/")
+        blob = self.gcp_client.bucket(ud.host).blob(path)
+        blob.download_to_filename(ud.localpath)
+
+        # Additional sanity checks copied from the wget class (although there
+        # are no known issues which mean these are required, treat the GCP API
+        # tool with a little healthy suspicion).
+        if not os.path.exists(ud.localpath):
+            raise FetchError(f"The GCP API returned success for gs://{ud.host}{ud.path} but {ud.localpath} doesn't exist?!")
+
+        if os.path.getsize(ud.localpath) == 0:
+            os.remove(ud.localpath)
+            raise FetchError(f"The downloaded file for gs://{ud.host}{ud.path} resulted in a zero size file?! Deleting and failing since this isn't right.")
+
+        return True
+
+    def checkstatus(self, fetch, ud, d):
+        """
+        Check the status of a URL.
+        """
+        logger.debug2(f"Checking status of gs://{ud.host}{ud.path}")
+        if self.gcp_client is None:
+            self.get_gcp_client()
+
+        bb.fetch2.check_network_access(d, "gsutil stat", ud.url)
+
+        # Path sometimes has leading slash, so strip it
+        path = ud.path.lstrip("/")
+        if self.gcp_client.bucket(ud.host).blob(path).exists() == False:
+            raise FetchError(f"The GCP API reported that gs://{ud.host}{ud.path} does not exist")
+        else:
+            return True

bitbake/lib/bb/fetch2/git.py

@@ -65,6 +65,7 @@ import fnmatch
 import os
 import re
 import shlex
+import shutil
 import subprocess
 import tempfile
 import bb
@@ -365,8 +366,32 @@ class Git(FetchMethod):
                 runfetchcmd(fetch_cmd, d, workdir=ud.clonedir)
         repourl = self._get_repo_url(ud)
 
+        needs_clone = False
+        if os.path.exists(ud.clonedir):
+            # The directory may exist, but not be the top level of a bare git
+            # repository in which case it needs to be deleted and re-cloned.
+            try:
+                # Since clones can be bare, use --absolute-git-dir instead of --show-toplevel
+                output = runfetchcmd("LANG=C %s rev-parse --absolute-git-dir" % ud.basecmd, d, workdir=ud.clonedir)
+                toplevel = output.rstrip()
+
+                if not bb.utils.path_is_descendant(toplevel, ud.clonedir):
+                    logger.warning("Top level directory '%s' is not a descendant of '%s'. Re-cloning", toplevel, ud.clonedir)
+                    needs_clone = True
+            except bb.fetch2.FetchError as e:
+                logger.warning("Unable to get top level for %s (not a git directory?): %s", ud.clonedir, e)
+                needs_clone = True
+            except FileNotFoundError as e:
+                logger.warning("%s", e)
+                needs_clone = True
+
+            if needs_clone:
+                shutil.rmtree(ud.clonedir)
+        else:
+            needs_clone = True
+
         # If the repo still doesn't exist, fallback to cloning it
-        if not os.path.exists(ud.clonedir):
+        if needs_clone:
             # We do this since git will use a "-l" option automatically for local urls where possible,
             # but it doesn't work when git/objects is a symlink, only works when it is a directory.
             if repourl.startswith("file://"):
@@ -564,6 +589,8 @@ class Git(FetchMethod):
         destdir = ud.destdir = os.path.join(destdir, destsuffix)
         if os.path.exists(destdir):
             bb.utils.prunedir(destdir)
+        if not ud.bareclone:
+            ud.unpack_tracer.unpack("git", destdir)
 
         need_lfs = self._need_lfs(ud)
 

bitbake/lib/bb/fetch2/gitsm.py

@@ -123,6 +123,13 @@ class GitSM(Git):
             url += ";name=%s" % module
             url += ";subpath=%s" % module
             url += ";nobranch=1"
+            url += ";lfs=%s" % self._need_lfs(ud)
+            # Note that adding "user=" here to give credentials to the
+            # submodule is not supported. Since using SRC_URI to give git://
+            # URL a password is not supported, one have to use one of the
+            # recommended way (eg. ~/.netrc or SSH config) which does specify
+            # the user (See comment in git.py).
+            # So, we will not take patches adding "user=" support here.
 
             ld = d.createCopy()
             # Not necessary to set SRC_URI, since we're passing the URI to
@@ -211,6 +218,10 @@ class GitSM(Git):
 
             try:
                 newfetch = Fetch([url], d, cache=False)
+                # modpath is needed by unpack tracer to calculate submodule
+                # checkout dir
+                new_ud = newfetch.ud[url]
+                new_ud.modpath = modpath
                 newfetch.unpack(root=os.path.dirname(os.path.join(repo_conf, 'modules', module)))
             except Exception as e:
                 logger.error('gitsm: submodule unpack failed: %s %s' % (type(e).__name__, str(e)))
@@ -236,10 +247,12 @@ class GitSM(Git):
         ret = self.process_submodules(ud, ud.destdir, unpack_submodules, d)
 
         if not ud.bareclone and ret:
-            # All submodules should already be downloaded and configured in the tree.  This simply sets
-            # up the configuration and checks out the files.  The main project config should remain
-            # unmodified, and no download from the internet should occur.
-            runfetchcmd("%s submodule update --recursive --no-fetch" % (ud.basecmd), d, quiet=True, workdir=ud.destdir)
+            # All submodules should already be downloaded and configured in the tree.  This simply
+            # sets up the configuration and checks out the files.  The main project config should
+            # remain unmodified, and no download from the internet should occur. As such, lfs smudge
+            # should also be skipped as these files were already smudged in the fetch stage if lfs
+            # was enabled.
+            runfetchcmd("GIT_LFS_SKIP_SMUDGE=1 %s submodule update --recursive --no-fetch" % (ud.basecmd), d, quiet=True, workdir=ud.destdir)
 
     def implicit_urldata(self, ud, d):
         import shutil, subprocess, tempfile

bitbake/lib/bb/fetch2/hg.py

@@ -242,6 +242,7 @@ class Hg(FetchMethod):
         revflag = "-r %s" % ud.revision
         subdir = ud.parm.get("destsuffix", ud.module)
         codir = "%s/%s" % (destdir, subdir)
+        ud.unpack_tracer.unpack("hg", codir)
 
         scmdata = ud.parm.get("scmdata", "")
         if scmdata != "nokeep":

bitbake/lib/bb/fetch2/npm.py

@@ -298,6 +298,7 @@ class Npm(FetchMethod):
         destsuffix = ud.parm.get("destsuffix", "npm")
         destdir = os.path.join(rootdir, destsuffix)
         npm_unpack(ud.localpath, destdir, d)
+        ud.unpack_tracer.unpack("npm", destdir)
 
     def clean(self, ud, d):
         """Clean any existing full or partial download"""

bitbake/lib/bb/fetch2/npmsw.py

@@ -191,7 +191,9 @@ class NpmShrinkWrap(FetchMethod):
             else:
                 raise ParameterError("Unsupported dependency: %s" % name, ud.url)
 
+            # name is needed by unpack tracer for module mapping
             ud.deps.append({
+                "name": name,
                 "url": url,
                 "localpath": localpath,
                 "extrapaths": extrapaths,
@@ -225,13 +227,15 @@ class NpmShrinkWrap(FetchMethod):
     @staticmethod
     def _foreach_proxy_method(ud, handle):
         returns = []
-        for proxy_url in ud.proxy.urls:
-            proxy_ud = ud.proxy.ud[proxy_url]
-            proxy_d = ud.proxy.d
-            proxy_ud.setup_localpath(proxy_d)
-            lf = lockfile(proxy_ud.lockfile)
-            returns.append(handle(proxy_ud.method, proxy_ud, proxy_d))
-            unlockfile(lf)
+        #Check if there are dependencies before try to fetch them
+        if len(ud.deps) > 0:
+            for proxy_url in ud.proxy.urls:
+                proxy_ud = ud.proxy.ud[proxy_url]
+                proxy_d = ud.proxy.d
+                proxy_ud.setup_localpath(proxy_d)
+                lf = lockfile(proxy_ud.lockfile)
+                returns.append(handle(proxy_ud.method, proxy_ud, proxy_d))
+                unlockfile(lf)
         return returns
 
     def verify_donestamp(self, ud, d):
@@ -268,6 +272,7 @@ class NpmShrinkWrap(FetchMethod):
         destsuffix = ud.parm.get("destsuffix")
         if destsuffix:
             destdir = os.path.join(rootdir, destsuffix)
+        ud.unpack_tracer.unpack("npm-shrinkwrap", destdir)
 
         bb.utils.mkdirhier(destdir)
         bb.utils.copyfile(ud.shrinkwrap_file,

bitbake/lib/bb/monitordisk.py

@@ -234,9 +234,10 @@ class diskMonitor:
                 freeInode = st.f_favail
 
                 if minInode and freeInode < minInode:
-                    # Some filesystems use dynamic inodes so can't run out
-                    # (e.g. btrfs). This is reported by the inode count being 0.
-                    if st.f_files == 0:
+                    # Some filesystems use dynamic inodes so can't run out.
+                    # This is reported by the inode count being 0 (btrfs) or the free
+                    # inode count being -1 (cephfs).
+                    if st.f_files == 0 or st.f_favail == -1:
                         self.devDict[k][2] = None
                         continue
                     # Always show warning, the self.checked would always be False if the action is WARN

bitbake/lib/bb/parse/__init__.py

@@ -60,6 +60,14 @@ def cached_mtime_noerror(f):
             return 0
     return __mtime_cache[f]
 
+def check_mtime(f, mtime):
+    try:
+        current_mtime = os.stat(f)[stat.ST_MTIME]
+        __mtime_cache[f] = current_mtime
+    except OSError:
+        current_mtime = 0
+    return current_mtime == mtime
+
 def update_mtime(f):
     try:
         __mtime_cache[f] = os.stat(f)[stat.ST_MTIME]

bitbake/lib/bb/runqueue.py

@@ -200,21 +200,24 @@ class RunQueueScheduler(object):
                 curr_memory_pressure = memory_pressure_fds.readline().split()[4].split("=")[1]
                 now = time.time()
                 tdiff = now - self.prev_pressure_time
-                if tdiff > 1.0:
-                    exceeds_cpu_pressure =  self.rq.max_cpu_pressure and (float(curr_cpu_pressure) - float(self.prev_cpu_pressure)) / tdiff > self.rq.max_cpu_pressure
-                    exceeds_io_pressure =  self.rq.max_io_pressure and (float(curr_io_pressure) - float(self.prev_io_pressure)) / tdiff > self.rq.max_io_pressure
-                    exceeds_memory_pressure = self.rq.max_memory_pressure and (float(curr_memory_pressure) - float(self.prev_memory_pressure)) / tdiff > self.rq.max_memory_pressure
+                psi_accumulation_interval = 1.0
+                cpu_pressure = (float(curr_cpu_pressure) - float(self.prev_cpu_pressure)) / tdiff
+                io_pressure = (float(curr_io_pressure) - float(self.prev_io_pressure)) / tdiff
+                memory_pressure = (float(curr_memory_pressure) - float(self.prev_memory_pressure)) / tdiff
+                exceeds_cpu_pressure =  self.rq.max_cpu_pressure and cpu_pressure > self.rq.max_cpu_pressure
+                exceeds_io_pressure =  self.rq.max_io_pressure and io_pressure > self.rq.max_io_pressure
+                exceeds_memory_pressure =  self.rq.max_memory_pressure and memory_pressure > self.rq.max_memory_pressure
+
+                if tdiff > psi_accumulation_interval:
                     self.prev_cpu_pressure = curr_cpu_pressure
                     self.prev_io_pressure = curr_io_pressure
                     self.prev_memory_pressure = curr_memory_pressure
                     self.prev_pressure_time = now
-                else:
-                    exceeds_cpu_pressure =  self.rq.max_cpu_pressure and (float(curr_cpu_pressure) - float(self.prev_cpu_pressure)) > self.rq.max_cpu_pressure
-                    exceeds_io_pressure =  self.rq.max_io_pressure and (float(curr_io_pressure) - float(self.prev_io_pressure)) > self.rq.max_io_pressure
-                    exceeds_memory_pressure = self.rq.max_memory_pressure and (float(curr_memory_pressure) - float(self.prev_memory_pressure)) > self.rq.max_memory_pressure
+
             pressure_state = (exceeds_cpu_pressure, exceeds_io_pressure, exceeds_memory_pressure)
+            pressure_values = (round(cpu_pressure,1), self.rq.max_cpu_pressure, round(io_pressure,1), self.rq.max_io_pressure, round(memory_pressure,1), self.rq.max_memory_pressure)
             if hasattr(self, "pressure_state") and pressure_state != self.pressure_state:
-                bb.note("Pressure status changed to CPU: %s, IO: %s, Mem: %s" % pressure_state)
+                bb.note("Pressure status changed to CPU: %s, IO: %s, Mem: %s (CPU: %s/%s, IO: %s/%s, Mem: %s/%s) - using %s/%s bitbake threads" % (pressure_state + pressure_values + (len(self.rq.runq_running.difference(self.rq.runq_complete)), self.rq.number_tasks)))
             self.pressure_state = pressure_state
             return (exceeds_cpu_pressure or exceeds_io_pressure or exceeds_memory_pressure)
         return False
@@ -1321,6 +1324,8 @@ class RunQueue:
         if self.cooker.configuration.profile:
             magic = "decafbadbad"
         fakerootlogs = None
+
+        workerscript = os.path.realpath(os.path.dirname(__file__) + "/../../bin/bitbake-worker")
         if fakeroot:
             magic = magic + "beef"
             mcdata = self.cooker.databuilder.mcdata[mc]
@@ -1329,10 +1334,10 @@ class RunQueue:
             env = os.environ.copy()
             for key, value in (var.split('=') for var in fakerootenv):
                 env[key] = value
-            worker = subprocess.Popen(fakerootcmd + ["bitbake-worker", magic], stdout=subprocess.PIPE, stdin=subprocess.PIPE, env=env)
+            worker = subprocess.Popen(fakerootcmd + [sys.executable, workerscript, magic], stdout=subprocess.PIPE, stdin=subprocess.PIPE, env=env)
             fakerootlogs = self.rqdata.dataCaches[mc].fakerootlogs
         else:
-            worker = subprocess.Popen(["bitbake-worker", magic], stdout=subprocess.PIPE, stdin=subprocess.PIPE)
+            worker = subprocess.Popen([sys.executable, workerscript, magic], stdout=subprocess.PIPE, stdin=subprocess.PIPE)
         bb.utils.nonblockingfd(worker.stdout)
         workerpipe = runQueuePipe(worker.stdout, None, self.cfgData, self, rqexec, fakerootlogs=fakerootlogs)
 
@@ -3156,7 +3161,7 @@ class runQueuePipe():
         if pipeout:
             pipeout.close()
         bb.utils.nonblockingfd(self.input)
-        self.queue = b""
+        self.queue = bytearray()
         self.d = d
         self.rq = rq
         self.rqexec = rqexec
@@ -3175,7 +3180,7 @@ class runQueuePipe():
 
         start = len(self.queue)
         try:
-            self.queue = self.queue + (self.input.read(102400) or b"")
+            self.queue.extend(self.input.read(102400) or b"")
         except (OSError, IOError) as e:
             if e.errno != errno.EAGAIN:
                 raise

bitbake/lib/bb/server/process.py

@@ -38,9 +38,13 @@ logger = logging.getLogger('BitBake')
 class ProcessTimeout(SystemExit):
     pass
 
+def currenttime():
+    return datetime.datetime.now().strftime('%H:%M:%S.%f')
+
 def serverlog(msg):
-    print(str(os.getpid()) + " " +  datetime.datetime.now().strftime('%H:%M:%S.%f') + " " + msg)
-    sys.stdout.flush()
+    print(str(os.getpid()) + " " +  currenttime() + " " + msg)
+    #Seems a flush here triggers filesytem sync like behaviour and long hangs in the server
+    #sys.stdout.flush()
 
 #
 # When we have lockfile issues, try and find infomation about which process is
@@ -289,7 +293,9 @@ class ProcessServer():
                     continue
                 try:
                     serverlog("Running command %s" % command)
-                    self.command_channel_reply.send(self.cooker.command.runCommand(command, self))
+                    reply = self.cooker.command.runCommand(command, self)
+                    serverlog("Sending reply %s" % repr(reply))
+                    self.command_channel_reply.send(reply)
                     serverlog("Command Completed (socket: %s)" % os.path.exists(self.sockname))
                 except Exception as e:
                    stack = traceback.format_exc()
@@ -405,12 +411,6 @@ class ProcessServer():
             nextsleep = 0.1
             fds = []
 
-            try:
-                self.cooker.process_inotify_updates()
-            except Exception as exc:
-                serverlog("Exception %s in inofify updates broke the idle_thread, exiting" % traceback.format_exc())
-                self.quit = True
-
             with bb.utils.lock_timeout(self._idlefuncsLock):
                 items = list(self._idlefuns.items())
 
@@ -502,9 +502,9 @@ class ServerCommunicator():
     def runCommand(self, command):
         self.connection.send(command)
         if not self.recv.poll(30):
-            logger.info("No reply from server in 30s (for command %s)" % command[0])
+            logger.info("No reply from server in 30s (for command %s at %s)" % (command[0], currenttime()))
             if not self.recv.poll(30):
-                raise ProcessTimeout("Timeout while waiting for a reply from the bitbake server (60s)")
+                raise ProcessTimeout("Timeout while waiting for a reply from the bitbake server (60s at %s)" % currenttime())
         ret, exc = self.recv.get()
         # Should probably turn all exceptions in exc back into exceptions?
         # For now, at least handle BBHandledException
@@ -620,7 +620,7 @@ class BitBakeServer(object):
         os.set_inheritable(self.bitbake_lock.fileno(), True)
         os.set_inheritable(self.readypipein, True)
         serverscript = os.path.realpath(os.path.dirname(__file__) + "/../../../bin/bitbake-server")
-        os.execl(sys.executable, "bitbake-server", serverscript, "decafbad", str(self.bitbake_lock.fileno()), str(self.readypipein), self.logfile, self.bitbake_lock.name, self.sockname,  str(self.server_timeout or 0), str(int(self.profile)), str(self.xmlrpcinterface[0]), str(self.xmlrpcinterface[1]))
+        os.execl(sys.executable, sys.executable, serverscript, "decafbad", str(self.bitbake_lock.fileno()), str(self.readypipein), self.logfile, self.bitbake_lock.name, self.sockname,  str(self.server_timeout or 0), str(int(self.profile)), str(self.xmlrpcinterface[0]), str(self.xmlrpcinterface[1]))
 
 def execServer(lockfd, readypipeinfd, lockname, sockname, server_timeout, xmlrpcinterface, profile):
 
@@ -860,11 +860,10 @@ class ConnectionWriter(object):
                 process.queue_signals = True
                 self._send(obj)
                 process.queue_signals = False
-                try:
-                    for sig in process.signal_received.pop():
-                        process.handle_sig(sig, None)
-                except IndexError:
-                    pass
+
+                while len(process.signal_received) > 0:
+                    sig = process.signal_received.pop()
+                    process.handle_sig(sig, None)
         else:
             self._send(obj)
 

bitbake/lib/bb/siggen.py

@@ -182,6 +182,11 @@ class SignatureGenerator(object):
     def exit(self):
         return
 
+def build_pnid(mc, pn, taskname):
+    if mc:
+        return "mc:" + mc + ":" + pn + ":" + taskname
+    return pn + ":" + taskname
+
 class SignatureGeneratorBasic(SignatureGenerator):
     """
     """
@@ -256,10 +261,6 @@ class SignatureGeneratorBasic(SignatureGenerator):
             bb.warn("Error during finalise of %s" % mcfn)
             raise
 
-        #Slow but can be useful for debugging mismatched basehashes
-        #for task in self.taskdeps[mcfn]:
-        #    self.dump_sigtask(mcfn, task, d.getVar("STAMP"), False)
-
         basehashes = {}
         for task in taskdeps:
             basehashes[task] = self.basehash[mcfn + ":" + task]
@@ -269,6 +270,11 @@ class SignatureGeneratorBasic(SignatureGenerator):
         d.setVar("__siggen_varvals", lookupcache)
         d.setVar("__siggen_taskdeps", taskdeps)
 
+        #Slow but can be useful for debugging mismatched basehashes
+        #self.setup_datacache_from_datastore(mcfn, d)
+        #for task in taskdeps:
+        #    self.dump_sigtask(mcfn, task, d.getVar("STAMP"), False)
+
     def setup_datacache_from_datastore(self, mcfn, d):
         super().setup_datacache_from_datastore(mcfn, d)
 
@@ -309,15 +315,19 @@ class SignatureGeneratorBasic(SignatureGenerator):
         recipename = dataCaches[mc].pkg_fn[mcfn]
 
         self.tidtopn[tid] = recipename
+        # save hashfn for deps into siginfo?
+        for dep in deps:
+            (depmc, _, deptask, depmcfn) = bb.runqueue.split_tid_mcfn(dep)
+            dep_pn = dataCaches[depmc].pkg_fn[depmcfn]
 
-        for dep in sorted(deps, key=clean_basepath):
-            (depmc, _, _, depmcfn) = bb.runqueue.split_tid_mcfn(dep)
-            depname = dataCaches[depmc].pkg_fn[depmcfn]
-            if not self.rundep_check(mcfn, recipename, task, dep, depname, dataCaches):
+            if not self.rundep_check(mcfn, recipename, task, dep, dep_pn, dataCaches):
                 continue
+
             if dep not in self.taskhash:
                 bb.fatal("%s is not in taskhash, caller isn't calling in dependency order?" % dep)
-            self.runtaskdeps[tid].append(dep)
+
+            dep_pnid = build_pnid(depmc, dep_pn, deptask)
+            self.runtaskdeps[tid].append((dep_pnid, dep))
 
         if task in dataCaches[mc].file_checksums[mcfn]:
             if self.checksum_cache:
@@ -348,10 +358,10 @@ class SignatureGeneratorBasic(SignatureGenerator):
     def get_taskhash(self, tid, deps, dataCaches):
 
         data = self.basehash[tid]
-        for dep in self.runtaskdeps[tid]:
-            data += self.get_unihash(dep)
+        for dep in sorted(self.runtaskdeps[tid]):
+            data += self.get_unihash(dep[1])
 
-        for (f, cs) in self.file_checksum_values[tid]:
+        for (f, cs) in sorted(self.file_checksum_values[tid], key=clean_checksum_file_path):
             if cs:
                 if "/./" in f:
                     data += "./" + f.split("/./")[1]
@@ -409,21 +419,21 @@ class SignatureGeneratorBasic(SignatureGenerator):
         data['varvals'][task] = self.datacaches[mc].siggen_varvals[mcfn][task]
         for dep in self.datacaches[mc].siggen_taskdeps[mcfn][task]:
             if dep in self.basehash_ignore_vars:
-               continue
+                continue
             data['gendeps'][dep] = self.datacaches[mc].siggen_gendeps[mcfn][dep]
             data['varvals'][dep] = self.datacaches[mc].siggen_varvals[mcfn][dep]
 
         if runtime and tid in self.taskhash:
-            data['runtaskdeps'] = self.runtaskdeps[tid]
+            data['runtaskdeps'] = [dep[0] for dep in sorted(self.runtaskdeps[tid])]
             data['file_checksum_values'] = []
-            for f,cs in self.file_checksum_values[tid]:
+            for f,cs in sorted(self.file_checksum_values[tid], key=clean_checksum_file_path):
                 if "/./" in f:
                     data['file_checksum_values'].append(("./" + f.split("/./")[1], cs))
                 else:
                     data['file_checksum_values'].append((os.path.basename(f), cs))
             data['runtaskhashes'] = {}
-            for dep in data['runtaskdeps']:
-                data['runtaskhashes'][dep] = self.get_unihash(dep)
+            for dep in self.runtaskdeps[tid]:
+                data['runtaskhashes'][dep[0]] = self.get_unihash(dep[1])
             data['taskhash'] = self.taskhash[tid]
             data['unihash'] = self.get_unihash(tid)
 
@@ -735,6 +745,12 @@ class SignatureGeneratorTestEquivHash(SignatureGeneratorUniHashMixIn, SignatureG
         self.server = data.getVar('BB_HASHSERVE')
         self.method = "sstate_output_hash"
 
+def clean_checksum_file_path(file_checksum_tuple):
+    f, cs = file_checksum_tuple
+    if "/./" in f:
+        return "./" + f.split("/./")[1]
+    return f
+
 def dump_this_task(outfile, d):
     import bb.parse
     mcfn = d.getVar("BB_FILENAME")
@@ -793,39 +809,6 @@ def list_inline_diff(oldlist, newlist, colors=None):
             ret.append(item)
     return '[%s]' % (', '.join(ret))
 
-def clean_basepath(basepath):
-    basepath, dir, recipe_task = basepath.rsplit("/", 2)
-    cleaned = dir + '/' + recipe_task
-
-    if basepath[0] == '/':
-        return cleaned
-
-    if basepath.startswith("mc:") and basepath.count(':') >= 2:
-        mc, mc_name, basepath = basepath.split(":", 2)
-        mc_suffix = ':mc:' + mc_name
-    else:
-        mc_suffix = ''
-
-    # mc stuff now removed from basepath. Whatever was next, if present will be the first
-    # suffix. ':/', recipe path start, marks the end of this. Something like
-    # 'virtual:a[:b[:c]]:/path...' (b and c being optional)
-    if basepath[0] != '/':
-        cleaned += ':' + basepath.split(':/', 1)[0]
-
-    return cleaned + mc_suffix
-
-def clean_basepaths(a):
-    b = {}
-    for x in a:
-        b[clean_basepath(x)] = a[x]
-    return b
-
-def clean_basepaths_list(a):
-    b = []
-    for x in a:
-        b.append(clean_basepath(x))
-    return b
-
 # Handled renamed fields
 def handle_renames(data):
     if 'basewhitelist' in data:
@@ -994,11 +977,11 @@ def compare_sigfiles(a, b, recursecb=None, color=False, collapsed=False):
                 a = a_data['runtaskdeps'][idx]
                 b = b_data['runtaskdeps'][idx]
                 if a_data['runtaskhashes'][a] != b_data['runtaskhashes'][b] and not collapsed:
-                    changed.append("%s with hash %s\n changed to\n%s with hash %s" % (clean_basepath(a), a_data['runtaskhashes'][a], clean_basepath(b), b_data['runtaskhashes'][b]))
+                    changed.append("%s with hash %s\n changed to\n%s with hash %s" % (a, a_data['runtaskhashes'][a], b, b_data['runtaskhashes'][b]))
 
         if changed:
-            clean_a = clean_basepaths_list(a_data['runtaskdeps'])
-            clean_b = clean_basepaths_list(b_data['runtaskdeps'])
+            clean_a = a_data['runtaskdeps']
+            clean_b = b_data['runtaskdeps']
             if clean_a != clean_b:
                 output.append(color_format("{color_title}runtaskdeps changed:{color_default}\n%s") % list_inline_diff(clean_a, clean_b, colors))
             else:
@@ -1007,8 +990,8 @@ def compare_sigfiles(a, b, recursecb=None, color=False, collapsed=False):
 
 
     if 'runtaskhashes' in a_data and 'runtaskhashes' in b_data:
-        a = clean_basepaths(a_data['runtaskhashes'])
-        b = clean_basepaths(b_data['runtaskhashes'])
+        a = a_data['runtaskhashes']
+        b = b_data['runtaskhashes']
         changed, added, removed = dict_diff(a, b)
         if added:
             for dep in sorted(added):

bitbake/lib/bb/tests/codeparser.py

@@ -44,6 +44,7 @@ class VariableReferenceTest(ReferenceTest):
     def parseExpression(self, exp):
         parsedvar = self.d.expandWithRefs(exp, None)
         self.references = parsedvar.references
+        self.execs = parsedvar.execs
 
     def test_simple_reference(self):
         self.setEmptyVars(["FOO"])
@@ -61,6 +62,11 @@ class VariableReferenceTest(ReferenceTest):
         self.parseExpression("${@d.getVar('BAR') + 'foo'}")
         self.assertReferences(set(["BAR"]))
 
+    def test_python_exec_reference(self):
+        self.parseExpression("${@eval('3 * 5')}")
+        self.assertReferences(set())
+        self.assertExecs(set(["eval"]))
+
 class ShellReferenceTest(ReferenceTest):
 
     def parseExpression(self, exp):
@@ -430,6 +436,32 @@ esac
         self.assertEqual(deps, set(["TESTVAR2"]))
         self.assertEqual(self.d.getVar('ANOTHERVAR').split(), ['testval3', 'anothervalue'])
 
+    def test_contains_vardeps_override_operators(self):
+        # Check override operators handle dependencies correctly with the contains functionality
+        expr_plain = 'testval'
+        expr_prepend = '${@bb.utils.filter("TESTVAR1", "testval1", d)} '
+        expr_append = ' ${@bb.utils.filter("TESTVAR2", "testval2", d)}'
+        expr_remove = '${@bb.utils.contains("TESTVAR3", "no-testval", "testval", "", d)}'
+        # Check dependencies
+        self.d.setVar('ANOTHERVAR', expr_plain)
+        self.d.prependVar('ANOTHERVAR', expr_prepend)
+        self.d.appendVar('ANOTHERVAR', expr_append)
+        self.d.setVar('ANOTHERVAR:remove', expr_remove)
+        self.d.setVar('TESTVAR1', 'blah')
+        self.d.setVar('TESTVAR2', 'testval2')
+        self.d.setVar('TESTVAR3', 'no-testval')
+        deps, values = bb.data.build_dependencies("ANOTHERVAR", set(self.d.keys()), set(), set(), set(), set(), self.d, self.d)
+        self.assertEqual(sorted(values.splitlines()),
+                         sorted([
+                          expr_prepend + expr_plain + expr_append,
+                          '_remove of ' + expr_remove,
+                          'TESTVAR1{testval1} = Unset',
+                          'TESTVAR2{testval2} = Set',
+                          'TESTVAR3{no-testval} = Set',
+                          ]))
+        # Check final value
+        self.assertEqual(self.d.getVar('ANOTHERVAR').split(), ['testval2'])
+
     #Currently no wildcard support
     #def test_vardeps_wildcards(self):
     #    self.d.setVar("oe_libinstall", "echo test")

bitbake/lib/bb/tests/data.py

@@ -77,6 +77,18 @@ class DataExpansions(unittest.TestCase):
         val = self.d.expand("${@d.getVar('foo') + ' ${bar}'}")
         self.assertEqual(str(val), "value_of_foo value_of_bar")
 
+    def test_python_snippet_function_reference(self):
+        self.d.setVar("TESTVAL", "testvalue")
+        self.d.setVar("testfunc", 'd.getVar("TESTVAL")')
+        context = bb.utils.get_context()
+        context["testfunc"] = lambda d: d.getVar("TESTVAL")
+        val = self.d.expand("${@testfunc(d)}")
+        self.assertEqual(str(val), "testvalue")
+
+    def test_python_snippet_builtin_metadata(self):
+        self.d.setVar("eval", "INVALID")
+        self.d.expand("${@eval('3')}")
+
     def test_python_unexpanded(self):
         self.d.setVar("bar", "${unsetvar}")
         val = self.d.expand("${@d.getVar('foo') + ' ${bar}'}")

bitbake/lib/bb/tests/fetch.py

@@ -684,11 +684,13 @@ class CleanTarballTest(FetcherTest):
         archive = tarfile.open(os.path.join(self.dldir, self.recipe_tarball))
         self.assertNotEqual(len(archive.members), 0)
         for member in archive.members:
-            self.assertEqual(member.uname, 'oe')
-            self.assertEqual(member.uid, 0)
-            self.assertEqual(member.gname, 'oe')
-            self.assertEqual(member.gid, 0)
-            self.assertEqual(member.mtime, mtime)
+            if member.name == ".":
+                continue
+            self.assertEqual(member.uname, 'oe', "user name for %s differs" % member.name)
+            self.assertEqual(member.uid, 0, "uid for %s differs" % member.name)
+            self.assertEqual(member.gname, 'oe', "group name for %s differs" % member.name)
+            self.assertEqual(member.gid, 0, "gid for %s differs" % member.name)
+            self.assertEqual(member.mtime, mtime, "mtime for %s differs" % member.name)
 
 
 class FetcherLocalTest(FetcherTest):
@@ -1342,6 +1344,7 @@ class URLHandle(unittest.TestCase):
        "cvs://anoncvs:anonymous@cvs.handhelds.org/cvs;tag=V0-99-81;module=familiar/dist/ipkg" : ('cvs', 'cvs.handhelds.org', '/cvs', 'anoncvs', 'anonymous', collections.OrderedDict([('tag', 'V0-99-81'), ('module', 'familiar/dist/ipkg')])),
        "git://git.openembedded.org/bitbake;branch=@foo;protocol=https" : ('git', 'git.openembedded.org', '/bitbake', '', '', {'branch': '@foo', 'protocol' : 'https'}),
        "file://somelocation;someparam=1": ('file', '', 'somelocation', '', '', {'someparam': '1'}),
+       "https://somesite.com/somerepo.git;user=anyUser:idtoken=1234" : ('https', 'somesite.com', '/somerepo.git', '', '', {'user': 'anyUser:idtoken=1234'}),
        r'git://s.o-me_ONE:!#$%^&*()-_={}[]\|:?,.<>~`@git.openembedded.org/bitbake;branch=main;protocol=https': ('git', 'git.openembedded.org', '/bitbake', 's.o-me_ONE', r'!#$%^&*()-_={}[]\|:?,.<>~`', {'branch': 'main', 'protocol' : 'https'}),
     }
     # we require a pathname to encodeurl but users can still pass such urls to 

bitbake/lib/bb/tests/siggen.py

@@ -17,75 +17,12 @@ import bb.siggen
 
 class SiggenTest(unittest.TestCase):
 
-    def test_clean_basepath_simple_target_basepath(self):
-        basepath = '/full/path/to/poky/meta/recipes-whatever/helloworld/helloworld_1.2.3.bb:do_sometask'
-        expected_cleaned = 'helloworld/helloworld_1.2.3.bb:do_sometask'
+    def test_build_pnid(self):
+        tests = {
+            ('', 'helloworld', 'do_sometask') : 'helloworld:do_sometask',
+            ('XX', 'helloworld', 'do_sometask') : 'mc:XX:helloworld:do_sometask',
+        }
 
-        actual_cleaned = bb.siggen.clean_basepath(basepath)
+        for t in tests:
+            self.assertEqual(bb.siggen.build_pnid(*t), tests[t])
 
-        self.assertEqual(actual_cleaned, expected_cleaned)
-
-    def test_clean_basepath_basic_virtual_basepath(self):
-        basepath = 'virtual:something:/full/path/to/poky/meta/recipes-whatever/helloworld/helloworld_1.2.3.bb:do_sometask'
-        expected_cleaned = 'helloworld/helloworld_1.2.3.bb:do_sometask:virtual:something'
-
-        actual_cleaned = bb.siggen.clean_basepath(basepath)
-
-        self.assertEqual(actual_cleaned, expected_cleaned)
-
-    def test_clean_basepath_mc_basepath(self):
-        basepath = 'mc:somemachine:/full/path/to/poky/meta/recipes-whatever/helloworld/helloworld_1.2.3.bb:do_sometask'
-        expected_cleaned = 'helloworld/helloworld_1.2.3.bb:do_sometask:mc:somemachine'
-
-        actual_cleaned = bb.siggen.clean_basepath(basepath)
-
-        self.assertEqual(actual_cleaned, expected_cleaned)
-
-    def test_clean_basepath_virtual_long_prefix_basepath(self):
-        basepath = 'virtual:something:A:B:C:/full/path/to/poky/meta/recipes-whatever/helloworld/helloworld_1.2.3.bb:do_sometask'
-        expected_cleaned = 'helloworld/helloworld_1.2.3.bb:do_sometask:virtual:something:A:B:C'
-
-        actual_cleaned = bb.siggen.clean_basepath(basepath)
-
-        self.assertEqual(actual_cleaned, expected_cleaned)
-
-    def test_clean_basepath_mc_virtual_basepath(self):
-        basepath = 'mc:somemachine:virtual:something:/full/path/to/poky/meta/recipes-whatever/helloworld/helloworld_1.2.3.bb:do_sometask'
-        expected_cleaned = 'helloworld/helloworld_1.2.3.bb:do_sometask:virtual:something:mc:somemachine'
-
-        actual_cleaned = bb.siggen.clean_basepath(basepath)
-
-        self.assertEqual(actual_cleaned, expected_cleaned)
-
-    def test_clean_basepath_mc_virtual_long_prefix_basepath(self):
-        basepath = 'mc:X:virtual:something:C:B:A:/full/path/to/poky/meta/recipes-whatever/helloworld/helloworld_1.2.3.bb:do_sometask'
-        expected_cleaned = 'helloworld/helloworld_1.2.3.bb:do_sometask:virtual:something:C:B:A:mc:X'
-
-        actual_cleaned = bb.siggen.clean_basepath(basepath)
-
-        self.assertEqual(actual_cleaned, expected_cleaned)
-
-
-    # def test_clean_basepath_performance(self):
-    #     input_basepaths = [
-    #         'mc:X:/full/path/to/poky/meta/recipes-whatever/helloworld/helloworld_1.2.3.bb:do_sometask',
-    #         'mc:X:virtual:something:C:B:A:/full/path/to/poky/meta/recipes-whatever/helloworld/helloworld_1.2.3.bb:do_sometask',
-    #         'virtual:something:C:B:A:/different/path/to/poky/meta/recipes-whatever/helloworld/helloworld_1.2.3.bb:do_sometask',
-    #         'virtual:something:A:/full/path/to/poky/meta/recipes-whatever/helloworld/helloworld_1.2.3.bb:do_sometask',
-    #         '/this/is/most/common/input/recipes-whatever/helloworld/helloworld_1.2.3.bb:do_sometask',
-    #         '/and/should/be/tested/with/recipes-whatever/helloworld/helloworld_1.2.3.bb:do_sometask',
-    #         '/more/weight/recipes-whatever/helloworld/helloworld_1.2.3.bb:do_sometask',
-    #     ]
-
-    #     time_start = time.time()
-
-    #     i = 2000000
-    #     while i >= 0:
-    #         for basepath in input_basepaths:
-    #             bb.siggen.clean_basepath(basepath)
-    #         i -= 1
-
-    #     elapsed = time.time() - time_start
-    #     print('{} ({}s)'.format(self.id(), round(elapsed, 3)))
-
-    #     self.assertTrue(False)

bitbake/lib/bb/tinfoil.py

@@ -325,11 +325,11 @@ class Tinfoil:
         self.recipes_parsed = False
         self.quiet = 0
         self.oldhandlers = self.logger.handlers[:]
+        self.localhandlers = []
         if setup_logging:
             # This is the *client-side* logger, nothing to do with
             # logging messages from the server
             bb.msg.logger_create('BitBake', output)
-            self.localhandlers = []
             for handler in self.logger.handlers:
                 if handler not in self.oldhandlers:
                     self.localhandlers.append(handler)
@@ -449,6 +449,12 @@ class Tinfoil:
         self.run_actions(config_params)
         self.recipes_parsed = True
 
+    def modified_files(self):
+        """
+        Notify the server it needs to revalidate it's caches since the client has modified files
+        """
+        self.run_command("revalidateCaches")
+
     def run_command(self, command, *params, handle_events=True):
         """
         Run a command on the server (as implemented in bb.command).

bitbake/lib/bb/ui/buildinfohelper.py

@@ -1746,7 +1746,6 @@ class BuildInfoHelper(object):
 
         buildname = self.server.runCommand(['getVariable', 'BUILDNAME'])[0]
         machine = self.server.runCommand(['getVariable', 'MACHINE'])[0]
-        image_name = self.server.runCommand(['getVariable', 'IMAGE_NAME'])[0]
 
         # location of the manifest files for this build;
         # note that this file is only produced if an image is produced
@@ -1767,6 +1766,18 @@ class BuildInfoHelper(object):
         # filter out anything which isn't an image target
         image_targets = [target for target in targets if target.is_image]
 
+        if len(image_targets) > 0:
+            #if there are image targets retrieve image_name
+            image_name = self.server.runCommand(['getVariable', 'IMAGE_NAME'])[0]
+            if not image_name:
+                #When build target is an image and image_name is not found as an environment variable
+                logger.info("IMAGE_NAME not found, extracting from bitbake command")
+                cmd = self.server.runCommand(['getVariable','BB_CMDLINE'])[0]
+                #filter out tokens that are command line options
+                cmd = [token for token in cmd if not token.startswith('-')]
+                image_name = cmd[1].split(':', 1)[0] # remove everything after : in image name
+                logger.info("IMAGE_NAME found as : %s " % image_name)
+
         for image_target in image_targets:
             # this is set to True if we find at least one file relating to
             # this target; if this remains False after the scan, we copy the

bitbake/lib/bb/utils.py

@@ -1828,6 +1828,29 @@ def mkstemp(suffix=None, prefix=None, dir=None, text=False):
         prefix = tempfile.gettempprefix() + entropy
     return tempfile.mkstemp(suffix=suffix, prefix=prefix, dir=dir, text=text)
 
+def path_is_descendant(descendant, ancestor):
+    """
+    Returns True if the path `descendant` is a descendant of `ancestor`
+    (including being equivalent to `ancestor` itself). Otherwise returns False.
+    Correctly accounts for symlinks, bind mounts, etc. by using
+    os.path.samestat() to compare paths
+
+    May raise any exception that os.stat() raises
+    """
+
+    ancestor_stat = os.stat(ancestor)
+
+    # Recurse up each directory component of the descendant to see if it is
+    # equivalent to the ancestor
+    check_dir = os.path.abspath(descendant).rstrip("/")
+    while check_dir:
+        check_stat = os.stat(check_dir)
+        if os.path.samestat(check_stat, ancestor_stat):
+            return True
+        check_dir = os.path.dirname(check_dir).rstrip("/")
+
+    return False
+
 # If we don't have a timeout of some kind and a process/thread exits badly (for example
 # OOM killed) and held a lock, we'd just hang in the lock futex forever. It is better
 # we exit at some point than hang. 5 minutes with no progress means we're probably deadlocked.

bitbake/lib/bb/xattr.py

@@ -0,0 +1,126 @@
+#! /usr/bin/env python3
+#
+# Copyright 2023 by Garmin Ltd. or its subsidiaries
+#
+# SPDX-License-Identifier: MIT
+
+import sys
+import ctypes
+import os
+import errno
+
+libc = ctypes.CDLL("libc.so.6", use_errno=True)
+fsencoding = sys.getfilesystemencoding()
+
+
+libc.listxattr.argtypes = [ctypes.c_char_p, ctypes.c_char_p, ctypes.c_size_t]
+libc.llistxattr.argtypes = [ctypes.c_char_p, ctypes.c_char_p, ctypes.c_size_t]
+
+
+def listxattr(path, follow=True):
+    func = libc.listxattr if follow else libc.llistxattr
+
+    os_path = os.fsencode(path)
+
+    while True:
+        length = func(os_path, None, 0)
+
+        if length < 0:
+            err = ctypes.get_errno()
+            raise OSError(err, os.strerror(err), str(path))
+
+        if length == 0:
+            return []
+
+        arr = ctypes.create_string_buffer(length)
+
+        read_length = func(os_path, arr, length)
+        if read_length != length:
+            # Race!
+            continue
+
+        return [a.decode(fsencoding) for a in arr.raw.split(b"\x00") if a]
+
+
+libc.getxattr.argtypes = [
+    ctypes.c_char_p,
+    ctypes.c_char_p,
+    ctypes.c_char_p,
+    ctypes.c_size_t,
+]
+libc.lgetxattr.argtypes = [
+    ctypes.c_char_p,
+    ctypes.c_char_p,
+    ctypes.c_char_p,
+    ctypes.c_size_t,
+]
+
+
+def getxattr(path, name, follow=True):
+    func = libc.getxattr if follow else libc.lgetxattr
+
+    os_path = os.fsencode(path)
+    os_name = os.fsencode(name)
+
+    while True:
+        length = func(os_path, os_name, None, 0)
+
+        if length < 0:
+            err = ctypes.get_errno()
+            if err == errno.ENODATA:
+                return None
+            raise OSError(err, os.strerror(err), str(path))
+
+        if length == 0:
+            return ""
+
+        arr = ctypes.create_string_buffer(length)
+
+        read_length = func(os_path, os_name, arr, length)
+        if read_length != length:
+            # Race!
+            continue
+
+        return arr.raw
+
+
+def get_all_xattr(path, follow=True):
+    attrs = {}
+
+    names = listxattr(path, follow)
+
+    for name in names:
+        value = getxattr(path, name, follow)
+        if value is None:
+            # This can happen if a value is erased after listxattr is called,
+            # so ignore it
+            continue
+        attrs[name] = value
+
+    return attrs
+
+
+def main():
+    import argparse
+    from pathlib import Path
+
+    parser = argparse.ArgumentParser()
+    parser.add_argument("path", help="File Path", type=Path)
+
+    args = parser.parse_args()
+
+    attrs = get_all_xattr(args.path)
+
+    for name, value in attrs.items():
+        try:
+            value = value.decode(fsencoding)
+        except UnicodeDecodeError:
+            pass
+
+        print(f"{name} = {value}")
+
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())

bitbake/lib/bblayers/action.py

@@ -50,12 +50,14 @@ class ActionPlugin(LayerPlugin):
 
         try:
             notadded, _ = bb.utils.edit_bblayers_conf(bblayers_conf, layerdirs, None)
+            self.tinfoil.modified_files()
             if not (args.force or notadded):
                 try:
                     self.tinfoil.run_command('parseConfiguration')
                 except (bb.tinfoil.TinfoilUIException, bb.BBHandledException):
                     # Restore the back up copy of bblayers.conf
                     shutil.copy2(backup, bblayers_conf)
+                    self.tinfoil.modified_files()
                     bb.fatal("Parse failure with the specified layer added, exiting.")
                 else:
                     for item in notadded:
@@ -81,6 +83,7 @@ class ActionPlugin(LayerPlugin):
                 layerdir = os.path.abspath(item)
             layerdirs.append(layerdir)
         (_, notremoved) = bb.utils.edit_bblayers_conf(bblayers_conf, None, layerdirs)
+        self.tinfoil.modified_files()
         if notremoved:
             for item in notremoved:
                 sys.stderr.write("No layers matching %s found in BBLAYERS\n" % item)
@@ -240,6 +243,9 @@ build results (as the layer priority order has effectively changed).
                             if not entry_found:
                                 logger.warning("File %s does not match the flattened layer's BBFILES setting, you may need to edit conf/layer.conf or move the file elsewhere" % f1full)
 
+        self.tinfoil.modified_files()
+
+
     def get_file_layer(self, filename):
         layerdir = self.get_file_layerdir(filename)
         if layerdir:

bitbake/lib/bblayers/query.py

@@ -282,7 +282,10 @@ Lists recipes with the bbappends that apply to them as subitems.
         else:
             logger.plain('=== Appended recipes ===')
 
-        pnlist = list(self.tinfoil.cooker_data.pkg_pn.keys())
+
+        cooker_data = self.tinfoil.cooker.recipecaches[args.mc]
+
+        pnlist = list(cooker_data.pkg_pn.keys())
         pnlist.sort()
         appends = False
         for pn in pnlist:
@@ -295,7 +298,7 @@ Lists recipes with the bbappends that apply to them as subitems.
                 if not found:
                     continue
 
-            if self.show_appends_for_pn(pn):
+            if self.show_appends_for_pn(pn, cooker_data, args.mc):
                 appends = True
 
         if not args.pnspec and self.show_appends_for_skipped():
@@ -304,8 +307,10 @@ Lists recipes with the bbappends that apply to them as subitems.
         if not appends:
             logger.plain('No append files found')
 
-    def show_appends_for_pn(self, pn):
-        filenames = self.tinfoil.cooker_data.pkg_pn[pn]
+    def show_appends_for_pn(self, pn, cooker_data, mc):
+        filenames = cooker_data.pkg_pn[pn]
+        if mc:
+            pn = "mc:%s:%s" % (mc, pn)
 
         best = self.tinfoil.find_best_provider(pn)
         best_filename = os.path.basename(best[3])
@@ -530,6 +535,7 @@ NOTE: .bbappend files can impact the dependencies.
 
         parser_show_appends = self.add_command(sp, 'show-appends', self.do_show_appends)
         parser_show_appends.add_argument('pnspec', nargs='*', help='optional recipe name specification (wildcards allowed, enclose in quotes to avoid shell expansion)')
+        parser_show_appends.add_argument('--mc', help='use specified multiconfig', default='')
 
         parser_show_cross_depends = self.add_command(sp, 'show-cross-depends', self.do_show_cross_depends)
         parser_show_cross_depends.add_argument('-f', '--filenames', help='show full file path', action='store_true')

bitbake/lib/hashserv/client.py

@@ -83,10 +83,10 @@ class AsyncClient(bb.asyncrpc.AsyncClient):
             {"get": {"taskhash": taskhash, "method": method, "all": all_properties}}
         )
 
-    async def get_outhash(self, method, outhash, taskhash):
+    async def get_outhash(self, method, outhash, taskhash, with_unihash=True):
         await self._set_mode(self.MODE_NORMAL)
         return await self.send_message(
-            {"get-outhash": {"outhash": outhash, "taskhash": taskhash, "method": method}}
+            {"get-outhash": {"outhash": outhash, "taskhash": taskhash, "method": method, "with_unihash": with_unihash}}
         )
 
     async def get_stats(self):
@@ -101,6 +101,14 @@ class AsyncClient(bb.asyncrpc.AsyncClient):
         await self._set_mode(self.MODE_NORMAL)
         return (await self.send_message({"backfill-wait": None}))["tasks"]
 
+    async def remove(self, where):
+        await self._set_mode(self.MODE_NORMAL)
+        return await self.send_message({"remove": {"where": where}})
+
+    async def clean_unused(self, max_age):
+        await self._set_mode(self.MODE_NORMAL)
+        return await self.send_message({"clean-unused": {"max_age_seconds": max_age}})
+
 
 class Client(bb.asyncrpc.Client):
     def __init__(self):
@@ -115,6 +123,8 @@ class Client(bb.asyncrpc.Client):
             "get_stats",
             "reset_stats",
             "backfill_wait",
+            "remove",
+            "clean_unused",
         )
 
     def _get_async_client(self):

bitbake/lib/hashserv/server.py

@@ -4,7 +4,7 @@
 #
 
 from contextlib import closing, contextmanager
-from datetime import datetime
+from datetime import datetime, timedelta
 import enum
 import asyncio
 import logging
@@ -186,6 +186,8 @@ class ServerClient(bb.asyncrpc.AsyncServerConnection):
                 'report-equiv': self.handle_equivreport,
                 'reset-stats': self.handle_reset_stats,
                 'backfill-wait': self.handle_backfill_wait,
+                'remove': self.handle_remove,
+                'clean-unused': self.handle_clean_unused,
             })
 
     def validate_proto_version(self):
@@ -269,27 +271,42 @@ class ServerClient(bb.asyncrpc.AsyncServerConnection):
         method = request['method']
         outhash = request['outhash']
         taskhash = request['taskhash']
+        with_unihash = request.get("with_unihash", True)
 
         with closing(self.db.cursor()) as cursor:
-            d = await self.get_outhash(cursor, method, outhash, taskhash)
+            d = await self.get_outhash(cursor, method, outhash, taskhash, with_unihash)
 
         self.write_message(d)
 
-    async def get_outhash(self, cursor, method, outhash, taskhash):
+    async def get_outhash(self, cursor, method, outhash, taskhash, with_unihash=True):
         d = None
-        cursor.execute(
-            '''
-            SELECT *, unihashes_v2.unihash AS unihash FROM outhashes_v2
-            INNER JOIN unihashes_v2 ON unihashes_v2.method=outhashes_v2.method AND unihashes_v2.taskhash=outhashes_v2.taskhash
-            WHERE outhashes_v2.method=:method AND outhashes_v2.outhash=:outhash
-            ORDER BY outhashes_v2.created ASC
-            LIMIT 1
-            ''',
-            {
-                'method': method,
-                'outhash': outhash,
-            }
-        )
+        if with_unihash:
+            cursor.execute(
+                '''
+                SELECT *, unihashes_v2.unihash AS unihash FROM outhashes_v2
+                INNER JOIN unihashes_v2 ON unihashes_v2.method=outhashes_v2.method AND unihashes_v2.taskhash=outhashes_v2.taskhash
+                WHERE outhashes_v2.method=:method AND outhashes_v2.outhash=:outhash
+                ORDER BY outhashes_v2.created ASC
+                LIMIT 1
+                ''',
+                {
+                    'method': method,
+                    'outhash': outhash,
+                }
+            )
+        else:
+            cursor.execute(
+                """
+                SELECT * FROM outhashes_v2
+                WHERE outhashes_v2.method=:method AND outhashes_v2.outhash=:outhash
+                ORDER BY outhashes_v2.created ASC
+                LIMIT 1
+                """,
+                {
+                    'method': method,
+                    'outhash': outhash,
+                }
+            )
         row = cursor.fetchone()
 
         if row is not None:
@@ -499,6 +516,50 @@ class ServerClient(bb.asyncrpc.AsyncServerConnection):
         await self.backfill_queue.join()
         self.write_message(d)
 
+    async def handle_remove(self, request):
+        condition = request["where"]
+        if not isinstance(condition, dict):
+            raise TypeError("Bad condition type %s" % type(condition))
+
+        def do_remove(columns, table_name, cursor):
+            nonlocal condition
+            where = {}
+            for c in columns:
+                if c in condition and condition[c] is not None:
+                    where[c] = condition[c]
+
+            if where:
+                query = ('DELETE FROM %s WHERE ' % table_name) + ' AND '.join("%s=:%s" % (k, k) for k in where.keys())
+                cursor.execute(query, where)
+                return cursor.rowcount
+
+            return 0
+
+        count = 0
+        with closing(self.db.cursor()) as cursor:
+            count += do_remove(OUTHASH_TABLE_COLUMNS, "outhashes_v2", cursor)
+            count += do_remove(UNIHASH_TABLE_COLUMNS, "unihashes_v2", cursor)
+            self.db.commit()
+
+        self.write_message({"count": count})
+
+    async def handle_clean_unused(self, request):
+        max_age = request["max_age_seconds"]
+        with closing(self.db.cursor()) as cursor:
+            cursor.execute(
+                """
+                DELETE FROM outhashes_v2 WHERE created<:oldest AND NOT EXISTS (
+                    SELECT unihashes_v2.id FROM unihashes_v2 WHERE unihashes_v2.method=outhashes_v2.method AND unihashes_v2.taskhash=outhashes_v2.taskhash LIMIT 1
+                )
+                """,
+                {
+                    "oldest": datetime.now() - timedelta(seconds=-max_age)
+                }
+            )
+            count = cursor.rowcount
+
+        self.write_message({"count": count})
+
     def query_equivalent(self, cursor, method, taskhash):
         # This is part of the inner loop and must be as fast as possible
         cursor.execute(

bitbake/lib/hashserv/tests.py

@@ -84,6 +84,7 @@ class HashEquivalenceCommonTests(object):
 
         result = self.client.report_unihash(taskhash, self.METHOD, outhash, unihash)
         self.assertEqual(result['unihash'], unihash, 'Server returned bad unihash')
+        return taskhash, outhash, unihash
 
     def test_create_equivalent(self):
         # Tests that a second reported task with the same outhash will be
@@ -125,6 +126,57 @@ class HashEquivalenceCommonTests(object):
 
         self.assertClientGetHash(self.client, taskhash, unihash)
 
+    def test_remove_taskhash(self):
+        taskhash, outhash, unihash = self.test_create_hash()
+        result = self.client.remove({"taskhash": taskhash})
+        self.assertGreater(result["count"], 0)
+        self.assertClientGetHash(self.client, taskhash, None)
+
+        result_outhash = self.client.get_outhash(self.METHOD, outhash, taskhash)
+        self.assertIsNone(result_outhash)
+
+    def test_remove_unihash(self):
+        taskhash, outhash, unihash = self.test_create_hash()
+        result = self.client.remove({"unihash": unihash})
+        self.assertGreater(result["count"], 0)
+        self.assertClientGetHash(self.client, taskhash, None)
+
+    def test_remove_outhash(self):
+        taskhash, outhash, unihash = self.test_create_hash()
+        result = self.client.remove({"outhash": outhash})
+        self.assertGreater(result["count"], 0)
+
+        result_outhash = self.client.get_outhash(self.METHOD, outhash, taskhash)
+        self.assertIsNone(result_outhash)
+
+    def test_remove_method(self):
+        taskhash, outhash, unihash = self.test_create_hash()
+        result = self.client.remove({"method": self.METHOD})
+        self.assertGreater(result["count"], 0)
+        self.assertClientGetHash(self.client, taskhash, None)
+
+        result_outhash = self.client.get_outhash(self.METHOD, outhash, taskhash)
+        self.assertIsNone(result_outhash)
+
+    def test_clean_unused(self):
+        taskhash, outhash, unihash = self.test_create_hash()
+
+        # Clean the database, which should not remove anything because all hashes an in-use
+        result = self.client.clean_unused(0)
+        self.assertEqual(result["count"], 0)
+        self.assertClientGetHash(self.client, taskhash, unihash)
+
+        # Remove the unihash. The row in the outhash table should still be present
+        self.client.remove({"unihash": unihash})
+        result_outhash = self.client.get_outhash(self.METHOD, outhash, taskhash, False)
+        self.assertIsNotNone(result_outhash)
+
+        # Now clean with no minimum age which will remove the outhash
+        result = self.client.clean_unused(0)
+        self.assertEqual(result["count"], 1)
+        result_outhash = self.client.get_outhash(self.METHOD, outhash, taskhash, False)
+        self.assertIsNone(result_outhash)
+
     def test_huge_message(self):
         # Simple test that hashes can be created
         taskhash = 'c665584ee6817aa99edfc77a44dd853828279370'

bitbake/lib/prserv/serv.py

@@ -344,9 +344,9 @@ def auto_shutdown():
 def ping(host, port):
     from . import client
 
-    conn = client.PRClient()
-    conn.connect_tcp(host, port)
-    return conn.ping()
+    with client.PRClient() as conn:
+        conn.connect_tcp(host, port)
+        return conn.ping()
 
 def connect(host, port):
     from . import client

bitbake/lib/toaster/bldcollector/urls.py

@@ -6,7 +6,7 @@
 # SPDX-License-Identifier: GPL-2.0-only
 #
 
-from django.conf.urls import url
+from django.urls import re_path as url
 
 import bldcollector.views
 

bitbake/lib/toaster/bldcollector/views.py

@@ -14,8 +14,11 @@ import subprocess
 import toastermain
 from django.views.decorators.csrf import csrf_exempt
 
+from toastermain.logs import log_view_mixin
+
 
 @csrf_exempt
+@log_view_mixin
 def eventfile(request):
     """ Receives a file by POST, and runs toaster-eventreply on this file """
     if request.method != "POST":

bitbake/lib/toaster/bldcontrol/models.py

@@ -4,7 +4,7 @@
 
 from __future__ import unicode_literals
 from django.db import models
-from django.utils.encoding import force_text
+from django.utils.encoding import force_str
 from orm.models import Project, Build, Layer_Version
 
 import logging
@@ -124,7 +124,7 @@ class BuildRequest(models.Model):
         return self.brvariable_set.get(name="MACHINE").value
 
     def __str__(self):
-        return force_text('%s %s' % (self.project, self.get_state_display()))
+        return force_str('%s %s' % (self.project, self.get_state_display()))
 
 # These tables specify the settings for running an actual build.
 # They MUST be kept in sync with the tables in orm.models.Project*

bitbake/lib/toaster/logs/.gitignore

@@ -0,0 +1 @@
+*.log*

bitbake/lib/toaster/orm/models.py

@@ -107,7 +107,7 @@ class ToasterSetting(models.Model):
 
 
 class ProjectManager(models.Manager):
-    def create_project(self, name, release, existing_project=None):
+    def create_project(self, name, release, existing_project=None, imported=False):
         if existing_project and (release is not None):
             prj = existing_project
             prj.bitbake_version = release.bitbake_version
@@ -134,19 +134,19 @@ class ProjectManager(models.Manager):
 
         if release is None:
             return prj
+        if not imported:
+            for rdl in release.releasedefaultlayer_set.all():
+                lv = Layer_Version.objects.filter(
+                    layer__name=rdl.layer_name,
+                    release=release).first()
 
-        for rdl in release.releasedefaultlayer_set.all():
-            lv = Layer_Version.objects.filter(
-                layer__name=rdl.layer_name,
-                release=release).first()
-
-            if lv:
-                ProjectLayer.objects.create(project=prj,
-                                            layercommit=lv,
-                                            optional=False)
-            else:
-                logger.warning("Default project layer %s not found" %
-                               rdl.layer_name)
+                if lv:
+                    ProjectLayer.objects.create(project=prj,
+                                                layercommit=lv,
+                                                optional=False)
+                else:
+                    logger.warning("Default project layer %s not found" %
+                                rdl.layer_name)
 
         return prj
 
@@ -1733,7 +1733,7 @@ class CustomImageRecipe(Recipe):
         packages_conf += "\""
 
         base_recipe_path = self.get_base_recipe_file()
-        if base_recipe_path:
+        if base_recipe_path and os.path.isfile(base_recipe_path):
             base_recipe = open(base_recipe_path, 'r').read()
         else:
             # Pass back None to trigger error message to user

bitbake/lib/toaster/tests/browser/selenium_helpers_base.py

@@ -21,6 +21,7 @@ import unittest
 
 from selenium import webdriver
 from selenium.webdriver.support.ui import WebDriverWait
+from selenium.webdriver.common.by import By
 from selenium.webdriver.common.desired_capabilities import DesiredCapabilities
 from selenium.common.exceptions import NoSuchElementException, \
         StaleElementReferenceException, TimeoutException
@@ -32,9 +33,7 @@ def create_selenium_driver(cls,browser='chrome'):
         browser = env_browser
 
     if browser == 'chrome':
-        return webdriver.Chrome(
-            service_args=["--verbose", "--log-path=selenium.log"]
-        )
+        return webdriver.Chrome()
     elif browser == 'firefox':
         return webdriver.Firefox()
     elif browser == 'marionette':
@@ -153,11 +152,11 @@ class SeleniumTestCaseBase(unittest.TestCase):
 
     def find(self, selector):
         """ Find single element by CSS selector """
-        return self.driver.find_element_by_css_selector(selector)
+        return self.driver.find_element(By.CSS_SELECTOR, selector)
 
     def find_all(self, selector):
         """ Find all elements matching CSS selector """
-        return self.driver.find_elements_by_css_selector(selector)
+        return self.driver.find_elements(By.CSS_SELECTOR, selector)
 
     def element_exists(self, selector):
         """

bitbake/lib/toaster/tests/browser/test_all_builds_page.py

@@ -7,7 +7,7 @@
 # SPDX-License-Identifier: GPL-2.0-only
 #
 
-import re
+import re, time
 
 from django.urls import reverse
 from django.utils import timezone
@@ -15,6 +15,8 @@ from tests.browser.selenium_helpers import SeleniumTestCase
 
 from orm.models import BitbakeVersion, Release, Project, Build, Target
 
+from selenium.webdriver.common.by import By
+
 
 class TestAllBuildsPage(SeleniumTestCase):
     """ Tests for all builds page /builds/ """
@@ -91,7 +93,7 @@ class TestAllBuildsPage(SeleniumTestCase):
         found_row = None
         for row in rows:
 
-            outcome_links = row.find_elements_by_css_selector(selector)
+            outcome_links = row.find_elements(By.CSS_SELECTOR, selector)
             if len(outcome_links) == 1:
                 found_row = row
                 break
@@ -131,17 +133,19 @@ class TestAllBuildsPage(SeleniumTestCase):
         url = reverse('all-builds')
         self.get(url)
 
+        # should see a rebuild button for non-command-line builds
+        selector = 'div[data-latest-build-result="%s"] .rebuild-btn' % build1.id
+        time.sleep(2)
+        run_again_button = self.find_all(selector)
+        self.assertEqual(len(run_again_button), 1,
+                         'should see a rebuild button for non-cli builds')
+
         # shouldn't see a rebuild button for command-line builds
         selector = 'div[data-latest-build-result="%s"] .rebuild-btn' % default_build.id
         run_again_button = self.find_all(selector)
         self.assertEqual(len(run_again_button), 0,
                          'should not see a rebuild button for cli builds')
 
-        # should see a rebuild button for non-command-line builds
-        selector = 'div[data-latest-build-result="%s"] .rebuild-btn' % build1.id
-        run_again_button = self.find_all(selector)
-        self.assertEqual(len(run_again_button), 1,
-                         'should see a rebuild button for non-cli builds')
 
     def test_tooltips_on_project_name(self):
         """
@@ -198,24 +202,24 @@ class TestAllBuildsPage(SeleniumTestCase):
 
         # test recent builds area for successful build
         element = self._get_build_time_element(build1)
-        links = element.find_elements_by_css_selector('a')
+        links = element.find_elements(By.CSS_SELECTOR, 'a')
         msg = 'should be a link on the build time for a successful recent build'
         self.assertEquals(len(links), 1, msg)
 
         # test recent builds area for failed build
         element = self._get_build_time_element(build2)
-        links = element.find_elements_by_css_selector('a')
+        links = element.find_elements(By.CSS_SELECTOR, 'a')
         msg = 'should not be a link on the build time for a failed recent build'
         self.assertEquals(len(links), 0, msg)
 
         # test the time column for successful build
         build1_row = self._get_row_for_build(build1)
-        links = build1_row.find_elements_by_css_selector('td.time a')
+        links = build1_row.find_elements(By.CSS_SELECTOR, 'td.time a')
         msg = 'should be a link on the build time for a successful build'
         self.assertEquals(len(links), 1, msg)
 
         # test the time column for failed build
         build2_row = self._get_row_for_build(build2)
-        links = build2_row.find_elements_by_css_selector('td.time a')
+        links = build2_row.find_elements(By.CSS_SELECTOR, 'td.time a')
         msg = 'should not be a link on the build time for a failed build'
         self.assertEquals(len(links), 0, msg)

bitbake/lib/toaster/tests/browser/test_all_projects_page.py

@@ -16,6 +16,8 @@ from tests.browser.selenium_helpers import SeleniumTestCase
 from orm.models import BitbakeVersion, Release, Project, Build
 from orm.models import ProjectVariable
 
+from selenium.webdriver.common.by import By
+
 class TestAllProjectsPage(SeleniumTestCase):
     """ Browser tests for projects page /projects/ """
 
@@ -117,7 +119,7 @@ class TestAllProjectsPage(SeleniumTestCase):
 
         # check the release text for the default project
         selector = 'span[data-project-field="release"] span.text-muted'
-        element = default_project_row.find_element_by_css_selector(selector)
+        element = default_project_row.find_element(By.CSS_SELECTOR, selector)
         text = element.text.strip()
         self.assertEqual(text, 'Not applicable',
                          'release should be "not applicable" for default project')
@@ -127,7 +129,7 @@ class TestAllProjectsPage(SeleniumTestCase):
 
         # check the link in the release cell for the other project
         selector = 'span[data-project-field="release"]'
-        element = other_project_row.find_element_by_css_selector(selector)
+        element = other_project_row.find_element(By.CSS_SELECTOR, selector)
         text = element.text.strip()
         self.assertEqual(text, self.release.name,
                          'release name should be shown for non-default project')
@@ -152,7 +154,7 @@ class TestAllProjectsPage(SeleniumTestCase):
 
         # check the machine cell for the default project
         selector = 'span[data-project-field="machine"] span.text-muted'
-        element = default_project_row.find_element_by_css_selector(selector)
+        element = default_project_row.find_element(By.CSS_SELECTOR, selector)
         text = element.text.strip()
         self.assertEqual(text, 'Not applicable',
                          'machine should be not applicable for default project')
@@ -162,7 +164,7 @@ class TestAllProjectsPage(SeleniumTestCase):
 
         # check the link in the machine cell for the other project
         selector = 'span[data-project-field="machine"]'
-        element = other_project_row.find_element_by_css_selector(selector)
+        element = other_project_row.find_element(By.CSS_SELECTOR, selector)
         text = element.text.strip()
         self.assertEqual(text, self.MACHINE_NAME,
                          'machine name should be shown for non-default project')
@@ -187,7 +189,7 @@ class TestAllProjectsPage(SeleniumTestCase):
 
         # check the link on the name field
         selector = 'span[data-project-field="name"] a'
-        element = default_project_row.find_element_by_css_selector(selector)
+        element = default_project_row.find_element(By.CSS_SELECTOR, selector)
         link_url = element.get_attribute('href').strip()
         expected_url = reverse('projectbuilds', args=(self.default_project.id,))
         msg = 'link on default project name should point to builds but was %s' % link_url
@@ -198,7 +200,7 @@ class TestAllProjectsPage(SeleniumTestCase):
 
         # check the link for the other project
         selector = 'span[data-project-field="name"] a'
-        element = other_project_row.find_element_by_css_selector(selector)
+        element = other_project_row.find_element(By.CSS_SELECTOR, selector)
         link_url = element.get_attribute('href').strip()
         expected_url = reverse('project', args=(self.project.id,))
         msg = 'link on project name should point to configuration but was %s' % link_url

bitbake/lib/toaster/tests/browser/test_builddashboard_page.py

@@ -15,6 +15,8 @@ from tests.browser.selenium_helpers import SeleniumTestCase
 from orm.models import Project, Release, BitbakeVersion, Build, LogMessage
 from orm.models import Layer, Layer_Version, Recipe, CustomImageRecipe, Variable
 
+from selenium.webdriver.common.by import By
+
 class TestBuildDashboardPage(SeleniumTestCase):
     """ Tests for the build dashboard /build/X """
 
@@ -183,7 +185,7 @@ class TestBuildDashboardPage(SeleniumTestCase):
 
         found = False
         for element in message_elements:
-            log_message_text = element.find_element_by_tag_name('pre').text.strip()
+            log_message_text = element.find_element(By.TAG_NAME, 'pre').text.strip()
             text_matches = (log_message_text == expected_text)
 
             log_message_pk = element.get_attribute('data-log-message-id')
@@ -213,7 +215,7 @@ class TestBuildDashboardPage(SeleniumTestCase):
         the WebElement modal match the list of text values in expected
         """
         # labels containing the radio buttons we're testing for
-        labels = modal.find_elements_by_css_selector(".radio")
+        labels = modal.find_elements(By.CSS_SELECTOR,".radio")
 
         labels_text = [lab.text for lab in labels]
         self.assertEqual(len(labels_text), len(expected))
@@ -248,7 +250,7 @@ class TestBuildDashboardPage(SeleniumTestCase):
         selector = '[data-role="edit-custom-image-trigger"]'
         self.click(selector)
 
-        modal = self.driver.find_element_by_id('edit-custom-image-modal')
+        modal = self.driver.find_element(By.ID, 'edit-custom-image-modal')
         self.wait_until_visible("#edit-custom-image-modal")
 
         # recipes we expect to see in the edit custom image modal
@@ -270,7 +272,7 @@ class TestBuildDashboardPage(SeleniumTestCase):
         selector = '[data-role="new-custom-image-trigger"]'
         self.click(selector)
 
-        modal = self.driver.find_element_by_id('new-custom-image-modal')
+        modal = self.driver.find_element(By.ID,'new-custom-image-modal')
         self.wait_until_visible("#new-custom-image-modal")
 
         # recipes we expect to see in the new custom image modal

bitbake/lib/toaster/tests/browser/test_layerdetails_page.py

@@ -97,6 +97,8 @@ class TestLayerDetailsPage(SeleniumTestCase):
                             "Expecting any of \"%s\"but got \"%s\"" %
                             (self.initial_values, value))
 
+            # Make sure the input visible beofre sending keys
+            self.wait_until_visible("#layer-git input[type=text]")
             inputs.send_keys("-edited")
 
         # Save the new values

bitbake/lib/toaster/tests/browser/test_most_recent_builds_states.py

@@ -6,7 +6,7 @@
 #
 # Copyright (C) 2013-2016 Intel Corporation
 #
-
+import time
 from django.urls import reverse
 from django.utils import timezone
 from tests.browser.selenium_helpers import SeleniumTestCase
@@ -14,6 +14,8 @@ from tests.browser.selenium_helpers_base import Wait
 from orm.models import Project, Build, Task, Recipe, Layer, Layer_Version
 from bldcontrol.models import BuildRequest
 
+from selenium.webdriver.common.by import By
+
 class TestMostRecentBuildsStates(SeleniumTestCase):
     """ Test states update correctly in most recent builds area """
 
@@ -52,6 +54,7 @@ class TestMostRecentBuildsStates(SeleniumTestCase):
         build.outcome = Build.IN_PROGRESS
         build.recipes_to_parse = recipes_to_parse
         build.recipes_parsed = 0
+        build.save()
 
         build_request.state = BuildRequest.REQ_INPROGRESS
         build_request.save()
@@ -62,7 +65,7 @@ class TestMostRecentBuildsStates(SeleniumTestCase):
         element = self.wait_until_visible(selector)
 
         bar_selector = '#recipes-parsed-percentage-bar-%s' % build.id
-        bar_element = element.find_element_by_css_selector(bar_selector)
+        bar_element = element.find_element(By.CSS_SELECTOR, bar_selector)
         self.assertEqual(bar_element.value_of_css_property('width'), '0px',
             'recipe parse progress should be at 0')
 
@@ -73,7 +76,7 @@ class TestMostRecentBuildsStates(SeleniumTestCase):
         self.get(url)
 
         element = self.wait_until_visible(selector)
-        bar_element = element.find_element_by_css_selector(bar_selector)
+        bar_element = element.find_element(By.CSS_SELECTOR, bar_selector)
         recipe_bar_updated = lambda driver: \
             bar_element.get_attribute('style') == 'width: 50%;'
         msg = 'recipe parse progress bar should update to 50%'
@@ -98,7 +101,7 @@ class TestMostRecentBuildsStates(SeleniumTestCase):
             'Tasks starting', 'build should show "tasks starting" status')
 
         # first task finished; check tasks progress bar
-        task1.order = 1
+        task1.outcome = Task.OUTCOME_SUCCESS
         task1.save()
 
         self.get(url)
@@ -107,7 +110,7 @@ class TestMostRecentBuildsStates(SeleniumTestCase):
         element = self.wait_until_visible(selector)
 
         bar_selector = '#build-pc-done-bar-%s' % build.id
-        bar_element = element.find_element_by_css_selector(bar_selector)
+        bar_element = element.find_element(By.CSS_SELECTOR, bar_selector)
 
         task_bar_updated = lambda driver: \
             bar_element.get_attribute('style') == 'width: 50%;'
@@ -115,13 +118,13 @@ class TestMostRecentBuildsStates(SeleniumTestCase):
         element = Wait(self.driver).until(task_bar_updated, msg)
 
         # last task finished; check tasks progress bar updates
-        task2.order = 2
+        task2.outcome = Task.OUTCOME_SUCCESS
         task2.save()
 
         self.get(url)
 
         element = self.wait_until_visible(selector)
-        bar_element = element.find_element_by_css_selector(bar_selector)
+        bar_element = element.find_element(By.CSS_SELECTOR, bar_selector)
         task_bar_updated = lambda driver: \
             bar_element.get_attribute('style') == 'width: 100%;'
         msg = 'tasks progress bar should update to 100%'

bitbake/lib/toaster/tests/browser/test_new_custom_image_page.py

@@ -6,6 +6,7 @@
 #
 # SPDX-License-Identifier: GPL-2.0-only
 #
+from bldcontrol.models import BuildEnvironment
 
 from django.urls import reverse
 from tests.browser.selenium_helpers import SeleniumTestCase
@@ -18,6 +19,9 @@ class TestNewCustomImagePage(SeleniumTestCase):
     CUSTOM_IMAGE_NAME = 'roopa-doopa'
 
     def setUp(self):
+        BuildEnvironment.objects.get_or_create(
+            betype=BuildEnvironment.TYPE_LOCAL,
+        )
         release = Release.objects.create(
             name='baz',
             bitbake_version=BitbakeVersion.objects.create(name='v1')

bitbake/lib/toaster/tests/browser/test_new_project_page.py

@@ -6,11 +6,13 @@
 #
 # SPDX-License-Identifier: GPL-2.0-only
 #
+import time
 
 from django.urls import reverse
 from tests.browser.selenium_helpers import SeleniumTestCase
 from selenium.webdriver.support.ui import Select
 from selenium.common.exceptions import InvalidElementStateException
+from selenium.webdriver.common.by import By
 
 from orm.models import Project, Release, BitbakeVersion
 
@@ -47,13 +49,14 @@ class TestNewProjectPage(SeleniumTestCase):
 
         url = reverse('newproject')
         self.get(url)
-
         self.enter_text('#new-project-name', project_name)
 
         select = Select(self.find('#projectversion'))
         select.select_by_value(str(self.release.pk))
 
+        time.sleep(1)
         self.click("#create-project-button")
+        time.sleep(2)
 
         # We should get redirected to the new project's page with the
         # notification at the top
@@ -84,6 +87,12 @@ class TestNewProjectPage(SeleniumTestCase):
         select = Select(self.find('#projectversion'))
         select.select_by_value(str(self.release.pk))
 
+        radio = self.driver.find_element(By.ID, 'type-new')
+        radio.click()
+
+        self.click("#create-project-button")
+        time.sleep(2)
+
         element = self.wait_until_visible('#hint-error-project-name')
 
         self.assertTrue(("Project names must be unique" in element.text),
@@ -96,6 +105,7 @@ class TestNewProjectPage(SeleniumTestCase):
         except InvalidElementStateException:
             pass
 
+        time.sleep(2)
         self.assertTrue(
             (Project.objects.filter(name=project_name).count() == 1),
             "New project not found in database")

bitbake/lib/toaster/tests/browser/test_project_config_page.py

@@ -11,6 +11,7 @@ from django.urls import reverse
 from tests.browser.selenium_helpers import SeleniumTestCase
 
 from orm.models import BitbakeVersion, Release, Project, ProjectVariable
+from selenium.webdriver.common.by import By
 
 class TestProjectConfigsPage(SeleniumTestCase):
     """ Test data at /project/X/builds is displayed correctly """
@@ -66,7 +67,7 @@ class TestProjectConfigsPage(SeleniumTestCase):
 
         self.enter_text('#new-imagefs_types', imagefs_type)
 
-        checkboxes = self.driver.find_elements_by_xpath("//input[@class='fs-checkbox-fstypes']")
+        checkboxes = self.driver.find_elements(By.XPATH, "//input[@class='fs-checkbox-fstypes']")
 
         for checkbox in checkboxes:
             if checkbox.get_attribute("value") == "btrfs":
@@ -95,7 +96,7 @@ class TestProjectConfigsPage(SeleniumTestCase):
         for checkbox in checkboxes:
             if checkbox.get_attribute("value") == "cpio":
                checkbox.click()
-               element = self.driver.find_element_by_id('new-imagefs_types')
+               element = self.driver.find_element(By.ID, 'new-imagefs_types')
 
                self.wait_until_visible('#new-imagefs_types')
 
@@ -129,7 +130,7 @@ class TestProjectConfigsPage(SeleniumTestCase):
         self.assertTrue((self.INVALID_PATH_START_TEXT in element.text), msg)
 
         # downloads dir path has a space
-        self.driver.find_element_by_id('new-dl_dir').clear()
+        self.driver.find_element(By.ID, 'new-dl_dir').clear()
         self.enter_text('#new-dl_dir', '/foo/bar a')
 
         element = self.wait_until_visible('#hintError-dl_dir')
@@ -137,7 +138,7 @@ class TestProjectConfigsPage(SeleniumTestCase):
         self.assertTrue((self.INVALID_PATH_CHAR_TEXT in element.text), msg)
 
         # downloads dir path starts with ${...} but has a space
-        self.driver.find_element_by_id('new-dl_dir').clear()
+        self.driver.find_element(By.ID,'new-dl_dir').clear()
         self.enter_text('#new-dl_dir', '${TOPDIR}/down foo')
 
         element = self.wait_until_visible('#hintError-dl_dir')
@@ -145,18 +146,18 @@ class TestProjectConfigsPage(SeleniumTestCase):
         self.assertTrue((self.INVALID_PATH_CHAR_TEXT in element.text), msg)
 
         # downloads dir path starts with /
-        self.driver.find_element_by_id('new-dl_dir').clear()
+        self.driver.find_element(By.ID,'new-dl_dir').clear()
         self.enter_text('#new-dl_dir', '/bar/foo')
 
-        hidden_element = self.driver.find_element_by_id('hintError-dl_dir')
+        hidden_element = self.driver.find_element(By.ID,'hintError-dl_dir')
         self.assertEqual(hidden_element.is_displayed(), False,
             'downloads directory path valid but treated as invalid')
 
         # downloads dir path starts with ${...}
-        self.driver.find_element_by_id('new-dl_dir').clear()
+        self.driver.find_element(By.ID,'new-dl_dir').clear()
         self.enter_text('#new-dl_dir', '${TOPDIR}/down')
 
-        hidden_element = self.driver.find_element_by_id('hintError-dl_dir')
+        hidden_element = self.driver.find_element(By.ID,'hintError-dl_dir')
         self.assertEqual(hidden_element.is_displayed(), False,
             'downloads directory path valid but treated as invalid')
 
@@ -184,7 +185,7 @@ class TestProjectConfigsPage(SeleniumTestCase):
         self.assertTrue((self.INVALID_PATH_START_TEXT in element.text), msg)
 
         # path has a space
-        self.driver.find_element_by_id('new-sstate_dir').clear()
+        self.driver.find_element(By.ID, 'new-sstate_dir').clear()
         self.enter_text('#new-sstate_dir', '/foo/bar a')
 
         element = self.wait_until_visible('#hintError-sstate_dir')
@@ -192,7 +193,7 @@ class TestProjectConfigsPage(SeleniumTestCase):
         self.assertTrue((self.INVALID_PATH_CHAR_TEXT in element.text), msg)
 
         # path starts with ${...} but has a space
-        self.driver.find_element_by_id('new-sstate_dir').clear()
+        self.driver.find_element(By.ID,'new-sstate_dir').clear()
         self.enter_text('#new-sstate_dir', '${TOPDIR}/down foo')
 
         element = self.wait_until_visible('#hintError-sstate_dir')
@@ -200,18 +201,18 @@ class TestProjectConfigsPage(SeleniumTestCase):
         self.assertTrue((self.INVALID_PATH_CHAR_TEXT in element.text), msg)
 
         # path starts with /
-        self.driver.find_element_by_id('new-sstate_dir').clear()
+        self.driver.find_element(By.ID,'new-sstate_dir').clear()
         self.enter_text('#new-sstate_dir', '/bar/foo')
 
-        hidden_element = self.driver.find_element_by_id('hintError-sstate_dir')
+        hidden_element = self.driver.find_element(By.ID, 'hintError-sstate_dir')
         self.assertEqual(hidden_element.is_displayed(), False,
             'sstate directory path valid but treated as invalid')
 
         # paths starts with ${...}
-        self.driver.find_element_by_id('new-sstate_dir').clear()
+        self.driver.find_element(By.ID, 'new-sstate_dir').clear()
         self.enter_text('#new-sstate_dir', '${TOPDIR}/down')
 
-        hidden_element = self.driver.find_element_by_id('hintError-sstate_dir')
+        hidden_element = self.driver.find_element(By.ID, 'hintError-sstate_dir')
         self.assertEqual(hidden_element.is_displayed(), False,
             'sstate directory path valid but treated as invalid')
 

bitbake/lib/toaster/tests/browser/test_toastertable_ui.py

@@ -13,6 +13,7 @@ from django.urls import reverse
 from django.utils import timezone
 from tests.browser.selenium_helpers import SeleniumTestCase
 from orm.models import BitbakeVersion, Release, Project, Build
+from selenium.webdriver.common.by import By
 
 class TestToasterTableUI(SeleniumTestCase):
     """
@@ -33,7 +34,7 @@ class TestToasterTableUI(SeleniumTestCase):
         table: WebElement for a ToasterTable
         """
         selector = 'thead a.sorted'
-        heading = table.find_element_by_css_selector(selector)
+        heading = table.find_element(By.CSS_SELECTOR, selector)
         return heading.get_attribute('innerHTML').strip()
 
     def _get_datetime_from_cell(self, row, selector):
@@ -45,7 +46,7 @@ class TestToasterTableUI(SeleniumTestCase):
         selector: CSS selector to use to find the cell containing the date time
         string
         """
-        cell = row.find_element_by_css_selector(selector)
+        cell = row.find_element(By.CSS_SELECTOR, selector)
         cell_text = cell.get_attribute('innerHTML').strip()
         return datetime.strptime(cell_text, '%d/%m/%y %H:%M')
 
@@ -105,7 +106,7 @@ class TestToasterTableUI(SeleniumTestCase):
         self.click('#checkbox-started_on')
 
         # sort by started_on column
-        links = table.find_elements_by_css_selector('th.started_on a')
+        links = table.find_elements(By.CSS_SELECTOR, 'th.started_on a')
         for link in links:
             if link.get_attribute('innerHTML').strip() == 'Started on':
                 link.click()

bitbake/lib/toaster/tests/builds/test_core_image_min.py

@@ -26,6 +26,7 @@ class BuildCoreImageMinimal(BuildTest):
 
     def setUp(self):
         self.completed_build = self.build("core-image-minimal")
+        self.built = self.target_already_built("core-image-minimal")
 
     # Check if build name is unique - tc_id=795
     def test_Build_Unique_Name(self):

bitbake/lib/toaster/tests/functional/functional_helpers.py

@@ -16,6 +16,9 @@ import re
 
 from tests.browser.selenium_helpers_base import SeleniumTestCaseBase
 from tests.builds.buildtest import load_build_environment
+from bldcontrol.models import BuildEnvironment
+from selenium.webdriver.common.by import By
+from selenium.common.exceptions import NoSuchElementException
 
 logger = logging.getLogger("toaster")
 
@@ -30,6 +33,8 @@ class SeleniumFunctionalTestCase(SeleniumTestCaseBase):
             raise RuntimeError("Please initialise django with the tests settings:  " \
                 "DJANGO_SETTINGS_MODULE='toastermain.settings_test'")
 
+        if BuildEnvironment.objects.count() == 0:
+            BuildEnvironment.objects.create(betype=BuildEnvironment.TYPE_LOCAL)
         load_build_environment()
 
         # start toaster
@@ -74,8 +79,8 @@ class SeleniumFunctionalTestCase(SeleniumTestCaseBase):
         """
         try:
             table_element = self.get_table_element(table_id)
-            element = table_element.find_element_by_link_text(link_text)
-        except self.NoSuchElementException:
+            element = table_element.find_element(By.LINK_TEXT, link_text)
+        except NoSuchElementException:
             print('no element found')
             raise
         return element
@@ -85,8 +90,8 @@ class SeleniumFunctionalTestCase(SeleniumTestCaseBase):
 #return whole-table element
             element_xpath = "//*[@id='" + table_id + "']"
             try:
-                element = self.driver.find_element_by_xpath(element_xpath)
-            except self.NoSuchElementException:
+                element = self.driver.find_element(By.XPATH, element_xpath)
+            except NoSuchElementException:
                 raise
             return element
         row = coordinate[0]
@@ -95,8 +100,8 @@ class SeleniumFunctionalTestCase(SeleniumTestCaseBase):
 #return whole-row element
             element_xpath = "//*[@id='" + table_id + "']/tbody/tr[" + str(row) + "]"
             try:
-                element = self.driver.find_element_by_xpath(element_xpath)
-            except self.NoSuchElementException:
+                element = self.driver.find_element(By.XPATH, element_xpath)
+            except NoSuchElementException:
                 return False
             return element
 #now we are looking for an element with specified X and Y
@@ -104,7 +109,7 @@ class SeleniumFunctionalTestCase(SeleniumTestCaseBase):
 
         element_xpath = "//*[@id='" + table_id + "']/tbody/tr[" + str(row) + "]/td[" + str(column) + "]"
         try:
-            element = self.driver.find_element_by_xpath(element_xpath)
-        except self.NoSuchElementException:
+            element = self.driver.find_element(By.XPATH, element_xpath)
+        except NoSuchElementException:
             return False
         return element

bitbake/lib/toaster/tests/functional/test_functional_basic.py

@@ -7,20 +7,22 @@
 # SPDX-License-Identifier: GPL-2.0-only
 #
 
-import re
+import re, time
+from django.urls import reverse
 from tests.functional.functional_helpers import SeleniumFunctionalTestCase
 from orm.models import Project
+from selenium.webdriver.common.by import By
 
 class FuntionalTestBasic(SeleniumFunctionalTestCase):
 
 #   testcase (1514)
     def test_create_slenium_project(self):
         project_name = 'selenium-project'
-        self.get('')
-        self.driver.find_element_by_link_text("To start building, create your first Toaster project").click()
-        self.driver.find_element_by_id("new-project-name").send_keys(project_name)
-        self.driver.find_element_by_id('projectversion').click()
-        self.driver.find_element_by_id("create-project-button").click()
+        self.get(reverse('newproject'))
+        self.driver.find_element(By.ID, "new-project-name").send_keys(project_name)
+        self.driver.find_element(By.ID, 'projectversion').click()
+        self.driver.find_element(By.ID, "create-project-button").click()
+        time.sleep(2)
         element = self.wait_until_visible('#project-created-notification')
         self.assertTrue(self.element_exists('#project-created-notification'),'Project creation notification not shown')
         self.assertTrue(project_name in element.text,
@@ -30,82 +32,87 @@ class FuntionalTestBasic(SeleniumFunctionalTestCase):
 
  #  testcase (1515)
     def test_verify_left_bar_menu(self):
-        self.get('')
+        self.get(reverse('all-projects'))
         self.wait_until_visible('#projectstable')
         self.find_element_by_link_text_in_table('projectstable', 'selenium-project').click()
+        time.sleep(2)
         self.assertTrue(self.element_exists('#config-nav'),'Configuration Tab does not exist')
         project_URL=self.get_URL()
-        self.driver.find_element_by_xpath('//a[@href="'+project_URL+'"]').click()
+        self.driver.find_element(By.XPATH, '//a[@href="'+project_URL+'"]').click()
+        time.sleep(2)
 
         try:
-            self.driver.find_element_by_xpath("//*[@id='config-nav']/ul/li/a[@href="+'"'+project_URL+'customimages/"'+"]").click()
-            self.assertTrue(re.search("Custom images",self.driver.find_element_by_xpath("//div[@class='col-md-10']").text),'Custom images information is not loading properly')
+            self.driver.find_element(By.XPATH, "//*[@id='config-nav']/ul/li/a[@href="+'"'+project_URL+'customimages/"'+"]").click()
+            time.sleep(2)
+            self.assertTrue(re.search("Custom images",self.driver.find_element(By.XPATH, "//div[@class='col-md-10']").text),'Custom images information is not loading properly')
         except:
             self.fail(msg='No Custom images tab available')
 
         try:
-            self.driver.find_element_by_xpath("//*[@id='config-nav']/ul/li/a[@href="+'"'+project_URL+'images/"'+"]").click()
-            self.assertTrue(re.search("Compatible image recipes",self.driver.find_element_by_xpath("//div[@class='col-md-10']").text),'The Compatible image recipes information is not loading properly')
+            self.driver.find_element(By.XPATH, "//*[@id='config-nav']/ul/li/a[@href="+'"'+project_URL+'images/"'+"]").click()
+            self.assertTrue(re.search("Compatible image recipes",self.driver.find_element(By.XPATH, "//div[@class='col-md-10']").text),'The Compatible image recipes information is not loading properly')
         except:
             self.fail(msg='No Compatible image tab available')
 
         try:
-            self.driver.find_element_by_xpath("//*[@id='config-nav']/ul/li/a[@href="+'"'+project_URL+'softwarerecipes/"'+"]").click()
-            self.assertTrue(re.search("Compatible software recipes",self.driver.find_element_by_xpath("//div[@class='col-md-10']").text),'The Compatible software recipe information is not loading properly')
+            self.driver.find_element(By.XPATH, "//*[@id='config-nav']/ul/li/a[@href="+'"'+project_URL+'softwarerecipes/"'+"]").click()
+            self.assertTrue(re.search("Compatible software recipes",self.driver.find_element(By.XPATH, "//div[@class='col-md-10']").text),'The Compatible software recipe information is not loading properly')
         except:
             self.fail(msg='No Compatible software recipe tab available')
 
         try:
-            self.driver.find_element_by_xpath("//*[@id='config-nav']/ul/li/a[@href="+'"'+project_URL+'machines/"'+"]").click()
-            self.assertTrue(re.search("Compatible machines",self.driver.find_element_by_xpath("//div[@class='col-md-10']").text),'The Compatible machine information is not loading properly')
+            self.driver.find_element(By.XPATH, "//*[@id='config-nav']/ul/li/a[@href="+'"'+project_URL+'machines/"'+"]").click()
+            self.assertTrue(re.search("Compatible machines",self.driver.find_element(By.XPATH, "//div[@class='col-md-10']").text),'The Compatible machine information is not loading properly')
         except:
             self.fail(msg='No Compatible machines tab available')
 
         try:
-            self.driver.find_element_by_xpath("//*[@id='config-nav']/ul/li/a[@href="+'"'+project_URL+'layers/"'+"]").click()
-            self.assertTrue(re.search("Compatible layers",self.driver.find_element_by_xpath("//div[@class='col-md-10']").text),'The Compatible layer information is not loading properly')
+            self.driver.find_element(By.XPATH, "//*[@id='config-nav']/ul/li/a[@href="+'"'+project_URL+'layers/"'+"]").click()
+            self.assertTrue(re.search("Compatible layers",self.driver.find_element(By.XPATH, "//div[@class='col-md-10']").text),'The Compatible layer information is not loading properly')
         except:
             self.fail(msg='No Compatible layers tab available')
 
         try:
-            self.driver.find_element_by_xpath("//*[@id='config-nav']/ul/li/a[@href="+'"'+project_URL+'configuration"'+"]").click()
-            self.assertTrue(re.search("Bitbake variables",self.driver.find_element_by_xpath("//div[@class='col-md-10']").text),'The Bitbake variables information is not loading properly')
+            self.driver.find_element(By.XPATH, "//*[@id='config-nav']/ul/li/a[@href="+'"'+project_URL+'configuration"'+"]").click()
+            self.assertTrue(re.search("Bitbake variables",self.driver.find_element(By.XPATH, "//div[@class='col-md-10']").text),'The Bitbake variables information is not loading properly')
         except:
             self.fail(msg='No Bitbake variables tab available')
 
 #   testcase (1516)
     def test_review_configuration_information(self):
         self.get('')
-        self.driver.find_element_by_xpath("//div[@id='global-nav']/ul/li/a[@href="+'"'+'/toastergui/projects/'+'"'+"]").click()
+        self.driver.find_element(By.XPATH, "//div[@id='global-nav']/ul/li/a[@href="+'"'+'/toastergui/projects/'+'"'+"]").click()
+        time.sleep(2)
         self.wait_until_visible('#projectstable')
         self.find_element_by_link_text_in_table('projectstable', 'selenium-project').click()
         project_URL=self.get_URL()
-
+        time.sleep(2)
         try:
            self.assertTrue(self.element_exists('#machine-section'),'Machine section for the project configuration page does not exist')
-           self.assertTrue(re.search("qemux86",self.driver.find_element_by_xpath("//span[@id='project-machine-name']").text),'The machine type is not assigned')
-           self.driver.find_element_by_xpath("//span[@id='change-machine-toggle']").click()
+           self.assertTrue(re.search("qemux86",self.driver.find_element(By.XPATH, "//span[@id='project-machine-name']").text),'The machine type is not assigned')
+           self.driver.find_element(By.XPATH, "//span[@id='change-machine-toggle']").click()
+           time.sleep(2)
            self.wait_until_visible('#select-machine-form')
            self.wait_until_visible('#cancel-machine-change')
-           self.driver.find_element_by_xpath("//form[@id='select-machine-form']/a[@id='cancel-machine-change']").click()
+           self.driver.find_element(By.XPATH, "//form[@id='select-machine-form']/a[@id='cancel-machine-change']").click()
         except:
            self.fail(msg='The machine information is wrong in the configuration page')
 
         try:
-           self.driver.find_element_by_id('no-most-built')
+           self.driver.find_element(By.ID, 'no-most-built')
         except:
            self.fail(msg='No Most built information in project detail page')
 
         try:
-           self.assertTrue(re.search("Yocto Project master",self.driver.find_element_by_xpath("//span[@id='project-release-title']").text),'The project release is not defined')
+           self.assertTrue(re.search("Yocto Project master",self.driver.find_element(By.XPATH, "//span[@id='project-release-title']").text),'The project release is not defined')
         except:
            self.fail(msg='No project release title information in project detail page')
 
         try:
-           self.driver.find_element_by_xpath("//div[@id='layer-container']")
-           self.assertTrue(re.search("3",self.driver.find_element_by_id("project-layers-count").text),'There should be 3 layers listed in the layer count')
-           layer_list = self.driver.find_element_by_id("layers-in-project-list")
-           layers = layer_list.find_elements_by_tag_name("li")
+           self.driver.find_element(By.XPATH, "//div[@id='layer-container']")
+           self.assertTrue(re.search("3",self.driver.find_element(By.ID, "project-layers-count").text),'There should be 3 layers listed in the layer count')
+           layer_list = self.driver.find_element(By.ID, "layers-in-project-list")
+           layers = layer_list.find_elements(By.TAG_NAME, "li")
            for layer in layers:
                if re.match ("openembedded-core",layer.text):
                     print ("openembedded-core layer is a default layer in the project configuration")
@@ -121,60 +128,67 @@ class FuntionalTestBasic(SeleniumFunctionalTestCase):
 #   testcase (1517)
     def test_verify_machine_information(self):
         self.get('')
-        self.driver.find_element_by_xpath("//div[@id='global-nav']/ul/li/a[@href="+'"'+'/toastergui/projects/'+'"'+"]").click()
+        self.driver.find_element(By.XPATH, "//div[@id='global-nav']/ul/li/a[@href="+'"'+'/toastergui/projects/'+'"'+"]").click()
+        time.sleep(2)
         self.wait_until_visible('#projectstable')
         self.find_element_by_link_text_in_table('projectstable', 'selenium-project').click()
+        time.sleep(2)
 
         try:
             self.assertTrue(self.element_exists('#machine-section'),'Machine section for the project configuration page does not exist')
-            self.assertTrue(re.search("qemux86",self.driver.find_element_by_id("project-machine-name").text),'The machine type is not assigned')
-            self.driver.find_element_by_id("change-machine-toggle").click()
+            self.assertTrue(re.search("qemux86",self.driver.find_element(By.ID, "project-machine-name").text),'The machine type is not assigned')
+            self.driver.find_element(By.ID, "change-machine-toggle").click()
+            time.sleep(2)
             self.wait_until_visible('#select-machine-form')
             self.wait_until_visible('#cancel-machine-change')
-            self.driver.find_element_by_id("cancel-machine-change").click()
+            self.driver.find_element(By.ID, "cancel-machine-change").click()
         except:
             self.fail(msg='The machine information is wrong in the configuration page')
 
 #   testcase (1518)
     def test_verify_most_built_recipes_information(self):
         self.get('')
-        self.driver.find_element_by_xpath("//div[@id='global-nav']/ul/li/a[@href="+'"'+'/toastergui/projects/'+'"'+"]").click()
+        self.driver.find_element(By.XPATH, "//div[@id='global-nav']/ul/li/a[@href="+'"'+'/toastergui/projects/'+'"'+"]").click()
+        time.sleep(2)
         self.wait_until_visible('#projectstable')
         self.find_element_by_link_text_in_table('projectstable', 'selenium-project').click()
         project_URL=self.get_URL()
-
+        time.sleep(2)
         try:
-            self.assertTrue(re.search("You haven't built any recipes yet",self.driver.find_element_by_id("no-most-built").text),'Default message of no builds is not present')
-            self.driver.find_element_by_xpath("//div[@id='no-most-built']/p/a[@href="+'"'+project_URL+'images/"'+"]").click()
-            self.assertTrue(re.search("Compatible image recipes",self.driver.find_element_by_xpath("//div[@class='col-md-10']").text),'The Choose a recipe to build link  is not working  properly')
+            self.assertTrue(re.search("You haven't built any recipes yet",self.driver.find_element(By.ID, "no-most-built").text),'Default message of no builds is not present')
+            self.driver.find_element(By.XPATH, "//div[@id='no-most-built']/p/a[@href="+'"'+project_URL+'images/"'+"]").click()
+            time.sleep(2)
+            self.assertTrue(re.search("Compatible image recipes",self.driver.find_element(By.XPATH, "//div[@class='col-md-10']").text),'The Choose a recipe to build link  is not working  properly')
         except:
             self.fail(msg='No Most built information in project detail page')
 
 #   testcase (1519)
     def test_verify_project_release_information(self):
         self.get('')
-        self.driver.find_element_by_xpath("//div[@id='global-nav']/ul/li/a[@href="+'"'+'/toastergui/projects/'+'"'+"]").click()
+        self.driver.find_element(By.XPATH, "//div[@id='global-nav']/ul/li/a[@href="+'"'+'/toastergui/projects/'+'"'+"]").click()
+        time.sleep(2)
         self.wait_until_visible('#projectstable')
         self.find_element_by_link_text_in_table('projectstable', 'selenium-project').click()
+        time.sleep(2)
 
         try:
-            self.assertTrue(re.search("Yocto Project master",self.driver.find_element_by_id("project-release-title").text),'The project release is not defined')
+            self.assertTrue(re.search("Yocto Project master",self.driver.find_element(By.ID, "project-release-title").text),'The project release is not defined')
         except:
             self.fail(msg='No project release title information in project detail page')
 
 #   testcase (1520)
     def test_verify_layer_information(self):
         self.get('')
-        self.driver.find_element_by_xpath("//div[@id='global-nav']/ul/li/a[@href="+'"'+'/toastergui/projects/'+'"'+"]").click()
+        self.driver.find_element(By.XPATH, "//div[@id='global-nav']/ul/li/a[@href="+'"'+'/toastergui/projects/'+'"'+"]").click()
         self.wait_until_visible('#projectstable')
         self.find_element_by_link_text_in_table('projectstable', 'selenium-project').click()
         project_URL=self.get_URL()
-
+        time.sleep(2)
         try:
-           self.driver.find_element_by_xpath("//div[@id='layer-container']")
-           self.assertTrue(re.search("3",self.driver.find_element_by_id("project-layers-count").text),'There should be 3 layers listed in the layer count')
-           layer_list = self.driver.find_element_by_id("layers-in-project-list")
-           layers = layer_list.find_elements_by_tag_name("li")
+           self.driver.find_element(By.XPATH, "//div[@id='layer-container']")
+           self.assertTrue(re.search("3",self.driver.find_element(By.ID, "project-layers-count").text),'There should be 3 layers listed in the layer count')
+           layer_list = self.driver.find_element(By.ID, "layers-in-project-list")
+           layers = layer_list.find_elements(By.TAG_NAME, "li")
 
            for layer in layers:
                if re.match ("openembedded-core",layer.text):
@@ -186,43 +200,44 @@ class FuntionalTestBasic(SeleniumFunctionalTestCase):
                else:
                   self.fail(msg='default layers are missing from the project configuration')
 
-           self.driver.find_element_by_xpath("//input[@id='layer-add-input']")
-           self.driver.find_element_by_xpath("//button[@id='add-layer-btn']")
-           self.driver.find_element_by_xpath("//div[@id='layer-container']/form[@class='form-inline']/p/a[@id='view-compatible-layers']")
-           self.driver.find_element_by_xpath("//div[@id='layer-container']/form[@class='form-inline']/p/a[@href="+'"'+project_URL+'importlayer"'+"]")
+           self.driver.find_element(By.XPATH, "//input[@id='layer-add-input']")
+           self.driver.find_element(By.XPATH, "//button[@id='add-layer-btn']")
+           self.driver.find_element(By.XPATH, "//div[@id='layer-container']/form[@class='form-inline']/p/a[@id='view-compatible-layers']")
+           self.driver.find_element(By.XPATH, "//div[@id='layer-container']/form[@class='form-inline']/p/a[@href="+'"'+project_URL+'importlayer"'+"]")
         except:
             self.fail(msg='No Layer information in project detail page')
 
 #   testcase (1521)
     def test_verify_project_detail_links(self):
         self.get('')
-        self.driver.find_element_by_xpath("//div[@id='global-nav']/ul/li/a[@href="+'"'+'/toastergui/projects/'+'"'+"]").click()
+        self.driver.find_element(By.XPATH, "//div[@id='global-nav']/ul/li/a[@href="+'"'+'/toastergui/projects/'+'"'+"]").click()
+        time.sleep(2)
         self.wait_until_visible('#projectstable')
         self.find_element_by_link_text_in_table('projectstable', 'selenium-project').click()
         project_URL=self.get_URL()
-
-        self.driver.find_element_by_xpath("//div[@id='project-topbar']/ul[@class='nav nav-tabs']/li[@id='topbar-configuration-tab']/a[@href="+'"'+project_URL+'"'+"]").click()
-        self.assertTrue(re.search("Configuration",self.driver.find_element_by_xpath("//div[@id='project-topbar']/ul[@class='nav nav-tabs']/li[@id='topbar-configuration-tab']/a[@href="+'"'+project_URL+'"'+"]").text), 'Configuration tab in project topbar is misspelled')
+        time.sleep(2)
+        self.driver.find_element(By.XPATH, "//div[@id='project-topbar']/ul[@class='nav nav-tabs']/li[@id='topbar-configuration-tab']/a[@href="+'"'+project_URL+'"'+"]").click()
+        self.assertTrue(re.search("Configuration",self.driver.find_element(By.XPATH, "//div[@id='project-topbar']/ul[@class='nav nav-tabs']/li[@id='topbar-configuration-tab']/a[@href="+'"'+project_URL+'"'+"]").text), 'Configuration tab in project topbar is misspelled')
 
         try:
-            self.driver.find_element_by_xpath("//div[@id='project-topbar']/ul[@class='nav nav-tabs']/li/a[@href="+'"'+project_URL+'builds/"'+"]").click()
-            self.assertTrue(re.search("Builds",self.driver.find_element_by_xpath("//div[@id='project-topbar']/ul[@class='nav nav-tabs']/li/a[@href="+'"'+project_URL+'builds/"'+"]").text), 'Builds tab in project topbar is misspelled')
-            self.driver.find_element_by_xpath("//div[@id='empty-state-projectbuildstable']")
+            self.driver.find_element(By.XPATH, "//div[@id='project-topbar']/ul[@class='nav nav-tabs']/li/a[@href="+'"'+project_URL+'builds/"'+"]").click()
+            self.assertTrue(re.search("Builds",self.driver.find_element(By.XPATH, "//div[@id='project-topbar']/ul[@class='nav nav-tabs']/li/a[@href="+'"'+project_URL+'builds/"'+"]").text), 'Builds tab in project topbar is misspelled')
+            self.driver.find_element(By.XPATH, "//div[@id='empty-state-projectbuildstable']")
         except:
             self.fail(msg='Builds tab information is not present')
 
         try:
-            self.driver.find_element_by_xpath("//div[@id='project-topbar']/ul[@class='nav nav-tabs']/li/a[@href="+'"'+project_URL+'importlayer"'+"]").click()
-            self.assertTrue(re.search("Import layer",self.driver.find_element_by_xpath("//div[@id='project-topbar']/ul[@class='nav nav-tabs']/li/a[@href="+'"'+project_URL+'importlayer"'+"]").text), 'Import layer tab in project topbar is misspelled')
-            self.driver.find_element_by_xpath("//fieldset[@id='repo-select']")
-            self.driver.find_element_by_xpath("//fieldset[@id='git-repo']")
+            self.driver.find_element(By.XPATH, "//div[@id='project-topbar']/ul[@class='nav nav-tabs']/li/a[@href="+'"'+project_URL+'importlayer"'+"]").click()
+            self.assertTrue(re.search("Import layer",self.driver.find_element(By.XPATH, "//div[@id='project-topbar']/ul[@class='nav nav-tabs']/li/a[@href="+'"'+project_URL+'importlayer"'+"]").text), 'Import layer tab in project topbar is misspelled')
+            self.driver.find_element(By.XPATH, "//fieldset[@id='repo-select']")
+            self.driver.find_element(By.XPATH, "//fieldset[@id='git-repo']")
         except:
             self.fail(msg='Import layer tab not loading properly')
 
         try:
-            self.driver.find_element_by_xpath("//div[@id='project-topbar']/ul[@class='nav nav-tabs']/li/a[@href="+'"'+project_URL+'newcustomimage/"'+"]").click()
-            self.assertTrue(re.search("New custom image",self.driver.find_element_by_xpath("//div[@id='project-topbar']/ul[@class='nav nav-tabs']/li/a[@href="+'"'+project_URL+'newcustomimage/"'+"]").text), 'New custom image tab in project topbar is misspelled')
-            self.assertTrue(re.search("Select the image recipe you want to customise",self.driver.find_element_by_xpath("//div[@class='col-md-12']/h2").text),'The new custom image tab is not loading correctly')
+            self.driver.find_element(By.XPATH, "//div[@id='project-topbar']/ul[@class='nav nav-tabs']/li/a[@href="+'"'+project_URL+'newcustomimage/"'+"]").click()
+            self.assertTrue(re.search("New custom image",self.driver.find_element(By.XPATH, "//div[@id='project-topbar']/ul[@class='nav nav-tabs']/li/a[@href="+'"'+project_URL+'newcustomimage/"'+"]").text), 'New custom image tab in project topbar is misspelled')
+            self.assertTrue(re.search("Select the image recipe you want to customise",self.driver.find_element(By.XPATH, "//div[@class='col-md-12']/h2").text),'The new custom image tab is not loading correctly')
         except:
             self.fail(msg='New custom image tab not loading properly')
 

bitbake/lib/toaster/tests/toaster-tests-requirements.txt

@@ -1 +1 @@
-selenium==2.49.2
+selenium>=4.13.0

bitbake/lib/toaster/tests/views/test_views.py

@@ -19,6 +19,7 @@ from orm.models import Layer_Version, Recipe
 from orm.models import CustomImageRecipe
 from orm.models import CustomImagePackage
 
+from bldcontrol.models import BuildEnvironment
 import inspect
 import toastergui
 
@@ -45,6 +46,9 @@ class ViewTests(TestCase):
         self.cust_package = CustomImagePackage.objects.first()
         self.package = Package.objects.first()
         self.lver = Layer_Version.objects.first()
+        if BuildEnvironment.objects.count() == 0:
+            BuildEnvironment.objects.create(betype=BuildEnvironment.TYPE_LOCAL)
+
 
     def test_get_base_call_returns_html(self):
         """Basic test for all-projects view"""

bitbake/lib/toaster/toastergui/api.py

@@ -11,7 +11,7 @@ import os
 import re
 import logging
 import json
-import subprocess
+import glob
 from collections import Counter
 
 from orm.models import Project, ProjectTarget, Build, Layer_Version
@@ -227,20 +227,18 @@ class XhrSetDefaultImageUrl(View):
 #    same logical name
 #  * Each project that uses a layer will have its own
 #    LayerVersion and Project Layer for it
-#  * During the Paroject delete process, when the last
+#  * During the Project delete process, when the last
 #    LayerVersion for a 'local_source_dir' layer is deleted
 #    then the Layer record is deleted to remove orphans
 #
 
 def scan_layer_content(layer,layer_version):
     # if this is a local layer directory, we can immediately scan its content
-    if layer.local_source_dir:
+    if os.path.isdir(layer.local_source_dir):
         try:
             # recipes-*/*/*.bb
-            cmd = '%s %s' % ('ls', os.path.join(layer.local_source_dir,'recipes-*/*/*.bb'))
-            recipes_list = subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE,stderr=subprocess.STDOUT).stdout.read()
-            recipes_list = recipes_list.decode("utf-8").strip()
-            if recipes_list and 'No such' not in recipes_list:
+            recipes_list = glob.glob(os.path.join(layer.local_source_dir, 'recipes-*/*/*.bb'))
+            for recipe in recipes_list:
                 for recipe in recipes_list.split('\n'):
                     recipe_path = recipe[recipe.rfind('recipes-'):]
                     recipe_name = recipe[recipe.rfind('/')+1:].replace('.bb','')
@@ -260,6 +258,9 @@ def scan_layer_content(layer,layer_version):
 
         except Exception as e:
             logger.warning("ERROR:scan_layer_content: %s" % e)
+    else:
+        logger.warning("ERROR: wrong path given")
+        raise KeyError("local_source_dir")
 
 class XhrLayer(View):
     """ Delete, Get, Add and Update Layer information
@@ -456,15 +457,18 @@ class XhrLayer(View):
                              'layerdetailurl':
                              layer_dep.get_detailspage_url(project.pk)})
 
-            # Scan the layer's content and update components
-            scan_layer_content(layer,layer_version)
+            # Only scan_layer_content if layer is local
+            if layer_data.get('local_source_dir', None):
+                # Scan the layer's content and update components
+                scan_layer_content(layer,layer_version)
 
         except Layer_Version.DoesNotExist:
             return error_response("layer-dep-not-found")
         except Project.DoesNotExist:
             return error_response("project-not-found")
-        except KeyError:
-            return error_response("incorrect-parameters")
+        except KeyError as e:
+            _log("KeyError: %s" % e)
+            return error_response(f"incorrect-parameters")
 
         return JsonResponse({'error': "ok",
                              'imported_layer': {

bitbake/lib/toaster/toastergui/static/js/bootstrap-3.4.1.js

@@ -1,6 +1,6 @@
 /*!
- * Bootstrap v3.3.6 (http://getbootstrap.com)
- * Copyright 2011-2016 Twitter, Inc.
+ * Bootstrap v3.4.1 (https://getbootstrap.com/)
+ * Copyright 2011-2019 Twitter, Inc.
  * Licensed under the MIT license
  */
 
@@ -11,16 +11,16 @@ if (typeof jQuery === 'undefined') {
 +function ($) {
   'use strict';
   var version = $.fn.jquery.split(' ')[0].split('.')
-  if ((version[0] < 2 && version[1] < 9) || (version[0] == 1 && version[1] == 9 && version[2] < 1) || (version[0] > 2)) {
-    throw new Error('Bootstrap\'s JavaScript requires jQuery version 1.9.1 or higher, but lower than version 3')
+  if ((version[0] < 2 && version[1] < 9) || (version[0] == 1 && version[1] == 9 && version[2] < 1) || (version[0] > 3)) {
+    throw new Error('Bootstrap\'s JavaScript requires jQuery version 1.9.1 or higher, but lower than version 4')
   }
 }(jQuery);
 
 /* ========================================================================
- * Bootstrap: transition.js v3.3.6
- * http://getbootstrap.com/javascript/#transitions
+ * Bootstrap: transition.js v3.4.1
+ * https://getbootstrap.com/docs/3.4/javascript/#transitions
  * ========================================================================
- * Copyright 2011-2015 Twitter, Inc.
+ * Copyright 2011-2019 Twitter, Inc.
  * Licensed under MIT (https://github.com/twbs/bootstrap/blob/master/LICENSE)
  * ======================================================================== */
 
@@ -28,7 +28,7 @@ if (typeof jQuery === 'undefined') {
 +function ($) {
   'use strict';
 
-  // CSS TRANSITION SUPPORT (Shoutout: http://www.modernizr.com/)
+  // CSS TRANSITION SUPPORT (Shoutout: https://modernizr.com/)
   // ============================================================
 
   function transitionEnd() {
@@ -50,7 +50,7 @@ if (typeof jQuery === 'undefined') {
     return false // explicit for ie8 (  ._.)
   }
 
-  // http://blog.alexmaccaw.com/css-transitions
+  // https://blog.alexmaccaw.com/css-transitions
   $.fn.emulateTransitionEnd = function (duration) {
     var called = false
     var $el = this
@@ -77,10 +77,10 @@ if (typeof jQuery === 'undefined') {
 }(jQuery);
 
 /* ========================================================================
- * Bootstrap: alert.js v3.3.6
- * http://getbootstrap.com/javascript/#alerts
+ * Bootstrap: alert.js v3.4.1
+ * https://getbootstrap.com/docs/3.4/javascript/#alerts
  * ========================================================================
- * Copyright 2011-2015 Twitter, Inc.
+ * Copyright 2011-2019 Twitter, Inc.
  * Licensed under MIT (https://github.com/twbs/bootstrap/blob/master/LICENSE)
  * ======================================================================== */
 
@@ -96,7 +96,7 @@ if (typeof jQuery === 'undefined') {
     $(el).on('click', dismiss, this.close)
   }
 
-  Alert.VERSION = '3.3.6'
+  Alert.VERSION = '3.4.1'
 
   Alert.TRANSITION_DURATION = 150
 
@@ -109,7 +109,8 @@ if (typeof jQuery === 'undefined') {
       selector = selector && selector.replace(/.*(?=#[^\s]*$)/, '') // strip for ie7
     }
 
-    var $parent = $(selector)
+    selector    = selector === '#' ? [] : selector
+    var $parent = $(document).find(selector)
 
     if (e) e.preventDefault()
 
@@ -172,10 +173,10 @@ if (typeof jQuery === 'undefined') {
 }(jQuery);
 
 /* ========================================================================
- * Bootstrap: button.js v3.3.6
- * http://getbootstrap.com/javascript/#buttons
+ * Bootstrap: button.js v3.4.1
+ * https://getbootstrap.com/docs/3.4/javascript/#buttons
  * ========================================================================
- * Copyright 2011-2015 Twitter, Inc.
+ * Copyright 2011-2019 Twitter, Inc.
  * Licensed under MIT (https://github.com/twbs/bootstrap/blob/master/LICENSE)
  * ======================================================================== */
 
@@ -192,7 +193,7 @@ if (typeof jQuery === 'undefined') {
     this.isLoading = false
   }
 
-  Button.VERSION  = '3.3.6'
+  Button.VERSION  = '3.4.1'
 
   Button.DEFAULTS = {
     loadingText: 'loading...'
@@ -214,10 +215,10 @@ if (typeof jQuery === 'undefined') {
 
       if (state == 'loadingText') {
         this.isLoading = true
-        $el.addClass(d).attr(d, d)
+        $el.addClass(d).attr(d, d).prop(d, true)
       } else if (this.isLoading) {
         this.isLoading = false
-        $el.removeClass(d).removeAttr(d)
+        $el.removeClass(d).removeAttr(d).prop(d, false)
       }
     }, this), 0)
   }
@@ -281,10 +282,15 @@ if (typeof jQuery === 'undefined') {
 
   $(document)
     .on('click.bs.button.data-api', '[data-toggle^="button"]', function (e) {
-      var $btn = $(e.target)
-      if (!$btn.hasClass('btn')) $btn = $btn.closest('.btn')
+      var $btn = $(e.target).closest('.btn')
       Plugin.call($btn, 'toggle')
-      if (!($(e.target).is('input[type="radio"]') || $(e.target).is('input[type="checkbox"]'))) e.preventDefault()
+      if (!($(e.target).is('input[type="radio"], input[type="checkbox"]'))) {
+        // Prevent double click on radios, and the double selections (so cancellation) on checkboxes
+        e.preventDefault()
+        // The target component still receive the focus
+        if ($btn.is('input,button')) $btn.trigger('focus')
+        else $btn.find('input:visible,button:visible').first().trigger('focus')
+      }
     })
     .on('focus.bs.button.data-api blur.bs.button.data-api', '[data-toggle^="button"]', function (e) {
       $(e.target).closest('.btn').toggleClass('focus', /^focus(in)?$/.test(e.type))
@@ -293,10 +299,10 @@ if (typeof jQuery === 'undefined') {
 }(jQuery);
 
 /* ========================================================================
- * Bootstrap: carousel.js v3.3.6
- * http://getbootstrap.com/javascript/#carousel
+ * Bootstrap: carousel.js v3.4.1
+ * https://getbootstrap.com/docs/3.4/javascript/#carousel
  * ========================================================================
- * Copyright 2011-2015 Twitter, Inc.
+ * Copyright 2011-2019 Twitter, Inc.
  * Licensed under MIT (https://github.com/twbs/bootstrap/blob/master/LICENSE)
  * ======================================================================== */
 
@@ -324,7 +330,7 @@ if (typeof jQuery === 'undefined') {
       .on('mouseleave.bs.carousel', $.proxy(this.cycle, this))
   }
 
-  Carousel.VERSION  = '3.3.6'
+  Carousel.VERSION  = '3.4.1'
 
   Carousel.TRANSITION_DURATION = 600
 
@@ -438,7 +444,9 @@ if (typeof jQuery === 'undefined') {
     var slidEvent = $.Event('slid.bs.carousel', { relatedTarget: relatedTarget, direction: direction }) // yes, "slid"
     if ($.support.transition && this.$element.hasClass('slide')) {
       $next.addClass(type)
-      $next[0].offsetWidth // force reflow
+      if (typeof $next === 'object' && $next.length) {
+        $next[0].offsetWidth // force reflow
+      }
       $active.addClass(direction)
       $next.addClass(direction)
       $active
@@ -500,10 +508,17 @@ if (typeof jQuery === 'undefined') {
   // =================
 
   var clickHandler = function (e) {
-    var href
     var $this   = $(this)
-    var $target = $($this.attr('data-target') || (href = $this.attr('href')) && href.replace(/.*(?=#[^\s]+$)/, '')) // strip for ie7
+    var href    = $this.attr('href')
+    if (href) {
+      href = href.replace(/.*(?=#[^\s]+$)/, '') // strip for ie7
+    }
+
+    var target  = $this.attr('data-target') || href
+    var $target = $(document).find(target)
+
     if (!$target.hasClass('carousel')) return
+
     var options = $.extend({}, $target.data(), $this.data())
     var slideIndex = $this.attr('data-slide-to')
     if (slideIndex) options.interval = false
@@ -531,13 +546,14 @@ if (typeof jQuery === 'undefined') {
 }(jQuery);
 
 /* ========================================================================
- * Bootstrap: collapse.js v3.3.6
- * http://getbootstrap.com/javascript/#collapse
+ * Bootstrap: collapse.js v3.4.1
+ * https://getbootstrap.com/docs/3.4/javascript/#collapse
  * ========================================================================
- * Copyright 2011-2015 Twitter, Inc.
+ * Copyright 2011-2019 Twitter, Inc.
  * Licensed under MIT (https://github.com/twbs/bootstrap/blob/master/LICENSE)
  * ======================================================================== */
 
+/* jshint latedef: false */
 
 +function ($) {
   'use strict';
@@ -561,7 +577,7 @@ if (typeof jQuery === 'undefined') {
     if (this.options.toggle) this.toggle()
   }
 
-  Collapse.VERSION  = '3.3.6'
+  Collapse.VERSION  = '3.4.1'
 
   Collapse.TRANSITION_DURATION = 350
 
@@ -668,7 +684,7 @@ if (typeof jQuery === 'undefined') {
   }
 
   Collapse.prototype.getParent = function () {
-    return $(this.options.parent)
+    return $(document).find(this.options.parent)
       .find('[data-toggle="collapse"][data-parent="' + this.options.parent + '"]')
       .each($.proxy(function (i, element) {
         var $element = $(element)
@@ -691,7 +707,7 @@ if (typeof jQuery === 'undefined') {
     var target = $trigger.attr('data-target')
       || (href = $trigger.attr('href')) && href.replace(/.*(?=#[^\s]+$)/, '') // strip for ie7
 
-    return $(target)
+    return $(document).find(target)
   }
 
 
@@ -743,10 +759,10 @@ if (typeof jQuery === 'undefined') {
 }(jQuery);
 
 /* ========================================================================
- * Bootstrap: dropdown.js v3.3.6
- * http://getbootstrap.com/javascript/#dropdowns
+ * Bootstrap: dropdown.js v3.4.1
+ * https://getbootstrap.com/docs/3.4/javascript/#dropdowns
  * ========================================================================
- * Copyright 2011-2015 Twitter, Inc.
+ * Copyright 2011-2019 Twitter, Inc.
  * Licensed under MIT (https://github.com/twbs/bootstrap/blob/master/LICENSE)
  * ======================================================================== */
 
@@ -763,7 +779,7 @@ if (typeof jQuery === 'undefined') {
     $(element).on('click.bs.dropdown', this.toggle)
   }
 
-  Dropdown.VERSION = '3.3.6'
+  Dropdown.VERSION = '3.4.1'
 
   function getParent($this) {
     var selector = $this.attr('data-target')
@@ -773,7 +789,7 @@ if (typeof jQuery === 'undefined') {
       selector = selector && /#[A-Za-z]/.test(selector) && selector.replace(/.*(?=#[^\s]*$)/, '') // strip for ie7
     }
 
-    var $parent = selector && $(selector)
+    var $parent = selector !== '#' ? $(document).find(selector) : null
 
     return $parent && $parent.length ? $parent : $this.parent()
   }
@@ -909,10 +925,10 @@ if (typeof jQuery === 'undefined') {
 }(jQuery);
 
 /* ========================================================================
- * Bootstrap: modal.js v3.3.6
- * http://getbootstrap.com/javascript/#modals
+ * Bootstrap: modal.js v3.4.1
+ * https://getbootstrap.com/docs/3.4/javascript/#modals
  * ========================================================================
- * Copyright 2011-2015 Twitter, Inc.
+ * Copyright 2011-2019 Twitter, Inc.
  * Licensed under MIT (https://github.com/twbs/bootstrap/blob/master/LICENSE)
  * ======================================================================== */
 
@@ -924,15 +940,16 @@ if (typeof jQuery === 'undefined') {
   // ======================
 
   var Modal = function (element, options) {
-    this.options             = options
-    this.$body               = $(document.body)
-    this.$element            = $(element)
-    this.$dialog             = this.$element.find('.modal-dialog')
-    this.$backdrop           = null
-    this.isShown             = null
-    this.originalBodyPad     = null
-    this.scrollbarWidth      = 0
+    this.options = options
+    this.$body = $(document.body)
+    this.$element = $(element)
+    this.$dialog = this.$element.find('.modal-dialog')
+    this.$backdrop = null
+    this.isShown = null
+    this.originalBodyPad = null
+    this.scrollbarWidth = 0
     this.ignoreBackdropClick = false
+    this.fixedContent = '.navbar-fixed-top, .navbar-fixed-bottom'
 
     if (this.options.remote) {
       this.$element
@@ -943,7 +960,7 @@ if (typeof jQuery === 'undefined') {
     }
   }
 
-  Modal.VERSION  = '3.3.6'
+  Modal.VERSION = '3.4.1'
 
   Modal.TRANSITION_DURATION = 300
   Modal.BACKDROP_TRANSITION_DURATION = 150
@@ -960,7 +977,7 @@ if (typeof jQuery === 'undefined') {
 
   Modal.prototype.show = function (_relatedTarget) {
     var that = this
-    var e    = $.Event('show.bs.modal', { relatedTarget: _relatedTarget })
+    var e = $.Event('show.bs.modal', { relatedTarget: _relatedTarget })
 
     this.$element.trigger(e)
 
@@ -1050,7 +1067,9 @@ if (typeof jQuery === 'undefined') {
     $(document)
       .off('focusin.bs.modal') // guard against infinite focus loop
       .on('focusin.bs.modal', $.proxy(function (e) {
-        if (this.$element[0] !== e.target && !this.$element.has(e.target).length) {
+        if (document !== e.target &&
+          this.$element[0] !== e.target &&
+          !this.$element.has(e.target).length) {
           this.$element.trigger('focus')
         }
       }, this))
@@ -1152,7 +1171,7 @@ if (typeof jQuery === 'undefined') {
     var modalIsOverflowing = this.$element[0].scrollHeight > document.documentElement.clientHeight
 
     this.$element.css({
-      paddingLeft:  !this.bodyIsOverflowing && modalIsOverflowing ? this.scrollbarWidth : '',
+      paddingLeft: !this.bodyIsOverflowing && modalIsOverflowing ? this.scrollbarWidth : '',
       paddingRight: this.bodyIsOverflowing && !modalIsOverflowing ? this.scrollbarWidth : ''
     })
   }
@@ -1177,11 +1196,26 @@ if (typeof jQuery === 'undefined') {
   Modal.prototype.setScrollbar = function () {
     var bodyPad = parseInt((this.$body.css('padding-right') || 0), 10)
     this.originalBodyPad = document.body.style.paddingRight || ''
-    if (this.bodyIsOverflowing) this.$body.css('padding-right', bodyPad + this.scrollbarWidth)
+    var scrollbarWidth = this.scrollbarWidth
+    if (this.bodyIsOverflowing) {
+      this.$body.css('padding-right', bodyPad + scrollbarWidth)
+      $(this.fixedContent).each(function (index, element) {
+        var actualPadding = element.style.paddingRight
+        var calculatedPadding = $(element).css('padding-right')
+        $(element)
+          .data('padding-right', actualPadding)
+          .css('padding-right', parseFloat(calculatedPadding) + scrollbarWidth + 'px')
+      })
+    }
   }
 
   Modal.prototype.resetScrollbar = function () {
     this.$body.css('padding-right', this.originalBodyPad)
+    $(this.fixedContent).each(function (index, element) {
+      var padding = $(element).data('padding-right')
+      $(element).removeData('padding-right')
+      element.style.paddingRight = padding ? padding : ''
+    })
   }
 
   Modal.prototype.measureScrollbar = function () { // thx walsh
@@ -1199,8 +1233,8 @@ if (typeof jQuery === 'undefined') {
 
   function Plugin(option, _relatedTarget) {
     return this.each(function () {
-      var $this   = $(this)
-      var data    = $this.data('bs.modal')
+      var $this = $(this)
+      var data = $this.data('bs.modal')
       var options = $.extend({}, Modal.DEFAULTS, $this.data(), typeof option == 'object' && option)
 
       if (!data) $this.data('bs.modal', (data = new Modal(this, options)))
@@ -1211,7 +1245,7 @@ if (typeof jQuery === 'undefined') {
 
   var old = $.fn.modal
 
-  $.fn.modal             = Plugin
+  $.fn.modal = Plugin
   $.fn.modal.Constructor = Modal
 
 
@@ -1228,10 +1262,13 @@ if (typeof jQuery === 'undefined') {
   // ==============
 
   $(document).on('click.bs.modal.data-api', '[data-toggle="modal"]', function (e) {
-    var $this   = $(this)
-    var href    = $this.attr('href')
-    var $target = $($this.attr('data-target') || (href && href.replace(/.*(?=#[^\s]+$)/, ''))) // strip for ie7
-    var option  = $target.data('bs.modal') ? 'toggle' : $.extend({ remote: !/#/.test(href) && href }, $target.data(), $this.data())
+    var $this = $(this)
+    var href = $this.attr('href')
+    var target = $this.attr('data-target') ||
+      (href && href.replace(/.*(?=#[^\s]+$)/, '')) // strip for ie7
+
+    var $target = $(document).find(target)
+    var option = $target.data('bs.modal') ? 'toggle' : $.extend({ remote: !/#/.test(href) && href }, $target.data(), $this.data())
 
     if ($this.is('a')) e.preventDefault()
 
@@ -1247,18 +1284,148 @@ if (typeof jQuery === 'undefined') {
 }(jQuery);
 
 /* ========================================================================
- * Bootstrap: tooltip.js v3.3.6
- * http://getbootstrap.com/javascript/#tooltip
+ * Bootstrap: tooltip.js v3.4.1
+ * https://getbootstrap.com/docs/3.4/javascript/#tooltip
  * Inspired by the original jQuery.tipsy by Jason Frame
  * ========================================================================
- * Copyright 2011-2015 Twitter, Inc.
+ * Copyright 2011-2019 Twitter, Inc.
  * Licensed under MIT (https://github.com/twbs/bootstrap/blob/master/LICENSE)
  * ======================================================================== */
 
-
 +function ($) {
   'use strict';
 
+  var DISALLOWED_ATTRIBUTES = ['sanitize', 'whiteList', 'sanitizeFn']
+
+  var uriAttrs = [
+    'background',
+    'cite',
+    'href',
+    'itemtype',
+    'longdesc',
+    'poster',
+    'src',
+    'xlink:href'
+  ]
+
+  var ARIA_ATTRIBUTE_PATTERN = /^aria-[\w-]*$/i
+
+  var DefaultWhitelist = {
+    // Global attributes allowed on any supplied element below.
+    '*': ['class', 'dir', 'id', 'lang', 'role', ARIA_ATTRIBUTE_PATTERN],
+    a: ['target', 'href', 'title', 'rel'],
+    area: [],
+    b: [],
+    br: [],
+    col: [],
+    code: [],
+    div: [],
+    em: [],
+    hr: [],
+    h1: [],
+    h2: [],
+    h3: [],
+    h4: [],
+    h5: [],
+    h6: [],
+    i: [],
+    img: ['src', 'alt', 'title', 'width', 'height'],
+    li: [],
+    ol: [],
+    p: [],
+    pre: [],
+    s: [],
+    small: [],
+    span: [],
+    sub: [],
+    sup: [],
+    strong: [],
+    u: [],
+    ul: []
+  }
+
+  /**
+   * A pattern that recognizes a commonly useful subset of URLs that are safe.
+   *
+   * Shoutout to Angular 7 https://github.com/angular/angular/blob/7.2.4/packages/core/src/sanitization/url_sanitizer.ts
+   */
+  var SAFE_URL_PATTERN = /^(?:(?:https?|mailto|ftp|tel|file):|[^&:/?#]*(?:[/?#]|$))/gi
+
+  /**
+   * A pattern that matches safe data URLs. Only matches image, video and audio types.
+   *
+   * Shoutout to Angular 7 https://github.com/angular/angular/blob/7.2.4/packages/core/src/sanitization/url_sanitizer.ts
+   */
+  var DATA_URL_PATTERN = /^data:(?:image\/(?:bmp|gif|jpeg|jpg|png|tiff|webp)|video\/(?:mpeg|mp4|ogg|webm)|audio\/(?:mp3|oga|ogg|opus));base64,[a-z0-9+/]+=*$/i
+
+  function allowedAttribute(attr, allowedAttributeList) {
+    var attrName = attr.nodeName.toLowerCase()
+
+    if ($.inArray(attrName, allowedAttributeList) !== -1) {
+      if ($.inArray(attrName, uriAttrs) !== -1) {
+        return Boolean(attr.nodeValue.match(SAFE_URL_PATTERN) || attr.nodeValue.match(DATA_URL_PATTERN))
+      }
+
+      return true
+    }
+
+    var regExp = $(allowedAttributeList).filter(function (index, value) {
+      return value instanceof RegExp
+    })
+
+    // Check if a regular expression validates the attribute.
+    for (var i = 0, l = regExp.length; i < l; i++) {
+      if (attrName.match(regExp[i])) {
+        return true
+      }
+    }
+
+    return false
+  }
+
+  function sanitizeHtml(unsafeHtml, whiteList, sanitizeFn) {
+    if (unsafeHtml.length === 0) {
+      return unsafeHtml
+    }
+
+    if (sanitizeFn && typeof sanitizeFn === 'function') {
+      return sanitizeFn(unsafeHtml)
+    }
+
+    // IE 8 and below don't support createHTMLDocument
+    if (!document.implementation || !document.implementation.createHTMLDocument) {
+      return unsafeHtml
+    }
+
+    var createdDocument = document.implementation.createHTMLDocument('sanitization')
+    createdDocument.body.innerHTML = unsafeHtml
+
+    var whitelistKeys = $.map(whiteList, function (el, i) { return i })
+    var elements = $(createdDocument.body).find('*')
+
+    for (var i = 0, len = elements.length; i < len; i++) {
+      var el = elements[i]
+      var elName = el.nodeName.toLowerCase()
+
+      if ($.inArray(elName, whitelistKeys) === -1) {
+        el.parentNode.removeChild(el)
+
+        continue
+      }
+
+      var attributeList = $.map(el.attributes, function (el) { return el })
+      var whitelistedAttributes = [].concat(whiteList['*'] || [], whiteList[elName] || [])
+
+      for (var j = 0, len2 = attributeList.length; j < len2; j++) {
+        if (!allowedAttribute(attributeList[j], whitelistedAttributes)) {
+          el.removeAttribute(attributeList[j].nodeName)
+        }
+      }
+    }
+
+    return createdDocument.body.innerHTML
+  }
+
   // TOOLTIP PUBLIC CLASS DEFINITION
   // ===============================
 
@@ -1274,7 +1441,7 @@ if (typeof jQuery === 'undefined') {
     this.init('tooltip', element, options)
   }
 
-  Tooltip.VERSION  = '3.3.6'
+  Tooltip.VERSION  = '3.4.1'
 
   Tooltip.TRANSITION_DURATION = 150
 
@@ -1291,7 +1458,10 @@ if (typeof jQuery === 'undefined') {
     viewport: {
       selector: 'body',
       padding: 0
-    }
+    },
+    sanitize : true,
+    sanitizeFn : null,
+    whiteList : DefaultWhitelist
   }
 
   Tooltip.prototype.init = function (type, element, options) {
@@ -1299,7 +1469,7 @@ if (typeof jQuery === 'undefined') {
     this.type      = type
     this.$element  = $(element)
     this.options   = this.getOptions(options)
-    this.$viewport = this.options.viewport && $($.isFunction(this.options.viewport) ? this.options.viewport.call(this, this.$element) : (this.options.viewport.selector || this.options.viewport))
+    this.$viewport = this.options.viewport && $(document).find($.isFunction(this.options.viewport) ? this.options.viewport.call(this, this.$element) : (this.options.viewport.selector || this.options.viewport))
     this.inState   = { click: false, hover: false, focus: false }
 
     if (this.$element[0] instanceof document.constructor && !this.options.selector) {
@@ -1332,7 +1502,15 @@ if (typeof jQuery === 'undefined') {
   }
 
   Tooltip.prototype.getOptions = function (options) {
-    options = $.extend({}, this.getDefaults(), this.$element.data(), options)
+    var dataAttributes = this.$element.data()
+
+    for (var dataAttr in dataAttributes) {
+      if (dataAttributes.hasOwnProperty(dataAttr) && $.inArray(dataAttr, DISALLOWED_ATTRIBUTES) !== -1) {
+        delete dataAttributes[dataAttr]
+      }
+    }
+
+    options = $.extend({}, this.getDefaults(), dataAttributes, options)
 
     if (options.delay && typeof options.delay == 'number') {
       options.delay = {
@@ -1341,6 +1519,10 @@ if (typeof jQuery === 'undefined') {
       }
     }
 
+    if (options.sanitize) {
+      options.template = sanitizeHtml(options.template, options.whiteList, options.sanitizeFn)
+    }
+
     return options
   }
 
@@ -1452,7 +1634,7 @@ if (typeof jQuery === 'undefined') {
         .addClass(placement)
         .data('bs.' + this.type, this)
 
-      this.options.container ? $tip.appendTo(this.options.container) : $tip.insertAfter(this.$element)
+      this.options.container ? $tip.appendTo($(document).find(this.options.container)) : $tip.insertAfter(this.$element)
       this.$element.trigger('inserted.bs.' + this.type)
 
       var pos          = this.getPosition()
@@ -1554,7 +1736,16 @@ if (typeof jQuery === 'undefined') {
     var $tip  = this.tip()
     var title = this.getTitle()
 
-    $tip.find('.tooltip-inner')[this.options.html ? 'html' : 'text'](title)
+    if (this.options.html) {
+      if (this.options.sanitize) {
+        title = sanitizeHtml(title, this.options.whiteList, this.options.sanitizeFn)
+      }
+
+      $tip.find('.tooltip-inner').html(title)
+    } else {
+      $tip.find('.tooltip-inner').text(title)
+    }
+
     $tip.removeClass('fade in top bottom left right')
   }
 
@@ -1565,9 +1756,11 @@ if (typeof jQuery === 'undefined') {
 
     function complete() {
       if (that.hoverState != 'in') $tip.detach()
-      that.$element
-        .removeAttr('aria-describedby')
-        .trigger('hidden.bs.' + that.type)
+      if (that.$element) { // TODO: Check whether guarding this code with this `if` is really necessary.
+        that.$element
+          .removeAttr('aria-describedby')
+          .trigger('hidden.bs.' + that.type)
+      }
       callback && callback()
     }
 
@@ -1610,7 +1803,10 @@ if (typeof jQuery === 'undefined') {
       // width and height are missing in IE8, so compute them manually; see https://github.com/twbs/bootstrap/issues/14093
       elRect = $.extend({}, elRect, { width: elRect.right - elRect.left, height: elRect.bottom - elRect.top })
     }
-    var elOffset  = isBody ? { top: 0, left: 0 } : $element.offset()
+    var isSvg = window.SVGElement && el instanceof window.SVGElement
+    // Avoid using $.offset() on SVGs since it gives incorrect results in jQuery 3.
+    // See https://github.com/twbs/bootstrap/issues/20280
+    var elOffset  = isBody ? { top: 0, left: 0 } : (isSvg ? null : $element.offset())
     var scroll    = { scroll: isBody ? document.documentElement.scrollTop || document.body.scrollTop : $element.scrollTop() }
     var outerDims = isBody ? { width: $(window).width(), height: $(window).height() } : null
 
@@ -1726,9 +1922,13 @@ if (typeof jQuery === 'undefined') {
       that.$tip = null
       that.$arrow = null
       that.$viewport = null
+      that.$element = null
     })
   }
 
+  Tooltip.prototype.sanitizeHtml = function (unsafeHtml) {
+    return sanitizeHtml(unsafeHtml, this.options.whiteList, this.options.sanitizeFn)
+  }
 
   // TOOLTIP PLUGIN DEFINITION
   // =========================
@@ -1762,10 +1962,10 @@ if (typeof jQuery === 'undefined') {
 }(jQuery);
 
 /* ========================================================================
- * Bootstrap: popover.js v3.3.6
- * http://getbootstrap.com/javascript/#popovers
+ * Bootstrap: popover.js v3.4.1
+ * https://getbootstrap.com/docs/3.4/javascript/#popovers
  * ========================================================================
- * Copyright 2011-2015 Twitter, Inc.
+ * Copyright 2011-2019 Twitter, Inc.
  * Licensed under MIT (https://github.com/twbs/bootstrap/blob/master/LICENSE)
  * ======================================================================== */
 
@@ -1782,7 +1982,7 @@ if (typeof jQuery === 'undefined') {
 
   if (!$.fn.tooltip) throw new Error('Popover requires tooltip.js')
 
-  Popover.VERSION  = '3.3.6'
+  Popover.VERSION  = '3.4.1'
 
   Popover.DEFAULTS = $.extend({}, $.fn.tooltip.Constructor.DEFAULTS, {
     placement: 'right',
@@ -1808,10 +2008,25 @@ if (typeof jQuery === 'undefined') {
     var title   = this.getTitle()
     var content = this.getContent()
 
-    $tip.find('.popover-title')[this.options.html ? 'html' : 'text'](title)
-    $tip.find('.popover-content').children().detach().end()[ // we use append for html objects to maintain js events
-      this.options.html ? (typeof content == 'string' ? 'html' : 'append') : 'text'
-    ](content)
+    if (this.options.html) {
+      var typeContent = typeof content
+
+      if (this.options.sanitize) {
+        title = this.sanitizeHtml(title)
+
+        if (typeContent === 'string') {
+          content = this.sanitizeHtml(content)
+        }
+      }
+
+      $tip.find('.popover-title').html(title)
+      $tip.find('.popover-content').children().detach().end()[
+        typeContent === 'string' ? 'html' : 'append'
+      ](content)
+    } else {
+      $tip.find('.popover-title').text(title)
+      $tip.find('.popover-content').children().detach().end().text(content)
+    }
 
     $tip.removeClass('fade top bottom left right in')
 
@@ -1830,8 +2045,8 @@ if (typeof jQuery === 'undefined') {
 
     return $e.attr('data-content')
       || (typeof o.content == 'function' ?
-            o.content.call($e[0]) :
-            o.content)
+        o.content.call($e[0]) :
+        o.content)
   }
 
   Popover.prototype.arrow = function () {
@@ -1871,10 +2086,10 @@ if (typeof jQuery === 'undefined') {
 }(jQuery);
 
 /* ========================================================================
- * Bootstrap: scrollspy.js v3.3.6
- * http://getbootstrap.com/javascript/#scrollspy
+ * Bootstrap: scrollspy.js v3.4.1
+ * https://getbootstrap.com/docs/3.4/javascript/#scrollspy
  * ========================================================================
- * Copyright 2011-2015 Twitter, Inc.
+ * Copyright 2011-2019 Twitter, Inc.
  * Licensed under MIT (https://github.com/twbs/bootstrap/blob/master/LICENSE)
  * ======================================================================== */
 
@@ -1900,7 +2115,7 @@ if (typeof jQuery === 'undefined') {
     this.process()
   }
 
-  ScrollSpy.VERSION  = '3.3.6'
+  ScrollSpy.VERSION  = '3.4.1'
 
   ScrollSpy.DEFAULTS = {
     offset: 10
@@ -2044,10 +2259,10 @@ if (typeof jQuery === 'undefined') {
 }(jQuery);
 
 /* ========================================================================
- * Bootstrap: tab.js v3.3.6
- * http://getbootstrap.com/javascript/#tabs
+ * Bootstrap: tab.js v3.4.1
+ * https://getbootstrap.com/docs/3.4/javascript/#tabs
  * ========================================================================
- * Copyright 2011-2015 Twitter, Inc.
+ * Copyright 2011-2019 Twitter, Inc.
  * Licensed under MIT (https://github.com/twbs/bootstrap/blob/master/LICENSE)
  * ======================================================================== */
 
@@ -2064,7 +2279,7 @@ if (typeof jQuery === 'undefined') {
     // jscs:enable requireDollarBeforejQueryAssignment
   }
 
-  Tab.VERSION = '3.3.6'
+  Tab.VERSION = '3.4.1'
 
   Tab.TRANSITION_DURATION = 150
 
@@ -2093,7 +2308,7 @@ if (typeof jQuery === 'undefined') {
 
     if (showEvent.isDefaultPrevented() || hideEvent.isDefaultPrevented()) return
 
-    var $target = $(selector)
+    var $target = $(document).find(selector)
 
     this.activate($this.closest('li'), $ul)
     this.activate($target, $target.parent(), function () {
@@ -2118,15 +2333,15 @@ if (typeof jQuery === 'undefined') {
       $active
         .removeClass('active')
         .find('> .dropdown-menu > .active')
-          .removeClass('active')
+        .removeClass('active')
         .end()
         .find('[data-toggle="tab"]')
-          .attr('aria-expanded', false)
+        .attr('aria-expanded', false)
 
       element
         .addClass('active')
         .find('[data-toggle="tab"]')
-          .attr('aria-expanded', true)
+        .attr('aria-expanded', true)
 
       if (transition) {
         element[0].offsetWidth // reflow for transition
@@ -2138,10 +2353,10 @@ if (typeof jQuery === 'undefined') {
       if (element.parent('.dropdown-menu').length) {
         element
           .closest('li.dropdown')
-            .addClass('active')
+          .addClass('active')
           .end()
           .find('[data-toggle="tab"]')
-            .attr('aria-expanded', true)
+          .attr('aria-expanded', true)
       }
 
       callback && callback()
@@ -2200,10 +2415,10 @@ if (typeof jQuery === 'undefined') {
 }(jQuery);
 
 /* ========================================================================
- * Bootstrap: affix.js v3.3.6
- * http://getbootstrap.com/javascript/#affix
+ * Bootstrap: affix.js v3.4.1
+ * https://getbootstrap.com/docs/3.4/javascript/#affix
  * ========================================================================
- * Copyright 2011-2015 Twitter, Inc.
+ * Copyright 2011-2019 Twitter, Inc.
  * Licensed under MIT (https://github.com/twbs/bootstrap/blob/master/LICENSE)
  * ======================================================================== */
 
@@ -2217,7 +2432,9 @@ if (typeof jQuery === 'undefined') {
   var Affix = function (element, options) {
     this.options = $.extend({}, Affix.DEFAULTS, options)
 
-    this.$target = $(this.options.target)
+    var target = this.options.target === Affix.DEFAULTS.target ? $(this.options.target) : $(document).find(this.options.target)
+
+    this.$target = target
       .on('scroll.bs.affix.data-api', $.proxy(this.checkPosition, this))
       .on('click.bs.affix.data-api',  $.proxy(this.checkPositionWithEventLoop, this))
 
@@ -2229,7 +2446,7 @@ if (typeof jQuery === 'undefined') {
     this.checkPosition()
   }
 
-  Affix.VERSION  = '3.3.6'
+  Affix.VERSION  = '3.4.1'
 
   Affix.RESET    = 'affix affix-top affix-bottom'
 

bitbake/lib/toaster/toastergui/static/js/libtoaster.js

@@ -657,7 +657,7 @@ $(document).ready(function() {
     hljs.initHighlightingOnLoad();
 
     // Prevent invalid links from jumping page scroll
-    $('a[href=#]').click(function() {
+    $('a[href="#"]').click(function() {
         return false;
     });
 

bitbake/lib/toaster/toastergui/static/js/projectpage.js

@@ -61,7 +61,7 @@ function projectPageInit(ctx) {
         distroChangeInput.val(urlParams.setDistro);
         distroChangeBtn.click();
     } else {
-      updateDistroName(prjInfo.distro.name);
+      updateDistroName(prjInfo.distro?.name);
     }
 
    /* Now we're really ready show the page */

bitbake/lib/toaster/toastergui/templates/base.html

@@ -14,11 +14,11 @@
 
     <meta name="viewport" content="width=device-width, initial-scale=1.0" />
     <meta http-equiv="Content-Type" content="text/html;charset=UTF-8" />
-    <script src="{% static 'js/jquery-2.0.3.min.js' %}">
+    <script src="{% static 'js/jquery-3.7.1.min.js' %}">
     </script>
     <script src="{% static 'js/jquery.cookie.js' %}">
     </script>
-    <script src="{% static 'js/bootstrap.min.js' %}">
+    <script src="{% static 'js/bootstrap-3.4.1.min.js' %}">
     </script>
     <script src="{% static 'js/typeahead.jquery.js' %}">
     </script>

bitbake/lib/toaster/toastergui/templates/base_specific.html

@@ -14,11 +14,11 @@
 
     <meta name="viewport" content="width=device-width, initial-scale=1.0" />
     <meta http-equiv="Content-Type" content="text/html;charset=UTF-8" />
-    <script src="{% static 'js/jquery-2.0.3.min.js' %}">
+    <script src="{% static 'js/jquery-3.7.1.min.js' %}">
     </script>
     <script src="{% static 'js/jquery.cookie.js' %}">
     </script>
-    <script src="{% static 'js/bootstrap.min.js' %}">
+    <script src="{% static 'js/bootstrap-3.4.1.min.js' %}">
     </script>
     <script src="{% static 'js/typeahead.jquery.js' %}">
     </script>

bitbake/lib/toaster/toastergui/templates/js-unit-tests.html

@@ -11,7 +11,7 @@
 <script src="{% static 'js/layerDepsModal.js' %}"></script>
 <script src="{% static 'js/projectpage.js' %}"></script>
 
-<script src="{% static 'js/bootstrap.min.js' %}"></script>
+<script src="{% static 'js/bootstrap-3.4.1.min.js' %}"></script>
 <script src="{% static 'js/filtersnippet.js' %}"></script>
 <script src="{% static 'js/importlayer.js' %}"></script>
 <script src="{% static 'js/highlight.pack.js' %}"></script>

bitbake/lib/toaster/toastergui/urls.py

@@ -6,7 +6,7 @@
 # SPDX-License-Identifier: GPL-2.0-only
 #
 
-from django.conf.urls import url
+from django.urls import re_path as url
 from django.views.generic import RedirectView
 
 from toastergui import tables

bitbake/lib/toaster/toastergui/views.py

@@ -34,6 +34,8 @@ import mimetypes
 
 import logging
 
+from toastermain.logs import log_view_mixin
+
 logger = logging.getLogger("toaster")
 
 # Project creation and managed build enable
@@ -56,6 +58,7 @@ class MimeTypeFinder(object):
         return guessed_type
 
 # single point to add global values into the context before rendering
+@log_view_mixin
 def toaster_render(request, page, context):
     context['project_enable'] = project_enable
     context['project_specific'] = is_project_specific
@@ -665,16 +668,17 @@ def recipe_packages(request, build_id, recipe_id):
     return response
 
 from django.http import HttpResponse
+@log_view_mixin
 def xhr_dirinfo(request, build_id, target_id):
     top = request.GET.get('start', '/')
     return HttpResponse(_get_dir_entries(build_id, target_id, top), content_type = "application/json")
 
 from django.utils.functional import Promise
-from django.utils.encoding import force_text
+from django.utils.encoding import force_str
 class LazyEncoder(json.JSONEncoder):
     def default(self, obj):
         if isinstance(obj, Promise):
-            return force_text(obj)
+            return force_str(obj)
         return super(LazyEncoder, self).default(obj)
 
 from toastergui.templatetags.projecttags import filtered_filesizeformat
@@ -1404,7 +1408,7 @@ if True:
                     if not os.path.isdir('%s/conf' % request.POST['importdir']):
                         raise BadParameterException("Bad path or missing 'conf' directory (%s)" % request.POST['importdir'])
                     from django.core import management
-                    management.call_command('buildimport', '--command=import', '--name=%s' % request.POST['projectname'], '--path=%s' % request.POST['importdir'], interactive=False)
+                    management.call_command('buildimport', '--command=import', '--name=%s' % request.POST['projectname'], '--path=%s' % request.POST['importdir'])
                     prj = Project.objects.get(name = request.POST['projectname'])
                     prj.merged_attr = True
                     prj.save()
@@ -1612,6 +1616,7 @@ if True:
 
     from django.views.decorators.csrf import csrf_exempt
     @csrf_exempt
+    @log_view_mixin
     def xhr_testreleasechange(request, pid):
         def response(data):
             return HttpResponse(jsonfilter(data),
@@ -1648,6 +1653,7 @@ if True:
         except Exception as e:
             return response({"error": str(e) })
 
+    @log_view_mixin
     def xhr_configvaredit(request, pid):
         try:
             prj = Project.objects.get(id = pid)
@@ -1726,6 +1732,7 @@ if True:
             return HttpResponse(json.dumps({"error":str(e) + "\n" + traceback.format_exc()}), content_type = "application/json")
 
 
+    @log_view_mixin
     def customrecipe_download(request, pid, recipe_id):
         recipe = get_object_or_404(CustomImageRecipe, pk=recipe_id)
 

bitbake/lib/toaster/toastergui/widgets.py

@@ -7,6 +7,7 @@
 #
 
 from django.views.generic import View, TemplateView
+from django.utils.decorators import method_decorator
 from django.views.decorators.cache import cache_control
 from django.shortcuts import HttpResponse
 from django.core.cache import cache
@@ -31,6 +32,7 @@ import re
 import os
 
 from toastergui.tablefilter import TableFilterMap
+from toastermain.logs import log_view_mixin
 
 try:
     from urllib import unquote_plus
@@ -63,8 +65,8 @@ class ToasterTable(TemplateView):
         self.default_orderby = ""
 
     # prevent HTTP caching of table data
-    @cache_control(must_revalidate=True,
-                   max_age=0, no_store=True, no_cache=True)
+    @method_decorator(cache_control(must_revalidate=True,
+                   max_age=0, no_store=True, no_cache=True))
     def dispatch(self, *args, **kwargs):
         return super(ToasterTable, self).dispatch(*args, **kwargs)
 
@@ -83,6 +85,7 @@ class ToasterTable(TemplateView):
 
         return context
 
+    @log_view_mixin
     def get(self, request, *args, **kwargs):
         if request.GET.get('format', None) == 'json':
 
@@ -304,6 +307,7 @@ class ToasterTable(TemplateView):
 
         self.setup_columns(**kwargs)
 
+        self.apply_orderby('pk')
         if search:
             self.apply_search(search)
         if filters:
@@ -413,6 +417,7 @@ class ToasterTypeAhead(View):
     def __init__(self, *args, **kwargs):
         super(ToasterTypeAhead, self).__init__()
 
+    @log_view_mixin
     def get(self, request, *args, **kwargs):
         def response(data):
             return HttpResponse(json.dumps(data,
@@ -468,6 +473,7 @@ class MostRecentBuildsView(View):
 
         return False
 
+    @log_view_mixin
     def get(self, request, *args, **kwargs):
         """
         Returns a list of builds in JSON format.

bitbake/lib/toaster/toastermain/logs.py

@@ -0,0 +1,153 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+
+import logging
+import json
+from pathlib import Path
+from django.http import HttpRequest
+
+BASE_DIR = Path(__file__).resolve(strict=True).parent.parent
+
+
+def log_api_request(request, response, view, logger_name='api'):
+    """Helper function for LogAPIMixin"""
+
+    repjson = {
+        'view': view,
+        'path': request.path,
+        'method': request.method,
+        'status': response.status_code
+    }
+
+    logger = logging.getLogger(logger_name)
+    logger.info(
+        json.dumps(repjson, indent=4, separators=(", ", " : "))
+    )
+
+
+def log_view_mixin(view):
+    def log_view_request(*args, **kwargs):
+        # get request from args else kwargs
+        request = None
+        if len(args) > 0:
+            for req in args:
+                if isinstance(req, HttpRequest):
+                    request = req
+                    break 
+        elif request is None:
+            request = kwargs.get('request')
+
+        response = view(*args, **kwargs)
+        log_api_request(
+            request, response, request.resolver_match.view_name, 'toaster')
+        return response
+    return log_view_request
+
+
+
+class LogAPIMixin:
+    """Logs API requests
+
+    tested with:
+        - APIView
+        - ModelViewSet
+        - ReadOnlyModelViewSet
+        - GenericAPIView
+
+    Note: you can set `view_name` attribute in View to override get_view_name()
+    """
+
+    def get_view_name(self):
+        if hasattr(self, 'view_name'):
+            return self.view_name
+        return super().get_view_name()
+
+    def finalize_response(self, request, response, *args, **kwargs):
+        log_api_request(request, response, self.get_view_name())
+        return super().finalize_response(request, response, *args, **kwargs)
+
+
+LOGGING_SETTINGS = {
+    'version': 1,
+    'disable_existing_loggers': False,
+    'filters': {
+        'require_debug_false': {
+            '()': 'django.utils.log.RequireDebugFalse'
+        }
+    },
+    'formatters': {
+        'datetime': {
+            'format': '%(asctime)s %(levelname)s %(message)s'
+        },
+        'verbose': {
+            'format': '{levelname} {asctime} {module} {name}.{funcName} {process:d} {thread:d} {message}',
+            'datefmt': "%d/%b/%Y %H:%M:%S",
+            'style': '{',
+        },
+        'api': {
+            'format': '\n{levelname} {asctime} {name}.{funcName}:\n{message}',
+            'style': '{'
+        }
+    },
+    'handlers': {
+        'mail_admins': {
+            'level': 'ERROR',
+            'filters': ['require_debug_false'],
+            'class': 'django.utils.log.AdminEmailHandler'
+        },
+        'console': {
+            'level': 'DEBUG',
+            'class': 'logging.StreamHandler',
+            'formatter': 'datetime',
+        },
+        'file_django': {
+            'level': 'INFO',
+            'class': 'logging.handlers.TimedRotatingFileHandler',
+            'filename': BASE_DIR / 'logs/django.log',
+            'when': 'D',  # interval type
+            'interval': 1,  # defaults to 1
+            'backupCount': 10,  # how many files to keep
+            'formatter': 'verbose',
+        },
+        'file_api': {
+            'level': 'INFO',
+            'class': 'logging.handlers.TimedRotatingFileHandler',
+            'filename': BASE_DIR / 'logs/api.log',
+            'when': 'D',
+            'interval': 1,
+            'backupCount': 10,
+            'formatter': 'verbose',
+        },
+        'file_toaster': {
+            'level': 'INFO',
+            'class': 'logging.handlers.TimedRotatingFileHandler',
+            'filename': BASE_DIR / 'logs/toaster.log',
+            'when': 'D',
+            'interval': 1,
+            'backupCount': 10,
+            'formatter': 'verbose',
+        },
+    },
+    'loggers': {
+        'django.request': {
+            'handlers': ['file_django', 'console'],
+            'level': 'WARN',
+            'propagate': True,
+        },
+        'django': {
+            'handlers': ['file_django', 'console'],
+            'level': 'WARNING',
+            'propogate': True,
+        },
+        'toaster': {
+            'handlers': ['file_toaster'],
+            'level': 'INFO',
+            'propagate': False,
+        },
+        'api': {
+            'handlers': ['file_api'],
+            'level': 'INFO',
+            'propagate': False,
+        }
+    }
+}

bitbake/lib/toaster/toastermain/management/commands/buildimport.py

@@ -545,7 +545,7 @@ class Command(BaseCommand):
             # Find the directory's release, and promote to default_release if local paths
             release = self.find_import_release(layers_list,lv_dict,default_release)
             # create project, SANITY: reuse any project of same name
-            project = Project.objects.create_project(project_name,release,project)
+            project = Project.objects.create_project(project_name,release,project, imported=True)
             # Apply any new layers or variables
             self.apply_conf_variables(project,layers_list,lv_dict,release)
             # WORKAROUND: since we now derive the release, redirect 'newproject_specific' to 'project_specific'

bitbake/lib/toaster/toastermain/management/commands/checksocket.py

@@ -13,7 +13,7 @@ import errno
 import socket
 
 from django.core.management.base import BaseCommand, CommandError
-from django.utils.encoding import force_text
+from django.utils.encoding import force_str
 
 DEFAULT_ADDRPORT = "0.0.0.0:8000"
 
@@ -51,7 +51,7 @@ class Command(BaseCommand):
             if hasattr(err, 'errno') and err.errno in errors:
                 errtext = errors[err.errno]
             else:
-                errtext = force_text(err)
+                errtext = force_str(err)
             raise CommandError(errtext)
 
         self.stdout.write("OK")

bitbake/lib/toaster/toastermain/settings.py

@@ -9,6 +9,8 @@
 # Django settings for Toaster project.
 
 import os
+from pathlib import Path
+from toastermain.logs import LOGGING_SETTINGS
 
 DEBUG = True
 
@@ -186,7 +188,13 @@ TEMPLATES = [
                 'django.template.loaders.app_directories.Loader',
                 #'django.template.loaders.eggs.Loader',
             ],
-            'string_if_invalid': InvalidString("%s"),
+            # https://docs.djangoproject.com/en/4.2/ref/templates/api/#how-invalid-variables-are-handled
+            # Generally, string_if_invalid should only be enabled in order to debug
+            # a specific template problem, then cleared once debugging is complete.
+            # If you assign a value other than '' to string_if_invalid,
+            # you will experience rendering problems with these templates and sites.
+            #  'string_if_invalid': InvalidString("%s"),
+            'string_if_invalid': "",
             'debug': DEBUG,
         },
     },
@@ -242,6 +250,9 @@ INSTALLED_APPS = (
     'django.contrib.humanize',
     'bldcollector',
     'toastermain',
+
+    # 3rd-lib
+    "log_viewer",
 )
 
 
@@ -302,43 +313,22 @@ for t in os.walk(os.path.dirname(currentdir)):
 # the site admins on every HTTP 500 error when DEBUG=False.
 # See http://docs.djangoproject.com/en/dev/topics/logging for
 # more details on how to customize your logging configuration.
-LOGGING = {
-    'version': 1,
-    'disable_existing_loggers': False,
-    'filters': {
-        'require_debug_false': {
-            '()': 'django.utils.log.RequireDebugFalse'
-        }
-    },
-    'formatters': {
-        'datetime': {
-            'format': '%(asctime)s %(levelname)s %(message)s'
-        }
-    },
-    'handlers': {
-        'mail_admins': {
-            'level': 'ERROR',
-            'filters': ['require_debug_false'],
-            'class': 'django.utils.log.AdminEmailHandler'
-        },
-        'console': {
-            'level': 'DEBUG',
-            'class': 'logging.StreamHandler',
-            'formatter': 'datetime',
-        }
-    },
-    'loggers': {
-        'toaster' : {
-            'handlers': ['console'],
-            'level': 'DEBUG',
-        },
-        'django.request': {
-            'handlers': ['console'],
-            'level': 'WARN',
-            'propagate': True,
-        },
-    }
-}
+LOGGING = LOGGING_SETTINGS
+
+# Build paths inside the project like this: BASE_DIR / 'subdir'.
+BASE_DIR = Path(__file__).resolve(strict=True).parent.parent
+
+# LOG VIEWER
+# https://pypi.org/project/django-log-viewer/
+LOG_VIEWER_FILES_PATTERN = '*.log*'
+LOG_VIEWER_FILES_DIR = os.path.join(BASE_DIR, 'logs')
+LOG_VIEWER_PAGE_LENGTH = 25      # total log lines per-page
+LOG_VIEWER_MAX_READ_LINES = 100000  # total log lines will be read
+LOG_VIEWER_PATTERNS = ['INFO', 'DEBUG', 'WARNING', 'ERROR', 'CRITICAL']
+
+# Optionally you can set the next variables in order to customize the admin:
+LOG_VIEWER_FILE_LIST_TITLE = "Logs list"
+
 
 if DEBUG and SQL_DEBUG:
     LOGGING['loggers']['django.db.backends'] = {

bitbake/lib/toaster/toastermain/urls.py

@@ -6,7 +6,7 @@
 # SPDX-License-Identifier: GPL-2.0-only
 #
 
-from django.conf.urls import include, url
+from django.urls import re_path as url, include
 from django.views.generic import RedirectView, TemplateView
 from django.views.decorators.cache import never_cache
 import bldcollector.views
@@ -28,6 +28,8 @@ urlpatterns = [
     # url(r'^admin/doc/', include('django.contrib.admindocs.urls')),
 
 
+    url(r'^logs/', include('log_viewer.urls')),
+
     # This is here to maintain backward compatibility and will be deprecated
     # in the future.
     url(r'^orm/eventfile$', bldcollector.views.eventfile),

bitbake/toaster-requirements.txt

@@ -1,3 +1,4 @@
-Django>3.2,<3.3
+Django>4.2,<4.3
 beautifulsoup4>=4.4.0
 pytz
+django-log-viewer==1.1.7

documentation/.gitignore

@@ -7,3 +7,5 @@ releases.rst
 .vscode/
 */svg/*.png
 */svg/*.pdf
+styles/*
+!styles/config

documentation/.vale.ini

@@ -0,0 +1,7 @@
+StylesPath = styles
+MinAlertLevel = suggestion
+Packages = RedHat, proselint, write-good, alex, Readability, Joblint
+Vocab = Yocto, OpenSource
+[*.rst]
+BasedOnStyles = Vale, RedHat, proselint, write-good, alex, Readability, Joblint
+

documentation/Makefile

@@ -5,6 +5,9 @@
 # from the environment for the first two.
 SPHINXOPTS    ?= -W --keep-going -j auto
 SPHINXBUILD   ?= sphinx-build
+# Release notes are excluded because they contain contributor names and commit messages which can't be modified
+VALEOPTS      ?= --no-wrap --glob '!migration-guides/release-notes-*.rst'
+VALEDOCS      ?= .
 SOURCEDIR     = .
 IMAGEDIRS     = */svg
 BUILDDIR      = _build
@@ -20,7 +23,7 @@ endif
 help:
 	@$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)
 
-.PHONY: all help Makefile clean publish epub latexpdf
+.PHONY: all help Makefile clean stylecheck publish epub latexpdf
 
 publish: Makefile html singlehtml
 	rm -rf $(BUILDDIR)/$(DESTDIR)/
@@ -44,7 +47,15 @@ PNGs := $(foreach dir, $(IMAGEDIRS), $(patsubst %.svg,%.png,$(wildcard $(SOURCED
 	$(SVG2PNG) --export-filename=$@ $<
 
 clean:
-	@rm -rf $(BUILDDIR) $(PNGs) $(PDFs) poky.yaml sphinx-static/switchers.js
+	@rm -rf $(BUILDDIR) $(PNGs) $(PDFs) poky.yaml sphinx-static/switchers.js releases.rst
+
+stylecheck:
+	vale sync
+	vale $(VALEOPTS) $(VALEDOCS)
+
+stylecheck:
+	vale sync
+	vale $(VALEOPTS) $(VALEDOCS)
 
 epub: $(PNGs)
 	$(SOURCEDIR)/set_versions.py

documentation/README

@@ -34,16 +34,18 @@ Manual Organization
 
 Here the folders corresponding to individual manuals:
 
+* brief-yoctoprojectqs - Yocto Project Quick Start
 * overview-manual      - Yocto Project Overview and Concepts Manual
-* sdk-manual           - Yocto Project Software Development Kit (SDK) Developer's Guide.
+* contributor-guide    - Yocto Project and OpenEmbedded Contributor Guide
+* ref-manual           - Yocto Project Reference Manual
 * bsp-guide            - Yocto Project Board Support Package (BSP) Developer's Guide
 * dev-manual           - Yocto Project Development Tasks Manual
 * kernel-dev           - Yocto Project Linux Kernel Development Manual
-* ref-manual           - Yocto Project Reference Manual
-* brief-yoctoprojectqs - Yocto Project Quick Start
 * profile-manual       - Yocto Project Profiling and Tracing Manual
+* sdk-manual           - Yocto Project Software Development Kit (SDK) Developer's Guide.
 * toaster-manual       - Toaster User Manual
 * test-manual          - Yocto Project Test Environment Manual
+* migration-guides     - Yocto Project Release and Migration Notes
 
 Each folder is self-contained regarding content and figures.
 
@@ -149,6 +151,20 @@ dependencies in a virtual environment:
  $ pipenv install
  $ pipenv run make html
 
+Style checking the Yocto Project documentation
+==============================================
+
+The project is starting to use Vale (https://vale.sh/)
+to validate the text style.
+
+To install Vale:
+
+ $ pip install vale
+
+To run Vale:
+
+ $ make stylecheck
+
 Sphinx theme and CSS customization
 ==================================
 

documentation/brief-yoctoprojectqs/index.rst

@@ -251,10 +251,10 @@ an entire Linux distribution, including the toolchain, from source.
       To use such mirrors, uncomment the below lines in your ``conf/local.conf``
       file in the :term:`Build Directory`::
 
-         BB_SIGNATURE_HANDLER = "OEEquivHash"
-         BB_HASHSERVE = "auto"
          BB_HASHSERVE_UPSTREAM = "hashserv.yocto.io:8687"
-         SSTATE_MIRRORS ?= "file://.* https://sstate.yoctoproject.org/all/PATH;downloadfilename=PATH"
+         SSTATE_MIRRORS ?= "file://.* http://cdn.jsdelivr.net/yocto/sstate/all/PATH;downloadfilename=PATH"
+         BB_HASHSERVE = "auto"
+         BB_SIGNATURE_HANDLER = "OEEquivHash"
 
 #. **Start the Build:** Continue with the following command to build an OS
    image for the target, which is ``core-image-sato`` in this example:

documentation/bsp-guide/bsp.rst

@@ -64,8 +64,8 @@ Here is an example that clones the Raspberry Pi BSP layer::
 
 In addition to BSP layers, the ``meta-yocto-bsp`` layer is part of the
 shipped ``poky`` repository. The ``meta-yocto-bsp`` layer maintains
-several "reference" BSPs including the ARM-based Beaglebone, MIPS-based
-EdgeRouter, and generic versions of both 32-bit and 64-bit IA machines.
+several "reference" BSPs including the ARM-based Beaglebone and generic
+versions of both 32-bit and 64-bit IA machines.
 
 For information on typical BSP development workflow, see the
 :ref:`bsp-guide/bsp:developing a board support package (bsp)`
@@ -764,29 +764,13 @@ workflow.
 
    .. note::
 
-      -  There are four hardware reference BSPs in the Yocto
+      -  There are three hardware reference BSPs in the Yocto
          Project release, located in the ``poky/meta-yocto-bsp``
          BSP layer:
 
          -  Texas Instruments Beaglebone (``beaglebone-yocto``)
 
-         -  Ubiquiti Networks EdgeRouter Lite (``edgerouter``)
-
-         -  Two general IA platforms (``genericx86`` and ``genericx86-64``)
-
-      -  There are three core Intel BSPs in the Yocto Project
-         release, in the ``meta-intel`` layer:
-
-         -  ``intel-core2-32``, which is a BSP optimized for the Core2
-            family of CPUs as well as all CPUs prior to the Silvermont
-            core.
-
-         -  ``intel-corei7-64``, which is a BSP optimized for Nehalem
-            and later Core and Xeon CPUs as well as Silvermont and later
-            Atom CPUs, such as the Baytrail SoCs.
-
-         -  ``intel-quark``, which is a BSP optimized for the Intel
-            Galileo gen1 & gen2 development boards.
+         -  Two generic IA platforms (``genericx86`` and ``genericx86-64``)
 
    When you set up a layer for a new BSP, you should follow a standard
    layout. This layout is described in the ":ref:`bsp-guide/bsp:example filesystem layout`"
@@ -867,8 +851,7 @@ Before looking at BSP requirements, you should consider the following:
    dictating that a specific kernel or kernel version be used in a given
    BSP.
 
-Following are the requirements for a released BSP that conform to the
-Yocto Project:
+The requirements for a released BSP that conform to the Yocto Project are:
 
 -  *Layer Name:* The BSP must have a layer name that follows the Yocto
    Project standards. For information on BSP layer names, see the
@@ -893,8 +876,8 @@ Yocto Project:
    ``recipes-*`` subdirectories specific to the recipe's function, or
    within a subdirectory containing a set of closely-related recipes.
    The recipes themselves should follow the general guidelines for
-   recipes used in the Yocto Project found in the ":oe_wiki:`OpenEmbedded
-   Style Guide </Styleguide>`".
+   recipes found in the ":doc:`../contributor-guide/recipe-style-guide`"
+   in the Yocto Project and OpenEmbedded Contributor Guide.
 
 -  *License File:* You must include a license file in the
    ``meta-bsp_root_name`` directory. This license covers the BSP
@@ -927,8 +910,8 @@ Yocto Project:
    -  The name and contact information for the BSP layer maintainer.
       This is the person to whom patches and questions should be sent.
       For information on how to find the right person, see the
-      ":ref:`dev-manual/changes:submitting a change to the yocto project`"
-      section in the Yocto Project Development Tasks Manual.
+      :doc:`../contributor-guide/submit-changes` section in the Yocto Project and
+      OpenEmbedded Contributor Guide.
 
    -  Instructions on how to build the BSP using the BSP layer.
 
@@ -972,7 +955,7 @@ Yocto Project:
 Released BSP Recommendations
 ----------------------------
 
-Following are recommendations for released BSPs that conform to the
+Here are recommendations for released BSPs that conform to the
 Yocto Project:
 
 -  *Bootable Images:* Released BSPs can contain one or more bootable
@@ -1034,7 +1017,7 @@ the following:
    that additional hierarchy and the files would obviously not be able
    to reside in a machine-specific directory.
 
-Following is a specific example to help you better understand the
+Here is a specific example to help you better understand the
 process. This example customizes a recipe by adding a
 BSP-specific configuration file named ``interfaces`` to the
 ``init-ifupdown_1.0.bb`` recipe for machine "xyz" where the BSP layer
@@ -1194,7 +1177,7 @@ Use these steps to create a BSP layer:
 
 -  *Create a Kernel Recipe:* Create a kernel recipe in
    ``recipes-kernel/linux`` by either using a kernel append file or a
-   new custom kernel recipe file (e.g. ``yocto-linux_4.12.bb``). The BSP
+   new custom kernel recipe file (e.g. ``linux-yocto_4.12.bb``). The BSP
    layers mentioned in the previous step also contain different kernel
    examples. See the ":ref:`kernel-dev/common:modifying an existing recipe`"
    section in the Yocto Project Linux Kernel Development Manual for
@@ -1250,21 +1233,18 @@ There are one or more machine configuration files in the
 
 For example, the machine configuration file for the `BeagleBone and
 BeagleBone Black development boards <https://beagleboard.org/bone>`__ is
-located in the layer ``poky/meta-yocto-bsp/conf/machine`` and is named
-``beaglebone-yocto.conf``::
+located in :yocto_git:`poky/meta-yocto-bsp/conf/machine/beaglebone-yocto.conf
+</poky/tree/meta-yocto-bsp/conf/machine/beaglebone-yocto.conf>`::
 
    #@TYPE: Machine
    #@NAME: Beaglebone-yocto machine
    #@DESCRIPTION: Reference machine configuration for http://beagleboard.org/bone and http://beagleboard.org/black boards
 
    PREFERRED_PROVIDER_virtual/xserver ?= "xserver-xorg"
-   XSERVER ?= "xserver-xorg \
-               xf86-video-modesetting \
-              "
 
    MACHINE_EXTRA_RRECOMMENDS = "kernel-modules kernel-devicetree"
 
-   EXTRA_IMAGEDEPENDS += "u-boot"
+   EXTRA_IMAGEDEPENDS += "virtual/bootloader"
 
    DEFAULTTUNE ?= "cortexa8hf-neon"
    include conf/machine/include/arm/armv7a/tune-cortexa8.inc
@@ -1272,19 +1252,20 @@ located in the layer ``poky/meta-yocto-bsp/conf/machine`` and is named
    IMAGE_FSTYPES += "tar.bz2 jffs2 wic wic.bmap"
    EXTRA_IMAGECMD:jffs2 = "-lnp "
    WKS_FILE ?= "beaglebone-yocto.wks"
-   IMAGE_INSTALL:append = " kernel-devicetree kernel-image-zimage"
-   do_image_wic[depends] += "mtools-native:do_populate_sysroot dosfstools-native:do_populate_sysroot"
+   MACHINE_ESSENTIAL_EXTRA_RDEPENDS += "kernel-image kernel-devicetree"
+   do_image_wic[depends] += "mtools-native:do_populate_sysroot dosfstools-native:do_populate_sysroot virtual/bootloader:do_deploy"
 
-   SERIAL_CONSOLES ?= "115200;ttyS0 115200;ttyO0"
-   SERIAL_CONSOLES_CHECK = "${SERIAL_CONSOLES}"
+   SERIAL_CONSOLES ?= "115200;ttyS0 115200;ttyO0 115200;ttyAMA0"
 
    PREFERRED_PROVIDER_virtual/kernel ?= "linux-yocto"
-   PREFERRED_VERSION_linux-yocto ?= "5.0%"
+   PREFERRED_VERSION_linux-yocto ?= "6.1%"
 
    KERNEL_IMAGETYPE = "zImage"
    KERNEL_DEVICETREE = "am335x-bone.dtb am335x-boneblack.dtb am335x-bonegreen.dtb"
    KERNEL_EXTRA_ARGS += "LOADADDR=${UBOOT_ENTRYPOINT}"
 
+   PREFERRED_PROVIDER_virtual/bootloader ?= "u-boot"
+
    SPL_BINARY = "MLO"
    UBOOT_SUFFIX = "img"
    UBOOT_MACHINE = "am335x_evm_defconfig"
@@ -1293,7 +1274,24 @@ located in the layer ``poky/meta-yocto-bsp/conf/machine`` and is named
 
    MACHINE_FEATURES = "usbgadget usbhost vfat alsa"
 
-   IMAGE_BOOT_FILES ?= "u-boot.${UBOOT_SUFFIX} MLO zImage am335x-bone.dtb am335x-boneblack.dtb am335x-bonegreen.dtb"
+   IMAGE_BOOT_FILES ?= "u-boot.${UBOOT_SUFFIX} ${SPL_BINARY} ${KERNEL_IMAGETYPE} ${KERNEL_DEVICETREE}"
+
+   # support runqemu
+   EXTRA_IMAGEDEPENDS += "qemu-native qemu-helper-native"
+   IMAGE_CLASSES += "qemuboot"
+   QB_DEFAULT_FSTYPE = "wic"
+   QB_FSINFO = "wic:no-kernel-in-fs"
+   QB_KERNEL_ROOT = "/dev/vda2"
+   QB_SYSTEM_NAME = "qemu-system-arm"
+   QB_MACHINE = "-machine virt"
+   QB_CPU = "-cpu cortex-a15"
+   QB_KERNEL_CMDLINE_APPEND = "console=ttyAMA0 systemd.mask=systemd-networkd"
+   QB_OPT_APPEND = "-device virtio-rng-device"
+   QB_TAP_OPT = "-netdev tap,id=net0,ifname=@TAP@,script=no,downscript=no"
+   QB_NETWORK_DEVICE = "-device virtio-net-device,netdev=net0,mac=@MAC@"
+   QB_ROOTFS_OPT = "-drive id=disk0,file=@ROOTFS@,if=none,format=raw -device virtio-blk-device,drive=disk0"
+   QB_SERIAL_OPT = ""
+   QB_TCPSERIAL_OPT = "-device virtio-serial-device -chardev socket,id=virtcon,port=@PORT@,host=127.0.0.1 -device virtconsole,chardev=virtcon"
 
 The variables used to configure the machine define machine-specific properties; for
 example, machine-dependent packages, machine tunings, the type of kernel
@@ -1313,11 +1311,6 @@ Project Reference Manual.
    "virtual/xserver" is "xserver-xorg", available in
    ``poky/meta/recipes-graphics/xorg-xserver``.
 
--  :term:`XSERVER`: The packages that
-   should be installed to provide an X server and drivers for the
-   machine. In this example, the "xserver-xorg" and
-   "xf86-video-modesetting" are installed.
-
 -  :term:`MACHINE_EXTRA_RRECOMMENDS`:
    A list of machine-dependent packages not essential for booting the
    image. Thus, the build does not fail if the packages do not exist.
@@ -1335,12 +1328,15 @@ Project Reference Manual.
    needed in the root filesystem. In this case, the U-Boot recipe must
    be built for the image.
 
+   At the end of the file, we also use this setings to implement
+   ``runqemu`` support on the host machine.
+
 -  :term:`DEFAULTTUNE`: Machines
    use tunings to optimize machine, CPU, and application performance.
    These features, which are collectively known as "tuning features",
-   are set in the :term:`OpenEmbedded-Core (OE-Core)` layer (e.g.
-   ``poky/meta/conf/machine/include``). In this example, the default
-   tuning file is ``cortexa8hf-neon``.
+   are set in the :term:`OpenEmbedded-Core (OE-Core)` layer. In this
+   example, the default tuning file is :oe_git:`tune-cortexa8
+   </openembedded-core/tree/meta/conf/machine/include/arm/armv7a/tune-cortexa8.inc>`.
 
    .. note::
 
@@ -1360,13 +1356,7 @@ Project Reference Manual.
 
 -  :term:`WKS_FILE`: The location of
    the :ref:`Wic kickstart <ref-manual/kickstart:openembedded kickstart (\`\`.wks\`\`) reference>` file used
-   by the OpenEmbedded build system to create a partitioned image
-   (image.wic).
-
--  :term:`IMAGE_INSTALL`:
-   Specifies packages to install into an image through the
-   :ref:`ref-classes-image` class. Recipes
-   use the :term:`IMAGE_INSTALL` variable.
+   by the OpenEmbedded build system to create a partitioned image.
 
 -  ``do_image_wic[depends]``: A task that is constructed during the
    build. In this example, the task depends on specific tools in order
@@ -1384,7 +1374,7 @@ Project Reference Manual.
 
 -  :term:`PREFERRED_VERSION_linux-yocto <PREFERRED_VERSION>`:
    Defines the version of the recipe used to build the kernel, which is
-   "5.0" in this case.
+   "6.1" in this case.
 
 -  :term:`KERNEL_IMAGETYPE`:
    The type of kernel to build for the device. In this case, the
@@ -1449,39 +1439,35 @@ The kernel recipe used to build the kernel image for the BeagleBone
 device was established in the machine configuration::
 
    PREFERRED_PROVIDER_virtual/kernel ?= "linux-yocto"
-   PREFERRED_VERSION_linux-yocto ?= "5.0%"
+   PREFERRED_VERSION_linux-yocto ?= "6.1%"
 
 The ``meta-yocto-bsp/recipes-kernel/linux`` directory in the layer contains
 metadata used to build the kernel. In this case, a kernel append file
-(i.e. ``linux-yocto_5.0.bbappend``) is used to override an established
-kernel recipe (i.e. ``linux-yocto_5.0.bb``), which is located in
+(i.e. ``linux-yocto_6.1.bbappend``) is used to override an established
+kernel recipe (i.e. ``linux-yocto_6.1.bb``), which is located in
 :yocto_git:`/poky/tree/meta/recipes-kernel/linux`.
 
-Following is the contents of the append file::
+The contents of the append file are::
 
-   KBRANCH:genericx86 = "v5.0/standard/base"
-   KBRANCH:genericx86-64 = "v5.0/standard/base"
-   KBRANCH:edgerouter = "v5.0/standard/edgerouter"
-   KBRANCH:beaglebone-yocto = "v5.0/standard/beaglebone"
+   KBRANCH:genericx86  = "v6.1/standard/base"
+   KBRANCH:genericx86-64  = "v6.1/standard/base"
+   KBRANCH:beaglebone-yocto = "v6.1/standard/beaglebone"
 
    KMACHINE:genericx86 ?= "common-pc"
    KMACHINE:genericx86-64 ?= "common-pc-64"
    KMACHINE:beaglebone-yocto ?= "beaglebone"
 
-   SRCREV_machine:genericx86 ?= "3df4aae6074e94e794e27fe7f17451d9353cdf3d"
-   SRCREV_machine:genericx86-64 ?= "3df4aae6074e94e794e27fe7f17451d9353cdf3d"
-   SRCREV_machine:edgerouter ?= "3df4aae6074e94e794e27fe7f17451d9353cdf3d"
-   SRCREV_machine:beaglebone-yocto ?= "3df4aae6074e94e794e27fe7f17451d9353cdf3d"
+   SRCREV_machine:genericx86 ?= "6ec439b4b456ce929c4c07fe457b5d6a4b468e86"
+   SRCREV_machine:genericx86-64 ?= "6ec439b4b456ce929c4c07fe457b5d6a4b468e86"
+   SRCREV_machine:beaglebone-yocto ?= "423e1996694b61fbfc8ec3bf062fc6461d64fde1"
 
    COMPATIBLE_MACHINE:genericx86 = "genericx86"
    COMPATIBLE_MACHINE:genericx86-64 = "genericx86-64"
-   COMPATIBLE_MACHINE:edgerouter = "edgerouter"
    COMPATIBLE_MACHINE:beaglebone-yocto = "beaglebone-yocto"
 
-   LINUX_VERSION:genericx86 = "5.0.3"
-   LINUX_VERSION:genericx86-64 = "5.0.3"
-   LINUX_VERSION:edgerouter = "5.0.3"
-   LINUX_VERSION:beaglebone-yocto = "5.0.3"
+   LINUX_VERSION:genericx86 = "6.1.30"
+   LINUX_VERSION:genericx86-64 = "6.1.30"
+   LINUX_VERSION:beaglebone-yocto = "6.1.20"
 
 This particular append file works for all the machines that are
 part of the ``meta-yocto-bsp`` layer. The relevant statements are

documentation/contributor-guide/identify-component.rst

@@ -0,0 +1,31 @@
+.. SPDX-License-Identifier: CC-BY-SA-2.0-UK
+
+Identify the component
+**********************
+
+The Yocto Project and OpenEmbedded ecosystem is built of :term:`layers <Layer>`
+so the first step is to identify the component where the issue likely lies.
+For example, if you have a hardware issue, it is likely related to the BSP
+you are using and the best place to seek advice would be from the BSP provider
+or :term:`layer`. If the issue is a build/configuration one and a distro is in
+use, they would likely be the first place to ask questions. If the issue is a
+generic one and/or in the core classes or metadata, the core layer or BitBake
+might be the appropriate component.
+
+Each metadata layer being used should contain a ``README`` file and that should
+explain where to report issues, where to send changes and how to contact the
+maintainers.
+
+If the issue is in the core metadata layer (OpenEmbedded-Core) or in BitBake,
+issues can be reported in the :yocto_bugs:`Yocto Project Bugzilla <>`. The
+:yocto_lists:`yocto </g/yocto>` mailing list is a general “catch-all” location
+where questions can be sent if you can’t work out where something should go.
+
+:term:`Poky` is a commonly used “combination” repository where multiple
+components have been combined (:oe_git:`bitbake </bitbake>`,
+:oe_git:`openembedded-core </openembedded-core>`,
+:yocto_git:`meta-yocto </meta-yocto>` and
+:yocto_git:`yocto-docs </yocto-docs>`). Patches should be submitted against the
+appropriate individual component rather than :term:`Poky` itself as detailed in
+the appropriate ``README`` file.
+

documentation/contributor-guide/index.rst

@@ -0,0 +1,26 @@
+.. SPDX-License-Identifier: CC-BY-SA-2.0-UK
+
+================================================
+Yocto Project and OpenEmbedded Contributor Guide
+================================================
+
+The Yocto Project and OpenEmbedded are open-source, community-based projects so
+contributions are very welcome, it is how the code evolves and everyone can
+effect change. Contributions take different forms, if you have a fix for an
+issue you’ve run into, a patch is the most appropriate way to contribute it.
+If you run into an issue but don’t have a solution, opening a defect in
+:yocto_bugs:`Bugzilla <>` or asking questions on the mailing lists might be
+more appropriate. This guide intends to point you in the right direction to
+this.
+
+
+.. toctree::
+   :caption: Table of Contents
+   :numbered:
+
+   identify-component
+   report-defect
+   recipe-style-guide
+   submit-changes
+
+.. include:: /boilerplate.rst

documentation/contributor-guide/recipe-style-guide.rst

@@ -0,0 +1,411 @@
+.. SPDX-License-Identifier: CC-BY-SA-2.0-UK
+
+Recipe Style Guide
+******************
+
+Recipe Naming Conventions
+=========================
+
+In general, most recipes should follow the naming convention
+``recipes-category/recipename/recipename_version.bb``. Recipes for related
+projects may share the same recipe directory. ``recipename`` and ``category``
+may contain hyphens, but hyphens are not allowed in ``version``.
+
+If the recipe is tracking a Git revision that does not correspond to a released
+version of the software, ``version`` may be ``git`` (e.g. ``recipename_git.bb``)
+and the recipe would set :term:`PV`.
+
+Version Policy
+==============
+
+Our versions follow the form ``<epoch>:<version>-<revision>``
+or in BitBake variable terms ${:term:`PE`}:${:term:`PV`}-${:term:`PR`}. We
+generally follow the `Debian <https://www.debian.org/doc/debian-policy/ch-controlfields.html#version>`__
+version policy which defines these terms.
+
+In most cases the version :term:`PV` will be set automatically from the recipe
+file name. It is recommended to use released versions of software as these are
+revisions that upstream are expecting people to use.
+
+Recipe versions should always compare and sort correctly so that upgrades work
+as expected. With conventional versions such as ``1.4`` upgrading ``to 1.5``
+this happens naturally, but some versions don't sort. For example,
+``1.5 Release Candidate 2`` could be written as ``1.5rc2`` but this sorts after
+``1.5``, so upgrades from feeds won't happen correctly.
+
+Instead the tilde (``~``) operator can be used, which sorts before the empty
+string so ``1.5~rc2`` comes before ``1.5``. There is a historical syntax which
+may be found where :term:`PV` is set as a combination of the prior version
+``+`` the pre-release version, for example ``PV=1.4+1.5rc2``. This is a valid
+syntax but the tilde form is preferred.
+
+For version comparisons, the ``opkg-compare-versions`` program from
+``opkg-utils`` can be useful when attempting to determine how two version
+numbers compare to each other. Our definitive version comparison algorithm is
+the one within bitbake which aims to match those of the package managers and
+Debian policy closely.
+
+When a recipe references a git revision that does not correspond to a released
+version of software (e.g. is not a tagged version), the :term:`PV` variable
+should include the Git revision using the following to make the
+version clear::
+
+    PV = "<version>+git${SRCPV}"
+
+In this case, ``<version>`` should be the most recently released version of the
+software from the current source revision (``git describe`` can be useful for
+determining this). Whilst not recommended for published layers, this format is
+also useful when using :term:`AUTOREV` to set the recipe to increment source
+control revisions automatically, which can be useful during local development.
+
+Version Number Changes
+======================
+
+The :term:`PR` variable is used to indicate different revisions of a recipe
+that reference the same upstream source version. It can be used to force a
+new version of a recipe to be installed onto a device from a package feed.
+These once had to be set manually but in most cases these can now be set and
+incremented automatically by a PR Server connected with a package feed.
+
+When :term:`PV` increases, any existing :term:`PR` value can and should be
+removed.
+
+If :term:`PV` changes in such a way that it does not increase with respect to
+the previous value, you need to increase :term:`PE` to ensure package managers
+will upgrade it correctly. If unset you should set :term:`PE` to "1" since
+the default of empty is easily confused with "0" depending on the package
+manager. :term:`PE` can only have an integer value.
+
+Recipe formatting
+=================
+
+Variable Formatting
+-------------------
+
+-  Variable assignment should a space around each side of the operator, e.g.
+   ``FOO = "bar"``, not ``FOO="bar"``.
+
+-  Double quotes should be used on the right-hand side of the assignment,
+   e.g. ``FOO = "bar"`` not ``FOO = 'bar'``
+
+-  Spaces should be used for indenting variables, with 4 spaces per tab
+
+-  Long variables should be split over multiple lines when possible by using
+   the continuation character (``\``)
+
+-  When splitting a long variable over multiple lines, all continuation lines
+   should be indented (with spaces) to align with the start of the quote on the
+   first line::
+
+       FOO = "this line is \
+              long \
+              "
+
+   Instead of::
+
+       FOO = "this line is \
+       long \
+       "
+
+Python Function formatting
+--------------------------
+
+-  Spaces must be used for indenting Python code, with 4 spaces per tab
+
+Shell Function formatting
+-------------------------
+
+-  The formatting of shell functions should be consistent within layers.
+   Some use tabs, some use spaces.
+
+Recipe metadata
+===============
+
+Required Variables
+------------------
+
+The following variables should be included in all recipes:
+
+-  :term:`SUMMARY`: a one line description of the upstream project
+
+-  :term:`DESCRIPTION`: an extended description of the upstream project,
+   possibly with multiple lines. If no reasonable description can be written,
+   this may be omitted as it defaults to :term:`SUMMARY`.
+
+-  :term:`HOMEPAGE`: the URL to the upstream projects homepage.
+
+-  :term:`BUGTRACKER`: the URL upstream projects bug tracking website,
+   if applicable.
+
+Recipe Ordering
+---------------
+
+When a variable is defined in recipes and classes, variables should follow the
+general order when possible:
+
+-  :term:`SUMMARY`
+-  :term:`DESCRIPTION`
+-  :term:`HOMEPAGE`
+-  :term:`BUGTRACKER`
+-  :term:`SECTION`
+-  :term:`LICENSE`
+-  :term:`LIC_FILES_CHKSUM`
+-  :term:`DEPENDS`
+-  :term:`PROVIDES`
+-  :term:`PV`
+-  :term:`SRC_URI`
+-  :term:`SRCREV`
+-  :term:`S`
+-  ``inherit ...``
+-  :term:`PACKAGECONFIG`
+-  Build class specific variables such as ``EXTRA_QMAKEVARS_POST`` and :term:`EXTRA_OECONF`
+-  Tasks such as :ref:`ref-tasks-configure`
+-  :term:`PACKAGE_ARCH`
+-  :term:`PACKAGES`
+-  :term:`FILES`
+-  :term:`RDEPENDS`
+-  :term:`RRECOMMENDS`
+-  :term:`RSUGGESTS`
+-  :term:`RPROVIDES`
+-  :term:`RCONFLICTS`
+-  :term:`BBCLASSEXTEND`
+
+There are some cases where ordering is important and these cases would override
+this default order. Examples include:
+
+-  :term:`PACKAGE_ARCH` needing to be set before ``inherit packagegroup``
+
+Tasks should be ordered based on the order they generally execute. For commonly
+used tasks this would be:
+
+-  :ref:`ref-tasks-fetch`
+-  :ref:`ref-tasks-unpack`
+-  :ref:`ref-tasks-patch`
+-  :ref:`ref-tasks-prepare_recipe_sysroot`
+-  :ref:`ref-tasks-configure`
+-  :ref:`ref-tasks-compile`
+-  :ref:`ref-tasks-install`
+-  :ref:`ref-tasks-populate_sysroot`
+-  :ref:`ref-tasks-package`
+
+Custom tasks should be sorted similarly.
+
+Package specific variables are typically grouped together, e.g.::
+
+    RDEPENDS:${PN} = “foo”
+    RDEPENDS:${PN}-libs = “bar”
+
+    RRECOMMENDS:${PN} = “one”
+    RRECOMMENDS:${PN}-libs = “two”
+
+Recipe License Fields
+---------------------
+
+Recipes need to define both the :term:`LICENSE` and
+:term:`LIC_FILES_CHKSUM` variables:
+
+-  :term:`LICENSE`: This variable specifies the license for the software.
+   If you do not know the license under which the software you are
+   building is distributed, you should go to the source code and look
+   for that information. Typical files containing this information
+   include ``COPYING``, :term:`LICENSE`, and ``README`` files. You could
+   also find the information near the top of a source file. For example,
+   given a piece of software licensed under the GNU General Public
+   License version 2, you would set :term:`LICENSE` as follows::
+
+      LICENSE = "GPL-2.0-only"
+
+   The licenses you specify within :term:`LICENSE` can have any name as long
+   as you do not use spaces, since spaces are used as separators between
+   license names. For standard licenses, use the names of the files in
+   ``meta/files/common-licenses/`` or the :term:`SPDXLICENSEMAP` flag names
+   defined in ``meta/conf/licenses.conf``.
+
+-  :term:`LIC_FILES_CHKSUM`: The OpenEmbedded build system uses this
+   variable to make sure the license text has not changed. If it has,
+   the build produces an error and it affords you the chance to figure
+   it out and correct the problem.
+
+   You need to specify all applicable licensing files for the software.
+   At the end of the configuration step, the build process will compare
+   the checksums of the files to be sure the text has not changed. Any
+   differences result in an error with the message containing the
+   current checksum. For more explanation and examples of how to set the
+   :term:`LIC_FILES_CHKSUM` variable, see the
+   ":ref:`dev-manual/licenses:tracking license changes`" section.
+
+   To determine the correct checksum string, you can list the
+   appropriate files in the :term:`LIC_FILES_CHKSUM` variable with incorrect
+   md5 strings, attempt to build the software, and then note the
+   resulting error messages that will report the correct md5 strings.
+   See the ":ref:`dev-manual/new-recipe:fetching code`" section for
+   additional information.
+
+   Here is an example that assumes the software has a ``COPYING`` file::
+
+      LIC_FILES_CHKSUM = "file://COPYING;md5=xxx"
+
+   When you try to build the
+   software, the build system will produce an error and give you the
+   correct string that you can substitute into the recipe file for a
+   subsequent build.
+
+License Updates
+~~~~~~~~~~~~~~~
+
+When you change the :term:`LICENSE` or :term:`LIC_FILES_CHKSUM` in the recipe
+you need to briefly explain the reason for the change via a ``License-Update:``
+tag.  Often it's quite trivial, such as::
+
+    License-Update: copyright years refreshed
+
+Less often, the actual licensing terms themselves will have changed.  If so, do
+try to link to upstream making/justifying that decision.
+
+Tips and Guidelines for Writing Recipes
+---------------------------------------
+
+-  Use :term:`BBCLASSEXTEND` instead of creating separate recipes such as ``-native``
+   and ``-nativesdk`` ones, whenever possible. This avoids having to maintain multiple
+   recipe files at the same time.
+
+-  Recipes should have tasks which are idempotent, i.e. that executing a given task
+   multiple times shouldn't change the end result. The build environment is built upon
+   this assumption and breaking it can cause obscure build failures.
+
+-  For idempotence when modifying files in tasks, it is usually best to:
+
+   - copy a file ``X`` to ``X.orig`` (only if it doesn't exist already)
+   - then, copy ``X.orig`` back to ``X``,
+   - and, finally, modify ``X``.
+
+   This ensures if rerun the task always has the same end result and the
+   original file can be preserved to reuse. It also guards against an
+   interrupted build corrupting the file.
+
+Patch Upstream Status
+=====================
+
+In order to keep track of patches applied by recipes and ultimately reduce the
+number of patches that need maintaining, the OpenEmbedded build system
+requires information about the upstream status of each patch.
+
+In its description, each patch should provide detailed information about the
+bug that it addresses, such as the URL in a bug tracking system and links
+to relevant mailing list archives.
+
+Then, you should also add an ``Upstream-Status:`` tag containing one of the
+following status strings:
+
+``Pending``
+   No determination has been made yet, or patch has not yet been submitted to
+   upstream.
+
+   Keep in mind that every patch submitted upstream reduces the maintainance
+   burden in OpenEmbedded and Yocto Project in the long run, so this patch
+   status should only be used in exceptional cases if there are genuine
+   obstacles to submitting a patch upstream; the reason for that should be
+   included in the patch.
+
+``Submitted [where]``
+   Submitted to upstream, waiting for approval. Optionally include where
+   it was submitted, such as the author, mailing list, etc.
+
+``Backport [version]``
+   Accepted upstream and included in the next release, or backported from newer
+   upstream version, because we are at a fixed version.
+   Include upstream version info (e.g. commit ID or next expected version).
+
+``Denied``
+   Not accepted by upstream, include reason in patch.
+
+``Inactive-Upstream [lastcommit: when (and/or) lastrelease: when]``
+   The upstream is no longer available. This typically means a defunct project
+   where no activity has happened for a long time --- measured in years. To make
+   that judgement, it is recommended to look at not only when the last release
+   happened, but also when the last commit happened, and whether newly made bug
+   reports and merge requests since that time receive no reaction. It is also
+   recommended to add to the patch description any relevant links where the
+   inactivity can be clearly seen.
+
+``Inappropriate [reason]``
+   The patch is not appropriate for upstream, include a brief reason on the
+   same line enclosed with ``[]``. In the past, there were several different
+   reasons not to submit patches upstream, but we have to consider that every
+   non-upstreamed patch means a maintainance burden for recipe maintainers.
+   Currently, the only reasons to mark patches as inappropriate for upstream
+   submission are:
+
+   -  ``oe specific``: the issue is specific to how OpenEmbedded performs builds
+      or sets things up at runtime, and can be resolved only with a patch that
+      is not however relevant or appropriate for general upstream submission.
+   -  ``upstream ticket <link>``: the issue is not specific to Open-Embedded
+      and should be fixed upstream, but the patch in its current form is not
+      suitable for merging upstream, and the author lacks sufficient expertise
+      to develop a proper patch. Instead the issue is handled via a bug report
+      (include link).
+
+Of course, if another person later takes care of submitting this patch upstream,
+the status should be changed to ``Submitted [where]``, and an additional
+``Signed-off-by:`` line should be added to the patch by the person claiming
+responsibility for upstreaming.
+
+Examples
+--------
+
+Here's an example of a patch that has been submitted upstream::
+
+   rpm: Adjusted the foo setting in bar
+
+   [RPM Ticket #65] -- http://rpm5.org/cvs/tktview?tn=65,5
+
+   The foo setting in bar was decreased from X to X-50% in order to
+   ensure we don't exhaust all system memory with foobar threads.
+
+   Upstream-Status: Submitted [rpm5-devel@rpm5.org]
+
+   Signed-off-by: Joe Developer <joe.developer@example.com>
+
+A future update can change the value to ``Backport`` or ``Denied`` as
+appropriate.
+
+Another example of a patch that is specific to OpenEmbedded::
+
+   Do not treat warnings as errors
+
+   There are additional warnings found with musl which are
+   treated as errors and fails the build, we have more combinations
+   than upstream supports to handle.
+
+   Upstream-Status: Inappropriate [oe specific]
+
+Here's a patch that has been backported from an upstream commit::
+
+   include missing sys/file.h for LOCK_EX
+
+   Upstream-Status: Backport [https://github.com/systemd/systemd/commit/ac8db36cbc26694ee94beecc8dca208ec4b5fd45]
+
+CVE patches
+===========
+
+In order to have a better control of vulnerabilities, patches that fix CVEs must
+contain a ``CVE:`` tag. This tag list all CVEs fixed by the patch. If more than
+one CVE is fixed, separate them using spaces.
+
+CVE Examples
+------------
+
+This should be the header of patch that fixes :cve:`2015-8370` in GRUB2::
+
+   grub2: Fix CVE-2015-8370
+
+   [No upstream tracking] -- https://bugzilla.redhat.com/show_bug.cgi?id=1286966
+
+   Back to 28; Grub2 Authentication
+
+   Two functions suffer from integer underflow fault; the grub_username_get() and grub_password_get()located in
+   grub-core/normal/auth.c and lib/crypto.c respectively. This can be exploited to obtain a Grub rescue shell.
+
+   Upstream-Status: Backport [http://git.savannah.gnu.org/cgit/grub.git/commit/?id=451d80e52d851432e109771bb8febafca7a5f1f2]
+   CVE: CVE-2015-8370
+   Signed-off-by: Joe Developer <joe.developer@example.com>

documentation/contributor-guide/report-defect.rst

@@ -0,0 +1,67 @@
+.. SPDX-License-Identifier: CC-BY-SA-2.0-UK
+
+Reporting a Defect Against the Yocto Project and OpenEmbedded
+**************************************************************
+
+You can use the Yocto Project instance of
+`Bugzilla <https://www.bugzilla.org/about/>`__ to submit a defect (bug)
+against BitBake, OpenEmbedded-Core, against any other Yocto Project component
+or for tool issues. For additional information on this implementation of
+Bugzilla see the ":ref:`Yocto Project Bugzilla <resources-bugtracker>`" section
+in the Yocto Project Reference Manual. For more detail on any of the following
+steps, see the Yocto Project
+:yocto_wiki:`Bugzilla wiki page </Bugzilla_Configuration_and_Bug_Tracking>`.
+
+Use the following general steps to submit a bug:
+
+#.  Open the Yocto Project implementation of :yocto_bugs:`Bugzilla <>`.
+
+#.  Click "File a Bug" to enter a new bug.
+
+#.  Choose the appropriate "Classification", "Product", and "Component"
+    for which the bug was found. Bugs for the Yocto Project fall into
+    one of several classifications, which in turn break down into
+    several products and components. For example, for a bug against the
+    ``meta-intel`` layer, you would choose "Build System, Metadata &
+    Runtime", "BSPs", and "bsps-meta-intel", respectively.
+
+#.  Choose the "Version" of the Yocto Project for which you found the
+    bug (e.g. &DISTRO;).
+
+#.  Determine and select the "Severity" of the bug. The severity
+    indicates how the bug impacted your work.
+
+#.  Choose the "Hardware" that the bug impacts.
+
+#.  Choose the "Architecture" that the bug impacts.
+
+#.  Choose a "Documentation change" item for the bug. Fixing a bug might
+    or might not affect the Yocto Project documentation. If you are
+    unsure of the impact to the documentation, select "Don't Know".
+
+#.  Provide a brief "Summary" of the bug. Try to limit your summary to
+    just a line or two and be sure to capture the essence of the bug.
+
+#.  Provide a detailed "Description" of the bug. You should provide as
+    much detail as you can about the context, behavior, output, and so
+    forth that surrounds the bug. You can even attach supporting files
+    for output from logs by using the "Add an attachment" button.
+
+#.  Click the "Submit Bug" button submit the bug. A new Bugzilla number
+    is assigned to the bug and the defect is logged in the bug tracking
+    system.
+
+Once you file a bug, the bug is processed by the Yocto Project Bug
+Triage Team and further details concerning the bug are assigned (e.g.
+priority and owner). You are the "Submitter" of the bug and any further
+categorization, progress, or comments on the bug result in Bugzilla
+sending you an automated email concerning the particular change or
+progress to the bug.
+
+There are no guarantees about if or when a bug might be worked on since an
+open-source project has no dedicated engineering resources. However, the
+project does have a good track record of resolving common issues over the
+medium and long term. We do encourage people to file bugs so issues are
+at least known about. It helps other users when they find somebody having
+the same issue as they do, and an issue that is unknown is much less likely
+to ever be fixed!

documentation/contributor-guide/submit-changes.rst

@@ -0,0 +1,827 @@
+.. SPDX-License-Identifier: CC-BY-SA-2.0-UK
+
+Contributing Changes to a Component
+************************************
+
+Contributions to the Yocto Project and OpenEmbedded are very welcome.
+Because the system is extremely configurable and flexible, we recognize
+that developers will want to extend, configure or optimize it for their
+specific uses.
+
+.. _ref-why-mailing-lists:
+
+Contributing through mailing lists --- Why not using web-based workflows?
+=========================================================================
+
+Both Yocto Project and OpenEmbedded have many key components that are
+maintained by patches being submitted on mailing lists. We appreciate this
+approach does look a little old fashioned when other workflows are available
+through web technology such as GitHub, GitLab and others. Since we are often
+asked this question, we’ve decided to document the reasons for using mailing
+lists.
+
+One significant factor is that we value peer review. When a change is proposed
+to many of the core pieces of the project, it helps to have many eyes of review
+go over them. Whilst there is ultimately one maintainer who needs to make the
+final call on accepting or rejecting a patch, the review is made by many eyes
+and the exact people reviewing it are likely unknown to the maintainer. It is
+often the surprise reviewer that catches the most interesting issues!
+
+This is in contrast to the "GitHub" style workflow where either just a
+maintainer makes that review, or review is specifically requested from
+nominated people. We believe there is significant value added to the codebase
+by this peer review and that moving away from mailing lists would be to the
+detriment of our code.
+
+We also need to acknowledge that many of our developers are used to this
+mailing list workflow and have worked with it for years, with tools and
+processes built around it. Changing away from this would result in a loss
+of key people from the project, which would again be to its detriment.
+
+The projects are acutely aware that potential new contributors find the
+mailing list approach off-putting and would prefer a web-based GUI.
+Since we don’t believe that can work for us, the project is aiming to ensure
+`patchwork <https://patchwork.yoctoproject.org/>`__ is available to help track
+patch status and also looking at how tooling can provide more feedback to users
+about patch status. We are looking at improving tools such as ``patchtest`` to
+test user contributions before they hit the mailing lists and also at better
+documenting how to use such workflows since we recognise that whilst this was
+common knowledge a decade ago, it might not be as familiar now.
+
+Preparing Changes for Submission
+================================
+
+Set up Git
+----------
+
+The first thing to do is to install Git packages. Here is an example
+on Debian and Ubuntu::
+
+   sudo apt install git-core git-email
+
+Then, you need to set a name and e-mail address that Git will
+use to identify your commits::
+
+   git config --global user.name "Ada Lovelace"
+   git config --global user.email "ada.lovelace@gmail.com"
+
+Clone the Git repository for the component to modify
+----------------------------------------------------
+
+After identifying the component to modify as described in the
+":doc:`../contributor-guide/identify-component`" section, clone the
+corresponding Git repository. Here is an example for OpenEmbedded-Core::
+
+  git clone https://git.openembedded.org/openembedded-core
+  cd openembedded-core
+
+Create a new branch
+-------------------
+
+Then, create a new branch in your local Git repository
+for your changes, starting from the reference branch in the upstream
+repository (often called ``master``)::
+
+   $ git checkout <ref-branch>
+   $ git checkout -b my-changes
+
+If you have completely unrelated sets of changes to submit, you should even
+create one branch for each set.
+
+Implement and commit changes
+----------------------------
+
+In each branch, you should group your changes into small, controlled and
+isolated ones. Keeping changes small and isolated aids review, makes
+merging/rebasing easier and keeps the change history clean should anyone need
+to refer to it in future.
+
+To this purpose, you should create *one Git commit per change*,
+corresponding to each of the patches you will eventually submit.
+See `further guidance <https://www.kernel.org/doc/html/latest/process/submitting-patches.html#separate-your-changes>`__
+in the Linux kernel documentation if needed.
+
+For example, when you intend to add multiple new recipes, each recipe
+should be added in a separate commit. For upgrades to existing recipes,
+the previous version should usually be deleted as part of the same commit
+to add the upgraded version.
+
+#. *Stage Your Changes:* Stage your changes by using the ``git add``
+   command on each file you modified. If you want to stage all the
+   files you modified, you can even use the ``git add -A`` command.
+
+#. *Commit Your Changes:* This is when you can create separate commits. For
+   each commit to create, use the ``git commit -s`` command with the files
+   or directories you want to include in the commit::
+
+      $ git commit -s file1 file2 dir1 dir2 ...
+
+   To include **a**\ ll staged files::
+
+      $ git commit -sa
+
+   -  The ``-s`` option of ``git commit`` adds a "Signed-off-by:" line
+      to your commit message. There is the same requirement for contributing
+      to the Linux kernel. Adding such a line signifies that you, the
+      submitter, have agreed to the `Developer's Certificate of Origin 1.1
+      <https://www.kernel.org/doc/html/latest/process/submitting-patches.html#sign-your-work-the-developer-s-certificate-of-origin>`__
+      as follows:
+
+      .. code-block:: none
+
+         Developer's Certificate of Origin 1.1
+
+         By making a contribution to this project, I certify that:
+
+         (a) The contribution was created in whole or in part by me and I
+             have the right to submit it under the open source license
+             indicated in the file; or
+
+         (b) The contribution is based upon previous work that, to the best
+             of my knowledge, is covered under an appropriate open source
+             license and I have the right under that license to submit that
+             work with modifications, whether created in whole or in part
+             by me, under the same open source license (unless I am
+             permitted to submit under a different license), as indicated
+             in the file; or
+
+         (c) The contribution was provided directly to me by some other
+             person who certified (a), (b) or (c) and I have not modified
+             it.
+
+         (d) I understand and agree that this project and the contribution
+             are public and that a record of the contribution (including all
+             personal information I submit with it, including my sign-off) is
+             maintained indefinitely and may be redistributed consistent with
+             this project or the open source license(s) involved.
+
+   -  Provide a single-line summary of the change and, if more
+      explanation is needed, provide more detail in the body of the
+      commit. This summary is typically viewable in the "shortlist" of
+      changes. Thus, providing something short and descriptive that
+      gives the reader a summary of the change is useful when viewing a
+      list of many commits. You should prefix this short description
+      with the recipe name (if changing a recipe), or else with the
+      short form path to the file being changed.
+
+      .. note::
+
+         To find a suitable prefix for the commit summary, a good idea
+         is to look for prefixes used in previous commits touching the
+         same files or directories::
+
+            git log --oneline <paths>
+
+   -  For the body of the commit message, provide detailed information
+      that describes what you changed, why you made the change, and the
+      approach you used. It might also be helpful if you mention how you
+      tested the change. Provide as much detail as you can in the body
+      of the commit message.
+
+      .. note::
+
+         If the single line summary is enough to describe a simple
+         change, the body of the commit message can be left empty.
+
+   -  If the change addresses a specific bug or issue that is associated
+      with a bug-tracking ID, include a reference to that ID in your
+      detailed description. For example, the Yocto Project uses a
+      specific convention for bug references --- any commit that addresses
+      a specific bug should use the following form for the detailed
+      description. Be sure to use the actual bug-tracking ID from
+      Bugzilla for bug-id::
+
+         Fixes [YOCTO #bug-id]
+
+         detailed description of change
+
+#. *Crediting contributors:* By using the ``git commit --amend`` command,
+   you can add some tags to the commit description to credit other contributors
+   to the change:
+
+   -  ``Reported-by``: name and email of a person reporting a bug
+      that your commit is trying to fix. This is a good practice
+      to encourage people to go on reporting bugs and let them
+      know that their reports are taken into account.
+
+   -  ``Suggested-by``: name and email of a person to credit for the
+      idea of making the change.
+
+   -  ``Tested-by``, ``Reviewed-by``: name and email for people having
+      tested your changes or reviewed their code. These fields are
+      usually added by the maintainer accepting a patch, or by
+      yourself if you submitted your patches to early reviewers,
+      or are submitting an unmodified patch again as part of a
+      new iteration of your patch series.
+
+   -  ``CC:`` Name and email of people you want to send a copy
+      of your changes to. This field will be used by ``git send-email``.
+
+   See `more guidance about using such tags
+   <https://www.kernel.org/doc/html/latest/process/submitting-patches.html#using-reported-by-tested-by-reviewed-by-suggested-by-and-fixes>`__
+   in the Linux kernel documentation.
+
+Test your changes
+-----------------
+
+For each contributions you make, you should test your changes as well.
+For this the Yocto Project offers several types of tests. Those tests cover
+different areas and it depends on your changes which are feasible. For example run:
+
+   -  For changes that affect the build environment:
+
+      -  ``bitbake-selftest``: for changes within BitBake
+
+      -  ``oe-selftest``: to test combinations of BitBake runs
+
+      -  ``oe-build-perf-test``: to test the performance of common build scenarios
+
+   -  For changes in a recipe:
+
+      - ``ptest``: run package specific tests, if they exist
+
+      - ``testimage``: build an image, boot it and run testcases on it
+
+      - If applicable, ensure also the ``native`` and ``nativesdk`` variants builds
+
+   -  For changes relating to the SDK:
+
+      - ``testsdk``: to build, install and run tests against a SDK
+
+      - ``testsdk_ext``: to build, install and run tests against an extended SDK
+
+Note that this list just gives suggestions and is not exhaustive. More details can
+be found here: :ref:`test-manual/intro:Yocto Project Tests --- Types of Testing Overview`.
+
+Creating Patches
+================
+
+Here is the general procedure on how to create patches to be sent through email:
+
+#. *Describe the Changes in your Branch:* If you have more than one commit
+   in your branch, it's recommended to provide a cover letter describing
+   the series of patches you are about to send.
+
+   For this purpose, a good solution is to store the cover letter contents
+   in the branch itself::
+
+      git branch --edit-description
+
+   This will open a text editor to fill in the description for your
+   changes. This description can be updated when necessary and will
+   be used by Git to create the cover letter together with the patches.
+
+   It is recommended to start this description with a title line which
+   will serve a the subject line for the cover letter.
+
+#. *Generate Patches for your Branch:* The ``git format-patch`` command will
+   generate patch files for each of the commits in your branch. You need
+   to pass the reference branch your branch starts from.
+
+   If you branch didn't need a description in the previous step::
+
+      $ git format-patch <ref-branch>
+
+   If you filled a description for your branch, you will want to generate
+   a cover letter too::
+
+      $ git format-patch --cover-letter --cover-from-description=auto <ref-branch>
+
+   After the command is run, the current directory contains numbered
+   ``.patch`` files for the commits in your branch. If you have a cover
+   letter, it will be in the ``0000-cover-letter.patch``.
+
+   .. note::
+
+      The ``--cover-from-description=auto`` option makes ``git format-patch``
+      use the first paragraph of the branch description as the cover
+      letter title. Another possibility, which is easier to remember, is to pass
+      only the ``--cover-letter`` option, but you will have to edit the
+      subject line manually every time you generate the patches.
+
+      See the `git format-patch manual page <https://git-scm.com/docs/git-format-patch>`__
+      for details.
+
+#. *Review each of the Patch Files:* This final review of the patches
+   before sending them often allows to view your changes from a different
+   perspective and discover defects such as typos, spacing issues or lines
+   or even files that you didn't intend to modify. This review should
+   include the cover letter patch too.
+
+   If necessary, rework your commits as described in
+   ":ref:`contributor-guide/submit-changes:taking patch review into account`".
+
+Validating Patches with Patchtest
+=================================
+
+``patchtest`` is available in ``openembedded-core`` as a tool for making
+sure that your patches are well-formatted and contain important info for
+maintenance purposes, such as ``Signed-off-by`` and ``Upstream-Status``
+tags. Note that no functional testing of the changes will be performed by ``patchtest``.
+Currently, it only supports testing patches for ``openembedded-core`` branches.
+To setup, perform the following::
+
+    pip install -r meta/lib/patchtest/requirements.txt
+    source oe-init-build-env
+    bitbake-layers add-layer ../meta-selftest
+
+Once these steps are complete and you have generated your patch files,
+you can run ``patchtest`` like so::
+
+    patchtest --patch <patch_name>
+
+Alternatively, if you want ``patchtest`` to iterate over and test
+multiple patches stored in a directory, you can use::
+
+    patchtest --directory <directory_name>
+
+By default, ``patchtest`` uses its own modules' file paths to determine what
+repository and test suite to check patches against. If you wish to test
+patches against a repository other than ``openembedded-core`` and/or use
+a different set of tests, you can use the ``--repodir`` and ``--testdir``
+flags::
+
+    patchtest --patch <patch_name> --repodir <path/to/repo> --testdir <path/to/testdir>
+
+Finally, note that ``patchtest`` is designed to test patches in a standalone
+way, so if your patches are meant to apply on top of changes made by
+previous patches in a series, it is possible that ``patchtest`` will report
+false failures regarding the "merge on head" test.
+
+Using ``patchtest`` in this manner provides a final check for the overall
+quality of your changes before they are submitted for review by the
+maintainers.
+
+Sending the Patches via Email
+=============================
+
+Using Git to Send Patches
+-------------------------
+
+To submit patches through email, it is very important that you send them
+without any whitespace or HTML formatting that either you or your mailer
+introduces. The maintainer that receives your patches needs to be able
+to save and apply them directly from your emails, using the ``git am``
+command.
+
+Using the ``git send-email`` command is the only error-proof way of sending
+your patches using email since there is no risk of compromising whitespace
+in the body of the message, which can occur when you use your own mail
+client. It will also properly include your patches as *inline attachments*,
+which is not easy to do with standard e-mail clients without breaking lines.
+If you used your regular e-mail client and shared your patches as regular
+attachments, reviewers wouldn't be able to quote specific sections of your
+changes and make comments about them.
+
+Setting up Git to Send Email
+----------------------------
+
+The ``git send-email`` command can send email by using a local or remote
+Mail Transport Agent (MTA) such as ``msmtp``, ``sendmail``, or
+through a direct SMTP configuration in your Git ``~/.gitconfig`` file.
+
+Here are the settings for letting ``git send-email`` send e-mail through your
+regular STMP server, using a Google Mail account as an example::
+
+   git config --global sendemail.smtpserver smtp.gmail.com
+   git config --global sendemail.smtpserverport 587
+   git config --global sendemail.smtpencryption tls
+   git config --global sendemail.smtpuser ada.lovelace@gmail.com
+   git config --global sendemail.smtppass = XXXXXXXX
+
+These settings will appear in the ``.gitconfig`` file in your home directory.
+
+If you neither can use a local MTA nor SMTP,  make sure you use an email client
+that does not touch the message (turning spaces in tabs, wrapping lines, etc.).
+A good mail client to do so is Pine (or Alpine) or Mutt. For more
+information about suitable clients, see `Email clients info for Linux
+<https://www.kernel.org/doc/html/latest/process/email-clients.html>`__
+in the Linux kernel sources.
+
+If you use such clients, just include the patch in the body of your email.
+
+Finding a Suitable Mailing List
+-------------------------------
+
+You should send patches to the appropriate mailing list so that they can be
+reviewed by the right contributors and merged by the appropriate maintainer.
+The specific mailing list you need to use depends on the location of the code
+you are changing.
+
+If people have concerns with any of the patches, they will usually voice
+their concern over the mailing list. If patches do not receive any negative
+reviews, the maintainer of the affected layer typically takes them, tests them,
+and then based on successful testing, merges them.
+
+In general, each component (e.g. layer) should have a ``README`` file
+that indicates where to send the changes and which process to follow.
+
+The "poky" repository, which is the Yocto Project's reference build
+environment, is a hybrid repository that contains several individual
+pieces (e.g. BitBake, Metadata, documentation, and so forth) built using
+the combo-layer tool. The upstream location used for submitting changes
+varies by component:
+
+-  *Core Metadata:* Send your patches to the
+   :oe_lists:`openembedded-core </g/openembedded-core>`
+   mailing list. For example, a change to anything under the ``meta`` or
+   ``scripts`` directories should be sent to this mailing list.
+
+-  *BitBake:* For changes to BitBake (i.e. anything under the
+   ``bitbake`` directory), send your patches to the
+   :oe_lists:`bitbake-devel </g/bitbake-devel>`
+   mailing list.
+
+-  *meta-poky* and *meta-yocto-bsp* trees: These trees contain Metadata. Use the
+   :yocto_lists:`poky </g/poky>` mailing list.
+
+-  *Documentation*: For changes to the Yocto Project documentation, use the
+   :yocto_lists:`docs </g/docs>` mailing list.
+
+For changes to other layers and tools hosted in the Yocto Project source
+repositories (i.e. :yocto_git:`git.yoctoproject.org <>`), use the
+:yocto_lists:`yocto </g/yocto/>` general mailing list.
+
+For changes to other layers hosted in the OpenEmbedded source
+repositories (i.e. :oe_git:`git.openembedded.org <>`), use
+the :oe_lists:`openembedded-devel </g/openembedded-devel>`
+mailing list, unless specified otherwise in the layer's ``README`` file.
+
+If you intend to submit a new recipe that neither fits into the core Metadata,
+nor into :oe_git:`meta-openembedded </meta-openembedded/>`, you should
+look for a suitable layer in https://layers.openembedded.org. If similar
+recipes can be expected, you may consider :ref:`dev-manual/layers:creating your own layer`.
+
+If in doubt, please ask on the :yocto_lists:`yocto </g/yocto/>` general mailing list
+or on the :oe_lists:`openembedded-devel </g/openembedded-devel>` mailing list.
+
+Subscribing to the Mailing List
+-------------------------------
+
+After identifying the right mailing list to use, you will have to subscribe to
+it if you haven't done it yet.
+
+If you attempt to send patches to a list you haven't subscribed to, your email
+will be returned as undelivered.
+
+However, if you don't want to be receive all the messages sent to a mailing list,
+you can set your subscription to "no email". You will still be a subscriber able
+to send messages, but you won't receive any e-mail. If people reply to your message,
+their e-mail clients will default to including your email address in the
+conversation anyway.
+
+Anyway, you'll also be able to access the new messages on mailing list archives,
+either through a web browser, or for the lists archived on https://lore.kernel.org,
+through an individual newsgroup feed or a git repository.
+
+Sending Patches via Email
+-------------------------
+
+At this stage, you are ready to send your patches via email. Here's the
+typical usage of ``git send-email``::
+
+   git send-email --to <mailing-list-address> *.patch
+
+Then, review each subject line and list of recipients carefully, and then
+and then allow the command to send each message.
+
+You will see that ``git send-email`` will automatically copy the people listed
+in any commit tags such as ``Signed-off-by`` or ``Reported-by``.
+
+In case you are sending patches for :oe_git:`meta-openembedded </meta-openembedded/>`
+or any layer other than :oe_git:`openembedded-core </openembedded-core/>`,
+please add the appropriate prefix so that it is clear which layer the patch is intended
+to be applied to::
+
+   git format-patch --subject-prefix="meta-oe][PATCH" ...
+
+.. note::
+
+   It is actually possible to send patches without generating them
+   first. However, make sure you have reviewed your changes carefully
+   because ``git send-email`` will just show you the title lines of
+   each patch.
+
+   Here's a command you can use if you just have one patch in your
+   branch::
+
+      git send-email --to <mailing-list-address> -1
+
+   If you have multiple patches and a cover letter, you can send
+   patches for all the commits between the reference branch
+   and the tip of your branch::
+
+      git send-email --cover-letter --cover-from-description=auto --to <mailing-list-address> -M <ref-branch>
+
+See the `git send-email manual page <https://git-scm.com/docs/git-send-email>`__
+for details.
+
+Troubleshooting Email Issues
+----------------------------
+
+Fixing your From identity
+~~~~~~~~~~~~~~~~~~~~~~~~~
+
+We have a frequent issue with contributors whose patches are received through
+a ``From`` field which doesn't match the ``Signed-off-by`` information. Here is
+a typical example for people sending from a domain name with :wikipedia:`DMARC`::
+
+   From: "Linus Torvalds via lists.openembedded.org <linus.torvalds=kernel.org@lists.openembedded.org>"
+
+This ``From`` field is used by ``git am`` to recreate commits with the right
+author name. The following will ensure that your e-mails have an additional
+``From`` field at the beginning of the Email body, and therefore that
+maintainers accepting your patches don't have to fix commit author information
+manually::
+
+   git config --global sendemail.from "linus.torvalds@kernel.org"
+
+The ``sendemail.from`` should match your ``user.email`` setting,
+which appears in the ``Signed-off-by`` line of your commits.
+
+Streamlining git send-email usage
+---------------------------------
+
+If you want to save time and not be forced to remember the right options to use
+with ``git send-email``, you can use Git configuration settings.
+
+-  To set the right mailing list address for a given repository::
+
+      git config --local sendemail.to openembedded-devel@lists.openembedded.org
+
+-  If the mailing list requires a subject prefix for the layer
+   (this only works when the repository only contains one layer)::
+
+      git config --local format.subjectprefix "meta-something][PATCH"
+
+Using Scripts to Push a Change Upstream and Request a Pull
+==========================================================
+
+For larger patch series it is preferable to send a pull request which not
+only includes the patch but also a pointer to a branch that can be pulled
+from. This involves making a local branch for your changes, pushing this
+branch to an accessible repository and then using the ``create-pull-request``
+and ``send-pull-request`` scripts from openembedded-core to create and send a
+patch series with a link to the branch for review.
+
+Follow this procedure to push a change to an upstream "contrib" Git
+repository once the steps in
+":ref:`contributor-guide/submit-changes:preparing changes for submission`"
+have been followed:
+
+.. note::
+
+   You can find general Git information on how to push a change upstream
+   in the
+   `Git Community Book <https://git-scm.com/book/en/v2/Distributed-Git-Distributed-Workflows>`__.
+
+#. *Request Push Access to an "Upstream" Contrib Repository:* Send an email to
+   ``helpdesk@yoctoproject.org``:
+
+    -  Attach your SSH public key which usually named ``id_rsa.pub.``.
+       If you don't have one generate it by running ``ssh-keygen -t rsa -b 4096 -C "your_email@example.com"``.
+
+    -  List the repositories you're planning to contribute to.
+
+    -  Include your preferred branch prefix for ``-contrib`` repositories.
+
+#. *Push Your Commits to the "Contrib" Upstream:* Push your
+   changes to that repository::
+
+      $ git push upstream_remote_repo local_branch_name
+
+   For example, suppose you have permissions to push
+   into the upstream ``meta-intel-contrib`` repository and you are
+   working in a local branch named `your_name`\ ``/README``. The following
+   command pushes your local commits to the ``meta-intel-contrib``
+   upstream repository and puts the commit in a branch named
+   `your_name`\ ``/README``::
+
+      $ git push meta-intel-contrib your_name/README
+
+#. *Determine Who to Notify:* Determine the maintainer or the mailing
+   list that you need to notify for the change.
+
+   Before submitting any change, you need to be sure who the maintainer
+   is or what mailing list that you need to notify. Use either these
+   methods to find out:
+
+   -  *Maintenance File:* Examine the ``maintainers.inc`` file, which is
+      located in the :term:`Source Directory` at
+      ``meta/conf/distro/include``, to see who is responsible for code.
+
+   -  *Search by File:* Using :ref:`overview-manual/development-environment:git`, you can
+      enter the following command to bring up a short list of all
+      commits against a specific file::
+
+         git shortlog -- filename
+
+      Just provide the name of the file for which you are interested. The
+      information returned is not ordered by history but does include a
+      list of everyone who has committed grouped by name. From the list,
+      you can see who is responsible for the bulk of the changes against
+      the file.
+
+   -  *Find the Mailing List to Use:* See the
+      ":ref:`contributor-guide/submit-changes:finding a suitable mailing list`"
+      section above.
+
+#. *Make a Pull Request:* Notify the maintainer or the mailing list that
+   you have pushed a change by making a pull request.
+
+   The Yocto Project provides two scripts that conveniently let you
+   generate and send pull requests to the Yocto Project. These scripts
+   are ``create-pull-request`` and ``send-pull-request``. You can find
+   these scripts in the ``scripts`` directory within the
+   :term:`Source Directory` (e.g.
+   ``poky/scripts``).
+
+   Using these scripts correctly formats the requests without
+   introducing any whitespace or HTML formatting. The maintainer that
+   receives your patches either directly or through the mailing list
+   needs to be able to save and apply them directly from your emails.
+   Using these scripts is the preferred method for sending patches.
+
+   First, create the pull request. For example, the following command
+   runs the script, specifies the upstream repository in the contrib
+   directory into which you pushed the change, and provides a subject
+   line in the created patch files::
+
+      $ poky/scripts/create-pull-request -u meta-intel-contrib -s "Updated Manual Section Reference in README"
+
+   Running this script forms ``*.patch`` files in a folder named
+   ``pull-``\ `PID` in the current directory. One of the patch files is a
+   cover letter.
+
+   Before running the ``send-pull-request`` script, you must edit the
+   cover letter patch to insert information about your change. After
+   editing the cover letter, send the pull request. For example, the
+   following command runs the script and specifies the patch directory
+   and email address. In this example, the email address is a mailing
+   list::
+
+      $ poky/scripts/send-pull-request -p ~/meta-intel/pull-10565 -t meta-intel@lists.yoctoproject.org
+
+   You need to follow the prompts as the script is interactive.
+
+   .. note::
+
+      For help on using these scripts, simply provide the ``-h``
+      argument as follows::
+
+              $ poky/scripts/create-pull-request -h
+              $ poky/scripts/send-pull-request -h
+
+Submitting Changes to Stable Release Branches
+=============================================
+
+The process for proposing changes to a Yocto Project stable branch differs
+from the steps described above. Changes to a stable branch must address
+identified bugs or CVEs and should be made carefully in order to avoid the
+risk of introducing new bugs or breaking backwards compatibility. Typically
+bug fixes must already be accepted into the master branch before they can be
+backported to a stable branch unless the bug in question does not affect the
+master branch or the fix on the master branch is unsuitable for backporting.
+
+The list of stable branches along with the status and maintainer for each
+branch can be obtained from the
+:yocto_wiki:`Releases wiki page </Releases>`.
+
+.. note::
+
+   Changes will not typically be accepted for branches which are marked as
+   End-Of-Life (EOL).
+
+With this in mind, the steps to submit a change for a stable branch are as
+follows:
+
+#. *Identify the bug or CVE to be fixed:* This information should be
+   collected so that it can be included in your submission.
+
+   See :ref:`dev-manual/vulnerabilities:checking for vulnerabilities`
+   for details about CVE tracking.
+
+#. *Check if the fix is already present in the master branch:* This will
+   result in the most straightforward path into the stable branch for the
+   fix.
+
+   #. *If the fix is present in the master branch --- submit a backport request
+      by email:* You should send an email to the relevant stable branch
+      maintainer and the mailing list with details of the bug or CVE to be
+      fixed, the commit hash on the master branch that fixes the issue and
+      the stable branches which you would like this fix to be backported to.
+
+   #. *If the fix is not present in the master branch --- submit the fix to the
+      master branch first:* This will ensure that the fix passes through the
+      project's usual patch review and test processes before being accepted.
+      It will also ensure that bugs are not left unresolved in the master
+      branch itself. Once the fix is accepted in the master branch a backport
+      request can be submitted as above.
+
+   #. *If the fix is unsuitable for the master branch --- submit a patch
+      directly for the stable branch:* This method should be considered as a
+      last resort. It is typically necessary when the master branch is using
+      a newer version of the software which includes an upstream fix for the
+      issue or when the issue has been fixed on the master branch in a way
+      that introduces backwards incompatible changes. In this case follow the
+      steps in ":ref:`contributor-guide/submit-changes:preparing changes for submission`"
+      and in the following sections but modify the subject header of your patch
+      email to include the name of the stable branch which you are
+      targetting. This can be done using the ``--subject-prefix`` argument to
+      ``git format-patch``, for example to submit a patch to the
+      "&DISTRO_NAME_NO_CAP_MINUS_ONE;" branch use::
+
+         git format-patch --subject-prefix='&DISTRO_NAME_NO_CAP_MINUS_ONE;][PATCH' ...
+
+Taking Patch Review into Account
+================================
+
+You may get feedback on your submitted patches from other community members
+or from the automated patchtest service. If issues are identified in your
+patches then it is usually necessary to address these before the patches are
+accepted into the project. In this case you should your commits according
+to the feedback and submit an updated version to the relevant mailing list.
+
+In any case, never fix reported issues by fixing them in new commits
+on the tip of your branch. Always come up with a new series of commits
+without the reported issues.
+
+.. note::
+
+   It is a good idea to send a copy to the reviewers who provided feedback
+   to the previous version of the patch. You can make sure this happens
+   by adding a ``CC`` tag to the commit description::
+
+      CC: William Shakespeare <bill@yoctoproject.org>
+
+A single patch can be amended using ``git commit --amend``, and multiple
+patches can be easily reworked and reordered through an interactive Git rebase::
+
+   git rebase -i <ref-branch>
+
+See `this tutorial <https://hackernoon.com/beginners-guide-to-interactive-rebasing-346a3f9c3a6d>`__
+for practical guidance about using Git interactive rebasing.
+
+You should also modify the ``[PATCH]`` tag in the email subject line when
+sending the revised patch to mark the new iteration as ``[PATCH v2]``,
+``[PATCH v3]``, etc as appropriate. This can be done by passing the ``-v``
+argument to ``git format-patch`` with a version number::
+
+   git format-patch -v2 <ref-branch>
+
+Lastly please ensure that you also test your revised changes. In particular
+please don't just edit the patch file written out by ``git format-patch`` and
+resend it.
+
+Tracking the Status of Patches
+==============================
+
+The Yocto Project uses a `Patchwork instance <https://patchwork.yoctoproject.org/>`__
+to track the status of patches submitted to the various mailing lists and to
+support automated patch testing. Each submitted patch is checked for common
+mistakes and deviations from the expected patch format and submitters are
+notified by ``patchtest`` if such mistakes are found. This process helps to
+reduce the burden of patch review on maintainers.
+
+.. note::
+
+   This system is imperfect and changes can sometimes get lost in the flow.
+   Asking about the status of a patch or change is reasonable if the change
+   has been idle for a while with no feedback.
+
+If your patches have not had any feedback in a few days, they may have already
+been merged. You can run ``git pull``  branch to check this. Note that many if
+not most layer maintainers do not send out acknowledgement emails when they
+accept patches. Alternatively, if there is no response or merge after a few days
+the patch may have been missed or the appropriate reviewers may not currently be
+around. It is then perfectly fine to reply to it yourself with a reminder asking
+for feedback.
+
+.. note::
+
+      Patch reviews for feature and recipe upgrade patches are likely be delayed
+      during a feature freeze because these types of patches aren't merged during
+      at that time --- you may have to wait until after the freeze is lifted.
+
+Maintainers also commonly use ``-next`` branches to test submissions prior to
+merging patches. Thus, you can get an idea of the status of a patch based on
+whether the patch has been merged into one of these branches. The commonly
+used testing branches for OpenEmbedded-Core are as follows:
+
+-  *openembedded-core "master-next" branch:* This branch is part of the
+   :oe_git:`openembedded-core </openembedded-core/>` repository and contains
+   proposed changes to the core metadata.
+
+-  *poky "master-next" branch:* This branch is part of the
+   :yocto_git:`poky </poky/>` repository and combines proposed
+   changes to BitBake, the core metadata and the poky distro.
+
+Similarly, stable branches maintained by the project may have corresponding
+``-next`` branches which collect proposed changes. For example,
+``&DISTRO_NAME_NO_CAP;-next`` and ``&DISTRO_NAME_NO_CAP_MINUS_ONE;-next``
+branches in both the "openembdedded-core" and "poky" repositories.
+
+Other layers may have similar testing branches but there is no formal
+requirement or standard for these so please check the documentation for the
+layers you are contributing to.
+

documentation/dev-manual/building.rst

@@ -32,6 +32,10 @@ build host running Linux.
       OpenEmbedded build system, see the
       :doc:`/brief-yoctoprojectqs/index` document.
 
+   -  You can also use the `Yocto Project BitBake
+      <https://marketplace.visualstudio.com/items?itemName=yocto-project.yocto-bitbake>`__
+      extension for Visual Studio Code to build images.
+
 The build process creates an entire Linux distribution from source and
 places it in your :term:`Build Directory` under ``tmp/deploy/images``. For
 detailed information on the build process using BitBake, see the
@@ -156,7 +160,7 @@ Follow these steps to set up and execute multiple configuration builds:
    The location for these multiconfig configuration files is specific.
    They must reside in the current :term:`Build Directory` in a sub-directory of
    ``conf`` named ``multiconfig`` or within a layer's ``conf`` directory
-   under a directory named ``multiconfig``. Following is an example that defines
+   under a directory named ``multiconfig``. Here is an example that defines
    two configuration files for the "x86" and "arm" multiconfigs:
 
    .. image:: figures/multiconfig_files.png
@@ -273,12 +277,12 @@ loading modules needed to locate and mount the final root filesystem.
 
 Follow these steps to create an :term:`Initramfs` image:
 
-#. *Create the :term:`Initramfs` Image Recipe:* You can reference the
+#. *Create the Initramfs Image Recipe:* You can reference the
    ``core-image-minimal-initramfs.bb`` recipe found in the
    ``meta/recipes-core`` directory of the :term:`Source Directory`
    as an example from which to work.
 
-#. *Decide if You Need to Bundle the :term:`Initramfs` Image Into the Kernel
+#. *Decide if You Need to Bundle the Initramfs Image Into the Kernel
    Image:* If you want the :term:`Initramfs` image that is built to be bundled
    in with the kernel image, set the :term:`INITRAMFS_IMAGE_BUNDLE`
    variable to ``"1"`` in your ``local.conf`` configuration file and set the
@@ -771,10 +775,9 @@ your tunings to best consider build times and package feed maintenance.
       in the script for information on how to use the tool.
 
    -  *BitBake's "-S printdiff" Option:* Using this option causes
-      BitBake to try to establish the closest signature match it can
-      (e.g. in the shared state cache) and then run ``bitbake-diffsigs``
-      over the matches to determine the stamps and delta where these two
-      stamp trees diverge.
+      BitBake to try to establish the most recent signature match
+      (e.g. in the shared state cache) and then compare matched signatures
+      to determine the stamps and delta where these two stamp trees diverge.
 
 Building Software from an External Source
 =========================================