build-appliance-image: Update to master head revision

(From OE-Core rev: c4fc226c2e3856b942bb4f57ead21a64c3dc8c0d)

Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>

.gitignore

@@ -30,4 +30,5 @@ hob-image-*.bb
 pull-*/
 bitbake/lib/toaster/contrib/tts/backlog.txt
 bitbake/lib/toaster/contrib/tts/log/*
-bitbake/lib/toaster/contrib/tts/.cache/*
+bitbake/lib/toaster/contrib/tts/.cache/*
+bitbake/lib/bb/tests/runqueue-tests/bitbake-cookerdaemon.log

README.hardware

@@ -1 +0,0 @@
-meta-yocto-bsp/README.hardware

README.hardware.md

@@ -0,0 +1 @@
+meta-yocto-bsp/README.hardware.md

README.md

@@ -0,0 +1 @@
+README.poky.md

README.poky

@@ -1 +0,0 @@
-meta-poky/README.poky

README.poky.md

@@ -0,0 +1 @@
+meta-poky/README.poky.md

bitbake/README

@@ -11,7 +11,7 @@ For information about Bitbake, see the OpenEmbedded website:
 
 Bitbake plain documentation can be found under the doc directory or its integrated
 html version at the Yocto Project website:
-    http://yoctoproject.org/documentation
+    https://docs.yoctoproject.org
 
 Contributing
 ------------

bitbake/bin/bitbake

@@ -26,7 +26,7 @@ from bb.main import bitbake_main, BitBakeConfigParameters, BBMainException
 if sys.getfilesystemencoding() != "utf-8":
     sys.exit("Please use a locale setting which supports UTF-8 (such as LANG=en_US.UTF-8).\nPython can't change the filesystem locale after loading so we need a UTF-8 when Python starts or things won't work.")
 
-__version__ = "1.48.0"
+__version__ = "1.51.0"
 
 if __name__ == "__main__":
     if __version__ != bb.__version__:

bitbake/bin/bitbake-getvar

@@ -0,0 +1,48 @@
+#! /usr/bin/env python3
+#
+# Copyright (C) 2021 Richard Purdie
+#
+# SPDX-License-Identifier: GPL-2.0-only
+#
+
+import argparse
+import io
+import os
+import sys
+
+bindir = os.path.dirname(__file__)
+topdir = os.path.dirname(bindir)
+sys.path[0:0] = [os.path.join(topdir, 'lib')]
+
+import bb.tinfoil
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser(description="Bitbake Query Variable")
+    parser.add_argument("variable", help="variable name to query")
+    parser.add_argument("-r", "--recipe", help="Recipe name to query", default=None, required=False)
+    parser.add_argument('-u', '--unexpand', help='Do not expand the value (with --value)', action="store_true")
+    parser.add_argument('-f', '--flag', help='Specify a variable flag to query (with --value)', default=None)
+    parser.add_argument('--value', help='Only report the value, no history and no variable name', action="store_true")
+    args = parser.parse_args()
+
+    if args.unexpand and not args.value:
+        print("--unexpand only makes sense with --value")
+        sys.exit(1)
+
+    if args.flag and not args.value:
+        print("--flag only makes sense with --value")
+        sys.exit(1)
+
+    with bb.tinfoil.Tinfoil(tracking=True) as tinfoil:
+        if args.recipe:
+            tinfoil.prepare(quiet=2)
+            d = tinfoil.parse_recipe(args.recipe)
+        else:
+            tinfoil.prepare(quiet=2, config_only=True)
+            d = tinfoil.config_data
+        if args.flag:
+            print(str(d.getVarFlag(args.variable, args.flag, expand=(not args.unexpand))))
+        elif args.value:
+            print(str(d.getVar(args.variable, expand=(not args.unexpand))))
+        else:
+            bb.data.emit_var(args.variable, d=d, all=True)

bitbake/bin/bitbake-hashclient

@@ -151,9 +151,6 @@ def main():
     func = getattr(args, 'func', None)
     if func:
         client = hashserv.create_client(args.address)
-        # Try to establish a connection to the server now to detect failures
-        # early
-        client.connect()
 
         return func(args, client)
 

bitbake/bin/bitbake-hashserv

@@ -30,9 +30,11 @@ def main():
                                                "--bind [::1]:8686"'''
                                      )
 
-    parser.add_argument('--bind', default=DEFAULT_BIND, help='Bind address (default "%(default)s")')
-    parser.add_argument('--database', default='./hashserv.db', help='Database file (default "%(default)s")')
-    parser.add_argument('--log', default='WARNING', help='Set logging level')
+    parser.add_argument('-b', '--bind', default=DEFAULT_BIND, help='Bind address (default "%(default)s")')
+    parser.add_argument('-d', '--database', default='./hashserv.db', help='Database file (default "%(default)s")')
+    parser.add_argument('-l', '--log', default='WARNING', help='Set logging level')
+    parser.add_argument('-u', '--upstream', help='Upstream hashserv to pull hashes from')
+    parser.add_argument('-r', '--read-only', action='store_true', help='Disallow write operations from clients')
 
     args = parser.parse_args()
 
@@ -47,7 +49,7 @@ def main():
     console.setLevel(level)
     logger.addHandler(console)
 
-    server = hashserv.create_server(args.bind, args.database)
+    server = hashserv.create_server(args.bind, args.database, upstream=args.upstream, read_only=args.read_only)
     server.serve_forever()
     return 0
 

bitbake/bin/bitbake-selftest

@@ -29,6 +29,7 @@ tests = ["bb.tests.codeparser",
          "bb.tests.runqueue",
          "bb.tests.siggen",
          "bb.tests.utils",
+         "bb.tests.compression",
          "hashserv.tests",
          "layerindexlib.tests.layerindexobj",
          "layerindexlib.tests.restapi",

bitbake/bin/bitbake-server

@@ -26,12 +26,10 @@ readypipeinfd = int(sys.argv[3])
 logfile = sys.argv[4]
 lockname = sys.argv[5]
 sockname = sys.argv[6]
-timeout = sys.argv[7]
+timeout = float(sys.argv[7])
 xmlrpcinterface = (sys.argv[8], int(sys.argv[9]))
 if xmlrpcinterface[0] == "None":
     xmlrpcinterface = (None, xmlrpcinterface[1])
-if timeout == "None":
-    timeout = None
 
 # Replace standard fds with our own
 with open('/dev/null', 'r') as si:

bitbake/bin/bitbake-worker

@@ -118,7 +118,9 @@ def worker_child_fire(event, d):
     data = b"<event>" + pickle.dumps(event) + b"</event>"
     try:
         worker_pipe_lock.acquire()
-        worker_pipe.write(data)
+        while(len(data)):
+            written = worker_pipe.write(data)
+            data = data[written:]
         worker_pipe_lock.release()
     except IOError:
         sigterm_handler(None, None)
@@ -148,11 +150,15 @@ def fork_off_task(cfg, data, databuilder, workerdata, fn, task, taskname, taskha
 
     taskdep = workerdata["taskdeps"][fn]
     if 'umask' in taskdep and taskname in taskdep['umask']:
+        umask = taskdep['umask'][taskname]
+    elif workerdata["umask"]:
+        umask = workerdata["umask"]
+    if umask:
         # umask might come in as a number or text string..
         try:
-             umask = int(taskdep['umask'][taskname],8)
+             umask = int(umask, 8)
         except TypeError:
-             umask = taskdep['umask'][taskname]
+             pass
 
     dry_run = cfg.dry_run or dry_run_exec
 
@@ -167,7 +173,7 @@ def fork_off_task(cfg, data, databuilder, workerdata, fn, task, taskname, taskha
         fakedirs = (workerdata["fakerootdirs"][fn] or "").split()
         for p in fakedirs:
             bb.utils.mkdirhier(p)
-        logger.debug(2, 'Running %s:%s under fakeroot, fakedirs: %s' %
+        logger.debug2('Running %s:%s under fakeroot, fakedirs: %s' %
                         (fn, taskname, ', '.join(fakedirs)))
     else:
         envvars = (workerdata["fakerootnoenv"][fn] or "").split()
@@ -321,7 +327,9 @@ class runQueueWorkerPipe():
         end = len(self.queue)
         index = self.queue.find(b"</event>")
         while index != -1:
-            worker_fire_prepickled(self.queue[:index+8])
+            msg = self.queue[:index+8]
+            assert msg.startswith(b"<event>") and msg.count(b"<event>") == 1
+            worker_fire_prepickled(msg)
             self.queue = self.queue[index+8:]
             index = self.queue.find(b"</event>")
         return (end > start)

bitbake/contrib/hashserv/Dockerfile

@@ -0,0 +1,19 @@
+# SPDX-License-Identifier: MIT
+#
+# Copyright (c) 2021 Joshua Watt <JPEWhacker@gmail.com>
+# 
+# Dockerfile to build a bitbake hash equivalence server container
+#
+# From the root of the bitbake repository, run:
+#
+#   docker build -f contrib/hashserv/Dockerfile .
+#
+
+FROM alpine:3.13.1
+
+RUN apk add --no-cache python3
+
+COPY bin/bitbake-hashserv /opt/bbhashserv/bin/
+COPY lib/hashserv /opt/bbhashserv/lib/hashserv/
+
+ENTRYPOINT ["/opt/bbhashserv/bin/bitbake-hashserv"]

bitbake/doc/Makefile

@@ -3,7 +3,7 @@
 
 # You can set these variables from the command line, and also
 # from the environment for the first two.
-SPHINXOPTS    ?=
+SPHINXOPTS    ?= -j auto
 SPHINXBUILD   ?= sphinx-build
 SOURCEDIR     = .
 BUILDDIR      = _build

bitbake/doc/bitbake-user-manual/bitbake-user-manual-execution.rst

@@ -16,7 +16,7 @@ data, or simply return information about the execution environment.
 
 This chapter describes BitBake's execution process from start to finish
 when you use it to create an image. The execution process is launched
-using the following command form: ::
+using the following command form::
 
   $ bitbake target
 
@@ -32,7 +32,7 @@ the BitBake command and its options, see ":ref:`The BitBake Command
    your project's ``local.conf`` configuration file.
 
    A common method to determine this value for your build host is to run
-   the following: ::
+   the following::
 
      $ grep processor /proc/cpuinfo
 
@@ -139,7 +139,7 @@ in ``BBPATH`` in the same way as configuration files.
 
 A good way to get an idea of the configuration files and the class files
 used in your execution environment is to run the following BitBake
-command: ::
+command::
 
   $ bitbake -e > mybb.log
 
@@ -155,7 +155,7 @@ execution environment.
    pair of curly braces in a shell function, the closing curly brace
    must not be located at the start of the line without leading spaces.
 
-   Here is an example that causes BitBake to produce a parsing error: ::
+   Here is an example that causes BitBake to produce a parsing error::
 
       fakeroot create_shar() {
          cat << "EOF" > ${SDK_DEPLOY}/${TOOLCHAIN_OUTPUTNAME}.sh
@@ -185,7 +185,7 @@ During the configuration phase, BitBake will have set
 :term:`BBFILES`. BitBake now uses it to construct a
 list of recipes to parse, along with any append files (``.bbappend``) to
 apply. ``BBFILES`` is a space-separated list of available files and
-supports wildcards. An example would be: ::
+supports wildcards. An example would be::
 
   BBFILES = "/path/to/bbfiles/*.bb /path/to/appends/*.bbappend"
 
@@ -206,7 +206,7 @@ parses in order any append files found in ``BBFILES``.
 One common convention is to use the recipe filename to define pieces of
 metadata. For example, in ``bitbake.conf`` the recipe name and version
 are used to set the variables :term:`PN` and
-:term:`PV`: ::
+:term:`PV`::
 
    PN = "${@bb.parse.BBHandler.vars_from_file(d.getVar('FILE', False),d)[0] or 'defaultpkgname'}"
    PV = "${@bb.parse.BBHandler.vars_from_file(d.getVar('FILE', False),d)[1] or '1.0'}"
@@ -238,13 +238,14 @@ Recipe file collections exist to allow the user to have multiple
 repositories of ``.bb`` files that contain the same exact package. For
 example, one could easily use them to make one's own local copy of an
 upstream repository, but with custom modifications that one does not
-want upstream. Here is an example: ::
+want upstream. Here is an example::
 
   BBFILES = "/stuff/openembedded/*/*.bb /stuff/openembedded.modified/*/*.bb"
   BBFILE_COLLECTIONS = "upstream local"
   BBFILE_PATTERN_upstream = "^/stuff/openembedded/"
   BBFILE_PATTERN_local = "^/stuff/openembedded.modified/"
-  BBFILE_PRIORITY_upstream = "5" BBFILE_PRIORITY_local = "10"
+  BBFILE_PRIORITY_upstream = "5"
+  BBFILE_PRIORITY_local = "10"
 
 .. note::
 
@@ -269,7 +270,7 @@ variable, which is optional.
 When a recipe uses ``PROVIDES``, that recipe's functionality can be
 found under an alternative name or names other than the implicit ``PN``
 name. As an example, suppose a recipe named ``keyboard_1.0.bb``
-contained the following: ::
+contained the following::
 
   PROVIDES += "fullkeyboard"
 
@@ -290,7 +291,7 @@ needs to prioritize providers by determining provider preferences.
 A common example in which a target has multiple providers is
 "virtual/kernel", which is on the ``PROVIDES`` list for each kernel
 recipe. Each machine often selects the best kernel provider by using a
-line similar to the following in the machine configuration file: ::
+line similar to the following in the machine configuration file::
 
   PREFERRED_PROVIDER_virtual/kernel = "linux-yocto"
 
@@ -330,7 +331,7 @@ If the first recipe is named ``a_1.1.bb``, then the
 
 Thus, if a recipe named ``a_1.2.bb`` exists, BitBake will choose 1.2 by
 default. However, if you define the following variable in a ``.conf``
-file that BitBake parses, you can change that preference: ::
+file that BitBake parses, you can change that preference::
 
   PREFERRED_VERSION_a = "1.1"
 
@@ -497,7 +498,7 @@ to the task.
 
 Like the working directory case, situations exist where dependencies
 should be ignored. For these cases, you can instruct the build process
-to ignore a dependency by using a line like the following: ::
+to ignore a dependency by using a line like the following::
 
   PACKAGE_ARCHS[vardepsexclude] = "MACHINE"
 
@@ -507,7 +508,7 @@ even if it does reference it.
 
 Equally, there are cases where we need to add dependencies BitBake is
 not able to find. You can accomplish this by using a line like the
-following: ::
+following::
 
   PACKAGE_ARCHS[vardeps] = "MACHINE"
 
@@ -535,7 +536,7 @@ configuration file, we can give BitBake some extra information to help
 it construct the basehash. The following statement effectively results
 in a list of global variable dependency excludes - variables never
 included in any checksum. This example uses variables from OpenEmbedded
-to help illustrate the concept: ::
+to help illustrate the concept::
 
    BB_HASHBASE_WHITELIST ?= "TMPDIR FILE PATH PWD BB_TASKHASH BBPATH DL_DIR \
        SSTATE_DIR THISDIR FILESEXTRAPATHS FILE_DIRNAME HOME LOGNAME SHELL \
@@ -556,7 +557,7 @@ OpenEmbedded-Core uses: "OEBasic" and "OEBasicHash". By default, there
 is a dummy "noop" signature handler enabled in BitBake. This means that
 behavior is unchanged from previous versions. ``OE-Core`` uses the
 "OEBasicHash" signature handler by default through this setting in the
-``bitbake.conf`` file: ::
+``bitbake.conf`` file::
 
   BB_SIGNATURE_HANDLER ?= "OEBasicHash"
 
@@ -647,13 +648,6 @@ compiled binary. To handle this, BitBake calls the
 each successful setscene task to know whether or not it needs to obtain
 the dependencies of that task.
 
-Finally, after all the setscene tasks have executed, BitBake calls the
-function listed in
-:term:`BB_SETSCENE_VERIFY_FUNCTION2`
-with the list of tasks BitBake thinks has been "covered". The metadata
-can then ensure that this list is correct and can inform BitBake that it
-wants specific tasks to be run regardless of the setscene result.
-
 You can find more information on setscene metadata in the
 :ref:`bitbake-user-manual/bitbake-user-manual-metadata:task checksums and setscene`
 section.

bitbake/doc/bitbake-user-manual/bitbake-user-manual-fetching.rst

@@ -27,7 +27,7 @@ and unpacking the files is often optionally followed by patching.
 Patching, however, is not covered by this module.
 
 The code to execute the first part of this process, a fetch, looks
-something like the following: ::
+something like the following::
 
    src_uri = (d.getVar('SRC_URI') or "").split()
    fetcher = bb.fetch2.Fetch(src_uri, d)
@@ -37,7 +37,7 @@ This code sets up an instance of the fetch class. The instance uses a
 space-separated list of URLs from the :term:`SRC_URI`
 variable and then calls the ``download`` method to download the files.
 
-The instantiation of the fetch class is usually followed by: ::
+The instantiation of the fetch class is usually followed by::
 
    rootdir = l.getVar('WORKDIR')
    fetcher.unpack(rootdir)
@@ -72,7 +72,7 @@ URLs by looking for source files in a specific search order:
 For each URL passed to the fetcher, the fetcher calls the submodule that
 handles that particular URL type. This behavior can be the source of
 some confusion when you are providing URLs for the ``SRC_URI`` variable.
-Consider the following two URLs: ::
+Consider the following two URLs::
 
    http://git.yoctoproject.org/git/poky;protocol=git
    git://git.yoctoproject.org/git/poky;protocol=http
@@ -81,7 +81,7 @@ In the former case, the URL is passed to the ``wget`` fetcher, which does not
 understand "git". Therefore, the latter case is the correct form since the Git
 fetcher does know how to use HTTP as a transport.
 
-Here are some examples that show commonly used mirror definitions: ::
+Here are some examples that show commonly used mirror definitions::
 
    PREMIRRORS ?= "\
       bzr://.*/.\*  http://somemirror.org/sources/ \\n \
@@ -111,19 +111,19 @@ File integrity is of key importance for reproducing builds. For
 non-local archive downloads, the fetcher code can verify SHA-256 and MD5
 checksums to ensure the archives have been downloaded correctly. You can
 specify these checksums by using the ``SRC_URI`` variable with the
-appropriate varflags as follows: ::
+appropriate varflags as follows::
 
    SRC_URI[md5sum] = "value"
    SRC_URI[sha256sum] = "value"
 
 You can also specify the checksums as
-parameters on the ``SRC_URI`` as shown below: ::
+parameters on the ``SRC_URI`` as shown below::
 
   SRC_URI = "http://example.com/foobar.tar.bz2;md5sum=4a8e0f237e961fd7785d19d07fdb994d"
 
 If multiple URIs exist, you can specify the checksums either directly as
 in the previous example, or you can name the URLs. The following syntax
-shows how you name the URIs: ::
+shows how you name the URIs::
 
    SRC_URI = "http://example.com/foobar.tar.bz2;name=foo"
    SRC_URI[foo.md5sum] = 4a8e0f237e961fd7785d19d07fdb994d
@@ -163,9 +163,6 @@ govern the behavior of the unpack stage:
 -  *dos:* Applies to ``.zip`` and ``.jar`` files and specifies whether
    to use DOS line ending conversion on text files.
 
--  *basepath:* Instructs the unpack stage to strip the specified
-   directories from the source path when unpacking.
-
 -  *subdir:* Unpacks the specific URL to the specified subdirectory
    within the root directory.
 
@@ -204,7 +201,7 @@ time the ``download()`` method is called.
 If you specify a directory, the entire directory is unpacked.
 
 Here are a couple of example URLs, the first relative and the second
-absolute: ::
+absolute::
 
    SRC_URI = "file://relativefile.patch"
    SRC_URI = "file:///Users/ich/very_important_software"
@@ -225,7 +222,7 @@ downloaded file is useful for avoiding collisions in
 :term:`DL_DIR` when dealing with multiple files that
 have the same name.
 
-Some example URLs are as follows: ::
+Some example URLs are as follows::
 
    SRC_URI = "http://oe.handhelds.org/not_there.aac"
    SRC_URI = "ftp://oe.handhelds.org/not_there_as_well.aac"
@@ -235,15 +232,13 @@ Some example URLs are as follows: ::
 
    Because URL parameters are delimited by semi-colons, this can
    introduce ambiguity when parsing URLs that also contain semi-colons,
-   for example:
-   ::
+   for example::
 
            SRC_URI = "http://abc123.org/git/?p=gcc/gcc.git;a=snapshot;h=a5dd47"
 
 
    Such URLs should should be modified by replacing semi-colons with '&'
-   characters:
-   ::
+   characters::
 
            SRC_URI = "http://abc123.org/git/?p=gcc/gcc.git&a=snapshot&h=a5dd47"
 
@@ -251,8 +246,7 @@ Some example URLs are as follows: ::
    In most cases this should work. Treating semi-colons and '&' in
    queries identically is recommended by the World Wide Web Consortium
    (W3C). Note that due to the nature of the URL, you may have to
-   specify the name of the downloaded file as well:
-   ::
+   specify the name of the downloaded file as well::
 
            SRC_URI = "http://abc123.org/git/?p=gcc/gcc.git&a=snapshot&h=a5dd47;downloadfilename=myfile.bz2"
 
@@ -321,7 +315,7 @@ The supported parameters are as follows:
 
 -  *"port":* The port to which the CVS server connects.
 
-Some example URLs are as follows: ::
+Some example URLs are as follows::
 
    SRC_URI = "cvs://CVSROOT;module=mymodule;tag=some-version;method=ext"
    SRC_URI = "cvs://CVSROOT;module=mymodule;date=20060126;localdir=usethat"
@@ -363,7 +357,7 @@ The supported parameters are as follows:
    username is different than the username used in the main URL, which
    is passed to the subversion command.
 
-Following are three examples using svn: ::
+Following are three examples using svn::
 
    SRC_URI = "svn://myrepos/proj1;module=vip;protocol=http;rev=667"
    SRC_URI = "svn://myrepos/proj1;module=opie;protocol=svn+ssh"
@@ -436,11 +430,20 @@ This fetcher supports the following parameters:
    parameter implies no branch and only works when the transfer protocol
    is ``file://``.
 
-Here are some example URLs: ::
+Here are some example URLs::
 
    SRC_URI = "git://git.oe.handhelds.org/git/vip.git;tag=version-1"
    SRC_URI = "git://git.oe.handhelds.org/git/vip.git;protocol=http"
 
+.. note::
+
+   Specifying passwords directly in ``git://`` urls is not supported.
+   There are several reasons: ``SRC_URI`` is often written out to logs and
+   other places, and that could easily leak passwords; it is also all too
+   easy to share metadata without removing passwords. SSH keys, ``~/.netrc``
+   and ``~/.ssh/config`` files can be used as alternatives.
+
+
 .. _gitsm-fetcher:
 
 Git Submodule Fetcher (``gitsm://``)
@@ -475,7 +478,7 @@ repository.
 
 To use this fetcher, make sure your recipe has proper
 :term:`SRC_URI`, :term:`SRCREV`, and
-:term:`PV` settings. Here is an example: ::
+:term:`PV` settings. Here is an example::
 
    SRC_URI = "ccrc://cc.example.org/ccrc;vob=/example_vob;module=/example_module"
    SRCREV = "EXAMPLE_CLEARCASE_TAG"
@@ -497,7 +500,7 @@ Following are options for the ``SRC_URI`` statement:
       The module and vob options are combined to create the load rule in the
       view config spec. As an example, consider the vob and module values from
       the SRC_URI statement at the start of this section. Combining those values
-      results in the following: ::
+      results in the following::
 
          load /example_vob/example_module
 
@@ -549,7 +552,7 @@ the server's URL and port number, and you can specify a username and
 password directly in your recipe within ``SRC_URI``.
 
 Here is an example that relies on ``P4CONFIG`` to specify the server URL
-and port, username, and password, and fetches the Head Revision: ::
+and port, username, and password, and fetches the Head Revision::
 
    SRC_URI = "p4://example-depot/main/source/..."
    SRCREV = "${AUTOREV}"
@@ -557,7 +560,7 @@ and port, username, and password, and fetches the Head Revision: ::
    S = "${WORKDIR}/p4"
 
 Here is an example that specifies the server URL and port, username, and
-password, and fetches a Revision based on a Label: ::
+password, and fetches a Revision based on a Label::
 
    P4PORT = "tcp:p4server.example.net:1666"
    SRC_URI = "p4://user:passwd@example-depot/main/source/..."
@@ -583,7 +586,7 @@ paths locally is desirable, the fetcher supports two parameters:
     paths locally for the specified location, even in combination with the
     ``module`` parameter.
 
-Here is an example use of the the ``module`` parameter: ::
+Here is an example use of the the ``module`` parameter::
 
    SRC_URI = "p4://user:passwd@example-depot/main;module=source/..."
 
@@ -591,7 +594,7 @@ In this case, the content of the top-level directory ``source/`` will be fetched
 to ``${P4DIR}``, including the directory itself.  The top-level directory will
 be accesible at ``${P4DIR}/source/``.
 
-Here is an example use of the the ``remotepath`` parameter: ::
+Here is an example use of the the ``remotepath`` parameter::
 
    SRC_URI = "p4://user:passwd@example-depot/main;module=source/...;remotepath=keep"
 
@@ -619,11 +622,39 @@ This fetcher supports the following parameters:
 
 -  *"manifest":* Name of the manifest file (default: ``default.xml``).
 
-Here are some example URLs: ::
+Here are some example URLs::
 
    SRC_URI = "repo://REPOROOT;protocol=git;branch=some_branch;manifest=my_manifest.xml"
    SRC_URI = "repo://REPOROOT;protocol=file;branch=some_branch;manifest=my_manifest.xml"
 
+.. _az-fetcher:
+
+Az Fetcher (``az://``)
+--------------------------
+
+This submodule fetches data from an
+`Azure Storage account <https://docs.microsoft.com/en-us/azure/storage/>`__ ,
+it inherits its functionality from the HTTP wget fetcher, but modifies its
+behavior to accomodate the usage of a
+`Shared Access Signature (SAS) <https://docs.microsoft.com/en-us/azure/storage/common/storage-sas-overview>`__
+for non-public data.
+
+Such functionality is set by the variable:
+
+-  :term:`AZ_SAS`: The Azure Storage Shared Access Signature provides secure
+   delegate access to resources, if this variable is set, the Az Fetcher will
+   use it when fetching artifacts from the cloud.
+
+You can specify the AZ_SAS variable as shown below::
+
+   AZ_SAS = "se=2021-01-01&sp=r&sv=2018-11-09&sr=c&skoid=<skoid>&sig=<signature>"
+
+Here is an example URL::
+
+   SRC_URI = "az://<azure-storage-account>.blob.core.windows.net/<foo_container>/<bar_file>"
+
+It can also be used when setting mirrors definitions using the :term:`PREMIRRORS` variable.
+
 Other Fetchers
 --------------
 

bitbake/doc/bitbake-user-manual/bitbake-user-manual-hello.rst

@@ -20,7 +20,7 @@ Obtaining BitBake
 
 See the :ref:`bitbake-user-manual/bitbake-user-manual-hello:obtaining bitbake` section for
 information on how to obtain BitBake. Once you have the source code on
-your machine, the BitBake directory appears as follows: ::
+your machine, the BitBake directory appears as follows::
 
    $ ls -al
    total 100
@@ -49,7 +49,7 @@ Setting Up the BitBake Environment
 
 First, you need to be sure that you can run BitBake. Set your working
 directory to where your local BitBake files are and run the following
-command: ::
+command::
 
   $ ./bin/bitbake --version
   BitBake Build Tool Core version 1.23.0, bitbake version 1.23.0
@@ -61,14 +61,14 @@ The recommended method to run BitBake is from a directory of your
 choice. To be able to run BitBake from any directory, you need to add
 the executable binary to your binary to your shell's environment
 ``PATH`` variable. First, look at your current ``PATH`` variable by
-entering the following: ::
+entering the following::
 
   $ echo $PATH
 
 Next, add the directory location
 for the BitBake binary to the ``PATH``. Here is an example that adds the
 ``/home/scott-lenovo/bitbake/bin`` directory to the front of the
-``PATH`` variable: ::
+``PATH`` variable::
 
   $ export PATH=/home/scott-lenovo/bitbake/bin:$PATH
 
@@ -116,7 +116,7 @@ Following is the complete "Hello World" example.
 
 #.  **Create a Project Directory:** First, set up a directory for the
     "Hello World" project. Here is how you can do so in your home
-    directory: ::
+    directory::
 
       $ mkdir ~/hello
       $ cd ~/hello
@@ -127,7 +127,7 @@ Following is the complete "Hello World" example.
     directory is a good way to isolate your project.
 
 #.  **Run BitBake:** At this point, you have nothing but a project
-    directory. Run the ``bitbake`` command and see what it does: ::
+    directory. Run the ``bitbake`` command and see what it does::
 
        $ bitbake
        The BBPATH variable is not set and bitbake did not
@@ -161,7 +161,7 @@ Following is the complete "Hello World" example.
     ``BBPATH`` variable up in a configuration file for each project.
 
     From your shell, enter the following commands to set and export the
-    ``BBPATH`` variable: ::
+    ``BBPATH`` variable::
 
       $ BBPATH="projectdirectory"
       $ export BBPATH
@@ -176,7 +176,7 @@ Following is the complete "Hello World" example.
        shell would.
 
 #.  **Run BitBake:** Now that you have ``BBPATH`` defined, run the
-    ``bitbake`` command again: ::
+    ``bitbake`` command again::
 
        $ bitbake
        ERROR: Traceback (most recent call last):
@@ -208,13 +208,13 @@ Following is the complete "Hello World" example.
     http://git.openembedded.org/bitbake/tree/conf/bitbake.conf.
 
     Use the following commands to create the ``conf`` directory in the
-    project directory: ::
+    project directory::
 
       $ mkdir conf
 
     From within the ``conf`` directory,
     use some editor to create the ``bitbake.conf`` so that it contains
-    the following: ::
+    the following::
 
        PN  = "${@bb.parse.BBHandler.vars_from_file(d.getVar('FILE', False),d)[0] or 'defaultpkgname'}"
 
@@ -251,7 +251,7 @@ Following is the complete "Hello World" example.
     glossary.
 
 #.  **Run BitBake:** After making sure that the ``conf/bitbake.conf`` file
-    exists, you can run the ``bitbake`` command again: ::
+    exists, you can run the ``bitbake`` command again::
 
        $ bitbake
        ERROR: Traceback (most recent call last):
@@ -278,7 +278,7 @@ Following is the complete "Hello World" example.
     in the ``classes`` directory of the project (i.e ``hello/classes``
     in this example).
 
-    Create the ``classes`` directory as follows: ::
+    Create the ``classes`` directory as follows::
 
       $ cd $HOME/hello
       $ mkdir classes
@@ -291,7 +291,7 @@ Following is the complete "Hello World" example.
     environments BitBake is supporting.
 
 #.  **Run BitBake:** After making sure that the ``classes/base.bbclass``
-    file exists, you can run the ``bitbake`` command again: ::
+    file exists, you can run the ``bitbake`` command again::
 
        $ bitbake
        Nothing to do. Use 'bitbake world' to build everything, or run 'bitbake --help' for usage information.
@@ -314,7 +314,7 @@ Following is the complete "Hello World" example.
     Minimally, you need a recipe file and a layer configuration file in
     your layer. The configuration file needs to be in the ``conf``
     directory inside the layer. Use these commands to set up the layer
-    and the ``conf`` directory: ::
+    and the ``conf`` directory::
 
        $ cd $HOME
        $ mkdir mylayer
@@ -322,12 +322,12 @@ Following is the complete "Hello World" example.
        $ mkdir conf
 
     Move to the ``conf`` directory and create a ``layer.conf`` file that has the
-    following: ::
+    following::
 
       BBPATH .= ":${LAYERDIR}"
-      BBFILES += "${LAYERDIR}/\*.bb"
+      BBFILES += "${LAYERDIR}/*.bb"
       BBFILE_COLLECTIONS += "mylayer"
-     `BBFILE_PATTERN_mylayer := "^${LAYERDIR_RE}/"
+      BBFILE_PATTERN_mylayer := "^${LAYERDIR_RE}/"
 
     For information on these variables, click on :term:`BBFILES`,
     :term:`LAYERDIR`, :term:`BBFILE_COLLECTIONS` or :term:`BBFILE_PATTERN_mylayer <BBFILE_PATTERN>`
@@ -335,7 +335,7 @@ Following is the complete "Hello World" example.
 
     You need to create the recipe file next. Inside your layer at the
     top-level, use an editor and create a recipe file named
-    ``printhello.bb`` that has the following: ::
+    ``printhello.bb`` that has the following::
 
        DESCRIPTION = "Prints Hello World"
        PN = 'printhello'
@@ -356,7 +356,7 @@ Following is the complete "Hello World" example.
     follow the links to the glossary.
 
 #. **Run BitBake With a Target:** Now that a BitBake target exists, run
-    the command and provide that target: ::
+    the command and provide that target::
 
       $ cd $HOME/hello
       $ bitbake printhello
@@ -376,7 +376,7 @@ Following is the complete "Hello World" example.
     ``hello/conf`` for this example).
 
     Set your working directory to the ``hello/conf`` directory and then
-    create the ``bblayers.conf`` file so that it contains the following: ::
+    create the ``bblayers.conf`` file so that it contains the following::
 
        BBLAYERS ?= " \
            /home/<you>/mylayer \
@@ -386,7 +386,7 @@ Following is the complete "Hello World" example.
 
 #. **Run BitBake With a Target:** Now that you have supplied the
     ``bblayers.conf`` file, run the ``bitbake`` command and provide the
-    target: ::
+    target::
 
        $ bitbake printhello
        Parsing recipes: 100% |##################################################################################|

bitbake/doc/bitbake-user-manual/bitbake-user-manual-intro.rst

@@ -248,13 +248,13 @@ underlying, similarly-named recipe files.
 
 When you name an append file, you can use the "``%``" wildcard character
 to allow for matching recipe names. For example, suppose you have an
-append file named as follows: ::
+append file named as follows::
 
   busybox_1.21.%.bbappend
 
 That append file
 would match any ``busybox_1.21.``\ x\ ``.bb`` version of the recipe. So,
-the append file would match the following recipe names: ::
+the append file would match the following recipe names::
 
   busybox_1.21.1.bb
   busybox_1.21.2.bb
@@ -290,7 +290,7 @@ You can obtain BitBake several different ways:
    are using. The metadata is generally backwards compatible but not
    forward compatible.
 
-   Here is an example that clones the BitBake repository: ::
+   Here is an example that clones the BitBake repository::
 
      $ git clone git://git.openembedded.org/bitbake
 
@@ -298,7 +298,7 @@ You can obtain BitBake several different ways:
    Git repository into a directory called ``bitbake``. Alternatively,
    you can designate a directory after the ``git clone`` command if you
    want to call the new directory something other than ``bitbake``. Here
-   is an example that names the directory ``bbdev``: ::
+   is an example that names the directory ``bbdev``::
 
      $ git clone git://git.openembedded.org/bitbake bbdev
 
@@ -317,7 +317,7 @@ You can obtain BitBake several different ways:
          method for getting BitBake. Cloning the repository makes it easier
          to update as patches are added to the stable branches.
 
-   The following example downloads a snapshot of BitBake version 1.17.0: ::
+   The following example downloads a snapshot of BitBake version 1.17.0::
 
      $ wget http://git.openembedded.org/bitbake/snapshot/bitbake-1.17.0.tar.gz
      $ tar zxpvf bitbake-1.17.0.tar.gz
@@ -347,7 +347,7 @@ execution examples.
 Usage and syntax
 ----------------
 
-Following is the usage and syntax for BitBake: ::
+Following is the usage and syntax for BitBake::
 
    $ bitbake -h
    Usage: bitbake [options] [recipename/target recipe:do_task ...]
@@ -469,11 +469,11 @@ default task, which is "build". BitBake obeys inter-task dependencies
 when doing so.
 
 The following command runs the build task, which is the default task, on
-the ``foo_1.0.bb`` recipe file: ::
+the ``foo_1.0.bb`` recipe file::
 
   $ bitbake -b foo_1.0.bb
 
-The following command runs the clean task on the ``foo.bb`` recipe file: ::
+The following command runs the clean task on the ``foo.bb`` recipe file::
 
   $ bitbake -b foo.bb -c clean
 
@@ -497,13 +497,13 @@ functionality, or when there are multiple versions of a recipe.
 The ``bitbake`` command, when not using "--buildfile" or "-b" only
 accepts a "PROVIDES". You cannot provide anything else. By default, a
 recipe file generally "PROVIDES" its "packagename" as shown in the
-following example: ::
+following example::
 
   $ bitbake foo
 
 This next example "PROVIDES" the
 package name and also uses the "-c" option to tell BitBake to just
-execute the ``do_clean`` task: ::
+execute the ``do_clean`` task::
 
   $ bitbake -c clean foo
 
@@ -514,7 +514,7 @@ The BitBake command line supports specifying different tasks for
 individual targets when you specify multiple targets. For example,
 suppose you had two targets (or recipes) ``myfirstrecipe`` and
 ``mysecondrecipe`` and you needed BitBake to run ``taskA`` for the first
-recipe and ``taskB`` for the second recipe: ::
+recipe and ``taskB`` for the second recipe::
 
   $ bitbake myfirstrecipe:do_taskA mysecondrecipe:do_taskB
 
@@ -540,7 +540,7 @@ produce more readable graphs. This way, you can remove from the graph
 ``DEPENDS`` from inherited classes such as ``base.bbclass``.
 
 Here are two examples that create dependency graphs. The second example
-omits depends common in OpenEmbedded from the graph: ::
+omits depends common in OpenEmbedded from the graph::
 
   $ bitbake -g foo
 
@@ -582,17 +582,17 @@ accomplished by setting the
 configuration files for ``target1`` and ``target2`` defined in the build
 directory. The following statement in the ``local.conf`` file both
 enables BitBake to perform multiple configuration builds and specifies
-the two extra multiconfigs: ::
+the two extra multiconfigs::
 
   BBMULTICONFIG = "target1 target2"
 
 Once the target configuration files are in place and BitBake has been
 enabled to perform multiple configuration builds, use the following
-command form to start the builds: ::
+command form to start the builds::
 
   $ bitbake [mc:multiconfigname:]target [[[mc:multiconfigname:]target] ... ]
 
-Here is an example for two extra multiconfigs: ``target1`` and ``target2``: ::
+Here is an example for two extra multiconfigs: ``target1`` and ``target2``::
 
   $ bitbake mc::target mc:target1:target mc:target2:target
 
@@ -613,12 +613,12 @@ multiconfig.
 
 To enable dependencies in a multiple configuration build, you must
 declare the dependencies in the recipe using the following statement
-form: ::
+form::
 
   task_or_package[mcdepends] = "mc:from_multiconfig:to_multiconfig:recipe_name:task_on_which_to_depend"
 
 To better show how to use this statement, consider an example with two
-multiconfigs: ``target1`` and ``target2``: ::
+multiconfigs: ``target1`` and ``target2``::
 
   image_task[mcdepends] = "mc:target1:target2:image2:rootfs_task"
 
@@ -629,7 +629,7 @@ completion of the rootfs_task used to build out image2, which is
 associated with the "target2" multiconfig.
 
 Once you set up this dependency, you can build the "target1" multiconfig
-using a BitBake command as follows: ::
+using a BitBake command as follows::
 
   $ bitbake mc:target1:image1
 
@@ -639,7 +639,7 @@ the ``rootfs_task`` for the "target2" multiconfig build.
 
 Having a recipe depend on the root filesystem of another build might not
 seem that useful. Consider this change to the statement in the image1
-recipe: ::
+recipe::
 
   image_task[mcdepends] = "mc:target1:target2:image2:image_task"
 

bitbake/doc/bitbake-user-manual/bitbake-user-manual-metadata.rst

@@ -26,7 +26,7 @@ assignment. ::
    VARIABLE = "value"
 
 As expected, if you include leading or
-trailing spaces as part of an assignment, the spaces are retained: ::
+trailing spaces as part of an assignment, the spaces are retained::
 
    VARIABLE = " value"
    VARIABLE = "value "
@@ -40,7 +40,7 @@ blank space (i.e. these are not the same values). ::
 
 You can use single quotes instead of double quotes when setting a
 variable's value. Doing so allows you to use values that contain the
-double quote character: ::
+double quote character::
 
    VARIABLE = 'I have a " in my value'
 
@@ -77,7 +77,7 @@ occurs, you can use BitBake to check the actual value of the suspect
 variable. You can make these checks for both configuration and recipe
 level changes:
 
--  For configuration changes, use the following: ::
+-  For configuration changes, use the following::
 
       $ bitbake -e
 
@@ -91,7 +91,7 @@ level changes:
       Variables that are exported to the environment are preceded by the
       string "export" in the command's output.
 
--  For recipe changes, use the following: ::
+-  For recipe changes, use the following::
 
       $ bitbake recipe -e \| grep VARIABLE="
 
@@ -105,7 +105,7 @@ Outside of :ref:`functions <bitbake-user-manual/bitbake-user-manual-metadata:fun
 BitBake joins any line ending in
 a backslash character ("\") with the following line before parsing
 statements. The most common use for the "\" character is to split
-variable assignments over multiple lines, as in the following example: ::
+variable assignments over multiple lines, as in the following example::
 
    FOO = "bar \
           baz \
@@ -116,7 +116,7 @@ character that follow it are removed when joining lines. Thus, no
 newline characters end up in the value of ``FOO``.
 
 Consider this additional example where the two assignments both assign
-"barbaz" to ``FOO``: ::
+"barbaz" to ``FOO``::
 
    FOO = "barbaz"
    FOO = "bar\
@@ -149,7 +149,7 @@ The "=" operator does not immediately expand variable references in the
 right-hand side. Instead, expansion is deferred until the variable
 assigned to is actually used. The result depends on the current values
 of the referenced variables. The following example should clarify this
-behavior: ::
+behavior::
 
    A = "${B} baz"
    B = "${C} bar"
@@ -177,7 +177,7 @@ Setting a default value (?=)
 You can use the "?=" operator to achieve a "softer" assignment for a
 variable. This type of assignment allows you to define a variable if it
 is undefined when the statement is parsed, but to leave the value alone
-if the variable has a value. Here is an example: ::
+if the variable has a value. Here is an example::
 
    A ?= "aval"
 
@@ -199,7 +199,7 @@ by using the "??=" operator. This assignment behaves identical to "?="
 except that the assignment is made at the end of the parsing process
 rather than immediately. Consequently, when multiple "??=" assignments
 exist, the last one is used. Also, any "=" or "?=" assignment will
-override the value set with "??=". Here is an example: ::
+override the value set with "??=". Here is an example::
 
    A ??= "somevalue"
    A ??= "someothervalue"
@@ -215,7 +215,7 @@ Immediate variable expansion (:=)
 ---------------------------------
 
 The ":=" operator results in a variable's contents being expanded
-immediately, rather than when the variable is actually used: ::
+immediately, rather than when the variable is actually used::
 
    T = "123"
    A := "test ${T}"
@@ -241,7 +241,7 @@ the "+=" and "=+" operators. These operators insert a space between the
 current value and prepended or appended value.
 
 These operators take immediate effect during parsing. Here are some
-examples: ::
+examples::
 
    B = "bval"
    B += "additionaldata"
@@ -260,7 +260,7 @@ If you want to append or prepend values without an inserted space, use
 the ".=" and "=." operators.
 
 These operators take immediate effect during parsing. Here are some
-examples: ::
+examples::
 
    B = "bval"
    B .= "additionaldata"
@@ -278,7 +278,7 @@ style syntax. When you use this syntax, no spaces are inserted.
 
 These operators differ from the ":=", ".=", "=.", "+=", and "=+"
 operators in that their effects are applied at variable expansion time
-rather than being immediately applied. Here are some examples: ::
+rather than being immediately applied. Here are some examples::
 
    B = "bval"
    B_append = " additional data"
@@ -309,7 +309,7 @@ syntax. Specifying a value for removal causes all occurrences of that
 value to be removed from the variable.
 
 When you use this syntax, BitBake expects one or more strings.
-Surrounding spaces and spacing are preserved. Here is an example: ::
+Surrounding spaces and spacing are preserved. Here is an example::
 
    FOO = "123 456 789 123456 123 456 123 456"
    FOO_remove = "123"
@@ -334,27 +334,27 @@ An advantage of the override style operations "_append", "_prepend", and
 "_remove" as compared to the "+=" and "=+" operators is that the
 override style operators provide guaranteed operations. For example,
 consider a class ``foo.bbclass`` that needs to add the value "val" to
-the variable ``FOO``, and a recipe that uses ``foo.bbclass`` as follows: ::
+the variable ``FOO``, and a recipe that uses ``foo.bbclass`` as follows::
 
    inherit foo
    FOO = "initial"
 
 If ``foo.bbclass`` uses the "+=" operator,
 as follows, then the final value of ``FOO`` will be "initial", which is
-not what is desired: ::
+not what is desired::
 
    FOO += "val"
 
 If, on the other hand, ``foo.bbclass``
 uses the "_append" operator, then the final value of ``FOO`` will be
-"initial val", as intended: ::
+"initial val", as intended::
 
    FOO_append = " val"
 
 .. note::
 
    It is never necessary to use "+=" together with "_append". The following
-   sequence of assignments appends "barbaz" to FOO: ::
+   sequence of assignments appends "barbaz" to FOO::
 
        FOO_append = "bar"
        FOO_append = "baz"
@@ -381,7 +381,7 @@ standard syntax operations previously mentioned work for variable flags
 except for override style syntax (i.e. "_prepend", "_append", and
 "_remove").
 
-Here are some examples showing how to set variable flags: ::
+Here are some examples showing how to set variable flags::
 
    FOO[a] = "abc"
    FOO[b] = "123"
@@ -393,7 +393,7 @@ respectively. The ``[a]`` flag becomes "abc 456".
 
 No need exists to pre-define variable flags. You can simply start using
 them. One extremely common application is to attach some brief
-documentation to a BitBake variable as follows: ::
+documentation to a BitBake variable as follows::
 
    CACHE[doc] = "The directory holding the cache of the metadata."
 
@@ -401,7 +401,7 @@ Inline Python Variable Expansion
 --------------------------------
 
 You can use inline Python variable expansion to set variables. Here is
-an example: ::
+an example::
 
    DATE = "${@time.strftime('%Y%m%d',time.gmtime())}"
 
@@ -410,7 +410,7 @@ This example results in the ``DATE`` variable being set to the current date.
 Probably the most common use of this feature is to extract the value of
 variables from BitBake's internal data dictionary, ``d``. The following
 lines select the values of a package name and its version number,
-respectively: ::
+respectively::
 
    PN = "${@bb.parse.BBHandler.vars_from_file(d.getVar('FILE', False),d)[0] or 'defaultpkgname'}"
    PV = "${@bb.parse.BBHandler.vars_from_file(d.getVar('FILE', False),d)[1] or '1.0'}"
@@ -419,12 +419,12 @@ respectively: ::
 
    Inline Python expressions work just like variable expansions insofar as the
    "=" and ":=" operators are concerned. Given the following assignment, foo()
-   is called each time FOO is expanded: ::
+   is called each time FOO is expanded::
 
       FOO = "${@foo()}"
 
    Contrast this with the following immediate assignment, where foo() is only
-   called once, while the assignment is parsed: ::
+   called once, while the assignment is parsed::
 
       FOO := "${@foo()}"
 
@@ -437,7 +437,7 @@ Unsetting variables
 
 It is possible to completely remove a variable or a variable flag from
 BitBake's internal data dictionary by using the "unset" keyword. Here is
-an example: ::
+an example::
 
    unset DATE
    unset do_fetch[noexec]
@@ -452,7 +452,7 @@ When specifying pathnames for use with BitBake, do not use the tilde
 cause BitBake to not recognize the path since BitBake does not expand
 this character in the same way a shell would.
 
-Instead, provide a fuller path as the following example illustrates: ::
+Instead, provide a fuller path as the following example illustrates::
 
    BBLAYERS ?= " \
        /home/scott-lenovo/LayerA \
@@ -463,7 +463,7 @@ Exporting Variables to the Environment
 
 You can export variables to the environment of running tasks by using
 the ``export`` keyword. For example, in the following example, the
-``do_foo`` task prints "value from the environment" when run: ::
+``do_foo`` task prints "value from the environment" when run::
 
    export ENV_VARIABLE
    ENV_VARIABLE = "value from the environment"
@@ -481,7 +481,7 @@ It does not matter whether ``export ENV_VARIABLE`` appears before or
 after assignments to ``ENV_VARIABLE``.
 
 It is also possible to combine ``export`` with setting a value for the
-variable. Here is an example: ::
+variable. Here is an example::
 
    export ENV_VARIABLE = "variable-value"
 
@@ -518,7 +518,7 @@ variable.
    to satisfy conditions. Thus, if you have a variable that is
    conditional on "arm", and "arm" is in ``OVERRIDES``, then the
    "arm"-specific version of the variable is used rather than the
-   non-conditional version. Here is an example: ::
+   non-conditional version. Here is an example::
 
       OVERRIDES = "architecture:os:machine"
       TEST = "default"
@@ -535,7 +535,7 @@ variable.
    an OpenEmbedded metadata-based Linux kernel recipe file. The
    following lines from the recipe file first set the kernel branch
    variable ``KBRANCH`` to a default value, then conditionally override
-   that value based on the architecture of the build: ::
+   that value based on the architecture of the build::
 
       KBRANCH = "standard/base"
       KBRANCH_qemuarm = "standard/arm-versatile-926ejs"
@@ -547,7 +547,7 @@ variable.
 
 -  *Appending and Prepending:* BitBake also supports append and prepend
    operations to variable values based on whether a specific item is
-   listed in ``OVERRIDES``. Here is an example: ::
+   listed in ``OVERRIDES``. Here is an example::
 
       DEPENDS = "glibc ncurses"
       OVERRIDES = "machine:local"
@@ -557,14 +557,14 @@ variable.
 
    Again, using an OpenEmbedded metadata-based kernel recipe file as an
    example, the following lines will conditionally append to the
-   ``KERNEL_FEATURES`` variable based on the architecture: ::
+   ``KERNEL_FEATURES`` variable based on the architecture::
 
       KERNEL_FEATURES_append = " ${KERNEL_EXTRA_FEATURES}"
       KERNEL_FEATURES_append_qemux86=" cfg/sound.scc cfg/paravirt_kvm.scc"
       KERNEL_FEATURES_append_qemux86-64=" cfg/sound.scc cfg/paravirt_kvm.scc"
 
 -  *Setting a Variable for a Single Task:* BitBake supports setting a
-   variable just for the duration of a single task. Here is an example: ::
+   variable just for the duration of a single task. Here is an example::
 
       FOO_task-configure = "val 1"
       FOO_task-compile = "val 2"
@@ -580,7 +580,7 @@ variable.
    ``do_compile`` task.
 
    You can also use this syntax with other combinations (e.g.
-   "``_prepend``") as shown in the following example: ::
+   "``_prepend``") as shown in the following example::
 
       EXTRA_OEMAKE_prepend_task-compile = "${PARALLEL_MAKE} "
 
@@ -588,7 +588,7 @@ Key Expansion
 -------------
 
 Key expansion happens when the BitBake datastore is finalized. To better
-understand this, consider the following example: ::
+understand this, consider the following example::
 
    A${B} = "X"
    B = "2"
@@ -614,7 +614,7 @@ There is often confusion concerning the order in which overrides and
 various "append" operators take effect. Recall that an append or prepend
 operation using "_append" and "_prepend" does not result in an immediate
 assignment as would "+=", ".=", "=+", or "=.". Consider the following
-example: ::
+example::
 
    OVERRIDES = "foo"
    A = "Z"
@@ -631,7 +631,7 @@ Applying overrides, however, changes things. Since "foo" is listed in
 version, which is equal to "X". So effectively, ``A_foo`` replaces
 ``A``.
 
-This next example changes the order of the override and the append: ::
+This next example changes the order of the override and the append::
 
    OVERRIDES = "foo"
    A = "Z"
@@ -644,7 +644,7 @@ appended with "X". Consequently, ``A`` becomes "ZX". Notice that spaces
 are not appended.
 
 This next example has the order of the appends and overrides reversed
-back as in the first example: ::
+back as in the first example::
 
    OVERRIDES = "foo"
    A = "Y"
@@ -658,7 +658,7 @@ leaving the variable set to "ZX". Finally, applying the override for
 "foo" results in the conditional variable ``A`` becoming "ZX" (i.e.
 ``A`` is replaced with ``A_foo``).
 
-This final example mixes in some varying operators: ::
+This final example mixes in some varying operators::
 
    A = "1"
    A_append = "2"
@@ -720,7 +720,7 @@ file and then have your recipe inherit that class file.
 
 As an example, your recipes could use the following directive to inherit
 an ``autotools.bbclass`` file. The class file would contain common
-functionality for using Autotools that could be shared across recipes: ::
+functionality for using Autotools that could be shared across recipes::
 
    inherit autotools
 
@@ -734,7 +734,7 @@ In this case, BitBake would search for the directory
 
 If you want to use the directive to inherit multiple classes, separate
 them with spaces. The following example shows how to inherit both the
-``buildhistory`` and ``rm_work`` classes: ::
+``buildhistory`` and ``rm_work`` classes::
 
    inherit buildhistory rm_work
 
@@ -742,19 +742,19 @@ An advantage with the inherit directive as compared to both the
 :ref:`include <bitbake-user-manual/bitbake-user-manual-metadata:\`\`include\`\` directive>` and :ref:`require <bitbake-user-manual/bitbake-user-manual-metadata:\`\`require\`\` directive>`
 directives is that you can inherit class files conditionally. You can
 accomplish this by using a variable expression after the ``inherit``
-statement. Here is an example: ::
+statement. Here is an example::
 
    inherit ${VARNAME}
 
 If ``VARNAME`` is
 going to be set, it needs to be set before the ``inherit`` statement is
 parsed. One way to achieve a conditional inherit in this case is to use
-overrides: ::
+overrides::
 
    VARIABLE = ""
    VARIABLE_someoverride = "myclass"
 
-Another method is by using anonymous Python. Here is an example: ::
+Another method is by using anonymous Python. Here is an example::
 
    python () {
        if condition == value:
@@ -764,7 +764,7 @@ Another method is by using anonymous Python. Here is an example: ::
    }
 
 Alternatively, you could use an in-line Python expression in the
-following form: ::
+following form::
 
    inherit ${@'classname' if condition else ''}
    inherit ${@functionname(params)}
@@ -790,7 +790,7 @@ encapsulated functionality or configuration that does not suit a
 ``.bbclass`` file.
 
 As an example, suppose you needed a recipe to include some self-test
-definitions: ::
+definitions::
 
    include test_defs.inc
 
@@ -831,7 +831,7 @@ include file named ``foo.inc`` that contains the common definitions
 needed to build "foo". You need to be sure ``foo.inc`` is located in the
 same directory as your two recipe files as well. Once these conditions
 are set up, you can share the functionality using a ``require``
-directive from within each recipe: ::
+directive from within each recipe::
 
    require foo.inc
 
@@ -844,7 +844,7 @@ class. BitBake only supports this directive when used within a
 configuration file.
 
 As an example, suppose you needed to inherit a class file called
-``abc.bbclass`` from a configuration file as follows: ::
+``abc.bbclass`` from a configuration file as follows::
 
    INHERIT += "abc"
 
@@ -862,7 +862,7 @@ subdirectory in one of the directories specified in ``BBPATH``.
 If you want to use the directive to inherit multiple classes, you can
 provide them on the same line in the ``local.conf`` file. Use spaces to
 separate the classes. The following example shows how to inherit both
-the ``autotools`` and ``pkgconfig`` classes: ::
+the ``autotools`` and ``pkgconfig`` classes::
 
    INHERIT += "autotools pkgconfig"
 
@@ -895,7 +895,7 @@ Shell Functions
 
 Functions written in shell script and executed either directly as
 functions, tasks, or both. They can also be called by other shell
-functions. Here is an example shell function definition: ::
+functions. Here is an example shell function definition::
 
    some_function () {
        echo "Hello World"
@@ -912,7 +912,7 @@ can also be applied to shell functions. Most commonly, this application
 would be used in a ``.bbappend`` file to modify functions in the main
 recipe. It can also be used to modify functions inherited from classes.
 
-As an example, consider the following: ::
+As an example, consider the following::
 
    do_foo() {
        bbplain first
@@ -931,7 +931,7 @@ As an example, consider the following: ::
        bbplain fourth
    }
 
-Running ``do_foo`` prints the following: ::
+Running ``do_foo`` prints the following::
 
    recipename do_foo: first
    recipename do_foo: second
@@ -952,7 +952,7 @@ BitBake-Style Python Functions
 These functions are written in Python and executed by BitBake or other
 Python functions using ``bb.build.exec_func()``.
 
-An example BitBake function is: ::
+An example BitBake function is::
 
    python some_python_function () {
        d.setVar("TEXT", "Hello World")
@@ -975,7 +975,7 @@ import these modules. Also in these types of functions, the datastore
 Similar to shell functions, you can also apply overrides and
 override-style operators to BitBake-style Python functions.
 
-As an example, consider the following: ::
+As an example, consider the following::
 
    python do_foo_prepend() {
        bb.plain("first")
@@ -989,7 +989,7 @@ As an example, consider the following: ::
        bb.plain("third")
    }
 
-Running ``do_foo`` prints the following: ::
+Running ``do_foo`` prints the following::
 
    recipename do_foo: first
    recipename do_foo: second
@@ -1004,7 +1004,7 @@ Python Functions
 These functions are written in Python and are executed by other Python
 code. Examples of Python functions are utility functions that you intend
 to call from in-line Python or from within other Python functions. Here
-is an example: ::
+is an example::
 
    def get_depends(d):
        if d.getVar('SOMECONDITION'):
@@ -1056,7 +1056,7 @@ functions and regular Python functions defined with "def":
 -  Regular Python functions are called with the usual Python syntax.
    BitBake-style Python functions are usually tasks and are called
    directly by BitBake, but can also be called manually from Python code
-   by using the ``bb.build.exec_func()`` function. Here is an example: ::
+   by using the ``bb.build.exec_func()`` function. Here is an example::
 
       bb.build.exec_func("my_bitbake_style_function", d)
 
@@ -1094,7 +1094,7 @@ Sometimes it is useful to set variables or perform other operations
 programmatically during parsing. To do this, you can define special
 Python functions, called anonymous Python functions, that run at the end
 of parsing. For example, the following conditionally sets a variable
-based on the value of another variable: ::
+based on the value of another variable::
 
    python () {
        if d.getVar('SOMEVAR') == 'value':
@@ -1107,7 +1107,7 @@ the name "__anonymous", rather than no name.
 Anonymous Python functions always run at the end of parsing, regardless
 of where they are defined. If a recipe contains many anonymous
 functions, they run in the same order as they are defined within the
-recipe. As an example, consider the following snippet: ::
+recipe. As an example, consider the following snippet::
 
    python () {
        d.setVar('FOO', 'foo 2')
@@ -1122,7 +1122,7 @@ recipe. As an example, consider the following snippet: ::
    BAR = "bar 1"
 
 The previous example is conceptually
-equivalent to the following snippet: ::
+equivalent to the following snippet::
 
    FOO = "foo 1"
    BAR = "bar 1"
@@ -1136,7 +1136,7 @@ available to tasks, which always run after parsing.
 
 Overrides and override-style operators such as "``_append``" are applied
 before anonymous functions run. In the following example, ``FOO`` ends
-up with the value "foo from anonymous": ::
+up with the value "foo from anonymous"::
 
    FOO = "foo"
    FOO_append = " from outside"
@@ -1173,24 +1173,24 @@ version of the function.
 
 To make use of this technique, you need the following things in place:
 
--  The class needs to define the function as follows: ::
+-  The class needs to define the function as follows::
 
       classname_functionname
 
    For example, if you have a class file
    ``bar.bbclass`` and a function named ``do_foo``, the class must
-   define the function as follows: ::
+   define the function as follows::
 
       bar_do_foo
 
 -  The class needs to contain the ``EXPORT_FUNCTIONS`` statement as
-   follows: ::
+   follows::
 
       EXPORT_FUNCTIONS functionname
 
    For example, continuing with
    the same example, the statement in the ``bar.bbclass`` would be as
-   follows: ::
+   follows::
 
       EXPORT_FUNCTIONS do_foo
 
@@ -1199,7 +1199,7 @@ To make use of this technique, you need the following things in place:
    class version of the function, it should call ``bar_do_foo``.
    Assuming ``do_foo`` was a shell function and ``EXPORT_FUNCTIONS`` was
    used as above, the recipe's function could conditionally call the
-   class version of the function as follows: ::
+   class version of the function as follows::
 
       do_foo() {
           if [ somecondition ] ; then
@@ -1233,7 +1233,7 @@ Tasks are either :ref:`shell functions <bitbake-user-manual/bitbake-user-manual-
 that have been promoted to tasks by using the ``addtask`` command. The
 ``addtask`` command can also optionally describe dependencies between
 the task and other tasks. Here is an example that shows how to define a
-task and declare some dependencies: ::
+task and declare some dependencies::
 
    python do_printdate () {
        import time
@@ -1264,12 +1264,12 @@ Additionally, the ``do_printdate`` task becomes dependent upon the
    rerun for experimentation purposes, you can make BitBake always
    consider the task "out-of-date" by using the
    :ref:`[nostamp] <bitbake-user-manual/bitbake-user-manual-metadata:Variable Flags>`
-   variable flag, as follows: ::
+   variable flag, as follows::
 
       do_printdate[nostamp] = "1"
 
    You can also explicitly run the task and provide the
-   -f option as follows: ::
+   -f option as follows::
 
       $ bitbake recipe -c printdate -f
 
@@ -1278,7 +1278,7 @@ Additionally, the ``do_printdate`` task becomes dependent upon the
    name.
 
 You might wonder about the practical effects of using ``addtask``
-without specifying any dependencies as is done in the following example: ::
+without specifying any dependencies as is done in the following example::
 
    addtask printdate
 
@@ -1286,7 +1286,7 @@ In this example, assuming dependencies have not been
 added through some other means, the only way to run the task is by
 explicitly selecting it with ``bitbake`` recipe ``-c printdate``. You
 can use the ``do_listtasks`` task to list all tasks defined in a recipe
-as shown in the following example: ::
+as shown in the following example::
 
    $ bitbake recipe -c listtasks
 
@@ -1296,12 +1296,23 @@ For more information on task dependencies, see the
 See the ":ref:`bitbake-user-manual/bitbake-user-manual-metadata:variable flags`" section for information
 on variable flags you can use with tasks.
 
+.. note::
+
+   While it's infrequent, it's possible to define multiple tasks as
+   dependencies when calling ``addtask``. For example, here's a snippet
+   from the OpenEmbedded class file ``package_tar.bbclass``::
+
+     addtask package_write_tar before do_build after do_packagedata do_package
+
+   Note how the ``package_write_tar`` task has to wait until both of
+   ``do_packagedata`` and ``do_package`` complete.
+
 Deleting a Task
 ---------------
 
 As well as being able to add tasks, you can delete them. Simply use the
 ``deltask`` command to delete a task. For example, to delete the example
-task used in the previous sections, you would use: ::
+task used in the previous sections, you would use::
 
    deltask printdate
 
@@ -1317,7 +1328,7 @@ to run before ``do_a``.
 
 If you want dependencies such as these to remain intact, use the
 ``[noexec]`` varflag to disable the task instead of using the
-``deltask`` command to delete it: ::
+``deltask`` command to delete it::
 
    do_b[noexec] = "1"
 
@@ -1345,7 +1356,7 @@ environment, you must take these two steps:
    example, assume you want to prevent the build system from accessing
    your ``$HOME/.ccache`` directory. The following command "whitelists"
    the environment variable ``CCACHE_DIR`` causing BitBake to allow that
-   variable into the datastore: ::
+   variable into the datastore::
 
       export BB_ENV_EXTRAWHITE="$BB_ENV_EXTRAWHITE CCACHE_DIR"
 
@@ -1355,7 +1366,7 @@ environment, you must take these two steps:
    available in the datastore. To export it to the task environment of
    every running task, use a command similar to the following in your
    local configuration file ``local.conf`` or your distribution
-   configuration file: ::
+   configuration file::
 
       export CCACHE_DIR
 
@@ -1374,7 +1385,7 @@ environment into a special variable named :term:`BB_ORIGENV`.
 The ``BB_ORIGENV`` variable returns a datastore object that can be
 queried using the standard datastore operators such as
 ``getVar(, False)``. The datastore object is useful, for example, to
-find the original ``DISPLAY`` variable. Here is an example: ::
+find the original ``DISPLAY`` variable. Here is an example::
 
    origenv = d.getVar("BB_ORIGENV", False)
    bar = origenv.getVar("BAR", False)
@@ -1387,7 +1398,7 @@ Variable Flags
 
 Variable flags (varflags) help control a task's functionality and
 dependencies. BitBake reads and writes varflags to the datastore using
-the following command forms: ::
+the following command forms::
 
    variable = d.getVarFlags("variable")
    self.d.setVarFlags("FOO", {"func": True})
@@ -1526,7 +1537,7 @@ intent is to make it easy to do things like email notification on build
 failures.
 
 Following is an example event handler that prints the name of the event
-and the content of the ``FILE`` variable: ::
+and the content of the ``FILE`` variable::
 
    addhandler myclass_eventhandler
    python myclass_eventhandler() {
@@ -1569,7 +1580,7 @@ might have an interest in viewing:
    events when each of the workers parse the base configuration or if
    the server changes configuration and reparses. Any given datastore
    only has one such event executed against it, however. If
-   ```BB_INVALIDCONF`` <#>`__ is set in the datastore by the event
+   :term:`BB_INVALIDCONF` is set in the datastore by the event
    handler, the configuration is reparsed and a new event triggered,
    allowing the metadata to update configuration.
 
@@ -1639,10 +1650,15 @@ user interfaces:
 Variants - Class Extension Mechanism
 ====================================
 
-BitBake supports two features that facilitate creating from a single
-recipe file multiple incarnations of that recipe file where all
-incarnations are buildable. These features are enabled through the
-:term:`BBCLASSEXTEND` and :term:`BBVERSIONS` variables.
+BitBake supports multiple incarnations of a recipe file via the
+:term:`BBCLASSEXTEND` variable.
+
+The :term:`BBCLASSEXTEND` variable is a space separated list of classes used
+to "extend" the recipe for each variant. Here is an example that results in a
+second incarnation of the current recipe being available. This second
+incarnation will have the "native" class inherited. ::
+
+      BBCLASSEXTEND = "native"
 
 .. note::
 
@@ -1652,34 +1668,6 @@ incarnations are buildable. These features are enabled through the
    class. For specific examples, see the OE-Core native , nativesdk , and
    multilib classes.
 
--  ``BBCLASSEXTEND``: This variable is a space separated list of
-   classes used to "extend" the recipe for each variant. Here is an
-   example that results in a second incarnation of the current recipe
-   being available. This second incarnation will have the "native" class
-   inherited. ::
-
-      BBCLASSEXTEND = "native"
-
--  ``BBVERSIONS``: This variable allows a single recipe to build
-   multiple versions of a project from a single recipe file. You can
-   also specify conditional metadata (using the
-   :term:`OVERRIDES` mechanism) for a single
-   version, or an optionally named range of versions. Here is an
-   example: ::
-
-      BBVERSIONS = "1.0 2.0 git"
-      SRC_URI_git = "git://someurl/somepath.git"
-
-      BBVERSIONS = "1.0.[0-6]:1.0.0+ 1.0.[7-9]:1.0.7+"
-      SRC_URI_append_1.0.7+ = "file://some_patch_which_the_new_versions_need.patch;patch=1"
-
-   The name of the range defaults to the original version of the recipe. For
-   example, in OpenEmbedded, the recipe file ``foo_1.0.0+.bb`` creates a default
-   name range of ``1.0.0+``. This is useful because the range name is not only
-   placed into overrides, but it is also made available for the metadata to use
-   in the variable that defines the base recipe versions for use in ``file://``
-   search paths (:term:`FILESPATH`).
-
 Dependencies
 ============
 
@@ -1708,7 +1696,7 @@ Dependencies Internal to the ``.bb`` File
 BitBake uses the ``addtask`` directive to manage dependencies that are
 internal to a given recipe file. You can use the ``addtask`` directive
 to indicate when a task is dependent on other tasks or when other tasks
-depend on that recipe. Here is an example: ::
+depend on that recipe. Here is an example::
 
    addtask printdate after do_fetch before do_build
 
@@ -1732,7 +1720,7 @@ task depends on the completion of the ``do_printdate`` task.
 
    -  The directive ``addtask mytask after do_configure`` by itself
       never causes ``do_mytask`` to run. ``do_mytask`` can still be run
-      manually as follows: ::
+      manually as follows::
 
          $ bitbake recipe -c mytask
 
@@ -1746,7 +1734,7 @@ Build Dependencies
 BitBake uses the :term:`DEPENDS` variable to manage
 build time dependencies. The ``[deptask]`` varflag for tasks signifies
 the task of each item listed in ``DEPENDS`` that must complete before
-that task can be executed. Here is an example: ::
+that task can be executed. Here is an example::
 
    do_configure[deptask] = "do_populate_sysroot"
 
@@ -1788,7 +1776,7 @@ dependencies are discovered and added.
 
 The ``[recrdeptask]`` flag is most commonly used in high-level recipes
 that need to wait for some task to finish "globally". For example,
-``image.bbclass`` has the following: ::
+``image.bbclass`` has the following::
 
    do_rootfs[recrdeptask] += "do_packagedata"
 
@@ -1797,7 +1785,7 @@ the current recipe and all recipes reachable (by way of dependencies)
 from the image recipe must run before the ``do_rootfs`` task can run.
 
 BitBake allows a task to recursively depend on itself by
-referencing itself in the task list: ::
+referencing itself in the task list::
 
    do_a[recrdeptask] = "do_a do_b"
 
@@ -1814,7 +1802,7 @@ Inter-Task Dependencies
 BitBake uses the ``[depends]`` flag in a more generic form to manage
 inter-task dependencies. This more generic form allows for
 inter-dependency checks for specific tasks rather than checks for the
-data in ``DEPENDS``. Here is an example: ::
+data in ``DEPENDS``. Here is an example::
 
    do_patch[depends] = "quilt-native:do_populate_sysroot"
 
@@ -1909,7 +1897,7 @@ To help understand how BitBake does this, the section assumes an
 OpenEmbedded metadata-based example.
 
 These checksums are stored in :term:`STAMP`. You can
-examine the checksums using the following BitBake command: ::
+examine the checksums using the following BitBake command::
 
    $ bitbake-dumpsigs
 
@@ -1932,10 +1920,6 @@ The following list describes related variables:
    Specifies a function BitBake calls that determines whether BitBake
    requires a setscene dependency to be met.
 
--  :term:`BB_SETSCENE_VERIFY_FUNCTION2`:
-   Specifies a function to call that verifies the list of planned task
-   execution before the main task execution happens.
-
 -  :term:`BB_STAMP_POLICY`: Defines the mode
    for comparing timestamps of stamp files.
 

bitbake/doc/bitbake-user-manual/bitbake-user-manual-ref-variables.rst

@@ -23,10 +23,6 @@ overview of their function and contents.
       systems extend the functionality of the variable as it is
       described here in this glossary.
 
-   -  Finally, there are variables mentioned in this glossary that do
-      not appear in the BitBake glossary. These other variables are
-      variables used in systems that use BitBake.
-
 .. glossary::
 
    :term:`ASSUME_PROVIDED`
@@ -39,6 +35,19 @@ overview of their function and contents.
       when specified allows for the Git binary from the host to be used
       rather than building ``git-native``.
 
+   :term:`AZ_SAS`
+      Azure Storage Shared Access Signature, when using the
+      :ref:`Azure Storage fetcher <bitbake-user-manual/bitbake-user-manual-fetching:fetchers>`
+      This variable can be defined to be used by the fetcher to authenticate
+      and gain access to non-public artifacts.
+      ::
+
+         AZ_SAS = ""se=2021-01-01&sp=r&sv=2018-11-09&sr=c&skoid=<skoid>&sig=<signature>""
+
+      For more information see Microsoft's Azure Storage documentation at
+      https://docs.microsoft.com/en-us/azure/storage/common/storage-sas-overview
+
+
    :term:`B`
       The directory in which BitBake executes functions during a recipe's
       build process.
@@ -108,12 +117,16 @@ overview of their function and contents.
       command line option). The task name specified should not include the
       ``do_`` prefix.
 
+   :term:`BB_DEFAULT_UMASK`
+      The default umask to apply to tasks if specified and no task specific
+      umask flag is set.
+
    :term:`BB_DISKMON_DIRS`
       Monitors disk space and available inodes during the build and allows
       you to control the build based on these parameters.
 
       Disk space monitoring is disabled by default. When setting this
-      variable, use the following form: ::
+      variable, use the following form::
 
          BB_DISKMON_DIRS = "<action>,<dir>,<threshold> [...]"
 
@@ -149,7 +162,7 @@ overview of their function and contents.
                not specify G, M, or K, Kbytes is assumed by
                default.  Do not use GB, MB, or KB.
 
-      Here are some examples: ::
+      Here are some examples::
 
          BB_DISKMON_DIRS = "ABORT,${TMPDIR},1G,100K WARN,${SSTATE_DIR},1G,100K"
          BB_DISKMON_DIRS = "STOPTASKS,${TMPDIR},1G"
@@ -190,7 +203,7 @@ overview of their function and contents.
       BB_DISKMON_WARNINTERVAL = "50M,5K"
 
       When specifying the variable in your configuration file, use the
-      following form: ::
+      following form::
 
          BB_DISKMON_WARNINTERVAL = "<disk_space_interval>,<disk_inode_interval>"
 
@@ -206,7 +219,7 @@ overview of their function and contents.
                G, M, or K for Gbytes, Mbytes, or Kbytes,
                respectively. You cannot use GB, MB, or KB.
 
-      Here is an example: ::
+      Here is an example::
 
          BB_DISKMON_DIRS = "WARN,${SSTATE_DIR},1G,100K"
          BB_DISKMON_WARNINTERVAL = "50M,5K"
@@ -253,45 +266,6 @@ overview of their function and contents.
       ``my-recipe.bb`` is executing, the ``BB_FILENAME`` variable contains
       "/foo/path/my-recipe.bb".
 
-   :term:`BBFILES_DYNAMIC`
-      Activates content depending on presence of identified layers.  You
-      identify the layers by the collections that the layers define.
-
-      Use the ``BBFILES_DYNAMIC`` variable to avoid ``.bbappend`` files whose
-      corresponding ``.bb`` file is in a layer that attempts to modify other
-      layers through ``.bbappend`` but does not want to introduce a hard
-      dependency on those other layers.
-
-      Additionally you can prefix the rule with "!" to add ``.bbappend`` and
-      ``.bb`` files in case a layer is not present.  Use this avoid hard
-      dependency on those other layers.
-
-      Use the following form for ``BBFILES_DYNAMIC``: ::
-
-         collection_name:filename_pattern
-
-      The following example identifies two collection names and two filename
-      patterns: ::
-
-         BBFILES_DYNAMIC += "\
-             clang-layer:${LAYERDIR}/bbappends/meta-clang/*/*/*.bbappend \
-             core:${LAYERDIR}/bbappends/openembedded-core/meta/*/*/*.bbappend \
-         "
-
-      When the collection name is prefixed with "!" it will add the file pattern in case
-      the layer is absent: ::
-
-         BBFILES_DYNAMIC += "\
-             !clang-layer:${LAYERDIR}/backfill/meta-clang/*/*/*.bb \
-         "
-
-      This next example shows an error message that occurs because invalid
-      entries are found, which cause parsing to abort: ::
-
-         ERROR: BBFILES_DYNAMIC entries must be of the form {!}<collection name>:<filename pattern>, not:
-         /work/my-layer/bbappends/meta-security-isafw/*/*/*.bbappend
-         /work/my-layer/bbappends/openembedded-core/meta/*/*/*.bbappend
-
    :term:`BB_GENERATE_MIRROR_TARBALLS`
       Causes tarballs of the Git repositories, including the Git metadata,
       to be placed in the :term:`DL_DIR` directory. Anyone
@@ -335,6 +309,16 @@ overview of their function and contents.
       However, the more accurate the data returned, the more efficient the
       build will be.
 
+   :term:`BB_HASHSERVE`
+      Specifies the Hash Equivalence server to use.
+
+      If set to ``auto``, BitBake automatically starts its own server
+      over a UNIX domain socket.
+
+      If set to ``host:port``, BitBake will use a remote server on the
+      specified host. This allows multiple clients to share the same
+      hash equivalence data.
+
    :term:`BB_INVALIDCONF`
       Used in combination with the ``ConfigParsed`` event to trigger
       re-parsing the base metadata (i.e. all the recipes). The
@@ -351,7 +335,7 @@ overview of their function and contents.
       Specifies the name of the log files saved into
       ``${``\ :term:`T`\ ``}``. By default, the ``BB_LOGFMT``
       variable is undefined and the log file names get created using the
-      following form: ::
+      following form::
 
          log.{task}.{pid}
 
@@ -405,7 +389,7 @@ overview of their function and contents.
       Specifies the name of the executable script files (i.e. run files)
       saved into ``${``\ :term:`T`\ ``}``. By default, the
       ``BB_RUNFMT`` variable is undefined and the run file names get
-      created using the following form: ::
+      created using the following form::
 
          run.{task}.{pid}
 
@@ -448,17 +432,6 @@ overview of their function and contents.
       The function specified by this variable returns a "True" or "False"
       depending on whether the dependency needs to be met.
 
-   :term:`BB_SETSCENE_VERIFY_FUNCTION2`
-      Specifies a function to call that verifies the list of planned task
-      execution before the main task execution happens. The function is
-      called once BitBake has a list of setscene tasks that have run and
-      either succeeded or failed.
-
-      The function allows for a task list check to see if they make sense.
-      Even if BitBake was planning to skip a task, the returned value of
-      the function can force BitBake to run the task, which is necessary
-      under certain metadata defined circumstances.
-
    :term:`BB_SIGNATURE_EXCLUDE_FLAGS`
       Lists variable flags (varflags) that can be safely excluded from
       checksum and dependency data for keys in the datastore. When
@@ -533,7 +506,7 @@ overview of their function and contents.
          This variable works similarly to the :term:`BB_TASK_NICE_LEVEL`
          variable except with a task's I/O priorities.
 
-      Set the variable as follows: ::
+      Set the variable as follows::
 
          BB_TASK_IONICE_LEVEL = "class.prio"
 
@@ -551,7 +524,7 @@ overview of their function and contents.
          In order for your I/O priority settings to take effect, you need the
          Completely Fair Queuing (CFQ) Scheduler selected for the backing block
          device. To select the scheduler, use the following command form where
-         device is the device (e.g. sda, sdb, and so forth): ::
+         device is the device (e.g. sda, sdb, and so forth)::
 
             $ sudo sh -c "echo cfq > /sys/block/device/queu/scheduler"
 
@@ -592,7 +565,7 @@ overview of their function and contents.
       To build a different variant of the recipe with a minimal amount of
       code, it usually is as simple as adding the variable to your recipe.
       Here are two examples. The "native" variants are from the
-      OpenEmbedded-Core metadata: ::
+      OpenEmbedded-Core metadata::
 
          BBCLASSEXTEND =+ "native nativesdk"
          BBCLASSEXTEND =+ "multilib:multilib_name"
@@ -667,6 +640,45 @@ overview of their function and contents.
       For details on the syntax, see the documentation by following the
       previous link.
 
+   :term:`BBFILES_DYNAMIC`
+      Activates content depending on presence of identified layers.  You
+      identify the layers by the collections that the layers define.
+
+      Use the ``BBFILES_DYNAMIC`` variable to avoid ``.bbappend`` files whose
+      corresponding ``.bb`` file is in a layer that attempts to modify other
+      layers through ``.bbappend`` but does not want to introduce a hard
+      dependency on those other layers.
+
+      Additionally you can prefix the rule with "!" to add ``.bbappend`` and
+      ``.bb`` files in case a layer is not present.  Use this avoid hard
+      dependency on those other layers.
+
+      Use the following form for ``BBFILES_DYNAMIC``::
+
+         collection_name:filename_pattern
+
+      The following example identifies two collection names and two filename
+      patterns::
+
+         BBFILES_DYNAMIC += "\
+             clang-layer:${LAYERDIR}/bbappends/meta-clang/*/*/*.bbappend \
+             core:${LAYERDIR}/bbappends/openembedded-core/meta/*/*/*.bbappend \
+         "
+
+      When the collection name is prefixed with "!" it will add the file pattern in case
+      the layer is absent::
+
+         BBFILES_DYNAMIC += "\
+             !clang-layer:${LAYERDIR}/backfill/meta-clang/*/*/*.bb \
+         "
+
+      This next example shows an error message that occurs because invalid
+      entries are found, which cause parsing to abort::
+
+         ERROR: BBFILES_DYNAMIC entries must be of the form {!}<collection name>:<filename pattern>, not:
+         /work/my-layer/bbappends/meta-security-isafw/*/*/*.bbappend
+         /work/my-layer/bbappends/openembedded-core/meta/*/*/*.bbappend
+
    :term:`BBINCLUDED`
       Contains a space-separated list of all of all files that BitBake's
       parser included during parsing of the current file.
@@ -684,7 +696,7 @@ overview of their function and contents.
    :term:`BBLAYERS`
       Lists the layers to enable during the build. This variable is defined
       in the ``bblayers.conf`` configuration file in the build directory.
-      Here is an example: ::
+      Here is an example::
 
          BBLAYERS = " \
              /home/scottrif/poky/meta \
@@ -718,13 +730,13 @@ overview of their function and contents.
 
       The following example uses a complete regular expression to tell
       BitBake to ignore all recipe and recipe append files in the
-      ``meta-ti/recipes-misc/`` directory: ::
+      ``meta-ti/recipes-misc/`` directory::
 
          BBMASK = "meta-ti/recipes-misc/"
 
       If you want to mask out multiple directories or recipes, you can
       specify multiple regular expression fragments. This next example
-      masks out multiple directories and individual recipes: ::
+      masks out multiple directories and individual recipes::
 
          BBMASK += "/meta-ti/recipes-misc/ meta-ti/recipes-ti/packagegroup/"
          BBMASK += "/meta-oe/recipes-support/"
@@ -745,7 +757,7 @@ overview of their function and contents.
       ``conf/local.conf`` configuration file.
 
       As an example, the following line specifies three multiconfigs, each
-      having a separate configuration file: ::
+      having a separate configuration file::
 
          BBMULTIFONFIG = "configA configB configC"
 
@@ -766,7 +778,7 @@ overview of their function and contents.
       If you run BitBake from a directory outside of the build directory,
       you must be sure to set ``BBPATH`` to point to the build directory.
       Set the variable as you would any environment variable and then run
-      BitBake: ::
+      BitBake::
 
          $ BBPATH="build_directory"
          $ export BBPATH
@@ -780,16 +792,6 @@ overview of their function and contents.
       Allows you to use a configuration file to add to the list of
       command-line target recipes you want to build.
 
-   :term:`BBVERSIONS`
-      Allows a single recipe to build multiple versions of a project from a
-      single recipe file. You also able to specify conditional metadata
-      using the :term:`OVERRIDES` mechanism for a
-      single version or for an optionally named range of versions.
-
-      For more information on ``BBVERSIONS``, see the
-      ":ref:`bitbake-user-manual/bitbake-user-manual-metadata:variants - class extension mechanism`"
-      section.
-
    :term:`BITBAKE_UI`
       Used to specify the UI module to use when running BitBake. Using this
       variable is equivalent to using the ``-u`` command-line option.
@@ -835,7 +837,7 @@ overview of their function and contents.
 
       Consider this simple example for two recipes named "a" and "b" that
       produce similarly named packages. In this example, the ``DEPENDS``
-      statement appears in the "a" recipe: ::
+      statement appears in the "a" recipe::
 
          DEPENDS = "b"
 
@@ -1057,7 +1059,7 @@ overview of their function and contents.
       recipes provide the same item. You should always suffix the variable
       with the name of the provided item, and you should set it to the
       :term:`PN` of the recipe to which you want to give
-      precedence. Some examples: ::
+      precedence. Some examples::
 
          PREFERRED_PROVIDER_virtual/kernel ?= "linux-yocto"
          PREFERRED_PROVIDER_virtual/xserver = "xserver-xf86"
@@ -1069,18 +1071,18 @@ overview of their function and contents.
       ``PREFERRED_PROVIDERS`` is identical to
       :term:`PREFERRED_PROVIDER`. However, the ``PREFERRED_PROVIDERS`` variable
       lets you define preferences for multiple situations using the following
-      form: ::
+      form::
 
          PREFERRED_PROVIDERS = "xxx:yyy aaa:bbb ..."
 
-      This form is a convenient replacement for the following: ::
+      This form is a convenient replacement for the following::
 
          PREFERRED_PROVIDER_xxx = "yyy"
          PREFERRED_PROVIDER_aaa = "bbb"
 
    :term:`PREFERRED_VERSION`
-      If there are multiple versions of recipes available, this variable
-      determines which recipe should be given preference. You must always
+      If there are multiple versions of a recipe available, this variable
+      determines which version should be given preference. You must always
       suffix the variable with the :term:`PN` you want to
       select, and you should set :term:`PV` accordingly for
       precedence.
@@ -1089,7 +1091,7 @@ overview of their function and contents.
       through the "``%``" character. You can use the character to match any
       number of characters, which can be useful when specifying versions
       that contain long revision numbers that potentially change. Here are
-      two examples: ::
+      two examples::
 
          PREFERRED_VERSION_python = "2.7.3"
          PREFERRED_VERSION_linux-yocto = "4.12%"
@@ -1100,6 +1102,10 @@ overview of their function and contents.
          end of the string. You cannot use the wildcard character in any other
          location of the string.
 
+      If a recipe with the specified version is not available, a warning
+      message will be shown. See :term:`REQUIRED_VERSION` if you want this
+      to be an error instead.
+
    :term:`PREMIRRORS`
       Specifies additional paths from which BitBake gets source code. When
       the build system searches for source code, it first tries the local
@@ -1109,7 +1115,7 @@ overview of their function and contents.
 
       Typically, you would add a specific server for the build system to
       attempt before any others by adding something like the following to
-      your configuration: ::
+      your configuration::
 
          PREMIRRORS_prepend = "\
          git://.*/.* http://www.yoctoproject.org/sources/ \n \
@@ -1131,7 +1137,7 @@ overview of their function and contents.
       ``DEPENDS``.
 
       Consider the following example ``PROVIDES`` statement from a recipe
-      file ``libav_0.8.11.bb``: ::
+      file ``libav_0.8.11.bb``::
 
          PROVIDES += "libpostproc"
 
@@ -1154,7 +1160,7 @@ overview of their function and contents.
    :term:`PRSERV_HOST`
       The network based :term:`PR` service host and port.
 
-      Following is an example of how the ``PRSERV_HOST`` variable is set: ::
+      Following is an example of how the ``PRSERV_HOST`` variable is set::
 
          PRSERV_HOST = "localhost:0"
 
@@ -1175,7 +1181,7 @@ overview of their function and contents.
       you should always use the variable in a form with an attached package
       name. For example, suppose you are building a development package
       that depends on the ``perl`` package. In this case, you would use the
-      following ``RDEPENDS`` statement: ::
+      following ``RDEPENDS`` statement::
 
          RDEPENDS_${PN}-dev += "perl"
 
@@ -1186,11 +1192,11 @@ overview of their function and contents.
       BitBake supports specifying versioned dependencies. Although the
       syntax varies depending on the packaging format, BitBake hides these
       differences from you. Here is the general syntax to specify versions
-      with the ``RDEPENDS`` variable: ::
+      with the ``RDEPENDS`` variable::
 
          RDEPENDS_${PN} = "package (operator version)"
 
-      For ``operator``, you can specify the following: ::
+      For ``operator``, you can specify the following::
 
          =
          <
@@ -1199,7 +1205,7 @@ overview of their function and contents.
          >=
 
       For example, the following sets up a dependency on version 1.2 or
-      greater of the package ``foo``: ::
+      greater of the package ``foo``::
 
          RDEPENDS_${PN} = "foo (>= 1.2)"
 
@@ -1210,6 +1216,16 @@ overview of their function and contents.
       The directory in which a local copy of a ``google-repo`` directory is
       stored when it is synced.
 
+   :term:`REQUIRED_VERSION`
+      If there are multiple versions of a recipe available, this variable
+      determines which version should be given preference. ``REQUIRED_VERSION``
+      works in exactly the same manner as :term:`PREFERRED_VERSION`, except
+      that if the specified version is not available then an error message
+      is shown and the build fails immediately.
+
+      If both ``REQUIRED_VERSION`` and ``PREFERRED_VERSION`` are set for
+      the same recipe, the ``REQUIRED_VERSION`` value applies.
+
    :term:`RPROVIDES`
       A list of package name aliases that a package also provides. These
       aliases are useful for satisfying runtime dependencies of other
@@ -1218,7 +1234,7 @@ overview of their function and contents.
 
       As with all package-controlling variables, you must always use the
       variable in conjunction with a package name override. Here is an
-      example: ::
+      example::
 
          RPROVIDES_${PN} = "widget-abi-2"
 
@@ -1232,11 +1248,11 @@ overview of their function and contents.
       BitBake supports specifying versioned recommends. Although the syntax
       varies depending on the packaging format, BitBake hides these
       differences from you. Here is the general syntax to specify versions
-      with the ``RRECOMMENDS`` variable: ::
+      with the ``RRECOMMENDS`` variable::
 
          RRECOMMENDS_${PN} = "package (operator version)"
 
-      For ``operator``, you can specify the following: ::
+      For ``operator``, you can specify the following::
 
          =
          <
@@ -1245,7 +1261,7 @@ overview of their function and contents.
          >=
 
       For example, the following sets up a recommend on version
-      1.2 or greater of the package ``foo``: ::
+      1.2 or greater of the package ``foo``::
 
          RRECOMMENDS_${PN} = "foo (>= 1.2)"
 
@@ -1299,6 +1315,8 @@ overview of their function and contents.
       -  ``svn://`` : Fetches files from a Subversion (``svn``) revision
          control repository.
 
+      -  ``az://`` : Fetches files from an Azure Storage account using HTTPS.
+
       Here are some additional options worth mentioning:
 
       -  ``unpack`` : Controls whether or not to unpack the file if it is

bitbake/doc/conf.py

@@ -14,6 +14,7 @@
 # import sys
 # sys.path.insert(0, os.path.abspath('.'))
 
+import sys
 import datetime
 
 current_version = "dev"

bitbake/lib/bb/__init__.py

@@ -9,7 +9,7 @@
 # SPDX-License-Identifier: GPL-2.0-only
 #
 
-__version__ = "1.48.0"
+__version__ = "1.51.0"
 
 import sys
 if sys.version_info < (3, 5, 0):
@@ -21,8 +21,8 @@ class BBHandledException(Exception):
     The big dilemma for generic bitbake code is what information to give the user
     when an exception occurs. Any exception inheriting this base exception class
     has already provided information to the user via some 'fired' message type such as
-    an explicitly fired event using bb.fire, or a bb.error message. If bitbake 
-    encounters an exception derived from this class, no backtrace or other information 
+    an explicitly fired event using bb.fire, or a bb.error message. If bitbake
+    encounters an exception derived from this class, no backtrace or other information
     will be given to the user, its assumed the earlier event provided the relevant information.
     """
     pass
@@ -42,7 +42,16 @@ class BBLoggerMixin(object):
 
     def setup_bblogger(self, name):
         if name.split(".")[0] == "BitBake":
-            self.debug = self.bbdebug
+            self.debug = self._debug_helper
+
+    def _debug_helper(self, *args, **kwargs):
+        return self.bbdebug(1, *args, **kwargs)
+
+    def debug2(self, *args, **kwargs):
+        return self.bbdebug(2, *args, **kwargs)
+
+    def debug3(self, *args, **kwargs):
+        return self.bbdebug(3, *args, **kwargs)
 
     def bbdebug(self, level, msg, *args, **kwargs):
         loglevel = logging.DEBUG - level + 1
@@ -128,7 +137,7 @@ def debug(lvl, *args):
         mainlogger.warning("Passed invalid debug level '%s' to bb.debug", lvl)
         args = (lvl,) + args
         lvl = 1
-    mainlogger.debug(lvl, ''.join(args))
+    mainlogger.bbdebug(lvl, ''.join(args))
 
 def note(*args):
     mainlogger.info(''.join(args))

bitbake/lib/bb/asyncrpc/__init__.py

@@ -0,0 +1,31 @@
+#
+# SPDX-License-Identifier: GPL-2.0-only
+#
+
+import itertools
+import json
+
+# The Python async server defaults to a 64K receive buffer, so we hardcode our
+# maximum chunk size. It would be better if the client and server reported to
+# each other what the maximum chunk sizes were, but that will slow down the
+# connection setup with a round trip delay so I'd rather not do that unless it
+# is necessary
+DEFAULT_MAX_CHUNK = 32 * 1024
+
+
+def chunkify(msg, max_chunk):
+    if len(msg) < max_chunk - 1:
+        yield ''.join((msg, "\n"))
+    else:
+        yield ''.join((json.dumps({
+                'chunk-stream': None
+            }), "\n"))
+
+        args = [iter(msg)] * (max_chunk - 1)
+        for m in map(''.join, itertools.zip_longest(*args, fillvalue='')):
+            yield ''.join(itertools.chain(m, "\n"))
+        yield "\n"
+
+
+from .client import AsyncClient, Client
+from .serv import AsyncServer, AsyncServerConnection

bitbake/lib/bb/asyncrpc/client.py

@@ -0,0 +1,150 @@
+#
+# SPDX-License-Identifier: GPL-2.0-only
+#
+
+import abc
+import asyncio
+import json
+import os
+import socket
+from . import chunkify, DEFAULT_MAX_CHUNK
+
+
+class AsyncClient(object):
+    def __init__(self, proto_name, proto_version, logger):
+        self.reader = None
+        self.writer = None
+        self.max_chunk = DEFAULT_MAX_CHUNK
+        self.proto_name = proto_name
+        self.proto_version = proto_version
+        self.logger = logger
+
+    async def connect_tcp(self, address, port):
+        async def connect_sock():
+            return await asyncio.open_connection(address, port)
+
+        self._connect_sock = connect_sock
+
+    async def connect_unix(self, path):
+        async def connect_sock():
+            return await asyncio.open_unix_connection(path)
+
+        self._connect_sock = connect_sock
+
+    async def setup_connection(self):
+        s = '%s %s\n\n' % (self.proto_name, self.proto_version)
+        self.writer.write(s.encode("utf-8"))
+        await self.writer.drain()
+
+    async def connect(self):
+        if self.reader is None or self.writer is None:
+            (self.reader, self.writer) = await self._connect_sock()
+            await self.setup_connection()
+
+    async def close(self):
+        self.reader = None
+
+        if self.writer is not None:
+            self.writer.close()
+            self.writer = None
+
+    async def _send_wrapper(self, proc):
+        count = 0
+        while True:
+            try:
+                await self.connect()
+                return await proc()
+            except (
+                OSError,
+                ConnectionError,
+                json.JSONDecodeError,
+                UnicodeDecodeError,
+            ) as e:
+                self.logger.warning("Error talking to server: %s" % e)
+                if count >= 3:
+                    if not isinstance(e, ConnectionError):
+                        raise ConnectionError(str(e))
+                    raise e
+                await self.close()
+                count += 1
+
+    async def send_message(self, msg):
+        async def get_line():
+            line = await self.reader.readline()
+            if not line:
+                raise ConnectionError("Connection closed")
+
+            line = line.decode("utf-8")
+
+            if not line.endswith("\n"):
+                raise ConnectionError("Bad message %r" % msg)
+
+            return line
+
+        async def proc():
+            for c in chunkify(json.dumps(msg), self.max_chunk):
+                self.writer.write(c.encode("utf-8"))
+            await self.writer.drain()
+
+            l = await get_line()
+
+            m = json.loads(l)
+            if m and "chunk-stream" in m:
+                lines = []
+                while True:
+                    l = (await get_line()).rstrip("\n")
+                    if not l:
+                        break
+                    lines.append(l)
+
+                m = json.loads("".join(lines))
+
+            return m
+
+        return await self._send_wrapper(proc)
+
+    async def ping(self):
+        return await self.send_message(
+            {'ping': {}}
+        )
+
+
+class Client(object):
+    def __init__(self):
+        self.client = self._get_async_client()
+        self.loop = asyncio.new_event_loop()
+
+        self._add_methods('connect_tcp', 'close', 'ping')
+
+    @abc.abstractmethod
+    def _get_async_client(self):
+        pass
+
+    def _get_downcall_wrapper(self, downcall):
+        def wrapper(*args, **kwargs):
+            return self.loop.run_until_complete(downcall(*args, **kwargs))
+
+        return wrapper
+
+    def _add_methods(self, *methods):
+        for m in methods:
+            downcall = getattr(self.client, m)
+            setattr(self, m, self._get_downcall_wrapper(downcall))
+
+    def connect_unix(self, path):
+        # AF_UNIX has path length issues so chdir here to workaround
+        cwd = os.getcwd()
+        try:
+            os.chdir(os.path.dirname(path))
+            self.loop.run_until_complete(self.client.connect_unix(os.path.basename(path)))
+            self.loop.run_until_complete(self.client.connect())
+        finally:
+            os.chdir(cwd)
+
+    @property
+    def max_chunk(self):
+        return self.client.max_chunk
+
+    @max_chunk.setter
+    def max_chunk(self, value):
+        self.client.max_chunk = value

bitbake/lib/bb/asyncrpc/serv.py

@@ -0,0 +1,223 @@
+#
+# SPDX-License-Identifier: GPL-2.0-only
+#
+
+import abc
+import asyncio
+import json
+import os
+import signal
+import socket
+import sys
+from . import chunkify, DEFAULT_MAX_CHUNK
+
+
+class ClientError(Exception):
+    pass
+
+
+class ServerError(Exception):
+    pass
+
+
+class AsyncServerConnection(object):
+    def __init__(self, reader, writer, proto_name, logger):
+        self.reader = reader
+        self.writer = writer
+        self.proto_name = proto_name
+        self.max_chunk = DEFAULT_MAX_CHUNK
+        self.handlers = {
+            'chunk-stream': self.handle_chunk,
+            'ping': self.handle_ping,
+        }
+        self.logger = logger
+
+    async def process_requests(self):
+        try:
+            self.addr = self.writer.get_extra_info('peername')
+            self.logger.debug('Client %r connected' % (self.addr,))
+
+            # Read protocol and version
+            client_protocol = await self.reader.readline()
+            if client_protocol is None:
+                return
+
+            (client_proto_name, client_proto_version) = client_protocol.decode('utf-8').rstrip().split()
+            if client_proto_name != self.proto_name:
+                self.logger.debug('Rejecting invalid protocol %s' % (self.proto_name))
+                return
+
+            self.proto_version = tuple(int(v) for v in client_proto_version.split('.'))
+            if not self.validate_proto_version():
+                self.logger.debug('Rejecting invalid protocol version %s' % (client_proto_version))
+                return
+
+            # Read headers. Currently, no headers are implemented, so look for
+            # an empty line to signal the end of the headers
+            while True:
+                line = await self.reader.readline()
+                if line is None:
+                    return
+
+                line = line.decode('utf-8').rstrip()
+                if not line:
+                    break
+
+            # Handle messages
+            while True:
+                d = await self.read_message()
+                if d is None:
+                    break
+                await self.dispatch_message(d)
+                await self.writer.drain()
+        except ClientError as e:
+            self.logger.error(str(e))
+        finally:
+            self.writer.close()
+
+    async def dispatch_message(self, msg):
+        for k in self.handlers.keys():
+            if k in msg:
+                self.logger.debug('Handling %s' % k)
+                await self.handlers[k](msg[k])
+                return
+
+        raise ClientError("Unrecognized command %r" % msg)
+
+    def write_message(self, msg):
+        for c in chunkify(json.dumps(msg), self.max_chunk):
+            self.writer.write(c.encode('utf-8'))
+
+    async def read_message(self):
+        l = await self.reader.readline()
+        if not l:
+            return None
+
+        try:
+            message = l.decode('utf-8')
+
+            if not message.endswith('\n'):
+                return None
+
+            return json.loads(message)
+        except (json.JSONDecodeError, UnicodeDecodeError) as e:
+            self.logger.error('Bad message from client: %r' % message)
+            raise e
+
+    async def handle_chunk(self, request):
+        lines = []
+        try:
+            while True:
+                l = await self.reader.readline()
+                l = l.rstrip(b"\n").decode("utf-8")
+                if not l:
+                    break
+                lines.append(l)
+
+            msg = json.loads(''.join(lines))
+        except (json.JSONDecodeError, UnicodeDecodeError) as e:
+            self.logger.error('Bad message from client: %r' % lines)
+            raise e
+
+        if 'chunk-stream' in msg:
+            raise ClientError("Nested chunks are not allowed")
+
+        await self.dispatch_message(msg)
+
+    async def handle_ping(self, request):
+        response = {'alive': True}
+        self.write_message(response)
+
+
+class AsyncServer(object):
+    def __init__(self, logger, loop=None):
+        if loop is None:
+            self.loop = asyncio.new_event_loop()
+            self.close_loop = True
+        else:
+            self.loop = loop
+            self.close_loop = False
+
+        self._cleanup_socket = None
+        self.logger = logger
+
+    def start_tcp_server(self, host, port):
+        self.server = self.loop.run_until_complete(
+            asyncio.start_server(self.handle_client, host, port, loop=self.loop)
+        )
+
+        for s in self.server.sockets:
+            self.logger.debug('Listening on %r' % (s.getsockname(),))
+            # Newer python does this automatically. Do it manually here for
+            # maximum compatibility
+            s.setsockopt(socket.SOL_TCP, socket.TCP_NODELAY, 1)
+            s.setsockopt(socket.SOL_TCP, socket.TCP_QUICKACK, 1)
+
+        name = self.server.sockets[0].getsockname()
+        if self.server.sockets[0].family == socket.AF_INET6:
+            self.address = "[%s]:%d" % (name[0], name[1])
+        else:
+            self.address = "%s:%d" % (name[0], name[1])
+
+    def start_unix_server(self, path):
+        def cleanup():
+            os.unlink(path)
+
+        cwd = os.getcwd()
+        try:
+            # Work around path length limits in AF_UNIX
+            os.chdir(os.path.dirname(path))
+            self.server = self.loop.run_until_complete(
+                asyncio.start_unix_server(self.handle_client, os.path.basename(path), loop=self.loop)
+            )
+        finally:
+            os.chdir(cwd)
+
+        self.logger.debug('Listening on %r' % path)
+
+        self._cleanup_socket = cleanup
+        self.address = "unix://%s" % os.path.abspath(path)
+
+    @abc.abstractmethod
+    def accept_client(self, reader, writer):
+        pass
+
+    async def handle_client(self, reader, writer):
+        # writer.transport.set_write_buffer_limits(0)
+        try:
+            client = self.accept_client(reader, writer)
+            await client.process_requests()
+        except Exception as e:
+            import traceback
+            self.logger.error('Error from client: %s' % str(e), exc_info=True)
+            traceback.print_exc()
+            writer.close()
+        self.logger.debug('Client disconnected')
+
+    def run_loop_forever(self):
+        try:
+            self.loop.run_forever()
+        except KeyboardInterrupt:
+            pass
+
+    def signal_handler(self):
+        self.loop.stop()
+
+    def serve_forever(self):
+        asyncio.set_event_loop(self.loop)
+        try:
+            self.loop.add_signal_handler(signal.SIGTERM, self.signal_handler)
+
+            self.run_loop_forever()
+            self.server.close()
+
+            self.loop.run_until_complete(self.server.wait_closed())
+            self.logger.debug('Server shutting down')
+        finally:
+            if self.close_loop:
+                if sys.version_info >= (3, 6):
+                    self.loop.run_until_complete(self.loop.shutdown_asyncgens())
+                self.loop.close()
+
+            if self._cleanup_socket is not None:
+                self._cleanup_socket()

bitbake/lib/bb/build.py

@@ -583,7 +583,7 @@ def _exec_task(fn, task, d, quieterr):
         logger.error("No such task: %s" % task)
         return 1
 
-    logger.debug(1, "Executing task %s", task)
+    logger.debug("Executing task %s", task)
 
     localdata = _task_data(fn, task, d)
     tempdir = localdata.getVar('T')
@@ -596,7 +596,7 @@ def _exec_task(fn, task, d, quieterr):
         curnice = os.nice(0)
         nice = int(nice) - curnice
         newnice = os.nice(nice)
-        logger.debug(1, "Renice to %s " % newnice)
+        logger.debug("Renice to %s " % newnice)
     ionice = localdata.getVar("BB_TASK_IONICE_LEVEL")
     if ionice:
         try:
@@ -720,7 +720,7 @@ def _exec_task(fn, task, d, quieterr):
 
         logfile.close()
         if os.path.exists(logfn) and os.path.getsize(logfn) == 0:
-            logger.debug(2, "Zero size logfn %s, removing", logfn)
+            logger.debug2("Zero size logfn %s, removing", logfn)
             bb.utils.remove(logfn)
             bb.utils.remove(loglink)
     event.fire(TaskSucceeded(task, fn, logfn, localdata), localdata)
@@ -854,6 +854,23 @@ def make_stamp(task, d, file_name = None):
         file_name = d.getVar('BB_FILENAME')
         bb.parse.siggen.dump_sigtask(file_name, task, stampbase, True)
 
+def find_stale_stamps(task, d, file_name=None):
+    current = stamp_internal(task, d, file_name)
+    current2 = stamp_internal(task + "_setscene", d, file_name)
+    cleanmask = stamp_cleanmask_internal(task, d, file_name)
+    found = []
+    for mask in cleanmask:
+        for name in glob.glob(mask):
+            if "sigdata" in name or "sigbasedata" in name:
+                continue
+            if name.endswith('.taint'):
+                continue
+            if name == current or name == current2:
+                continue
+            logger.debug2("Stampfile %s does not match %s or %s" % (name, current, current2))
+            found.append(name)
+    return found
+
 def del_stamp(task, d, file_name = None):
     """
     Removes a stamp for a given task
@@ -910,6 +927,11 @@ def add_tasks(tasklist, d):
                 task_deps[name] = {}
             if name in flags:
                 deptask = d.expand(flags[name])
+                if name in ['noexec', 'fakeroot', 'nostamp']:
+                    if deptask != '1':
+                        bb.warn("In a future version of BitBake, setting the '{}' flag to something other than '1' "
+                                "will result in the flag not being set. See YP bug #13808.".format(name))
+
                 task_deps[name][task] = deptask
         getTask('mcdepends')
         getTask('depends')

bitbake/lib/bb/cache.py

@@ -26,7 +26,7 @@ import re
 
 logger = logging.getLogger("BitBake.Cache")
 
-__cache_version__ = "153"
+__cache_version__ = "154"
 
 def getCacheFile(path, filename, mc, data_hash):
     mcspec = ''
@@ -94,6 +94,7 @@ class CoreRecipeInfo(RecipeInfoCommon):
         if not self.packages:
             self.packages.append(self.pn)
         self.packages_dynamic = self.listvar('PACKAGES_DYNAMIC', metadata)
+        self.rprovides_pkg = self.pkgvar('RPROVIDES', self.packages, metadata)
 
         self.skipreason = self.getvar('__SKIPPED', metadata)
         if self.skipreason:
@@ -120,12 +121,12 @@ class CoreRecipeInfo(RecipeInfoCommon):
         self.depends          = self.depvar('DEPENDS', metadata)
         self.rdepends         = self.depvar('RDEPENDS', metadata)
         self.rrecommends      = self.depvar('RRECOMMENDS', metadata)
-        self.rprovides_pkg    = self.pkgvar('RPROVIDES', self.packages, metadata)
         self.rdepends_pkg     = self.pkgvar('RDEPENDS', self.packages, metadata)
         self.rrecommends_pkg  = self.pkgvar('RRECOMMENDS', self.packages, metadata)
         self.inherits         = self.getvar('__inherit_cache', metadata, expand=False)
         self.fakerootenv      = self.getvar('FAKEROOTENV', metadata)
         self.fakerootdirs     = self.getvar('FAKEROOTDIRS', metadata)
+        self.fakerootlogs     = self.getvar('FAKEROOTLOGS', metadata)
         self.fakerootnoenv    = self.getvar('FAKEROOTNOENV', metadata)
         self.extradepsfunc    = self.getvar('calculate_extra_depends', metadata)
 
@@ -163,6 +164,7 @@ class CoreRecipeInfo(RecipeInfoCommon):
         cachedata.fakerootenv = {}
         cachedata.fakerootnoenv = {}
         cachedata.fakerootdirs = {}
+        cachedata.fakerootlogs = {}
         cachedata.extradepsfunc = {}
 
     def add_cacheData(self, cachedata, fn):
@@ -215,7 +217,7 @@ class CoreRecipeInfo(RecipeInfoCommon):
         if not self.not_world:
             cachedata.possible_world.append(fn)
         #else:
-        #    logger.debug(2, "EXCLUDE FROM WORLD: %s", fn)
+        #    logger.debug2("EXCLUDE FROM WORLD: %s", fn)
 
         # create a collection of all targets for sanity checking
         # tasks, such as upstream versions, license, and tools for
@@ -231,6 +233,7 @@ class CoreRecipeInfo(RecipeInfoCommon):
         cachedata.fakerootenv[fn] = self.fakerootenv
         cachedata.fakerootnoenv[fn] = self.fakerootnoenv
         cachedata.fakerootdirs[fn] = self.fakerootdirs
+        cachedata.fakerootlogs[fn] = self.fakerootlogs
         cachedata.extradepsfunc[fn] = self.extradepsfunc
 
 def virtualfn2realfn(virtualfn):
@@ -238,7 +241,7 @@ def virtualfn2realfn(virtualfn):
     Convert a virtual file name to a real one + the associated subclass keyword
     """
     mc = ""
-    if virtualfn.startswith('mc:'):
+    if virtualfn.startswith('mc:') and virtualfn.count(':') >= 2:
         elems = virtualfn.split(':')
         mc = elems[1]
         virtualfn = ":".join(elems[2:])
@@ -268,7 +271,7 @@ def variant2virtual(realfn, variant):
     """
     if variant == "":
         return realfn
-    if variant.startswith("mc:"):
+    if variant.startswith("mc:") and variant.count(':') >= 2:
         elems = variant.split(":")
         if elems[2]:
             return "mc:" + elems[1] + ":virtual:" + ":".join(elems[2:]) + ":" + realfn
@@ -323,7 +326,7 @@ class NoCache(object):
         Return a complete set of data for fn.
         To do this, we need to parse the file.
         """
-        logger.debug(1, "Parsing %s (full)" % virtualfn)
+        logger.debug("Parsing %s (full)" % virtualfn)
         (fn, virtual, mc) = virtualfn2realfn(virtualfn)
         bb_data = self.load_bbfile(virtualfn, appends, virtonly=True)
         return bb_data[virtual]
@@ -400,7 +403,7 @@ class Cache(NoCache):
 
         self.cachefile = self.getCacheFile("bb_cache.dat")
 
-        self.logger.debug(1, "Cache dir: %s", self.cachedir)
+        self.logger.debug("Cache dir: %s", self.cachedir)
         bb.utils.mkdirhier(self.cachedir)
 
         cache_ok = True
@@ -408,7 +411,7 @@ class Cache(NoCache):
             for cache_class in self.caches_array:
                 cachefile = self.getCacheFile(cache_class.cachefile)
                 cache_exists = os.path.exists(cachefile)
-                self.logger.debug(2, "Checking if %s exists: %r", cachefile, cache_exists)
+                self.logger.debug2("Checking if %s exists: %r", cachefile, cache_exists)
                 cache_ok = cache_ok and cache_exists
                 cache_class.init_cacheData(self)
         if cache_ok:
@@ -416,7 +419,7 @@ class Cache(NoCache):
         elif os.path.isfile(self.cachefile):
             self.logger.info("Out of date cache found, rebuilding...")
         else:
-            self.logger.debug(1, "Cache file %s not found, building..." % self.cachefile)
+            self.logger.debug("Cache file %s not found, building..." % self.cachefile)
 
         # We don't use the symlink, its just for debugging convinience
         if self.mc:
@@ -449,13 +452,11 @@ class Cache(NoCache):
         return cachesize
 
     def load_cachefile(self, progress):
-        cachesize = self.cachesize()
         previous_progress = 0
-        previous_percent = 0
 
         for cache_class in self.caches_array:
             cachefile = self.getCacheFile(cache_class.cachefile)
-            self.logger.debug(1, 'Loading cache file: %s' % cachefile)
+            self.logger.debug('Loading cache file: %s' % cachefile)
             with open(cachefile, "rb") as cachefile:
                 pickled = pickle.Unpickler(cachefile)
                 # Check cache version information
@@ -502,7 +503,7 @@ class Cache(NoCache):
 
     def parse(self, filename, appends):
         """Parse the specified filename, returning the recipe information"""
-        self.logger.debug(1, "Parsing %s", filename)
+        self.logger.debug("Parsing %s", filename)
         infos = []
         datastores = self.load_bbfile(filename, appends, mc=self.mc)
         depends = []
@@ -556,7 +557,7 @@ class Cache(NoCache):
         cached, infos = self.load(fn, appends)
         for virtualfn, info_array in infos:
             if info_array[0].skipped:
-                self.logger.debug(1, "Skipping %s: %s", virtualfn, info_array[0].skipreason)
+                self.logger.debug("Skipping %s: %s", virtualfn, info_array[0].skipreason)
                 skipped += 1
             else:
                 self.add_info(virtualfn, info_array, cacheData, not cached)
@@ -592,21 +593,21 @@ class Cache(NoCache):
 
         # File isn't in depends_cache
         if not fn in self.depends_cache:
-            self.logger.debug(2, "%s is not cached", fn)
+            self.logger.debug2("%s is not cached", fn)
             return False
 
         mtime = bb.parse.cached_mtime_noerror(fn)
 
         # Check file still exists
         if mtime == 0:
-            self.logger.debug(2, "%s no longer exists", fn)
+            self.logger.debug2("%s no longer exists", fn)
             self.remove(fn)
             return False
 
         info_array = self.depends_cache[fn]
         # Check the file's timestamp
         if mtime != info_array[0].timestamp:
-            self.logger.debug(2, "%s changed", fn)
+            self.logger.debug2("%s changed", fn)
             self.remove(fn)
             return False
 
@@ -617,13 +618,13 @@ class Cache(NoCache):
                 fmtime = bb.parse.cached_mtime_noerror(f)
                 # Check if file still exists
                 if old_mtime != 0 and fmtime == 0:
-                    self.logger.debug(2, "%s's dependency %s was removed",
+                    self.logger.debug2("%s's dependency %s was removed",
                                          fn, f)
                     self.remove(fn)
                     return False
 
                 if (fmtime != old_mtime):
-                    self.logger.debug(2, "%s's dependency %s changed",
+                    self.logger.debug2("%s's dependency %s changed",
                                          fn, f)
                     self.remove(fn)
                     return False
@@ -640,14 +641,14 @@ class Cache(NoCache):
                         continue
                     f, exist = f.split(":")
                     if (exist == "True" and not os.path.exists(f)) or (exist == "False" and os.path.exists(f)):
-                        self.logger.debug(2, "%s's file checksum list file %s changed",
+                        self.logger.debug2("%s's file checksum list file %s changed",
                                              fn, f)
                         self.remove(fn)
                         return False
 
         if tuple(appends) != tuple(info_array[0].appends):
-            self.logger.debug(2, "appends for %s changed", fn)
-            self.logger.debug(2, "%s to %s" % (str(appends), str(info_array[0].appends)))
+            self.logger.debug2("appends for %s changed", fn)
+            self.logger.debug2("%s to %s" % (str(appends), str(info_array[0].appends)))
             self.remove(fn)
             return False
 
@@ -656,10 +657,10 @@ class Cache(NoCache):
             virtualfn = variant2virtual(fn, cls)
             self.clean.add(virtualfn)
             if virtualfn not in self.depends_cache:
-                self.logger.debug(2, "%s is not cached", virtualfn)
+                self.logger.debug2("%s is not cached", virtualfn)
                 invalid = True
             elif len(self.depends_cache[virtualfn]) != len(self.caches_array):
-                self.logger.debug(2, "Extra caches missing for %s?" % virtualfn)
+                self.logger.debug2("Extra caches missing for %s?" % virtualfn)
                 invalid = True
 
         # If any one of the variants is not present, mark as invalid for all
@@ -667,10 +668,10 @@ class Cache(NoCache):
             for cls in info_array[0].variants:
                 virtualfn = variant2virtual(fn, cls)
                 if virtualfn in self.clean:
-                    self.logger.debug(2, "Removing %s from cache", virtualfn)
+                    self.logger.debug2("Removing %s from cache", virtualfn)
                     self.clean.remove(virtualfn)
             if fn in self.clean:
-                self.logger.debug(2, "Marking %s as not clean", fn)
+                self.logger.debug2("Marking %s as not clean", fn)
                 self.clean.remove(fn)
             return False
 
@@ -683,10 +684,10 @@ class Cache(NoCache):
         Called from the parser in error cases
         """
         if fn in self.depends_cache:
-            self.logger.debug(1, "Removing %s from cache", fn)
+            self.logger.debug("Removing %s from cache", fn)
             del self.depends_cache[fn]
         if fn in self.clean:
-            self.logger.debug(1, "Marking %s as unclean", fn)
+            self.logger.debug("Marking %s as unclean", fn)
             self.clean.remove(fn)
 
     def sync(self):
@@ -699,13 +700,13 @@ class Cache(NoCache):
             return
 
         if self.cacheclean:
-            self.logger.debug(2, "Cache is clean, not saving.")
+            self.logger.debug2("Cache is clean, not saving.")
             return
 
         for cache_class in self.caches_array:
             cache_class_name = cache_class.__name__
             cachefile = self.getCacheFile(cache_class.cachefile)
-            self.logger.debug(2, "Writing %s", cachefile)
+            self.logger.debug2("Writing %s", cachefile)
             with open(cachefile, "wb") as f:
                 p = pickle.Pickler(f, pickle.HIGHEST_PROTOCOL)
                 p.dump(__cache_version__)
@@ -816,10 +817,6 @@ class MulticonfigCache(Mapping):
         for k in self.__caches:
             yield k
 
-    def keys(self):
-        return self.__caches[key]
-
-
 def init(cooker):
     """
     The Objective: Cache the minimum amount of data possible yet get to the
@@ -885,7 +882,7 @@ class MultiProcessCache(object):
         bb.utils.mkdirhier(cachedir)
         self.cachefile = os.path.join(cachedir,
                                       cache_file_name or self.__class__.cache_file_name)
-        logger.debug(1, "Using cache in '%s'", self.cachefile)
+        logger.debug("Using cache in '%s'", self.cachefile)
 
         glf = bb.utils.lockfile(self.cachefile + ".lock")
 
@@ -991,7 +988,7 @@ class SimpleCache(object):
         bb.utils.mkdirhier(cachedir)
         self.cachefile = os.path.join(cachedir,
                                       cache_file_name or self.__class__.cache_file_name)
-        logger.debug(1, "Using cache in '%s'", self.cachefile)
+        logger.debug("Using cache in '%s'", self.cachefile)
 
         glf = bb.utils.lockfile(self.cachefile + ".lock")
 

bitbake/lib/bb/codeparser.py

@@ -212,9 +212,9 @@ class PythonParser():
             funcstr = codegen.to_source(func)
             argstr = codegen.to_source(arg)
         except TypeError:
-            self.log.debug(2, 'Failed to convert function and argument to source form')
+            self.log.debug2('Failed to convert function and argument to source form')
         else:
-            self.log.debug(1, self.unhandled_message % (funcstr, argstr))
+            self.log.debug(self.unhandled_message % (funcstr, argstr))
 
     def visit_Call(self, node):
         name = self.called_node_name(node.func)
@@ -450,7 +450,7 @@ class ShellParser():
 
                 cmd = word[1]
                 if cmd.startswith("$"):
-                    self.log.debug(1, self.unhandled_template % cmd)
+                    self.log.debug(self.unhandled_template % cmd)
                 elif cmd == "eval":
                     command = " ".join(word for _, word in words[1:])
                     self._parse_shell(command)

bitbake/lib/bb/command.py

@@ -20,6 +20,7 @@ Commands are queued in a CommandQueue
 
 from collections import OrderedDict, defaultdict
 
+import io
 import bb.event
 import bb.cooker
 import bb.remotedata
@@ -500,6 +501,17 @@ class CommandsSync:
         d = command.remotedatastores[dsindex].varhistory
         return getattr(d, method)(*args, **kwargs)
 
+    def dataStoreConnectorVarHistCmdEmit(self, command, params):
+        dsindex = params[0]
+        var = params[1]
+        oval = params[2]
+        val = params[3]
+        d = command.remotedatastores[params[4]]
+
+        o = io.StringIO()
+        command.remotedatastores[dsindex].varhistory.emit(var, oval, val, o, d)
+        return o.getvalue()
+
     def dataStoreConnectorIncHistCmd(self, command, params):
         dsindex = params[0]
         method = params[1]

bitbake/lib/bb/compress/_pipecompress.py

@@ -0,0 +1,194 @@
+#
+# SPDX-License-Identifier: GPL-2.0-only
+#
+# Helper library to implement streaming compression and decompression using an
+# external process
+#
+# This library should be used directly by end users; a wrapper library for the
+# specific compression tool should be created
+
+import builtins
+import io
+import os
+import subprocess
+
+
+def open_wrap(
+    cls, filename, mode="rb", *, encoding=None, errors=None, newline=None, **kwargs
+):
+    """
+    Open a compressed file in binary or text mode.
+
+    Users should not call this directly. A specific compression library can use
+    this helper to provide it's own "open" command
+
+    The filename argument can be an actual filename (a str or bytes object), or
+    an existing file object to read from or write to.
+
+    The mode argument can be "r", "rb", "w", "wb", "x", "xb", "a" or "ab" for
+    binary mode, or "rt", "wt", "xt" or "at" for text mode. The default mode is
+    "rb".
+
+    For binary mode, this function is equivalent to the cls constructor:
+    cls(filename, mode). In this case, the encoding, errors and newline
+    arguments must not be provided.
+
+    For text mode, a cls object is created, and wrapped in an
+    io.TextIOWrapper instance with the specified encoding, error handling
+    behavior, and line ending(s).
+    """
+    if "t" in mode:
+        if "b" in mode:
+            raise ValueError("Invalid mode: %r" % (mode,))
+    else:
+        if encoding is not None:
+            raise ValueError("Argument 'encoding' not supported in binary mode")
+        if errors is not None:
+            raise ValueError("Argument 'errors' not supported in binary mode")
+        if newline is not None:
+            raise ValueError("Argument 'newline' not supported in binary mode")
+
+    file_mode = mode.replace("t", "")
+    if isinstance(filename, (str, bytes, os.PathLike)):
+        binary_file = cls(filename, file_mode, **kwargs)
+    elif hasattr(filename, "read") or hasattr(filename, "write"):
+        binary_file = cls(None, file_mode, fileobj=filename, **kwargs)
+    else:
+        raise TypeError("filename must be a str or bytes object, or a file")
+
+    if "t" in mode:
+        return io.TextIOWrapper(
+            binary_file, encoding, errors, newline, write_through=True
+        )
+    else:
+        return binary_file
+
+
+class CompressionError(OSError):
+    pass
+
+
+class PipeFile(io.RawIOBase):
+    """
+    Class that implements generically piping to/from a compression program
+
+    Derived classes should add the function get_compress() and get_decompress()
+    that return the required commands. Input will be piped into stdin and the
+    (de)compressed output should be written to stdout, e.g.:
+
+        class FooFile(PipeCompressionFile):
+            def get_decompress(self):
+                return ["fooc", "--decompress", "--stdout"]
+
+            def get_compress(self):
+                return ["fooc", "--compress", "--stdout"]
+
+    """
+
+    READ = 0
+    WRITE = 1
+
+    def __init__(self, filename=None, mode="rb", *, stderr=None, fileobj=None):
+        if "t" in mode or "U" in mode:
+            raise ValueError("Invalid mode: {!r}".format(mode))
+
+        if not "b" in mode:
+            mode += "b"
+
+        if mode.startswith("r"):
+            self.mode = self.READ
+        elif mode.startswith("w"):
+            self.mode = self.WRITE
+        else:
+            raise ValueError("Invalid mode %r" % mode)
+
+        if fileobj is not None:
+            self.fileobj = fileobj
+        else:
+            self.fileobj = builtins.open(filename, mode or "rb")
+
+        if self.mode == self.READ:
+            self.p = subprocess.Popen(
+                self.get_decompress(),
+                stdin=self.fileobj,
+                stdout=subprocess.PIPE,
+                stderr=stderr,
+                close_fds=True,
+            )
+            self.pipe = self.p.stdout
+        else:
+            self.p = subprocess.Popen(
+                self.get_compress(),
+                stdin=subprocess.PIPE,
+                stdout=self.fileobj,
+                stderr=stderr,
+                close_fds=True,
+            )
+            self.pipe = self.p.stdin
+
+        self.__closed = False
+
+    def _check_process(self):
+        if self.p is None:
+            return
+
+        returncode = self.p.wait()
+        if returncode:
+            raise CompressionError("Process died with %d" % returncode)
+        self.p = None
+
+    def close(self):
+        if self.closed:
+            return
+
+        self.pipe.close()
+        if self.p is not None:
+            self._check_process()
+        self.fileobj.close()
+
+        self.__closed = True
+
+    @property
+    def closed(self):
+        return self.__closed
+
+    def fileno(self):
+        return self.pipe.fileno()
+
+    def flush(self):
+        self.pipe.flush()
+
+    def isatty(self):
+        return self.pipe.isatty()
+
+    def readable(self):
+        return self.mode == self.READ
+
+    def writable(self):
+        return self.mode == self.WRITE
+
+    def readinto(self, b):
+        if self.mode != self.READ:
+            import errno
+
+            raise OSError(
+                errno.EBADF, "read() on write-only %s object" % self.__class__.__name__
+            )
+        size = self.pipe.readinto(b)
+        if size == 0:
+            self._check_process()
+        return size
+
+    def write(self, data):
+        if self.mode != self.WRITE:
+            import errno
+
+            raise OSError(
+                errno.EBADF, "write() on read-only %s object" % self.__class__.__name__
+            )
+        data = self.pipe.write(data)
+
+        if not data:
+            self._check_process()
+
+        return data

bitbake/lib/bb/compress/lz4.py

@@ -0,0 +1,17 @@
+#
+# SPDX-License-Identifier: GPL-2.0-only
+#
+
+import bb.compress._pipecompress
+
+
+def open(*args, **kwargs):
+    return bb.compress._pipecompress.open_wrap(LZ4File, *args, **kwargs)
+
+
+class LZ4File(bb.compress._pipecompress.PipeFile):
+    def get_compress(self):
+        return ["lz4c", "-z", "-c"]
+
+    def get_decompress(self):
+        return ["lz4c", "-d", "-c"]

bitbake/lib/bb/compress/zstd.py

@@ -0,0 +1,28 @@
+#
+# SPDX-License-Identifier: GPL-2.0-only
+#
+
+import bb.compress._pipecompress
+import shutil
+
+
+def open(*args, **kwargs):
+    return bb.compress._pipecompress.open_wrap(ZstdFile, *args, **kwargs)
+
+
+class ZstdFile(bb.compress._pipecompress.PipeFile):
+    def __init__(self, *args, num_threads=1, compresslevel=3, **kwargs):
+        self.num_threads = num_threads
+        self.compresslevel = compresslevel
+        super().__init__(*args, **kwargs)
+
+    def _get_zstd(self):
+        if self.num_threads == 1 or not shutil.which("pzstd"):
+            return ["zstd"]
+        return ["pzstd", "-p", "%d" % self.num_threads]
+
+    def get_compress(self):
+        return self._get_zstd() + ["-c", "-%d" % self.compresslevel]
+
+    def get_decompress(self):
+        return self._get_zstd() + ["-d", "-c"]

bitbake/lib/bb/cooker.py

@@ -73,7 +73,9 @@ class SkippedPackage:
             self.pn = info.pn
             self.skipreason = info.skipreason
             self.provides = info.provides
-            self.rprovides = info.rprovides
+            self.rprovides = info.packages + info.rprovides
+            for package in info.packages:
+                self.rprovides += info.rprovides_pkg[package]
         elif reason:
             self.skipreason = reason
 
@@ -409,6 +411,8 @@ class BBCooker:
             self.data.disableTracking()
 
     def parseConfiguration(self):
+        self.updateCacheSync()
+
         # Change nice level if we're asked to
         nice = self.data.getVar("BB_NICE_LEVEL")
         if nice:
@@ -439,7 +443,7 @@ class BBCooker:
                         continue
                 except AttributeError:
                     pass
-                logger.debug(1, "Marking as dirty due to '%s' option change to '%s'" % (o, options[o]))
+                logger.debug("Marking as dirty due to '%s' option change to '%s'" % (o, options[o]))
                 print("Marking as dirty due to '%s' option change to '%s'" % (o, options[o]))
                 clean = False
             if hasattr(self.configuration, o):
@@ -466,17 +470,17 @@ class BBCooker:
 
         for k in bb.utils.approved_variables():
             if k in environment and k not in self.configuration.env:
-                logger.debug(1, "Updating new environment variable %s to %s" % (k, environment[k]))
+                logger.debug("Updating new environment variable %s to %s" % (k, environment[k]))
                 self.configuration.env[k] = environment[k]
                 clean = False
             if k in self.configuration.env and k not in environment:
-                logger.debug(1, "Updating environment variable %s (deleted)" % (k))
+                logger.debug("Updating environment variable %s (deleted)" % (k))
                 del self.configuration.env[k]
                 clean = False
             if k not in self.configuration.env and k not in environment:
                 continue
             if environment[k] != self.configuration.env[k]:
-                logger.debug(1, "Updating environment variable %s from %s to %s" % (k, self.configuration.env[k], environment[k]))
+                logger.debug("Updating environment variable %s from %s to %s" % (k, self.configuration.env[k], environment[k]))
                 self.configuration.env[k] = environment[k]
                 clean = False
 
@@ -484,7 +488,7 @@ class BBCooker:
         self.configuration.env = environment
 
         if not clean:
-            logger.debug(1, "Base environment change, triggering reparse")
+            logger.debug("Base environment change, triggering reparse")
             self.reset()
 
     def runCommands(self, server, data, abort):
@@ -498,22 +502,30 @@ class BBCooker:
 
     def showVersions(self):
 
-        (latest_versions, preferred_versions) = self.findProviders()
+        (latest_versions, preferred_versions, required) = self.findProviders()
 
-        logger.plain("%-35s %25s %25s", "Recipe Name", "Latest Version", "Preferred Version")
-        logger.plain("%-35s %25s %25s\n", "===========", "==============", "=================")
+        logger.plain("%-35s %25s %25s %25s", "Recipe Name", "Latest Version", "Preferred Version", "Required Version")
+        logger.plain("%-35s %25s %25s %25s\n", "===========", "==============", "=================", "================")
 
         for p in sorted(self.recipecaches[''].pkg_pn):
-            pref = preferred_versions[p]
+            preferred = preferred_versions[p]
             latest = latest_versions[p]
+            requiredstr = ""
+            preferredstr = ""
+            if required[p]:
+                if preferred[0] is not None:
+                    requiredstr = preferred[0][0] + ":" + preferred[0][1] + '-' + preferred[0][2]
+                else:
+                    bb.fatal("REQUIRED_VERSION of package %s not available" % p)
+            else:
+                preferredstr = preferred[0][0] + ":" + preferred[0][1] + '-' + preferred[0][2]
 
-            prefstr = pref[0][0] + ":" + pref[0][1] + '-' + pref[0][2]
             lateststr = latest[0][0] + ":" + latest[0][1] + "-" + latest[0][2]
 
-            if pref == latest:
-                prefstr = ""
+            if preferred == latest:
+                preferredstr = ""
 
-            logger.plain("%-35s %25s %25s", p, lateststr, prefstr)
+            logger.plain("%-35s %25s %25s %25s", p, lateststr, preferredstr, requiredstr)
 
     def showEnvironment(self, buildfile=None, pkgs_to_build=None):
         """
@@ -612,7 +624,7 @@ class BBCooker:
         # Replace string such as "mc:*:bash"
         # into "mc:A:bash mc:B:bash bash"
         for k in targetlist:
-            if k.startswith("mc:"):
+            if k.startswith("mc:") and k.count(':') >= 2:
                 if wildcard:
                     bb.fatal('multiconfig conflict')
                 if k.split(":")[1] == "*":
@@ -646,7 +658,7 @@ class BBCooker:
         for k in fulltargetlist:
             origk = k
             mc = ""
-            if k.startswith("mc:"):
+            if k.startswith("mc:") and k.count(':') >= 2:
                 mc = k.split(":")[1]
                 k = ":".join(k.split(":")[2:])
             ktask = task
@@ -695,7 +707,7 @@ class BBCooker:
                         if depmc not in self.multiconfigs:
                             bb.fatal("Multiconfig dependency %s depends on nonexistent multiconfig configuration named configuration %s" % (k,depmc))
                         else:
-                            logger.debug(1, "Adding providers for multiconfig dependency %s" % l[3])
+                            logger.debug("Adding providers for multiconfig dependency %s" % l[3])
                             taskdata[depmc].add_provider(localdata[depmc], self.recipecaches[depmc], l[3])
                             seen.add(k)
                             new = True
@@ -1059,10 +1071,16 @@ class BBCooker:
         if pn in self.recipecaches[mc].providers:
             filenames = self.recipecaches[mc].providers[pn]
             eligible, foundUnique = bb.providers.filterProviders(filenames, pn, self.databuilder.mcdata[mc], self.recipecaches[mc])
-            filename = eligible[0]
+            if eligible is not None:
+                filename = eligible[0]
+            else:
+                filename = None
             return None, None, None, filename
         elif pn in self.recipecaches[mc].pkg_pn:
-            return bb.providers.findBestProvider(pn, self.databuilder.mcdata[mc], self.recipecaches[mc], self.recipecaches[mc].pkg_pn)
+            (latest, latest_f, preferred_ver, preferred_file, required) = bb.providers.findBestProvider(pn, self.databuilder.mcdata[mc], self.recipecaches[mc], self.recipecaches[mc].pkg_pn)
+            if required and preferred_file is None:
+                return None, None, None, None
+            return (latest, latest_f, preferred_ver, preferred_file)
         else:
             return None, None, None, None
 
@@ -1551,7 +1569,7 @@ class BBCooker:
         self.inotify_modified_files = []
 
         if not self.baseconfig_valid:
-            logger.debug(1, "Reloading base configuration data")
+            logger.debug("Reloading base configuration data")
             self.initConfigurationData()
             self.handlePRServ()
 

bitbake/lib/bb/cookerdata.py

@@ -23,8 +23,8 @@ logger      = logging.getLogger("BitBake")
 parselog    = logging.getLogger("BitBake.Parsing")
 
 class ConfigParameters(object):
-    def __init__(self, argv=sys.argv):
-        self.options, targets = self.parseCommandLine(argv)
+    def __init__(self, argv=None):
+        self.options, targets = self.parseCommandLine(argv or sys.argv)
         self.environment = self.parseEnvironment()
 
         self.options.pkgs_to_build = targets or []
@@ -209,7 +209,7 @@ def findConfigFile(configfile, data):
     return None
 
 #
-# We search for a conf/bblayers.conf under an entry in BBPATH or in cwd working 
+# We search for a conf/bblayers.conf under an entry in BBPATH or in cwd working
 # up to /. If that fails, we search for a conf/bitbake.conf in BBPATH.
 #
 
@@ -429,7 +429,7 @@ class CookerDataBuilder(object):
                 parselog.critical("Undefined event handler function '%s'" % var)
                 raise bb.BBHandledException()
             handlerln = int(data.getVarFlag(var, "lineno", False))
-            bb.event.register(var, data.getVar(var, False),  (data.getVarFlag(var, "eventmask") or "").split(), handlerfn, handlerln)
+            bb.event.register(var, data.getVar(var, False),  (data.getVarFlag(var, "eventmask") or "").split(), handlerfn, handlerln, data)
 
         data.setVar('BBINCLUDED',bb.parse.get_file_depends(data))
 

bitbake/lib/bb/data_smart.py

@@ -481,6 +481,7 @@ class DataSmart(MutableMapping):
 
     def setVar(self, var, value, **loginfo):
         #print("var=" + str(var) + "  val=" + str(value))
+        var = var.replace(":", "_")
         self.expand_cache = {}
         parsing=False
         if 'parsing' in loginfo:
@@ -589,6 +590,8 @@ class DataSmart(MutableMapping):
         """
         Rename the variable key to newkey
         """
+        key = key.replace(":", "_")
+        newkey = newkey.replace(":", "_")
         if key == newkey:
             bb.warn("Calling renameVar with equivalent keys (%s) is invalid" % key)
             return
@@ -637,6 +640,7 @@ class DataSmart(MutableMapping):
         self.setVar(var + "_prepend", value, ignore=True, parsing=True)
 
     def delVar(self, var, **loginfo):
+        var = var.replace(":", "_")
         self.expand_cache = {}
 
         loginfo['detail'] = ""
@@ -664,6 +668,7 @@ class DataSmart(MutableMapping):
                          override = None
 
     def setVarFlag(self, var, flag, value, **loginfo):
+        var = var.replace(":", "_")
         self.expand_cache = {}
 
         if 'op' not in loginfo:
@@ -687,6 +692,7 @@ class DataSmart(MutableMapping):
             self.dict["__exportlist"]["_content"].add(var)
 
     def getVarFlag(self, var, flag, expand=True, noweakdefault=False, parsing=False, retparser=False):
+        var = var.replace(":", "_")
         if flag == "_content":
             cachename = var
         else:
@@ -814,6 +820,7 @@ class DataSmart(MutableMapping):
         return value
 
     def delVarFlag(self, var, flag, **loginfo):
+        var = var.replace(":", "_")
         self.expand_cache = {}
 
         local_var, _ = self._findVar(var)
@@ -831,6 +838,7 @@ class DataSmart(MutableMapping):
             del self.dict[var][flag]
 
     def appendVarFlag(self, var, flag, value, **loginfo):
+        var = var.replace(":", "_")
         loginfo['op'] = 'append'
         loginfo['flag'] = flag
         self.varhistory.record(**loginfo)
@@ -838,6 +846,7 @@ class DataSmart(MutableMapping):
         self.setVarFlag(var, flag, newvalue, ignore=True)
 
     def prependVarFlag(self, var, flag, value, **loginfo):
+        var = var.replace(":", "_")
         loginfo['op'] = 'prepend'
         loginfo['flag'] = flag
         self.varhistory.record(**loginfo)
@@ -845,6 +854,7 @@ class DataSmart(MutableMapping):
         self.setVarFlag(var, flag, newvalue, ignore=True)
 
     def setVarFlags(self, var, flags, **loginfo):
+        var = var.replace(":", "_")
         self.expand_cache = {}
         infer_caller_details(loginfo)
         if not var in self.dict:
@@ -859,6 +869,7 @@ class DataSmart(MutableMapping):
             self.dict[var][i] = flags[i]
 
     def getVarFlags(self, var, expand = False, internalflags=False):
+        var = var.replace(":", "_")
         local_var, _ = self._findVar(var)
         flags = {}
 
@@ -875,6 +886,7 @@ class DataSmart(MutableMapping):
 
 
     def delVarFlags(self, var, **loginfo):
+        var = var.replace(":", "_")
         self.expand_cache = {}
         if not var in self.dict:
             self._makeShadowCopy(var)

bitbake/lib/bb/event.py

@@ -118,6 +118,8 @@ def fire_class_handlers(event, d):
             if _eventfilter:
                 if not _eventfilter(name, handler, event, d):
                     continue
+            if d is not None and not name in (d.getVar("__BBHANDLERS_MC") or set()):
+                continue
             execute_handler(name, handler, event, d)
 
 ui_queue = []
@@ -227,11 +229,19 @@ def fire_from_worker(event, d):
     fire_ui_handlers(event, d)
 
 noop = lambda _: None
-def register(name, handler, mask=None, filename=None, lineno=None):
+def register(name, handler, mask=None, filename=None, lineno=None, data=None):
     """Register an Event handler"""
 
+    if data is not None and data.getVar("BB_CURRENT_MC"):
+        mc = data.getVar("BB_CURRENT_MC")
+        name = '%s%s' % (mc.replace('-', '_'), name)
+
     # already registered
     if name in _handlers:
+        if data is not None:
+            bbhands_mc = (data.getVar("__BBHANDLERS_MC") or set())
+            bbhands_mc.add(name)
+            data.setVar("__BBHANDLERS_MC", bbhands_mc)
         return AlreadyRegistered
 
     if handler is not None:
@@ -268,10 +278,20 @@ def register(name, handler, mask=None, filename=None, lineno=None):
                     _event_handler_map[m] = {}
                 _event_handler_map[m][name] = True
 
+        if data is not None:
+            bbhands_mc = (data.getVar("__BBHANDLERS_MC") or set())
+            bbhands_mc.add(name)
+            data.setVar("__BBHANDLERS_MC", bbhands_mc)
+
         return Registered
 
-def remove(name, handler):
+def remove(name, handler, data=None):
     """Remove an Event handler"""
+    if data is not None:
+        if data.getVar("BB_CURRENT_MC"):
+            mc = data.getVar("BB_CURRENT_MC")
+            name = '%s%s' % (mc.replace('-', '_'), name)
+
     _handlers.pop(name)
     if name in _catchall_handlers:
         _catchall_handlers.pop(name)
@@ -279,6 +299,12 @@ def remove(name, handler):
         if name in _event_handler_map[event]:
             _event_handler_map[event].pop(name)
 
+    if data is not None:
+        bbhands_mc = (data.getVar("__BBHANDLERS_MC") or set())
+        if name in bbhands_mc:
+            bbhands_mc.remove(name)
+            data.setVar("__BBHANDLERS_MC", bbhands_mc)
+
 def get_handlers():
     return _handlers
 
@@ -644,6 +670,17 @@ class ReachableStamps(Event):
         Event.__init__(self)
         self.stamps = stamps
 
+class StaleSetSceneTasks(Event):
+    """
+    An event listing setscene tasks which are 'stale' and will
+    be rerun. The metadata may use to clean up stale data.
+    tasks is a mapping of tasks and matching stale stamps.
+    """
+
+    def __init__(self, tasks):
+        Event.__init__(self)
+        self.tasks = tasks
+
 class FilesMatchingFound(Event):
     """
     Event when a list of files matching the supplied pattern has

bitbake/lib/bb/fetch2/__init__.py

@@ -290,7 +290,7 @@ class URI(object):
 
     def _param_str_split(self, string, elmdelim, kvdelim="="):
         ret = collections.OrderedDict()
-        for k, v in [x.split(kvdelim, 1) for x in string.split(elmdelim)]:
+        for k, v in [x.split(kvdelim, 1) for x in string.split(elmdelim) if x]:
             ret[k] = v
         return ret
 
@@ -428,7 +428,7 @@ def uri_replace(ud, uri_find, uri_replace, replacements, d, mirrortarball=None):
     uri_decoded = list(decodeurl(ud.url))
     uri_find_decoded = list(decodeurl(uri_find))
     uri_replace_decoded = list(decodeurl(uri_replace))
-    logger.debug(2, "For url %s comparing %s to %s" % (uri_decoded, uri_find_decoded, uri_replace_decoded))
+    logger.debug2("For url %s comparing %s to %s" % (uri_decoded, uri_find_decoded, uri_replace_decoded))
     result_decoded = ['', '', '', '', '', {}]
     for loc, i in enumerate(uri_find_decoded):
         result_decoded[loc] = uri_decoded[loc]
@@ -474,7 +474,7 @@ def uri_replace(ud, uri_find, uri_replace, replacements, d, mirrortarball=None):
     result = encodeurl(result_decoded)
     if result == ud.url:
         return None
-    logger.debug(2, "For url %s returning %s" % (ud.url, result))
+    logger.debug2("For url %s returning %s" % (ud.url, result))
     return result
 
 methods = []
@@ -499,9 +499,9 @@ def fetcher_init(d):
     # When to drop SCM head revisions controlled by user policy
     srcrev_policy = d.getVar('BB_SRCREV_POLICY') or "clear"
     if srcrev_policy == "cache":
-        logger.debug(1, "Keeping SRCREV cache due to cache policy of: %s", srcrev_policy)
+        logger.debug("Keeping SRCREV cache due to cache policy of: %s", srcrev_policy)
     elif srcrev_policy == "clear":
-        logger.debug(1, "Clearing SRCREV cache due to cache policy of: %s", srcrev_policy)
+        logger.debug("Clearing SRCREV cache due to cache policy of: %s", srcrev_policy)
         revs.clear()
     else:
         raise FetchError("Invalid SRCREV cache policy of: %s" % srcrev_policy)
@@ -562,6 +562,9 @@ def verify_checksum(ud, d, precomputed={}):
 
         checksum_expected = getattr(ud, "%s_expected" % checksum_id)
 
+        if checksum_expected == '':
+            checksum_expected = None
+
         return {
             "id": checksum_id,
             "name": checksum_name,
@@ -612,7 +615,7 @@ def verify_checksum(ud, d, precomputed={}):
 
     for ci in checksum_infos:
         if ci["expected"] and ci["expected"] != ci["data"]:
-            messages.append("File: '%s' has %s checksum %s when %s was " \
+            messages.append("File: '%s' has %s checksum '%s' when '%s' was " \
                             "expected" % (ud.localpath, ci["id"], ci["data"], ci["expected"]))
             bad_checksum = ci["data"]
 
@@ -831,7 +834,10 @@ def runfetchcmd(cmd, d, quiet=False, cleanup=None, log=None, workdir=None):
                   'SSH_AUTH_SOCK', 'SSH_AGENT_PID',
                   'SOCKS5_USER', 'SOCKS5_PASSWD',
                   'DBUS_SESSION_BUS_ADDRESS',
-                  'P4CONFIG']
+                  'P4CONFIG',
+                  'AWS_ACCESS_KEY_ID',
+                  'AWS_SECRET_ACCESS_KEY',
+                  'AWS_DEFAULT_REGION']
 
     if not cleanup:
         cleanup = []
@@ -857,9 +863,9 @@ def runfetchcmd(cmd, d, quiet=False, cleanup=None, log=None, workdir=None):
     cmd = 'export PSEUDO_DISABLED=1; ' + cmd
 
     if workdir:
-        logger.debug(1, "Running '%s' in %s" % (cmd, workdir))
+        logger.debug("Running '%s' in %s" % (cmd, workdir))
     else:
-        logger.debug(1, "Running %s", cmd)
+        logger.debug("Running %s", cmd)
 
     success = False
     error_message = ""
@@ -900,7 +906,7 @@ def check_network_access(d, info, url):
     elif not trusted_network(d, url):
         raise UntrustedUrl(url, info)
     else:
-        logger.debug(1, "Fetcher accessed the network with the command %s" % info)
+        logger.debug("Fetcher accessed the network with the command %s" % info)
 
 def build_mirroruris(origud, mirrors, ld):
     uris = []
@@ -926,7 +932,7 @@ def build_mirroruris(origud, mirrors, ld):
                     continue
 
                 if not trusted_network(ld, newuri):
-                    logger.debug(1, "Mirror %s not in the list of trusted networks, skipping" %  (newuri))
+                    logger.debug("Mirror %s not in the list of trusted networks, skipping" %  (newuri))
                     continue
 
                 # Create a local copy of the mirrors minus the current line
@@ -939,8 +945,8 @@ def build_mirroruris(origud, mirrors, ld):
                     newud = FetchData(newuri, ld)
                     newud.setup_localpath(ld)
                 except bb.fetch2.BBFetchException as e:
-                    logger.debug(1, "Mirror fetch failure for url %s (original url: %s)" % (newuri, origud.url))
-                    logger.debug(1, str(e))
+                    logger.debug("Mirror fetch failure for url %s (original url: %s)" % (newuri, origud.url))
+                    logger.debug(str(e))
                     try:
                         # setup_localpath of file:// urls may fail, we should still see
                         # if mirrors of the url exist
@@ -1043,8 +1049,8 @@ def try_mirror_url(fetch, origud, ud, ld, check = False):
         elif isinstance(e, NoChecksumError):
             raise
         else:
-            logger.debug(1, "Mirror fetch failure for url %s (original url: %s)" % (ud.url, origud.url))
-            logger.debug(1, str(e))
+            logger.debug("Mirror fetch failure for url %s (original url: %s)" % (ud.url, origud.url))
+            logger.debug(str(e))
         try:
             ud.method.clean(ud, ld)
         except UnboundLocalError:
@@ -1243,7 +1249,7 @@ class FetchData(object):
 
             if checksum_name in self.parm:
                 checksum_expected = self.parm[checksum_name]
-            elif self.type not in ["http", "https", "ftp", "ftps", "sftp", "s3"]:
+            elif self.type not in ["http", "https", "ftp", "ftps", "sftp", "s3", "az"]:
                 checksum_expected = None
             else:
                 checksum_expected = d.getVarFlag("SRC_URI", checksum_name)
@@ -1456,6 +1462,10 @@ class FetchMethod(object):
                 cmd = '7z x -so %s | tar x --no-same-owner -f -' % file
             elif file.endswith('.7z'):
                 cmd = '7za x -y %s 1>/dev/null' % file
+            elif file.endswith('.tzst') or file.endswith('.tar.zst'):
+                cmd = 'zstd --decompress --stdout %s | tar x --no-same-owner -f -' % file
+            elif file.endswith('.zst'):
+                cmd = 'zstd --decompress --stdout %s > %s' % (file, efile)
             elif file.endswith('.zip') or file.endswith('.jar'):
                 try:
                     dos = bb.utils.to_boolean(urldata.parm.get('dos'), False)
@@ -1684,7 +1694,7 @@ class Fetch(object):
                 if m.verify_donestamp(ud, self.d) and not m.need_update(ud, self.d):
                     done = True
                 elif m.try_premirror(ud, self.d):
-                    logger.debug(1, "Trying PREMIRRORS")
+                    logger.debug("Trying PREMIRRORS")
                     mirrors = mirror_from_string(self.d.getVar('PREMIRRORS'))
                     done = m.try_mirrors(self, ud, self.d, mirrors)
                     if done:
@@ -1694,7 +1704,7 @@ class Fetch(object):
                             m.update_donestamp(ud, self.d)
                         except ChecksumError as e:
                             logger.warning("Checksum failure encountered with premirror download of %s - will attempt other sources." % u)
-                            logger.debug(1, str(e))
+                            logger.debug(str(e))
                             done = False
 
                 if premirroronly:
@@ -1706,7 +1716,7 @@ class Fetch(object):
                     try:
                         if not trusted_network(self.d, ud.url):
                             raise UntrustedUrl(ud.url)
-                        logger.debug(1, "Trying Upstream")
+                        logger.debug("Trying Upstream")
                         m.download(ud, self.d)
                         if hasattr(m, "build_mirror_data"):
                             m.build_mirror_data(ud, self.d)
@@ -1721,19 +1731,19 @@ class Fetch(object):
                     except BBFetchException as e:
                         if isinstance(e, ChecksumError):
                             logger.warning("Checksum failure encountered with download of %s - will attempt other sources if available" % u)
-                            logger.debug(1, str(e))
+                            logger.debug(str(e))
                             if os.path.exists(ud.localpath):
                                 rename_bad_checksum(ud, e.checksum)
                         elif isinstance(e, NoChecksumError):
                             raise
                         else:
                             logger.warning('Failed to fetch URL %s, attempting MIRRORS if available' % u)
-                            logger.debug(1, str(e))
+                            logger.debug(str(e))
                         firsterr = e
                         # Remove any incomplete fetch
                         if not verified_stamp:
                             m.clean(ud, self.d)
-                        logger.debug(1, "Trying MIRRORS")
+                        logger.debug("Trying MIRRORS")
                         mirrors = mirror_from_string(self.d.getVar('MIRRORS'))
                         done = m.try_mirrors(self, ud, self.d, mirrors)
 
@@ -1770,7 +1780,7 @@ class Fetch(object):
             ud = self.ud[u]
             ud.setup_localpath(self.d)
             m = ud.method
-            logger.debug(1, "Testing URL %s", u)
+            logger.debug("Testing URL %s", u)
             # First try checking uri, u, from PREMIRRORS
             mirrors = mirror_from_string(self.d.getVar('PREMIRRORS'))
             ret = m.try_mirrors(self, ud, self.d, mirrors, True)
@@ -1904,6 +1914,7 @@ from . import repo
 from . import clearcase
 from . import npm
 from . import npmsw
+from . import az
 
 methods.append(local.Local())
 methods.append(wget.Wget())
@@ -1923,3 +1934,4 @@ methods.append(repo.Repo())
 methods.append(clearcase.ClearCase())
 methods.append(npm.Npm())
 methods.append(npmsw.NpmShrinkWrap())
+methods.append(az.Az())

bitbake/lib/bb/fetch2/az.py

@@ -0,0 +1,93 @@
+"""
+BitBake 'Fetch' Azure Storage implementation
+
+"""
+
+# Copyright (C) 2021 Alejandro Hernandez Samaniego
+#
+# Based on bb.fetch2.wget:
+# Copyright (C) 2003, 2004  Chris Larson
+#
+# SPDX-License-Identifier: GPL-2.0-only
+#
+# Based on functions from the base bb module, Copyright 2003 Holger Schurig
+
+import shlex
+import os
+import bb
+from   bb.fetch2 import FetchError
+from   bb.fetch2 import logger
+from   bb.fetch2.wget import Wget
+
+
+class Az(Wget):
+
+    def supports(self, ud, d):
+        """
+        Check to see if a given url can be fetched from Azure Storage
+        """
+        return ud.type in ['az']
+
+
+    def checkstatus(self, fetch, ud, d, try_again=True):
+
+        # checkstatus discards parameters either way, we need to do this before adding the SAS
+        ud.url = ud.url.replace('az://','https://').split(';')[0]
+
+        az_sas = d.getVar('AZ_SAS')
+        if az_sas and az_sas not in ud.url:
+            ud.url += az_sas
+
+        return Wget.checkstatus(self, fetch, ud, d, try_again)
+
+    # Override download method, include retries
+    def download(self, ud, d, retries=3):
+        """Fetch urls"""
+
+        # If were reaching the account transaction limit we might be refused a connection,
+        # retrying allows us to avoid false negatives since the limit changes over time
+        fetchcmd = self.basecmd + ' --retry-connrefused --waitretry=5'
+
+        # We need to provide a localpath to avoid wget using the SAS
+        # ud.localfile either has the downloadfilename or ud.path
+        localpath = os.path.join(d.getVar("DL_DIR"), ud.localfile)
+        bb.utils.mkdirhier(os.path.dirname(localpath))
+        fetchcmd += " -O %s" % shlex.quote(localpath)
+
+
+        if ud.user and ud.pswd:
+            fetchcmd += " --user=%s --password=%s --auth-no-challenge" % (ud.user, ud.pswd)
+
+        # Check if a Shared Access Signature was given and use it
+        az_sas = d.getVar('AZ_SAS')
+
+        if az_sas:
+            azuri = '%s%s%s%s' % ('https://', ud.host, ud.path, az_sas)
+        else:
+            azuri = '%s%s%s' % ('https://', ud.host, ud.path)
+
+        if os.path.exists(ud.localpath):
+            # file exists, but we didnt complete it.. trying again.
+            fetchcmd += d.expand(" -c -P ${DL_DIR} '%s'" % azuri)
+        else:
+            fetchcmd += d.expand(" -P ${DL_DIR} '%s'" % azuri)
+
+        try:
+            self._runwget(ud, d, fetchcmd, False)
+        except FetchError as e:
+            # Azure fails on handshake sometimes when using wget after some stress, producing a
+            # FetchError from the fetcher, if the artifact exists retyring should succeed
+            if 'Unable to establish SSL connection' in str(e):
+                logger.debug2('Unable to establish SSL connection: Retries remaining: %s, Retrying...' % retries)
+                self.download(ud, d, retries -1)
+
+        # Sanity check since wget can pretend it succeed when it didn't
+        # Also, this used to happen if sourceforge sent us to the mirror page
+        if not os.path.exists(ud.localpath):
+            raise FetchError("The fetch command returned success for url %s but %s doesn't exist?!" % (azuri, ud.localpath), azuri)
+
+        if os.path.getsize(ud.localpath) == 0:
+            os.remove(ud.localpath)
+            raise FetchError("The fetch of %s resulted in a zero size file?! Deleting and failing since this isn't right." % (azuri), azuri)
+
+        return True

bitbake/lib/bb/fetch2/bzr.py

@@ -74,16 +74,16 @@ class Bzr(FetchMethod):
 
         if os.access(os.path.join(ud.pkgdir, os.path.basename(ud.pkgdir), '.bzr'), os.R_OK):
             bzrcmd = self._buildbzrcommand(ud, d, "update")
-            logger.debug(1, "BZR Update %s", ud.url)
+            logger.debug("BZR Update %s", ud.url)
             bb.fetch2.check_network_access(d, bzrcmd, ud.url)
             runfetchcmd(bzrcmd, d, workdir=os.path.join(ud.pkgdir, os.path.basename(ud.path)))
         else:
             bb.utils.remove(os.path.join(ud.pkgdir, os.path.basename(ud.pkgdir)), True)
             bzrcmd = self._buildbzrcommand(ud, d, "fetch")
             bb.fetch2.check_network_access(d, bzrcmd, ud.url)
-            logger.debug(1, "BZR Checkout %s", ud.url)
+            logger.debug("BZR Checkout %s", ud.url)
             bb.utils.mkdirhier(ud.pkgdir)
-            logger.debug(1, "Running %s", bzrcmd)
+            logger.debug("Running %s", bzrcmd)
             runfetchcmd(bzrcmd, d, workdir=ud.pkgdir)
 
         scmdata = ud.parm.get("scmdata", "")
@@ -109,7 +109,7 @@ class Bzr(FetchMethod):
         """
         Return the latest upstream revision number
         """
-        logger.debug(2, "BZR fetcher hitting network for %s", ud.url)
+        logger.debug2("BZR fetcher hitting network for %s", ud.url)
 
         bb.fetch2.check_network_access(d, self._buildbzrcommand(ud, d, "revno"), ud.url)
 

bitbake/lib/bb/fetch2/clearcase.py

@@ -70,7 +70,7 @@ class ClearCase(FetchMethod):
         return ud.type in ['ccrc']
 
     def debug(self, msg):
-        logger.debug(1, "ClearCase: %s", msg)
+        logger.debug("ClearCase: %s", msg)
 
     def urldata_init(self, ud, d):
         """

bitbake/lib/bb/fetch2/cvs.py

@@ -109,7 +109,7 @@ class Cvs(FetchMethod):
             cvsupdatecmd = "CVS_RSH=\"%s\" %s" % (cvs_rsh, cvsupdatecmd)
 
         # create module directory
-        logger.debug(2, "Fetch: checking for module directory")
+        logger.debug2("Fetch: checking for module directory")
         moddir = os.path.join(ud.pkgdir, localdir)
         workdir = None
         if os.access(os.path.join(moddir, 'CVS'), os.R_OK):
@@ -123,7 +123,7 @@ class Cvs(FetchMethod):
             # check out sources there
             bb.utils.mkdirhier(ud.pkgdir)
             workdir = ud.pkgdir
-            logger.debug(1, "Running %s", cvscmd)
+            logger.debug("Running %s", cvscmd)
             bb.fetch2.check_network_access(d, cvscmd, ud.url)
             cmd = cvscmd
 

bitbake/lib/bb/fetch2/git.py

@@ -168,7 +168,11 @@ class Git(FetchMethod):
         if len(branches) != len(ud.names):
             raise bb.fetch2.ParameterError("The number of name and branch parameters is not balanced", ud.url)
 
-        ud.cloneflags = "-s -n"
+        ud.noshared = d.getVar("BB_GIT_NOSHARED") == "1"
+
+        ud.cloneflags = "-n"
+        if not ud.noshared:
+            ud.cloneflags += " -s"
         if ud.bareclone:
             ud.cloneflags += " --mirror"
 
@@ -220,9 +224,14 @@ class Git(FetchMethod):
             ud.shallow = False
 
         if ud.usehead:
-            ud.unresolvedrev['default'] = 'HEAD'
+            # When usehead is set let's associate 'HEAD' with the unresolved
+            # rev of this repository. This will get resolved into a revision
+            # later. If an actual revision happens to have also been provided
+            # then this setting will be overridden.
+            for name in ud.names:
+                ud.unresolvedrev[name] = 'HEAD'
 
-        ud.basecmd = d.getVar("FETCHCMD_git") or "git -c core.fsyncobjectfiles=0"
+        ud.basecmd = d.getVar("FETCHCMD_git") or "git -c core.fsyncobjectfiles=0 -c gc.autoDetach=false"
 
         write_tarballs = d.getVar("BB_GENERATE_MIRROR_TARBALLS") or "0"
         ud.write_tarballs = write_tarballs != "0" or ud.rebaseable
@@ -389,7 +398,7 @@ class Git(FetchMethod):
             tmpdir = tempfile.mkdtemp(dir=d.getVar('DL_DIR'))
             try:
                 # Do the checkout. This implicitly involves a Git LFS fetch.
-                self.unpack(ud, tmpdir, d)
+                Git.unpack(self, ud, tmpdir, d)
 
                 # Scoop up a copy of any stuff that Git LFS downloaded. Merge them into
                 # the bare clonedir.
@@ -633,6 +642,11 @@ class Git(FetchMethod):
         """
         Return the repository URL
         """
+        # Note that we do not support passwords directly in the git urls. There are several
+        # reasons. SRC_URI can be written out to things like buildhistory and people don't
+        # want to leak passwords like that. Its also all too easy to share metadata without 
+        # removing the password. ssh keys, ~/.netrc and ~/.ssh/config files can be used as
+        # alternatives so we will not take patches adding password support here.
         if ud.user:
             username = ud.user + '@'
         else:

bitbake/lib/bb/fetch2/gitsm.py

@@ -78,7 +78,7 @@ class GitSM(Git):
                     module_hash = ""
 
                 if not module_hash:
-                    logger.debug(1, "submodule %s is defined, but is not initialized in the repository. Skipping", m)
+                    logger.debug("submodule %s is defined, but is not initialized in the repository. Skipping", m)
                     continue
 
                 submodules.append(m)
@@ -179,7 +179,7 @@ class GitSM(Git):
                             (ud.basecmd, ud.revisions[ud.names[0]]), d, workdir=ud.clonedir)
 
         if len(need_update_list) > 0:
-            logger.debug(1, 'gitsm: Submodules requiring update: %s' % (' '.join(need_update_list)))
+            logger.debug('gitsm: Submodules requiring update: %s' % (' '.join(need_update_list)))
             return True
 
         return False

bitbake/lib/bb/fetch2/hg.py

@@ -150,7 +150,7 @@ class Hg(FetchMethod):
     def download(self, ud, d):
         """Fetch url"""
 
-        logger.debug(2, "Fetch: checking for module directory '" + ud.moddir + "'")
+        logger.debug2("Fetch: checking for module directory '" + ud.moddir + "'")
 
         # If the checkout doesn't exist and the mirror tarball does, extract it
         if not os.path.exists(ud.pkgdir) and os.path.exists(ud.fullmirror):
@@ -160,7 +160,7 @@ class Hg(FetchMethod):
         if os.access(os.path.join(ud.moddir, '.hg'), os.R_OK):
             # Found the source, check whether need pull
             updatecmd = self._buildhgcommand(ud, d, "update")
-            logger.debug(1, "Running %s", updatecmd)
+            logger.debug("Running %s", updatecmd)
             try:
                 runfetchcmd(updatecmd, d, workdir=ud.moddir)
             except bb.fetch2.FetchError:
@@ -168,7 +168,7 @@ class Hg(FetchMethod):
                 pullcmd = self._buildhgcommand(ud, d, "pull")
                 logger.info("Pulling " + ud.url)
                 # update sources there
-                logger.debug(1, "Running %s", pullcmd)
+                logger.debug("Running %s", pullcmd)
                 bb.fetch2.check_network_access(d, pullcmd, ud.url)
                 runfetchcmd(pullcmd, d, workdir=ud.moddir)
                 try:
@@ -183,14 +183,14 @@ class Hg(FetchMethod):
             logger.info("Fetch " + ud.url)
             # check out sources there
             bb.utils.mkdirhier(ud.pkgdir)
-            logger.debug(1, "Running %s", fetchcmd)
+            logger.debug("Running %s", fetchcmd)
             bb.fetch2.check_network_access(d, fetchcmd, ud.url)
             runfetchcmd(fetchcmd, d, workdir=ud.pkgdir)
 
         # Even when we clone (fetch), we still need to update as hg's clone
         # won't checkout the specified revision if its on a branch
         updatecmd = self._buildhgcommand(ud, d, "update")
-        logger.debug(1, "Running %s", updatecmd)
+        logger.debug("Running %s", updatecmd)
         runfetchcmd(updatecmd, d, workdir=ud.moddir)
 
     def clean(self, ud, d):
@@ -247,9 +247,9 @@ class Hg(FetchMethod):
         if scmdata != "nokeep":
             proto = ud.parm.get('protocol', 'http')
             if not os.access(os.path.join(codir, '.hg'), os.R_OK):
-                logger.debug(2, "Unpack: creating new hg repository in '" + codir + "'")
+                logger.debug2("Unpack: creating new hg repository in '" + codir + "'")
                 runfetchcmd("%s init %s" % (ud.basecmd, codir), d)
-            logger.debug(2, "Unpack: updating source in '" + codir + "'")
+            logger.debug2("Unpack: updating source in '" + codir + "'")
             if ud.user and ud.pswd:
                 runfetchcmd("%s --config auth.default.prefix=* --config auth.default.username=%s --config auth.default.password=%s --config \"auth.default.schemes=%s\" pull %s" % (ud.basecmd, ud.user, ud.pswd, proto, ud.moddir), d, workdir=codir)
             else:
@@ -259,5 +259,5 @@ class Hg(FetchMethod):
             else:
                 runfetchcmd("%s up -C %s" % (ud.basecmd, revflag), d, workdir=codir)
         else:
-            logger.debug(2, "Unpack: extracting source to '" + codir + "'")
+            logger.debug2("Unpack: extracting source to '" + codir + "'")
             runfetchcmd("%s archive -t files %s %s" % (ud.basecmd, revflag, codir), d, workdir=ud.moddir)

bitbake/lib/bb/fetch2/local.py

@@ -54,12 +54,12 @@ class Local(FetchMethod):
             return [path]
         filespath = d.getVar('FILESPATH')
         if filespath:
-            logger.debug(2, "Searching for %s in paths:\n    %s" % (path, "\n    ".join(filespath.split(":"))))
+            logger.debug2("Searching for %s in paths:\n    %s" % (path, "\n    ".join(filespath.split(":"))))
             newpath, hist = bb.utils.which(filespath, path, history=True)
             searched.extend(hist)
         if not os.path.exists(newpath):
             dldirfile = os.path.join(d.getVar("DL_DIR"), path)
-            logger.debug(2, "Defaulting to %s for %s" % (dldirfile, path))
+            logger.debug2("Defaulting to %s for %s" % (dldirfile, path))
             bb.utils.mkdirhier(os.path.dirname(dldirfile))
             searched.append(dldirfile)
             return searched

bitbake/lib/bb/fetch2/osc.py

@@ -84,13 +84,13 @@ class Osc(FetchMethod):
         Fetch url
         """
 
-        logger.debug(2, "Fetch: checking for module directory '" + ud.moddir + "'")
+        logger.debug2("Fetch: checking for module directory '" + ud.moddir + "'")
 
         if os.access(os.path.join(d.getVar('OSCDIR'), ud.path, ud.module), os.R_OK):
             oscupdatecmd = self._buildosccommand(ud, d, "update")
             logger.info("Update "+ ud.url)
             # update sources there
-            logger.debug(1, "Running %s", oscupdatecmd)
+            logger.debug("Running %s", oscupdatecmd)
             bb.fetch2.check_network_access(d, oscupdatecmd, ud.url)
             runfetchcmd(oscupdatecmd, d, workdir=ud.moddir)
         else:
@@ -98,7 +98,7 @@ class Osc(FetchMethod):
             logger.info("Fetch " + ud.url)
             # check out sources there
             bb.utils.mkdirhier(ud.pkgdir)
-            logger.debug(1, "Running %s", oscfetchcmd)
+            logger.debug("Running %s", oscfetchcmd)
             bb.fetch2.check_network_access(d, oscfetchcmd, ud.url)
             runfetchcmd(oscfetchcmd, d, workdir=ud.pkgdir)
 

bitbake/lib/bb/fetch2/perforce.py

@@ -90,16 +90,16 @@ class Perforce(FetchMethod):
         p4port = d.getVar('P4PORT')
 
         if p4port:
-            logger.debug(1, 'Using recipe provided P4PORT: %s' % p4port)
+            logger.debug('Using recipe provided P4PORT: %s' % p4port)
             ud.host = p4port
         else:
-            logger.debug(1, 'Trying to use P4CONFIG to automatically set P4PORT...')
+            logger.debug('Trying to use P4CONFIG to automatically set P4PORT...')
             ud.usingp4config = True
             p4cmd = '%s info | grep "Server address"' % ud.basecmd
             bb.fetch2.check_network_access(d, p4cmd, ud.url)
             ud.host = runfetchcmd(p4cmd, d, True)
             ud.host = ud.host.split(': ')[1].strip()
-            logger.debug(1, 'Determined P4PORT to be: %s' % ud.host)
+            logger.debug('Determined P4PORT to be: %s' % ud.host)
             if not ud.host:
                 raise FetchError('Could not determine P4PORT from P4CONFIG')
 
@@ -119,6 +119,7 @@ class Perforce(FetchMethod):
         cleanedpath = ud.path.replace('/...', '').replace('/', '.')
         cleanedhost = ud.host.replace(':', '.')
 
+        cleanedmodule = ""
         # Merge the path and module into the final depot location
         if ud.module:
             if ud.module.find('/') == 0:
@@ -133,7 +134,7 @@ class Perforce(FetchMethod):
 
         ud.setup_revisions(d)
 
-        ud.localfile = d.expand('%s_%s_%s.tar.gz' % (cleanedhost, cleanedpath, ud.revision))
+        ud.localfile = d.expand('%s_%s_%s_%s.tar.gz' % (cleanedhost, cleanedpath, cleandedmodule, ud.revision))
 
     def _buildp4command(self, ud, d, command, depot_filename=None):
         """
@@ -207,7 +208,7 @@ class Perforce(FetchMethod):
         for filename in p4fileslist:
             item = filename.split(' - ')
             lastaction = item[1].split()
-            logger.debug(1, 'File: %s Last Action: %s' % (item[0], lastaction[0]))
+            logger.debug('File: %s Last Action: %s' % (item[0], lastaction[0]))
             if lastaction[0] == 'delete':
                 continue
             filelist.append(item[0])
@@ -254,7 +255,7 @@ class Perforce(FetchMethod):
             raise FetchError('Could not determine the latest perforce changelist')
 
         tipcset = tip.split(' ')[1]
-        logger.debug(1, 'p4 tip found to be changelist %s' % tipcset)
+        logger.debug('p4 tip found to be changelist %s' % tipcset)
         return tipcset
 
     def sortable_revision(self, ud, d, name):

bitbake/lib/bb/fetch2/repo.py

@@ -47,7 +47,7 @@ class Repo(FetchMethod):
         """Fetch url"""
 
         if os.access(os.path.join(d.getVar("DL_DIR"), ud.localfile), os.R_OK):
-            logger.debug(1, "%s already exists (or was stashed). Skipping repo init / sync.", ud.localpath)
+            logger.debug("%s already exists (or was stashed). Skipping repo init / sync.", ud.localpath)
             return
 
         repodir = d.getVar("REPODIR") or (d.getVar("DL_DIR") + "/repo")

bitbake/lib/bb/fetch2/s3.py

@@ -18,10 +18,47 @@ The aws tool must be correctly installed and configured prior to use.
 import os
 import bb
 import urllib.request, urllib.parse, urllib.error
+import re
 from bb.fetch2 import FetchMethod
 from bb.fetch2 import FetchError
 from bb.fetch2 import runfetchcmd
 
+def convertToBytes(value, unit):
+    value = float(value)
+    if (unit == "KiB"):
+        value = value*1024.0;
+    elif (unit == "MiB"):
+        value = value*1024.0*1024.0;
+    elif (unit == "GiB"):
+        value = value*1024.0*1024.0*1024.0;
+    return value
+
+class S3ProgressHandler(bb.progress.LineFilterProgressHandler):
+    """
+    Extract progress information from s3 cp output, e.g.:
+    Completed 5.1 KiB/8.8 GiB (12.0 MiB/s) with 1 file(s) remaining
+    """
+    def __init__(self, d):
+        super(S3ProgressHandler, self).__init__(d)
+        # Send an initial progress event so the bar gets shown
+        self._fire_progress(0)
+
+    def writeline(self, line):
+        percs = re.findall(r'^Completed (\d+.{0,1}\d*) (\w+)\/(\d+.{0,1}\d*) (\w+) (\(.+\)) with\s+', line)
+        if percs:
+            completed = (percs[-1][0])
+            completedUnit = (percs[-1][1])
+            total = (percs[-1][2])
+            totalUnit = (percs[-1][3])
+            completed = convertToBytes(completed, completedUnit)
+            total = convertToBytes(total, totalUnit)
+            progress = (completed/total)*100.0
+            rate = percs[-1][4]
+            self.update(progress, rate)
+            return False
+        return True
+
+
 class S3(FetchMethod):
     """Class to fetch urls via 'aws s3'"""
 
@@ -52,7 +89,9 @@ class S3(FetchMethod):
 
         cmd = '%s cp s3://%s%s %s' % (ud.basecmd, ud.host, ud.path, ud.localpath)
         bb.fetch2.check_network_access(d, cmd, ud.url)
-        runfetchcmd(cmd, d)
+
+        progresshandler = S3ProgressHandler(d)
+        runfetchcmd(cmd, d, False, log=progresshandler)
 
         # Additional sanity checks copied from the wget class (although there
         # are no known issues which mean these are required, treat the aws cli

bitbake/lib/bb/fetch2/svn.py

@@ -86,7 +86,7 @@ class Svn(FetchMethod):
         if command == "info":
             svncmd = "%s info %s %s://%s/%s/" % (ud.basecmd, " ".join(options), proto, svnroot, ud.module)
         elif command == "log1":
-            svncmd = "%s log --limit 1 %s %s://%s/%s/" % (ud.basecmd, " ".join(options), proto, svnroot, ud.module)
+            svncmd = "%s log --limit 1 --quiet %s %s://%s/%s/" % (ud.basecmd, " ".join(options), proto, svnroot, ud.module)
         else:
             suffix = ""
 
@@ -116,7 +116,7 @@ class Svn(FetchMethod):
     def download(self, ud, d):
         """Fetch url"""
 
-        logger.debug(2, "Fetch: checking for module directory '" + ud.moddir + "'")
+        logger.debug2("Fetch: checking for module directory '" + ud.moddir + "'")
 
         lf = bb.utils.lockfile(ud.svnlock)
 
@@ -129,7 +129,7 @@ class Svn(FetchMethod):
                     runfetchcmd(ud.basecmd + " upgrade", d, workdir=ud.moddir)
                 except FetchError:
                     pass
-                logger.debug(1, "Running %s", svncmd)
+                logger.debug("Running %s", svncmd)
                 bb.fetch2.check_network_access(d, svncmd, ud.url)
                 runfetchcmd(svncmd, d, workdir=ud.moddir)
             else:
@@ -137,7 +137,7 @@ class Svn(FetchMethod):
                 logger.info("Fetch " + ud.url)
                 # check out sources there
                 bb.utils.mkdirhier(ud.pkgdir)
-                logger.debug(1, "Running %s", svncmd)
+                logger.debug("Running %s", svncmd)
                 bb.fetch2.check_network_access(d, svncmd, ud.url)
                 runfetchcmd(svncmd, d, workdir=ud.pkgdir)
 

bitbake/lib/bb/fetch2/wget.py

@@ -52,6 +52,12 @@ class WgetProgressHandler(bb.progress.LineFilterProgressHandler):
 
 
 class Wget(FetchMethod):
+
+    # CDNs like CloudFlare may do a 'browser integrity test' which can fail
+    # with the standard wget/urllib User-Agent, so pretend to be a modern
+    # browser.
+    user_agent = "Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:84.0) Gecko/20100101 Firefox/84.0"
+
     """Class to fetch urls via 'wget'"""
     def supports(self, ud, d):
         """
@@ -82,7 +88,7 @@ class Wget(FetchMethod):
 
         progresshandler = WgetProgressHandler(d)
 
-        logger.debug(2, "Fetching %s using command '%s'" % (ud.url, command))
+        logger.debug2("Fetching %s using command '%s'" % (ud.url, command))
         bb.fetch2.check_network_access(d, command, ud.url)
         runfetchcmd(command + ' --progress=dot -v', d, quiet, log=progresshandler, workdir=workdir)
 
@@ -297,7 +303,7 @@ class Wget(FetchMethod):
             # Some servers (FusionForge, as used on Alioth) require that the
             # optional Accept header is set.
             r.add_header("Accept", "*/*")
-            r.add_header("User-Agent", "Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.9.2.12) Gecko/20101027 Ubuntu/9.10 (karmic) Firefox/3.6.12")
+            r.add_header("User-Agent", self.user_agent)
             def add_basic_auth(login_str, request):
                 '''Adds Basic auth to http request, pass in login:password as string'''
                 import base64
@@ -320,11 +326,19 @@ class Wget(FetchMethod):
                 pass
         except urllib.error.URLError as e:
             if try_again:
-                logger.debug(2, "checkstatus: trying again")
+                logger.debug2("checkstatus: trying again")
                 return self.checkstatus(fetch, ud, d, False)
             else:
                 # debug for now to avoid spamming the logs in e.g. remote sstate searches
-                logger.debug(2, "checkstatus() urlopen failed: %s" % e)
+                logger.debug2("checkstatus() urlopen failed: %s" % e)
+                return False
+        except ConnectionResetError as e:
+            if try_again:
+                logger.debug2("checkstatus: trying again")
+                return self.checkstatus(fetch, ud, d, False)
+            else:
+                # debug for now to avoid spamming the logs in e.g. remote sstate searches
+                logger.debug2("checkstatus() urlopen failed: %s" % e)
                 return False
         return True
 
@@ -401,9 +415,8 @@ class Wget(FetchMethod):
         """
         f = tempfile.NamedTemporaryFile()
         with tempfile.TemporaryDirectory(prefix="wget-index-") as workdir, tempfile.NamedTemporaryFile(dir=workdir, prefix="wget-listing-") as f:
-            agent = "Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.9.2.12) Gecko/20101027 Ubuntu/9.10 (karmic) Firefox/3.6.12"
             fetchcmd = self.basecmd
-            fetchcmd += " -O " + f.name + " --user-agent='" + agent + "' '" + uri + "'"
+            fetchcmd += " -O " + f.name + " --user-agent='" + self.user_agent + "' '" + uri + "'"
             try:
                 self._runwget(ud, d, fetchcmd, True, workdir=workdir)
                 fetchresult = f.read()
@@ -459,7 +472,7 @@ class Wget(FetchMethod):
         version_dir = ['', '', '']
         version = ['', '', '']
 
-        dirver_regex = re.compile(r"(?P<pfx>\D*)(?P<ver>(\d+[\.\-_])+(\d+))")
+        dirver_regex = re.compile(r"(?P<pfx>\D*)(?P<ver>(\d+[\.\-_])*(\d+))")
         s = dirver_regex.search(dirver)
         if s:
             version_dir[1] = s.group('ver')

bitbake/lib/bb/main.py

@@ -119,178 +119,181 @@ warnings.filterwarnings("ignore", category=ImportWarning)
 warnings.filterwarnings("ignore", category=DeprecationWarning, module="<string>$")
 warnings.filterwarnings("ignore", message="With-statements now directly support multiple context managers")
 
-class BitBakeConfigParameters(cookerdata.ConfigParameters):
 
-    def parseCommandLine(self, argv=sys.argv):
-        parser = optparse.OptionParser(
-            formatter=BitbakeHelpFormatter(),
-            version="BitBake Build Tool Core version %s" % bb.__version__,
-            usage="""%prog [options] [recipename/target recipe:do_task ...]
+def create_bitbake_parser():
+    parser = optparse.OptionParser(
+        formatter=BitbakeHelpFormatter(),
+        version="BitBake Build Tool Core version %s" % bb.__version__,
+        usage="""%prog [options] [recipename/target recipe:do_task ...]
 
     Executes the specified task (default is 'build') for a given set of target recipes (.bb files).
     It is assumed there is a conf/bblayers.conf available in cwd or in BBPATH which
     will provide the layer, BBFILES and other configuration information.""")
 
-        parser.add_option("-b", "--buildfile", action="store", dest="buildfile", default=None,
-                          help="Execute tasks from a specific .bb recipe directly. WARNING: Does "
-                               "not handle any dependencies from other recipes.")
+    parser.add_option("-b", "--buildfile", action="store", dest="buildfile", default=None,
+                      help="Execute tasks from a specific .bb recipe directly. WARNING: Does "
+                           "not handle any dependencies from other recipes.")
 
-        parser.add_option("-k", "--continue", action="store_false", dest="abort", default=True,
-                          help="Continue as much as possible after an error. While the target that "
-                               "failed and anything depending on it cannot be built, as much as "
-                               "possible will be built before stopping.")
+    parser.add_option("-k", "--continue", action="store_false", dest="abort", default=True,
+                      help="Continue as much as possible after an error. While the target that "
+                           "failed and anything depending on it cannot be built, as much as "
+                           "possible will be built before stopping.")
 
-        parser.add_option("-f", "--force", action="store_true", dest="force", default=False,
-                          help="Force the specified targets/task to run (invalidating any "
-                               "existing stamp file).")
+    parser.add_option("-f", "--force", action="store_true", dest="force", default=False,
+                      help="Force the specified targets/task to run (invalidating any "
+                           "existing stamp file).")
 
-        parser.add_option("-c", "--cmd", action="store", dest="cmd",
-                          help="Specify the task to execute. The exact options available "
-                               "depend on the metadata. Some examples might be 'compile'"
-                               " or 'populate_sysroot' or 'listtasks' may give a list of "
-                               "the tasks available.")
+    parser.add_option("-c", "--cmd", action="store", dest="cmd",
+                      help="Specify the task to execute. The exact options available "
+                           "depend on the metadata. Some examples might be 'compile'"
+                           " or 'populate_sysroot' or 'listtasks' may give a list of "
+                           "the tasks available.")
 
-        parser.add_option("-C", "--clear-stamp", action="store", dest="invalidate_stamp",
-                          help="Invalidate the stamp for the specified task such as 'compile' "
-                               "and then run the default task for the specified target(s).")
+    parser.add_option("-C", "--clear-stamp", action="store", dest="invalidate_stamp",
+                      help="Invalidate the stamp for the specified task such as 'compile' "
+                           "and then run the default task for the specified target(s).")
 
-        parser.add_option("-r", "--read", action="append", dest="prefile", default=[],
-                          help="Read the specified file before bitbake.conf.")
+    parser.add_option("-r", "--read", action="append", dest="prefile", default=[],
+                      help="Read the specified file before bitbake.conf.")
 
-        parser.add_option("-R", "--postread", action="append", dest="postfile", default=[],
-                          help="Read the specified file after bitbake.conf.")
+    parser.add_option("-R", "--postread", action="append", dest="postfile", default=[],
+                      help="Read the specified file after bitbake.conf.")
 
-        parser.add_option("-v", "--verbose", action="store_true", dest="verbose", default=False,
-                          help="Enable tracing of shell tasks (with 'set -x'). "
-                               "Also print bb.note(...) messages to stdout (in "
-                               "addition to writing them to ${T}/log.do_<task>).")
+    parser.add_option("-v", "--verbose", action="store_true", dest="verbose", default=False,
+                      help="Enable tracing of shell tasks (with 'set -x'). "
+                           "Also print bb.note(...) messages to stdout (in "
+                           "addition to writing them to ${T}/log.do_<task>).")
 
-        parser.add_option("-D", "--debug", action="count", dest="debug", default=0,
-                          help="Increase the debug level. You can specify this "
-                               "more than once. -D sets the debug level to 1, "
-                               "where only bb.debug(1, ...) messages are printed "
-                               "to stdout; -DD sets the debug level to 2, where "
-                               "both bb.debug(1, ...) and bb.debug(2, ...) "
-                               "messages are printed; etc. Without -D, no debug "
-                               "messages are printed. Note that -D only affects "
-                               "output to stdout. All debug messages are written "
-                               "to ${T}/log.do_taskname, regardless of the debug "
-                               "level.")
+    parser.add_option("-D", "--debug", action="count", dest="debug", default=0,
+                      help="Increase the debug level. You can specify this "
+                           "more than once. -D sets the debug level to 1, "
+                           "where only bb.debug(1, ...) messages are printed "
+                           "to stdout; -DD sets the debug level to 2, where "
+                           "both bb.debug(1, ...) and bb.debug(2, ...) "
+                           "messages are printed; etc. Without -D, no debug "
+                           "messages are printed. Note that -D only affects "
+                           "output to stdout. All debug messages are written "
+                           "to ${T}/log.do_taskname, regardless of the debug "
+                           "level.")
 
-        parser.add_option("-q", "--quiet", action="count", dest="quiet", default=0,
-                          help="Output less log message data to the terminal. You can specify this more than once.")
+    parser.add_option("-q", "--quiet", action="count", dest="quiet", default=0,
+                      help="Output less log message data to the terminal. You can specify this more than once.")
 
-        parser.add_option("-n", "--dry-run", action="store_true", dest="dry_run", default=False,
-                          help="Don't execute, just go through the motions.")
+    parser.add_option("-n", "--dry-run", action="store_true", dest="dry_run", default=False,
+                      help="Don't execute, just go through the motions.")
 
-        parser.add_option("-S", "--dump-signatures", action="append", dest="dump_signatures",
-                          default=[], metavar="SIGNATURE_HANDLER",
-                          help="Dump out the signature construction information, with no task "
-                               "execution. The SIGNATURE_HANDLER parameter is passed to the "
-                               "handler. Two common values are none and printdiff but the handler "
-                               "may define more/less. none means only dump the signature, printdiff"
-                               " means compare the dumped signature with the cached one.")
+    parser.add_option("-S", "--dump-signatures", action="append", dest="dump_signatures",
+                      default=[], metavar="SIGNATURE_HANDLER",
+                      help="Dump out the signature construction information, with no task "
+                           "execution. The SIGNATURE_HANDLER parameter is passed to the "
+                           "handler. Two common values are none and printdiff but the handler "
+                           "may define more/less. none means only dump the signature, printdiff"
+                           " means compare the dumped signature with the cached one.")
 
-        parser.add_option("-p", "--parse-only", action="store_true",
-                          dest="parse_only", default=False,
-                          help="Quit after parsing the BB recipes.")
+    parser.add_option("-p", "--parse-only", action="store_true",
+                      dest="parse_only", default=False,
+                      help="Quit after parsing the BB recipes.")
 
-        parser.add_option("-s", "--show-versions", action="store_true",
-                          dest="show_versions", default=False,
-                          help="Show current and preferred versions of all recipes.")
+    parser.add_option("-s", "--show-versions", action="store_true",
+                      dest="show_versions", default=False,
+                      help="Show current and preferred versions of all recipes.")
 
-        parser.add_option("-e", "--environment", action="store_true",
-                          dest="show_environment", default=False,
-                          help="Show the global or per-recipe environment complete with information"
-                               " about where variables were set/changed.")
+    parser.add_option("-e", "--environment", action="store_true",
+                      dest="show_environment", default=False,
+                      help="Show the global or per-recipe environment complete with information"
+                           " about where variables were set/changed.")
 
-        parser.add_option("-g", "--graphviz", action="store_true", dest="dot_graph", default=False,
-                          help="Save dependency tree information for the specified "
-                               "targets in the dot syntax.")
+    parser.add_option("-g", "--graphviz", action="store_true", dest="dot_graph", default=False,
+                      help="Save dependency tree information for the specified "
+                           "targets in the dot syntax.")
 
-        parser.add_option("-I", "--ignore-deps", action="append",
-                          dest="extra_assume_provided", default=[],
-                          help="Assume these dependencies don't exist and are already provided "
-                               "(equivalent to ASSUME_PROVIDED). Useful to make dependency "
-                               "graphs more appealing")
+    parser.add_option("-I", "--ignore-deps", action="append",
+                      dest="extra_assume_provided", default=[],
+                      help="Assume these dependencies don't exist and are already provided "
+                           "(equivalent to ASSUME_PROVIDED). Useful to make dependency "
+                           "graphs more appealing")
 
-        parser.add_option("-l", "--log-domains", action="append", dest="debug_domains", default=[],
-                          help="Show debug logging for the specified logging domains")
+    parser.add_option("-l", "--log-domains", action="append", dest="debug_domains", default=[],
+                      help="Show debug logging for the specified logging domains")
 
-        parser.add_option("-P", "--profile", action="store_true", dest="profile", default=False,
-                          help="Profile the command and save reports.")
+    parser.add_option("-P", "--profile", action="store_true", dest="profile", default=False,
+                      help="Profile the command and save reports.")
 
-        # @CHOICES@ is substituted out by BitbakeHelpFormatter above
-        parser.add_option("-u", "--ui", action="store", dest="ui",
-                          default=os.environ.get('BITBAKE_UI', 'knotty'),
-                          help="The user interface to use (@CHOICES@ - default %default).")
+    # @CHOICES@ is substituted out by BitbakeHelpFormatter above
+    parser.add_option("-u", "--ui", action="store", dest="ui",
+                      default=os.environ.get('BITBAKE_UI', 'knotty'),
+                      help="The user interface to use (@CHOICES@ - default %default).")
 
-        parser.add_option("", "--token", action="store", dest="xmlrpctoken",
-                          default=os.environ.get("BBTOKEN"),
-                          help="Specify the connection token to be used when connecting "
-                               "to a remote server.")
+    parser.add_option("", "--token", action="store", dest="xmlrpctoken",
+                      default=os.environ.get("BBTOKEN"),
+                      help="Specify the connection token to be used when connecting "
+                           "to a remote server.")
 
-        parser.add_option("", "--revisions-changed", action="store_true",
-                          dest="revisions_changed", default=False,
-                          help="Set the exit code depending on whether upstream floating "
-                               "revisions have changed or not.")
+    parser.add_option("", "--revisions-changed", action="store_true",
+                      dest="revisions_changed", default=False,
+                      help="Set the exit code depending on whether upstream floating "
+                           "revisions have changed or not.")
 
-        parser.add_option("", "--server-only", action="store_true",
-                          dest="server_only", default=False,
-                          help="Run bitbake without a UI, only starting a server "
-                               "(cooker) process.")
+    parser.add_option("", "--server-only", action="store_true",
+                      dest="server_only", default=False,
+                      help="Run bitbake without a UI, only starting a server "
+                           "(cooker) process.")
 
-        parser.add_option("-B", "--bind", action="store", dest="bind", default=False,
-                          help="The name/address for the bitbake xmlrpc server to bind to.")
+    parser.add_option("-B", "--bind", action="store", dest="bind", default=False,
+                      help="The name/address for the bitbake xmlrpc server to bind to.")
 
-        parser.add_option("-T", "--idle-timeout", type=float, dest="server_timeout",
-                          default=os.getenv("BB_SERVER_TIMEOUT"),
-                          help="Set timeout to unload bitbake server due to inactivity, "
-                                "set to -1 means no unload, "
-                                "default: Environment variable BB_SERVER_TIMEOUT.")
+    parser.add_option("-T", "--idle-timeout", type=float, dest="server_timeout",
+                      default=os.getenv("BB_SERVER_TIMEOUT"),
+                      help="Set timeout to unload bitbake server due to inactivity, "
+                           "set to -1 means no unload, "
+                           "default: Environment variable BB_SERVER_TIMEOUT.")
 
-        parser.add_option("", "--no-setscene", action="store_true",
-                          dest="nosetscene", default=False,
-                          help="Do not run any setscene tasks. sstate will be ignored and "
-                               "everything needed, built.")
+    parser.add_option("", "--no-setscene", action="store_true",
+                      dest="nosetscene", default=False,
+                      help="Do not run any setscene tasks. sstate will be ignored and "
+                           "everything needed, built.")
 
-        parser.add_option("", "--skip-setscene", action="store_true",
-                          dest="skipsetscene", default=False,
-                          help="Skip setscene tasks if they would be executed. Tasks previously "
-                               "restored from sstate will be kept, unlike --no-setscene")
+    parser.add_option("", "--skip-setscene", action="store_true",
+                      dest="skipsetscene", default=False,
+                      help="Skip setscene tasks if they would be executed. Tasks previously "
+                           "restored from sstate will be kept, unlike --no-setscene")
 
-        parser.add_option("", "--setscene-only", action="store_true",
-                          dest="setsceneonly", default=False,
-                          help="Only run setscene tasks, don't run any real tasks.")
+    parser.add_option("", "--setscene-only", action="store_true",
+                      dest="setsceneonly", default=False,
+                      help="Only run setscene tasks, don't run any real tasks.")
 
-        parser.add_option("", "--remote-server", action="store", dest="remote_server",
-                          default=os.environ.get("BBSERVER"),
-                          help="Connect to the specified server.")
+    parser.add_option("", "--remote-server", action="store", dest="remote_server",
+                      default=os.environ.get("BBSERVER"),
+                      help="Connect to the specified server.")
 
-        parser.add_option("-m", "--kill-server", action="store_true",
-                          dest="kill_server", default=False,
-                          help="Terminate any running bitbake server.")
+    parser.add_option("-m", "--kill-server", action="store_true",
+                      dest="kill_server", default=False,
+                      help="Terminate any running bitbake server.")
 
-        parser.add_option("", "--observe-only", action="store_true",
-                          dest="observe_only", default=False,
-                          help="Connect to a server as an observing-only client.")
+    parser.add_option("", "--observe-only", action="store_true",
+                      dest="observe_only", default=False,
+                      help="Connect to a server as an observing-only client.")
 
-        parser.add_option("", "--status-only", action="store_true",
-                          dest="status_only", default=False,
-                          help="Check the status of the remote bitbake server.")
+    parser.add_option("", "--status-only", action="store_true",
+                      dest="status_only", default=False,
+                      help="Check the status of the remote bitbake server.")
 
-        parser.add_option("-w", "--write-log", action="store", dest="writeeventlog",
-                          default=os.environ.get("BBEVENTLOG"),
-                          help="Writes the event log of the build to a bitbake event json file. "
-                               "Use '' (empty string) to assign the name automatically.")
+    parser.add_option("-w", "--write-log", action="store", dest="writeeventlog",
+                      default=os.environ.get("BBEVENTLOG"),
+                      help="Writes the event log of the build to a bitbake event json file. "
+                           "Use '' (empty string) to assign the name automatically.")
 
-        parser.add_option("", "--runall", action="append", dest="runall",
-                          help="Run the specified task for any recipe in the taskgraph of the specified target (even if it wouldn't otherwise have run).")
+    parser.add_option("", "--runall", action="append", dest="runall",
+                      help="Run the specified task for any recipe in the taskgraph of the specified target (even if it wouldn't otherwise have run).")
 
-        parser.add_option("", "--runonly", action="append", dest="runonly",
-                          help="Run only the specified task within the taskgraph of the specified targets (and any task dependencies those tasks may have).")
+    parser.add_option("", "--runonly", action="append", dest="runonly",
+                      help="Run only the specified task within the taskgraph of the specified targets (and any task dependencies those tasks may have).")
+    return parser
 
 
+class BitBakeConfigParameters(cookerdata.ConfigParameters):
+    def parseCommandLine(self, argv=sys.argv):
+        parser = create_bitbake_parser()
         options, targets = parser.parse_args(argv)
 
         if options.quiet and options.verbose:
@@ -466,7 +469,7 @@ def setup_bitbake(configParams, extrafeatures=None):
                     logger.info("Retrying server connection (#%d)..." % tryno)
                 else:
                     logger.info("Retrying server connection (#%d)... (%s)" % (tryno, traceback.format_exc()))
-                
+
             if not retries:
                 bb.fatal("Unable to connect to bitbake server, or start one (server startup failures would be in bitbake-cookerdaemon.log).")
             bb.event.print_ui_queue()

bitbake/lib/bb/monitordisk.py

@@ -59,7 +59,7 @@ def getMountedDev(path):
         pass
     return None
 
-def getDiskData(BBDirs, configuration):
+def getDiskData(BBDirs):
 
     """Prepare disk data for disk space monitor"""
 
@@ -168,7 +168,7 @@ class diskMonitor:
 
         BBDirs = configuration.getVar("BB_DISKMON_DIRS") or None
         if BBDirs:
-            self.devDict = getDiskData(BBDirs, configuration)
+            self.devDict = getDiskData(BBDirs)
             if self.devDict:
                 self.spaceInterval, self.inodeInterval = getInterval(configuration)
                 if self.spaceInterval and self.inodeInterval:

bitbake/lib/bb/msg.py

@@ -278,7 +278,7 @@ def setLoggingConfig(defaultconfig, userconfigfile=None):
         with open(os.path.normpath(userconfigfile), 'r') as f:
             if userconfigfile.endswith('.yml') or userconfigfile.endswith('.yaml'):
                 import yaml
-                userconfig = yaml.load(f)
+                userconfig = yaml.safe_load(f)
             elif userconfigfile.endswith('.json') or userconfigfile.endswith('.cfg'):
                 import json
                 userconfig = json.load(f)

bitbake/lib/bb/parse/__init__.py

@@ -71,7 +71,7 @@ def update_mtime(f):
 
 def update_cache(f):
     if f in __mtime_cache:
-        logger.debug(1, "Updating mtime cache for %s" % f)
+        logger.debug("Updating mtime cache for %s" % f)
         update_mtime(f)
 
 def clear_cache():

bitbake/lib/bb/parse/ast.py

@@ -34,7 +34,7 @@ class IncludeNode(AstNode):
         Include the file and evaluate the statements
         """
         s = data.expand(self.what_file)
-        logger.debug(2, "CONF %s:%s: including %s", self.filename, self.lineno, s)
+        logger.debug2("CONF %s:%s: including %s", self.filename, self.lineno, s)
 
         # TODO: Cache those includes... maybe not here though
         if self.force:
@@ -97,6 +97,7 @@ class DataNode(AstNode):
     def eval(self, data):
         groupd = self.groupd
         key = groupd["var"]
+        key = key.replace(":", "_")
         loginfo = {
             'variable': key,
             'file': self.filename,
@@ -207,6 +208,7 @@ class ExportFuncsNode(AstNode):
     def eval(self, data):
 
         for func in self.n:
+            func = func.replace(":", "_")
             calledfunc = self.classname + "_" + func
 
             if data.getVar(func, False) and not data.getVarFlag(func, 'export_func', False):
@@ -335,7 +337,7 @@ def finalize(fn, d, variant = None):
             if not handlerfn:
                 bb.fatal("Undefined event handler function '%s'" % var)
             handlerln = int(d.getVarFlag(var, "lineno", False))
-            bb.event.register(var, d.getVar(var, False), (d.getVarFlag(var, "eventmask") or "").split(), handlerfn, handlerln)
+            bb.event.register(var, d.getVar(var, False), (d.getVarFlag(var, "eventmask") or "").split(), handlerfn, handlerln, data=d)
 
         bb.event.fire(bb.event.RecipePreFinalise(fn), d)
 
@@ -376,7 +378,7 @@ def _create_variants(datastores, names, function, onlyfinalise):
 def multi_finalize(fn, d):
     appends = (d.getVar("__BBAPPEND") or "").split()
     for append in appends:
-        logger.debug(1, "Appending .bbappend file %s to %s", append, fn)
+        logger.debug("Appending .bbappend file %s to %s", append, fn)
         bb.parse.BBHandler.handle(append, d, True)
 
     onlyfinalise = d.getVar("__ONLYFINALISE", False)

bitbake/lib/bb/parse/parse_py/BBHandler.py

@@ -13,7 +13,7 @@
 #
 
 import re, bb, os
-import bb.build, bb.utils
+import bb.build, bb.utils, bb.data_smart
 
 from . import ConfHandler
 from .. import resolve_file, ast, logger, ParseError
@@ -22,7 +22,7 @@ from .ConfHandler import include, init
 # For compatibility
 bb.deprecate_import(__name__, "bb.parse", ["vars_from_file"])
 
-__func_start_regexp__    = re.compile(r"(((?P<py>python)|(?P<fr>fakeroot))\s*)*(?P<func>[\w\.\-\+\{\}\$]+)?\s*\(\s*\)\s*{$" )
+__func_start_regexp__    = re.compile(r"(((?P<py>python(?=(\s|\()))|(?P<fr>fakeroot(?=\s)))\s*)*(?P<func>[\w\.\-\+\{\}\$:]+)?\s*\(\s*\)\s*{$" )
 __inherit_regexp__       = re.compile(r"inherit\s+(.+)" )
 __export_func_regexp__   = re.compile(r"EXPORT_FUNCTIONS\s+(.+)" )
 __addtask_regexp__       = re.compile(r"addtask\s+(?P<func>\w+)\s*((before\s*(?P<before>((.*(?=after))|(.*))))|(after\s*(?P<after>((.*(?=before))|(.*)))))*")
@@ -60,7 +60,7 @@ def inherit(files, fn, lineno, d):
                 file = abs_fn
 
         if not file in __inherit_cache:
-            logger.debug(1, "Inheriting %s (from %s:%d)" % (file, fn, lineno))
+            logger.debug("Inheriting %s (from %s:%d)" % (file, fn, lineno))
             __inherit_cache.append( file )
             d.setVar('__inherit_cache', __inherit_cache)
             include(fn, file, lineno, d, "inherit")
@@ -233,6 +233,10 @@ def feeder(lineno, s, fn, root, statements, eof=False):
             if taskexpression.count(word) > 1:
                 logger.warning("addtask contained multiple '%s' keywords, only one is supported" % word)
 
+        # Check and warn for having task with exprssion as part of task name
+        for te in taskexpression:
+            if any( ( "%s_" % keyword ) in te for keyword in bb.data_smart.__setvar_keyword__ ):
+                raise ParseError("Task name '%s' contains a keyword which is not recommended/supported.\nPlease rename the task not to include the keyword.\n%s" % (te, ("\n".join(map(str, bb.data_smart.__setvar_keyword__)))), fn)
         ast.handleAddTask(statements, fn, lineno, m)
         return
 

bitbake/lib/bb/parse/parse_py/ConfHandler.py

@@ -20,7 +20,7 @@ from bb.parse import ParseError, resolve_file, ast, logger, handle
 __config_regexp__  = re.compile( r"""
     ^
     (?P<exp>export\s+)?
-    (?P<var>[a-zA-Z0-9\-_+.${}/~]+?)
+    (?P<var>[a-zA-Z0-9\-_+.${}/~:]+?)
     (\[(?P<flag>[a-zA-Z0-9\-_+.]+)\])?
 
     \s* (
@@ -95,7 +95,7 @@ def include_single_file(parentfn, fn, lineno, data, error_out):
         if exc.errno == errno.ENOENT:
             if error_out:
                 raise ParseError("Could not %s file %s" % (error_out, fn), parentfn, lineno)
-            logger.debug(2, "CONF file '%s' not found", fn)
+            logger.debug2("CONF file '%s' not found", fn)
         else:
             if error_out:
                 raise ParseError("Could not %s file %s: %s" % (error_out, fn, exc.strerror), parentfn, lineno)

bitbake/lib/bb/persist_data.py

@@ -248,7 +248,7 @@ class PersistData(object):
                       stacklevel=2)
 
         self.data = persist(d)
-        logger.debug(1, "Using '%s' as the persistent data cache",
+        logger.debug("Using '%s' as the persistent data cache",
                      self.data.filename)
 
     def addDomain(self, domain):

bitbake/lib/bb/progress.py

@@ -94,12 +94,15 @@ class LineFilterProgressHandler(ProgressHandler):
         while True:
             breakpos = self._linebuffer.find('\n') + 1
             if breakpos == 0:
-                break
+                # for the case when the line with progress ends with only '\r'
+                breakpos = self._linebuffer.find('\r') + 1
+                if breakpos == 0:
+                    break
             line = self._linebuffer[:breakpos]
             self._linebuffer = self._linebuffer[breakpos:]
             # Drop any line feeds and anything that precedes them
             lbreakpos = line.rfind('\r') + 1
-            if lbreakpos:
+            if lbreakpos and lbreakpos != breakpos:
                 line = line[lbreakpos:]
             if self.writeline(filter_color(line)):
                 super().write(line)

bitbake/lib/bb/providers.py

@@ -38,16 +38,17 @@ def findProviders(cfgData, dataCache, pkg_pn = None):
     localdata = data.createCopy(cfgData)
     bb.data.expandKeys(localdata)
 
+    required = {}
     preferred_versions = {}
     latest_versions = {}
 
     for pn in pkg_pn:
-        (last_ver, last_file, pref_ver, pref_file) = findBestProvider(pn, localdata, dataCache, pkg_pn)
+        (last_ver, last_file, pref_ver, pref_file, req) = findBestProvider(pn, localdata, dataCache, pkg_pn)
         preferred_versions[pn] = (pref_ver, pref_file)
         latest_versions[pn] = (last_ver, last_file)
+        required[pn] = req
 
-    return (latest_versions, preferred_versions)
-
+    return (latest_versions, preferred_versions, required)
 
 def allProviders(dataCache):
     """
@@ -59,7 +60,6 @@ def allProviders(dataCache):
         all_providers[pn].append((ver, fn))
     return all_providers
 
-
 def sortPriorities(pn, dataCache, pkg_pn = None):
     """
     Reorder pkg_pn by file priority and default preference
@@ -87,6 +87,21 @@ def sortPriorities(pn, dataCache, pkg_pn = None):
 
     return tmp_pn
 
+def versionVariableMatch(cfgData, keyword, pn):
+    """
+    Return the value of the <keyword>_VERSION variable if set.
+    """
+
+    # pn can contain '_', e.g. gcc-cross-x86_64 and an override cannot
+    # hence we do this manually rather than use OVERRIDES
+    ver = cfgData.getVar("%s_VERSION_pn-%s" % (keyword, pn))
+    if not ver:
+        ver = cfgData.getVar("%s_VERSION_%s" % (keyword, pn))
+    if not ver:
+        ver = cfgData.getVar("%s_VERSION" % keyword)
+
+    return ver
+
 def preferredVersionMatch(pe, pv, pr, preferred_e, preferred_v, preferred_r):
     """
     Check if the version pe,pv,pr is the preferred one.
@@ -102,19 +117,28 @@ def preferredVersionMatch(pe, pv, pr, preferred_e, preferred_v, preferred_r):
 
 def findPreferredProvider(pn, cfgData, dataCache, pkg_pn = None, item = None):
     """
-    Find the first provider in pkg_pn with a PREFERRED_VERSION set.
+    Find the first provider in pkg_pn with REQUIRED_VERSION or PREFERRED_VERSION set.
     """
 
     preferred_file = None
     preferred_ver = None
+    required = False
 
-    # pn can contain '_', e.g. gcc-cross-x86_64 and an override cannot
-    # hence we do this manually rather than use OVERRIDES
-    preferred_v = cfgData.getVar("PREFERRED_VERSION_pn-%s" % pn)
-    if not preferred_v:
-        preferred_v = cfgData.getVar("PREFERRED_VERSION_%s" % pn)
-    if not preferred_v:
-        preferred_v = cfgData.getVar("PREFERRED_VERSION")
+    required_v = versionVariableMatch(cfgData, "REQUIRED", pn)
+    preferred_v = versionVariableMatch(cfgData, "PREFERRED", pn)
+
+    itemstr = ""
+    if item:
+        itemstr = " (for item %s)" % item
+
+    if required_v is not None:
+        if preferred_v is not None:
+            logger.warning("REQUIRED_VERSION and PREFERRED_VERSION for package %s%s are both set using REQUIRED_VERSION %s", pn, itemstr, required_v)
+        else:
+            logger.debug("REQUIRED_VERSION is set for package %s%s", pn, itemstr)
+        # REQUIRED_VERSION always takes precedence over PREFERRED_VERSION
+        preferred_v = required_v
+        required = True
 
     if preferred_v:
         m = re.match(r'(\d+:)*(.*)(_.*)*', preferred_v)
@@ -147,11 +171,9 @@ def findPreferredProvider(pn, cfgData, dataCache, pkg_pn = None, item = None):
             pv_str = preferred_v
         if not (preferred_e is None):
             pv_str = '%s:%s' % (preferred_e, pv_str)
-        itemstr = ""
-        if item:
-            itemstr = " (for item %s)" % item
         if preferred_file is None:
-            logger.info("preferred version %s of %s not available%s", pv_str, pn, itemstr)
+            if not required:
+                logger.warning("preferred version %s of %s not available%s", pv_str, pn, itemstr)
             available_vers = []
             for file_set in pkg_pn:
                 for f in file_set:
@@ -163,12 +185,16 @@ def findPreferredProvider(pn, cfgData, dataCache, pkg_pn = None, item = None):
                         available_vers.append(ver_str)
             if available_vers:
                 available_vers.sort()
-                logger.info("versions of %s available: %s", pn, ' '.join(available_vers))
+                logger.warning("versions of %s available: %s", pn, ' '.join(available_vers))
+            if required:
+                logger.error("required version %s of %s not available%s", pv_str, pn, itemstr)
         else:
-            logger.debug(1, "selecting %s as PREFERRED_VERSION %s of package %s%s", preferred_file, pv_str, pn, itemstr)
-
-    return (preferred_ver, preferred_file)
+            if required:
+                logger.debug("selecting %s as REQUIRED_VERSION %s of package %s%s", preferred_file, pv_str, pn, itemstr)
+            else:
+                logger.debug("selecting %s as PREFERRED_VERSION %s of package %s%s", preferred_file, pv_str, pn, itemstr)
 
+    return (preferred_ver, preferred_file, required)
 
 def findLatestProvider(pn, cfgData, dataCache, file_set):
     """
@@ -189,7 +215,6 @@ def findLatestProvider(pn, cfgData, dataCache, file_set):
 
     return (latest, latest_f)
 
-
 def findBestProvider(pn, cfgData, dataCache, pkg_pn = None, item = None):
     """
     If there is a PREFERRED_VERSION, find the highest-priority bbfile
@@ -198,17 +223,16 @@ def findBestProvider(pn, cfgData, dataCache, pkg_pn = None, item = None):
     """
 
     sortpkg_pn = sortPriorities(pn, dataCache, pkg_pn)
-    # Find the highest priority provider with a PREFERRED_VERSION set
-    (preferred_ver, preferred_file) = findPreferredProvider(pn, cfgData, dataCache, sortpkg_pn, item)
+    # Find the highest priority provider with a REQUIRED_VERSION or PREFERRED_VERSION set
+    (preferred_ver, preferred_file, required) = findPreferredProvider(pn, cfgData, dataCache, sortpkg_pn, item)
     # Find the latest version of the highest priority provider
     (latest, latest_f) = findLatestProvider(pn, cfgData, dataCache, sortpkg_pn[0])
 
-    if preferred_file is None:
+    if not required and preferred_file is None:
         preferred_file = latest_f
         preferred_ver = latest
 
-    return (latest, latest_f, preferred_ver, preferred_file)
-
+    return (latest, latest_f, preferred_ver, preferred_file, required)
 
 def _filterProviders(providers, item, cfgData, dataCache):
     """
@@ -232,12 +256,15 @@ def _filterProviders(providers, item, cfgData, dataCache):
             pkg_pn[pn] = []
         pkg_pn[pn].append(p)
 
-    logger.debug(1, "providers for %s are: %s", item, list(sorted(pkg_pn.keys())))
+    logger.debug("providers for %s are: %s", item, list(sorted(pkg_pn.keys())))
 
-    # First add PREFERRED_VERSIONS
+    # First add REQUIRED_VERSIONS or PREFERRED_VERSIONS
     for pn in sorted(pkg_pn):
         sortpkg_pn[pn] = sortPriorities(pn, dataCache, pkg_pn)
-        preferred_versions[pn] = findPreferredProvider(pn, cfgData, dataCache, sortpkg_pn[pn], item)
+        preferred_ver, preferred_file, required = findPreferredProvider(pn, cfgData, dataCache, sortpkg_pn[pn], item)
+        if required and preferred_file is None:
+            return eligible
+        preferred_versions[pn] = (preferred_ver, preferred_file)
         if preferred_versions[pn][1]:
             eligible.append(preferred_versions[pn][1])
 
@@ -248,9 +275,8 @@ def _filterProviders(providers, item, cfgData, dataCache):
         preferred_versions[pn] = findLatestProvider(pn, cfgData, dataCache, sortpkg_pn[pn][0])
         eligible.append(preferred_versions[pn][1])
 
-    if len(eligible) == 0:
-        logger.error("no eligible providers for %s", item)
-        return 0
+    if not eligible:
+        return eligible
 
     # If pn == item, give it a slight default preference
     # This means PREFERRED_PROVIDER_foobar defaults to foobar if available
@@ -266,7 +292,6 @@ def _filterProviders(providers, item, cfgData, dataCache):
 
     return eligible
 
-
 def filterProviders(providers, item, cfgData, dataCache):
     """
     Take a list of providers and filter/reorder according to the
@@ -291,7 +316,7 @@ def filterProviders(providers, item, cfgData, dataCache):
                 foundUnique = True
                 break
 
-    logger.debug(1, "sorted providers for %s are: %s", item, eligible)
+    logger.debug("sorted providers for %s are: %s", item, eligible)
 
     return eligible, foundUnique
 
@@ -333,7 +358,7 @@ def filterProvidersRunTime(providers, item, cfgData, dataCache):
             provides = dataCache.pn_provides[pn]
             for provide in provides:
                 prefervar = cfgData.getVar('PREFERRED_PROVIDER_%s' % provide)
-                #logger.debug(1, "checking PREFERRED_PROVIDER_%s (value %s) against %s", provide, prefervar, pns.keys())
+                #logger.debug("checking PREFERRED_PROVIDER_%s (value %s) against %s", provide, prefervar, pns.keys())
                 if prefervar in pns and pns[prefervar] not in preferred:
                     var = "PREFERRED_PROVIDER_%s = %s" % (provide, prefervar)
                     logger.verbose("selecting %s to satisfy runtime %s due to %s", prefervar, item, var)
@@ -349,7 +374,7 @@ def filterProvidersRunTime(providers, item, cfgData, dataCache):
     if numberPreferred > 1:
         logger.error("Trying to resolve runtime dependency %s resulted in conflicting PREFERRED_PROVIDER entries being found.\nThe providers found were: %s\nThe PREFERRED_PROVIDER entries resulting in this conflict were: %s. You could set PREFERRED_RPROVIDER_%s" % (item, preferred, preferred_vars, item))
 
-    logger.debug(1, "sorted runtime providers for %s are: %s", item, eligible)
+    logger.debug("sorted runtime providers for %s are: %s", item, eligible)
 
     return eligible, numberPreferred
 
@@ -384,11 +409,10 @@ def getRuntimeProviders(dataCache, rdepend):
             regexp_cache[pattern] = regexp
         if regexp.match(rdepend):
             rproviders += dataCache.packages_dynamic[pattern]
-            logger.debug(1, "Assuming %s is a dynamic package, but it may not exist" % rdepend)
+            logger.debug("Assuming %s is a dynamic package, but it may not exist" % rdepend)
 
     return rproviders
 
-
 def buildWorldTargetList(dataCache, task=None):
     """
     Build package list for "bitbake world"
@@ -396,22 +420,22 @@ def buildWorldTargetList(dataCache, task=None):
     if dataCache.world_target:
         return
 
-    logger.debug(1, "collating packages for \"world\"")
+    logger.debug("collating packages for \"world\"")
     for f in dataCache.possible_world:
         terminal = True
         pn = dataCache.pkg_fn[f]
         if task and task not in dataCache.task_deps[f]['tasks']:
-            logger.debug(2, "World build skipping %s as task %s doesn't exist", f, task)
+            logger.debug2("World build skipping %s as task %s doesn't exist", f, task)
             terminal = False
 
         for p in dataCache.pn_provides[pn]:
             if p.startswith('virtual/'):
-                logger.debug(2, "World build skipping %s due to %s provider starting with virtual/", f, p)
+                logger.debug2("World build skipping %s due to %s provider starting with virtual/", f, p)
                 terminal = False
                 break
             for pf in dataCache.providers[p]:
                 if dataCache.pkg_fn[pf] != pn:
-                    logger.debug(2, "World build skipping %s due to both us and %s providing %s", f, pf, p)
+                    logger.debug2("World build skipping %s due to both us and %s providing %s", f, pf, p)
                     terminal = False
                     break
         if terminal:

bitbake/lib/bb/runqueue.py

@@ -38,7 +38,7 @@ def taskname_from_tid(tid):
     return tid.rsplit(":", 1)[1]
 
 def mc_from_tid(tid):
-    if tid.startswith('mc:'):
+    if tid.startswith('mc:') and tid.count(':') >= 2:
         return tid.split(':')[1]
     return ""
 
@@ -47,13 +47,13 @@ def split_tid(tid):
     return (mc, fn, taskname)
 
 def split_mc(n):
-    if n.startswith("mc:"):
+    if n.startswith("mc:") and n.count(':') >= 2:
         _, mc, n = n.split(":", 2)
         return (mc, n)
     return ('', n)
 
 def split_tid_mcfn(tid):
-    if tid.startswith('mc:'):
+    if tid.startswith('mc:') and tid.count(':') >= 2:
         elems = tid.split(':')
         mc = elems[1]
         fn = ":".join(elems[2:-1])
@@ -544,8 +544,8 @@ class RunQueueData:
         for tid in self.runtaskentries:
             if task_done[tid] is False or deps_left[tid] != 0:
                 problem_tasks.append(tid)
-                logger.debug(2, "Task %s is not buildable", tid)
-                logger.debug(2, "(Complete marker was %s and the remaining dependency count was %s)\n", task_done[tid], deps_left[tid])
+                logger.debug2("Task %s is not buildable", tid)
+                logger.debug2("(Complete marker was %s and the remaining dependency count was %s)\n", task_done[tid], deps_left[tid])
             self.runtaskentries[tid].weight = weight[tid]
 
         if problem_tasks:
@@ -643,7 +643,7 @@ class RunQueueData:
                 (mc, fn, taskname, taskfn) = split_tid_mcfn(tid)
                 #runtid = build_tid(mc, fn, taskname)
 
-                #logger.debug(2, "Processing %s,%s:%s", mc, fn, taskname)
+                #logger.debug2("Processing %s,%s:%s", mc, fn, taskname)
 
                 depends = set()
                 task_deps = self.dataCaches[mc].task_deps[taskfn]
@@ -1199,9 +1199,9 @@ class RunQueueData:
         """
         Dump some debug information on the internal data structures
         """
-        logger.debug(3, "run_tasks:")
+        logger.debug3("run_tasks:")
         for tid in self.runtaskentries:
-            logger.debug(3, " %s: %s   Deps %s RevDeps %s", tid,
+            logger.debug3(" %s: %s   Deps %s RevDeps %s", tid,
                          self.runtaskentries[tid].weight,
                          self.runtaskentries[tid].depends,
                          self.runtaskentries[tid].revdeps)
@@ -1238,10 +1238,11 @@ class RunQueue:
         self.fakeworker = {}
 
     def _start_worker(self, mc, fakeroot = False, rqexec = None):
-        logger.debug(1, "Starting bitbake-worker")
+        logger.debug("Starting bitbake-worker")
         magic = "decafbad"
         if self.cooker.configuration.profile:
             magic = "decafbadbad"
+        fakerootlogs = None
         if fakeroot:
             magic = magic + "beef"
             mcdata = self.cooker.databuilder.mcdata[mc]
@@ -1251,10 +1252,11 @@ class RunQueue:
             for key, value in (var.split('=') for var in fakerootenv):
                 env[key] = value
             worker = subprocess.Popen(fakerootcmd + ["bitbake-worker", magic], stdout=subprocess.PIPE, stdin=subprocess.PIPE, env=env)
+            fakerootlogs = self.rqdata.dataCaches[mc].fakerootlogs
         else:
             worker = subprocess.Popen(["bitbake-worker", magic], stdout=subprocess.PIPE, stdin=subprocess.PIPE)
         bb.utils.nonblockingfd(worker.stdout)
-        workerpipe = runQueuePipe(worker.stdout, None, self.cfgData, self, rqexec)
+        workerpipe = runQueuePipe(worker.stdout, None, self.cfgData, self, rqexec, fakerootlogs=fakerootlogs)
 
         workerdata = {
             "taskdeps" : self.rqdata.dataCaches[mc].task_deps,
@@ -1271,6 +1273,7 @@ class RunQueue:
             "date" : self.cfgData.getVar("DATE"),
             "time" : self.cfgData.getVar("TIME"),
             "hashservaddr" : self.cooker.hashservaddr,
+            "umask" : self.cfgData.getVar("BB_DEFAULT_UMASK"),
         }
 
         worker.stdin.write(b"<cookerconfig>" + pickle.dumps(self.cooker.configuration) + b"</cookerconfig>")
@@ -1283,7 +1286,7 @@ class RunQueue:
     def _teardown_worker(self, worker):
         if not worker:
             return
-        logger.debug(1, "Teardown for bitbake-worker")
+        logger.debug("Teardown for bitbake-worker")
         try:
            worker.process.stdin.write(b"<quit></quit>")
            worker.process.stdin.flush()
@@ -1356,12 +1359,12 @@ class RunQueue:
 
         # If the stamp is missing, it's not current
         if not os.access(stampfile, os.F_OK):
-            logger.debug(2, "Stampfile %s not available", stampfile)
+            logger.debug2("Stampfile %s not available", stampfile)
             return False
         # If it's a 'nostamp' task, it's not current
         taskdep = self.rqdata.dataCaches[mc].task_deps[taskfn]
         if 'nostamp' in taskdep and taskname in taskdep['nostamp']:
-            logger.debug(2, "%s.%s is nostamp\n", fn, taskname)
+            logger.debug2("%s.%s is nostamp\n", fn, taskname)
             return False
 
         if taskname != "do_setscene" and taskname.endswith("_setscene"):
@@ -1385,18 +1388,18 @@ class RunQueue:
                     continue
                 if fn == fn2 or (fulldeptree and fn2 not in stampwhitelist):
                     if not t2:
-                        logger.debug(2, 'Stampfile %s does not exist', stampfile2)
+                        logger.debug2('Stampfile %s does not exist', stampfile2)
                         iscurrent = False
                         break
                     if t1 < t2:
-                        logger.debug(2, 'Stampfile %s < %s', stampfile, stampfile2)
+                        logger.debug2('Stampfile %s < %s', stampfile, stampfile2)
                         iscurrent = False
                         break
                     if recurse and iscurrent:
                         if dep in cache:
                             iscurrent = cache[dep]
                             if not iscurrent:
-                                logger.debug(2, 'Stampfile for dependency %s:%s invalid (cached)' % (fn2, taskname2))
+                                logger.debug2('Stampfile for dependency %s:%s invalid (cached)' % (fn2, taskname2))
                         else:
                             iscurrent = self.check_stamp_task(dep, recurse=True, cache=cache)
                             cache[dep] = iscurrent
@@ -1466,7 +1469,7 @@ class RunQueue:
             if not self.dm_event_handler_registered:
                  res = bb.event.register(self.dm_event_handler_name,
                                          lambda x: self.dm.check(self) if self.state in [runQueueRunning, runQueueCleanUp] else False,
-                                         ('bb.event.HeartbeatEvent',))
+                                         ('bb.event.HeartbeatEvent',), data=self.cfgData)
                  self.dm_event_handler_registered = True
 
             dump = self.cooker.configuration.dump_signatures
@@ -1505,7 +1508,7 @@ class RunQueue:
         build_done = self.state is runQueueComplete or self.state is runQueueFailed
 
         if build_done and self.dm_event_handler_registered:
-            bb.event.remove(self.dm_event_handler_name, None)
+            bb.event.remove(self.dm_event_handler_name, None, data=self.cfgData)
             self.dm_event_handler_registered = False
 
         if build_done and self.rqexe:
@@ -1761,7 +1764,7 @@ class RunQueueExecute:
         for scheduler in schedulers:
             if self.scheduler == scheduler.name:
                 self.sched = scheduler(self, self.rqdata)
-                logger.debug(1, "Using runqueue scheduler '%s'", scheduler.name)
+                logger.debug("Using runqueue scheduler '%s'", scheduler.name)
                 break
         else:
             bb.fatal("Invalid scheduler '%s'.  Available schedulers: %s" %
@@ -1771,7 +1774,7 @@ class RunQueueExecute:
         self.sqdata = SQData()
         build_scenequeue_data(self.sqdata, self.rqdata, self.rq, self.cooker, self.stampcache, self)
 
-    def runqueue_process_waitpid(self, task, status):
+    def runqueue_process_waitpid(self, task, status, fakerootlog=None):
 
         # self.build_stamps[pid] may not exist when use shared work directory.
         if task in self.build_stamps:
@@ -1786,7 +1789,7 @@ class RunQueueExecute:
             self.sq_live.remove(task)
         else:
             if status != 0:
-                self.task_fail(task, status)
+                self.task_fail(task, status, fakerootlog=fakerootlog)
             else:
                 self.task_complete(task)
         return True
@@ -1899,7 +1902,7 @@ class RunQueueExecute:
                     break
             if alldeps:
                 self.setbuildable(revdep)
-                logger.debug(1, "Marking task %s as buildable", revdep)
+                logger.debug("Marking task %s as buildable", revdep)
 
     def task_complete(self, task):
         self.stats.taskCompleted()
@@ -1907,14 +1910,31 @@ class RunQueueExecute:
         self.task_completeoutright(task)
         self.runq_tasksrun.add(task)
 
-    def task_fail(self, task, exitcode):
+    def task_fail(self, task, exitcode, fakerootlog=None):
         """
         Called when a task has failed
         Updates the state engine with the failure
         """
         self.stats.taskFailed()
         self.failed_tids.append(task)
-        bb.event.fire(runQueueTaskFailed(task, self.stats, exitcode, self.rq), self.cfgData)
+
+        fakeroot_log = ""
+        if fakerootlog and os.path.exists(fakerootlog):
+            with open(fakerootlog) as fakeroot_log_file:
+                fakeroot_failed = False
+                for line in reversed(fakeroot_log_file.readlines()):
+                    for fakeroot_error in ['mismatch', 'error', 'fatal']:
+                        if fakeroot_error in line.lower():
+                            fakeroot_failed = True
+                    if 'doing new pid setup and server start' in line:
+                        break
+                    fakeroot_log = line + fakeroot_log
+
+            if not fakeroot_failed:
+                fakeroot_log = None
+
+        bb.event.fire(runQueueTaskFailed(task, self.stats, exitcode, self.rq, fakeroot_log=fakeroot_log), self.cfgData)
+
         if self.rqdata.taskData[''].abort:
             self.rq.state = runQueueCleanUp
 
@@ -1929,7 +1949,7 @@ class RunQueueExecute:
     def summarise_scenequeue_errors(self):
         err = False
         if not self.sqdone:
-            logger.debug(1, 'We could skip tasks %s', "\n".join(sorted(self.scenequeue_covered)))
+            logger.debug('We could skip tasks %s', "\n".join(sorted(self.scenequeue_covered)))
             completeevent = sceneQueueComplete(self.sq_stats, self.rq)
             bb.event.fire(completeevent, self.cfgData)
         if self.sq_deferred:
@@ -1990,7 +2010,7 @@ class RunQueueExecute:
                 if nexttask in self.sq_buildable and nexttask not in self.sq_running and self.sqdata.stamps[nexttask] not in self.build_stamps.values():
                     if nexttask not in self.sqdata.unskippable and len(self.sqdata.sq_revdeps[nexttask]) > 0 and self.sqdata.sq_revdeps[nexttask].issubset(self.scenequeue_covered) and self.check_dependencies(nexttask, self.sqdata.sq_revdeps[nexttask]):
                         if nexttask not in self.rqdata.target_tids:
-                            logger.debug(2, "Skipping setscene for task %s" % nexttask)
+                            logger.debug2("Skipping setscene for task %s" % nexttask)
                             self.sq_task_skip(nexttask)
                             self.scenequeue_notneeded.add(nexttask)
                             if nexttask in self.sq_deferred:
@@ -2003,28 +2023,26 @@ class RunQueueExecute:
                     if nexttask in self.sq_deferred:
                         if self.sq_deferred[nexttask] not in self.runq_complete:
                             continue
-                        logger.debug(1, "Task %s no longer deferred" % nexttask)
+                        logger.debug("Task %s no longer deferred" % nexttask)
                         del self.sq_deferred[nexttask]
                         valid = self.rq.validate_hashes(set([nexttask]), self.cooker.data, 0, False, summary=False)
                         if not valid:
-                            logger.debug(1, "%s didn't become valid, skipping setscene" % nexttask)
+                            logger.debug("%s didn't become valid, skipping setscene" % nexttask)
                             self.sq_task_failoutright(nexttask)
                             return True
-                        else:
-                            self.sqdata.outrightfail.remove(nexttask)
                     if nexttask in self.sqdata.outrightfail:
-                        logger.debug(2, 'No package found, so skipping setscene task %s', nexttask)
+                        logger.debug2('No package found, so skipping setscene task %s', nexttask)
                         self.sq_task_failoutright(nexttask)
                         return True
                     if nexttask in self.sqdata.unskippable:
-                        logger.debug(2, "Setscene task %s is unskippable" % nexttask)
+                        logger.debug2("Setscene task %s is unskippable" % nexttask)
                     task = nexttask
                     break
         if task is not None:
             (mc, fn, taskname, taskfn) = split_tid_mcfn(task)
             taskname = taskname + "_setscene"
             if self.rq.check_stamp_task(task, taskname_from_tid(task), recurse = True, cache=self.stampcache):
-                logger.debug(2, 'Stamp for underlying task %s is current, so skipping setscene variant', task)
+                logger.debug2('Stamp for underlying task %s is current, so skipping setscene variant', task)
                 self.sq_task_failoutright(task)
                 return True
 
@@ -2034,12 +2052,12 @@ class RunQueueExecute:
                     return True
 
             if self.rq.check_stamp_task(task, taskname, cache=self.stampcache):
-                logger.debug(2, 'Setscene stamp current task %s, so skip it and its dependencies', task)
+                logger.debug2('Setscene stamp current task %s, so skip it and its dependencies', task)
                 self.sq_task_skip(task)
                 return True
 
             if self.cooker.configuration.skipsetscene:
-                logger.debug(2, 'No setscene tasks should be executed. Skipping %s', task)
+                logger.debug2('No setscene tasks should be executed. Skipping %s', task)
                 self.sq_task_failoutright(task)
                 return True
 
@@ -2101,12 +2119,12 @@ class RunQueueExecute:
                     return True
 
             if task in self.tasks_covered:
-                logger.debug(2, "Setscene covered task %s", task)
+                logger.debug2("Setscene covered task %s", task)
                 self.task_skip(task, "covered")
                 return True
 
             if self.rq.check_stamp_task(task, taskname, cache=self.stampcache):
-                logger.debug(2, "Stamp current task %s", task)
+                logger.debug2("Stamp current task %s", task)
 
                 self.task_skip(task, "existing")
                 self.runq_tasksrun.add(task)
@@ -2276,10 +2294,16 @@ class RunQueueExecute:
             self.updated_taskhash_queue.remove((tid, unihash))
 
             if unihash != self.rqdata.runtaskentries[tid].unihash:
-                hashequiv_logger.verbose("Task %s unihash changed to %s" % (tid, unihash))
-                self.rqdata.runtaskentries[tid].unihash = unihash
-                bb.parse.siggen.set_unihash(tid, unihash)
-                toprocess.add(tid)
+                # Make sure we rehash any other tasks with the same task hash that we're deferred against.
+                torehash = [tid]
+                for deftid in self.sq_deferred:
+                    if self.sq_deferred[deftid] == tid:
+                        torehash.append(deftid)
+                for hashtid in torehash:
+                    hashequiv_logger.verbose("Task %s unihash changed to %s" % (hashtid, unihash))
+                    self.rqdata.runtaskentries[hashtid].unihash = unihash
+                    bb.parse.siggen.set_unihash(hashtid, unihash)
+                    toprocess.add(hashtid)
 
         # Work out all tasks which depend upon these
         total = set()
@@ -2326,7 +2350,7 @@ class RunQueueExecute:
                     remapped = True
 
                 if not remapped:
-                    #logger.debug(1, "Task %s hash changes: %s->%s %s->%s" % (tid, orighash, newhash, origuni, newuni))
+                    #logger.debug("Task %s hash changes: %s->%s %s->%s" % (tid, orighash, newhash, origuni, newuni))
                     self.rqdata.runtaskentries[tid].hash = newhash
                     self.rqdata.runtaskentries[tid].unihash = newuni
                     changed.add(tid)
@@ -2341,7 +2365,7 @@ class RunQueueExecute:
             for mc in self.rq.fakeworker:
                 self.rq.fakeworker[mc].process.stdin.write(b"<newtaskhashes>" + pickle.dumps(bb.parse.siggen.get_taskhashes()) + b"</newtaskhashes>")
 
-            hashequiv_logger.debug(1, pprint.pformat("Tasks changed:\n%s" % (changed)))
+            hashequiv_logger.debug(pprint.pformat("Tasks changed:\n%s" % (changed)))
 
         for tid in changed:
             if tid not in self.rqdata.runq_setscene_tids:
@@ -2360,7 +2384,7 @@ class RunQueueExecute:
             # Check no tasks this covers are running
             for dep in self.sqdata.sq_covered_tasks[tid]:
                 if dep in self.runq_running and dep not in self.runq_complete:
-                    hashequiv_logger.debug(2, "Task %s is running which blocks setscene for %s from running" % (dep, tid))
+                    hashequiv_logger.debug2("Task %s is running which blocks setscene for %s from running" % (dep, tid))
                     valid = False
                     break
             if not valid:
@@ -2437,7 +2461,10 @@ class RunQueueExecute:
                 if dep in self.scenequeue_covered or dep in self.scenequeue_notcovered:
                     # dependency could be already processed, e.g. noexec setscene task
                     continue
-                logger.debug(2, "%s was unavailable and is a hard dependency of %s so skipping" % (task, dep))
+                noexec, stamppresent = check_setscene_stamps(dep, self.rqdata, self.rq, self.stampcache)
+                if noexec or stamppresent:
+                    continue
+                logger.debug2("%s was unavailable and is a hard dependency of %s so skipping" % (task, dep))
                 self.sq_task_failoutright(dep)
                 continue
             if self.sqdata.sq_revdeps[dep].issubset(self.scenequeue_covered | self.scenequeue_notcovered):
@@ -2467,7 +2494,7 @@ class RunQueueExecute:
         completed dependencies as buildable
         """
 
-        logger.debug(1, 'Found task %s which could be accelerated', task)
+        logger.debug('Found task %s which could be accelerated', task)
         self.scenequeue_covered.add(task)
         self.scenequeue_updatecounters(task)
 
@@ -2761,6 +2788,40 @@ def build_scenequeue_data(sqdata, rqdata, rq, cooker, stampcache, sqrq):
 
     update_scenequeue_data(sqdata.sq_revdeps, sqdata, rqdata, rq, cooker, stampcache, sqrq, summary=True)
 
+    # Compute a list of 'stale' sstate tasks where the current hash does not match the one
+    # in any stamp files. Pass the list out to metadata as an event.
+    found = {}
+    for tid in rqdata.runq_setscene_tids:
+        (mc, fn, taskname, taskfn) = split_tid_mcfn(tid)
+        stamps = bb.build.find_stale_stamps(taskname, rqdata.dataCaches[mc], taskfn)
+        if stamps:
+            if mc not in found:
+                found[mc] = {}
+            found[mc][tid] = stamps
+    for mc in found:
+        event = bb.event.StaleSetSceneTasks(found[mc])
+        bb.event.fire(event, cooker.databuilder.mcdata[mc])
+
+def check_setscene_stamps(tid, rqdata, rq, stampcache, noexecstamp=False):
+
+    (mc, fn, taskname, taskfn) = split_tid_mcfn(tid)
+
+    taskdep = rqdata.dataCaches[mc].task_deps[taskfn]
+
+    if 'noexec' in taskdep and taskname in taskdep['noexec']:
+        bb.build.make_stamp(taskname + "_setscene", rqdata.dataCaches[mc], taskfn)
+        return True, False
+
+    if rq.check_stamp_task(tid, taskname + "_setscene", cache=stampcache):
+        logger.debug2('Setscene stamp current for task %s', tid)
+        return False, True
+
+    if rq.check_stamp_task(tid, taskname, recurse = True, cache=stampcache):
+        logger.debug2('Normal stamp current for task %s', tid)
+        return False, True
+
+    return False, False
+
 def update_scenequeue_data(tids, sqdata, rqdata, rq, cooker, stampcache, sqrq, summary=True):
 
     tocheck = set()
@@ -2770,25 +2831,17 @@ def update_scenequeue_data(tids, sqdata, rqdata, rq, cooker, stampcache, sqrq, s
             sqdata.stamppresent.remove(tid)
         if tid in sqdata.valid:
             sqdata.valid.remove(tid)
+        if tid in sqdata.outrightfail:
+            sqdata.outrightfail.remove(tid)
 
-        (mc, fn, taskname, taskfn) = split_tid_mcfn(tid)
+        noexec, stamppresent = check_setscene_stamps(tid, rqdata, rq, stampcache, noexecstamp=True)
 
-        taskdep = rqdata.dataCaches[mc].task_deps[taskfn]
-
-        if 'noexec' in taskdep and taskname in taskdep['noexec']:
+        if noexec:
             sqdata.noexec.add(tid)
             sqrq.sq_task_skip(tid)
-            bb.build.make_stamp(taskname + "_setscene", rqdata.dataCaches[mc], taskfn)
             continue
 
-        if rq.check_stamp_task(tid, taskname + "_setscene", cache=stampcache):
-            logger.debug(2, 'Setscene stamp current for task %s', tid)
-            sqdata.stamppresent.add(tid)
-            sqrq.sq_task_skip(tid)
-            continue
-
-        if rq.check_stamp_task(tid, taskname, recurse = True, cache=stampcache):
-            logger.debug(2, 'Normal stamp current for task %s', tid)
+        if stamppresent:
             sqdata.stamppresent.add(tid)
             sqrq.sq_task_skip(tid)
             continue
@@ -2798,6 +2851,7 @@ def update_scenequeue_data(tids, sqdata, rqdata, rq, cooker, stampcache, sqrq, s
     sqdata.valid |= rq.validate_hashes(tocheck, cooker.data, len(sqdata.stamppresent), False, summary=summary)
 
     sqdata.hashes = {}
+    sqrq.sq_deferred = {}
     for mc in sorted(sqdata.multiconfigs):
         for tid in sorted(sqdata.sq_revdeps):
             if mc_from_tid(tid) != mc:
@@ -2810,10 +2864,13 @@ def update_scenequeue_data(tids, sqdata, rqdata, rq, cooker, stampcache, sqrq, s
                 continue
             if tid in sqrq.scenequeue_notcovered:
                 continue
-            sqdata.outrightfail.add(tid)
+            if tid in sqrq.scenequeue_covered:
+                continue
 
             h = pending_hash_index(tid, rqdata)
             if h not in sqdata.hashes:
+                if tid in tids:
+                    sqdata.outrightfail.add(tid)
                 sqdata.hashes[h] = tid
             else:
                 sqrq.sq_deferred[tid] = sqdata.hashes[h]
@@ -2882,12 +2939,16 @@ class runQueueTaskFailed(runQueueEvent):
     """
     Event notifying a task failed
     """
-    def __init__(self, task, stats, exitcode, rq):
+    def __init__(self, task, stats, exitcode, rq, fakeroot_log=None):
         runQueueEvent.__init__(self, task, stats, rq)
         self.exitcode = exitcode
+        self.fakeroot_log = fakeroot_log
 
     def __str__(self):
-        return "Task (%s) failed with exit code '%s'" % (self.taskstring, self.exitcode)
+        if self.fakeroot_log:
+            return "Task (%s) failed with exit code '%s' \nPseudo log:\n%s" % (self.taskstring, self.exitcode, self.fakeroot_log)
+        else:
+            return "Task (%s) failed with exit code '%s'" % (self.taskstring, self.exitcode)
 
 class sceneQueueTaskFailed(sceneQueueEvent):
     """
@@ -2939,7 +3000,7 @@ class runQueuePipe():
     """
     Abstraction for a pipe between a worker thread and the server
     """
-    def __init__(self, pipein, pipeout, d, rq, rqexec):
+    def __init__(self, pipein, pipeout, d, rq, rqexec, fakerootlogs=None):
         self.input = pipein
         if pipeout:
             pipeout.close()
@@ -2948,6 +3009,7 @@ class runQueuePipe():
         self.d = d
         self.rq = rq
         self.rqexec = rqexec
+        self.fakerootlogs = fakerootlogs
 
     def setrunqueueexec(self, rqexec):
         self.rqexec = rqexec
@@ -2993,7 +3055,11 @@ class runQueuePipe():
                     task, status = pickle.loads(self.queue[10:index])
                 except (ValueError, pickle.UnpicklingError, AttributeError, IndexError) as e:
                     bb.msg.fatal("RunQueue", "failed load pickle '%s': '%s'" % (e, self.queue[10:index]))
-                self.rqexec.runqueue_process_waitpid(task, status)
+                (_, _, _, taskfn) = split_tid_mcfn(task)
+                fakerootlog = None
+                if self.fakerootlogs and taskfn and taskfn in self.fakerootlogs:
+                    fakerootlog = self.fakerootlogs[taskfn]
+                self.rqexec.runqueue_process_waitpid(task, status, fakerootlog=fakerootlog)
                 found = True
                 self.queue = self.queue[index+11:]
                 index = self.queue.find(b"</exitcode>")

bitbake/lib/bb/server/process.py

@@ -147,7 +147,7 @@ class ProcessServer():
                 conn = newconnections.pop(-1)
                 fds.append(conn)
                 self.controllersock = conn
-            elif self.timeout is None and not ready:
+            elif not self.timeout and not ready:
                 serverlog("No timeout, exiting.")
                 self.quit = True
 
@@ -367,7 +367,12 @@ class ProcessServer():
                 self.next_heartbeat = now + self.heartbeat_seconds
             if hasattr(self.cooker, "data"):
                 heartbeat = bb.event.HeartbeatEvent(now)
-                bb.event.fire(heartbeat, self.cooker.data)
+                try:
+                    bb.event.fire(heartbeat, self.cooker.data)
+                except Exception as exc:
+                    if not isinstance(exc, bb.BBHandledException):
+                        logger.exception('Running heartbeat function')
+                    self.quit = True
         if nextsleep and now + nextsleep > self.next_heartbeat:
             # Shorten timeout so that we we wake up in time for
             # the heartbeat.
@@ -509,7 +514,7 @@ class BitBakeServer(object):
         os.set_inheritable(self.bitbake_lock.fileno(), True)
         os.set_inheritable(self.readypipein, True)
         serverscript = os.path.realpath(os.path.dirname(__file__) + "/../../../bin/bitbake-server")
-        os.execl(sys.executable, "bitbake-server", serverscript, "decafbad", str(self.bitbake_lock.fileno()), str(self.readypipein), self.logfile, self.bitbake_lock.name, self.sockname,  str(self.server_timeout), str(self.xmlrpcinterface[0]), str(self.xmlrpcinterface[1]))
+        os.execl(sys.executable, "bitbake-server", serverscript, "decafbad", str(self.bitbake_lock.fileno()), str(self.readypipein), self.logfile, self.bitbake_lock.name, self.sockname,  str(self.server_timeout or 0), str(self.xmlrpcinterface[0]), str(self.xmlrpcinterface[1]))
 
 def execServer(lockfd, readypipeinfd, lockname, sockname, server_timeout, xmlrpcinterface):
 

bitbake/lib/bb/siggen.py

@@ -311,13 +311,7 @@ class SignatureGeneratorBasic(SignatureGenerator):
 
         data = self.basehash[tid]
         for dep in self.runtaskdeps[tid]:
-            if dep in self.unihash:
-                if self.unihash[dep] is None:
-                    data = data + self.taskhash[dep]
-                else:
-                    data = data + self.unihash[dep]
-            else:
-                data = data + self.get_unihash(dep)
+            data = data + self.get_unihash(dep)
 
         for (f, cs) in self.file_checksum_values[tid]:
             if cs:
@@ -408,7 +402,7 @@ class SignatureGeneratorBasic(SignatureGenerator):
                 p = pickle.dump(data, stream, -1)
                 stream.flush()
             os.chmod(tmpfile, 0o664)
-            os.rename(tmpfile, sigfile)
+            bb.utils.rename(tmpfile, sigfile)
         except (OSError, IOError) as err:
             try:
                 os.unlink(tmpfile)
@@ -547,8 +541,8 @@ class SignatureGeneratorUniHashMixIn(object):
                 # is much more interesting, so it is reported at debug level 1
                 hashequiv_logger.debug((1, 2)[unihash == taskhash], 'Found unihash %s in place of %s for %s from %s' % (unihash, taskhash, tid, self.server))
             else:
-                hashequiv_logger.debug(2, 'No reported unihash for %s:%s from %s' % (tid, taskhash, self.server))
-        except hashserv.client.HashConnectionError as e:
+                hashequiv_logger.debug2('No reported unihash for %s:%s from %s' % (tid, taskhash, self.server))
+        except ConnectionError as e:
             bb.warn('Error contacting Hash Equivalence Server %s: %s' % (self.server, str(e)))
 
         self.set_unihash(tid, unihash)
@@ -621,13 +615,13 @@ class SignatureGeneratorUniHashMixIn(object):
                 new_unihash = data['unihash']
 
                 if new_unihash != unihash:
-                    hashequiv_logger.debug(1, 'Task %s unihash changed %s -> %s by server %s' % (taskhash, unihash, new_unihash, self.server))
+                    hashequiv_logger.debug('Task %s unihash changed %s -> %s by server %s' % (taskhash, unihash, new_unihash, self.server))
                     bb.event.fire(bb.runqueue.taskUniHashUpdate(fn + ':do_' + task, new_unihash), d)
                     self.set_unihash(tid, new_unihash)
                     d.setVar('BB_UNIHASH', new_unihash)
                 else:
-                    hashequiv_logger.debug(1, 'Reported task %s as unihash %s to %s' % (taskhash, unihash, self.server))
-            except hashserv.client.HashConnectionError as e:
+                    hashequiv_logger.debug('Reported task %s as unihash %s to %s' % (taskhash, unihash, self.server))
+            except ConnectionError as e:
                 bb.warn('Error contacting Hash Equivalence Server %s: %s' % (self.server, str(e)))
         finally:
             if sigfile:
@@ -667,7 +661,7 @@ class SignatureGeneratorUniHashMixIn(object):
                 # TODO: What to do here?
                 hashequiv_logger.verbose('Task %s unihash reported as unwanted hash %s' % (tid, finalunihash))
 
-        except hashserv.client.HashConnectionError as e:
+        except ConnectionError as e:
             bb.warn('Error contacting Hash Equivalence Server %s: %s' % (self.server, str(e)))
 
         return False
@@ -754,7 +748,7 @@ def clean_basepath(basepath):
     if basepath[0] == '/':
         return cleaned
 
-    if basepath.startswith("mc:"):
+    if basepath.startswith("mc:") and basepath.count(':') >= 2:
         mc, mc_name, basepath = basepath.split(":", 2)
         mc_suffix = ':mc:' + mc_name
     else:

bitbake/lib/bb/taskdata.py

@@ -131,7 +131,7 @@ class TaskData:
             for depend in dataCache.deps[fn]:
                 dependids.add(depend)
             self.depids[fn] = list(dependids)
-            logger.debug(2, "Added dependencies %s for %s", str(dataCache.deps[fn]), fn)
+            logger.debug2("Added dependencies %s for %s", str(dataCache.deps[fn]), fn)
 
         # Work out runtime dependencies
         if not fn in self.rdepids:
@@ -149,9 +149,9 @@ class TaskData:
                     rreclist.append(rdepend)
                     rdependids.add(rdepend)
             if rdependlist:
-                logger.debug(2, "Added runtime dependencies %s for %s", str(rdependlist), fn)
+                logger.debug2("Added runtime dependencies %s for %s", str(rdependlist), fn)
             if rreclist:
-                logger.debug(2, "Added runtime recommendations %s for %s", str(rreclist), fn)
+                logger.debug2("Added runtime recommendations %s for %s", str(rreclist), fn)
             self.rdepids[fn] = list(rdependids)
 
         for dep in self.depids[fn]:
@@ -378,7 +378,7 @@ class TaskData:
         for fn in eligible:
             if fn in self.failed_fns:
                 continue
-            logger.debug(2, "adding %s to satisfy %s", fn, item)
+            logger.debug2("adding %s to satisfy %s", fn, item)
             self.add_build_target(fn, item)
             self.add_tasks(fn, dataCache)
 
@@ -431,7 +431,7 @@ class TaskData:
         for fn in eligible:
             if fn in self.failed_fns:
                 continue
-            logger.debug(2, "adding '%s' to satisfy runtime '%s'", fn, item)
+            logger.debug2("adding '%s' to satisfy runtime '%s'", fn, item)
             self.add_runtime_target(fn, item)
             self.add_tasks(fn, dataCache)
 
@@ -446,7 +446,7 @@ class TaskData:
             return
         if not missing_list:
             missing_list = []
-        logger.debug(1, "File '%s' is unbuildable, removing...", fn)
+        logger.debug("File '%s' is unbuildable, removing...", fn)
         self.failed_fns.append(fn)
         for target in self.build_targets:
             if fn in self.build_targets[target]:
@@ -526,7 +526,7 @@ class TaskData:
                     added = added + 1
                 except (bb.providers.NoRProvider, bb.providers.MultipleRProvider):
                     self.remove_runtarget(target)
-            logger.debug(1, "Resolved " + str(added) + " extra dependencies")
+            logger.debug("Resolved " + str(added) + " extra dependencies")
             if added == 0:
                 break
         # self.dump_data()
@@ -549,38 +549,38 @@ class TaskData:
         """
         Dump some debug information on the internal data structures
         """
-        logger.debug(3, "build_names:")
-        logger.debug(3, ", ".join(self.build_targets))
+        logger.debug3("build_names:")
+        logger.debug3(", ".join(self.build_targets))
 
-        logger.debug(3, "run_names:")
-        logger.debug(3, ", ".join(self.run_targets))
+        logger.debug3("run_names:")
+        logger.debug3(", ".join(self.run_targets))
 
-        logger.debug(3, "build_targets:")
+        logger.debug3("build_targets:")
         for target in self.build_targets:
             targets = "None"
             if target in self.build_targets:
                 targets = self.build_targets[target]
-            logger.debug(3, " %s: %s", target, targets)
+            logger.debug3(" %s: %s", target, targets)
 
-        logger.debug(3, "run_targets:")
+        logger.debug3("run_targets:")
         for target in self.run_targets:
             targets = "None"
             if target in self.run_targets:
                 targets = self.run_targets[target]
-            logger.debug(3, " %s: %s", target, targets)
+            logger.debug3(" %s: %s", target, targets)
 
-        logger.debug(3, "tasks:")
+        logger.debug3("tasks:")
         for tid in self.taskentries:
-            logger.debug(3, " %s: %s %s %s",
+            logger.debug3(" %s: %s %s %s",
                        tid,
                        self.taskentries[tid].idepends,
                        self.taskentries[tid].irdepends,
                        self.taskentries[tid].tdepends)
 
-        logger.debug(3, "dependency ids (per fn):")
+        logger.debug3("dependency ids (per fn):")
         for fn in self.depids:
-            logger.debug(3, " %s: %s", fn, self.depids[fn])
+            logger.debug3(" %s: %s", fn, self.depids[fn])
 
-        logger.debug(3, "runtime dependency ids (per fn):")
+        logger.debug3("runtime dependency ids (per fn):")
         for fn in self.rdepids:
-            logger.debug(3, " %s: %s", fn, self.rdepids[fn])
+            logger.debug3(" %s: %s", fn, self.rdepids[fn])

bitbake/lib/bb/tests/color.py

@@ -31,7 +31,7 @@ class ColorCodeTests(unittest.TestCase):
     def setUp(self):
         self.d = bb.data.init()
         self._progress_watcher = ProgressWatcher()
-        bb.event.register("bb.build.TaskProgress", self._progress_watcher.handle_event)
+        bb.event.register("bb.build.TaskProgress", self._progress_watcher.handle_event, data=self.d)
 
     def tearDown(self):
         bb.event.remove("bb.build.TaskProgress", None)

bitbake/lib/bb/tests/compression.py

@@ -0,0 +1,98 @@
+#
+# SPDX-License-Identifier: GPL-2.0-only
+#
+
+from pathlib import Path
+import bb.compress.lz4
+import bb.compress.zstd
+import contextlib
+import os
+import shutil
+import tempfile
+import unittest
+import subprocess
+
+
+class CompressionTests(object):
+    def setUp(self):
+        self._t = tempfile.TemporaryDirectory()
+        self.tmpdir = Path(self._t.name)
+        self.addCleanup(self._t.cleanup)
+
+    def _file_helper(self, mode_suffix, data):
+        tmp_file = self.tmpdir / "compressed"
+
+        with self.do_open(tmp_file, mode="w" + mode_suffix) as f:
+            f.write(data)
+
+        with self.do_open(tmp_file, mode="r" + mode_suffix) as f:
+            read_data = f.read()
+
+        self.assertEqual(read_data, data)
+
+    def test_text_file(self):
+        self._file_helper("t", "Hello")
+
+    def test_binary_file(self):
+        self._file_helper("b", "Hello".encode("utf-8"))
+
+    def _pipe_helper(self, mode_suffix, data):
+        rfd, wfd = os.pipe()
+        with open(rfd, "rb") as r, open(wfd, "wb") as w:
+            with self.do_open(r, mode="r" + mode_suffix) as decompress:
+                with self.do_open(w, mode="w" + mode_suffix) as compress:
+                    compress.write(data)
+                read_data = decompress.read()
+
+        self.assertEqual(read_data, data)
+
+    def test_text_pipe(self):
+        self._pipe_helper("t", "Hello")
+
+    def test_binary_pipe(self):
+        self._pipe_helper("b", "Hello".encode("utf-8"))
+
+    def test_bad_decompress(self):
+        tmp_file = self.tmpdir / "compressed"
+        with tmp_file.open("wb") as f:
+            f.write(b"\x00")
+
+        with self.assertRaises(OSError):
+            with self.do_open(tmp_file, mode="rb", stderr=subprocess.DEVNULL) as f:
+                data = f.read()
+
+
+class LZ4Tests(CompressionTests, unittest.TestCase):
+    def setUp(self):
+        if shutil.which("lz4c") is None:
+            self.skipTest("'lz4c' not found")
+        super().setUp()
+
+    @contextlib.contextmanager
+    def do_open(self, *args, **kwargs):
+        with bb.compress.lz4.open(*args, **kwargs) as f:
+            yield f
+
+
+class ZStdTests(CompressionTests, unittest.TestCase):
+    def setUp(self):
+        if shutil.which("zstd") is None:
+            self.skipTest("'zstd' not found")
+        super().setUp()
+
+    @contextlib.contextmanager
+    def do_open(self, *args, **kwargs):
+        with bb.compress.zstd.open(*args, **kwargs) as f:
+            yield f
+
+
+class PZStdTests(CompressionTests, unittest.TestCase):
+    def setUp(self):
+        if shutil.which("pzstd") is None:
+            self.skipTest("'pzstd' not found")
+        super().setUp()
+
+    @contextlib.contextmanager
+    def do_open(self, *args, **kwargs):
+        with bb.compress.zstd.open(*args, num_threads=2, **kwargs) as f:
+            yield f

bitbake/lib/bb/tests/fetch.py

@@ -87,6 +87,25 @@ class URITest(unittest.TestCase):
             },
             'relative': False
         },
+        # Check that trailing semicolons are handled correctly
+        "http://www.example.org/index.html?qparam1=qvalue1;param2=value2;" : {
+            'uri': 'http://www.example.org/index.html?qparam1=qvalue1;param2=value2',
+            'scheme': 'http',
+            'hostname': 'www.example.org',
+            'port': None,
+            'hostport': 'www.example.org',
+            'path': '/index.html',
+            'userinfo': '',
+            'username': '',
+            'password': '',
+            'params': {
+                'param2': 'value2'
+            },
+            'query': {
+                'qparam1': 'qvalue1'
+            },
+            'relative': False
+        },
         "http://www.example.com:8080/index.html" : {
             'uri': 'http://www.example.com:8080/index.html',
             'scheme': 'http',
@@ -371,6 +390,7 @@ class FetcherTest(unittest.TestCase):
         if os.environ.get("BB_TMPDIR_NOCLEAN") == "yes":
             print("Not cleaning up %s. Please remove manually." % self.tempdir)
         else:
+            bb.process.run('chmod u+rw -R %s' % self.tempdir)
             bb.utils.prunedir(self.tempdir)
 
 class MirrorUriTest(FetcherTest):
@@ -654,6 +674,52 @@ class FetcherLocalTest(FetcherTest):
         with self.assertRaises(bb.fetch2.UnpackError):
             self.fetchUnpack(['file://a;subdir=/bin/sh'])
 
+    def dummyGitTest(self, suffix):
+        # Create dummy local Git repo
+        src_dir = tempfile.mkdtemp(dir=self.tempdir,
+                                   prefix='gitfetch_localusehead_')
+        src_dir = os.path.abspath(src_dir)
+        bb.process.run("git init", cwd=src_dir)
+        bb.process.run("git config user.email 'you@example.com'", cwd=src_dir)
+        bb.process.run("git config user.name 'Your Name'", cwd=src_dir)
+        bb.process.run("git commit --allow-empty -m'Dummy commit'",
+                       cwd=src_dir)
+        # Use other branch than master
+        bb.process.run("git checkout -b my-devel", cwd=src_dir)
+        bb.process.run("git commit --allow-empty -m'Dummy commit 2'",
+                       cwd=src_dir)
+        stdout = bb.process.run("git rev-parse HEAD", cwd=src_dir)
+        orig_rev = stdout[0].strip()
+
+        # Fetch and check revision
+        self.d.setVar("SRCREV", "AUTOINC")
+        url = "git://" + src_dir + ";protocol=file;" + suffix
+        fetcher = bb.fetch.Fetch([url], self.d)
+        fetcher.download()
+        fetcher.unpack(self.unpackdir)
+        stdout = bb.process.run("git rev-parse HEAD",
+                                cwd=os.path.join(self.unpackdir, 'git'))
+        unpack_rev = stdout[0].strip()
+        self.assertEqual(orig_rev, unpack_rev)
+
+    def test_local_gitfetch_usehead(self):
+        self.dummyGitTest("usehead=1")
+
+    def test_local_gitfetch_usehead_withname(self):
+        self.dummyGitTest("usehead=1;name=newName")
+
+    def test_local_gitfetch_shared(self):
+        self.dummyGitTest("usehead=1;name=sharedName")
+        alt = os.path.join(self.unpackdir, 'git/.git/objects/info/alternates')
+        self.assertTrue(os.path.exists(alt))
+
+    def test_local_gitfetch_noshared(self):
+        self.d.setVar('BB_GIT_NOSHARED', '1')
+        self.unpackdir += '_noshared'
+        self.dummyGitTest("usehead=1;name=noSharedName")
+        alt = os.path.join(self.unpackdir, 'git/.git/objects/info/alternates')
+        self.assertFalse(os.path.exists(alt))
+
 class FetcherNoNetworkTest(FetcherTest):
     def setUp(self):
         super().setUp()
@@ -844,35 +910,21 @@ class FetcherNetworkTest(FetcherTest):
         self.assertRaises(bb.fetch.FetchError, self.gitfetcher, url1, url2)
 
     @skipIfNoNetwork()
-    def test_gitfetch_localusehead(self):
-        # Create dummy local Git repo
-        src_dir = tempfile.mkdtemp(dir=self.tempdir,
-                                   prefix='gitfetch_localusehead_')
-        src_dir = os.path.abspath(src_dir)
-        bb.process.run("git init", cwd=src_dir)
-        bb.process.run("git commit --allow-empty -m'Dummy commit'",
-                       cwd=src_dir)
-        # Use other branch than master
-        bb.process.run("git checkout -b my-devel", cwd=src_dir)
-        bb.process.run("git commit --allow-empty -m'Dummy commit 2'",
-                       cwd=src_dir)
-        stdout = bb.process.run("git rev-parse HEAD", cwd=src_dir)
-        orig_rev = stdout[0].strip()
-
-        # Fetch and check revision
-        self.d.setVar("SRCREV", "AUTOINC")
-        url = "git://" + src_dir + ";protocol=file;usehead=1"
-        fetcher = bb.fetch.Fetch([url], self.d)
-        fetcher.download()
-        fetcher.unpack(self.unpackdir)
-        stdout = bb.process.run("git rev-parse HEAD",
-                                cwd=os.path.join(self.unpackdir, 'git'))
-        unpack_rev = stdout[0].strip()
-        self.assertEqual(orig_rev, unpack_rev)
+    def test_gitfetch_usehead(self):
+        # Since self.gitfetcher() sets SRCREV we expect this to override
+        # `usehead=1' and instead fetch the specified SRCREV. See
+        # test_local_gitfetch_usehead() for a positive use of the usehead
+        # feature.
+        url = "git://git.openembedded.org/bitbake;usehead=1"
+        self.assertRaises(bb.fetch.ParameterError, self.gitfetcher, url, url)
 
     @skipIfNoNetwork()
-    def test_gitfetch_remoteusehead(self):
-        url = "git://git.openembedded.org/bitbake;usehead=1"
+    def test_gitfetch_usehead_withname(self):
+        # Since self.gitfetcher() sets SRCREV we expect this to override
+        # `usehead=1' and instead fetch the specified SRCREV. See
+        # test_local_gitfetch_usehead() for a positive use of the usehead
+        # feature.
+        url = "git://git.openembedded.org/bitbake;usehead=1;name=newName"
         self.assertRaises(bb.fetch.ParameterError, self.gitfetcher, url, url)
 
     @skipIfNoNetwork()
@@ -1288,7 +1340,7 @@ class FetchCheckStatusTest(FetcherTest):
                       "http://downloads.yoctoproject.org/releases/sato/sato-engine-0.2.tar.gz",
                       "http://downloads.yoctoproject.org/releases/sato/sato-engine-0.3.tar.gz",
                       "https://yoctoproject.org/",
-                      "https://yoctoproject.org/documentation",
+                      "https://docs.yoctoproject.org",
                       "http://downloads.yoctoproject.org/releases/opkg/opkg-0.1.7.tar.gz",
                       "http://downloads.yoctoproject.org/releases/opkg/opkg-0.3.0.tar.gz",
                       "ftp://sourceware.org/pub/libffi/libffi-1.20.tar.gz",
@@ -1333,6 +1385,8 @@ class GitMakeShallowTest(FetcherTest):
         self.gitdir = os.path.join(self.tempdir, 'gitshallow')
         bb.utils.mkdirhier(self.gitdir)
         bb.process.run('git init', cwd=self.gitdir)
+        bb.process.run('git config user.email "you@example.com"', cwd=self.gitdir)
+        bb.process.run('git config user.name "Your Name"', cwd=self.gitdir)
 
     def assertRefs(self, expected_refs):
         actual_refs = self.git(['for-each-ref', '--format=%(refname)']).splitlines()
@@ -1456,6 +1510,8 @@ class GitShallowTest(FetcherTest):
 
         bb.utils.mkdirhier(self.srcdir)
         self.git('init', cwd=self.srcdir)
+        self.git('config user.email "you@example.com"', cwd=self.srcdir)
+        self.git('config user.name "Your Name"', cwd=self.srcdir)
         self.d.setVar('WORKDIR', self.tempdir)
         self.d.setVar('S', self.gitdir)
         self.d.delVar('PREMIRRORS')
@@ -1537,6 +1593,7 @@ class GitShallowTest(FetcherTest):
 
         # fetch and unpack, from the shallow tarball
         bb.utils.remove(self.gitdir, recurse=True)
+        bb.process.run('chmod u+w -R "%s"' % ud.clonedir)
         bb.utils.remove(ud.clonedir, recurse=True)
         bb.utils.remove(ud.clonedir.replace('gitsource', 'gitsubmodule'), recurse=True)
 
@@ -1689,6 +1746,8 @@ class GitShallowTest(FetcherTest):
         smdir = os.path.join(self.tempdir, 'gitsubmodule')
         bb.utils.mkdirhier(smdir)
         self.git('init', cwd=smdir)
+        self.git('config user.email "you@example.com"', cwd=smdir)
+        self.git('config user.name "Your Name"', cwd=smdir)
         # Make this look like it was cloned from a remote...
         self.git('config --add remote.origin.url "%s"' % smdir, cwd=smdir)
         self.git('config --add remote.origin.fetch "+refs/heads/*:refs/remotes/origin/*"', cwd=smdir)
@@ -1719,6 +1778,8 @@ class GitShallowTest(FetcherTest):
         smdir = os.path.join(self.tempdir, 'gitsubmodule')
         bb.utils.mkdirhier(smdir)
         self.git('init', cwd=smdir)
+        self.git('config user.email "you@example.com"', cwd=smdir)
+        self.git('config user.name "Your Name"', cwd=smdir)
         # Make this look like it was cloned from a remote...
         self.git('config --add remote.origin.url "%s"' % smdir, cwd=smdir)
         self.git('config --add remote.origin.fetch "+refs/heads/*:refs/remotes/origin/*"', cwd=smdir)
@@ -1737,7 +1798,7 @@ class GitShallowTest(FetcherTest):
 
         # Set up the mirror
         mirrordir = os.path.join(self.tempdir, 'mirror')
-        os.rename(self.dldir, mirrordir)
+        bb.utils.rename(self.dldir, mirrordir)
         self.d.setVar('PREMIRRORS', 'gitsm://.*/.* file://%s/\n' % mirrordir)
 
         # Fetch from the mirror
@@ -1761,8 +1822,8 @@ class GitShallowTest(FetcherTest):
             self.git('annex init', cwd=self.srcdir)
             open(os.path.join(self.srcdir, 'c'), 'w').close()
             self.git('annex add c', cwd=self.srcdir)
-            self.git('commit -m annex-c -a', cwd=self.srcdir)
-            bb.process.run('chmod u+w -R %s' % os.path.join(self.srcdir, '.git', 'annex'))
+            self.git('commit --author "Foo Bar <foo@bar>" -m annex-c -a', cwd=self.srcdir)
+            bb.process.run('chmod u+w -R %s' % self.srcdir)
 
             uri = 'gitannex://%s;protocol=file;subdir=${S}' % self.srcdir
             fetcher, ud = self.fetch_shallow(uri)
@@ -1855,7 +1916,7 @@ class GitShallowTest(FetcherTest):
         bb.utils.mkdirhier(mirrordir)
         self.d.setVar('PREMIRRORS', 'git://.*/.* file://%s/\n' % mirrordir)
 
-        os.rename(os.path.join(self.dldir, mirrortarball),
+        bb.utils.rename(os.path.join(self.dldir, mirrortarball),
                   os.path.join(mirrordir, mirrortarball))
 
         # Fetch from the mirror
@@ -2037,6 +2098,8 @@ class GitLfsTest(FetcherTest):
 
         bb.utils.mkdirhier(self.srcdir)
         self.git('init', cwd=self.srcdir)
+        self.git('config user.email "you@example.com"', cwd=self.srcdir)
+        self.git('config user.name "Your Name"', cwd=self.srcdir)
         with open(os.path.join(self.srcdir, '.gitattributes'), 'wt') as attrs:
             attrs.write('*.mp3 filter=lfs -text')
         self.git(['add', '.gitattributes'], cwd=self.srcdir)
@@ -2577,3 +2640,29 @@ class NPMTest(FetcherTest):
         fetcher = bb.fetch.Fetch(['npmsw://' + swfile], self.d)
         fetcher.download()
         self.assertTrue(os.path.exists(ud.localpath))
+
+class GitSharedTest(FetcherTest):
+    def setUp(self):
+        super(GitSharedTest, self).setUp()
+        self.recipe_url = "git://git.openembedded.org/bitbake"
+        self.d.setVar('SRCREV', '82ea737a0b42a8b53e11c9cde141e9e9c0bd8c40')
+
+    @skipIfNoNetwork()
+    def test_shared_unpack(self):
+        fetcher = bb.fetch.Fetch([self.recipe_url], self.d)
+
+        fetcher.download()
+        fetcher.unpack(self.unpackdir)
+        alt = os.path.join(self.unpackdir, 'git/.git/objects/info/alternates')
+        self.assertTrue(os.path.exists(alt))
+
+    @skipIfNoNetwork()
+    def test_noshared_unpack(self):
+        self.d.setVar('BB_GIT_NOSHARED', '1')
+        self.unpackdir += '_noshared'
+        fetcher = bb.fetch.Fetch([self.recipe_url], self.d)
+
+        fetcher.download()
+        fetcher.unpack(self.unpackdir)
+        alt = os.path.join(self.unpackdir, 'git/.git/objects/info/alternates')
+        self.assertFalse(os.path.exists(alt))

bitbake/lib/bb/tests/runqueue-tests/recipes/f1.bb

@@ -1 +1 @@
-do_install[mcdepends] = "mc:mc1:mc2:a1:do_build"
+do_install[mcdepends] = "mc:mc-1:mc_2:a1:do_build"

bitbake/lib/bb/tests/runqueue-tests/recipes/fails-mc/fails-mc1.bb

@@ -1,5 +1,5 @@
 python () {
-    if d.getVar("BB_CURRENT_MC") == "mc1":
-        bb.fatal("Multiconfig is mc1")
+    if d.getVar("BB_CURRENT_MC") == "mc-1":
+        bb.fatal("Multiconfig is mc-1")
 }
 

bitbake/lib/bb/tests/runqueue-tests/recipes/fails-mc/fails-mc2.bb

@@ -1,4 +1,4 @@
 python () {
-    if d.getVar("BB_CURRENT_MC") == "mc2":
-        bb.fatal("Multiconfig is mc2")
+    if d.getVar("BB_CURRENT_MC") == "mc_2":
+        bb.fatal("Multiconfig is mc_2")
 }

bitbake/lib/bb/tests/runqueue.py

@@ -216,66 +216,66 @@ class RunQueueTests(unittest.TestCase):
     def test_multiconfig_setscene_optimise(self):
         with tempfile.TemporaryDirectory(prefix="runqueuetest") as tempdir:
             extraenv = {
-                "BBMULTICONFIG" : "mc1 mc2",
+                "BBMULTICONFIG" : "mc-1 mc_2",
                 "BB_SIGNATURE_HANDLER" : "basic"
             }
-            cmd = ["bitbake", "b1", "mc:mc1:b1", "mc:mc2:b1"]
+            cmd = ["bitbake", "b1", "mc:mc-1:b1", "mc:mc_2:b1"]
             setscenetasks = ['package_write_ipk_setscene', 'package_write_rpm_setscene', 'packagedata_setscene',
                              'populate_sysroot_setscene', 'package_qa_setscene']
             sstatevalid = ""
             tasks = self.run_bitbakecmd(cmd, tempdir, sstatevalid, extraenv=extraenv)
             expected = ['a1:' + x for x in self.alltasks] + ['b1:' + x for x in self.alltasks] + \
-                       ['mc1:b1:' + x for x in setscenetasks] + ['mc1:a1:' + x for x in setscenetasks] + \
-                       ['mc2:b1:' + x for x in setscenetasks] + ['mc2:a1:' + x for x in setscenetasks] + \
-                       ['mc1:b1:build', 'mc2:b1:build']
-            for x in ['mc1:a1:package_qa_setscene', 'mc2:a1:package_qa_setscene', 'a1:build', 'a1:package_qa']:
+                       ['mc-1:b1:' + x for x in setscenetasks] + ['mc-1:a1:' + x for x in setscenetasks] + \
+                       ['mc_2:b1:' + x for x in setscenetasks] + ['mc_2:a1:' + x for x in setscenetasks] + \
+                       ['mc-1:b1:build', 'mc_2:b1:build']
+            for x in ['mc-1:a1:package_qa_setscene', 'mc_2:a1:package_qa_setscene', 'a1:build', 'a1:package_qa']:
                 expected.remove(x)
             self.assertEqual(set(tasks), set(expected))
 
     def test_multiconfig_bbmask(self):
         # This test validates that multiconfigs can independently mask off
         # recipes they do not want with BBMASK. It works by having recipes
-        # that will fail to parse for mc1 and mc2, then making each multiconfig
+        # that will fail to parse for mc-1 and mc_2, then making each multiconfig
         # build the one that does parse. This ensures that the recipes are in
         # each multiconfigs BBFILES, but each is masking only the one that
         # doesn't parse
         with tempfile.TemporaryDirectory(prefix="runqueuetest") as tempdir:
             extraenv = {
-                "BBMULTICONFIG" : "mc1 mc2",
+                "BBMULTICONFIG" : "mc-1 mc_2",
                 "BB_SIGNATURE_HANDLER" : "basic",
                 "EXTRA_BBFILES": "${COREBASE}/recipes/fails-mc/*.bb",
             }
-            cmd = ["bitbake", "mc:mc1:fails-mc2", "mc:mc2:fails-mc1"]
+            cmd = ["bitbake", "mc:mc-1:fails-mc2", "mc:mc_2:fails-mc1"]
             self.run_bitbakecmd(cmd, tempdir, "", extraenv=extraenv)
 
     def test_multiconfig_mcdepends(self):
         with tempfile.TemporaryDirectory(prefix="runqueuetest") as tempdir:
             extraenv = {
-                "BBMULTICONFIG" : "mc1 mc2",
+                "BBMULTICONFIG" : "mc-1 mc_2",
                 "BB_SIGNATURE_HANDLER" : "TestMulticonfigDepends",
                 "EXTRA_BBFILES": "${COREBASE}/recipes/fails-mc/*.bb",
             }
-            tasks = self.run_bitbakecmd(["bitbake", "mc:mc1:f1"], tempdir, "", extraenv=extraenv, cleanup=True)
-            expected = ["mc1:f1:%s" % t for t in self.alltasks] + \
-                       ["mc2:a1:%s" % t for t in self.alltasks]
+            tasks = self.run_bitbakecmd(["bitbake", "mc:mc-1:f1"], tempdir, "", extraenv=extraenv, cleanup=True)
+            expected = ["mc-1:f1:%s" % t for t in self.alltasks] + \
+                       ["mc_2:a1:%s" % t for t in self.alltasks]
             self.assertEqual(set(tasks), set(expected))
 
             # A rebuild does nothing
-            tasks = self.run_bitbakecmd(["bitbake", "mc:mc1:f1"], tempdir, "", extraenv=extraenv, cleanup=True)
+            tasks = self.run_bitbakecmd(["bitbake", "mc:mc-1:f1"], tempdir, "", extraenv=extraenv, cleanup=True)
             self.assertEqual(set(tasks), set())
 
             # Test that a signature change in the dependent task causes
             # mcdepends to rebuild
-            tasks = self.run_bitbakecmd(["bitbake", "mc:mc2:a1", "-c", "compile", "-f"], tempdir, "", extraenv=extraenv, cleanup=True)
-            expected = ["mc2:a1:compile"]
+            tasks = self.run_bitbakecmd(["bitbake", "mc:mc_2:a1", "-c", "compile", "-f"], tempdir, "", extraenv=extraenv, cleanup=True)
+            expected = ["mc_2:a1:compile"]
             self.assertEqual(set(tasks), set(expected))
 
             rerun_tasks = self.alltasks[:]
             for x in ("fetch", "unpack", "patch", "prepare_recipe_sysroot", "configure", "compile"):
                 rerun_tasks.remove(x)
-            tasks = self.run_bitbakecmd(["bitbake", "mc:mc1:f1"], tempdir, "", extraenv=extraenv, cleanup=True)
-            expected = ["mc1:f1:%s" % t for t in rerun_tasks] + \
-                       ["mc2:a1:%s" % t for t in rerun_tasks]
+            tasks = self.run_bitbakecmd(["bitbake", "mc:mc-1:f1"], tempdir, "", extraenv=extraenv, cleanup=True)
+            expected = ["mc-1:f1:%s" % t for t in rerun_tasks] + \
+                       ["mc_2:a1:%s" % t for t in rerun_tasks]
             self.assertEqual(set(tasks), set(expected))
 
     @unittest.skipIf(sys.version_info < (3, 5, 0), 'Python 3.5 or later required')

bitbake/lib/bb/tinfoil.py

@@ -52,6 +52,10 @@ class TinfoilDataStoreConnectorVarHistory:
     def remoteCommand(self, cmd, *args, **kwargs):
         return self.tinfoil.run_command('dataStoreConnectorVarHistCmd', self.dsindex, cmd, args, kwargs)
 
+    def emit(self, var, oval, val, o, d):
+        ret = self.tinfoil.run_command('dataStoreConnectorVarHistCmdEmit', self.dsindex, var, oval, val, d.dsindex)
+        o.write(ret)
+
     def __getattr__(self, name):
         if not hasattr(bb.data_smart.VariableHistory, name):
             raise AttributeError("VariableHistory has no such method %s" % name)
@@ -440,7 +444,7 @@ class Tinfoil:
         to initialise Tinfoil and use it with config_only=True first and
         then conditionally call this function to parse recipes later.
         """
-        config_params = TinfoilConfigParameters(config_only=False)
+        config_params = TinfoilConfigParameters(config_only=False, quiet=self.quiet)
         self.run_actions(config_params)
         self.recipes_parsed = True
 

bitbake/lib/bb/ui/buildinfohelper.py

@@ -148,14 +148,14 @@ class ORMWrapper(object):
         buildrequest = None
         if brbe is not None:
             # Toaster-triggered build
-            logger.debug(1, "buildinfohelper: brbe is %s" % brbe)
+            logger.debug("buildinfohelper: brbe is %s" % brbe)
             br, _ = brbe.split(":")
             buildrequest = BuildRequest.objects.get(pk=br)
             prj = buildrequest.project
         else:
             # CLI build
             prj = Project.objects.get_or_create_default_project()
-            logger.debug(1, "buildinfohelper: project is not specified, defaulting to %s" % prj)
+            logger.debug("buildinfohelper: project is not specified, defaulting to %s" % prj)
 
         if buildrequest is not None:
             # reuse existing Build object
@@ -171,7 +171,7 @@ class ORMWrapper(object):
                 completed_on=now,
                 build_name='')
 
-        logger.debug(1, "buildinfohelper: build is created %s" % build)
+        logger.debug("buildinfohelper: build is created %s" % build)
 
         if buildrequest is not None:
             buildrequest.build = build
@@ -906,7 +906,7 @@ class BuildInfoHelper(object):
 
         self.project = None
 
-        logger.debug(1, "buildinfohelper: Build info helper inited %s" % vars(self))
+        logger.debug("buildinfohelper: Build info helper inited %s" % vars(self))
 
 
     ###################
@@ -1620,7 +1620,7 @@ class BuildInfoHelper(object):
             # if we have a backlog of events, do our best to save them here
             if len(self.internal_state['backlog']):
                 tempevent = self.internal_state['backlog'].pop()
-                logger.debug(1, "buildinfohelper: Saving stored event %s "
+                logger.debug("buildinfohelper: Saving stored event %s "
                              % tempevent)
                 self.store_log_event(tempevent,cli_backlog)
             else:

bitbake/lib/bb/ui/knotty.py

@@ -21,6 +21,7 @@ import fcntl
 import struct
 import copy
 import atexit
+from itertools import groupby
 
 from bb.ui import uihelper
 
@@ -539,6 +540,13 @@ def main(server, eventHandler, params, tf = TerminalFilter):
         except OSError:
            pass
 
+    # Add the logging domains specified by the user on the command line
+    for (domainarg, iterator) in groupby(params.debug_domains):
+        dlevel = len(tuple(iterator))
+        l = logconfig["loggers"].setdefault("BitBake.%s" % domainarg, {})
+        l["level"] = logging.DEBUG - dlevel + 1
+        l.setdefault("handlers", []).extend(["BitBake.verbconsole"])
+
     conf = bb.msg.setLoggingConfig(logconfig, logconfigfile)
 
     if sys.stdin.isatty() and sys.stdout.isatty():

bitbake/lib/bb/utils.py

@@ -129,6 +129,7 @@ def vercmp(ta, tb):
     return r
 
 def vercmp_string(a, b):
+    """ Split version strings and compare them """
     ta = split_version(a)
     tb = split_version(b)
     return vercmp(ta, tb)
@@ -247,6 +248,12 @@ def explode_dep_versions2(s, *, sort=True):
     return r
 
 def explode_dep_versions(s):
+    """
+    Take an RDEPENDS style string of format:
+    "DEPEND1 (optional version) DEPEND2 (optional version) ..."
+    skip null value and items appeared in dependancy string multiple times
+    and return a dictionary of dependencies and versions.
+    """
     r = explode_dep_versions2(s)
     for d in r:
         if not r[d]:
@@ -602,7 +609,7 @@ def filter_environment(good_vars):
     os.environ["LC_ALL"] = "en_US.UTF-8"
 
     if removed_vars:
-        logger.debug(1, "Removed the following variables from the environment: %s", ", ".join(removed_vars.keys()))
+        logger.debug("Removed the following variables from the environment: %s", ", ".join(removed_vars.keys()))
 
     return removed_vars
 
@@ -692,7 +699,7 @@ def remove(path, recurse=False, ionice=False):
                 raise
 
 def prunedir(topdir, ionice=False):
-    # Delete everything reachable from the directory named in 'topdir'.
+    """ Delete everything reachable from the directory named in 'topdir'. """
     # CAUTION:  This is dangerous!
     if _check_unsafe_delete_path(topdir):
         raise Exception('bb.utils.prunedir: called with dangerous path "%s", refusing to delete!' % topdir)
@@ -703,8 +710,10 @@ def prunedir(topdir, ionice=False):
 # but thats possibly insane and suffixes is probably going to be small
 #
 def prune_suffix(var, suffixes, d):
-    # See if var ends with any of the suffixes listed and
-    # remove it if found
+    """ 
+    See if var ends with any of the suffixes listed and
+    remove it if found 
+    """
     for suffix in suffixes:
         if suffix and var.endswith(suffix):
             return var[:-len(suffix)]
@@ -773,7 +782,7 @@ def movefile(src, dest, newmtime = None, sstat = None):
 
     if sstat[stat.ST_DEV] == dstat[stat.ST_DEV]:
         try:
-            os.rename(src, destpath)
+            bb.utils.rename(src, destpath)
             renamefailed = 0
         except Exception as e:
             if e.errno != errno.EXDEV:
@@ -787,7 +796,7 @@ def movefile(src, dest, newmtime = None, sstat = None):
         if stat.S_ISREG(sstat[stat.ST_MODE]):
             try: # For safety copy then move it over.
                 shutil.copyfile(src, destpath + "#new")
-                os.rename(destpath + "#new", destpath)
+                bb.utils.rename(destpath + "#new", destpath)
                 didcopy = 1
             except Exception as e:
                 print('movefile: copy', src, '->', dest, 'failed.', e)
@@ -865,7 +874,7 @@ def copyfile(src, dest, newmtime = None, sstat = None):
 
             # For safety copy then move it over.
             shutil.copyfile(src, dest + "#new")
-            os.rename(dest + "#new", dest)
+            bb.utils.rename(dest + "#new", dest)
         except Exception as e:
             logger.warning("copyfile: copy %s to %s failed (%s)" % (src, dest, e))
             return False
@@ -956,6 +965,10 @@ def umask(new_mask):
         os.umask(current_mask)
 
 def to_boolean(string, default=None):
+    """ 
+    Check input string and return boolean value True/False/None
+    depending upon the checks 
+    """
     if not string:
         return default
 
@@ -999,6 +1012,23 @@ def contains(variable, checkvalues, truevalue, falsevalue, d):
     return falsevalue
 
 def contains_any(variable, checkvalues, truevalue, falsevalue, d):
+    """Check if a variable contains any values specified.
+
+    Arguments:
+
+    variable -- the variable name. This will be fetched and expanded (using
+    d.getVar(variable)) and then split into a set().
+
+    checkvalues -- if this is a string it is split on whitespace into a set(),
+    otherwise coerced directly into a set().
+
+    truevalue -- the value to return if checkvalues is a subset of variable.
+
+    falsevalue -- the value to return if variable is empty or if checkvalues is
+    not a subset of variable.
+
+    d -- the data store.
+    """
     val = d.getVar(variable)
     if not val:
         return falsevalue
@@ -1560,8 +1590,8 @@ def set_process_name(name):
     except:
         pass
 
-# export common proxies variables from datastore to environment
 def export_proxies(d):
+    """ export common proxies variables from datastore to environment """
     import os
 
     variables = ['http_proxy', 'HTTP_PROXY', 'https_proxy', 'HTTPS_PROXY',
@@ -1583,12 +1613,12 @@ def export_proxies(d):
 
 def load_plugins(logger, plugins, pluginpath):
     def load_plugin(name):
-        logger.debug(1, 'Loading plugin %s' % name)
+        logger.debug('Loading plugin %s' % name)
         spec = importlib.machinery.PathFinder.find_spec(name, path=[pluginpath] )
         if spec:
             return spec.loader.load_module()
 
-    logger.debug(1, 'Loading plugins from %s...' % pluginpath)
+    logger.debug('Loading plugins from %s...' % pluginpath)
 
     expanded = (glob.glob(os.path.join(pluginpath, '*' + ext))
                 for ext in python_extensions)
@@ -1639,3 +1669,15 @@ def is_semver(version):
         return False
 
     return True
+
+# Wrapper around os.rename which can handle cross device problems
+# e.g. from container filesystems
+def rename(src, dst):
+    try:
+        os.rename(src, dst)
+    except OSError as err:
+        if err.errno == 18:
+            # Invalid cross-device link error
+            shutil.move(src, dst)
+        else:
+            raise err

bitbake/lib/bblayers/action.py

@@ -50,10 +50,10 @@ class ActionPlugin(LayerPlugin):
             if not (args.force or notadded):
                 try:
                     self.tinfoil.run_command('parseConfiguration')
-                except bb.tinfoil.TinfoilUIException:
+                except (bb.tinfoil.TinfoilUIException, bb.BBHandledException):
                     # Restore the back up copy of bblayers.conf
                     shutil.copy2(backup, bblayers_conf)
-                    bb.fatal("Parse failure with the specified layer added")
+                    bb.fatal("Parse failure with the specified layer added, aborting.")
                 else:
                     for item in notadded:
                         sys.stderr.write("Specified layer %s is already in BBLAYERS\n" % item)

bitbake/lib/bblayers/layerindex.py

@@ -79,7 +79,7 @@ class LayerIndexPlugin(ActionPlugin):
             branches = [args.branch]
         else:
             branches = (self.tinfoil.config_data.getVar('LAYERSERIES_CORENAMES') or 'master').split()
-        logger.debug(1, 'Trying branches: %s' % branches)
+        logger.debug('Trying branches: %s' % branches)
 
         ignore_layers = []
         if args.ignore:
@@ -159,12 +159,17 @@ class LayerIndexPlugin(ActionPlugin):
                 logger.plain('  recommended by: %s' % ' '.join(recommendedby))
 
         if dependencies:
-            fetchdir = self.tinfoil.config_data.getVar('BBLAYERS_FETCH_DIR')
-            if not fetchdir:
-                logger.error("Cannot get BBLAYERS_FETCH_DIR")
-                return 1
+            if args.fetchdir:
+                fetchdir = args.fetchdir
+            else:
+                fetchdir = self.tinfoil.config_data.getVar('BBLAYERS_FETCH_DIR')
+                if not fetchdir:
+                    logger.error("Cannot get BBLAYERS_FETCH_DIR")
+                    return 1
+
             if not os.path.exists(fetchdir):
                 os.makedirs(fetchdir)
+
             addlayers = []
 
             for deplayerbranch in dependencies:
@@ -206,6 +211,8 @@ class LayerIndexPlugin(ActionPlugin):
 """
         args.show_only = True
         args.ignore = []
+        args.fetchdir = ""
+        args.shallow = True
         self.do_layerindex_fetch(args)
 
     def register_commands(self, sp):
@@ -214,6 +221,7 @@ class LayerIndexPlugin(ActionPlugin):
         parser_layerindex_fetch.add_argument('-b', '--branch', help='branch name to fetch')
         parser_layerindex_fetch.add_argument('-s', '--shallow', help='do only shallow clones (--depth=1)', action='store_true')
         parser_layerindex_fetch.add_argument('-i', '--ignore', help='assume the specified layers do not need to be fetched/added (separate multiple layers with commas, no spaces)', metavar='LAYER')
+        parser_layerindex_fetch.add_argument('-f', '--fetchdir', help='directory to fetch the layer(s) into (will be created if it does not exist)')
         parser_layerindex_fetch.add_argument('layername', nargs='+', help='layer to fetch')
 
         parser_layerindex_show_depends = self.add_command(sp, 'layerindex-show-depends', self.do_layerindex_show_depends, parserecipes=False)

bitbake/lib/bblayers/query.py

@@ -128,7 +128,7 @@ skipped recipes will also be listed, with a " (skipped)" suffix.
                     sys.exit(1)
 
         pkg_pn = self.tinfoil.cooker.recipecaches[mc].pkg_pn
-        (latest_versions, preferred_versions) = self.tinfoil.find_providers(mc)
+        (latest_versions, preferred_versions, required_versions) = self.tinfoil.find_providers(mc)
         allproviders = self.tinfoil.get_all_providers(mc)
 
         # Ensure we list skipped recipes

bitbake/lib/hashserv/__init__.py

@@ -3,6 +3,7 @@
 # SPDX-License-Identifier: GPL-2.0-only
 #
 
+import asyncio
 from contextlib import closing
 import re
 import sqlite3
@@ -21,6 +22,24 @@ ADDR_TYPE_TCP = 1
 # is necessary
 DEFAULT_MAX_CHUNK = 32 * 1024
 
+TABLE_DEFINITION = (
+    ("method", "TEXT NOT NULL"),
+    ("outhash", "TEXT NOT NULL"),
+    ("taskhash", "TEXT NOT NULL"),
+    ("unihash", "TEXT NOT NULL"),
+    ("created", "DATETIME"),
+
+    # Optional fields
+    ("owner", "TEXT"),
+    ("PN", "TEXT"),
+    ("PV", "TEXT"),
+    ("PR", "TEXT"),
+    ("task", "TEXT"),
+    ("outhash_siginfo", "TEXT"),
+)
+
+TABLE_COLUMNS = tuple(name for name, _ in TABLE_DEFINITION)
+
 def setup_database(database, sync=True):
     db = sqlite3.connect(database)
     db.row_factory = sqlite3.Row
@@ -29,23 +48,10 @@ def setup_database(database, sync=True):
         cursor.execute('''
             CREATE TABLE IF NOT EXISTS tasks_v2 (
                 id INTEGER PRIMARY KEY AUTOINCREMENT,
-                method TEXT NOT NULL,
-                outhash TEXT NOT NULL,
-                taskhash TEXT NOT NULL,
-                unihash TEXT NOT NULL,
-                created DATETIME,
-
-                -- Optional fields
-                owner TEXT,
-                PN TEXT,
-                PV TEXT,
-                PR TEXT,
-                task TEXT,
-                outhash_siginfo TEXT,
-
+                %s
                 UNIQUE(method, outhash, taskhash)
                 )
-            ''')
+            ''' % " ".join("%s %s," % (name, typ) for name, typ in TABLE_DEFINITION))
         cursor.execute('PRAGMA journal_mode = WAL')
         cursor.execute('PRAGMA synchronous = %s' % ('NORMAL' if sync else 'OFF'))
 
@@ -88,10 +94,10 @@ def chunkify(msg, max_chunk):
         yield "\n"
 
 
-def create_server(addr, dbname, *, sync=True):
+def create_server(addr, dbname, *, sync=True, upstream=None, read_only=False):
     from . import server
     db = setup_database(dbname, sync=sync)
-    s = server.Server(db)
+    s = server.Server(db, upstream=upstream, read_only=read_only)
 
     (typ, a) = parse_address(addr)
     if typ == ADDR_TYPE_UNIX:
@@ -113,3 +119,15 @@ def create_client(addr):
         c.connect_tcp(*a)
 
     return c
+
+async def create_async_client(addr):
+    from . import client
+    c = client.AsyncClient()
+
+    (typ, a) = parse_address(addr)
+    if typ == ADDR_TYPE_UNIX:
+        await c.connect_unix(*a)
+    else:
+        await c.connect_tcp(*a)
+
+    return c

bitbake/lib/hashserv/client.py

@@ -3,189 +3,119 @@
 # SPDX-License-Identifier: GPL-2.0-only
 #
 
-import json
 import logging
 import socket
-import os
-from . import chunkify, DEFAULT_MAX_CHUNK
+import bb.asyncrpc
+from . import create_async_client
 
 
-logger = logging.getLogger('hashserv.client')
+logger = logging.getLogger("hashserv.client")
 
 
-class HashConnectionError(Exception):
-    pass
-
-
-class Client(object):
+class AsyncClient(bb.asyncrpc.AsyncClient):
     MODE_NORMAL = 0
     MODE_GET_STREAM = 1
 
     def __init__(self):
-        self._socket = None
-        self.reader = None
-        self.writer = None
+        super().__init__('OEHASHEQUIV', '1.1', logger)
         self.mode = self.MODE_NORMAL
-        self.max_chunk = DEFAULT_MAX_CHUNK
 
-    def connect_tcp(self, address, port):
-        def connect_sock():
-            s = socket.create_connection((address, port))
+    async def setup_connection(self):
+        await super().setup_connection()
+        cur_mode = self.mode
+        self.mode = self.MODE_NORMAL
+        await self._set_mode(cur_mode)
 
-            s.setsockopt(socket.SOL_TCP, socket.TCP_NODELAY, 1)
-            s.setsockopt(socket.SOL_TCP, socket.TCP_QUICKACK, 1)
-            s.setsockopt(socket.SOL_SOCKET, socket.SO_KEEPALIVE, 1)
-            return s
-
-        self._connect_sock = connect_sock
-
-    def connect_unix(self, path):
-        def connect_sock():
-            s = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
-            # AF_UNIX has path length issues so chdir here to workaround
-            cwd = os.getcwd()
-            try:
-                os.chdir(os.path.dirname(path))
-                s.connect(os.path.basename(path))
-            finally:
-                os.chdir(cwd)
-            return s
-
-        self._connect_sock = connect_sock
-
-    def connect(self):
-        if self._socket is None:
-            self._socket = self._connect_sock()
-
-            self.reader = self._socket.makefile('r', encoding='utf-8')
-            self.writer = self._socket.makefile('w', encoding='utf-8')
-
-            self.writer.write('OEHASHEQUIV 1.1\n\n')
-            self.writer.flush()
-
-            # Restore mode if the socket is being re-created
-            cur_mode = self.mode
-            self.mode = self.MODE_NORMAL
-            self._set_mode(cur_mode)
-
-        return self._socket
-
-    def close(self):
-        if self._socket is not None:
-            self._socket.close()
-            self._socket = None
-            self.reader = None
-            self.writer = None
-
-    def _send_wrapper(self, proc):
-        count = 0
-        while True:
-            try:
-                self.connect()
-                return proc()
-            except (OSError, HashConnectionError, json.JSONDecodeError, UnicodeDecodeError) as e:
-                logger.warning('Error talking to server: %s' % e)
-                if count >= 3:
-                    if not isinstance(e, HashConnectionError):
-                        raise HashConnectionError(str(e))
-                    raise e
-                self.close()
-                count += 1
-
-    def send_message(self, msg):
-        def get_line():
-            line = self.reader.readline()
-            if not line:
-                raise HashConnectionError('Connection closed')
-
-            if not line.endswith('\n'):
-                raise HashConnectionError('Bad message %r' % message)
-
-            return line
-
-        def proc():
-            for c in chunkify(json.dumps(msg), self.max_chunk):
-                self.writer.write(c)
-            self.writer.flush()
-
-            l = get_line()
-
-            m = json.loads(l)
-            if 'chunk-stream' in m:
-                lines = []
-                while True:
-                    l = get_line().rstrip('\n')
-                    if not l:
-                        break
-                    lines.append(l)
-
-                m = json.loads(''.join(lines))
-
-            return m
-
-        return self._send_wrapper(proc)
-
-    def send_stream(self, msg):
-        def proc():
-            self.writer.write("%s\n" % msg)
-            self.writer.flush()
-            l = self.reader.readline()
+    async def send_stream(self, msg):
+        async def proc():
+            self.writer.write(("%s\n" % msg).encode("utf-8"))
+            await self.writer.drain()
+            l = await self.reader.readline()
             if not l:
-                raise HashConnectionError('Connection closed')
-            return l.rstrip()
+                raise ConnectionError("Connection closed")
+            return l.decode("utf-8").rstrip()
 
-        return self._send_wrapper(proc)
+        return await self._send_wrapper(proc)
 
-    def _set_mode(self, new_mode):
+    async def _set_mode(self, new_mode):
         if new_mode == self.MODE_NORMAL and self.mode == self.MODE_GET_STREAM:
-            r = self.send_stream('END')
-            if r != 'ok':
-                raise HashConnectionError('Bad response from server %r' % r)
+            r = await self.send_stream("END")
+            if r != "ok":
+                raise ConnectionError("Bad response from server %r" % r)
         elif new_mode == self.MODE_GET_STREAM and self.mode == self.MODE_NORMAL:
-            r = self.send_message({'get-stream': None})
-            if r != 'ok':
-                raise HashConnectionError('Bad response from server %r' % r)
+            r = await self.send_message({"get-stream": None})
+            if r != "ok":
+                raise ConnectionError("Bad response from server %r" % r)
         elif new_mode != self.mode:
-            raise Exception('Undefined mode transition %r -> %r' % (self.mode, new_mode))
+            raise Exception(
+                "Undefined mode transition %r -> %r" % (self.mode, new_mode)
+            )
 
         self.mode = new_mode
 
-    def get_unihash(self, method, taskhash):
-        self._set_mode(self.MODE_GET_STREAM)
-        r = self.send_stream('%s %s' % (method, taskhash))
+    async def get_unihash(self, method, taskhash):
+        await self._set_mode(self.MODE_GET_STREAM)
+        r = await self.send_stream("%s %s" % (method, taskhash))
         if not r:
             return None
         return r
 
-    def report_unihash(self, taskhash, method, outhash, unihash, extra={}):
-        self._set_mode(self.MODE_NORMAL)
+    async def report_unihash(self, taskhash, method, outhash, unihash, extra={}):
+        await self._set_mode(self.MODE_NORMAL)
         m = extra.copy()
-        m['taskhash'] = taskhash
-        m['method'] = method
-        m['outhash'] = outhash
-        m['unihash'] = unihash
-        return self.send_message({'report': m})
+        m["taskhash"] = taskhash
+        m["method"] = method
+        m["outhash"] = outhash
+        m["unihash"] = unihash
+        return await self.send_message({"report": m})
 
-    def report_unihash_equiv(self, taskhash, method, unihash, extra={}):
-        self._set_mode(self.MODE_NORMAL)
+    async def report_unihash_equiv(self, taskhash, method, unihash, extra={}):
+        await self._set_mode(self.MODE_NORMAL)
         m = extra.copy()
-        m['taskhash'] = taskhash
-        m['method'] = method
-        m['unihash'] = unihash
-        return self.send_message({'report-equiv': m})
+        m["taskhash"] = taskhash
+        m["method"] = method
+        m["unihash"] = unihash
+        return await self.send_message({"report-equiv": m})
 
-    def get_taskhash(self, method, taskhash, all_properties=False):
-        self._set_mode(self.MODE_NORMAL)
-        return self.send_message({'get': {
-            'taskhash': taskhash,
-            'method': method,
-            'all': all_properties
-        }})
+    async def get_taskhash(self, method, taskhash, all_properties=False):
+        await self._set_mode(self.MODE_NORMAL)
+        return await self.send_message(
+            {"get": {"taskhash": taskhash, "method": method, "all": all_properties}}
+        )
 
-    def get_stats(self):
-        self._set_mode(self.MODE_NORMAL)
-        return self.send_message({'get-stats': None})
+    async def get_outhash(self, method, outhash, taskhash):
+        await self._set_mode(self.MODE_NORMAL)
+        return await self.send_message(
+            {"get-outhash": {"outhash": outhash, "taskhash": taskhash, "method": method}}
+        )
 
-    def reset_stats(self):
-        self._set_mode(self.MODE_NORMAL)
-        return self.send_message({'reset-stats': None})
+    async def get_stats(self):
+        await self._set_mode(self.MODE_NORMAL)
+        return await self.send_message({"get-stats": None})
+
+    async def reset_stats(self):
+        await self._set_mode(self.MODE_NORMAL)
+        return await self.send_message({"reset-stats": None})
+
+    async def backfill_wait(self):
+        await self._set_mode(self.MODE_NORMAL)
+        return (await self.send_message({"backfill-wait": None}))["tasks"]
+
+
+class Client(bb.asyncrpc.Client):
+    def __init__(self):
+        super().__init__()
+        self._add_methods(
+            "connect_tcp",
+            "close",
+            "get_unihash",
+            "report_unihash",
+            "report_unihash_equiv",
+            "get_taskhash",
+            "get_stats",
+            "reset_stats",
+            "backfill_wait",
+        )
+
+    def _get_async_client(self):
+        return AsyncClient()

bitbake/lib/hashserv/server.py

@@ -3,17 +3,15 @@
 # SPDX-License-Identifier: GPL-2.0-only
 #
 
-from contextlib import closing
+from contextlib import closing, contextmanager
 from datetime import datetime
 import asyncio
-import json
 import logging
 import math
-import os
-import signal
-import socket
 import time
-from . import chunkify, DEFAULT_MAX_CHUNK
+from . import create_async_client, TABLE_COLUMNS
+import bb.asyncrpc
+
 
 logger = logging.getLogger('hashserv.server')
 
@@ -108,70 +106,92 @@ class Stats(object):
         return {k: getattr(self, k) for k in ('num', 'total_time', 'max_time', 'average', 'stdev')}
 
 
-class ClientError(Exception):
-    pass
+def insert_task(cursor, data, ignore=False):
+    keys = sorted(data.keys())
+    query = '''INSERT%s INTO tasks_v2 (%s) VALUES (%s)''' % (
+        " OR IGNORE" if ignore else "",
+        ', '.join(keys),
+        ', '.join(':' + k for k in keys))
+    cursor.execute(query, data)
 
-class ServerClient(object):
+async def copy_from_upstream(client, db, method, taskhash):
+    d = await client.get_taskhash(method, taskhash, True)
+    if d is not None:
+        # Filter out unknown columns
+        d = {k: v for k, v in d.items() if k in TABLE_COLUMNS}
+
+        with closing(db.cursor()) as cursor:
+            insert_task(cursor, d)
+            db.commit()
+
+    return d
+
+async def copy_outhash_from_upstream(client, db, method, outhash, taskhash):
+    d = await client.get_outhash(method, outhash, taskhash)
+    if d is not None:
+        # Filter out unknown columns
+        d = {k: v for k, v in d.items() if k in TABLE_COLUMNS}
+
+        with closing(db.cursor()) as cursor:
+            insert_task(cursor, d)
+            db.commit()
+
+    return d
+
+class ServerClient(bb.asyncrpc.AsyncServerConnection):
     FAST_QUERY = 'SELECT taskhash, method, unihash FROM tasks_v2 WHERE method=:method AND taskhash=:taskhash ORDER BY created ASC LIMIT 1'
     ALL_QUERY =  'SELECT *                         FROM tasks_v2 WHERE method=:method AND taskhash=:taskhash ORDER BY created ASC LIMIT 1'
+    OUTHASH_QUERY = '''
+        -- Find tasks with a matching outhash (that is, tasks that
+        -- are equivalent)
+        SELECT * FROM tasks_v2 WHERE method=:method AND outhash=:outhash
 
-    def __init__(self, reader, writer, db, request_stats):
-        self.reader = reader
-        self.writer = writer
+        -- If there is an exact match on the taskhash, return it.
+        -- Otherwise return the oldest matching outhash of any
+        -- taskhash
+        ORDER BY CASE WHEN taskhash=:taskhash THEN 1 ELSE 2 END,
+            created ASC
+
+        -- Only return one row
+        LIMIT 1
+        '''
+
+    def __init__(self, reader, writer, db, request_stats, backfill_queue, upstream, read_only):
+        super().__init__(reader, writer, 'OEHASHEQUIV', logger)
         self.db = db
         self.request_stats = request_stats
-        self.max_chunk = DEFAULT_MAX_CHUNK
+        self.max_chunk = bb.asyncrpc.DEFAULT_MAX_CHUNK
+        self.backfill_queue = backfill_queue
+        self.upstream = upstream
 
-        self.handlers = {
+        self.handlers.update({
             'get': self.handle_get,
-            'report': self.handle_report,
-            'report-equiv': self.handle_equivreport,
+            'get-outhash': self.handle_get_outhash,
             'get-stream': self.handle_get_stream,
             'get-stats': self.handle_get_stats,
-            'reset-stats': self.handle_reset_stats,
-            'chunk-stream': self.handle_chunk,
-        }
+        })
+
+        if not read_only:
+            self.handlers.update({
+                'report': self.handle_report,
+                'report-equiv': self.handle_equivreport,
+                'reset-stats': self.handle_reset_stats,
+                'backfill-wait': self.handle_backfill_wait,
+            })
+
+    def validate_proto_version(self):
+        return (self.proto_version > (1, 0) and self.proto_version <= (1, 1))
 
     async def process_requests(self):
-        try:
-            self.addr = self.writer.get_extra_info('peername')
-            logger.debug('Client %r connected' % (self.addr,))
+        if self.upstream is not None:
+            self.upstream_client = await create_async_client(self.upstream)
+        else:
+            self.upstream_client = None
 
-            # Read protocol and version
-            protocol = await self.reader.readline()
-            if protocol is None:
-                return
+        await super().process_requests()
 
-            (proto_name, proto_version) = protocol.decode('utf-8').rstrip().split()
-            if proto_name != 'OEHASHEQUIV':
-                return
-
-            proto_version = tuple(int(v) for v in proto_version.split('.'))
-            if proto_version < (1, 0) or proto_version > (1, 1):
-                return
-
-            # Read headers. Currently, no headers are implemented, so look for
-            # an empty line to signal the end of the headers
-            while True:
-                line = await self.reader.readline()
-                if line is None:
-                    return
-
-                line = line.decode('utf-8').rstrip()
-                if not line:
-                    break
-
-            # Handle messages
-            while True:
-                d = await self.read_message()
-                if d is None:
-                    break
-                await self.dispatch_message(d)
-                await self.writer.drain()
-        except ClientError as e:
-            logger.error(str(e))
-        finally:
-            self.writer.close()
+        if self.upstream_client is not None:
+            await self.upstream_client.close()
 
     async def dispatch_message(self, msg):
         for k in self.handlers.keys():
@@ -185,47 +205,7 @@ class ServerClient(object):
                         await self.handlers[k](msg[k])
                 return
 
-        raise ClientError("Unrecognized command %r" % msg)
-
-    def write_message(self, msg):
-        for c in chunkify(json.dumps(msg), self.max_chunk):
-            self.writer.write(c.encode('utf-8'))
-
-    async def read_message(self):
-        l = await self.reader.readline()
-        if not l:
-            return None
-
-        try:
-            message = l.decode('utf-8')
-
-            if not message.endswith('\n'):
-                return None
-
-            return json.loads(message)
-        except (json.JSONDecodeError, UnicodeDecodeError) as e:
-            logger.error('Bad message from client: %r' % message)
-            raise e
-
-    async def handle_chunk(self, request):
-        lines = []
-        try:
-            while True:
-                l = await self.reader.readline()
-                l = l.rstrip(b"\n").decode("utf-8")
-                if not l:
-                    break
-                lines.append(l)
-
-            msg = json.loads(''.join(lines))
-        except (json.JSONDecodeError, UnicodeDecodeError) as e:
-            logger.error('Bad message from client: %r' % message)
-            raise e
-
-        if 'chunk-stream' in msg:
-            raise ClientError("Nested chunks are not allowed")
-
-        await self.dispatch_message(msg)
+        raise bb.asyncrpc.ClientError("Unrecognized command %r" % msg)
 
     async def handle_get(self, request):
         method = request['method']
@@ -239,15 +219,34 @@ class ServerClient(object):
         if row is not None:
             logger.debug('Found equivalent task %s -> %s', (row['taskhash'], row['unihash']))
             d = {k: row[k] for k in row.keys()}
-
-            self.write_message(d)
+        elif self.upstream_client is not None:
+            d = await copy_from_upstream(self.upstream_client, self.db, method, taskhash)
         else:
-            self.write_message(None)
+            d = None
+
+        self.write_message(d)
+
+    async def handle_get_outhash(self, request):
+        with closing(self.db.cursor()) as cursor:
+            cursor.execute(self.OUTHASH_QUERY,
+                           {k: request[k] for k in ('method', 'outhash', 'taskhash')})
+
+            row = cursor.fetchone()
+
+        if row is not None:
+            logger.debug('Found equivalent outhash %s -> %s', (row['outhash'], row['unihash']))
+            d = {k: row[k] for k in row.keys()}
+        else:
+            d = None
+
+        self.write_message(d)
 
     async def handle_get_stream(self, request):
         self.write_message('ok')
 
         while True:
+            upstream = None
+
             l = await self.reader.readline()
             if not l:
                 return
@@ -272,6 +271,12 @@ class ServerClient(object):
                 if row is not None:
                     msg = ('%s\n' % row['unihash']).encode('utf-8')
                     #logger.debug('Found equivalent task %s -> %s', (row['taskhash'], row['unihash']))
+                elif self.upstream_client is not None:
+                    upstream = await self.upstream_client.get_unihash(method, taskhash)
+                    if upstream:
+                        msg = ("%s\n" % upstream).encode("utf-8")
+                    else:
+                        msg = "\n".encode("utf-8")
                 else:
                     msg = '\n'.encode('utf-8')
 
@@ -282,25 +287,26 @@ class ServerClient(object):
 
             await self.writer.drain()
 
+            # Post to the backfill queue after writing the result to minimize
+            # the turn around time on a request
+            if upstream is not None:
+                await self.backfill_queue.put((method, taskhash))
+
     async def handle_report(self, data):
         with closing(self.db.cursor()) as cursor:
-            cursor.execute('''
-                -- Find tasks with a matching outhash (that is, tasks that
-                -- are equivalent)
-                SELECT taskhash, method, unihash FROM tasks_v2 WHERE method=:method AND outhash=:outhash
-
-                -- If there is an exact match on the taskhash, return it.
-                -- Otherwise return the oldest matching outhash of any
-                -- taskhash
-                ORDER BY CASE WHEN taskhash=:taskhash THEN 1 ELSE 2 END,
-                    created ASC
-
-                -- Only return one row
-                LIMIT 1
-                ''', {k: data[k] for k in ('method', 'outhash', 'taskhash')})
+            cursor.execute(self.OUTHASH_QUERY,
+                           {k: data[k] for k in ('method', 'outhash', 'taskhash')})
 
             row = cursor.fetchone()
 
+            if row is None and self.upstream_client:
+                # Try upstream
+                row = await copy_outhash_from_upstream(self.upstream_client,
+                                                       self.db,
+                                                       data['method'],
+                                                       data['outhash'],
+                                                       data['taskhash'])
+
             # If no matching outhash was found, or one *was* found but it
             # wasn't an exact match on the taskhash, a new entry for this
             # taskhash should be added
@@ -324,11 +330,7 @@ class ServerClient(object):
                     if k in data:
                         insert_data[k] = data[k]
 
-                cursor.execute('''INSERT INTO tasks_v2 (%s) VALUES (%s)''' % (
-                    ', '.join(sorted(insert_data.keys())),
-                    ', '.join(':' + k for k in sorted(insert_data.keys()))),
-                    insert_data)
-
+                insert_task(cursor, insert_data)
                 self.db.commit()
 
                 logger.info('Adding taskhash %s with unihash %s',
@@ -358,11 +360,7 @@ class ServerClient(object):
                 if k in data:
                     insert_data[k] = data[k]
 
-            cursor.execute('''INSERT OR IGNORE INTO tasks_v2 (%s) VALUES (%s)''' % (
-                ', '.join(sorted(insert_data.keys())),
-                ', '.join(':' + k for k in sorted(insert_data.keys()))),
-                insert_data)
-
+            insert_task(cursor, insert_data, ignore=True)
             self.db.commit()
 
             # Fetch the unihash that will be reported for the taskhash. If the
@@ -394,6 +392,13 @@ class ServerClient(object):
         self.request_stats.reset()
         self.write_message(d)
 
+    async def handle_backfill_wait(self, request):
+        d = {
+            'tasks': self.backfill_queue.qsize(),
+        }
+        await self.backfill_queue.join()
+        self.write_message(d)
+
     def query_equivalent(self, method, taskhash, query):
         # This is part of the inner loop and must be as fast as possible
         try:
@@ -404,86 +409,52 @@ class ServerClient(object):
             cursor.close()
 
 
-class Server(object):
-    def __init__(self, db, loop=None):
+class Server(bb.asyncrpc.AsyncServer):
+    def __init__(self, db, loop=None, upstream=None, read_only=False):
+        if upstream and read_only:
+            raise bb.asyncrpc.ServerError("Read-only hashserv cannot pull from an upstream server")
+
+        super().__init__(logger, loop)
+
         self.request_stats = Stats()
         self.db = db
+        self.upstream = upstream
+        self.read_only = read_only
 
-        if loop is None:
-            self.loop = asyncio.new_event_loop()
-            self.close_loop = True
+    def accept_client(self, reader, writer):
+        return ServerClient(reader, writer, self.db, self.request_stats, self.backfill_queue, self.upstream, self.read_only)
+
+    @contextmanager
+    def _backfill_worker(self):
+        async def backfill_worker_task():
+            client = await create_async_client(self.upstream)
+            try:
+                while True:
+                    item = await self.backfill_queue.get()
+                    if item is None:
+                        self.backfill_queue.task_done()
+                        break
+                    method, taskhash = item
+                    await copy_from_upstream(client, self.db, method, taskhash)
+                    self.backfill_queue.task_done()
+            finally:
+                await client.close()
+
+        async def join_worker(worker):
+            await self.backfill_queue.put(None)
+            await worker
+
+        if self.upstream is not None:
+            worker = asyncio.ensure_future(backfill_worker_task())
+            try:
+                yield
+            finally:
+                self.loop.run_until_complete(join_worker(worker))
         else:
-            self.loop = loop
-            self.close_loop = False
+            yield
 
-        self._cleanup_socket = None
+    def run_loop_forever(self):
+        self.backfill_queue = asyncio.Queue()
 
-    def start_tcp_server(self, host, port):
-        self.server = self.loop.run_until_complete(
-            asyncio.start_server(self.handle_client, host, port, loop=self.loop)
-        )
-
-        for s in self.server.sockets:
-            logger.info('Listening on %r' % (s.getsockname(),))
-            # Newer python does this automatically. Do it manually here for
-            # maximum compatibility
-            s.setsockopt(socket.SOL_TCP, socket.TCP_NODELAY, 1)
-            s.setsockopt(socket.SOL_TCP, socket.TCP_QUICKACK, 1)
-
-        name = self.server.sockets[0].getsockname()
-        if self.server.sockets[0].family == socket.AF_INET6:
-            self.address = "[%s]:%d" % (name[0], name[1])
-        else:
-            self.address = "%s:%d" % (name[0], name[1])
-
-    def start_unix_server(self, path):
-        def cleanup():
-            os.unlink(path)
-
-        cwd = os.getcwd()
-        try:
-            # Work around path length limits in AF_UNIX
-            os.chdir(os.path.dirname(path))
-            self.server = self.loop.run_until_complete(
-                asyncio.start_unix_server(self.handle_client, os.path.basename(path), loop=self.loop)
-            )
-        finally:
-            os.chdir(cwd)
-
-        logger.info('Listening on %r' % path)
-
-        self._cleanup_socket = cleanup
-        self.address = "unix://%s" % os.path.abspath(path)
-
-    async def handle_client(self, reader, writer):
-        # writer.transport.set_write_buffer_limits(0)
-        try:
-            client = ServerClient(reader, writer, self.db, self.request_stats)
-            await client.process_requests()
-        except Exception as e:
-            import traceback
-            logger.error('Error from client: %s' % str(e), exc_info=True)
-            traceback.print_exc()
-            writer.close()
-        logger.info('Client disconnected')
-
-    def serve_forever(self):
-        def signal_handler():
-            self.loop.stop()
-
-        self.loop.add_signal_handler(signal.SIGTERM, signal_handler)
-
-        try:
-            self.loop.run_forever()
-        except KeyboardInterrupt:
-            pass
-
-        self.server.close()
-        self.loop.run_until_complete(self.server.wait_closed())
-        logger.info('Server shutting down')
-
-        if self.close_loop:
-            self.loop.close()
-
-        if self._cleanup_socket is not None:
-            self._cleanup_socket()
+        with self._backfill_worker():
+            super().run_loop_forever()

bitbake/lib/hashserv/tests.py

@@ -16,44 +16,68 @@ import threading
 import unittest
 import socket
 
+def _run_server(server, idx):
+    # logging.basicConfig(level=logging.DEBUG, filename='bbhashserv.log', filemode='w',
+    #                     format='%(levelname)s %(filename)s:%(lineno)d %(message)s')
+    sys.stdout = open('bbhashserv-%d.log' % idx, 'w')
+    sys.stderr = sys.stdout
+    server.serve_forever()
 
-class TestHashEquivalenceServer(object):
+
+class HashEquivalenceTestSetup(object):
     METHOD = 'TestMethod'
 
-    def _run_server(self):
-        # logging.basicConfig(level=logging.DEBUG, filename='bbhashserv.log', filemode='w',
-        #                     format='%(levelname)s %(filename)s:%(lineno)d %(message)s')
-        self.server.serve_forever()
+    server_index = 0
+
+    def start_server(self, dbpath=None, upstream=None, read_only=False):
+        self.server_index += 1
+        if dbpath is None:
+            dbpath = os.path.join(self.temp_dir.name, "db%d.sqlite" % self.server_index)
+
+        def cleanup_thread(thread):
+            thread.terminate()
+            thread.join()
+
+        server = create_server(self.get_server_addr(self.server_index),
+                               dbpath,
+                               upstream=upstream,
+                               read_only=read_only)
+        server.dbpath = dbpath
+
+        server.thread = multiprocessing.Process(target=_run_server, args=(server, self.server_index))
+        server.thread.start()
+        self.addCleanup(cleanup_thread, server.thread)
+
+        def cleanup_client(client):
+            client.close()
+
+        client = create_client(server.address)
+        self.addCleanup(cleanup_client, client)
+
+        return (client, server)
 
     def setUp(self):
         if sys.version_info < (3, 5, 0):
             self.skipTest('Python 3.5 or later required')
 
         self.temp_dir = tempfile.TemporaryDirectory(prefix='bb-hashserv')
-        self.dbfile = os.path.join(self.temp_dir.name, 'db.sqlite')
+        self.addCleanup(self.temp_dir.cleanup)
 
-        self.server = create_server(self.get_server_addr(), self.dbfile)
-        self.server_thread = multiprocessing.Process(target=self._run_server)
-        self.server_thread.start()
-        self.client = create_client(self.server.address)
+        (self.client, self.server) = self.start_server()
 
-    def tearDown(self):
-        # Shutdown server
-        s = getattr(self, 'server', None)
-        if s is not None:
-            self.server_thread.terminate()
-            self.server_thread.join()
-        self.client.close()
-        self.temp_dir.cleanup()
+    def assertClientGetHash(self, client, taskhash, unihash):
+        result = client.get_unihash(self.METHOD, taskhash)
+        self.assertEqual(result, unihash)
 
+
+class HashEquivalenceCommonTests(object):
     def test_create_hash(self):
         # Simple test that hashes can be created
         taskhash = '35788efcb8dfb0a02659d81cf2bfd695fb30faf9'
         outhash = '2765d4a5884be49b28601445c2760c5f21e7e5c0ee2b7e3fce98fd7e5970796f'
         unihash = 'f46d3fbb439bd9b921095da657a4de906510d2cd'
 
-        result = self.client.get_unihash(self.METHOD, taskhash)
-        self.assertIsNone(result, msg='Found unexpected task, %r' % result)
+        self.assertClientGetHash(self.client, taskhash, None)
 
         result = self.client.report_unihash(taskhash, self.METHOD, outhash, unihash)
         self.assertEqual(result['unihash'], unihash, 'Server returned bad unihash')
@@ -84,22 +108,19 @@ class TestHashEquivalenceServer(object):
         unihash = '218e57509998197d570e2c98512d0105985dffc9'
         self.client.report_unihash(taskhash, self.METHOD, outhash, unihash)
 
-        result = self.client.get_unihash(self.METHOD, taskhash)
-        self.assertEqual(result, unihash)
+        self.assertClientGetHash(self.client, taskhash, unihash)
 
         outhash2 = '0904a7fe3dc712d9fd8a74a616ddca2a825a8ee97adf0bd3fc86082c7639914d'
         unihash2 = 'ae9a7d252735f0dafcdb10e2e02561ca3a47314c'
         self.client.report_unihash(taskhash, self.METHOD, outhash2, unihash2)
 
-        result = self.client.get_unihash(self.METHOD, taskhash)
-        self.assertEqual(result, unihash)
+        self.assertClientGetHash(self.client, taskhash, unihash)
 
         outhash3 = '77623a549b5b1a31e3732dfa8fe61d7ce5d44b3370f253c5360e136b852967b4'
         unihash3 = '9217a7d6398518e5dc002ed58f2cbbbc78696603'
         self.client.report_unihash(taskhash, self.METHOD, outhash3, unihash3)
 
-        result = self.client.get_unihash(self.METHOD, taskhash)
-        self.assertEqual(result, unihash)
+        self.assertClientGetHash(self.client, taskhash, unihash)
 
     def test_huge_message(self):
         # Simple test that hashes can be created
@@ -107,8 +128,7 @@ class TestHashEquivalenceServer(object):
         outhash = '3c979c3db45c569f51ab7626a4651074be3a9d11a84b1db076f5b14f7d39db44'
         unihash = '90e9bc1d1f094c51824adca7f8ea79a048d68824'
 
-        result = self.client.get_unihash(self.METHOD, taskhash)
-        self.assertIsNone(result, msg='Found unexpected task, %r' % result)
+        self.assertClientGetHash(self.client, taskhash, None)
 
         siginfo = "0" * (self.client.max_chunk * 4)
 
@@ -156,16 +176,140 @@ class TestHashEquivalenceServer(object):
 
         self.assertFalse(failures)
 
+    def test_upstream_server(self):
+        # Tests upstream server support. This is done by creating two servers
+        # that share a database file. The downstream server has it upstream
+        # set to the test server, whereas the side server doesn't. This allows
+        # verification that the hash requests are being proxied to the upstream
+        # server by verifying that they appear on the downstream client, but not
+        # the side client. It also verifies that the results are pulled into
+        # the downstream database by checking that the downstream and side servers
+        # match after the downstream is done waiting for all backfill tasks
+        (down_client, down_server) = self.start_server(upstream=self.server.address)
+        (side_client, side_server) = self.start_server(dbpath=down_server.dbpath)
 
-class TestHashEquivalenceUnixServer(TestHashEquivalenceServer, unittest.TestCase):
-    def get_server_addr(self):
-        return "unix://" + os.path.join(self.temp_dir.name, 'sock')
+        def check_hash(taskhash, unihash, old_sidehash):
+            nonlocal down_client
+            nonlocal side_client
+
+            # check upstream server
+            self.assertClientGetHash(self.client, taskhash, unihash)
+
+            # Hash should *not* be present on the side server
+            self.assertClientGetHash(side_client, taskhash, old_sidehash)
+
+            # Hash should be present on the downstream server, since it
+            # will defer to the upstream server. This will trigger
+            # the backfill in the downstream server
+            self.assertClientGetHash(down_client, taskhash, unihash)
+
+            # After waiting for the downstream client to finish backfilling the
+            # task from the upstream server, it should appear in the side server
+            # since the database is populated
+            down_client.backfill_wait()
+            self.assertClientGetHash(side_client, taskhash, unihash)
+
+        # Basic report
+        taskhash = '8aa96fcffb5831b3c2c0cb75f0431e3f8b20554a'
+        outhash = 'afe240a439959ce86f5e322f8c208e1fedefea9e813f2140c81af866cc9edf7e'
+        unihash = '218e57509998197d570e2c98512d0105985dffc9'
+        self.client.report_unihash(taskhash, self.METHOD, outhash, unihash)
+
+        check_hash(taskhash, unihash, None)
+
+        # Duplicated taskhash with multiple output hashes and unihashes.
+        # All servers should agree with the originally reported hash
+        outhash2 = '0904a7fe3dc712d9fd8a74a616ddca2a825a8ee97adf0bd3fc86082c7639914d'
+        unihash2 = 'ae9a7d252735f0dafcdb10e2e02561ca3a47314c'
+        self.client.report_unihash(taskhash, self.METHOD, outhash2, unihash2)
+
+        check_hash(taskhash, unihash, unihash)
+
+        # Report an equivalent task. The sideload will originally report
+        # no unihash until backfilled
+        taskhash3 = "044c2ec8aaf480685a00ff6ff49e6162e6ad34e1"
+        unihash3 = "def64766090d28f627e816454ed46894bb3aab36"
+        self.client.report_unihash(taskhash3, self.METHOD, outhash, unihash3)
+
+        check_hash(taskhash3, unihash, None)
+
+        # Test that reporting a unihash in the downstream client isn't
+        # propagating to the upstream server
+        taskhash4 = "e3da00593d6a7fb435c7e2114976c59c5fd6d561"
+        outhash4 = "1cf8713e645f491eb9c959d20b5cae1c47133a292626dda9b10709857cbe688a"
+        unihash4 = "3b5d3d83f07f259e9086fcb422c855286e18a57d"
+        down_client.report_unihash(taskhash4, self.METHOD, outhash4, unihash4)
+        down_client.backfill_wait()
+
+        self.assertClientGetHash(down_client, taskhash4, unihash4)
+        self.assertClientGetHash(side_client, taskhash4, unihash4)
+        self.assertClientGetHash(self.client, taskhash4, None)
+
+        # Test that reporting a unihash in the downstream is able to find a
+        # match which was previously reported to the upstream server
+        taskhash5 = '35788efcb8dfb0a02659d81cf2bfd695fb30faf9'
+        outhash5 = '2765d4a5884be49b28601445c2760c5f21e7e5c0ee2b7e3fce98fd7e5970796f'
+        unihash5 = 'f46d3fbb439bd9b921095da657a4de906510d2cd'
+        result = self.client.report_unihash(taskhash5, self.METHOD, outhash5, unihash5)
+
+        taskhash6 = '35788efcb8dfb0a02659d81cf2bfd695fb30fafa'
+        unihash6 = 'f46d3fbb439bd9b921095da657a4de906510d2ce'
+        result = down_client.report_unihash(taskhash6, self.METHOD, outhash5, unihash6)
+        self.assertEqual(result['unihash'], unihash5, 'Server failed to copy unihash from upstream')
+
+    def test_ro_server(self):
+        (ro_client, ro_server) = self.start_server(dbpath=self.server.dbpath, read_only=True)
+
+        # Report a hash via the read-write server
+        taskhash = '35788efcb8dfb0a02659d81cf2bfd695fb30faf9'
+        outhash = '2765d4a5884be49b28601445c2760c5f21e7e5c0ee2b7e3fce98fd7e5970796f'
+        unihash = 'f46d3fbb439bd9b921095da657a4de906510d2cd'
+
+        result = self.client.report_unihash(taskhash, self.METHOD, outhash, unihash)
+        self.assertEqual(result['unihash'], unihash, 'Server returned bad unihash')
+
+        # Check the hash via the read-only server
+        self.assertClientGetHash(ro_client, taskhash, unihash)
+
+        # Ensure that reporting via the read-only server fails
+        taskhash2 = 'c665584ee6817aa99edfc77a44dd853828279370'
+        outhash2 = '3c979c3db45c569f51ab7626a4651074be3a9d11a84b1db076f5b14f7d39db44'
+        unihash2 = '90e9bc1d1f094c51824adca7f8ea79a048d68824'
+
+        with self.assertRaises(ConnectionError):
+            ro_client.report_unihash(taskhash2, self.METHOD, outhash2, unihash2)
+
+        # Ensure that the database was not modified
+        self.assertClientGetHash(self.client, taskhash2, None)
 
 
-class TestHashEquivalenceTCPServer(TestHashEquivalenceServer, unittest.TestCase):
-    def get_server_addr(self):
+class TestHashEquivalenceUnixServer(HashEquivalenceTestSetup, HashEquivalenceCommonTests, unittest.TestCase):
+    def get_server_addr(self, server_idx):
+        return "unix://" + os.path.join(self.temp_dir.name, 'sock%d' % server_idx)
+
+
+class TestHashEquivalenceUnixServerLongPath(HashEquivalenceTestSetup, unittest.TestCase):
+    DEEP_DIRECTORY = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa/bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb/ccccccccccccccccccccccccccccccccccccccccccc"
+    def get_server_addr(self, server_idx):
+        os.makedirs(os.path.join(self.temp_dir.name, self.DEEP_DIRECTORY), exist_ok=True)
+        return "unix://" + os.path.join(self.temp_dir.name, self.DEEP_DIRECTORY, 'sock%d' % server_idx)
+
+
+    def test_long_sock_path(self):
+        # Simple test that hashes can be created
+        taskhash = '35788efcb8dfb0a02659d81cf2bfd695fb30faf9'
+        outhash = '2765d4a5884be49b28601445c2760c5f21e7e5c0ee2b7e3fce98fd7e5970796f'
+        unihash = 'f46d3fbb439bd9b921095da657a4de906510d2cd'
+
+        self.assertClientGetHash(self.client, taskhash, None)
+
+        result = self.client.report_unihash(taskhash, self.METHOD, outhash, unihash)
+        self.assertEqual(result['unihash'], unihash, 'Server returned bad unihash')
+
+
+class TestHashEquivalenceTCPServer(HashEquivalenceTestSetup, HashEquivalenceCommonTests, unittest.TestCase):
+    def get_server_addr(self, server_idx):
         # Some hosts cause asyncio module to misbehave, when IPv6 is not enabled.
         # If IPv6 is enabled, it should be safe to use localhost directly, in general
         # case it is more reliable to resolve the IP address explicitly.
         return socket.gethostbyname("localhost") + ":0"
-            

bitbake/lib/layerindexlib/__init__.py

@@ -94,7 +94,7 @@ class LayerIndex():
            if not param:
                continue
            item = param.split('=', 1)
-           logger.debug(1, item)
+           logger.debug(item)
            param_dict[item[0]] = item[1]
 
         return param_dict
@@ -123,7 +123,7 @@ class LayerIndex():
         up = urlparse(url)
 
         if username:
-            logger.debug(1, "Configuring authentication for %s..." % url)
+            logger.debug("Configuring authentication for %s..." % url)
             password_mgr = urllib.request.HTTPPasswordMgrWithDefaultRealm()
             password_mgr.add_password(None, "%s://%s" % (up.scheme, up.netloc), username, password)
             handler = urllib.request.HTTPBasicAuthHandler(password_mgr)
@@ -133,20 +133,20 @@ class LayerIndex():
 
         urllib.request.install_opener(opener)
 
-        logger.debug(1, "Fetching %s (%s)..." % (url, ["without authentication", "with authentication"][bool(username)]))
+        logger.debug("Fetching %s (%s)..." % (url, ["without authentication", "with authentication"][bool(username)]))
 
         try:
             res = urlopen(Request(url, headers={'User-Agent': 'Mozilla/5.0 (bitbake/lib/layerindex)'}, unverifiable=True))
         except urllib.error.HTTPError as e:
-            logger.debug(1, "HTTP Error: %s: %s" % (e.code, e.reason))
-            logger.debug(1, " Requested: %s" % (url))
-            logger.debug(1, " Actual:    %s" % (e.geturl()))
+            logger.debug("HTTP Error: %s: %s" % (e.code, e.reason))
+            logger.debug(" Requested: %s" % (url))
+            logger.debug(" Actual:    %s" % (e.geturl()))
 
             if e.code == 404:
-                logger.debug(1, "Request not found.")
+                logger.debug("Request not found.")
                 raise LayerIndexFetchError(url, e)
             else:
-                logger.debug(1, "Headers:\n%s" % (e.headers))
+                logger.debug("Headers:\n%s" % (e.headers))
                 raise LayerIndexFetchError(url, e)
         except OSError as e:
             error = 0
@@ -170,7 +170,7 @@ class LayerIndex():
                 raise LayerIndexFetchError(url, "Unable to fetch OSError exception: %s" % e)
 
         finally:
-            logger.debug(1, "...fetching %s (%s), done." % (url, ["without authentication", "with authentication"][bool(username)]))
+            logger.debug("...fetching %s (%s), done." % (url, ["without authentication", "with authentication"][bool(username)]))
 
         return res
 
@@ -205,14 +205,14 @@ The format of the indexURI:
         if reload:
             self.indexes = []
 
-        logger.debug(1, 'Loading: %s' % indexURI)
+        logger.debug('Loading: %s' % indexURI)
 
         if not self.plugins:
             raise LayerIndexException("No LayerIndex Plugins available")
 
         for plugin in self.plugins:
             # Check if the plugin was initialized
-            logger.debug(1, 'Trying %s' % plugin.__class__)
+            logger.debug('Trying %s' % plugin.__class__)
             if not hasattr(plugin, 'type') or not plugin.type:
                 continue
             try:
@@ -220,11 +220,11 @@ The format of the indexURI:
                 indexEnt = plugin.load_index(indexURI, load)
                 break
             except LayerIndexPluginUrlError as e:
-                logger.debug(1, "%s doesn't support %s" % (plugin.type, e.url))
+                logger.debug("%s doesn't support %s" % (plugin.type, e.url))
             except NotImplementedError:
                 pass
         else:
-            logger.debug(1, "No plugins support %s" % indexURI)
+            logger.debug("No plugins support %s" % indexURI)
             raise LayerIndexException("No plugins support %s" % indexURI)
 
         # Mark CONFIG data as something we've added...
@@ -255,19 +255,19 @@ will write out the individual elements split by layer and related components.
 
         for plugin in self.plugins:
             # Check if the plugin was initialized
-            logger.debug(1, 'Trying %s' % plugin.__class__)
+            logger.debug('Trying %s' % plugin.__class__)
             if not hasattr(plugin, 'type') or not plugin.type:
                 continue
             try:
                 plugin.store_index(indexURI, index)
                 break
             except LayerIndexPluginUrlError as e:
-                logger.debug(1, "%s doesn't support %s" % (plugin.type, e.url))
+                logger.debug("%s doesn't support %s" % (plugin.type, e.url))
             except NotImplementedError:
-                logger.debug(1, "Store not implemented in %s" % plugin.type)
+                logger.debug("Store not implemented in %s" % plugin.type)
                 pass
         else:
-            logger.debug(1, "No plugins support %s" % indexURI)
+            logger.debug("No plugins support %s" % indexURI)
             raise LayerIndexException("No plugins support %s" % indexURI)
 
 
@@ -292,7 +292,7 @@ layerBranches set.  If not, they are effectively blank.'''
            the default configuration until the first vcs_url/branch match.'''
 
         for index in self.indexes:
-            logger.debug(1, ' searching %s' % index.config['DESCRIPTION'])
+            logger.debug(' searching %s' % index.config['DESCRIPTION'])
             layerBranch = index.find_vcs_url(vcs_url, [branch])
             if layerBranch:
                 return layerBranch
@@ -304,7 +304,7 @@ layerBranches set.  If not, they are effectively blank.'''
            If a branch has not been specified, we will iterate over the branches in
            the default configuration until the first collection/branch match.'''
 
-        logger.debug(1, 'find_collection: %s (%s) %s' % (collection, version, branch))
+        logger.debug('find_collection: %s (%s) %s' % (collection, version, branch))
 
         if branch:
             branches = [branch]
@@ -312,12 +312,12 @@ layerBranches set.  If not, they are effectively blank.'''
             branches = None
 
         for index in self.indexes:
-            logger.debug(1, ' searching %s' % index.config['DESCRIPTION'])
+            logger.debug(' searching %s' % index.config['DESCRIPTION'])
             layerBranch = index.find_collection(collection, version, branches)
             if layerBranch:
                 return layerBranch
         else:
-            logger.debug(1, 'Collection %s (%s) not found for branch (%s)' % (collection, version, branch))
+            logger.debug('Collection %s (%s) not found for branch (%s)' % (collection, version, branch))
         return None
 
     def find_layerbranch(self, name, branch=None):
@@ -408,7 +408,7 @@ layerBranches set.  If not, they are effectively blank.'''
                                               version=deplayerbranch.version
                                           )
                         if rdeplayerbranch != deplayerbranch:
-                                logger.debug(1, 'Replaced %s:%s:%s with %s:%s:%s' % \
+                                logger.debug('Replaced %s:%s:%s with %s:%s:%s' % \
                                       (deplayerbranch.index.config['DESCRIPTION'],
                                        deplayerbranch.branch.name,
                                        deplayerbranch.layer.name,
@@ -1121,7 +1121,7 @@ class LayerBranch(LayerIndexItemObj):
     @property
     def branch(self):
         try:
-            logger.debug(1, "Get branch object from branches[%s]" % (self.branch_id))
+            logger.debug("Get branch object from branches[%s]" % (self.branch_id))
             return self.index.branches[self.branch_id]
         except KeyError:
             raise AttributeError('Unable to find branches in index to map branch_id %s' % self.branch_id)
@@ -1149,7 +1149,7 @@ class LayerBranch(LayerIndexItemObj):
 
     @actual_branch.setter
     def actual_branch(self, value):
-        logger.debug(1, "Set actual_branch to %s .. name is %s" % (value, self.branch.name))
+        logger.debug("Set actual_branch to %s .. name is %s" % (value, self.branch.name))
         if value != self.branch.name:
             self._setattr('actual_branch', value, prop=False)
         else:

bitbake/lib/layerindexlib/cooker.py

@@ -173,7 +173,7 @@ class CookerPlugin(layerindexlib.plugin.IndexPlugin):
         else:
             branches = ['HEAD']
 
-        logger.debug(1, "Loading cooker data branches %s" % branches)
+        logger.debug("Loading cooker data branches %s" % branches)
 
         index = self._load_bblayers(branches=branches)
 
@@ -220,7 +220,7 @@ class CookerPlugin(layerindexlib.plugin.IndexPlugin):
                                         required=required, layerbranch=layerBranchId,
                                         dependency=depLayerBranch.layer_id)
 
-                        logger.debug(1, '%s requires %s' % (layerDependency.layer.name, layerDependency.dependency.name))
+                        logger.debug('%s requires %s' % (layerDependency.layer.name, layerDependency.dependency.name))
                         index.add_element("layerDependencies", [layerDependency])
 
                     return layerDependencyId

bitbake/lib/layerindexlib/restapi.py

@@ -82,7 +82,7 @@ class RestApiPlugin(layerindexlib.plugin.IndexPlugin):
 
 
         def load_cache(path, index, branches=[]):
-            logger.debug(1, 'Loading json file %s' % path)
+            logger.debug('Loading json file %s' % path)
             with open(path, 'rt', encoding='utf-8') as f:
                 pindex = json.load(f)
 
@@ -102,7 +102,7 @@ class RestApiPlugin(layerindexlib.plugin.IndexPlugin):
             if newpBranch:
                 index.add_raw_element('branches', layerindexlib.Branch, newpBranch)
             else:
-                logger.debug(1, 'No matching branches (%s) in index file(s)' % branches)
+                logger.debug('No matching branches (%s) in index file(s)' % branches)
                 # No matching branches.. return nothing...
                 return
 
@@ -120,7 +120,7 @@ class RestApiPlugin(layerindexlib.plugin.IndexPlugin):
             load_cache(up.path, index, branches)
             return index
 
-        logger.debug(1, 'Loading from dir %s...' % (up.path))
+        logger.debug('Loading from dir %s...' % (up.path))
         for (dirpath, _, filenames) in os.walk(up.path):
             for filename in filenames:
                 if not filename.endswith('.json'):
@@ -144,7 +144,7 @@ class RestApiPlugin(layerindexlib.plugin.IndexPlugin):
         def _get_json_response(apiurl=None, username=None, password=None, retry=True):
             assert apiurl is not None
 
-            logger.debug(1, "fetching %s" % apiurl)
+            logger.debug("fetching %s" % apiurl)
 
             up = urlparse(apiurl)
 
@@ -163,9 +163,9 @@ class RestApiPlugin(layerindexlib.plugin.IndexPlugin):
                 parsed = json.loads(res.read().decode('utf-8'))
             except ConnectionResetError:
                 if retry:
-                    logger.debug(1, "%s: Connection reset by peer.  Retrying..." % url)
+                    logger.debug("%s: Connection reset by peer.  Retrying..." % url)
                     parsed = _get_json_response(apiurl=up_stripped.geturl(), username=username, password=password, retry=False)
-                    logger.debug(1, "%s: retry successful.")
+                    logger.debug("%s: retry successful.")
                 else:
                     raise layerindexlib.LayerIndexFetchError('%s: Connection reset by peer.  Is there a firewall blocking your connection?' % apiurl)
 
@@ -207,25 +207,25 @@ class RestApiPlugin(layerindexlib.plugin.IndexPlugin):
         if "*" not in branches:
             filter = "?filter=name:%s" % "OR".join(branches)
 
-        logger.debug(1, "Loading %s from %s" % (branches, index.apilinks['branches']))
+        logger.debug("Loading %s from %s" % (branches, index.apilinks['branches']))
 
         # The link won't include username/password, so pull it from the original url
         pindex['branches'] = _get_json_response(index.apilinks['branches'] + filter,
                                                     username=up.username, password=up.password)
         if not pindex['branches']:
-            logger.debug(1, "No valid branches (%s) found at url %s." % (branch, url))
+            logger.debug("No valid branches (%s) found at url %s." % (branch, url))
             return index
         index.add_raw_element("branches", layerindexlib.Branch, pindex['branches'])
 
         # Load all of the layerItems (these can not be easily filtered)
-        logger.debug(1, "Loading %s from %s" % ('layerItems', index.apilinks['layerItems']))
+        logger.debug("Loading %s from %s" % ('layerItems', index.apilinks['layerItems']))
 
 
         # The link won't include username/password, so pull it from the original url
         pindex['layerItems'] = _get_json_response(index.apilinks['layerItems'],
                                                   username=up.username, password=up.password)
         if not pindex['layerItems']:
-            logger.debug(1, "No layers were found at url %s." % (url))
+            logger.debug("No layers were found at url %s." % (url))
             return index
         index.add_raw_element("layerItems", layerindexlib.LayerItem, pindex['layerItems'])
 
@@ -235,13 +235,13 @@ class RestApiPlugin(layerindexlib.plugin.IndexPlugin):
         for branch in index.branches:
             filter = "?filter=branch__name:%s" % index.branches[branch].name
 
-            logger.debug(1, "Loading %s from %s" % ('layerBranches', index.apilinks['layerBranches']))
+            logger.debug("Loading %s from %s" % ('layerBranches', index.apilinks['layerBranches']))
 
             # The link won't include username/password, so pull it from the original url
             pindex['layerBranches'] = _get_json_response(index.apilinks['layerBranches'] + filter,
                                                   username=up.username, password=up.password)
             if not pindex['layerBranches']:
-                logger.debug(1, "No valid layer branches (%s) found at url %s." % (branches or "*", url))
+                logger.debug("No valid layer branches (%s) found at url %s." % (branches or "*", url))
                 return index
             index.add_raw_element("layerBranches", layerindexlib.LayerBranch, pindex['layerBranches'])
 
@@ -256,7 +256,7 @@ class RestApiPlugin(layerindexlib.plugin.IndexPlugin):
                                    ("distros", layerindexlib.Distro)]:
                 if lName not in load:
                     continue
-                logger.debug(1, "Loading %s from %s" % (lName, index.apilinks[lName]))
+                logger.debug("Loading %s from %s" % (lName, index.apilinks[lName]))
 
                 # The link won't include username/password, so pull it from the original url
                 pindex[lName] = _get_json_response(index.apilinks[lName] + filter,
@@ -283,7 +283,7 @@ class RestApiPlugin(layerindexlib.plugin.IndexPlugin):
         if up.scheme != 'file':
             raise layerindexlib.plugin.LayerIndexPluginUrlError(self.type, url)
 
-        logger.debug(1, "Storing to %s..." % up.path)
+        logger.debug("Storing to %s..." % up.path)
 
         try:
             layerbranches = index.layerBranches
@@ -299,12 +299,12 @@ class RestApiPlugin(layerindexlib.plugin.IndexPlugin):
                     if getattr(index, objects)[obj].layerbranch_id == layerbranchid:
                        filtered.append(getattr(index, objects)[obj]._data)
                 except AttributeError:
-                    logger.debug(1, 'No obj.layerbranch_id: %s' % objects)
+                    logger.debug('No obj.layerbranch_id: %s' % objects)
                     # No simple filter method, just include it...
                     try:
                         filtered.append(getattr(index, objects)[obj]._data)
                     except AttributeError:
-                        logger.debug(1, 'No obj._data: %s %s' % (objects, type(obj)))
+                        logger.debug('No obj._data: %s %s' % (objects, type(obj)))
                         filtered.append(obj)
             return filtered
 

bitbake/lib/layerindexlib/tests/cooker.py

@@ -72,7 +72,7 @@ class LayerIndexCookerTest(LayersTest):
 
     def test_find_collection(self):
         def _check(collection, expected):
-            self.logger.debug(1, "Looking for collection %s..." % collection)
+            self.logger.debug("Looking for collection %s..." % collection)
             result = self.layerindex.find_collection(collection)
             if expected:
                 self.assertIsNotNone(result, msg="Did not find %s when it shouldn't be there" % collection)
@@ -91,7 +91,7 @@ class LayerIndexCookerTest(LayersTest):
 
     def test_find_layerbranch(self):
         def _check(name, expected):
-            self.logger.debug(1, "Looking for layerbranch %s..." % name)
+            self.logger.debug("Looking for layerbranch %s..." % name)
             result = self.layerindex.find_layerbranch(name)
             if expected:
                 self.assertIsNotNone(result, msg="Did not find %s when it shouldn't be there" % collection)

bitbake/lib/layerindexlib/tests/restapi.py

@@ -57,11 +57,11 @@ class LayerIndexWebRestApiTest(LayersTest):
                type in self.layerindex.indexes[0].config['local']:
                 continue
             for id in getattr(self.layerindex.indexes[0], type):
-                self.logger.debug(1, "type %s" % (type))
+                self.logger.debug("type %s" % (type))
 
                 self.assertTrue(id in getattr(reload.indexes[0], type), msg="Id number not in reloaded index")
 
-                self.logger.debug(1, "%s ? %s" % (getattr(self.layerindex.indexes[0], type)[id], getattr(reload.indexes[0], type)[id]))
+                self.logger.debug("%s ? %s" % (getattr(self.layerindex.indexes[0], type)[id], getattr(reload.indexes[0], type)[id]))
 
                 self.assertEqual(getattr(self.layerindex.indexes[0], type)[id], getattr(reload.indexes[0], type)[id], msg="Reloaded contents different")
 
@@ -80,11 +80,11 @@ class LayerIndexWebRestApiTest(LayersTest):
                type in self.layerindex.indexes[0].config['local']:
                 continue
             for id in getattr(self.layerindex.indexes[0] ,type):
-                self.logger.debug(1, "type %s" % (type))
+                self.logger.debug("type %s" % (type))
 
                 self.assertTrue(id in getattr(reload.indexes[0], type), msg="Id number missing from reloaded data")
 
-                self.logger.debug(1, "%s ? %s" % (getattr(self.layerindex.indexes[0] ,type)[id], getattr(reload.indexes[0], type)[id]))
+                self.logger.debug("%s ? %s" % (getattr(self.layerindex.indexes[0] ,type)[id], getattr(reload.indexes[0], type)[id]))
 
                 self.assertEqual(getattr(self.layerindex.indexes[0] ,type)[id], getattr(reload.indexes[0], type)[id], msg="reloaded data does not match original")
 
@@ -111,14 +111,14 @@ class LayerIndexWebRestApiTest(LayersTest):
                 if dep.layer.name == 'meta-python':
                     break
             else:
-                self.logger.debug(1, "meta-python was not found")
+                self.logger.debug("meta-python was not found")
                 raise self.failureException
 
             # Only check the first element...
             break
         else:
             # Empty list, this is bad.
-            self.logger.debug(1, "Empty list of dependencies")
+            self.logger.debug("Empty list of dependencies")
             self.assertIsNotNone(first, msg="Empty list of dependencies")
 
             # Last dep should be the requested item
@@ -128,7 +128,7 @@ class LayerIndexWebRestApiTest(LayersTest):
     @skipIfNoNetwork()
     def test_find_collection(self):
         def _check(collection, expected):
-            self.logger.debug(1, "Looking for collection %s..." % collection)
+            self.logger.debug("Looking for collection %s..." % collection)
             result = self.layerindex.find_collection(collection)
             if expected:
                 self.assertIsNotNone(result, msg="Did not find %s when it should be there" % collection)
@@ -148,11 +148,11 @@ class LayerIndexWebRestApiTest(LayersTest):
     @skipIfNoNetwork()
     def test_find_layerbranch(self):
         def _check(name, expected):
-            self.logger.debug(1, "Looking for layerbranch %s..." % name)
+            self.logger.debug("Looking for layerbranch %s..." % name)
 
             for index in self.layerindex.indexes:
                 for layerbranchid in index.layerBranches:
-                    self.logger.debug(1, "Present: %s" % index.layerBranches[layerbranchid].layer.name)
+                    self.logger.debug("Present: %s" % index.layerBranches[layerbranchid].layer.name)
             result = self.layerindex.find_layerbranch(name)
             if expected:
                 self.assertIsNotNone(result, msg="Did not find %s when it should be there" % collection)

bitbake/lib/prserv/serv.py

@@ -5,8 +5,6 @@
 import os,sys,logging
 import signal, time
 from xmlrpc.server import SimpleXMLRPCServer, SimpleXMLRPCRequestHandler
-import threading
-import queue
 import socket
 import io
 import sqlite3
@@ -14,14 +12,10 @@ import bb.server.xmlrpcclient
 import prserv
 import prserv.db
 import errno
-import select
+import multiprocessing
 
 logger = logging.getLogger("BitBake.PRserv")
 
-if sys.hexversion < 0x020600F0:
-    print("Sorry, python 2.6 or later is required.")
-    sys.exit(1)
-
 class Handler(SimpleXMLRPCRequestHandler):
     def _dispatch(self,method,params):
         try:
@@ -37,7 +31,7 @@ singleton = None
 
 
 class PRServer(SimpleXMLRPCServer):
-    def __init__(self, dbfile, logfile, interface, daemon=True):
+    def __init__(self, dbfile, logfile, interface):
         ''' constructor '''
         try:
             SimpleXMLRPCServer.__init__(self, interface,
@@ -50,57 +44,18 @@ class PRServer(SimpleXMLRPCServer):
             raise PRServiceConfigError
 
         self.dbfile=dbfile
-        self.daemon=daemon
         self.logfile=logfile
-        self.working_thread=None
         self.host, self.port = self.socket.getsockname()
-        self.pidfile=PIDPREFIX % (self.host, self.port)
 
         self.register_function(self.getPR, "getPR")
-        self.register_function(self.quit, "quit")
         self.register_function(self.ping, "ping")
         self.register_function(self.export, "export")
-        self.register_function(self.dump_db, "dump_db")
         self.register_function(self.importone, "importone")
         self.register_introspection_functions()
 
-        self.quitpipein, self.quitpipeout = os.pipe()
-
-        self.requestqueue = queue.Queue()
-        self.handlerthread = threading.Thread(target = self.process_request_thread)
-        self.handlerthread.daemon = False
-
-    def process_request_thread(self):
-        """Same as in BaseServer but as a thread.
-
-        In addition, exception handling is done here.
-
-        """
-        iter_count = 1
+        self.iter_count = 0
         # 60 iterations between syncs or sync if dirty every ~30 seconds
-        iterations_between_sync = 60
-
-        bb.utils.set_process_name("PRServ Handler")
-
-        while not self.quitflag:
-            try:
-                (request, client_address) = self.requestqueue.get(True, 30)
-            except queue.Empty:
-                self.table.sync_if_dirty()
-                continue
-            if request is None:
-                continue
-            try:
-                self.finish_request(request, client_address)
-                self.shutdown_request(request)
-                iter_count = (iter_count + 1) % iterations_between_sync
-                if iter_count == 0:
-                    self.table.sync_if_dirty()
-            except:
-                self.handle_error(request, client_address)
-                self.shutdown_request(request)
-                self.table.sync()
-            self.table.sync_if_dirty()
+        self.iterations_between_sync = 60
 
     def sigint_handler(self, signum, stack):
         if self.table:
@@ -109,11 +64,30 @@ class PRServer(SimpleXMLRPCServer):
     def sigterm_handler(self, signum, stack):
         if self.table:
             self.table.sync()
-        self.quit()
-        self.requestqueue.put((None, None))
+        raise(SystemExit)
 
     def process_request(self, request, client_address):
-        self.requestqueue.put((request, client_address))
+        if request is None:
+            return
+        try:
+            self.finish_request(request, client_address)
+            self.shutdown_request(request)
+            self.iter_count = (self.iter_count + 1) % self.iterations_between_sync
+            if self.iter_count == 0:
+                self.table.sync_if_dirty()
+        except:
+            self.handle_error(request, client_address)
+            self.shutdown_request(request)
+            self.table.sync()
+        self.table.sync_if_dirty()
+
+    def serve_forever(self, poll_interval=0.5):
+        signal.signal(signal.SIGINT, self.sigint_handler)
+        signal.signal(signal.SIGTERM, self.sigterm_handler)
+
+        self.db = prserv.db.PRData(self.dbfile)
+        self.table = self.db["PRMAIN"]
+        return super().serve_forever(poll_interval)
 
     def export(self, version=None, pkgarch=None, checksum=None, colinfo=True):
         try:
@@ -122,31 +96,11 @@ class PRServer(SimpleXMLRPCServer):
             logger.error(str(exc))
             return None
 
-    def dump_db(self):
-        """
-        Returns a script (string) that reconstructs the state of the
-        entire database at the time this function is called. The script
-        language is defined by the backing database engine, which is a
-        function of server configuration.
-        Returns None if the database engine does not support dumping to
-        script or if some other error is encountered in processing.
-        """
-        buff = io.StringIO()
-        try:
-            self.table.sync()
-            self.table.dump_db(buff)
-            return buff.getvalue()
-        except Exception as exc:
-            logger.error(str(exc))
-            return None
-        finally:
-            buff.close()
-
     def importone(self, version, pkgarch, checksum, value):
         return self.table.importone(version, pkgarch, checksum, value)
 
     def ping(self):
-        return not self.quitflag
+        return True
 
     def getinfo(self):
         return (self.host, self.port)
@@ -161,144 +115,6 @@ class PRServer(SimpleXMLRPCServer):
             logger.error(str(exc))
             return None
 
-    def quit(self):
-        self.quitflag=True
-        os.write(self.quitpipeout, b"q")
-        os.close(self.quitpipeout)
-        return
-
-    def work_forever(self,):
-        self.quitflag = False
-        # This timeout applies to the poll in TCPServer, we need the select 
-        # below to wake on our quit pipe closing. We only ever call into handle_request
-        # if there is data there.
-        self.timeout = 0.01
-
-        bb.utils.set_process_name("PRServ")
-
-        # DB connection must be created after all forks
-        self.db = prserv.db.PRData(self.dbfile)
-        self.table = self.db["PRMAIN"]
-
-        logger.info("Started PRServer with DBfile: %s, IP: %s, PORT: %s, PID: %s" %
-                     (self.dbfile, self.host, self.port, str(os.getpid())))
-
-        self.handlerthread.start()
-        while not self.quitflag:
-            ready = select.select([self.fileno(), self.quitpipein], [], [], 30)
-            if self.quitflag:
-                break
-            if self.fileno() in ready[0]:
-                self.handle_request()
-        self.handlerthread.join()
-        self.db.disconnect()
-        logger.info("PRServer: stopping...")
-        self.server_close()
-        os.close(self.quitpipein)
-        return
-
-    def start(self):
-        if self.daemon:
-            pid = self.daemonize()
-        else:
-            pid = self.fork()
-            self.pid = pid
-
-        # Ensure both the parent sees this and the child from the work_forever log entry above
-        logger.info("Started PRServer with DBfile: %s, IP: %s, PORT: %s, PID: %s" %
-                     (self.dbfile, self.host, self.port, str(pid)))
-
-    def delpid(self):
-        os.remove(self.pidfile)
-
-    def daemonize(self):
-        """
-        See Advanced Programming in the UNIX, Sec 13.3
-        """
-        try:
-            pid = os.fork()
-            if pid > 0:
-                os.waitpid(pid, 0)
-                #parent return instead of exit to give control 
-                return pid
-        except OSError as e:
-            raise Exception("%s [%d]" % (e.strerror, e.errno))
-
-        os.setsid()
-        """
-        fork again to make sure the daemon is not session leader, 
-        which prevents it from acquiring controlling terminal
-        """
-        try:
-            pid = os.fork()
-            if pid > 0: #parent
-                os._exit(0)
-        except OSError as e:
-            raise Exception("%s [%d]" % (e.strerror, e.errno))
-
-        self.cleanup_handles()
-        os._exit(0)
-
-    def fork(self):
-        try:
-            pid = os.fork()
-            if pid > 0:
-                self.socket.close() # avoid ResourceWarning in parent
-                return pid
-        except OSError as e:
-            raise Exception("%s [%d]" % (e.strerror, e.errno))
-
-        bb.utils.signal_on_parent_exit("SIGTERM")
-        self.cleanup_handles()
-        os._exit(0)
-
-    def cleanup_handles(self):
-        signal.signal(signal.SIGINT, self.sigint_handler)
-        signal.signal(signal.SIGTERM, self.sigterm_handler)
-        os.chdir("/")
-
-        sys.stdout.flush()
-        sys.stderr.flush()
-
-        # We could be called from a python thread with io.StringIO as
-        # stdout/stderr or it could be 'real' unix fd forking where we need
-        # to physically close the fds to prevent the program launching us from
-        # potentially hanging on a pipe. Handle both cases.
-        si = open('/dev/null', 'r')
-        try:
-            os.dup2(si.fileno(),sys.stdin.fileno())
-        except (AttributeError, io.UnsupportedOperation):
-            sys.stdin = si
-        so = open(self.logfile, 'a+')
-        try:
-            os.dup2(so.fileno(),sys.stdout.fileno())
-        except (AttributeError, io.UnsupportedOperation):
-            sys.stdout = so
-        try:
-            os.dup2(so.fileno(),sys.stderr.fileno())
-        except (AttributeError, io.UnsupportedOperation):
-            sys.stderr = so
-
-        # Clear out all log handlers prior to the fork() to avoid calling
-        # event handlers not part of the PRserver
-        for logger_iter in logging.Logger.manager.loggerDict.keys():
-            logging.getLogger(logger_iter).handlers = []
-
-        # Ensure logging makes it to the logfile
-        streamhandler = logging.StreamHandler()
-        streamhandler.setLevel(logging.DEBUG)
-        formatter = bb.msg.BBLogFormatter("%(levelname)s: %(message)s")
-        streamhandler.setFormatter(formatter)
-        logger.addHandler(streamhandler)
-
-        # write pidfile
-        pid = str(os.getpid())
-        with open(self.pidfile, 'w') as pf:
-            pf.write("%s\n" % pid)
-
-        self.work_forever()
-        self.delpid()
-
 class PRServSingleton(object):
     def __init__(self, dbfile, logfile, interface):
         self.dbfile = dbfile
@@ -308,8 +124,10 @@ class PRServSingleton(object):
         self.port = None
 
     def start(self):
-        self.prserv = PRServer(self.dbfile, self.logfile, self.interface, daemon=False)
-        self.prserv.start()
+        self.prserv = PRServer(self.dbfile, self.logfile, self.interface)
+        self.process = multiprocessing.Process(target=self.prserv.serve_forever)
+        self.process.start()
+
         self.host, self.port = self.prserv.getinfo()
 
     def getinfo(self):
@@ -323,13 +141,6 @@ class PRServerConnection(object):
         self.port = port
         self.connection, self.transport = bb.server.xmlrpcclient._create_server(self.host, self.port)
 
-    def terminate(self):
-        try:
-            logger.info("Terminating PRServer...")
-            self.connection.quit()
-        except Exception as exc:
-            sys.stderr.write("%s\n" % str(exc))
-
     def getPR(self, version, pkgarch, checksum):
         return self.connection.getPR(version, pkgarch, checksum)
 
@@ -339,15 +150,82 @@ class PRServerConnection(object):
     def export(self,version=None, pkgarch=None, checksum=None, colinfo=True):
         return self.connection.export(version, pkgarch, checksum, colinfo)
 
-    def dump_db(self):
-        return self.connection.dump_db()
-
     def importone(self, version, pkgarch, checksum, value):
         return self.connection.importone(version, pkgarch, checksum, value)
 
     def getinfo(self):
         return self.host, self.port
 
+def run_as_daemon(func, pidfile, logfile):
+    """
+    See Advanced Programming in the UNIX, Sec 13.3
+    """
+    try:
+        pid = os.fork()
+        if pid > 0:
+            os.waitpid(pid, 0)
+            #parent return instead of exit to give control
+            return pid
+    except OSError as e:
+        raise Exception("%s [%d]" % (e.strerror, e.errno))
+
+    os.setsid()
+    """
+    fork again to make sure the daemon is not session leader,
+    which prevents it from acquiring controlling terminal
+    """
+    try:
+        pid = os.fork()
+        if pid > 0: #parent
+            os._exit(0)
+    except OSError as e:
+        raise Exception("%s [%d]" % (e.strerror, e.errno))
+
+    os.chdir("/")
+
+    sys.stdout.flush()
+    sys.stderr.flush()
+
+    # We could be called from a python thread with io.StringIO as
+    # stdout/stderr or it could be 'real' unix fd forking where we need
+    # to physically close the fds to prevent the program launching us from
+    # potentially hanging on a pipe. Handle both cases.
+    si = open('/dev/null', 'r')
+    try:
+        os.dup2(si.fileno(),sys.stdin.fileno())
+    except (AttributeError, io.UnsupportedOperation):
+        sys.stdin = si
+    so = open(logfile, 'a+')
+    try:
+        os.dup2(so.fileno(),sys.stdout.fileno())
+    except (AttributeError, io.UnsupportedOperation):
+        sys.stdout = so
+    try:
+        os.dup2(so.fileno(),sys.stderr.fileno())
+    except (AttributeError, io.UnsupportedOperation):
+        sys.stderr = so
+
+    # Clear out all log handlers prior to the fork() to avoid calling
+    # event handlers not part of the PRserver
+    for logger_iter in logging.Logger.manager.loggerDict.keys():
+        logging.getLogger(logger_iter).handlers = []
+
+    # Ensure logging makes it to the logfile
+    streamhandler = logging.StreamHandler()
+    streamhandler.setLevel(logging.DEBUG)
+    formatter = bb.msg.BBLogFormatter("%(levelname)s: %(message)s")
+    streamhandler.setFormatter(formatter)
+    logger.addHandler(streamhandler)
+
+    # write pidfile
+    pid = str(os.getpid())
+    with open(pidfile, 'w') as pf:
+        pf.write("%s\n" % pid)
+
+    func()
+    os.remove(pidfile)
+    os._exit(0)
+
 def start_daemon(dbfile, host, port, logfile):
     ip = socket.gethostbyname(host)
     pidfile = PIDPREFIX % (ip, port)
@@ -363,7 +241,7 @@ def start_daemon(dbfile, host, port, logfile):
         return 1
 
     server = PRServer(os.path.abspath(dbfile), os.path.abspath(logfile), (ip,port))
-    server.start()
+    run_as_daemon(server.serve_forever, pidfile, os.path.abspath(logfile))
 
     # Sometimes, the port (i.e. localhost:0) indicated by the user does not match with
     # the one the server actually is listening, so at least warn the user about it
@@ -400,25 +278,13 @@ def stop_daemon(host, port):
         return 1
 
     try:
-        PRServerConnection(ip, port).terminate()
-    except:
-        logger.critical("Stop PRService %s:%d failed" % (host,port))
+        if is_running(pid):
+            print("Sending SIGTERM to pr-server.")
+            os.kill(pid, signal.SIGTERM)
+            time.sleep(0.1)
 
-    try:
-        if pid:
-            wait_timeout = 0
-            print("Waiting for pr-server to exit.")
-            while is_running(pid) and wait_timeout < 50:
-                time.sleep(0.1)
-                wait_timeout += 1
-
-            if is_running(pid):
-                print("Sending SIGTERM to pr-server.")
-                os.kill(pid,signal.SIGTERM)
-                time.sleep(0.1)
-
-            if os.path.exists(pidfile):
-                os.remove(pidfile)
+        if os.path.exists(pidfile):
+            os.remove(pidfile)
 
     except OSError as e:
         err = str(e)
@@ -494,19 +360,14 @@ def auto_start(d):
 
 def auto_shutdown():
     global singleton
-    if singleton:
-        host, port = singleton.getinfo()
-        try:
-            PRServerConnection(host, port).terminate()
-        except:
-            logger.critical("Stop PRService %s:%d failed" % (host,port))
-
-        try:
-            os.waitpid(singleton.prserv.pid, 0)
-        except ChildProcessError:
-            pass
+    if singleton and singleton.process:
+        singleton.process.terminate()
+        singleton.process.join()
         singleton = None
 
 def ping(host, port):
     conn=PRServerConnection(host, port)
     return conn.ping()
+
+def connect(host, port):
+    return PRServerConnection(host, port)

contrib/git-hooks/sendemail-validate.sample

@@ -0,0 +1,78 @@
+#!/usr/bin/env python3
+
+# Copyright (C) 2020  Agilent Technologies, Inc.
+# Author: Chris Laplante <chris.laplante@agilent.com>
+
+# This sendemail-validate hook injects 'From: ' header lines into outgoing
+# emails sent via 'git send-email', to ensure that accurate commit authorship
+# information is present. It was created because some email servers
+# (notably Microsoft Exchange / Office 360) seem to butcher outgoing patches,
+# resulting in incorrect authorship.
+
+# Current limitations:
+#   1. Assumes one per patch per email
+#   2. Minimal error checking
+#
+# Installation:
+#   1. Copy to .git/hooks/sendemail-validate
+#   2. chmod +x .git/hooks/sendemail-validate
+
+
+import enum
+import re
+import subprocess
+import sys
+
+
+class Subject(enum.IntEnum):
+    NOT_SEEN = 0
+    CONSUMING = 1
+    SEEN = 2
+
+
+def make_from_line():
+    cmd = ["git", "var", "GIT_COMMITTER_IDENT"]
+    proc = subprocess.run(cmd, check=True, stdout=subprocess.PIPE, universal_newlines=True)
+    regex = re.compile(r"^(.*>).*$")
+    match = regex.match(proc.stdout)
+    assert match is not None
+    return "From: {0}".format(match.group(1))
+
+
+def main():
+    email = sys.argv[1]
+
+    with open(email, "r") as f:
+        email_lines = f.read().split("\n")
+
+    subject_seen = Subject.NOT_SEEN
+    first_body_line = None
+    for i, line in enumerate(email_lines):
+        if (subject_seen == Subject.NOT_SEEN) and line.startswith("Subject: "):
+            subject_seen = Subject.CONSUMING
+            continue
+        if subject_seen == Subject.CONSUMING:
+            if not line.strip():
+                subject_seen = Subject.SEEN
+                continue
+        if subject_seen == Subject.SEEN:
+            first_body_line = i
+            break
+
+    assert subject_seen == Subject.SEEN
+    assert first_body_line is not None
+
+    from_line = make_from_line()
+    # Only add FROM line if it is not already there
+    if email_lines[first_body_line] != from_line:
+        email_lines.insert(first_body_line, from_line)
+        email_lines.insert(first_body_line + 1, "")
+        with open(email, "w") as f:
+            f.write("\n".join(email_lines))
+
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())
+

documentation/.gitignore

@@ -1 +1,3 @@
 _build/
+Pipfile.lock
+.vscode/

documentation/Makefile

@@ -3,7 +3,7 @@
 
 # You can set these variables from the command line, and also
 # from the environment for the first two.
-SPHINXOPTS    ?=
+SPHINXOPTS    ?= -W --keep-going -j auto
 SPHINXBUILD   ?= sphinx-build
 SOURCEDIR     = .
 BUILDDIR      = _build

documentation/Pipfile

@@ -0,0 +1,14 @@
+[[source]]
+name = "pypi"
+url = "https://pypi.org/simple"
+verify_ssl = true
+
+[dev-packages]
+
+[packages]
+sphinx = "*"
+sphinx-rtd-theme = "*"
+pyyaml = "*"
+
+[requires]
+python_version = "3"

documentation/README

@@ -2,7 +2,7 @@ documentation
 =============
 
 This is the directory that contains the Yocto Project documentation.  The Yocto
-Project source repositories at http://git.yoctoproject.org/cgit.cgi have two
+Project source repositories at https://git.yoctoproject.org/cgit.cgi have two
 instances of the "documentation" directory.  You should understand each of
 these instances.
 
@@ -32,7 +32,7 @@ these instances.
 Manual Organization
 ===================
 
-Folders exist for individual manuals as follows:
+Here the folders corresponding to individual manuals:
 
 * sdk-manual           - The Yocto Project Software Development Kit (SDK) Developer's Guide.
 * bsp-guide            - The Yocto Project Board Support Package (BSP) Developer's Guide
@@ -47,12 +47,12 @@ Folders exist for individual manuals as follows:
 Each folder is self-contained regarding content and figures.
 
 If you want to find HTML versions of the Yocto Project manuals on the web,
-go to http://www.yoctoproject.org and click on the "Documentation" tab.  From
+go to https://www.yoctoproject.org and click on the "Docs" tab.  From
 there you have access to archived documentation from previous releases, current
 documentation for the latest release, and "Docs in Progress" for the release
 currently being developed.
 
-In general, the Yocto Project site (http://www.yoctoproject.org) is a great
+In general, the Yocto Project site (https://www.yoctoproject.org) is a great
 reference for both information and downloads.
 
 poky.yaml
@@ -88,16 +88,16 @@ documentation.
 Yocto Project documentation website
 ===================================
 
-A new website has been created to host the Yocto Project
-documentation, it can be found at: https://docs.yoctoproject.org/.
+The website hosting the Yocto Project documentation, can be found
+at: https://docs.yoctoproject.org/.
 
-The entire Yocto Project documentation, as well as the BitBake manual
+The entire Yocto Project documentation, as well as the BitBake manual,
 is published on this website, including all previously released
 versions. A version switcher was added, as a drop-down menu on the top
 of the page to switch back and forth between the various versions of
 the current active Yocto Project releases.
 
-Transition pages have been added (as rst file) to show links to old
+Transition pages have been added (as rst files) to show links to old
 versions of the Yocto Project documentation with links to each manual
 generated with DocBook.
 
@@ -109,7 +109,7 @@ obvious reasons, we will only support building the Yocto Project
 documentation with Python3.
 
 Sphinx might be available in your Linux distro packages repositories,
-however it is not recommend using distro packages, as they might be
+however it is not recommended to use distro packages, as they might be
 old versions, especially if you are using an LTS version of your
 distro. The recommended method to install Sphinx and all required
 dependencies is to use the Python Package Index (pip).
@@ -127,6 +127,13 @@ The resulting HTML index page will be _build/html/index.html, and you
 can browse your own copy of the locally generated documentation with
 your browser.
 
+Alternatively, you can use Pipenv to automatically install all required
+dependencies in a virtual environment:
+
+ $ cd documentation
+ $ pipenv install
+ $ pipenv run make html
+
 Sphinx theme and CSS customization
 ==================================
 
@@ -160,7 +167,7 @@ The layout of the Yocto Project manuals is organized as follows
           Section
             Section
 
-The following headings styles are defined in Sphinx:
+Here are the heading styles defined in Sphinx:
 
     Book              => overline ===
       Chapter         => overline ***
@@ -178,7 +185,7 @@ Sphinx has a glossary directive. From
 https://www.sphinx-doc.org/en/master/usage/restructuredtext/directives.html#glossary:
 
     This directive must contain a reST definition list with terms and
-    definitions. The definitions will then be referencable with the
+    definitions. It's then possible to refer to each definition through the
     [https://www.sphinx-doc.org/en/master/usage/restructuredtext/roles.html#role-term
     'term' role].
 
@@ -199,7 +206,7 @@ however there are important shortcomings. For example they cannot be
 used/nested inside code-block sections.
 
 A Sphinx extension was implemented to support variable substitutions
-to mimic the DocBook based documentation behavior. Variabes
+to mimic the DocBook based documentation behavior. Variable
 substitutions are done while reading/parsing the .rst files. The
 pattern for variables substitutions is the same as with DocBook,
 e.g. `&VAR;`.
@@ -215,13 +222,13 @@ For example, the following .rst content will produce the 'expected'
 content:
 
   .. code-block::
-      $ mkdir ~/poky-&DISTRO;
+      $ mkdir poky-&DISTRO;
       or
       $ git clone &YOCTO_GIT_URL;/git/poky -b &DISTRO_NAME_NO_CAP;
 
 Variables can be nested, like it was the case for DocBook:
 
-  YOCTO_HOME_URL : "http://www.yoctoproject.org"
+  YOCTO_HOME_URL : "https://www.yoctoproject.org"
   YOCTO_DOCS_URL : "&YOCTO_HOME_URL;/docs"
 
 Note directive
@@ -230,14 +237,14 @@ Note directive
 Sphinx has a builtin 'note' directive that produces clean Note section
 in the output file. There are various types of directives such as
 "attention", "caution", "danger", "error", "hint", "important", "tip",
-"warning", "admonition" that are supported, and additional directive
+"warning", "admonition" that are supported, and additional directives
 can be added as Sphinx extension if needed.
 
 Figures
 =======
 
 The Yocto Project documentation has many figures/images. Sphinx has a
-'figure' directive which is straight forward to use. To include a
+'figure' directive which is straightforward to use. To include a
 figure in the body of the documentation:
 
   .. image:: figures/YP-flow-diagram.png
@@ -252,10 +259,13 @@ websites.
 More information can be found here:
 https://sublime-and-sphinx-guide.readthedocs.io/en/latest/references.html.
 
+Anchor (<#link>) links are forbidden as they are not checked by Sphinx during
+the build and may be broken without knowing about it.
+
 References
 ==========
 
-The following extension is enabed by default:
+The following extension is enabled by default:
 sphinx.ext.autosectionlabel
 (https://www.sphinx-doc.org/en/master/usage/extensions/autosectionlabel.html).
 
@@ -264,7 +274,7 @@ autosectionlabel_prefix_document is enabled by default, so that we can
 insert references from any document.
 
 For example, to insert an HTML link to a section from
-documentaion/manual/intro.rst, use:
+documentation/manual/intro.rst, use:
 
   Please check this :ref:`manual/intro:Cross-References to Locations in the Same Document`
 
@@ -287,7 +297,8 @@ Extlinks
 
 The sphinx.ext.extlinks extension is enabled by default
 (https://sublime-and-sphinx-guide.readthedocs.io/en/latest/references.html#use-the-external-links-extension),
-and it is configured with:
+and it is configured with the 'extlinks' definitions in
+the 'documentation/conf.py' file:
 
   'yocto_home': ('https://yoctoproject.org%s', None),
   'yocto_wiki': ('https://wiki.yoctoproject.org%s', None),
@@ -299,6 +310,10 @@ and it is configured with:
   'yocto_git': ('https://git.yoctoproject.org%s', None),
   'oe_home': ('https://www.openembedded.org%s', None),
   'oe_lists': ('https://lists.openembedded.org%s', None),
+  'oe_git': ('https://git.openembedded.org%s', None),
+  'oe_wiki': ('https://www.openembedded.org/wiki%s', None),
+  'oe_layerindex': ('https://layers.openembedded.org%s', None),
+  'oe_layer': ('https://layers.openembedded.org/layerindex/branch/master/layer%s', None),
 
 It creates convenient shortcuts which can be used throughout the
 documentation rst files, as:
@@ -318,3 +333,9 @@ References to the bitbake manual can be done like this:
   See the ":ref:`-D <bitbake:bitbake-user-manual/bitbake-user-manual-intro:usage and syntax>`" option
 or
   :term:`bitbake:BB_NUMBER_PARSE_THREADS`
+
+Submitting documentation changes
+================================
+
+Please see the top level README file in this repository for details of where
+to send patches.