conf.py/poky.yaml: Move version information to poky.yaml and read in conf.py

Merge in the changes from master allowing conf.py to use information from
poky.yaml. This allows the head version mapped to X.999 on the website to
have the version information displayed clearly and correctly.

(From yocto-docs rev: f462982ddb6e32052217d44c0c44e8a4df115589)

Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>

.gitignore

@@ -30,4 +30,5 @@ hob-image-*.bb
 pull-*/
 bitbake/lib/toaster/contrib/tts/backlog.txt
 bitbake/lib/toaster/contrib/tts/log/*
-bitbake/lib/toaster/contrib/tts/.cache/*
+bitbake/lib/toaster/contrib/tts/.cache/*
+bitbake/lib/bb/tests/runqueue-tests/bitbake-cookerdaemon.log

README.OE-Core

@@ -6,24 +6,24 @@ of OpenEmbedded. It is distro-less (can build a functional image with
 DISTRO = "nodistro") and contains only emulated machine support.
 
 For information about OpenEmbedded, see the OpenEmbedded website:
-    http://www.openembedded.org/
+    https://www.openembedded.org/
 
 The Yocto Project has extensive documentation about OE including a reference manual
 which can be found at:
-    http://yoctoproject.org/documentation
+    https://docs.yoctoproject.org/
 
 
 Contributing
 ------------
 
 Please refer to
-http://www.openembedded.org/wiki/How_to_submit_a_patch_to_OpenEmbedded
+https://www.openembedded.org/wiki/How_to_submit_a_patch_to_OpenEmbedded
 for guidelines on how to submit patches.
 
 Mailing list:
 
-    http://lists.openembedded.org/mailman/listinfo/openembedded-core
+    https://lists.openembedded.org/g/openembedded-core
 
 Source code:
 
-    http://git.openembedded.org/openembedded-core/
+    https://git.openembedded.org/openembedded-core/

bitbake/README

@@ -11,7 +11,7 @@ For information about Bitbake, see the OpenEmbedded website:
 
 Bitbake plain documentation can be found under the doc directory or its integrated
 html version at the Yocto Project website:
-    http://yoctoproject.org/documentation
+    https://docs.yoctoproject.org
 
 Contributing
 ------------

bitbake/bin/bitbake

@@ -26,7 +26,7 @@ from bb.main import bitbake_main, BitBakeConfigParameters, BBMainException
 if sys.getfilesystemencoding() != "utf-8":
     sys.exit("Please use a locale setting which supports UTF-8 (such as LANG=en_US.UTF-8).\nPython can't change the filesystem locale after loading so we need a UTF-8 when Python starts or things won't work.")
 
-__version__ = "1.48.0"
+__version__ = "1.50.0"
 
 if __name__ == "__main__":
     if __version__ != bb.__version__:

bitbake/bin/bitbake-hashclient

@@ -151,9 +151,6 @@ def main():
     func = getattr(args, 'func', None)
     if func:
         client = hashserv.create_client(args.address)
-        # Try to establish a connection to the server now to detect failures
-        # early
-        client.connect()
 
         return func(args, client)
 

bitbake/bin/bitbake-hashserv

@@ -30,9 +30,11 @@ def main():
                                                "--bind [::1]:8686"'''
                                      )
 
-    parser.add_argument('--bind', default=DEFAULT_BIND, help='Bind address (default "%(default)s")')
-    parser.add_argument('--database', default='./hashserv.db', help='Database file (default "%(default)s")')
-    parser.add_argument('--log', default='WARNING', help='Set logging level')
+    parser.add_argument('-b', '--bind', default=DEFAULT_BIND, help='Bind address (default "%(default)s")')
+    parser.add_argument('-d', '--database', default='./hashserv.db', help='Database file (default "%(default)s")')
+    parser.add_argument('-l', '--log', default='WARNING', help='Set logging level')
+    parser.add_argument('-u', '--upstream', help='Upstream hashserv to pull hashes from')
+    parser.add_argument('-r', '--read-only', action='store_true', help='Disallow write operations from clients')
 
     args = parser.parse_args()
 
@@ -47,7 +49,7 @@ def main():
     console.setLevel(level)
     logger.addHandler(console)
 
-    server = hashserv.create_server(args.bind, args.database)
+    server = hashserv.create_server(args.bind, args.database, upstream=args.upstream, read_only=args.read_only)
     server.serve_forever()
     return 0
 

bitbake/bin/bitbake-server

@@ -26,7 +26,7 @@ readypipeinfd = int(sys.argv[3])
 logfile = sys.argv[4]
 lockname = sys.argv[5]
 sockname = sys.argv[6]
-timeout = sys.argv[7]
+timeout = float(sys.argv[7])
 xmlrpcinterface = (sys.argv[8], int(sys.argv[9]))
 if xmlrpcinterface[0] == "None":
     xmlrpcinterface = (None, xmlrpcinterface[1])

bitbake/bin/bitbake-worker

@@ -16,6 +16,8 @@ import signal
 import pickle
 import traceback
 import queue
+import shlex
+import subprocess
 from multiprocessing import Lock
 from threading import Thread
 
@@ -118,7 +120,9 @@ def worker_child_fire(event, d):
     data = b"<event>" + pickle.dumps(event) + b"</event>"
     try:
         worker_pipe_lock.acquire()
-        worker_pipe.write(data)
+        while(len(data)):
+            written = worker_pipe.write(data)
+            data = data[written:]
         worker_pipe_lock.release()
     except IOError:
         sigterm_handler(None, None)
@@ -143,21 +147,27 @@ def fork_off_task(cfg, data, databuilder, workerdata, fn, task, taskname, taskha
     # a fork() or exec*() activates PSEUDO...
 
     envbackup = {}
+    fakeroot = False
     fakeenv = {}
     umask = None
 
     taskdep = workerdata["taskdeps"][fn]
     if 'umask' in taskdep and taskname in taskdep['umask']:
+        umask = taskdep['umask'][taskname]
+    elif workerdata["umask"]:
+        umask = workerdata["umask"]
+    if umask:
         # umask might come in as a number or text string..
         try:
-             umask = int(taskdep['umask'][taskname],8)
+             umask = int(umask, 8)
         except TypeError:
-             umask = taskdep['umask'][taskname]
+             pass
 
     dry_run = cfg.dry_run or dry_run_exec
 
     # We can't use the fakeroot environment in a dry run as it possibly hasn't been built
     if 'fakeroot' in taskdep and taskname in taskdep['fakeroot'] and not dry_run:
+        fakeroot = True
         envvars = (workerdata["fakerootenv"][fn] or "").split()
         for key, value in (var.split('=') for var in envvars):
             envbackup[key] = os.environ.get(key)
@@ -167,7 +177,7 @@ def fork_off_task(cfg, data, databuilder, workerdata, fn, task, taskname, taskha
         fakedirs = (workerdata["fakerootdirs"][fn] or "").split()
         for p in fakedirs:
             bb.utils.mkdirhier(p)
-        logger.debug(2, 'Running %s:%s under fakeroot, fakedirs: %s' %
+        logger.debug2('Running %s:%s under fakeroot, fakedirs: %s' %
                         (fn, taskname, ', '.join(fakedirs)))
     else:
         envvars = (workerdata["fakerootnoenv"][fn] or "").split()
@@ -276,7 +286,13 @@ def fork_off_task(cfg, data, databuilder, workerdata, fn, task, taskname, taskha
             try:
                 if dry_run:
                     return 0
-                return bb.build.exec_task(fn, taskname, the_data, cfg.profile)
+                try:
+                    ret = bb.build.exec_task(fn, taskname, the_data, cfg.profile)
+                finally:
+                    if fakeroot:
+                        fakerootcmd = shlex.split(the_data.getVar("FAKEROOTCMD"))
+                        subprocess.run(fakerootcmd + ['-S'], check=True, stdout=subprocess.PIPE)
+                return ret
             except:
                 os._exit(1)
         if not profiling:
@@ -321,7 +337,9 @@ class runQueueWorkerPipe():
         end = len(self.queue)
         index = self.queue.find(b"</event>")
         while index != -1:
-            worker_fire_prepickled(self.queue[:index+8])
+            msg = self.queue[:index+8]
+            assert msg.startswith(b"<event>") and msg.count(b"<event>") == 1
+            worker_fire_prepickled(msg)
             self.queue = self.queue[index+8:]
             index = self.queue.find(b"</event>")
         return (end > start)
@@ -505,9 +523,11 @@ except BaseException as e:
         import traceback
         sys.stderr.write(traceback.format_exc())
         sys.stderr.write(str(e))
+finally:
+    worker_thread_exit = True
+    worker_thread.join()
 
-worker_thread_exit = True
-worker_thread.join()
-
-workerlog_write("exitting")
+workerlog_write("exiting")
+if not normalexit:
+    sys.exit(1)
 sys.exit(0)

bitbake/contrib/hashserv/Dockerfile

@@ -0,0 +1,19 @@
+# SPDX-License-Identifier: MIT
+#
+# Copyright (c) 2021 Joshua Watt <JPEWhacker@gmail.com>
+# 
+# Dockerfile to build a bitbake hash equivalence server container
+#
+# From the root of the bitbake repository, run:
+#
+#   docker build -f contrib/hashserv/Dockerfile .
+#
+
+FROM alpine:3.13.1
+
+RUN apk add --no-cache python3
+
+COPY bin/bitbake-hashserv /opt/bbhashserv/bin/
+COPY lib/hashserv /opt/bbhashserv/lib/hashserv/
+
+ENTRYPOINT ["/opt/bbhashserv/bin/bitbake-hashserv"]

bitbake/doc/Makefile

@@ -0,0 +1,35 @@
+# Minimal makefile for Sphinx documentation
+#
+
+# You can set these variables from the command line, and also
+# from the environment for the first two.
+SPHINXOPTS    ?= -j auto
+SPHINXBUILD   ?= sphinx-build
+SOURCEDIR     = .
+BUILDDIR      = _build
+DESTDIR       = final
+
+ifeq ($(shell if which $(SPHINXBUILD) >/dev/null 2>&1; then echo 1; else echo 0; fi),0)
+$(error "The '$(SPHINXBUILD)' command was not found. Make sure you have Sphinx installed")
+endif
+
+# Put it first so that "make" without argument is like "make help".
+help:
+	@$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)
+
+.PHONY: help Makefile clean publish
+
+publish: Makefile html singlehtml
+	rm -rf $(BUILDDIR)/$(DESTDIR)/
+	mkdir -p $(BUILDDIR)/$(DESTDIR)/
+	cp -r $(BUILDDIR)/html/* $(BUILDDIR)/$(DESTDIR)/
+	cp $(BUILDDIR)/singlehtml/index.html $(BUILDDIR)/$(DESTDIR)/singleindex.html
+	sed -i -e 's@index.html#@singleindex.html#@g' $(BUILDDIR)/$(DESTDIR)/singleindex.html
+
+clean:
+	@rm -rf $(BUILDDIR)
+
+# Catch-all target: route all unknown targets to Sphinx using the new
+# "make mode" option.  $(O) is meant as a shortcut for $(SPHINXOPTS).
+%: Makefile
+	@$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)

bitbake/doc/bitbake-user-manual/bitbake-user-manual-execution.rst

@@ -244,7 +244,8 @@ want upstream. Here is an example: ::
   BBFILE_COLLECTIONS = "upstream local"
   BBFILE_PATTERN_upstream = "^/stuff/openembedded/"
   BBFILE_PATTERN_local = "^/stuff/openembedded.modified/"
-  BBFILE_PRIORITY_upstream = "5" BBFILE_PRIORITY_local = "10"
+  BBFILE_PRIORITY_upstream = "5"
+  BBFILE_PRIORITY_local = "10"
 
 .. note::
 

bitbake/doc/bitbake-user-manual/bitbake-user-manual-fetching.rst

@@ -441,6 +441,15 @@ Here are some example URLs: ::
    SRC_URI = "git://git.oe.handhelds.org/git/vip.git;tag=version-1"
    SRC_URI = "git://git.oe.handhelds.org/git/vip.git;protocol=http"
 
+.. note::
+
+   Specifying passwords directly in ``git://`` urls is not supported.
+   There are several reasons: ``SRC_URI`` is often written out to logs and
+   other places, and that could easily leak passwords; it is also all too
+   easy to share metadata without removing passwords. SSH keys, ``~/.netrc``
+   and ``~/.ssh/config`` files can be used as alternatives.
+
+
 .. _gitsm-fetcher:
 
 Git Submodule Fetcher (``gitsm://``)
@@ -624,6 +633,34 @@ Here are some example URLs: ::
    SRC_URI = "repo://REPOROOT;protocol=git;branch=some_branch;manifest=my_manifest.xml"
    SRC_URI = "repo://REPOROOT;protocol=file;branch=some_branch;manifest=my_manifest.xml"
 
+.. _az-fetcher:
+
+Az Fetcher (``az://``)
+--------------------------
+
+This submodule fetches data from an
+`Azure Storage account <https://docs.microsoft.com/en-us/azure/storage/>`__ ,
+it inherits its functionality from the HTTP wget fetcher, but modifies its
+behavior to accomodate the usage of a
+`Shared Access Signature (SAS) <https://docs.microsoft.com/en-us/azure/storage/common/storage-sas-overview>`__
+for non-public data.
+
+Such functionality is set by the variable:
+
+-  :term:`AZ_SAS`: The Azure Storage Shared Access Signature provides secure
+   delegate access to resources, if this variable is set, the Az Fetcher will
+   use it when fetching artifacts from the cloud.
+
+You can specify the AZ_SAS variable as shown below: ::
+
+   AZ_SAS = "se=2021-01-01&sp=r&sv=2018-11-09&sr=c&skoid=<skoid>&sig=<signature>"
+
+Here is an example URL: ::
+
+   SRC_URI = "az://<azure-storage-account>.blob.core.windows.net/<foo_container>/<bar_file>"
+
+It can also be used when setting mirrors definitions using the :term:`PREMIRRORS` variable.
+
 Other Fetchers
 --------------
 

bitbake/doc/bitbake-user-manual/bitbake-user-manual-metadata.rst

@@ -1296,6 +1296,17 @@ For more information on task dependencies, see the
 See the ":ref:`bitbake-user-manual/bitbake-user-manual-metadata:variable flags`" section for information
 on variable flags you can use with tasks.
 
+.. note::
+
+   While it's infrequent, it's possible to define multiple tasks as
+   dependencies when calling ``addtask``. For example, here's a snippet
+   from the OpenEmbedded class file ``package_tar.bbclass``::
+
+     addtask package_write_tar before do_build after do_packagedata do_package
+
+   Note how the ``package_write_tar`` task has to wait until both of
+   ``do_packagedata`` and ``do_package`` complete.
+
 Deleting a Task
 ---------------
 
@@ -1569,7 +1580,7 @@ might have an interest in viewing:
    events when each of the workers parse the base configuration or if
    the server changes configuration and reparses. Any given datastore
    only has one such event executed against it, however. If
-   ```BB_INVALIDCONF`` <#>`__ is set in the datastore by the event
+   :term:`BB_INVALIDCONF` is set in the datastore by the event
    handler, the configuration is reparsed and a new event triggered,
    allowing the metadata to update configuration.
 

bitbake/doc/bitbake-user-manual/bitbake-user-manual-ref-variables.rst

@@ -39,6 +39,19 @@ overview of their function and contents.
       when specified allows for the Git binary from the host to be used
       rather than building ``git-native``.
 
+   :term:`AZ_SAS`
+      Azure Storage Shared Access Signature, when using the
+      :ref:`Azure Storage fetcher <bitbake-user-manual/bitbake-user-manual-fetching:fetchers>`
+      This variable can be defined to be used by the fetcher to authenticate
+      and gain access to non-public artifacts.
+      ::
+
+         AZ_SAS = ""se=2021-01-01&sp=r&sv=2018-11-09&sr=c&skoid=<skoid>&sig=<signature>""
+
+      For more information see Microsoft's Azure Storage documentation at
+      https://docs.microsoft.com/en-us/azure/storage/common/storage-sas-overview
+
+
    :term:`B`
       The directory in which BitBake executes functions during a recipe's
       build process.
@@ -108,6 +121,10 @@ overview of their function and contents.
       command line option). The task name specified should not include the
       ``do_`` prefix.
 
+   :term:`BB_DEFAULT_UMASK`
+      The default umask to apply to tasks if specified and no task specific
+      umask flag is set.
+
    :term:`BB_DISKMON_DIRS`
       Monitors disk space and available inodes during the build and allows
       you to control the build based on these parameters.
@@ -253,45 +270,6 @@ overview of their function and contents.
       ``my-recipe.bb`` is executing, the ``BB_FILENAME`` variable contains
       "/foo/path/my-recipe.bb".
 
-   :term:`BBFILES_DYNAMIC`
-      Activates content depending on presence of identified layers.  You
-      identify the layers by the collections that the layers define.
-
-      Use the ``BBFILES_DYNAMIC`` variable to avoid ``.bbappend`` files whose
-      corresponding ``.bb`` file is in a layer that attempts to modify other
-      layers through ``.bbappend`` but does not want to introduce a hard
-      dependency on those other layers.
-
-      Additionally you can prefix the rule with "!" to add ``.bbappend`` and
-      ``.bb`` files in case a layer is not present.  Use this avoid hard
-      dependency on those other layers.
-
-      Use the following form for ``BBFILES_DYNAMIC``: ::
-
-         collection_name:filename_pattern
-
-      The following example identifies two collection names and two filename
-      patterns: ::
-
-         BBFILES_DYNAMIC += "\
-             clang-layer:${LAYERDIR}/bbappends/meta-clang/*/*/*.bbappend \
-             core:${LAYERDIR}/bbappends/openembedded-core/meta/*/*/*.bbappend \
-         "
-
-      When the collection name is prefixed with "!" it will add the file pattern in case
-      the layer is absent: ::
-
-         BBFILES_DYNAMIC += "\
-             !clang-layer:${LAYERDIR}/backfill/meta-clang/*/*/*.bb \
-         "
-
-      This next example shows an error message that occurs because invalid
-      entries are found, which cause parsing to abort: ::
-
-         ERROR: BBFILES_DYNAMIC entries must be of the form {!}<collection name>:<filename pattern>, not:
-         /work/my-layer/bbappends/meta-security-isafw/*/*/*.bbappend
-         /work/my-layer/bbappends/openembedded-core/meta/*/*/*.bbappend
-
    :term:`BB_GENERATE_MIRROR_TARBALLS`
       Causes tarballs of the Git repositories, including the Git metadata,
       to be placed in the :term:`DL_DIR` directory. Anyone
@@ -667,6 +645,45 @@ overview of their function and contents.
       For details on the syntax, see the documentation by following the
       previous link.
 
+   :term:`BBFILES_DYNAMIC`
+      Activates content depending on presence of identified layers.  You
+      identify the layers by the collections that the layers define.
+
+      Use the ``BBFILES_DYNAMIC`` variable to avoid ``.bbappend`` files whose
+      corresponding ``.bb`` file is in a layer that attempts to modify other
+      layers through ``.bbappend`` but does not want to introduce a hard
+      dependency on those other layers.
+
+      Additionally you can prefix the rule with "!" to add ``.bbappend`` and
+      ``.bb`` files in case a layer is not present.  Use this avoid hard
+      dependency on those other layers.
+
+      Use the following form for ``BBFILES_DYNAMIC``: ::
+
+         collection_name:filename_pattern
+
+      The following example identifies two collection names and two filename
+      patterns: ::
+
+         BBFILES_DYNAMIC += "\
+             clang-layer:${LAYERDIR}/bbappends/meta-clang/*/*/*.bbappend \
+             core:${LAYERDIR}/bbappends/openembedded-core/meta/*/*/*.bbappend \
+         "
+
+      When the collection name is prefixed with "!" it will add the file pattern in case
+      the layer is absent: ::
+
+         BBFILES_DYNAMIC += "\
+             !clang-layer:${LAYERDIR}/backfill/meta-clang/*/*/*.bb \
+         "
+
+      This next example shows an error message that occurs because invalid
+      entries are found, which cause parsing to abort: ::
+
+         ERROR: BBFILES_DYNAMIC entries must be of the form {!}<collection name>:<filename pattern>, not:
+         /work/my-layer/bbappends/meta-security-isafw/*/*/*.bbappend
+         /work/my-layer/bbappends/openembedded-core/meta/*/*/*.bbappend
+
    :term:`BBINCLUDED`
       Contains a space-separated list of all of all files that BitBake's
       parser included during parsing of the current file.
@@ -1079,8 +1096,8 @@ overview of their function and contents.
          PREFERRED_PROVIDER_aaa = "bbb"
 
    :term:`PREFERRED_VERSION`
-      If there are multiple versions of recipes available, this variable
-      determines which recipe should be given preference. You must always
+      If there are multiple versions of a recipe available, this variable
+      determines which version should be given preference. You must always
       suffix the variable with the :term:`PN` you want to
       select, and you should set :term:`PV` accordingly for
       precedence.
@@ -1100,6 +1117,10 @@ overview of their function and contents.
          end of the string. You cannot use the wildcard character in any other
          location of the string.
 
+      If a recipe with the specified version is not available, a warning
+      message will be shown. See :term:`REQUIRED_VERSION` if you want this
+      to be an error instead.
+
    :term:`PREMIRRORS`
       Specifies additional paths from which BitBake gets source code. When
       the build system searches for source code, it first tries the local
@@ -1210,6 +1231,16 @@ overview of their function and contents.
       The directory in which a local copy of a ``google-repo`` directory is
       stored when it is synced.
 
+   :term:`REQUIRED_VERSION`
+      If there are multiple versions of a recipe available, this variable
+      determines which version should be given preference. ``REQUIRED_VERSION``
+      works in exactly the same manner as :term:`PREFERRED_VERSION`, except
+      that if the specified version is not available then an error message
+      is shown and the build fails immediately.
+
+      If both ``REQUIRED_VERSION`` and ``PREFERRED_VERSION`` are set for
+      the same recipe, the ``REQUIRED_VERSION`` value applies.
+
    :term:`RPROVIDES`
       A list of package name aliases that a package also provides. These
       aliases are useful for satisfying runtime dependencies of other
@@ -1299,6 +1330,8 @@ overview of their function and contents.
       -  ``svn://`` : Fetches files from a Subversion (``svn``) revision
          control repository.
 
+      -  ``az://`` : Fetches files from an Azure Storage account using HTTPS.
+
       Here are some additional options worth mentioning:
 
       -  ``unpack`` : Controls whether or not to unpack the file if it is

bitbake/doc/conf.py

@@ -14,6 +14,7 @@
 # import sys
 # sys.path.insert(0, os.path.abspath('.'))
 
+import sys
 import datetime
 
 current_version = "dev"

bitbake/lib/bb/__init__.py

@@ -9,7 +9,7 @@
 # SPDX-License-Identifier: GPL-2.0-only
 #
 
-__version__ = "1.48.0"
+__version__ = "1.50.0"
 
 import sys
 if sys.version_info < (3, 5, 0):
@@ -21,8 +21,8 @@ class BBHandledException(Exception):
     The big dilemma for generic bitbake code is what information to give the user
     when an exception occurs. Any exception inheriting this base exception class
     has already provided information to the user via some 'fired' message type such as
-    an explicitly fired event using bb.fire, or a bb.error message. If bitbake 
-    encounters an exception derived from this class, no backtrace or other information 
+    an explicitly fired event using bb.fire, or a bb.error message. If bitbake
+    encounters an exception derived from this class, no backtrace or other information
     will be given to the user, its assumed the earlier event provided the relevant information.
     """
     pass
@@ -42,14 +42,23 @@ class BBLoggerMixin(object):
 
     def setup_bblogger(self, name):
         if name.split(".")[0] == "BitBake":
-            self.debug = self.bbdebug
+            self.debug = self._debug_helper
+
+    def _debug_helper(self, *args, **kwargs):
+        return self.bbdebug(1, *args, **kwargs)
+
+    def debug2(self, *args, **kwargs):
+        return self.bbdebug(2, *args, **kwargs)
+
+    def debug3(self, *args, **kwargs):
+        return self.bbdebug(3, *args, **kwargs)
 
     def bbdebug(self, level, msg, *args, **kwargs):
         loglevel = logging.DEBUG - level + 1
         if not bb.event.worker_pid:
             if self.name in bb.msg.loggerDefaultDomains and loglevel > (bb.msg.loggerDefaultDomains[self.name]):
                 return
-            if loglevel > bb.msg.loggerDefaultLogLevel:
+            if loglevel < bb.msg.loggerDefaultLogLevel:
                 return
         return self.log(loglevel, msg, *args, **kwargs)
 
@@ -128,7 +137,7 @@ def debug(lvl, *args):
         mainlogger.warning("Passed invalid debug level '%s' to bb.debug", lvl)
         args = (lvl,) + args
         lvl = 1
-    mainlogger.debug(lvl, ''.join(args))
+    mainlogger.bbdebug(lvl, ''.join(args))
 
 def note(*args):
     mainlogger.info(''.join(args))

bitbake/lib/bb/build.py

@@ -298,6 +298,10 @@ def exec_func_python(func, d, runfile, cwd=None):
         comp = utils.better_compile(code, func, "exec_python_func() autogenerated")
         utils.better_exec(comp, {"d": d}, code, "exec_python_func() autogenerated")
     finally:
+        # We want any stdout/stderr to be printed before any other log messages to make debugging
+        # more accurate. In some cases we seem to lose stdout/stderr entirely in logging tests without this.
+        sys.stdout.flush()
+        sys.stderr.flush()
         bb.debug(2, "Python function %s finished" % func)
 
         if cwd and olddir:
@@ -583,7 +587,7 @@ def _exec_task(fn, task, d, quieterr):
         logger.error("No such task: %s" % task)
         return 1
 
-    logger.debug(1, "Executing task %s", task)
+    logger.debug("Executing task %s", task)
 
     localdata = _task_data(fn, task, d)
     tempdir = localdata.getVar('T')
@@ -596,7 +600,7 @@ def _exec_task(fn, task, d, quieterr):
         curnice = os.nice(0)
         nice = int(nice) - curnice
         newnice = os.nice(nice)
-        logger.debug(1, "Renice to %s " % newnice)
+        logger.debug("Renice to %s " % newnice)
     ionice = localdata.getVar("BB_TASK_IONICE_LEVEL")
     if ionice:
         try:
@@ -694,12 +698,16 @@ def _exec_task(fn, task, d, quieterr):
         except bb.BBHandledException:
             event.fire(TaskFailed(task, fn, logfn, localdata, True), localdata)
             return 1
-        except Exception as exc:
+        except (Exception, SystemExit) as exc:
             if quieterr:
                 event.fire(TaskFailedSilent(task, fn, logfn, localdata), localdata)
             else:
                 errprinted = errchk.triggered
-                logger.error(str(exc))
+                # If the output is already on stdout, we've printed the information in the
+                # logs once already so don't duplicate
+                if verboseStdoutLogging:
+                    errprinted = True
+                logger.error(repr(exc))
                 event.fire(TaskFailed(task, fn, logfn, localdata, errprinted), localdata)
             return 1
     finally:
@@ -720,7 +728,7 @@ def _exec_task(fn, task, d, quieterr):
 
         logfile.close()
         if os.path.exists(logfn) and os.path.getsize(logfn) == 0:
-            logger.debug(2, "Zero size logfn %s, removing", logfn)
+            logger.debug2("Zero size logfn %s, removing", logfn)
             bb.utils.remove(logfn)
             bb.utils.remove(loglink)
     event.fire(TaskSucceeded(task, fn, logfn, localdata), localdata)
@@ -854,6 +862,23 @@ def make_stamp(task, d, file_name = None):
         file_name = d.getVar('BB_FILENAME')
         bb.parse.siggen.dump_sigtask(file_name, task, stampbase, True)
 
+def find_stale_stamps(task, d, file_name=None):
+    current = stamp_internal(task, d, file_name)
+    current2 = stamp_internal(task + "_setscene", d, file_name)
+    cleanmask = stamp_cleanmask_internal(task, d, file_name)
+    found = []
+    for mask in cleanmask:
+        for name in glob.glob(mask):
+            if "sigdata" in name or "sigbasedata" in name:
+                continue
+            if name.endswith('.taint'):
+                continue
+            if name == current or name == current2:
+                continue
+            logger.debug2("Stampfile %s does not match %s or %s" % (name, current, current2))
+            found.append(name)
+    return found
+
 def del_stamp(task, d, file_name = None):
     """
     Removes a stamp for a given task
@@ -1008,6 +1033,8 @@ def tasksbetween(task_start, task_end, d):
     def follow_chain(task, endtask, chain=None):
         if not chain:
             chain = []
+        if task in chain:
+            bb.fatal("Circular task dependencies as %s depends on itself via the chain %s" % (task, " -> ".join(chain)))
         chain.append(task)
         for othertask in tasks:
             if othertask == task:

bitbake/lib/bb/cache.py

@@ -19,14 +19,15 @@
 import os
 import logging
 import pickle
-from collections import defaultdict, Mapping
+from collections import defaultdict
+from collections.abc import Mapping
 import bb.utils
 from bb import PrefixLoggerAdapter
 import re
 
 logger = logging.getLogger("BitBake.Cache")
 
-__cache_version__ = "153"
+__cache_version__ = "154"
 
 def getCacheFile(path, filename, mc, data_hash):
     mcspec = ''
@@ -94,6 +95,7 @@ class CoreRecipeInfo(RecipeInfoCommon):
         if not self.packages:
             self.packages.append(self.pn)
         self.packages_dynamic = self.listvar('PACKAGES_DYNAMIC', metadata)
+        self.rprovides_pkg = self.pkgvar('RPROVIDES', self.packages, metadata)
 
         self.skipreason = self.getvar('__SKIPPED', metadata)
         if self.skipreason:
@@ -120,12 +122,12 @@ class CoreRecipeInfo(RecipeInfoCommon):
         self.depends          = self.depvar('DEPENDS', metadata)
         self.rdepends         = self.depvar('RDEPENDS', metadata)
         self.rrecommends      = self.depvar('RRECOMMENDS', metadata)
-        self.rprovides_pkg    = self.pkgvar('RPROVIDES', self.packages, metadata)
         self.rdepends_pkg     = self.pkgvar('RDEPENDS', self.packages, metadata)
         self.rrecommends_pkg  = self.pkgvar('RRECOMMENDS', self.packages, metadata)
         self.inherits         = self.getvar('__inherit_cache', metadata, expand=False)
         self.fakerootenv      = self.getvar('FAKEROOTENV', metadata)
         self.fakerootdirs     = self.getvar('FAKEROOTDIRS', metadata)
+        self.fakerootlogs     = self.getvar('FAKEROOTLOGS', metadata)
         self.fakerootnoenv    = self.getvar('FAKEROOTNOENV', metadata)
         self.extradepsfunc    = self.getvar('calculate_extra_depends', metadata)
 
@@ -163,6 +165,7 @@ class CoreRecipeInfo(RecipeInfoCommon):
         cachedata.fakerootenv = {}
         cachedata.fakerootnoenv = {}
         cachedata.fakerootdirs = {}
+        cachedata.fakerootlogs = {}
         cachedata.extradepsfunc = {}
 
     def add_cacheData(self, cachedata, fn):
@@ -215,7 +218,7 @@ class CoreRecipeInfo(RecipeInfoCommon):
         if not self.not_world:
             cachedata.possible_world.append(fn)
         #else:
-        #    logger.debug(2, "EXCLUDE FROM WORLD: %s", fn)
+        #    logger.debug2("EXCLUDE FROM WORLD: %s", fn)
 
         # create a collection of all targets for sanity checking
         # tasks, such as upstream versions, license, and tools for
@@ -231,6 +234,7 @@ class CoreRecipeInfo(RecipeInfoCommon):
         cachedata.fakerootenv[fn] = self.fakerootenv
         cachedata.fakerootnoenv[fn] = self.fakerootnoenv
         cachedata.fakerootdirs[fn] = self.fakerootdirs
+        cachedata.fakerootlogs[fn] = self.fakerootlogs
         cachedata.extradepsfunc[fn] = self.extradepsfunc
 
 def virtualfn2realfn(virtualfn):
@@ -238,7 +242,7 @@ def virtualfn2realfn(virtualfn):
     Convert a virtual file name to a real one + the associated subclass keyword
     """
     mc = ""
-    if virtualfn.startswith('mc:'):
+    if virtualfn.startswith('mc:') and virtualfn.count(':') >= 2:
         elems = virtualfn.split(':')
         mc = elems[1]
         virtualfn = ":".join(elems[2:])
@@ -268,7 +272,7 @@ def variant2virtual(realfn, variant):
     """
     if variant == "":
         return realfn
-    if variant.startswith("mc:"):
+    if variant.startswith("mc:") and variant.count(':') >= 2:
         elems = variant.split(":")
         if elems[2]:
             return "mc:" + elems[1] + ":virtual:" + ":".join(elems[2:]) + ":" + realfn
@@ -323,7 +327,7 @@ class NoCache(object):
         Return a complete set of data for fn.
         To do this, we need to parse the file.
         """
-        logger.debug(1, "Parsing %s (full)" % virtualfn)
+        logger.debug("Parsing %s (full)" % virtualfn)
         (fn, virtual, mc) = virtualfn2realfn(virtualfn)
         bb_data = self.load_bbfile(virtualfn, appends, virtonly=True)
         return bb_data[virtual]
@@ -400,7 +404,7 @@ class Cache(NoCache):
 
         self.cachefile = self.getCacheFile("bb_cache.dat")
 
-        self.logger.debug(1, "Cache dir: %s", self.cachedir)
+        self.logger.debug("Cache dir: %s", self.cachedir)
         bb.utils.mkdirhier(self.cachedir)
 
         cache_ok = True
@@ -408,7 +412,7 @@ class Cache(NoCache):
             for cache_class in self.caches_array:
                 cachefile = self.getCacheFile(cache_class.cachefile)
                 cache_exists = os.path.exists(cachefile)
-                self.logger.debug(2, "Checking if %s exists: %r", cachefile, cache_exists)
+                self.logger.debug2("Checking if %s exists: %r", cachefile, cache_exists)
                 cache_ok = cache_ok and cache_exists
                 cache_class.init_cacheData(self)
         if cache_ok:
@@ -416,7 +420,7 @@ class Cache(NoCache):
         elif os.path.isfile(self.cachefile):
             self.logger.info("Out of date cache found, rebuilding...")
         else:
-            self.logger.debug(1, "Cache file %s not found, building..." % self.cachefile)
+            self.logger.debug("Cache file %s not found, building..." % self.cachefile)
 
         # We don't use the symlink, its just for debugging convinience
         if self.mc:
@@ -449,13 +453,11 @@ class Cache(NoCache):
         return cachesize
 
     def load_cachefile(self, progress):
-        cachesize = self.cachesize()
         previous_progress = 0
-        previous_percent = 0
 
         for cache_class in self.caches_array:
             cachefile = self.getCacheFile(cache_class.cachefile)
-            self.logger.debug(1, 'Loading cache file: %s' % cachefile)
+            self.logger.debug('Loading cache file: %s' % cachefile)
             with open(cachefile, "rb") as cachefile:
                 pickled = pickle.Unpickler(cachefile)
                 # Check cache version information
@@ -502,7 +504,7 @@ class Cache(NoCache):
 
     def parse(self, filename, appends):
         """Parse the specified filename, returning the recipe information"""
-        self.logger.debug(1, "Parsing %s", filename)
+        self.logger.debug("Parsing %s", filename)
         infos = []
         datastores = self.load_bbfile(filename, appends, mc=self.mc)
         depends = []
@@ -556,7 +558,7 @@ class Cache(NoCache):
         cached, infos = self.load(fn, appends)
         for virtualfn, info_array in infos:
             if info_array[0].skipped:
-                self.logger.debug(1, "Skipping %s: %s", virtualfn, info_array[0].skipreason)
+                self.logger.debug("Skipping %s: %s", virtualfn, info_array[0].skipreason)
                 skipped += 1
             else:
                 self.add_info(virtualfn, info_array, cacheData, not cached)
@@ -592,21 +594,21 @@ class Cache(NoCache):
 
         # File isn't in depends_cache
         if not fn in self.depends_cache:
-            self.logger.debug(2, "%s is not cached", fn)
+            self.logger.debug2("%s is not cached", fn)
             return False
 
         mtime = bb.parse.cached_mtime_noerror(fn)
 
         # Check file still exists
         if mtime == 0:
-            self.logger.debug(2, "%s no longer exists", fn)
+            self.logger.debug2("%s no longer exists", fn)
             self.remove(fn)
             return False
 
         info_array = self.depends_cache[fn]
         # Check the file's timestamp
         if mtime != info_array[0].timestamp:
-            self.logger.debug(2, "%s changed", fn)
+            self.logger.debug2("%s changed", fn)
             self.remove(fn)
             return False
 
@@ -617,13 +619,13 @@ class Cache(NoCache):
                 fmtime = bb.parse.cached_mtime_noerror(f)
                 # Check if file still exists
                 if old_mtime != 0 and fmtime == 0:
-                    self.logger.debug(2, "%s's dependency %s was removed",
+                    self.logger.debug2("%s's dependency %s was removed",
                                          fn, f)
                     self.remove(fn)
                     return False
 
                 if (fmtime != old_mtime):
-                    self.logger.debug(2, "%s's dependency %s changed",
+                    self.logger.debug2("%s's dependency %s changed",
                                          fn, f)
                     self.remove(fn)
                     return False
@@ -640,14 +642,14 @@ class Cache(NoCache):
                         continue
                     f, exist = f.split(":")
                     if (exist == "True" and not os.path.exists(f)) or (exist == "False" and os.path.exists(f)):
-                        self.logger.debug(2, "%s's file checksum list file %s changed",
+                        self.logger.debug2("%s's file checksum list file %s changed",
                                              fn, f)
                         self.remove(fn)
                         return False
 
         if tuple(appends) != tuple(info_array[0].appends):
-            self.logger.debug(2, "appends for %s changed", fn)
-            self.logger.debug(2, "%s to %s" % (str(appends), str(info_array[0].appends)))
+            self.logger.debug2("appends for %s changed", fn)
+            self.logger.debug2("%s to %s" % (str(appends), str(info_array[0].appends)))
             self.remove(fn)
             return False
 
@@ -656,10 +658,10 @@ class Cache(NoCache):
             virtualfn = variant2virtual(fn, cls)
             self.clean.add(virtualfn)
             if virtualfn not in self.depends_cache:
-                self.logger.debug(2, "%s is not cached", virtualfn)
+                self.logger.debug2("%s is not cached", virtualfn)
                 invalid = True
             elif len(self.depends_cache[virtualfn]) != len(self.caches_array):
-                self.logger.debug(2, "Extra caches missing for %s?" % virtualfn)
+                self.logger.debug2("Extra caches missing for %s?" % virtualfn)
                 invalid = True
 
         # If any one of the variants is not present, mark as invalid for all
@@ -667,10 +669,10 @@ class Cache(NoCache):
             for cls in info_array[0].variants:
                 virtualfn = variant2virtual(fn, cls)
                 if virtualfn in self.clean:
-                    self.logger.debug(2, "Removing %s from cache", virtualfn)
+                    self.logger.debug2("Removing %s from cache", virtualfn)
                     self.clean.remove(virtualfn)
             if fn in self.clean:
-                self.logger.debug(2, "Marking %s as not clean", fn)
+                self.logger.debug2("Marking %s as not clean", fn)
                 self.clean.remove(fn)
             return False
 
@@ -683,10 +685,10 @@ class Cache(NoCache):
         Called from the parser in error cases
         """
         if fn in self.depends_cache:
-            self.logger.debug(1, "Removing %s from cache", fn)
+            self.logger.debug("Removing %s from cache", fn)
             del self.depends_cache[fn]
         if fn in self.clean:
-            self.logger.debug(1, "Marking %s as unclean", fn)
+            self.logger.debug("Marking %s as unclean", fn)
             self.clean.remove(fn)
 
     def sync(self):
@@ -699,13 +701,13 @@ class Cache(NoCache):
             return
 
         if self.cacheclean:
-            self.logger.debug(2, "Cache is clean, not saving.")
+            self.logger.debug2("Cache is clean, not saving.")
             return
 
         for cache_class in self.caches_array:
             cache_class_name = cache_class.__name__
             cachefile = self.getCacheFile(cache_class.cachefile)
-            self.logger.debug(2, "Writing %s", cachefile)
+            self.logger.debug2("Writing %s", cachefile)
             with open(cachefile, "wb") as f:
                 p = pickle.Pickler(f, pickle.HIGHEST_PROTOCOL)
                 p.dump(__cache_version__)
@@ -816,10 +818,6 @@ class MulticonfigCache(Mapping):
         for k in self.__caches:
             yield k
 
-    def keys(self):
-        return self.__caches[key]
-
-
 def init(cooker):
     """
     The Objective: Cache the minimum amount of data possible yet get to the
@@ -885,7 +883,7 @@ class MultiProcessCache(object):
         bb.utils.mkdirhier(cachedir)
         self.cachefile = os.path.join(cachedir,
                                       cache_file_name or self.__class__.cache_file_name)
-        logger.debug(1, "Using cache in '%s'", self.cachefile)
+        logger.debug("Using cache in '%s'", self.cachefile)
 
         glf = bb.utils.lockfile(self.cachefile + ".lock")
 
@@ -991,7 +989,7 @@ class SimpleCache(object):
         bb.utils.mkdirhier(cachedir)
         self.cachefile = os.path.join(cachedir,
                                       cache_file_name or self.__class__.cache_file_name)
-        logger.debug(1, "Using cache in '%s'", self.cachefile)
+        logger.debug("Using cache in '%s'", self.cachefile)
 
         glf = bb.utils.lockfile(self.cachefile + ".lock")
 

bitbake/lib/bb/codeparser.py

@@ -212,9 +212,9 @@ class PythonParser():
             funcstr = codegen.to_source(func)
             argstr = codegen.to_source(arg)
         except TypeError:
-            self.log.debug(2, 'Failed to convert function and argument to source form')
+            self.log.debug2('Failed to convert function and argument to source form')
         else:
-            self.log.debug(1, self.unhandled_message % (funcstr, argstr))
+            self.log.debug(self.unhandled_message % (funcstr, argstr))
 
     def visit_Call(self, node):
         name = self.called_node_name(node.func)
@@ -450,7 +450,7 @@ class ShellParser():
 
                 cmd = word[1]
                 if cmd.startswith("$"):
-                    self.log.debug(1, self.unhandled_template % cmd)
+                    self.log.debug(self.unhandled_template % cmd)
                 elif cmd == "eval":
                     command = " ".join(word for _, word in words[1:])
                     self._parse_shell(command)

bitbake/lib/bb/cooker.py

@@ -73,7 +73,9 @@ class SkippedPackage:
             self.pn = info.pn
             self.skipreason = info.skipreason
             self.provides = info.provides
-            self.rprovides = info.rprovides
+            self.rprovides = info.packages + info.rprovides
+            for package in info.packages:
+                self.rprovides += info.rprovides_pkg[package]
         elif reason:
             self.skipreason = reason
 
@@ -380,14 +382,29 @@ class BBCooker:
         try:
             self.prhost = prserv.serv.auto_start(self.data)
         except prserv.serv.PRServiceConfigError as e:
-            bb.fatal("Unable to start PR Server, exitting")
+            bb.fatal("Unable to start PR Server, exiting")
 
         if self.data.getVar("BB_HASHSERVE") == "auto":
             # Create a new hash server bound to a unix domain socket
             if not self.hashserv:
                 dbfile = (self.data.getVar("PERSISTENT_DIR") or self.data.getVar("CACHE")) + "/hashserv.db"
+                upstream = self.data.getVar("BB_HASHSERVE_UPSTREAM") or None
+                if upstream:
+                    import socket
+                    try:
+                        sock = socket.create_connection(upstream.split(":"), 5)
+                        sock.close()
+                    except socket.error as e:
+                        bb.warn("BB_HASHSERVE_UPSTREAM is not valid, unable to connect hash equivalence server at '%s': %s" 
+                                 % (upstream, repr(e)))
+
                 self.hashservaddr = "unix://%s/hashserve.sock" % self.data.getVar("TOPDIR")
-                self.hashserv = hashserv.create_server(self.hashservaddr, dbfile, sync=False)
+                self.hashserv = hashserv.create_server(
+                    self.hashservaddr,
+                    dbfile,
+                    sync=False,
+                    upstream=upstream,
+                )
                 self.hashserv.process = multiprocessing.Process(target=self.hashserv.serve_forever)
                 self.hashserv.process.start()
             self.data.setVar("BB_HASHSERVE", self.hashservaddr)
@@ -409,6 +426,8 @@ class BBCooker:
             self.data.disableTracking()
 
     def parseConfiguration(self):
+        self.updateCacheSync()
+
         # Change nice level if we're asked to
         nice = self.data.getVar("BB_NICE_LEVEL")
         if nice:
@@ -439,7 +458,7 @@ class BBCooker:
                         continue
                 except AttributeError:
                     pass
-                logger.debug(1, "Marking as dirty due to '%s' option change to '%s'" % (o, options[o]))
+                logger.debug("Marking as dirty due to '%s' option change to '%s'" % (o, options[o]))
                 print("Marking as dirty due to '%s' option change to '%s'" % (o, options[o]))
                 clean = False
             if hasattr(self.configuration, o):
@@ -466,17 +485,17 @@ class BBCooker:
 
         for k in bb.utils.approved_variables():
             if k in environment and k not in self.configuration.env:
-                logger.debug(1, "Updating new environment variable %s to %s" % (k, environment[k]))
+                logger.debug("Updating new environment variable %s to %s" % (k, environment[k]))
                 self.configuration.env[k] = environment[k]
                 clean = False
             if k in self.configuration.env and k not in environment:
-                logger.debug(1, "Updating environment variable %s (deleted)" % (k))
+                logger.debug("Updating environment variable %s (deleted)" % (k))
                 del self.configuration.env[k]
                 clean = False
             if k not in self.configuration.env and k not in environment:
                 continue
             if environment[k] != self.configuration.env[k]:
-                logger.debug(1, "Updating environment variable %s from %s to %s" % (k, self.configuration.env[k], environment[k]))
+                logger.debug("Updating environment variable %s from %s to %s" % (k, self.configuration.env[k], environment[k]))
                 self.configuration.env[k] = environment[k]
                 clean = False
 
@@ -484,7 +503,7 @@ class BBCooker:
         self.configuration.env = environment
 
         if not clean:
-            logger.debug(1, "Base environment change, triggering reparse")
+            logger.debug("Base environment change, triggering reparse")
             self.reset()
 
     def runCommands(self, server, data, abort):
@@ -498,22 +517,30 @@ class BBCooker:
 
     def showVersions(self):
 
-        (latest_versions, preferred_versions) = self.findProviders()
+        (latest_versions, preferred_versions, required) = self.findProviders()
 
-        logger.plain("%-35s %25s %25s", "Recipe Name", "Latest Version", "Preferred Version")
-        logger.plain("%-35s %25s %25s\n", "===========", "==============", "=================")
+        logger.plain("%-35s %25s %25s %25s", "Recipe Name", "Latest Version", "Preferred Version", "Required Version")
+        logger.plain("%-35s %25s %25s %25s\n", "===========", "==============", "=================", "================")
 
         for p in sorted(self.recipecaches[''].pkg_pn):
-            pref = preferred_versions[p]
+            preferred = preferred_versions[p]
             latest = latest_versions[p]
+            requiredstr = ""
+            preferredstr = ""
+            if required[p]:
+                if preferred[0] is not None:
+                    requiredstr = preferred[0][0] + ":" + preferred[0][1] + '-' + preferred[0][2]
+                else:
+                    bb.fatal("REQUIRED_VERSION of package %s not available" % p)
+            else:
+                preferredstr = preferred[0][0] + ":" + preferred[0][1] + '-' + preferred[0][2]
 
-            prefstr = pref[0][0] + ":" + pref[0][1] + '-' + pref[0][2]
             lateststr = latest[0][0] + ":" + latest[0][1] + "-" + latest[0][2]
 
-            if pref == latest:
-                prefstr = ""
+            if preferred == latest:
+                preferredstr = ""
 
-            logger.plain("%-35s %25s %25s", p, lateststr, prefstr)
+            logger.plain("%-35s %25s %25s %25s", p, lateststr, preferredstr, requiredstr)
 
     def showEnvironment(self, buildfile=None, pkgs_to_build=None):
         """
@@ -612,7 +639,7 @@ class BBCooker:
         # Replace string such as "mc:*:bash"
         # into "mc:A:bash mc:B:bash bash"
         for k in targetlist:
-            if k.startswith("mc:"):
+            if k.startswith("mc:") and k.count(':') >= 2:
                 if wildcard:
                     bb.fatal('multiconfig conflict')
                 if k.split(":")[1] == "*":
@@ -646,7 +673,7 @@ class BBCooker:
         for k in fulltargetlist:
             origk = k
             mc = ""
-            if k.startswith("mc:"):
+            if k.startswith("mc:") and k.count(':') >= 2:
                 mc = k.split(":")[1]
                 k = ":".join(k.split(":")[2:])
             ktask = task
@@ -695,7 +722,7 @@ class BBCooker:
                         if depmc not in self.multiconfigs:
                             bb.fatal("Multiconfig dependency %s depends on nonexistent multiconfig configuration named configuration %s" % (k,depmc))
                         else:
-                            logger.debug(1, "Adding providers for multiconfig dependency %s" % l[3])
+                            logger.debug("Adding providers for multiconfig dependency %s" % l[3])
                             taskdata[depmc].add_provider(localdata[depmc], self.recipecaches[depmc], l[3])
                             seen.add(k)
                             new = True
@@ -788,7 +815,9 @@ class BBCooker:
             for dep in rq.rqdata.runtaskentries[tid].depends:
                 (depmc, depfn, _, deptaskfn) = bb.runqueue.split_tid_mcfn(dep)
                 deppn = self.recipecaches[depmc].pkg_fn[deptaskfn]
-                depend_tree["tdepends"][dotname].append("%s.%s" % (deppn, bb.runqueue.taskname_from_tid(dep)))
+                if depmc:
+                    depmc = "mc:" + depmc + ":"
+                depend_tree["tdepends"][dotname].append("%s%s.%s" % (depmc, deppn, bb.runqueue.taskname_from_tid(dep)))
             if taskfn not in seen_fns:
                 seen_fns.append(taskfn)
                 packages = []
@@ -1059,10 +1088,16 @@ class BBCooker:
         if pn in self.recipecaches[mc].providers:
             filenames = self.recipecaches[mc].providers[pn]
             eligible, foundUnique = bb.providers.filterProviders(filenames, pn, self.databuilder.mcdata[mc], self.recipecaches[mc])
-            filename = eligible[0]
+            if eligible is not None:
+                filename = eligible[0]
+            else:
+                filename = None
             return None, None, None, filename
         elif pn in self.recipecaches[mc].pkg_pn:
-            return bb.providers.findBestProvider(pn, self.databuilder.mcdata[mc], self.recipecaches[mc], self.recipecaches[mc].pkg_pn)
+            (latest, latest_f, preferred_ver, preferred_file, required) = bb.providers.findBestProvider(pn, self.databuilder.mcdata[mc], self.recipecaches[mc], self.recipecaches[mc].pkg_pn)
+            if required and preferred_file is None:
+                return None, None, None, None
+            return (latest, latest_f, preferred_ver, preferred_file)
         else:
             return None, None, None, None
 
@@ -1551,7 +1586,7 @@ class BBCooker:
         self.inotify_modified_files = []
 
         if not self.baseconfig_valid:
-            logger.debug(1, "Reloading base configuration data")
+            logger.debug("Reloading base configuration data")
             self.initConfigurationData()
             self.handlePRServ()
 
@@ -2181,21 +2216,33 @@ class CookerParser(object):
             yield not cached, mc, infos
 
     def parse_generator(self):
-        while True:
+        empty = False
+        while self.processes or not empty:
+            for process in self.processes.copy():
+                if not process.is_alive():
+                    process.join()
+                    self.processes.remove(process)
+
             if self.parsed >= self.toparse:
                 break
 
             try:
                 result = self.result_queue.get(timeout=0.25)
             except queue.Empty:
+                empty = True
                 pass
             else:
+                empty = False
                 value = result[1]
                 if isinstance(value, BaseException):
                     raise value
                 else:
                     yield result
 
+        if not (self.parsed >= self.toparse):
+            raise bb.parse.ParseError("Not all recipes parsed, parser thread killed/died? Exiting.", None)
+
+
     def parse_next(self):
         result = []
         parsed = None
@@ -2207,18 +2254,18 @@ class CookerParser(object):
         except bb.BBHandledException as exc:
             self.error += 1
             logger.error('Failed to parse recipe: %s' % exc.recipe)
-            self.shutdown(clean=False)
+            self.shutdown(clean=False, force=True)
             return False
         except ParsingFailure as exc:
             self.error += 1
             logger.error('Unable to parse %s: %s' %
                      (exc.recipe, bb.exceptions.to_string(exc.realexception)))
-            self.shutdown(clean=False)
+            self.shutdown(clean=False, force=True)
             return False
         except bb.parse.ParseError as exc:
             self.error += 1
             logger.error(str(exc))
-            self.shutdown(clean=False)
+            self.shutdown(clean=False, force=True)
             return False
         except bb.data_smart.ExpansionError as exc:
             self.error += 1
@@ -2227,7 +2274,7 @@ class CookerParser(object):
             tb = list(itertools.dropwhile(lambda e: e.filename.startswith(bbdir), exc.traceback))
             logger.error('ExpansionError during parsing %s', value.recipe,
                          exc_info=(etype, value, tb))
-            self.shutdown(clean=False)
+            self.shutdown(clean=False, force=True)
             return False
         except Exception as exc:
             self.error += 1
@@ -2239,7 +2286,7 @@ class CookerParser(object):
                 # Most likely, an exception occurred during raising an exception
                 import traceback
                 logger.error('Exception during parse: %s' % traceback.format_exc())
-            self.shutdown(clean=False)
+            self.shutdown(clean=False, force=True)
             return False
 
         self.current += 1

bitbake/lib/bb/cookerdata.py

@@ -23,8 +23,8 @@ logger      = logging.getLogger("BitBake")
 parselog    = logging.getLogger("BitBake.Parsing")
 
 class ConfigParameters(object):
-    def __init__(self, argv=sys.argv):
-        self.options, targets = self.parseCommandLine(argv)
+    def __init__(self, argv=None):
+        self.options, targets = self.parseCommandLine(argv or sys.argv)
         self.environment = self.parseEnvironment()
 
         self.options.pkgs_to_build = targets or []
@@ -209,7 +209,7 @@ def findConfigFile(configfile, data):
     return None
 
 #
-# We search for a conf/bblayers.conf under an entry in BBPATH or in cwd working 
+# We search for a conf/bblayers.conf under an entry in BBPATH or in cwd working
 # up to /. If that fails, we search for a conf/bitbake.conf in BBPATH.
 #
 
@@ -291,6 +291,8 @@ class CookerDataBuilder(object):
 
             multiconfig = (self.data.getVar("BBMULTICONFIG") or "").split()
             for config in multiconfig:
+                if config[0].isdigit():
+                    bb.fatal("Multiconfig name '%s' is invalid as multiconfigs cannot start with a digit" % config)
                 mcdata = self.parseConfigurationFiles(self.prefiles, self.postfiles, config)
                 bb.event.fire(bb.event.ConfigParsed(), mcdata)
                 self.mcdata[config] = mcdata
@@ -342,6 +344,9 @@ class CookerDataBuilder(object):
             layers = (data.getVar('BBLAYERS') or "").split()
             broken_layers = []
 
+            if not layers:
+                bb.fatal("The bblayers.conf file doesn't contain any BBLAYERS definition")
+
             data = bb.data.createCopy(data)
             approved = bb.utils.approved_variables()
 
@@ -396,6 +401,8 @@ class CookerDataBuilder(object):
                 if c in collections_tmp:
                     bb.fatal("Found duplicated BBFILE_COLLECTIONS '%s', check bblayers.conf or layer.conf to fix it." % c)
                 compat = set((data.getVar("LAYERSERIES_COMPAT_%s" % c) or "").split())
+                if compat and not layerseries:
+                    bb.fatal("No core layer found to work with layer '%s'. Missing entry in bblayers.conf?" % c)
                 if compat and not (compat & layerseries):
                     bb.fatal("Layer %s is not compatible with the core layer which only supports these series: %s (layer is compatible with %s)"
                               % (c, " ".join(layerseries), " ".join(compat)))
@@ -429,7 +436,7 @@ class CookerDataBuilder(object):
                 parselog.critical("Undefined event handler function '%s'" % var)
                 raise bb.BBHandledException()
             handlerln = int(data.getVarFlag(var, "lineno", False))
-            bb.event.register(var, data.getVar(var, False),  (data.getVarFlag(var, "eventmask") or "").split(), handlerfn, handlerln)
+            bb.event.register(var, data.getVar(var, False),  (data.getVarFlag(var, "eventmask") or "").split(), handlerfn, handlerln, data)
 
         data.setVar('BBINCLUDED',bb.parse.get_file_depends(data))
 

bitbake/lib/bb/data_smart.py

@@ -17,7 +17,7 @@ BitBake build tools.
 # Based on functions from the base bb module, Copyright 2003 Holger Schurig
 
 import copy, re, sys, traceback
-from collections import MutableMapping
+from collections.abc import MutableMapping
 import logging
 import hashlib
 import bb, bb.codeparser
@@ -28,7 +28,7 @@ logger = logging.getLogger("BitBake.Data")
 
 __setvar_keyword__ = ["_append", "_prepend", "_remove"]
 __setvar_regexp__ = re.compile(r'(?P<base>.*?)(?P<keyword>_append|_prepend|_remove)(_(?P<add>[^A-Z]*))?$')
-__expand_var_regexp__ = re.compile(r"\${[a-zA-Z0-9\-_+./~]+?}")
+__expand_var_regexp__ = re.compile(r"\${[a-zA-Z0-9\-_+./~:]+?}")
 __expand_python_regexp__ = re.compile(r"\${@.+?}")
 __whitespace_split__ = re.compile(r'(\s)')
 __override_regexp__ = re.compile(r'[a-z0-9]+')
@@ -403,7 +403,7 @@ class DataSmart(MutableMapping):
                     s = __expand_python_regexp__.sub(varparse.python_sub, s)
                 except SyntaxError as e:
                     # Likely unmatched brackets, just don't expand the expression
-                    if e.msg != "EOL while scanning string literal":
+                    if e.msg != "EOL while scanning string literal" and not e.msg.startswith("unterminated string literal"):
                         raise
                 if s == olds:
                     break
@@ -411,6 +411,8 @@ class DataSmart(MutableMapping):
                 raise
             except bb.parse.SkipRecipe:
                 raise
+            except bb.BBHandledException:
+                raise
             except Exception as exc:
                 tb = sys.exc_info()[2]
                 raise ExpansionError(varname, s, exc).with_traceback(tb) from exc
@@ -481,6 +483,7 @@ class DataSmart(MutableMapping):
 
     def setVar(self, var, value, **loginfo):
         #print("var=" + str(var) + "  val=" + str(value))
+        var = var.replace(":", "_")
         self.expand_cache = {}
         parsing=False
         if 'parsing' in loginfo:
@@ -589,6 +592,8 @@ class DataSmart(MutableMapping):
         """
         Rename the variable key to newkey
         """
+        key = key.replace(":", "_")
+        newkey = newkey.replace(":", "_")
         if key == newkey:
             bb.warn("Calling renameVar with equivalent keys (%s) is invalid" % key)
             return
@@ -637,6 +642,7 @@ class DataSmart(MutableMapping):
         self.setVar(var + "_prepend", value, ignore=True, parsing=True)
 
     def delVar(self, var, **loginfo):
+        var = var.replace(":", "_")
         self.expand_cache = {}
 
         loginfo['detail'] = ""
@@ -664,6 +670,7 @@ class DataSmart(MutableMapping):
                          override = None
 
     def setVarFlag(self, var, flag, value, **loginfo):
+        var = var.replace(":", "_")
         self.expand_cache = {}
 
         if 'op' not in loginfo:
@@ -687,6 +694,7 @@ class DataSmart(MutableMapping):
             self.dict["__exportlist"]["_content"].add(var)
 
     def getVarFlag(self, var, flag, expand=True, noweakdefault=False, parsing=False, retparser=False):
+        var = var.replace(":", "_")
         if flag == "_content":
             cachename = var
         else:
@@ -814,6 +822,7 @@ class DataSmart(MutableMapping):
         return value
 
     def delVarFlag(self, var, flag, **loginfo):
+        var = var.replace(":", "_")
         self.expand_cache = {}
 
         local_var, _ = self._findVar(var)
@@ -831,6 +840,7 @@ class DataSmart(MutableMapping):
             del self.dict[var][flag]
 
     def appendVarFlag(self, var, flag, value, **loginfo):
+        var = var.replace(":", "_")
         loginfo['op'] = 'append'
         loginfo['flag'] = flag
         self.varhistory.record(**loginfo)
@@ -838,6 +848,7 @@ class DataSmart(MutableMapping):
         self.setVarFlag(var, flag, newvalue, ignore=True)
 
     def prependVarFlag(self, var, flag, value, **loginfo):
+        var = var.replace(":", "_")
         loginfo['op'] = 'prepend'
         loginfo['flag'] = flag
         self.varhistory.record(**loginfo)
@@ -845,6 +856,7 @@ class DataSmart(MutableMapping):
         self.setVarFlag(var, flag, newvalue, ignore=True)
 
     def setVarFlags(self, var, flags, **loginfo):
+        var = var.replace(":", "_")
         self.expand_cache = {}
         infer_caller_details(loginfo)
         if not var in self.dict:
@@ -859,6 +871,7 @@ class DataSmart(MutableMapping):
             self.dict[var][i] = flags[i]
 
     def getVarFlags(self, var, expand = False, internalflags=False):
+        var = var.replace(":", "_")
         local_var, _ = self._findVar(var)
         flags = {}
 
@@ -875,6 +888,7 @@ class DataSmart(MutableMapping):
 
 
     def delVarFlags(self, var, **loginfo):
+        var = var.replace(":", "_")
         self.expand_cache = {}
         if not var in self.dict:
             self._makeShadowCopy(var)
@@ -1005,7 +1019,7 @@ class DataSmart(MutableMapping):
             else:
                 data.update({key:value})
 
-            varflags = d.getVarFlags(key, internalflags = True)
+            varflags = d.getVarFlags(key, internalflags = True, expand=["vardepvalue"])
             if not varflags:
                 continue
             for f in varflags:

bitbake/lib/bb/event.py

@@ -118,6 +118,8 @@ def fire_class_handlers(event, d):
             if _eventfilter:
                 if not _eventfilter(name, handler, event, d):
                     continue
+            if d is not None and not name in (d.getVar("__BBHANDLERS_MC") or set()):
+                continue
             execute_handler(name, handler, event, d)
 
 ui_queue = []
@@ -227,11 +229,19 @@ def fire_from_worker(event, d):
     fire_ui_handlers(event, d)
 
 noop = lambda _: None
-def register(name, handler, mask=None, filename=None, lineno=None):
+def register(name, handler, mask=None, filename=None, lineno=None, data=None):
     """Register an Event handler"""
 
+    if data is not None and data.getVar("BB_CURRENT_MC"):
+        mc = data.getVar("BB_CURRENT_MC")
+        name = '%s%s' % (mc.replace('-', '_'), name)
+
     # already registered
     if name in _handlers:
+        if data is not None:
+            bbhands_mc = (data.getVar("__BBHANDLERS_MC") or set())
+            bbhands_mc.add(name)
+            data.setVar("__BBHANDLERS_MC", bbhands_mc)
         return AlreadyRegistered
 
     if handler is not None:
@@ -268,10 +278,20 @@ def register(name, handler, mask=None, filename=None, lineno=None):
                     _event_handler_map[m] = {}
                 _event_handler_map[m][name] = True
 
+        if data is not None:
+            bbhands_mc = (data.getVar("__BBHANDLERS_MC") or set())
+            bbhands_mc.add(name)
+            data.setVar("__BBHANDLERS_MC", bbhands_mc)
+
         return Registered
 
-def remove(name, handler):
+def remove(name, handler, data=None):
     """Remove an Event handler"""
+    if data is not None:
+        if data.getVar("BB_CURRENT_MC"):
+            mc = data.getVar("BB_CURRENT_MC")
+            name = '%s%s' % (mc.replace('-', '_'), name)
+
     _handlers.pop(name)
     if name in _catchall_handlers:
         _catchall_handlers.pop(name)
@@ -279,6 +299,12 @@ def remove(name, handler):
         if name in _event_handler_map[event]:
             _event_handler_map[event].pop(name)
 
+    if data is not None:
+        bbhands_mc = (data.getVar("__BBHANDLERS_MC") or set())
+        if name in bbhands_mc:
+            bbhands_mc.remove(name)
+            data.setVar("__BBHANDLERS_MC", bbhands_mc)
+
 def get_handlers():
     return _handlers
 
@@ -644,6 +670,17 @@ class ReachableStamps(Event):
         Event.__init__(self)
         self.stamps = stamps
 
+class StaleSetSceneTasks(Event):
+    """
+    An event listing setscene tasks which are 'stale' and will
+    be rerun. The metadata may use to clean up stale data.
+    tasks is a mapping of tasks and matching stale stamps.
+    """
+
+    def __init__(self, tasks):
+        Event.__init__(self)
+        self.tasks = tasks
+
 class FilesMatchingFound(Event):
     """
     Event when a list of files matching the supplied pattern has

bitbake/lib/bb/fetch2/__init__.py

@@ -290,7 +290,7 @@ class URI(object):
 
     def _param_str_split(self, string, elmdelim, kvdelim="="):
         ret = collections.OrderedDict()
-        for k, v in [x.split(kvdelim, 1) for x in string.split(elmdelim)]:
+        for k, v in [x.split(kvdelim, 1) for x in string.split(elmdelim) if x]:
             ret[k] = v
         return ret
 
@@ -428,8 +428,9 @@ def uri_replace(ud, uri_find, uri_replace, replacements, d, mirrortarball=None):
     uri_decoded = list(decodeurl(ud.url))
     uri_find_decoded = list(decodeurl(uri_find))
     uri_replace_decoded = list(decodeurl(uri_replace))
-    logger.debug(2, "For url %s comparing %s to %s" % (uri_decoded, uri_find_decoded, uri_replace_decoded))
+    logger.debug2("For url %s comparing %s to %s" % (uri_decoded, uri_find_decoded, uri_replace_decoded))
     result_decoded = ['', '', '', '', '', {}]
+    # 0 - type, 1 - host, 2 - path, 3 - user,  4- pswd, 5 - params
     for loc, i in enumerate(uri_find_decoded):
         result_decoded[loc] = uri_decoded[loc]
         regexp = i
@@ -449,6 +450,9 @@ def uri_replace(ud, uri_find, uri_replace, replacements, d, mirrortarball=None):
                 for l in replacements:
                     uri_replace_decoded[loc][k] = uri_replace_decoded[loc][k].replace(l, replacements[l])
                 result_decoded[loc][k] = uri_replace_decoded[loc][k]
+        elif (loc == 3 or loc == 4) and uri_replace_decoded[loc]:
+            # User/password in the replacement is just a straight replacement
+            result_decoded[loc] = uri_replace_decoded[loc]
         elif (re.match(regexp, uri_decoded[loc])):
             if not uri_replace_decoded[loc]:
                 result_decoded[loc] = ""
@@ -474,7 +478,7 @@ def uri_replace(ud, uri_find, uri_replace, replacements, d, mirrortarball=None):
     result = encodeurl(result_decoded)
     if result == ud.url:
         return None
-    logger.debug(2, "For url %s returning %s" % (ud.url, result))
+    logger.debug2("For url %s returning %s" % (ud.url, result))
     return result
 
 methods = []
@@ -499,9 +503,9 @@ def fetcher_init(d):
     # When to drop SCM head revisions controlled by user policy
     srcrev_policy = d.getVar('BB_SRCREV_POLICY') or "clear"
     if srcrev_policy == "cache":
-        logger.debug(1, "Keeping SRCREV cache due to cache policy of: %s", srcrev_policy)
+        logger.debug("Keeping SRCREV cache due to cache policy of: %s", srcrev_policy)
     elif srcrev_policy == "clear":
-        logger.debug(1, "Clearing SRCREV cache due to cache policy of: %s", srcrev_policy)
+        logger.debug("Clearing SRCREV cache due to cache policy of: %s", srcrev_policy)
         revs.clear()
     else:
         raise FetchError("Invalid SRCREV cache policy of: %s" % srcrev_policy)
@@ -562,6 +566,9 @@ def verify_checksum(ud, d, precomputed={}):
 
         checksum_expected = getattr(ud, "%s_expected" % checksum_id)
 
+        if checksum_expected == '':
+            checksum_expected = None
+
         return {
             "id": checksum_id,
             "name": checksum_name,
@@ -612,7 +619,7 @@ def verify_checksum(ud, d, precomputed={}):
 
     for ci in checksum_infos:
         if ci["expected"] and ci["expected"] != ci["data"]:
-            messages.append("File: '%s' has %s checksum %s when %s was " \
+            messages.append("File: '%s' has %s checksum '%s' when '%s' was " \
                             "expected" % (ud.localpath, ci["id"], ci["data"], ci["expected"]))
             bad_checksum = ci["data"]
 
@@ -853,18 +860,13 @@ def runfetchcmd(cmd, d, quiet=False, cleanup=None, log=None, workdir=None):
         if val:
             cmd = 'export ' + var + '=\"%s\"; %s' % (val, cmd)
 
-    # Ensure that a _PYTHON_SYSCONFIGDATA_NAME value set by a recipe
-    # (for example via python3native.bbclass since warrior) is not set for
-    # host Python (otherwise tools like git-make-shallow will fail)
-    cmd = 'unset _PYTHON_SYSCONFIGDATA_NAME; ' + cmd
-
     # Disable pseudo as it may affect ssh, potentially causing it to hang.
     cmd = 'export PSEUDO_DISABLED=1; ' + cmd
 
     if workdir:
-        logger.debug(1, "Running '%s' in %s" % (cmd, workdir))
+        logger.debug("Running '%s' in %s" % (cmd, workdir))
     else:
-        logger.debug(1, "Running %s", cmd)
+        logger.debug("Running %s", cmd)
 
     success = False
     error_message = ""
@@ -873,7 +875,7 @@ def runfetchcmd(cmd, d, quiet=False, cleanup=None, log=None, workdir=None):
         (output, errors) = bb.process.run(cmd, log=log, shell=True, stderr=subprocess.PIPE, cwd=workdir)
         success = True
     except bb.process.NotFoundError as e:
-        error_message = "Fetch command %s" % (e.command)
+        error_message = "Fetch command %s not found" % (e.command)
     except bb.process.ExecutionError as e:
         if e.stdout:
             output = "output:\n%s\n%s" % (e.stdout, e.stderr)
@@ -905,7 +907,7 @@ def check_network_access(d, info, url):
     elif not trusted_network(d, url):
         raise UntrustedUrl(url, info)
     else:
-        logger.debug(1, "Fetcher accessed the network with the command %s" % info)
+        logger.debug("Fetcher accessed the network with the command %s" % info)
 
 def build_mirroruris(origud, mirrors, ld):
     uris = []
@@ -931,7 +933,7 @@ def build_mirroruris(origud, mirrors, ld):
                     continue
 
                 if not trusted_network(ld, newuri):
-                    logger.debug(1, "Mirror %s not in the list of trusted networks, skipping" %  (newuri))
+                    logger.debug("Mirror %s not in the list of trusted networks, skipping" %  (newuri))
                     continue
 
                 # Create a local copy of the mirrors minus the current line
@@ -944,8 +946,8 @@ def build_mirroruris(origud, mirrors, ld):
                     newud = FetchData(newuri, ld)
                     newud.setup_localpath(ld)
                 except bb.fetch2.BBFetchException as e:
-                    logger.debug(1, "Mirror fetch failure for url %s (original url: %s)" % (newuri, origud.url))
-                    logger.debug(1, str(e))
+                    logger.debug("Mirror fetch failure for url %s (original url: %s)" % (newuri, origud.url))
+                    logger.debug(str(e))
                     try:
                         # setup_localpath of file:// urls may fail, we should still see
                         # if mirrors of the url exist
@@ -1048,8 +1050,8 @@ def try_mirror_url(fetch, origud, ud, ld, check = False):
         elif isinstance(e, NoChecksumError):
             raise
         else:
-            logger.debug(1, "Mirror fetch failure for url %s (original url: %s)" % (ud.url, origud.url))
-            logger.debug(1, str(e))
+            logger.debug("Mirror fetch failure for url %s (original url: %s)" % (ud.url, origud.url))
+            logger.debug(str(e))
         try:
             ud.method.clean(ud, ld)
         except UnboundLocalError:
@@ -1248,7 +1250,7 @@ class FetchData(object):
 
             if checksum_name in self.parm:
                 checksum_expected = self.parm[checksum_name]
-            elif self.type not in ["http", "https", "ftp", "ftps", "sftp", "s3"]:
+            elif self.type not in ["http", "https", "ftp", "ftps", "sftp", "s3", "az"]:
                 checksum_expected = None
             else:
                 checksum_expected = d.getVarFlag("SRC_URI", checksum_name)
@@ -1461,6 +1463,10 @@ class FetchMethod(object):
                 cmd = '7z x -so %s | tar x --no-same-owner -f -' % file
             elif file.endswith('.7z'):
                 cmd = '7za x -y %s 1>/dev/null' % file
+            elif file.endswith('.tzst') or file.endswith('.tar.zst'):
+                cmd = 'zstd --decompress --stdout %s | tar x --no-same-owner -f -' % file
+            elif file.endswith('.zst'):
+                cmd = 'zstd --decompress --stdout %s > %s' % (file, efile)
             elif file.endswith('.zip') or file.endswith('.jar'):
                 try:
                     dos = bb.utils.to_boolean(urldata.parm.get('dos'), False)
@@ -1689,7 +1695,7 @@ class Fetch(object):
                 if m.verify_donestamp(ud, self.d) and not m.need_update(ud, self.d):
                     done = True
                 elif m.try_premirror(ud, self.d):
-                    logger.debug(1, "Trying PREMIRRORS")
+                    logger.debug("Trying PREMIRRORS")
                     mirrors = mirror_from_string(self.d.getVar('PREMIRRORS'))
                     done = m.try_mirrors(self, ud, self.d, mirrors)
                     if done:
@@ -1699,7 +1705,7 @@ class Fetch(object):
                             m.update_donestamp(ud, self.d)
                         except ChecksumError as e:
                             logger.warning("Checksum failure encountered with premirror download of %s - will attempt other sources." % u)
-                            logger.debug(1, str(e))
+                            logger.debug(str(e))
                             done = False
 
                 if premirroronly:
@@ -1711,7 +1717,7 @@ class Fetch(object):
                     try:
                         if not trusted_network(self.d, ud.url):
                             raise UntrustedUrl(ud.url)
-                        logger.debug(1, "Trying Upstream")
+                        logger.debug("Trying Upstream")
                         m.download(ud, self.d)
                         if hasattr(m, "build_mirror_data"):
                             m.build_mirror_data(ud, self.d)
@@ -1726,19 +1732,19 @@ class Fetch(object):
                     except BBFetchException as e:
                         if isinstance(e, ChecksumError):
                             logger.warning("Checksum failure encountered with download of %s - will attempt other sources if available" % u)
-                            logger.debug(1, str(e))
+                            logger.debug(str(e))
                             if os.path.exists(ud.localpath):
                                 rename_bad_checksum(ud, e.checksum)
                         elif isinstance(e, NoChecksumError):
                             raise
                         else:
                             logger.warning('Failed to fetch URL %s, attempting MIRRORS if available' % u)
-                            logger.debug(1, str(e))
+                            logger.debug(str(e))
                         firsterr = e
                         # Remove any incomplete fetch
                         if not verified_stamp:
                             m.clean(ud, self.d)
-                        logger.debug(1, "Trying MIRRORS")
+                        logger.debug("Trying MIRRORS")
                         mirrors = mirror_from_string(self.d.getVar('MIRRORS'))
                         done = m.try_mirrors(self, ud, self.d, mirrors)
 
@@ -1775,7 +1781,7 @@ class Fetch(object):
             ud = self.ud[u]
             ud.setup_localpath(self.d)
             m = ud.method
-            logger.debug(1, "Testing URL %s", u)
+            logger.debug("Testing URL %s", u)
             # First try checking uri, u, from PREMIRRORS
             mirrors = mirror_from_string(self.d.getVar('PREMIRRORS'))
             ret = m.try_mirrors(self, ud, self.d, mirrors, True)
@@ -1909,6 +1915,7 @@ from . import repo
 from . import clearcase
 from . import npm
 from . import npmsw
+from . import az
 
 methods.append(local.Local())
 methods.append(wget.Wget())
@@ -1928,3 +1935,4 @@ methods.append(repo.Repo())
 methods.append(clearcase.ClearCase())
 methods.append(npm.Npm())
 methods.append(npmsw.NpmShrinkWrap())
+methods.append(az.Az())

bitbake/lib/bb/fetch2/az.py

@@ -0,0 +1,93 @@
+"""
+BitBake 'Fetch' Azure Storage implementation
+
+"""
+
+# Copyright (C) 2021 Alejandro Hernandez Samaniego
+#
+# Based on bb.fetch2.wget:
+# Copyright (C) 2003, 2004  Chris Larson
+#
+# SPDX-License-Identifier: GPL-2.0-only
+#
+# Based on functions from the base bb module, Copyright 2003 Holger Schurig
+
+import shlex
+import os
+import bb
+from   bb.fetch2 import FetchError
+from   bb.fetch2 import logger
+from   bb.fetch2.wget import Wget
+
+
+class Az(Wget):
+
+    def supports(self, ud, d):
+        """
+        Check to see if a given url can be fetched from Azure Storage
+        """
+        return ud.type in ['az']
+
+
+    def checkstatus(self, fetch, ud, d, try_again=True):
+
+        # checkstatus discards parameters either way, we need to do this before adding the SAS
+        ud.url = ud.url.replace('az://','https://').split(';')[0]
+
+        az_sas = d.getVar('AZ_SAS')
+        if az_sas and az_sas not in ud.url:
+            ud.url += az_sas
+
+        return Wget.checkstatus(self, fetch, ud, d, try_again)
+
+    # Override download method, include retries
+    def download(self, ud, d, retries=3):
+        """Fetch urls"""
+
+        # If were reaching the account transaction limit we might be refused a connection,
+        # retrying allows us to avoid false negatives since the limit changes over time
+        fetchcmd = self.basecmd + ' --retry-connrefused --waitretry=5'
+
+        # We need to provide a localpath to avoid wget using the SAS
+        # ud.localfile either has the downloadfilename or ud.path
+        localpath = os.path.join(d.getVar("DL_DIR"), ud.localfile)
+        bb.utils.mkdirhier(os.path.dirname(localpath))
+        fetchcmd += " -O %s" % shlex.quote(localpath)
+
+
+        if ud.user and ud.pswd:
+            fetchcmd += " --user=%s --password=%s --auth-no-challenge" % (ud.user, ud.pswd)
+
+        # Check if a Shared Access Signature was given and use it
+        az_sas = d.getVar('AZ_SAS')
+
+        if az_sas:
+            azuri = '%s%s%s%s' % ('https://', ud.host, ud.path, az_sas)
+        else:
+            azuri = '%s%s%s' % ('https://', ud.host, ud.path)
+
+        if os.path.exists(ud.localpath):
+            # file exists, but we didnt complete it.. trying again.
+            fetchcmd += d.expand(" -c -P ${DL_DIR} '%s'" % azuri)
+        else:
+            fetchcmd += d.expand(" -P ${DL_DIR} '%s'" % azuri)
+
+        try:
+            self._runwget(ud, d, fetchcmd, False)
+        except FetchError as e:
+            # Azure fails on handshake sometimes when using wget after some stress, producing a
+            # FetchError from the fetcher, if the artifact exists retyring should succeed
+            if 'Unable to establish SSL connection' in str(e):
+                logger.debug2('Unable to establish SSL connection: Retries remaining: %s, Retrying...' % retries)
+                self.download(ud, d, retries -1)
+
+        # Sanity check since wget can pretend it succeed when it didn't
+        # Also, this used to happen if sourceforge sent us to the mirror page
+        if not os.path.exists(ud.localpath):
+            raise FetchError("The fetch command returned success for url %s but %s doesn't exist?!" % (azuri, ud.localpath), azuri)
+
+        if os.path.getsize(ud.localpath) == 0:
+            os.remove(ud.localpath)
+            raise FetchError("The fetch of %s resulted in a zero size file?! Deleting and failing since this isn't right." % (azuri), azuri)
+
+        return True

bitbake/lib/bb/fetch2/bzr.py

@@ -74,16 +74,16 @@ class Bzr(FetchMethod):
 
         if os.access(os.path.join(ud.pkgdir, os.path.basename(ud.pkgdir), '.bzr'), os.R_OK):
             bzrcmd = self._buildbzrcommand(ud, d, "update")
-            logger.debug(1, "BZR Update %s", ud.url)
+            logger.debug("BZR Update %s", ud.url)
             bb.fetch2.check_network_access(d, bzrcmd, ud.url)
             runfetchcmd(bzrcmd, d, workdir=os.path.join(ud.pkgdir, os.path.basename(ud.path)))
         else:
             bb.utils.remove(os.path.join(ud.pkgdir, os.path.basename(ud.pkgdir)), True)
             bzrcmd = self._buildbzrcommand(ud, d, "fetch")
             bb.fetch2.check_network_access(d, bzrcmd, ud.url)
-            logger.debug(1, "BZR Checkout %s", ud.url)
+            logger.debug("BZR Checkout %s", ud.url)
             bb.utils.mkdirhier(ud.pkgdir)
-            logger.debug(1, "Running %s", bzrcmd)
+            logger.debug("Running %s", bzrcmd)
             runfetchcmd(bzrcmd, d, workdir=ud.pkgdir)
 
         scmdata = ud.parm.get("scmdata", "")
@@ -109,7 +109,7 @@ class Bzr(FetchMethod):
         """
         Return the latest upstream revision number
         """
-        logger.debug(2, "BZR fetcher hitting network for %s", ud.url)
+        logger.debug2("BZR fetcher hitting network for %s", ud.url)
 
         bb.fetch2.check_network_access(d, self._buildbzrcommand(ud, d, "revno"), ud.url)
 

bitbake/lib/bb/fetch2/clearcase.py

@@ -70,7 +70,7 @@ class ClearCase(FetchMethod):
         return ud.type in ['ccrc']
 
     def debug(self, msg):
-        logger.debug(1, "ClearCase: %s", msg)
+        logger.debug("ClearCase: %s", msg)
 
     def urldata_init(self, ud, d):
         """

bitbake/lib/bb/fetch2/cvs.py

@@ -109,7 +109,7 @@ class Cvs(FetchMethod):
             cvsupdatecmd = "CVS_RSH=\"%s\" %s" % (cvs_rsh, cvsupdatecmd)
 
         # create module directory
-        logger.debug(2, "Fetch: checking for module directory")
+        logger.debug2("Fetch: checking for module directory")
         moddir = os.path.join(ud.pkgdir, localdir)
         workdir = None
         if os.access(os.path.join(moddir, 'CVS'), os.R_OK):
@@ -123,7 +123,7 @@ class Cvs(FetchMethod):
             # check out sources there
             bb.utils.mkdirhier(ud.pkgdir)
             workdir = ud.pkgdir
-            logger.debug(1, "Running %s", cvscmd)
+            logger.debug("Running %s", cvscmd)
             bb.fetch2.check_network_access(d, cvscmd, ud.url)
             cmd = cvscmd
 

bitbake/lib/bb/fetch2/git.py

@@ -68,6 +68,7 @@ import subprocess
 import tempfile
 import bb
 import bb.progress
+from contextlib import contextmanager
 from   bb.fetch2 import FetchMethod
 from   bb.fetch2 import runfetchcmd
 from   bb.fetch2 import logger
@@ -141,6 +142,10 @@ class Git(FetchMethod):
             ud.proto = 'file'
         else:
             ud.proto = "git"
+        if ud.host == "github.com" and ud.proto == "git":
+            # github stopped supporting git protocol
+            # https://github.blog/2021-09-01-improving-git-protocol-security-github/#no-more-unauthenticated-git
+            ud.proto = "https"
 
         if not ud.proto in ('git', 'file', 'ssh', 'http', 'https', 'rsync'):
             raise bb.fetch2.ParameterError("Invalid protocol type", ud.url)
@@ -220,7 +225,12 @@ class Git(FetchMethod):
             ud.shallow = False
 
         if ud.usehead:
-            ud.unresolvedrev['default'] = 'HEAD'
+            # When usehead is set let's associate 'HEAD' with the unresolved
+            # rev of this repository. This will get resolved into a revision
+            # later. If an actual revision happens to have also been provided
+            # then this setting will be overridden.
+            for name in ud.names:
+                ud.unresolvedrev[name] = 'HEAD'
 
         ud.basecmd = d.getVar("FETCHCMD_git") or "git -c core.fsyncobjectfiles=0"
 
@@ -379,7 +389,50 @@ class Git(FetchMethod):
             if missing_rev:
                 raise bb.fetch2.FetchError("Unable to find revision %s even from upstream" % missing_rev)
 
+        if self._contains_lfs(ud, d, ud.clonedir) and self._need_lfs(ud):
+            # Unpack temporary working copy, use it to run 'git checkout' to force pre-fetching
+            # of all LFS blobs needed at the the srcrev.
+            #
+            # It would be nice to just do this inline here by running 'git-lfs fetch'
+            # on the bare clonedir, but that operation requires a working copy on some
+            # releases of Git LFS.
+            tmpdir = tempfile.mkdtemp(dir=d.getVar('DL_DIR'))
+            try:
+                # Do the checkout. This implicitly involves a Git LFS fetch.
+                Git.unpack(self, ud, tmpdir, d)
+
+                # Scoop up a copy of any stuff that Git LFS downloaded. Merge them into
+                # the bare clonedir.
+                #
+                # As this procedure is invoked repeatedly on incremental fetches as
+                # a recipe's SRCREV is bumped throughout its lifetime, this will
+                # result in a gradual accumulation of LFS blobs in <ud.clonedir>/lfs
+                # corresponding to all the blobs reachable from the different revs
+                # fetched across time.
+                #
+                # Only do this if the unpack resulted in a .git/lfs directory being
+                # created; this only happens if at least one blob needed to be
+                # downloaded.
+                if os.path.exists(os.path.join(tmpdir, "git", ".git", "lfs")):
+                    runfetchcmd("tar -cf - lfs | tar -xf - -C %s" % ud.clonedir, d, workdir="%s/git/.git" % tmpdir)
+            finally:
+                bb.utils.remove(tmpdir, recurse=True)
+
     def build_mirror_data(self, ud, d):
+
+        # Create as a temp file and move atomically into position to avoid races
+        @contextmanager
+        def create_atomic(filename):
+            fd, tfile = tempfile.mkstemp(dir=os.path.dirname(filename))
+            try:
+                yield tfile
+                umask = os.umask(0o666)
+                os.umask(umask)
+                os.chmod(tfile, (0o666 & ~umask))
+                os.rename(tfile, filename)
+            finally:
+                os.close(fd)
+
         if ud.shallow and ud.write_shallow_tarballs:
             if not os.path.exists(ud.fullshallow):
                 if os.path.islink(ud.fullshallow):
@@ -390,7 +443,8 @@ class Git(FetchMethod):
                     self.clone_shallow_local(ud, shallowclone, d)
 
                     logger.info("Creating tarball of git repository")
-                    runfetchcmd("tar -czf %s ." % ud.fullshallow, d, workdir=shallowclone)
+                    with create_atomic(ud.fullshallow) as tfile:
+                        runfetchcmd("tar -czf %s ." % tfile, d, workdir=shallowclone)
                     runfetchcmd("touch %s.done" % ud.fullshallow, d)
                 finally:
                     bb.utils.remove(tempdir, recurse=True)
@@ -399,7 +453,8 @@ class Git(FetchMethod):
                 os.unlink(ud.fullmirror)
 
             logger.info("Creating tarball of git repository")
-            runfetchcmd("tar -czf %s ." % ud.fullmirror, d, workdir=ud.clonedir)
+            with create_atomic(ud.fullmirror) as tfile:
+                runfetchcmd("tar -czf %s ." % tfile, d, workdir=ud.clonedir)
             runfetchcmd("touch %s.done" % ud.fullmirror, d)
 
     def clone_shallow_local(self, ud, dest, d):
@@ -474,7 +529,7 @@ class Git(FetchMethod):
         if os.path.exists(destdir):
             bb.utils.prunedir(destdir)
 
-        need_lfs = ud.parm.get("lfs", "1") == "1"
+        need_lfs = self._need_lfs(ud)
 
         if not need_lfs:
             ud.basecmd = "GIT_LFS_SKIP_SMUDGE=1 " + ud.basecmd
@@ -563,6 +618,9 @@ class Git(FetchMethod):
             raise bb.fetch2.FetchError("The command '%s' gave output with more then 1 line unexpectedly, output: '%s'" % (cmd, output))
         return output.split()[0] != "0"
 
+    def _need_lfs(self, ud):
+        return ud.parm.get("lfs", "1") == "1"
+
     def _contains_lfs(self, ud, d, wd):
         """
         Check if the repository has 'lfs' (large file) content
@@ -573,8 +631,14 @@ class Git(FetchMethod):
         else:
             branchname = "master"
 
-        cmd = "%s grep lfs origin/%s:.gitattributes | wc -l" % (
-            ud.basecmd, ud.branches[ud.names[0]])
+        # The bare clonedir doesn't use the remote names; it has the branch immediately.
+        if wd == ud.clonedir:
+            refname = ud.branches[ud.names[0]]
+        else:
+            refname = "origin/%s" % ud.branches[ud.names[0]]
+
+        cmd = "%s grep lfs %s:.gitattributes | wc -l" % (
+            ud.basecmd, refname)
 
         try:
             output = runfetchcmd(cmd, d, quiet=True, workdir=wd)
@@ -595,6 +659,11 @@ class Git(FetchMethod):
         """
         Return the repository URL
         """
+        # Note that we do not support passwords directly in the git urls. There are several
+        # reasons. SRC_URI can be written out to things like buildhistory and people don't
+        # want to leak passwords like that. Its also all too easy to share metadata without 
+        # removing the password. ssh keys, ~/.netrc and ~/.ssh/config files can be used as
+        # alternatives so we will not take patches adding password support here.
         if ud.user:
             username = ud.user + '@'
         else:

bitbake/lib/bb/fetch2/gitsm.py

@@ -78,7 +78,7 @@ class GitSM(Git):
                     module_hash = ""
 
                 if not module_hash:
-                    logger.debug(1, "submodule %s is defined, but is not initialized in the repository. Skipping", m)
+                    logger.debug("submodule %s is defined, but is not initialized in the repository. Skipping", m)
                     continue
 
                 submodules.append(m)
@@ -179,7 +179,7 @@ class GitSM(Git):
                             (ud.basecmd, ud.revisions[ud.names[0]]), d, workdir=ud.clonedir)
 
         if len(need_update_list) > 0:
-            logger.debug(1, 'gitsm: Submodules requiring update: %s' % (' '.join(need_update_list)))
+            logger.debug('gitsm: Submodules requiring update: %s' % (' '.join(need_update_list)))
             return True
 
         return False

bitbake/lib/bb/fetch2/hg.py

@@ -150,7 +150,7 @@ class Hg(FetchMethod):
     def download(self, ud, d):
         """Fetch url"""
 
-        logger.debug(2, "Fetch: checking for module directory '" + ud.moddir + "'")
+        logger.debug2("Fetch: checking for module directory '" + ud.moddir + "'")
 
         # If the checkout doesn't exist and the mirror tarball does, extract it
         if not os.path.exists(ud.pkgdir) and os.path.exists(ud.fullmirror):
@@ -160,7 +160,7 @@ class Hg(FetchMethod):
         if os.access(os.path.join(ud.moddir, '.hg'), os.R_OK):
             # Found the source, check whether need pull
             updatecmd = self._buildhgcommand(ud, d, "update")
-            logger.debug(1, "Running %s", updatecmd)
+            logger.debug("Running %s", updatecmd)
             try:
                 runfetchcmd(updatecmd, d, workdir=ud.moddir)
             except bb.fetch2.FetchError:
@@ -168,7 +168,7 @@ class Hg(FetchMethod):
                 pullcmd = self._buildhgcommand(ud, d, "pull")
                 logger.info("Pulling " + ud.url)
                 # update sources there
-                logger.debug(1, "Running %s", pullcmd)
+                logger.debug("Running %s", pullcmd)
                 bb.fetch2.check_network_access(d, pullcmd, ud.url)
                 runfetchcmd(pullcmd, d, workdir=ud.moddir)
                 try:
@@ -183,14 +183,14 @@ class Hg(FetchMethod):
             logger.info("Fetch " + ud.url)
             # check out sources there
             bb.utils.mkdirhier(ud.pkgdir)
-            logger.debug(1, "Running %s", fetchcmd)
+            logger.debug("Running %s", fetchcmd)
             bb.fetch2.check_network_access(d, fetchcmd, ud.url)
             runfetchcmd(fetchcmd, d, workdir=ud.pkgdir)
 
         # Even when we clone (fetch), we still need to update as hg's clone
         # won't checkout the specified revision if its on a branch
         updatecmd = self._buildhgcommand(ud, d, "update")
-        logger.debug(1, "Running %s", updatecmd)
+        logger.debug("Running %s", updatecmd)
         runfetchcmd(updatecmd, d, workdir=ud.moddir)
 
     def clean(self, ud, d):
@@ -247,9 +247,9 @@ class Hg(FetchMethod):
         if scmdata != "nokeep":
             proto = ud.parm.get('protocol', 'http')
             if not os.access(os.path.join(codir, '.hg'), os.R_OK):
-                logger.debug(2, "Unpack: creating new hg repository in '" + codir + "'")
+                logger.debug2("Unpack: creating new hg repository in '" + codir + "'")
                 runfetchcmd("%s init %s" % (ud.basecmd, codir), d)
-            logger.debug(2, "Unpack: updating source in '" + codir + "'")
+            logger.debug2("Unpack: updating source in '" + codir + "'")
             if ud.user and ud.pswd:
                 runfetchcmd("%s --config auth.default.prefix=* --config auth.default.username=%s --config auth.default.password=%s --config \"auth.default.schemes=%s\" pull %s" % (ud.basecmd, ud.user, ud.pswd, proto, ud.moddir), d, workdir=codir)
             else:
@@ -259,5 +259,5 @@ class Hg(FetchMethod):
             else:
                 runfetchcmd("%s up -C %s" % (ud.basecmd, revflag), d, workdir=codir)
         else:
-            logger.debug(2, "Unpack: extracting source to '" + codir + "'")
+            logger.debug2("Unpack: extracting source to '" + codir + "'")
             runfetchcmd("%s archive -t files %s %s" % (ud.basecmd, revflag, codir), d, workdir=ud.moddir)

bitbake/lib/bb/fetch2/local.py

@@ -54,12 +54,12 @@ class Local(FetchMethod):
             return [path]
         filespath = d.getVar('FILESPATH')
         if filespath:
-            logger.debug(2, "Searching for %s in paths:\n    %s" % (path, "\n    ".join(filespath.split(":"))))
+            logger.debug2("Searching for %s in paths:\n    %s" % (path, "\n    ".join(filespath.split(":"))))
             newpath, hist = bb.utils.which(filespath, path, history=True)
             searched.extend(hist)
         if not os.path.exists(newpath):
             dldirfile = os.path.join(d.getVar("DL_DIR"), path)
-            logger.debug(2, "Defaulting to %s for %s" % (dldirfile, path))
+            logger.debug2("Defaulting to %s for %s" % (dldirfile, path))
             bb.utils.mkdirhier(os.path.dirname(dldirfile))
             searched.append(dldirfile)
             return searched

bitbake/lib/bb/fetch2/npmsw.py

@@ -29,6 +29,8 @@ from bb.fetch2.npm import npm_integrity
 from bb.fetch2.npm import npm_localfile
 from bb.fetch2.npm import npm_unpack
 from bb.utils import is_semver
+from bb.utils import lockfile
+from bb.utils import unlockfile
 
 def foreach_dependencies(shrinkwrap, callback=None, dev=False):
     """
@@ -187,7 +189,9 @@ class NpmShrinkWrap(FetchMethod):
             proxy_ud = ud.proxy.ud[proxy_url]
             proxy_d = ud.proxy.d
             proxy_ud.setup_localpath(proxy_d)
+            lf = lockfile(proxy_ud.lockfile)
             returns.append(handle(proxy_ud.method, proxy_ud, proxy_d))
+            unlockfile(lf)
         return returns
 
     def verify_donestamp(self, ud, d):

bitbake/lib/bb/fetch2/osc.py

@@ -84,13 +84,13 @@ class Osc(FetchMethod):
         Fetch url
         """
 
-        logger.debug(2, "Fetch: checking for module directory '" + ud.moddir + "'")
+        logger.debug2("Fetch: checking for module directory '" + ud.moddir + "'")
 
         if os.access(os.path.join(d.getVar('OSCDIR'), ud.path, ud.module), os.R_OK):
             oscupdatecmd = self._buildosccommand(ud, d, "update")
             logger.info("Update "+ ud.url)
             # update sources there
-            logger.debug(1, "Running %s", oscupdatecmd)
+            logger.debug("Running %s", oscupdatecmd)
             bb.fetch2.check_network_access(d, oscupdatecmd, ud.url)
             runfetchcmd(oscupdatecmd, d, workdir=ud.moddir)
         else:
@@ -98,7 +98,7 @@ class Osc(FetchMethod):
             logger.info("Fetch " + ud.url)
             # check out sources there
             bb.utils.mkdirhier(ud.pkgdir)
-            logger.debug(1, "Running %s", oscfetchcmd)
+            logger.debug("Running %s", oscfetchcmd)
             bb.fetch2.check_network_access(d, oscfetchcmd, ud.url)
             runfetchcmd(oscfetchcmd, d, workdir=ud.pkgdir)
 

bitbake/lib/bb/fetch2/perforce.py

@@ -90,16 +90,16 @@ class Perforce(FetchMethod):
         p4port = d.getVar('P4PORT')
 
         if p4port:
-            logger.debug(1, 'Using recipe provided P4PORT: %s' % p4port)
+            logger.debug('Using recipe provided P4PORT: %s' % p4port)
             ud.host = p4port
         else:
-            logger.debug(1, 'Trying to use P4CONFIG to automatically set P4PORT...')
+            logger.debug('Trying to use P4CONFIG to automatically set P4PORT...')
             ud.usingp4config = True
             p4cmd = '%s info | grep "Server address"' % ud.basecmd
             bb.fetch2.check_network_access(d, p4cmd, ud.url)
             ud.host = runfetchcmd(p4cmd, d, True)
             ud.host = ud.host.split(': ')[1].strip()
-            logger.debug(1, 'Determined P4PORT to be: %s' % ud.host)
+            logger.debug('Determined P4PORT to be: %s' % ud.host)
             if not ud.host:
                 raise FetchError('Could not determine P4PORT from P4CONFIG')
 
@@ -119,6 +119,7 @@ class Perforce(FetchMethod):
         cleanedpath = ud.path.replace('/...', '').replace('/', '.')
         cleanedhost = ud.host.replace(':', '.')
 
+        cleanedmodule = ""
         # Merge the path and module into the final depot location
         if ud.module:
             if ud.module.find('/') == 0:
@@ -133,7 +134,7 @@ class Perforce(FetchMethod):
 
         ud.setup_revisions(d)
 
-        ud.localfile = d.expand('%s_%s_%s.tar.gz' % (cleanedhost, cleanedpath, ud.revision))
+        ud.localfile = d.expand('%s_%s_%s_%s.tar.gz' % (cleanedhost, cleanedpath, cleanedmodule, ud.revision))
 
     def _buildp4command(self, ud, d, command, depot_filename=None):
         """
@@ -207,7 +208,7 @@ class Perforce(FetchMethod):
         for filename in p4fileslist:
             item = filename.split(' - ')
             lastaction = item[1].split()
-            logger.debug(1, 'File: %s Last Action: %s' % (item[0], lastaction[0]))
+            logger.debug('File: %s Last Action: %s' % (item[0], lastaction[0]))
             if lastaction[0] == 'delete':
                 continue
             filelist.append(item[0])
@@ -254,7 +255,7 @@ class Perforce(FetchMethod):
             raise FetchError('Could not determine the latest perforce changelist')
 
         tipcset = tip.split(' ')[1]
-        logger.debug(1, 'p4 tip found to be changelist %s' % tipcset)
+        logger.debug('p4 tip found to be changelist %s' % tipcset)
         return tipcset
 
     def sortable_revision(self, ud, d, name):

bitbake/lib/bb/fetch2/repo.py

@@ -47,7 +47,7 @@ class Repo(FetchMethod):
         """Fetch url"""
 
         if os.access(os.path.join(d.getVar("DL_DIR"), ud.localfile), os.R_OK):
-            logger.debug(1, "%s already exists (or was stashed). Skipping repo init / sync.", ud.localpath)
+            logger.debug("%s already exists (or was stashed). Skipping repo init / sync.", ud.localpath)
             return
 
         repodir = d.getVar("REPODIR") or (d.getVar("DL_DIR") + "/repo")

bitbake/lib/bb/fetch2/svn.py

@@ -86,7 +86,7 @@ class Svn(FetchMethod):
         if command == "info":
             svncmd = "%s info %s %s://%s/%s/" % (ud.basecmd, " ".join(options), proto, svnroot, ud.module)
         elif command == "log1":
-            svncmd = "%s log --limit 1 %s %s://%s/%s/" % (ud.basecmd, " ".join(options), proto, svnroot, ud.module)
+            svncmd = "%s log --limit 1 --quiet %s %s://%s/%s/" % (ud.basecmd, " ".join(options), proto, svnroot, ud.module)
         else:
             suffix = ""
 
@@ -116,7 +116,7 @@ class Svn(FetchMethod):
     def download(self, ud, d):
         """Fetch url"""
 
-        logger.debug(2, "Fetch: checking for module directory '" + ud.moddir + "'")
+        logger.debug2("Fetch: checking for module directory '" + ud.moddir + "'")
 
         lf = bb.utils.lockfile(ud.svnlock)
 
@@ -129,7 +129,7 @@ class Svn(FetchMethod):
                     runfetchcmd(ud.basecmd + " upgrade", d, workdir=ud.moddir)
                 except FetchError:
                     pass
-                logger.debug(1, "Running %s", svncmd)
+                logger.debug("Running %s", svncmd)
                 bb.fetch2.check_network_access(d, svncmd, ud.url)
                 runfetchcmd(svncmd, d, workdir=ud.moddir)
             else:
@@ -137,7 +137,7 @@ class Svn(FetchMethod):
                 logger.info("Fetch " + ud.url)
                 # check out sources there
                 bb.utils.mkdirhier(ud.pkgdir)
-                logger.debug(1, "Running %s", svncmd)
+                logger.debug("Running %s", svncmd)
                 bb.fetch2.check_network_access(d, svncmd, ud.url)
                 runfetchcmd(svncmd, d, workdir=ud.pkgdir)
 

bitbake/lib/bb/fetch2/wget.py

@@ -52,6 +52,12 @@ class WgetProgressHandler(bb.progress.LineFilterProgressHandler):
 
 
 class Wget(FetchMethod):
+
+    # CDNs like CloudFlare may do a 'browser integrity test' which can fail
+    # with the standard wget/urllib User-Agent, so pretend to be a modern
+    # browser.
+    user_agent = "Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:84.0) Gecko/20100101 Firefox/84.0"
+
     """Class to fetch urls via 'wget'"""
     def supports(self, ud, d):
         """
@@ -82,7 +88,7 @@ class Wget(FetchMethod):
 
         progresshandler = WgetProgressHandler(d)
 
-        logger.debug(2, "Fetching %s using command '%s'" % (ud.url, command))
+        logger.debug2("Fetching %s using command '%s'" % (ud.url, command))
         bb.fetch2.check_network_access(d, command, ud.url)
         runfetchcmd(command + ' --progress=dot -v', d, quiet, log=progresshandler, workdir=workdir)
 
@@ -297,7 +303,7 @@ class Wget(FetchMethod):
             # Some servers (FusionForge, as used on Alioth) require that the
             # optional Accept header is set.
             r.add_header("Accept", "*/*")
-            r.add_header("User-Agent", "Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.9.2.12) Gecko/20101027 Ubuntu/9.10 (karmic) Firefox/3.6.12")
+            r.add_header("User-Agent", self.user_agent)
             def add_basic_auth(login_str, request):
                 '''Adds Basic auth to http request, pass in login:password as string'''
                 import base64
@@ -316,15 +322,23 @@ class Wget(FetchMethod):
             except (TypeError, ImportError, IOError, netrc.NetrcParseError):
                 pass
 
-            with opener.open(r) as response:
+            with opener.open(r, timeout=30) as response:
                 pass
         except urllib.error.URLError as e:
             if try_again:
-                logger.debug(2, "checkstatus: trying again")
+                logger.debug2("checkstatus: trying again")
                 return self.checkstatus(fetch, ud, d, False)
             else:
                 # debug for now to avoid spamming the logs in e.g. remote sstate searches
-                logger.debug(2, "checkstatus() urlopen failed: %s" % e)
+                logger.debug2("checkstatus() urlopen failed: %s" % e)
+                return False
+        except ConnectionResetError as e:
+            if try_again:
+                logger.debug2("checkstatus: trying again")
+                return self.checkstatus(fetch, ud, d, False)
+            else:
+                # debug for now to avoid spamming the logs in e.g. remote sstate searches
+                logger.debug2("checkstatus() urlopen failed: %s" % e)
                 return False
         return True
 
@@ -401,9 +415,8 @@ class Wget(FetchMethod):
         """
         f = tempfile.NamedTemporaryFile()
         with tempfile.TemporaryDirectory(prefix="wget-index-") as workdir, tempfile.NamedTemporaryFile(dir=workdir, prefix="wget-listing-") as f:
-            agent = "Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.9.2.12) Gecko/20101027 Ubuntu/9.10 (karmic) Firefox/3.6.12"
             fetchcmd = self.basecmd
-            fetchcmd += " -O " + f.name + " --user-agent='" + agent + "' '" + uri + "'"
+            fetchcmd += " -O " + f.name + " --user-agent='" + self.user_agent + "' '" + uri + "'"
             try:
                 self._runwget(ud, d, fetchcmd, True, workdir=workdir)
                 fetchresult = f.read()
@@ -459,7 +472,7 @@ class Wget(FetchMethod):
         version_dir = ['', '', '']
         version = ['', '', '']
 
-        dirver_regex = re.compile(r"(?P<pfx>\D*)(?P<ver>(\d+[\.\-_])+(\d+))")
+        dirver_regex = re.compile(r"(?P<pfx>\D*)(?P<ver>(\d+[\.\-_])*(\d+))")
         s = dirver_regex.search(dirver)
         if s:
             version_dir[1] = s.group('ver')

bitbake/lib/bb/main.py

@@ -119,178 +119,181 @@ warnings.filterwarnings("ignore", category=ImportWarning)
 warnings.filterwarnings("ignore", category=DeprecationWarning, module="<string>$")
 warnings.filterwarnings("ignore", message="With-statements now directly support multiple context managers")
 
-class BitBakeConfigParameters(cookerdata.ConfigParameters):
 
-    def parseCommandLine(self, argv=sys.argv):
-        parser = optparse.OptionParser(
-            formatter=BitbakeHelpFormatter(),
-            version="BitBake Build Tool Core version %s" % bb.__version__,
-            usage="""%prog [options] [recipename/target recipe:do_task ...]
+def create_bitbake_parser():
+    parser = optparse.OptionParser(
+        formatter=BitbakeHelpFormatter(),
+        version="BitBake Build Tool Core version %s" % bb.__version__,
+        usage="""%prog [options] [recipename/target recipe:do_task ...]
 
     Executes the specified task (default is 'build') for a given set of target recipes (.bb files).
     It is assumed there is a conf/bblayers.conf available in cwd or in BBPATH which
     will provide the layer, BBFILES and other configuration information.""")
 
-        parser.add_option("-b", "--buildfile", action="store", dest="buildfile", default=None,
-                          help="Execute tasks from a specific .bb recipe directly. WARNING: Does "
-                               "not handle any dependencies from other recipes.")
+    parser.add_option("-b", "--buildfile", action="store", dest="buildfile", default=None,
+                      help="Execute tasks from a specific .bb recipe directly. WARNING: Does "
+                           "not handle any dependencies from other recipes.")
 
-        parser.add_option("-k", "--continue", action="store_false", dest="abort", default=True,
-                          help="Continue as much as possible after an error. While the target that "
-                               "failed and anything depending on it cannot be built, as much as "
-                               "possible will be built before stopping.")
+    parser.add_option("-k", "--continue", action="store_false", dest="abort", default=True,
+                      help="Continue as much as possible after an error. While the target that "
+                           "failed and anything depending on it cannot be built, as much as "
+                           "possible will be built before stopping.")
 
-        parser.add_option("-f", "--force", action="store_true", dest="force", default=False,
-                          help="Force the specified targets/task to run (invalidating any "
-                               "existing stamp file).")
+    parser.add_option("-f", "--force", action="store_true", dest="force", default=False,
+                      help="Force the specified targets/task to run (invalidating any "
+                           "existing stamp file).")
 
-        parser.add_option("-c", "--cmd", action="store", dest="cmd",
-                          help="Specify the task to execute. The exact options available "
-                               "depend on the metadata. Some examples might be 'compile'"
-                               " or 'populate_sysroot' or 'listtasks' may give a list of "
-                               "the tasks available.")
+    parser.add_option("-c", "--cmd", action="store", dest="cmd",
+                      help="Specify the task to execute. The exact options available "
+                           "depend on the metadata. Some examples might be 'compile'"
+                           " or 'populate_sysroot' or 'listtasks' may give a list of "
+                           "the tasks available.")
 
-        parser.add_option("-C", "--clear-stamp", action="store", dest="invalidate_stamp",
-                          help="Invalidate the stamp for the specified task such as 'compile' "
-                               "and then run the default task for the specified target(s).")
+    parser.add_option("-C", "--clear-stamp", action="store", dest="invalidate_stamp",
+                      help="Invalidate the stamp for the specified task such as 'compile' "
+                           "and then run the default task for the specified target(s).")
 
-        parser.add_option("-r", "--read", action="append", dest="prefile", default=[],
-                          help="Read the specified file before bitbake.conf.")
+    parser.add_option("-r", "--read", action="append", dest="prefile", default=[],
+                      help="Read the specified file before bitbake.conf.")
 
-        parser.add_option("-R", "--postread", action="append", dest="postfile", default=[],
-                          help="Read the specified file after bitbake.conf.")
+    parser.add_option("-R", "--postread", action="append", dest="postfile", default=[],
+                      help="Read the specified file after bitbake.conf.")
 
-        parser.add_option("-v", "--verbose", action="store_true", dest="verbose", default=False,
-                          help="Enable tracing of shell tasks (with 'set -x'). "
-                               "Also print bb.note(...) messages to stdout (in "
-                               "addition to writing them to ${T}/log.do_<task>).")
+    parser.add_option("-v", "--verbose", action="store_true", dest="verbose", default=False,
+                      help="Enable tracing of shell tasks (with 'set -x'). "
+                           "Also print bb.note(...) messages to stdout (in "
+                           "addition to writing them to ${T}/log.do_<task>).")
 
-        parser.add_option("-D", "--debug", action="count", dest="debug", default=0,
-                          help="Increase the debug level. You can specify this "
-                               "more than once. -D sets the debug level to 1, "
-                               "where only bb.debug(1, ...) messages are printed "
-                               "to stdout; -DD sets the debug level to 2, where "
-                               "both bb.debug(1, ...) and bb.debug(2, ...) "
-                               "messages are printed; etc. Without -D, no debug "
-                               "messages are printed. Note that -D only affects "
-                               "output to stdout. All debug messages are written "
-                               "to ${T}/log.do_taskname, regardless of the debug "
-                               "level.")
+    parser.add_option("-D", "--debug", action="count", dest="debug", default=0,
+                      help="Increase the debug level. You can specify this "
+                           "more than once. -D sets the debug level to 1, "
+                           "where only bb.debug(1, ...) messages are printed "
+                           "to stdout; -DD sets the debug level to 2, where "
+                           "both bb.debug(1, ...) and bb.debug(2, ...) "
+                           "messages are printed; etc. Without -D, no debug "
+                           "messages are printed. Note that -D only affects "
+                           "output to stdout. All debug messages are written "
+                           "to ${T}/log.do_taskname, regardless of the debug "
+                           "level.")
 
-        parser.add_option("-q", "--quiet", action="count", dest="quiet", default=0,
-                          help="Output less log message data to the terminal. You can specify this more than once.")
+    parser.add_option("-q", "--quiet", action="count", dest="quiet", default=0,
+                      help="Output less log message data to the terminal. You can specify this more than once.")
 
-        parser.add_option("-n", "--dry-run", action="store_true", dest="dry_run", default=False,
-                          help="Don't execute, just go through the motions.")
+    parser.add_option("-n", "--dry-run", action="store_true", dest="dry_run", default=False,
+                      help="Don't execute, just go through the motions.")
 
-        parser.add_option("-S", "--dump-signatures", action="append", dest="dump_signatures",
-                          default=[], metavar="SIGNATURE_HANDLER",
-                          help="Dump out the signature construction information, with no task "
-                               "execution. The SIGNATURE_HANDLER parameter is passed to the "
-                               "handler. Two common values are none and printdiff but the handler "
-                               "may define more/less. none means only dump the signature, printdiff"
-                               " means compare the dumped signature with the cached one.")
+    parser.add_option("-S", "--dump-signatures", action="append", dest="dump_signatures",
+                      default=[], metavar="SIGNATURE_HANDLER",
+                      help="Dump out the signature construction information, with no task "
+                           "execution. The SIGNATURE_HANDLER parameter is passed to the "
+                           "handler. Two common values are none and printdiff but the handler "
+                           "may define more/less. none means only dump the signature, printdiff"
+                           " means compare the dumped signature with the cached one.")
 
-        parser.add_option("-p", "--parse-only", action="store_true",
-                          dest="parse_only", default=False,
-                          help="Quit after parsing the BB recipes.")
+    parser.add_option("-p", "--parse-only", action="store_true",
+                      dest="parse_only", default=False,
+                      help="Quit after parsing the BB recipes.")
 
-        parser.add_option("-s", "--show-versions", action="store_true",
-                          dest="show_versions", default=False,
-                          help="Show current and preferred versions of all recipes.")
+    parser.add_option("-s", "--show-versions", action="store_true",
+                      dest="show_versions", default=False,
+                      help="Show current and preferred versions of all recipes.")
 
-        parser.add_option("-e", "--environment", action="store_true",
-                          dest="show_environment", default=False,
-                          help="Show the global or per-recipe environment complete with information"
-                               " about where variables were set/changed.")
+    parser.add_option("-e", "--environment", action="store_true",
+                      dest="show_environment", default=False,
+                      help="Show the global or per-recipe environment complete with information"
+                           " about where variables were set/changed.")
 
-        parser.add_option("-g", "--graphviz", action="store_true", dest="dot_graph", default=False,
-                          help="Save dependency tree information for the specified "
-                               "targets in the dot syntax.")
+    parser.add_option("-g", "--graphviz", action="store_true", dest="dot_graph", default=False,
+                      help="Save dependency tree information for the specified "
+                           "targets in the dot syntax.")
 
-        parser.add_option("-I", "--ignore-deps", action="append",
-                          dest="extra_assume_provided", default=[],
-                          help="Assume these dependencies don't exist and are already provided "
-                               "(equivalent to ASSUME_PROVIDED). Useful to make dependency "
-                               "graphs more appealing")
+    parser.add_option("-I", "--ignore-deps", action="append",
+                      dest="extra_assume_provided", default=[],
+                      help="Assume these dependencies don't exist and are already provided "
+                           "(equivalent to ASSUME_PROVIDED). Useful to make dependency "
+                           "graphs more appealing")
 
-        parser.add_option("-l", "--log-domains", action="append", dest="debug_domains", default=[],
-                          help="Show debug logging for the specified logging domains")
+    parser.add_option("-l", "--log-domains", action="append", dest="debug_domains", default=[],
+                      help="Show debug logging for the specified logging domains")
 
-        parser.add_option("-P", "--profile", action="store_true", dest="profile", default=False,
-                          help="Profile the command and save reports.")
+    parser.add_option("-P", "--profile", action="store_true", dest="profile", default=False,
+                      help="Profile the command and save reports.")
 
-        # @CHOICES@ is substituted out by BitbakeHelpFormatter above
-        parser.add_option("-u", "--ui", action="store", dest="ui",
-                          default=os.environ.get('BITBAKE_UI', 'knotty'),
-                          help="The user interface to use (@CHOICES@ - default %default).")
+    # @CHOICES@ is substituted out by BitbakeHelpFormatter above
+    parser.add_option("-u", "--ui", action="store", dest="ui",
+                      default=os.environ.get('BITBAKE_UI', 'knotty'),
+                      help="The user interface to use (@CHOICES@ - default %default).")
 
-        parser.add_option("", "--token", action="store", dest="xmlrpctoken",
-                          default=os.environ.get("BBTOKEN"),
-                          help="Specify the connection token to be used when connecting "
-                               "to a remote server.")
+    parser.add_option("", "--token", action="store", dest="xmlrpctoken",
+                      default=os.environ.get("BBTOKEN"),
+                      help="Specify the connection token to be used when connecting "
+                           "to a remote server.")
 
-        parser.add_option("", "--revisions-changed", action="store_true",
-                          dest="revisions_changed", default=False,
-                          help="Set the exit code depending on whether upstream floating "
-                               "revisions have changed or not.")
+    parser.add_option("", "--revisions-changed", action="store_true",
+                      dest="revisions_changed", default=False,
+                      help="Set the exit code depending on whether upstream floating "
+                           "revisions have changed or not.")
 
-        parser.add_option("", "--server-only", action="store_true",
-                          dest="server_only", default=False,
-                          help="Run bitbake without a UI, only starting a server "
-                               "(cooker) process.")
+    parser.add_option("", "--server-only", action="store_true",
+                      dest="server_only", default=False,
+                      help="Run bitbake without a UI, only starting a server "
+                           "(cooker) process.")
 
-        parser.add_option("-B", "--bind", action="store", dest="bind", default=False,
-                          help="The name/address for the bitbake xmlrpc server to bind to.")
+    parser.add_option("-B", "--bind", action="store", dest="bind", default=False,
+                      help="The name/address for the bitbake xmlrpc server to bind to.")
 
-        parser.add_option("-T", "--idle-timeout", type=float, dest="server_timeout",
-                          default=os.getenv("BB_SERVER_TIMEOUT"),
-                          help="Set timeout to unload bitbake server due to inactivity, "
-                                "set to -1 means no unload, "
-                                "default: Environment variable BB_SERVER_TIMEOUT.")
+    parser.add_option("-T", "--idle-timeout", type=float, dest="server_timeout",
+                      default=os.getenv("BB_SERVER_TIMEOUT"),
+                      help="Set timeout to unload bitbake server due to inactivity, "
+                           "set to -1 means no unload, "
+                           "default: Environment variable BB_SERVER_TIMEOUT.")
 
-        parser.add_option("", "--no-setscene", action="store_true",
-                          dest="nosetscene", default=False,
-                          help="Do not run any setscene tasks. sstate will be ignored and "
-                               "everything needed, built.")
+    parser.add_option("", "--no-setscene", action="store_true",
+                      dest="nosetscene", default=False,
+                      help="Do not run any setscene tasks. sstate will be ignored and "
+                           "everything needed, built.")
 
-        parser.add_option("", "--skip-setscene", action="store_true",
-                          dest="skipsetscene", default=False,
-                          help="Skip setscene tasks if they would be executed. Tasks previously "
-                               "restored from sstate will be kept, unlike --no-setscene")
+    parser.add_option("", "--skip-setscene", action="store_true",
+                      dest="skipsetscene", default=False,
+                      help="Skip setscene tasks if they would be executed. Tasks previously "
+                           "restored from sstate will be kept, unlike --no-setscene")
 
-        parser.add_option("", "--setscene-only", action="store_true",
-                          dest="setsceneonly", default=False,
-                          help="Only run setscene tasks, don't run any real tasks.")
+    parser.add_option("", "--setscene-only", action="store_true",
+                      dest="setsceneonly", default=False,
+                      help="Only run setscene tasks, don't run any real tasks.")
 
-        parser.add_option("", "--remote-server", action="store", dest="remote_server",
-                          default=os.environ.get("BBSERVER"),
-                          help="Connect to the specified server.")
+    parser.add_option("", "--remote-server", action="store", dest="remote_server",
+                      default=os.environ.get("BBSERVER"),
+                      help="Connect to the specified server.")
 
-        parser.add_option("-m", "--kill-server", action="store_true",
-                          dest="kill_server", default=False,
-                          help="Terminate any running bitbake server.")
+    parser.add_option("-m", "--kill-server", action="store_true",
+                      dest="kill_server", default=False,
+                      help="Terminate any running bitbake server.")
 
-        parser.add_option("", "--observe-only", action="store_true",
-                          dest="observe_only", default=False,
-                          help="Connect to a server as an observing-only client.")
+    parser.add_option("", "--observe-only", action="store_true",
+                      dest="observe_only", default=False,
+                      help="Connect to a server as an observing-only client.")
 
-        parser.add_option("", "--status-only", action="store_true",
-                          dest="status_only", default=False,
-                          help="Check the status of the remote bitbake server.")
+    parser.add_option("", "--status-only", action="store_true",
+                      dest="status_only", default=False,
+                      help="Check the status of the remote bitbake server.")
 
-        parser.add_option("-w", "--write-log", action="store", dest="writeeventlog",
-                          default=os.environ.get("BBEVENTLOG"),
-                          help="Writes the event log of the build to a bitbake event json file. "
-                               "Use '' (empty string) to assign the name automatically.")
+    parser.add_option("-w", "--write-log", action="store", dest="writeeventlog",
+                      default=os.environ.get("BBEVENTLOG"),
+                      help="Writes the event log of the build to a bitbake event json file. "
+                           "Use '' (empty string) to assign the name automatically.")
 
-        parser.add_option("", "--runall", action="append", dest="runall",
-                          help="Run the specified task for any recipe in the taskgraph of the specified target (even if it wouldn't otherwise have run).")
+    parser.add_option("", "--runall", action="append", dest="runall",
+                      help="Run the specified task for any recipe in the taskgraph of the specified target (even if it wouldn't otherwise have run).")
 
-        parser.add_option("", "--runonly", action="append", dest="runonly",
-                          help="Run only the specified task within the taskgraph of the specified targets (and any task dependencies those tasks may have).")
+    parser.add_option("", "--runonly", action="append", dest="runonly",
+                      help="Run only the specified task within the taskgraph of the specified targets (and any task dependencies those tasks may have).")
+    return parser
 
 
+class BitBakeConfigParameters(cookerdata.ConfigParameters):
+    def parseCommandLine(self, argv=sys.argv):
+        parser = create_bitbake_parser()
         options, targets = parser.parse_args(argv)
 
         if options.quiet and options.verbose:
@@ -466,7 +469,7 @@ def setup_bitbake(configParams, extrafeatures=None):
                     logger.info("Retrying server connection (#%d)..." % tryno)
                 else:
                     logger.info("Retrying server connection (#%d)... (%s)" % (tryno, traceback.format_exc()))
-                
+
             if not retries:
                 bb.fatal("Unable to connect to bitbake server, or start one (server startup failures would be in bitbake-cookerdaemon.log).")
             bb.event.print_ui_queue()

bitbake/lib/bb/monitordisk.py

@@ -59,7 +59,7 @@ def getMountedDev(path):
         pass
     return None
 
-def getDiskData(BBDirs, configuration):
+def getDiskData(BBDirs):
 
     """Prepare disk data for disk space monitor"""
 
@@ -168,7 +168,7 @@ class diskMonitor:
 
         BBDirs = configuration.getVar("BB_DISKMON_DIRS") or None
         if BBDirs:
-            self.devDict = getDiskData(BBDirs, configuration)
+            self.devDict = getDiskData(BBDirs)
             if self.devDict:
                 self.spaceInterval, self.inodeInterval = getInterval(configuration)
                 if self.spaceInterval and self.inodeInterval:

bitbake/lib/bb/msg.py

@@ -278,7 +278,7 @@ def setLoggingConfig(defaultconfig, userconfigfile=None):
         with open(os.path.normpath(userconfigfile), 'r') as f:
             if userconfigfile.endswith('.yml') or userconfigfile.endswith('.yaml'):
                 import yaml
-                userconfig = yaml.load(f)
+                userconfig = yaml.safe_load(f)
             elif userconfigfile.endswith('.json') or userconfigfile.endswith('.cfg'):
                 import json
                 userconfig = json.load(f)

bitbake/lib/bb/parse/__init__.py

@@ -71,7 +71,7 @@ def update_mtime(f):
 
 def update_cache(f):
     if f in __mtime_cache:
-        logger.debug(1, "Updating mtime cache for %s" % f)
+        logger.debug("Updating mtime cache for %s" % f)
         update_mtime(f)
 
 def clear_cache():

bitbake/lib/bb/parse/ast.py

@@ -34,7 +34,7 @@ class IncludeNode(AstNode):
         Include the file and evaluate the statements
         """
         s = data.expand(self.what_file)
-        logger.debug(2, "CONF %s:%s: including %s", self.filename, self.lineno, s)
+        logger.debug2("CONF %s:%s: including %s", self.filename, self.lineno, s)
 
         # TODO: Cache those includes... maybe not here though
         if self.force:
@@ -97,6 +97,7 @@ class DataNode(AstNode):
     def eval(self, data):
         groupd = self.groupd
         key = groupd["var"]
+        key = key.replace(":", "_")
         loginfo = {
             'variable': key,
             'file': self.filename,
@@ -207,6 +208,7 @@ class ExportFuncsNode(AstNode):
     def eval(self, data):
 
         for func in self.n:
+            func = func.replace(":", "_")
             calledfunc = self.classname + "_" + func
 
             if data.getVar(func, False) and not data.getVarFlag(func, 'export_func', False):
@@ -335,7 +337,7 @@ def finalize(fn, d, variant = None):
             if not handlerfn:
                 bb.fatal("Undefined event handler function '%s'" % var)
             handlerln = int(d.getVarFlag(var, "lineno", False))
-            bb.event.register(var, d.getVar(var, False), (d.getVarFlag(var, "eventmask") or "").split(), handlerfn, handlerln)
+            bb.event.register(var, d.getVar(var, False), (d.getVarFlag(var, "eventmask") or "").split(), handlerfn, handlerln, data=d)
 
         bb.event.fire(bb.event.RecipePreFinalise(fn), d)
 
@@ -376,7 +378,7 @@ def _create_variants(datastores, names, function, onlyfinalise):
 def multi_finalize(fn, d):
     appends = (d.getVar("__BBAPPEND") or "").split()
     for append in appends:
-        logger.debug(1, "Appending .bbappend file %s to %s", append, fn)
+        logger.debug("Appending .bbappend file %s to %s", append, fn)
         bb.parse.BBHandler.handle(append, d, True)
 
     onlyfinalise = d.getVar("__ONLYFINALISE", False)

bitbake/lib/bb/parse/parse_py/BBHandler.py

@@ -13,7 +13,7 @@
 #
 
 import re, bb, os
-import bb.build, bb.utils
+import bb.build, bb.utils, bb.data_smart
 
 from . import ConfHandler
 from .. import resolve_file, ast, logger, ParseError
@@ -22,7 +22,7 @@ from .ConfHandler import include, init
 # For compatibility
 bb.deprecate_import(__name__, "bb.parse", ["vars_from_file"])
 
-__func_start_regexp__    = re.compile(r"(((?P<py>python)|(?P<fr>fakeroot))\s*)*(?P<func>[\w\.\-\+\{\}\$]+)?\s*\(\s*\)\s*{$" )
+__func_start_regexp__    = re.compile(r"(((?P<py>python(?=(\s|\()))|(?P<fr>fakeroot(?=\s)))\s*)*(?P<func>[\w\.\-\+\{\}\$:]+)?\s*\(\s*\)\s*{$" )
 __inherit_regexp__       = re.compile(r"inherit\s+(.+)" )
 __export_func_regexp__   = re.compile(r"EXPORT_FUNCTIONS\s+(.+)" )
 __addtask_regexp__       = re.compile(r"addtask\s+(?P<func>\w+)\s*((before\s*(?P<before>((.*(?=after))|(.*))))|(after\s*(?P<after>((.*(?=before))|(.*)))))*")
@@ -60,7 +60,7 @@ def inherit(files, fn, lineno, d):
                 file = abs_fn
 
         if not file in __inherit_cache:
-            logger.debug(1, "Inheriting %s (from %s:%d)" % (file, fn, lineno))
+            logger.debug("Inheriting %s (from %s:%d)" % (file, fn, lineno))
             __inherit_cache.append( file )
             d.setVar('__inherit_cache', __inherit_cache)
             include(fn, file, lineno, d, "inherit")
@@ -233,6 +233,10 @@ def feeder(lineno, s, fn, root, statements, eof=False):
             if taskexpression.count(word) > 1:
                 logger.warning("addtask contained multiple '%s' keywords, only one is supported" % word)
 
+        # Check and warn for having task with exprssion as part of task name
+        for te in taskexpression:
+            if any( ( "%s_" % keyword ) in te for keyword in bb.data_smart.__setvar_keyword__ ):
+                raise ParseError("Task name '%s' contains a keyword which is not recommended/supported.\nPlease rename the task not to include the keyword.\n%s" % (te, ("\n".join(map(str, bb.data_smart.__setvar_keyword__)))), fn)
         ast.handleAddTask(statements, fn, lineno, m)
         return
 

bitbake/lib/bb/parse/parse_py/ConfHandler.py

@@ -20,7 +20,7 @@ from bb.parse import ParseError, resolve_file, ast, logger, handle
 __config_regexp__  = re.compile( r"""
     ^
     (?P<exp>export\s+)?
-    (?P<var>[a-zA-Z0-9\-_+.${}/~]+?)
+    (?P<var>[a-zA-Z0-9\-_+.${}/~:]+?)
     (\[(?P<flag>[a-zA-Z0-9\-_+.]+)\])?
 
     \s* (
@@ -95,7 +95,7 @@ def include_single_file(parentfn, fn, lineno, data, error_out):
         if exc.errno == errno.ENOENT:
             if error_out:
                 raise ParseError("Could not %s file %s" % (error_out, fn), parentfn, lineno)
-            logger.debug(2, "CONF file '%s' not found", fn)
+            logger.debug2("CONF file '%s' not found", fn)
         else:
             if error_out:
                 raise ParseError("Could not %s file %s: %s" % (error_out, fn, exc.strerror), parentfn, lineno)

bitbake/lib/bb/persist_data.py

@@ -12,6 +12,7 @@ currently, providing a key/value store accessed by 'domain'.
 #
 
 import collections
+import collections.abc
 import contextlib
 import functools
 import logging
@@ -19,7 +20,7 @@ import os.path
 import sqlite3
 import sys
 import warnings
-from collections import Mapping
+from collections.abc import Mapping
 
 sqlversion = sqlite3.sqlite_version_info
 if sqlversion[0] < 3 or (sqlversion[0] == 3 and sqlversion[1] < 3):
@@ -29,7 +30,7 @@ if sqlversion[0] < 3 or (sqlversion[0] == 3 and sqlversion[1] < 3):
 logger = logging.getLogger("BitBake.PersistData")
 
 @functools.total_ordering
-class SQLTable(collections.MutableMapping):
+class SQLTable(collections.abc.MutableMapping):
     class _Decorators(object):
         @staticmethod
         def retry(*, reconnect=True):
@@ -248,7 +249,7 @@ class PersistData(object):
                       stacklevel=2)
 
         self.data = persist(d)
-        logger.debug(1, "Using '%s' as the persistent data cache",
+        logger.debug("Using '%s' as the persistent data cache",
                      self.data.filename)
 
     def addDomain(self, domain):

bitbake/lib/bb/process.py

@@ -60,7 +60,7 @@ class Popen(subprocess.Popen):
         "close_fds": True,
         "preexec_fn": subprocess_setup,
         "stdout": subprocess.PIPE,
-        "stderr": subprocess.STDOUT,
+        "stderr": subprocess.PIPE,
         "stdin": subprocess.PIPE,
         "shell": False,
     }
@@ -181,5 +181,8 @@ def run(cmd, input=None, log=None, extrafiles=None, **options):
             stderr = stderr.decode("utf-8")
 
     if pipe.returncode != 0:
+        if log:
+            # Don't duplicate the output in the exception if logging it
+            raise ExecutionError(cmd, pipe.returncode, None, None)
         raise ExecutionError(cmd, pipe.returncode, stdout, stderr)
     return stdout, stderr

bitbake/lib/bb/providers.py

@@ -38,16 +38,17 @@ def findProviders(cfgData, dataCache, pkg_pn = None):
     localdata = data.createCopy(cfgData)
     bb.data.expandKeys(localdata)
 
+    required = {}
     preferred_versions = {}
     latest_versions = {}
 
     for pn in pkg_pn:
-        (last_ver, last_file, pref_ver, pref_file) = findBestProvider(pn, localdata, dataCache, pkg_pn)
+        (last_ver, last_file, pref_ver, pref_file, req) = findBestProvider(pn, localdata, dataCache, pkg_pn)
         preferred_versions[pn] = (pref_ver, pref_file)
         latest_versions[pn] = (last_ver, last_file)
+        required[pn] = req
 
-    return (latest_versions, preferred_versions)
-
+    return (latest_versions, preferred_versions, required)
 
 def allProviders(dataCache):
     """
@@ -59,7 +60,6 @@ def allProviders(dataCache):
         all_providers[pn].append((ver, fn))
     return all_providers
 
-
 def sortPriorities(pn, dataCache, pkg_pn = None):
     """
     Reorder pkg_pn by file priority and default preference
@@ -87,6 +87,21 @@ def sortPriorities(pn, dataCache, pkg_pn = None):
 
     return tmp_pn
 
+def versionVariableMatch(cfgData, keyword, pn):
+    """
+    Return the value of the <keyword>_VERSION variable if set.
+    """
+
+    # pn can contain '_', e.g. gcc-cross-x86_64 and an override cannot
+    # hence we do this manually rather than use OVERRIDES
+    ver = cfgData.getVar("%s_VERSION_pn-%s" % (keyword, pn))
+    if not ver:
+        ver = cfgData.getVar("%s_VERSION_%s" % (keyword, pn))
+    if not ver:
+        ver = cfgData.getVar("%s_VERSION" % keyword)
+
+    return ver
+
 def preferredVersionMatch(pe, pv, pr, preferred_e, preferred_v, preferred_r):
     """
     Check if the version pe,pv,pr is the preferred one.
@@ -102,19 +117,28 @@ def preferredVersionMatch(pe, pv, pr, preferred_e, preferred_v, preferred_r):
 
 def findPreferredProvider(pn, cfgData, dataCache, pkg_pn = None, item = None):
     """
-    Find the first provider in pkg_pn with a PREFERRED_VERSION set.
+    Find the first provider in pkg_pn with REQUIRED_VERSION or PREFERRED_VERSION set.
     """
 
     preferred_file = None
     preferred_ver = None
+    required = False
 
-    # pn can contain '_', e.g. gcc-cross-x86_64 and an override cannot
-    # hence we do this manually rather than use OVERRIDES
-    preferred_v = cfgData.getVar("PREFERRED_VERSION_pn-%s" % pn)
-    if not preferred_v:
-        preferred_v = cfgData.getVar("PREFERRED_VERSION_%s" % pn)
-    if not preferred_v:
-        preferred_v = cfgData.getVar("PREFERRED_VERSION")
+    required_v = versionVariableMatch(cfgData, "REQUIRED", pn)
+    preferred_v = versionVariableMatch(cfgData, "PREFERRED", pn)
+
+    itemstr = ""
+    if item:
+        itemstr = " (for item %s)" % item
+
+    if required_v is not None:
+        if preferred_v is not None:
+            logger.warn("REQUIRED_VERSION and PREFERRED_VERSION for package %s%s are both set using REQUIRED_VERSION %s", pn, itemstr, required_v)
+        else:
+            logger.debug("REQUIRED_VERSION is set for package %s%s", pn, itemstr)
+        # REQUIRED_VERSION always takes precedence over PREFERRED_VERSION
+        preferred_v = required_v
+        required = True
 
     if preferred_v:
         m = re.match(r'(\d+:)*(.*)(_.*)*', preferred_v)
@@ -147,11 +171,9 @@ def findPreferredProvider(pn, cfgData, dataCache, pkg_pn = None, item = None):
             pv_str = preferred_v
         if not (preferred_e is None):
             pv_str = '%s:%s' % (preferred_e, pv_str)
-        itemstr = ""
-        if item:
-            itemstr = " (for item %s)" % item
         if preferred_file is None:
-            logger.info("preferred version %s of %s not available%s", pv_str, pn, itemstr)
+            if not required:
+                logger.warn("preferred version %s of %s not available%s", pv_str, pn, itemstr)
             available_vers = []
             for file_set in pkg_pn:
                 for f in file_set:
@@ -163,12 +185,16 @@ def findPreferredProvider(pn, cfgData, dataCache, pkg_pn = None, item = None):
                         available_vers.append(ver_str)
             if available_vers:
                 available_vers.sort()
-                logger.info("versions of %s available: %s", pn, ' '.join(available_vers))
+                logger.warn("versions of %s available: %s", pn, ' '.join(available_vers))
+            if required:
+                logger.error("required version %s of %s not available%s", pv_str, pn, itemstr)
         else:
-            logger.debug(1, "selecting %s as PREFERRED_VERSION %s of package %s%s", preferred_file, pv_str, pn, itemstr)
-
-    return (preferred_ver, preferred_file)
+            if required:
+                logger.debug("selecting %s as REQUIRED_VERSION %s of package %s%s", preferred_file, pv_str, pn, itemstr)
+            else:
+                logger.debug("selecting %s as PREFERRED_VERSION %s of package %s%s", preferred_file, pv_str, pn, itemstr)
 
+    return (preferred_ver, preferred_file, required)
 
 def findLatestProvider(pn, cfgData, dataCache, file_set):
     """
@@ -189,7 +215,6 @@ def findLatestProvider(pn, cfgData, dataCache, file_set):
 
     return (latest, latest_f)
 
-
 def findBestProvider(pn, cfgData, dataCache, pkg_pn = None, item = None):
     """
     If there is a PREFERRED_VERSION, find the highest-priority bbfile
@@ -198,17 +223,16 @@ def findBestProvider(pn, cfgData, dataCache, pkg_pn = None, item = None):
     """
 
     sortpkg_pn = sortPriorities(pn, dataCache, pkg_pn)
-    # Find the highest priority provider with a PREFERRED_VERSION set
-    (preferred_ver, preferred_file) = findPreferredProvider(pn, cfgData, dataCache, sortpkg_pn, item)
+    # Find the highest priority provider with a REQUIRED_VERSION or PREFERRED_VERSION set
+    (preferred_ver, preferred_file, required) = findPreferredProvider(pn, cfgData, dataCache, sortpkg_pn, item)
     # Find the latest version of the highest priority provider
     (latest, latest_f) = findLatestProvider(pn, cfgData, dataCache, sortpkg_pn[0])
 
-    if preferred_file is None:
+    if not required and preferred_file is None:
         preferred_file = latest_f
         preferred_ver = latest
 
-    return (latest, latest_f, preferred_ver, preferred_file)
-
+    return (latest, latest_f, preferred_ver, preferred_file, required)
 
 def _filterProviders(providers, item, cfgData, dataCache):
     """
@@ -232,12 +256,15 @@ def _filterProviders(providers, item, cfgData, dataCache):
             pkg_pn[pn] = []
         pkg_pn[pn].append(p)
 
-    logger.debug(1, "providers for %s are: %s", item, list(sorted(pkg_pn.keys())))
+    logger.debug("providers for %s are: %s", item, list(sorted(pkg_pn.keys())))
 
-    # First add PREFERRED_VERSIONS
+    # First add REQUIRED_VERSIONS or PREFERRED_VERSIONS
     for pn in sorted(pkg_pn):
         sortpkg_pn[pn] = sortPriorities(pn, dataCache, pkg_pn)
-        preferred_versions[pn] = findPreferredProvider(pn, cfgData, dataCache, sortpkg_pn[pn], item)
+        preferred_ver, preferred_file, required = findPreferredProvider(pn, cfgData, dataCache, sortpkg_pn[pn], item)
+        if required and preferred_file is None:
+            return eligible
+        preferred_versions[pn] = (preferred_ver, preferred_file)
         if preferred_versions[pn][1]:
             eligible.append(preferred_versions[pn][1])
 
@@ -248,9 +275,8 @@ def _filterProviders(providers, item, cfgData, dataCache):
         preferred_versions[pn] = findLatestProvider(pn, cfgData, dataCache, sortpkg_pn[pn][0])
         eligible.append(preferred_versions[pn][1])
 
-    if len(eligible) == 0:
-        logger.error("no eligible providers for %s", item)
-        return 0
+    if not eligible:
+        return eligible
 
     # If pn == item, give it a slight default preference
     # This means PREFERRED_PROVIDER_foobar defaults to foobar if available
@@ -266,7 +292,6 @@ def _filterProviders(providers, item, cfgData, dataCache):
 
     return eligible
 
-
 def filterProviders(providers, item, cfgData, dataCache):
     """
     Take a list of providers and filter/reorder according to the
@@ -291,7 +316,7 @@ def filterProviders(providers, item, cfgData, dataCache):
                 foundUnique = True
                 break
 
-    logger.debug(1, "sorted providers for %s are: %s", item, eligible)
+    logger.debug("sorted providers for %s are: %s", item, eligible)
 
     return eligible, foundUnique
 
@@ -333,7 +358,7 @@ def filterProvidersRunTime(providers, item, cfgData, dataCache):
             provides = dataCache.pn_provides[pn]
             for provide in provides:
                 prefervar = cfgData.getVar('PREFERRED_PROVIDER_%s' % provide)
-                #logger.debug(1, "checking PREFERRED_PROVIDER_%s (value %s) against %s", provide, prefervar, pns.keys())
+                #logger.debug("checking PREFERRED_PROVIDER_%s (value %s) against %s", provide, prefervar, pns.keys())
                 if prefervar in pns and pns[prefervar] not in preferred:
                     var = "PREFERRED_PROVIDER_%s = %s" % (provide, prefervar)
                     logger.verbose("selecting %s to satisfy runtime %s due to %s", prefervar, item, var)
@@ -349,7 +374,7 @@ def filterProvidersRunTime(providers, item, cfgData, dataCache):
     if numberPreferred > 1:
         logger.error("Trying to resolve runtime dependency %s resulted in conflicting PREFERRED_PROVIDER entries being found.\nThe providers found were: %s\nThe PREFERRED_PROVIDER entries resulting in this conflict were: %s. You could set PREFERRED_RPROVIDER_%s" % (item, preferred, preferred_vars, item))
 
-    logger.debug(1, "sorted runtime providers for %s are: %s", item, eligible)
+    logger.debug("sorted runtime providers for %s are: %s", item, eligible)
 
     return eligible, numberPreferred
 
@@ -384,11 +409,10 @@ def getRuntimeProviders(dataCache, rdepend):
             regexp_cache[pattern] = regexp
         if regexp.match(rdepend):
             rproviders += dataCache.packages_dynamic[pattern]
-            logger.debug(1, "Assuming %s is a dynamic package, but it may not exist" % rdepend)
+            logger.debug("Assuming %s is a dynamic package, but it may not exist" % rdepend)
 
     return rproviders
 
-
 def buildWorldTargetList(dataCache, task=None):
     """
     Build package list for "bitbake world"
@@ -396,22 +420,22 @@ def buildWorldTargetList(dataCache, task=None):
     if dataCache.world_target:
         return
 
-    logger.debug(1, "collating packages for \"world\"")
+    logger.debug("collating packages for \"world\"")
     for f in dataCache.possible_world:
         terminal = True
         pn = dataCache.pkg_fn[f]
         if task and task not in dataCache.task_deps[f]['tasks']:
-            logger.debug(2, "World build skipping %s as task %s doesn't exist", f, task)
+            logger.debug2("World build skipping %s as task %s doesn't exist", f, task)
             terminal = False
 
         for p in dataCache.pn_provides[pn]:
             if p.startswith('virtual/'):
-                logger.debug(2, "World build skipping %s due to %s provider starting with virtual/", f, p)
+                logger.debug2("World build skipping %s due to %s provider starting with virtual/", f, p)
                 terminal = False
                 break
             for pf in dataCache.providers[p]:
                 if dataCache.pkg_fn[pf] != pn:
-                    logger.debug(2, "World build skipping %s due to both us and %s providing %s", f, pf, p)
+                    logger.debug2("World build skipping %s due to both us and %s providing %s", f, pf, p)
                     terminal = False
                     break
         if terminal:

bitbake/lib/bb/runqueue.py

@@ -38,7 +38,7 @@ def taskname_from_tid(tid):
     return tid.rsplit(":", 1)[1]
 
 def mc_from_tid(tid):
-    if tid.startswith('mc:'):
+    if tid.startswith('mc:') and tid.count(':') >= 2:
         return tid.split(':')[1]
     return ""
 
@@ -47,13 +47,13 @@ def split_tid(tid):
     return (mc, fn, taskname)
 
 def split_mc(n):
-    if n.startswith("mc:"):
+    if n.startswith("mc:") and n.count(':') >= 2:
         _, mc, n = n.split(":", 2)
         return (mc, n)
     return ('', n)
 
 def split_tid_mcfn(tid):
-    if tid.startswith('mc:'):
+    if tid.startswith('mc:') and tid.count(':') >= 2:
         elems = tid.split(':')
         mc = elems[1]
         fn = ":".join(elems[2:-1])
@@ -85,15 +85,19 @@ class RunQueueStats:
     """
     Holds statistics on the tasks handled by the associated runQueue
     """
-    def __init__(self, total):
+    def __init__(self, total, setscene_total):
         self.completed = 0
         self.skipped = 0
         self.failed = 0
         self.active = 0
+        self.setscene_active = 0
+        self.setscene_covered = 0
+        self.setscene_notcovered = 0
+        self.setscene_total = setscene_total
         self.total = total
 
     def copy(self):
-        obj = self.__class__(self.total)
+        obj = self.__class__(self.total, self.setscene_total)
         obj.__dict__.update(self.__dict__)
         return obj
 
@@ -112,6 +116,13 @@ class RunQueueStats:
     def taskActive(self):
         self.active = self.active + 1
 
+    def updateCovered(self, covered, notcovered):
+        self.setscene_covered = covered
+        self.setscene_notcovered = notcovered
+
+    def updateActiveSetscene(self, active):
+        self.setscene_active = active
+
 # These values indicate the next step due to be run in the
 # runQueue state machine
 runQueuePrepare = 2
@@ -544,8 +555,8 @@ class RunQueueData:
         for tid in self.runtaskentries:
             if task_done[tid] is False or deps_left[tid] != 0:
                 problem_tasks.append(tid)
-                logger.debug(2, "Task %s is not buildable", tid)
-                logger.debug(2, "(Complete marker was %s and the remaining dependency count was %s)\n", task_done[tid], deps_left[tid])
+                logger.debug2("Task %s is not buildable", tid)
+                logger.debug2("(Complete marker was %s and the remaining dependency count was %s)\n", task_done[tid], deps_left[tid])
             self.runtaskentries[tid].weight = weight[tid]
 
         if problem_tasks:
@@ -643,7 +654,7 @@ class RunQueueData:
                 (mc, fn, taskname, taskfn) = split_tid_mcfn(tid)
                 #runtid = build_tid(mc, fn, taskname)
 
-                #logger.debug(2, "Processing %s,%s:%s", mc, fn, taskname)
+                #logger.debug2("Processing %s,%s:%s", mc, fn, taskname)
 
                 depends = set()
                 task_deps = self.dataCaches[mc].task_deps[taskfn]
@@ -915,38 +926,36 @@ class RunQueueData:
         #
         # Once all active tasks are marked, prune the ones we don't need.
 
-        delcount = {}
-        for tid in list(self.runtaskentries.keys()):
-            if tid not in runq_build:
-                delcount[tid] = self.runtaskentries[tid]
-                del self.runtaskentries[tid]
-
         # Handle --runall
         if self.cooker.configuration.runall:
             # re-run the mark_active and then drop unused tasks from new list
+            reduced_tasklist = set(self.runtaskentries.keys())
+            for tid in list(self.runtaskentries.keys()):
+                if tid not in runq_build:
+                   reduced_tasklist.remove(tid)
             runq_build = {}
 
             for task in self.cooker.configuration.runall:
                 if not task.startswith("do_"):
                     task = "do_{0}".format(task)
                 runall_tids = set()
-                for tid in list(self.runtaskentries):
+                for tid in reduced_tasklist:
                     wanttid = "{0}:{1}".format(fn_from_tid(tid), task)
-                    if wanttid in delcount:
-                        self.runtaskentries[wanttid] = delcount[wanttid]
                     if wanttid in self.runtaskentries:
                         runall_tids.add(wanttid)
 
                 for tid in list(runall_tids):
-                    mark_active(tid,1)
+                    mark_active(tid, 1)
                     if self.cooker.configuration.force:
                         invalidate_task(tid, False)
 
-            for tid in list(self.runtaskentries.keys()):
-                if tid not in runq_build:
-                    delcount[tid] = self.runtaskentries[tid]
-                    del self.runtaskentries[tid]
+        delcount = set()
+        for tid in list(self.runtaskentries.keys()):
+            if tid not in runq_build:
+                delcount.add(tid)
+                del self.runtaskentries[tid]
 
+        if self.cooker.configuration.runall:
             if len(self.runtaskentries) == 0:
                 bb.msg.fatal("RunQueue", "Could not find any tasks with the tasknames %s to run within the recipes of the taskgraphs of the targets %s" % (str(self.cooker.configuration.runall), str(self.targets)))
 
@@ -960,16 +969,16 @@ class RunQueueData:
             for task in self.cooker.configuration.runonly:
                 if not task.startswith("do_"):
                     task = "do_{0}".format(task)
-                runonly_tids = { k: v for k, v in self.runtaskentries.items() if taskname_from_tid(k) == task }
+                runonly_tids = [k for k in self.runtaskentries.keys() if taskname_from_tid(k) == task]
 
-                for tid in list(runonly_tids):
-                    mark_active(tid,1)
+                for tid in runonly_tids:
+                    mark_active(tid, 1)
                     if self.cooker.configuration.force:
                         invalidate_task(tid, False)
 
             for tid in list(self.runtaskentries.keys()):
                 if tid not in runq_build:
-                    delcount[tid] = self.runtaskentries[tid]
+                    delcount.add(tid)
                     del self.runtaskentries[tid]
 
             if len(self.runtaskentries) == 0:
@@ -1199,9 +1208,9 @@ class RunQueueData:
         """
         Dump some debug information on the internal data structures
         """
-        logger.debug(3, "run_tasks:")
+        logger.debug3("run_tasks:")
         for tid in self.runtaskentries:
-            logger.debug(3, " %s: %s   Deps %s RevDeps %s", tid,
+            logger.debug3(" %s: %s   Deps %s RevDeps %s", tid,
                          self.runtaskentries[tid].weight,
                          self.runtaskentries[tid].depends,
                          self.runtaskentries[tid].revdeps)
@@ -1238,10 +1247,11 @@ class RunQueue:
         self.fakeworker = {}
 
     def _start_worker(self, mc, fakeroot = False, rqexec = None):
-        logger.debug(1, "Starting bitbake-worker")
+        logger.debug("Starting bitbake-worker")
         magic = "decafbad"
         if self.cooker.configuration.profile:
             magic = "decafbadbad"
+        fakerootlogs = None
         if fakeroot:
             magic = magic + "beef"
             mcdata = self.cooker.databuilder.mcdata[mc]
@@ -1251,10 +1261,11 @@ class RunQueue:
             for key, value in (var.split('=') for var in fakerootenv):
                 env[key] = value
             worker = subprocess.Popen(fakerootcmd + ["bitbake-worker", magic], stdout=subprocess.PIPE, stdin=subprocess.PIPE, env=env)
+            fakerootlogs = self.rqdata.dataCaches[mc].fakerootlogs
         else:
             worker = subprocess.Popen(["bitbake-worker", magic], stdout=subprocess.PIPE, stdin=subprocess.PIPE)
         bb.utils.nonblockingfd(worker.stdout)
-        workerpipe = runQueuePipe(worker.stdout, None, self.cfgData, self, rqexec)
+        workerpipe = runQueuePipe(worker.stdout, None, self.cfgData, self, rqexec, fakerootlogs=fakerootlogs)
 
         workerdata = {
             "taskdeps" : self.rqdata.dataCaches[mc].task_deps,
@@ -1271,6 +1282,7 @@ class RunQueue:
             "date" : self.cfgData.getVar("DATE"),
             "time" : self.cfgData.getVar("TIME"),
             "hashservaddr" : self.cooker.hashservaddr,
+            "umask" : self.cfgData.getVar("BB_DEFAULT_UMASK"),
         }
 
         worker.stdin.write(b"<cookerconfig>" + pickle.dumps(self.cooker.configuration) + b"</cookerconfig>")
@@ -1283,7 +1295,7 @@ class RunQueue:
     def _teardown_worker(self, worker):
         if not worker:
             return
-        logger.debug(1, "Teardown for bitbake-worker")
+        logger.debug("Teardown for bitbake-worker")
         try:
            worker.process.stdin.write(b"<quit></quit>")
            worker.process.stdin.flush()
@@ -1356,12 +1368,12 @@ class RunQueue:
 
         # If the stamp is missing, it's not current
         if not os.access(stampfile, os.F_OK):
-            logger.debug(2, "Stampfile %s not available", stampfile)
+            logger.debug2("Stampfile %s not available", stampfile)
             return False
         # If it's a 'nostamp' task, it's not current
         taskdep = self.rqdata.dataCaches[mc].task_deps[taskfn]
         if 'nostamp' in taskdep and taskname in taskdep['nostamp']:
-            logger.debug(2, "%s.%s is nostamp\n", fn, taskname)
+            logger.debug2("%s.%s is nostamp\n", fn, taskname)
             return False
 
         if taskname != "do_setscene" and taskname.endswith("_setscene"):
@@ -1385,18 +1397,18 @@ class RunQueue:
                     continue
                 if fn == fn2 or (fulldeptree and fn2 not in stampwhitelist):
                     if not t2:
-                        logger.debug(2, 'Stampfile %s does not exist', stampfile2)
+                        logger.debug2('Stampfile %s does not exist', stampfile2)
                         iscurrent = False
                         break
                     if t1 < t2:
-                        logger.debug(2, 'Stampfile %s < %s', stampfile, stampfile2)
+                        logger.debug2('Stampfile %s < %s', stampfile, stampfile2)
                         iscurrent = False
                         break
                     if recurse and iscurrent:
                         if dep in cache:
                             iscurrent = cache[dep]
                             if not iscurrent:
-                                logger.debug(2, 'Stampfile for dependency %s:%s invalid (cached)' % (fn2, taskname2))
+                                logger.debug2('Stampfile for dependency %s:%s invalid (cached)' % (fn2, taskname2))
                         else:
                             iscurrent = self.check_stamp_task(dep, recurse=True, cache=cache)
                             cache[dep] = iscurrent
@@ -1466,7 +1478,7 @@ class RunQueue:
             if not self.dm_event_handler_registered:
                  res = bb.event.register(self.dm_event_handler_name,
                                          lambda x: self.dm.check(self) if self.state in [runQueueRunning, runQueueCleanUp] else False,
-                                         ('bb.event.HeartbeatEvent',))
+                                         ('bb.event.HeartbeatEvent',), data=self.cfgData)
                  self.dm_event_handler_registered = True
 
             dump = self.cooker.configuration.dump_signatures
@@ -1505,7 +1517,7 @@ class RunQueue:
         build_done = self.state is runQueueComplete or self.state is runQueueFailed
 
         if build_done and self.dm_event_handler_registered:
-            bb.event.remove(self.dm_event_handler_name, None)
+            bb.event.remove(self.dm_event_handler_name, None, data=self.cfgData)
             self.dm_event_handler_registered = False
 
         if build_done and self.rqexe:
@@ -1732,8 +1744,7 @@ class RunQueueExecute:
         self.holdoff_need_update = True
         self.sqdone = False
 
-        self.stats = RunQueueStats(len(self.rqdata.runtaskentries))
-        self.sq_stats = RunQueueStats(len(self.rqdata.runq_setscene_tids))
+        self.stats = RunQueueStats(len(self.rqdata.runtaskentries), len(self.rqdata.runq_setscene_tids))
 
         for mc in rq.worker:
             rq.worker[mc].pipe.setrunqueueexec(self)
@@ -1761,7 +1772,7 @@ class RunQueueExecute:
         for scheduler in schedulers:
             if self.scheduler == scheduler.name:
                 self.sched = scheduler(self, self.rqdata)
-                logger.debug(1, "Using runqueue scheduler '%s'", scheduler.name)
+                logger.debug("Using runqueue scheduler '%s'", scheduler.name)
                 break
         else:
             bb.fatal("Invalid scheduler '%s'.  Available schedulers: %s" %
@@ -1771,7 +1782,7 @@ class RunQueueExecute:
         self.sqdata = SQData()
         build_scenequeue_data(self.sqdata, self.rqdata, self.rq, self.cooker, self.stampcache, self)
 
-    def runqueue_process_waitpid(self, task, status):
+    def runqueue_process_waitpid(self, task, status, fakerootlog=None):
 
         # self.build_stamps[pid] may not exist when use shared work directory.
         if task in self.build_stamps:
@@ -1784,9 +1795,10 @@ class RunQueueExecute:
             else:
                 self.sq_task_complete(task)
             self.sq_live.remove(task)
+            self.stats.updateActiveSetscene(len(self.sq_live))
         else:
             if status != 0:
-                self.task_fail(task, status)
+                self.task_fail(task, status, fakerootlog=fakerootlog)
             else:
                 self.task_complete(task)
         return True
@@ -1817,7 +1829,7 @@ class RunQueueExecute:
     def finish(self):
         self.rq.state = runQueueCleanUp
 
-        active = self.stats.active + self.sq_stats.active
+        active = self.stats.active + len(self.sq_live)
         if active > 0:
             bb.event.fire(runQueueExitWait(active), self.cfgData)
             self.rq.read_workers()
@@ -1850,7 +1862,7 @@ class RunQueueExecute:
         return valid
 
     def can_start_task(self):
-        active = self.stats.active + self.sq_stats.active
+        active = self.stats.active + len(self.sq_live)
         can_start = active < self.number_tasks
         return can_start
 
@@ -1899,7 +1911,13 @@ class RunQueueExecute:
                     break
             if alldeps:
                 self.setbuildable(revdep)
-                logger.debug(1, "Marking task %s as buildable", revdep)
+                logger.debug("Marking task %s as buildable", revdep)
+
+        for t in self.sq_deferred.copy():
+            if self.sq_deferred[t] == task:
+                logger.debug2("Deferred task %s now buildable" % t)
+                del self.sq_deferred[t]
+                update_scenequeue_data([t], self.sqdata, self.rqdata, self.rq, self.cooker, self.stampcache, self, summary=False)
 
     def task_complete(self, task):
         self.stats.taskCompleted()
@@ -1907,14 +1925,31 @@ class RunQueueExecute:
         self.task_completeoutright(task)
         self.runq_tasksrun.add(task)
 
-    def task_fail(self, task, exitcode):
+    def task_fail(self, task, exitcode, fakerootlog=None):
         """
         Called when a task has failed
         Updates the state engine with the failure
         """
         self.stats.taskFailed()
         self.failed_tids.append(task)
-        bb.event.fire(runQueueTaskFailed(task, self.stats, exitcode, self.rq), self.cfgData)
+
+        fakeroot_log = ""
+        if fakerootlog and os.path.exists(fakerootlog):
+            with open(fakerootlog) as fakeroot_log_file:
+                fakeroot_failed = False
+                for line in reversed(fakeroot_log_file.readlines()):
+                    for fakeroot_error in ['mismatch', 'error', 'fatal']:
+                        if fakeroot_error in line.lower():
+                            fakeroot_failed = True
+                    if 'doing new pid setup and server start' in line:
+                        break
+                    fakeroot_log = line + fakeroot_log
+
+            if not fakeroot_failed:
+                fakeroot_log = None
+
+        bb.event.fire(runQueueTaskFailed(task, self.stats, exitcode, self.rq, fakeroot_log=fakeroot_log), self.cfgData)
+
         if self.rqdata.taskData[''].abort:
             self.rq.state = runQueueCleanUp
 
@@ -1929,8 +1964,8 @@ class RunQueueExecute:
     def summarise_scenequeue_errors(self):
         err = False
         if not self.sqdone:
-            logger.debug(1, 'We could skip tasks %s', "\n".join(sorted(self.scenequeue_covered)))
-            completeevent = sceneQueueComplete(self.sq_stats, self.rq)
+            logger.debug('We could skip tasks %s', "\n".join(sorted(self.scenequeue_covered)))
+            completeevent = sceneQueueComplete(self.stats, self.rq)
             bb.event.fire(completeevent, self.cfgData)
         if self.sq_deferred:
             logger.error("Scenequeue had deferred entries: %s" % pprint.pformat(self.sq_deferred))
@@ -1942,6 +1977,10 @@ class RunQueueExecute:
             logger.error("Scenequeue had holdoff tasks: %s" % pprint.pformat(self.holdoff_tasks))
             err = True
 
+        for tid in self.scenequeue_covered.intersection(self.scenequeue_notcovered):
+            # No task should end up in both covered and uncovered, that is a bug.
+            logger.error("Setscene task %s in both covered and notcovered." % tid)
+
         for tid in self.rqdata.runq_setscene_tids:
             if tid not in self.scenequeue_covered and tid not in self.scenequeue_notcovered:
                 err = True
@@ -1986,7 +2025,7 @@ class RunQueueExecute:
                 if nexttask in self.sq_buildable and nexttask not in self.sq_running and self.sqdata.stamps[nexttask] not in self.build_stamps.values():
                     if nexttask not in self.sqdata.unskippable and len(self.sqdata.sq_revdeps[nexttask]) > 0 and self.sqdata.sq_revdeps[nexttask].issubset(self.scenequeue_covered) and self.check_dependencies(nexttask, self.sqdata.sq_revdeps[nexttask]):
                         if nexttask not in self.rqdata.target_tids:
-                            logger.debug(2, "Skipping setscene for task %s" % nexttask)
+                            logger.debug2("Skipping setscene for task %s" % nexttask)
                             self.sq_task_skip(nexttask)
                             self.scenequeue_notneeded.add(nexttask)
                             if nexttask in self.sq_deferred:
@@ -1999,28 +2038,26 @@ class RunQueueExecute:
                     if nexttask in self.sq_deferred:
                         if self.sq_deferred[nexttask] not in self.runq_complete:
                             continue
-                        logger.debug(1, "Task %s no longer deferred" % nexttask)
+                        logger.debug("Task %s no longer deferred" % nexttask)
                         del self.sq_deferred[nexttask]
                         valid = self.rq.validate_hashes(set([nexttask]), self.cooker.data, 0, False, summary=False)
                         if not valid:
-                            logger.debug(1, "%s didn't become valid, skipping setscene" % nexttask)
+                            logger.debug("%s didn't become valid, skipping setscene" % nexttask)
                             self.sq_task_failoutright(nexttask)
                             return True
-                        else:
-                            self.sqdata.outrightfail.remove(nexttask)
                     if nexttask in self.sqdata.outrightfail:
-                        logger.debug(2, 'No package found, so skipping setscene task %s', nexttask)
+                        logger.debug2('No package found, so skipping setscene task %s', nexttask)
                         self.sq_task_failoutright(nexttask)
                         return True
                     if nexttask in self.sqdata.unskippable:
-                        logger.debug(2, "Setscene task %s is unskippable" % nexttask)
+                        logger.debug2("Setscene task %s is unskippable" % nexttask)
                     task = nexttask
                     break
         if task is not None:
             (mc, fn, taskname, taskfn) = split_tid_mcfn(task)
             taskname = taskname + "_setscene"
             if self.rq.check_stamp_task(task, taskname_from_tid(task), recurse = True, cache=self.stampcache):
-                logger.debug(2, 'Stamp for underlying task %s is current, so skipping setscene variant', task)
+                logger.debug2('Stamp for underlying task %s is current, so skipping setscene variant', task)
                 self.sq_task_failoutright(task)
                 return True
 
@@ -2030,16 +2067,16 @@ class RunQueueExecute:
                     return True
 
             if self.rq.check_stamp_task(task, taskname, cache=self.stampcache):
-                logger.debug(2, 'Setscene stamp current task %s, so skip it and its dependencies', task)
+                logger.debug2('Setscene stamp current task %s, so skip it and its dependencies', task)
                 self.sq_task_skip(task)
                 return True
 
             if self.cooker.configuration.skipsetscene:
-                logger.debug(2, 'No setscene tasks should be executed. Skipping %s', task)
+                logger.debug2('No setscene tasks should be executed. Skipping %s', task)
                 self.sq_task_failoutright(task)
                 return True
 
-            startevent = sceneQueueTaskStarted(task, self.sq_stats, self.rq)
+            startevent = sceneQueueTaskStarted(task, self.stats, self.rq)
             bb.event.fire(startevent, self.cfgData)
 
             taskdepdata = self.sq_build_taskdepdata(task)
@@ -2060,7 +2097,7 @@ class RunQueueExecute:
             self.build_stamps2.append(self.build_stamps[task])
             self.sq_running.add(task)
             self.sq_live.add(task)
-            self.sq_stats.taskActive()
+            self.stats.updateActiveSetscene(len(self.sq_live))
             if self.can_start_task():
                 return True
 
@@ -2097,12 +2134,12 @@ class RunQueueExecute:
                     return True
 
             if task in self.tasks_covered:
-                logger.debug(2, "Setscene covered task %s", task)
+                logger.debug2("Setscene covered task %s", task)
                 self.task_skip(task, "covered")
                 return True
 
             if self.rq.check_stamp_task(task, taskname, cache=self.stampcache):
-                logger.debug(2, "Stamp current task %s", task)
+                logger.debug2("Stamp current task %s", task)
 
                 self.task_skip(task, "existing")
                 self.runq_tasksrun.add(task)
@@ -2150,7 +2187,7 @@ class RunQueueExecute:
             if self.can_start_task():
                 return True
 
-        if self.stats.active > 0 or self.sq_stats.active > 0:
+        if self.stats.active > 0 or len(self.sq_live) > 0:
             self.rq.read_workers()
             return self.rq.active_fds()
 
@@ -2158,7 +2195,8 @@ class RunQueueExecute:
         if self.sq_deferred:
             tid = self.sq_deferred.pop(list(self.sq_deferred.keys())[0])
             logger.warning("Runqeueue deadlocked on deferred tasks, forcing task %s" % tid)
-            self.sq_task_failoutright(tid)
+            if tid not in self.runq_complete:
+                self.sq_task_failoutright(tid)
             return True
 
         if len(self.failed_tids) != 0:
@@ -2272,10 +2310,16 @@ class RunQueueExecute:
             self.updated_taskhash_queue.remove((tid, unihash))
 
             if unihash != self.rqdata.runtaskentries[tid].unihash:
-                hashequiv_logger.verbose("Task %s unihash changed to %s" % (tid, unihash))
-                self.rqdata.runtaskentries[tid].unihash = unihash
-                bb.parse.siggen.set_unihash(tid, unihash)
-                toprocess.add(tid)
+                # Make sure we rehash any other tasks with the same task hash that we're deferred against.
+                torehash = [tid]
+                for deftid in self.sq_deferred:
+                    if self.sq_deferred[deftid] == tid:
+                        torehash.append(deftid)
+                for hashtid in torehash:
+                    hashequiv_logger.verbose("Task %s unihash changed to %s" % (hashtid, unihash))
+                    self.rqdata.runtaskentries[hashtid].unihash = unihash
+                    bb.parse.siggen.set_unihash(hashtid, unihash)
+                    toprocess.add(hashtid)
 
         # Work out all tasks which depend upon these
         total = set()
@@ -2322,7 +2366,7 @@ class RunQueueExecute:
                     remapped = True
 
                 if not remapped:
-                    #logger.debug(1, "Task %s hash changes: %s->%s %s->%s" % (tid, orighash, newhash, origuni, newuni))
+                    #logger.debug("Task %s hash changes: %s->%s %s->%s" % (tid, orighash, newhash, origuni, newuni))
                     self.rqdata.runtaskentries[tid].hash = newhash
                     self.rqdata.runtaskentries[tid].unihash = newuni
                     changed.add(tid)
@@ -2337,7 +2381,7 @@ class RunQueueExecute:
             for mc in self.rq.fakeworker:
                 self.rq.fakeworker[mc].process.stdin.write(b"<newtaskhashes>" + pickle.dumps(bb.parse.siggen.get_taskhashes()) + b"</newtaskhashes>")
 
-            hashequiv_logger.debug(1, pprint.pformat("Tasks changed:\n%s" % (changed)))
+            hashequiv_logger.debug(pprint.pformat("Tasks changed:\n%s" % (changed)))
 
         for tid in changed:
             if tid not in self.rqdata.runq_setscene_tids:
@@ -2356,7 +2400,7 @@ class RunQueueExecute:
             # Check no tasks this covers are running
             for dep in self.sqdata.sq_covered_tasks[tid]:
                 if dep in self.runq_running and dep not in self.runq_complete:
-                    hashequiv_logger.debug(2, "Task %s is running which blocks setscene for %s from running" % (dep, tid))
+                    hashequiv_logger.debug2("Task %s is running which blocks setscene for %s from running" % (dep, tid))
                     valid = False
                     break
             if not valid:
@@ -2415,6 +2459,11 @@ class RunQueueExecute:
 
         if update_tasks:
             self.sqdone = False
+            for tid in [t[0] for t in update_tasks]:
+                h = pending_hash_index(tid, self.rqdata)
+                if h in self.sqdata.hashes and tid != self.sqdata.hashes[h]:
+                    self.sq_deferred[tid] = self.sqdata.hashes[h]
+                    bb.note("Deferring %s after %s" % (tid, self.sqdata.hashes[h]))
             update_scenequeue_data([t[0] for t in update_tasks], self.sqdata, self.rqdata, self.rq, self.cooker, self.stampcache, self, summary=False)
 
         for (tid, harddepfail, origvalid) in update_tasks:
@@ -2424,13 +2473,20 @@ class RunQueueExecute:
                 self.sq_task_failoutright(tid)
 
         if changed:
+            self.stats.updateCovered(len(self.scenequeue_covered), len(self.scenequeue_notcovered))
             self.holdoff_need_update = True
 
     def scenequeue_updatecounters(self, task, fail=False):
 
         for dep in sorted(self.sqdata.sq_deps[task]):
             if fail and task in self.sqdata.sq_harddeps and dep in self.sqdata.sq_harddeps[task]:
-                logger.debug(2, "%s was unavailable and is a hard dependency of %s so skipping" % (task, dep))
+                if dep in self.scenequeue_covered or dep in self.scenequeue_notcovered:
+                    # dependency could be already processed, e.g. noexec setscene task
+                    continue
+                noexec, stamppresent = check_setscene_stamps(dep, self.rqdata, self.rq, self.stampcache)
+                if noexec or stamppresent:
+                    continue
+                logger.debug2("%s was unavailable and is a hard dependency of %s so skipping" % (task, dep))
                 self.sq_task_failoutright(dep)
                 continue
             if self.sqdata.sq_revdeps[dep].issubset(self.scenequeue_covered | self.scenequeue_notcovered):
@@ -2451,6 +2507,7 @@ class RunQueueExecute:
                         new.add(dep)
             next = new
 
+        self.stats.updateCovered(len(self.scenequeue_covered), len(self.scenequeue_notcovered))
         self.holdoff_need_update = True
 
     def sq_task_completeoutright(self, task):
@@ -2460,7 +2517,7 @@ class RunQueueExecute:
         completed dependencies as buildable
         """
 
-        logger.debug(1, 'Found task %s which could be accelerated', task)
+        logger.debug('Found task %s which could be accelerated', task)
         self.scenequeue_covered.add(task)
         self.scenequeue_updatecounters(task)
 
@@ -2474,13 +2531,11 @@ class RunQueueExecute:
                 self.rq.state = runQueueCleanUp
 
     def sq_task_complete(self, task):
-        self.sq_stats.taskCompleted()
-        bb.event.fire(sceneQueueTaskCompleted(task, self.sq_stats, self.rq), self.cfgData)
+        bb.event.fire(sceneQueueTaskCompleted(task, self.stats, self.rq), self.cfgData)
         self.sq_task_completeoutright(task)
 
     def sq_task_fail(self, task, result):
-        self.sq_stats.taskFailed()
-        bb.event.fire(sceneQueueTaskFailed(task, self.sq_stats, result, self), self.cfgData)
+        bb.event.fire(sceneQueueTaskFailed(task, self.stats, result, self), self.cfgData)
         self.scenequeue_notcovered.add(task)
         self.scenequeue_updatecounters(task, True)
         self.sq_check_taskfail(task)
@@ -2488,8 +2543,6 @@ class RunQueueExecute:
     def sq_task_failoutright(self, task):
         self.sq_running.add(task)
         self.sq_buildable.add(task)
-        self.sq_stats.taskSkipped()
-        self.sq_stats.taskCompleted()
         self.scenequeue_notcovered.add(task)
         self.scenequeue_updatecounters(task, True)
 
@@ -2497,8 +2550,6 @@ class RunQueueExecute:
         self.sq_running.add(task)
         self.sq_buildable.add(task)
         self.sq_task_completeoutright(task)
-        self.sq_stats.taskSkipped()
-        self.sq_stats.taskCompleted()
 
     def sq_build_taskdepdata(self, task):
         def getsetscenedeps(tid):
@@ -2752,8 +2803,55 @@ def build_scenequeue_data(sqdata, rqdata, rq, cooker, stampcache, sqrq):
     sqdata.stamppresent = set()
     sqdata.valid = set()
 
+    sqdata.hashes = {}
+    sqrq.sq_deferred = {}
+    for mc in sorted(sqdata.multiconfigs):
+        for tid in sorted(sqdata.sq_revdeps):
+            if mc_from_tid(tid) != mc:
+                continue
+            h = pending_hash_index(tid, rqdata)
+            if h not in sqdata.hashes:
+                sqdata.hashes[h] = tid
+            else:
+                sqrq.sq_deferred[tid] = sqdata.hashes[h]
+                bb.note("Deferring %s after %s" % (tid, sqdata.hashes[h]))
+
     update_scenequeue_data(sqdata.sq_revdeps, sqdata, rqdata, rq, cooker, stampcache, sqrq, summary=True)
 
+    # Compute a list of 'stale' sstate tasks where the current hash does not match the one
+    # in any stamp files. Pass the list out to metadata as an event.
+    found = {}
+    for tid in rqdata.runq_setscene_tids:
+        (mc, fn, taskname, taskfn) = split_tid_mcfn(tid)
+        stamps = bb.build.find_stale_stamps(taskname, rqdata.dataCaches[mc], taskfn)
+        if stamps:
+            if mc not in found:
+                found[mc] = {}
+            found[mc][tid] = stamps
+    for mc in found:
+        event = bb.event.StaleSetSceneTasks(found[mc])
+        bb.event.fire(event, cooker.databuilder.mcdata[mc])
+
+def check_setscene_stamps(tid, rqdata, rq, stampcache, noexecstamp=False):
+
+    (mc, fn, taskname, taskfn) = split_tid_mcfn(tid)
+
+    taskdep = rqdata.dataCaches[mc].task_deps[taskfn]
+
+    if 'noexec' in taskdep and taskname in taskdep['noexec']:
+        bb.build.make_stamp(taskname + "_setscene", rqdata.dataCaches[mc], taskfn)
+        return True, False
+
+    if rq.check_stamp_task(tid, taskname + "_setscene", cache=stampcache):
+        logger.debug2('Setscene stamp current for task %s', tid)
+        return False, True
+
+    if rq.check_stamp_task(tid, taskname, recurse = True, cache=stampcache):
+        logger.debug2('Normal stamp current for task %s', tid)
+        return False, True
+
+    return False, False
+
 def update_scenequeue_data(tids, sqdata, rqdata, rq, cooker, stampcache, sqrq, summary=True):
 
     tocheck = set()
@@ -2763,25 +2861,17 @@ def update_scenequeue_data(tids, sqdata, rqdata, rq, cooker, stampcache, sqrq, s
             sqdata.stamppresent.remove(tid)
         if tid in sqdata.valid:
             sqdata.valid.remove(tid)
+        if tid in sqdata.outrightfail:
+            sqdata.outrightfail.remove(tid)
 
-        (mc, fn, taskname, taskfn) = split_tid_mcfn(tid)
+        noexec, stamppresent = check_setscene_stamps(tid, rqdata, rq, stampcache, noexecstamp=True)
 
-        taskdep = rqdata.dataCaches[mc].task_deps[taskfn]
-
-        if 'noexec' in taskdep and taskname in taskdep['noexec']:
+        if noexec:
             sqdata.noexec.add(tid)
             sqrq.sq_task_skip(tid)
-            bb.build.make_stamp(taskname + "_setscene", rqdata.dataCaches[mc], taskfn)
             continue
 
-        if rq.check_stamp_task(tid, taskname + "_setscene", cache=stampcache):
-            logger.debug(2, 'Setscene stamp current for task %s', tid)
-            sqdata.stamppresent.add(tid)
-            sqrq.sq_task_skip(tid)
-            continue
-
-        if rq.check_stamp_task(tid, taskname, recurse = True, cache=stampcache):
-            logger.debug(2, 'Normal stamp current for task %s', tid)
+        if stamppresent:
             sqdata.stamppresent.add(tid)
             sqrq.sq_task_skip(tid)
             continue
@@ -2790,28 +2880,20 @@ def update_scenequeue_data(tids, sqdata, rqdata, rq, cooker, stampcache, sqrq, s
 
     sqdata.valid |= rq.validate_hashes(tocheck, cooker.data, len(sqdata.stamppresent), False, summary=summary)
 
-    sqdata.hashes = {}
-    for mc in sorted(sqdata.multiconfigs):
-        for tid in sorted(sqdata.sq_revdeps):
-            if mc_from_tid(tid) != mc:
-                continue
-            if tid in sqdata.stamppresent:
-                continue
-            if tid in sqdata.valid:
-                continue
-            if tid in sqdata.noexec:
-                continue
-            if tid in sqrq.scenequeue_notcovered:
-                continue
-            sqdata.outrightfail.add(tid)
-
-            h = pending_hash_index(tid, rqdata)
-            if h not in sqdata.hashes:
-                sqdata.hashes[h] = tid
-            else:
-                sqrq.sq_deferred[tid] = sqdata.hashes[h]
-                bb.note("Deferring %s after %s" % (tid, sqdata.hashes[h]))
-
+    for tid in tids:
+        if tid in sqdata.stamppresent:
+            continue
+        if tid in sqdata.valid:
+            continue
+        if tid in sqdata.noexec:
+            continue
+        if tid in sqrq.scenequeue_covered:
+            continue
+        if tid in sqrq.scenequeue_notcovered:
+            continue
+        if tid in sqrq.sq_deferred:
+            continue
+        sqdata.outrightfail.add(tid)
 
 class TaskFailure(Exception):
     """
@@ -2875,12 +2957,16 @@ class runQueueTaskFailed(runQueueEvent):
     """
     Event notifying a task failed
     """
-    def __init__(self, task, stats, exitcode, rq):
+    def __init__(self, task, stats, exitcode, rq, fakeroot_log=None):
         runQueueEvent.__init__(self, task, stats, rq)
         self.exitcode = exitcode
+        self.fakeroot_log = fakeroot_log
 
     def __str__(self):
-        return "Task (%s) failed with exit code '%s'" % (self.taskstring, self.exitcode)
+        if self.fakeroot_log:
+            return "Task (%s) failed with exit code '%s' \nPseudo log:\n%s" % (self.taskstring, self.exitcode, self.fakeroot_log)
+        else:
+            return "Task (%s) failed with exit code '%s'" % (self.taskstring, self.exitcode)
 
 class sceneQueueTaskFailed(sceneQueueEvent):
     """
@@ -2932,7 +3018,7 @@ class runQueuePipe():
     """
     Abstraction for a pipe between a worker thread and the server
     """
-    def __init__(self, pipein, pipeout, d, rq, rqexec):
+    def __init__(self, pipein, pipeout, d, rq, rqexec, fakerootlogs=None):
         self.input = pipein
         if pipeout:
             pipeout.close()
@@ -2941,6 +3027,7 @@ class runQueuePipe():
         self.d = d
         self.rq = rq
         self.rqexec = rqexec
+        self.fakerootlogs = fakerootlogs
 
     def setrunqueueexec(self, rqexec):
         self.rqexec = rqexec
@@ -2986,7 +3073,11 @@ class runQueuePipe():
                     task, status = pickle.loads(self.queue[10:index])
                 except (ValueError, pickle.UnpicklingError, AttributeError, IndexError) as e:
                     bb.msg.fatal("RunQueue", "failed load pickle '%s': '%s'" % (e, self.queue[10:index]))
-                self.rqexec.runqueue_process_waitpid(task, status)
+                (_, _, _, taskfn) = split_tid_mcfn(task)
+                fakerootlog = None
+                if self.fakerootlogs and taskfn and taskfn in self.fakerootlogs:
+                    fakerootlog = self.fakerootlogs[taskfn]
+                self.rqexec.runqueue_process_waitpid(task, status, fakerootlog=fakerootlog)
                 found = True
                 self.queue = self.queue[index+11:]
                 index = self.queue.find(b"</exitcode>")

bitbake/lib/bb/server/process.py

@@ -26,6 +26,7 @@ import errno
 import re
 import datetime
 import pickle
+import gc
 import bb.server.xmlrpcserver
 from bb import daemonize
 from multiprocessing import queues
@@ -147,7 +148,7 @@ class ProcessServer():
                 conn = newconnections.pop(-1)
                 fds.append(conn)
                 self.controllersock = conn
-            elif self.timeout is None and not ready:
+            elif not self.timeout and not ready:
                 serverlog("No timeout, exiting.")
                 self.quit = True
 
@@ -367,7 +368,12 @@ class ProcessServer():
                 self.next_heartbeat = now + self.heartbeat_seconds
             if hasattr(self.cooker, "data"):
                 heartbeat = bb.event.HeartbeatEvent(now)
-                bb.event.fire(heartbeat, self.cooker.data)
+                try:
+                    bb.event.fire(heartbeat, self.cooker.data)
+                except Exception as exc:
+                    if not isinstance(exc, bb.BBHandledException):
+                        logger.exception('Running heartbeat function')
+                    self.quit = True
         if nextsleep and now + nextsleep > self.next_heartbeat:
             # Shorten timeout so that we we wake up in time for
             # the heartbeat.
@@ -466,7 +472,7 @@ class BitBakeServer(object):
             try:
                 r = ready.get()
             except EOFError:
-                # Trap the child exitting/closing the pipe and error out
+                # Trap the child exiting/closing the pipe and error out
                 r = None
         if not r or r[0] != "r":
             ready.close()
@@ -509,7 +515,7 @@ class BitBakeServer(object):
         os.set_inheritable(self.bitbake_lock.fileno(), True)
         os.set_inheritable(self.readypipein, True)
         serverscript = os.path.realpath(os.path.dirname(__file__) + "/../../../bin/bitbake-server")
-        os.execl(sys.executable, "bitbake-server", serverscript, "decafbad", str(self.bitbake_lock.fileno()), str(self.readypipein), self.logfile, self.bitbake_lock.name, self.sockname,  str(self.server_timeout), str(self.xmlrpcinterface[0]), str(self.xmlrpcinterface[1]))
+        os.execl(sys.executable, "bitbake-server", serverscript, "decafbad", str(self.bitbake_lock.fileno()), str(self.readypipein), self.logfile, self.bitbake_lock.name, self.sockname,  str(self.server_timeout or 0), str(self.xmlrpcinterface[0]), str(self.xmlrpcinterface[1]))
 
 def execServer(lockfd, readypipeinfd, lockname, sockname, server_timeout, xmlrpcinterface):
 
@@ -654,7 +660,7 @@ class BBUIEventQueue:
         self.reader = ConnectionReader(readfd)
 
         self.t = threading.Thread()
-        self.t.setDaemon(True)
+        self.t.daemon = True
         self.t.run = self.startCallbackHandler
         self.t.start()
 
@@ -732,8 +738,10 @@ class ConnectionWriter(object):
 
     def send(self, obj):
         obj = multiprocessing.reduction.ForkingPickler.dumps(obj)
+        gc.disable()
         with self.wlock:
             self.writer.send_bytes(obj)
+        gc.enable()
 
     def fileno(self):
         return self.writer.fileno()

bitbake/lib/bb/server/xmlrpcserver.py

@@ -11,6 +11,7 @@ import hashlib
 import time
 import inspect
 from xmlrpc.server import SimpleXMLRPCServer, SimpleXMLRPCRequestHandler
+import bb.server.xmlrpcclient
 
 import bb
 

bitbake/lib/bb/siggen.py

@@ -311,13 +311,7 @@ class SignatureGeneratorBasic(SignatureGenerator):
 
         data = self.basehash[tid]
         for dep in self.runtaskdeps[tid]:
-            if dep in self.unihash:
-                if self.unihash[dep] is None:
-                    data = data + self.taskhash[dep]
-                else:
-                    data = data + self.unihash[dep]
-            else:
-                data = data + self.get_unihash(dep)
+            data = data + self.get_unihash(dep)
 
         for (f, cs) in self.file_checksum_values[tid]:
             if cs:
@@ -547,7 +541,7 @@ class SignatureGeneratorUniHashMixIn(object):
                 # is much more interesting, so it is reported at debug level 1
                 hashequiv_logger.debug((1, 2)[unihash == taskhash], 'Found unihash %s in place of %s for %s from %s' % (unihash, taskhash, tid, self.server))
             else:
-                hashequiv_logger.debug(2, 'No reported unihash for %s:%s from %s' % (tid, taskhash, self.server))
+                hashequiv_logger.debug2('No reported unihash for %s:%s from %s' % (tid, taskhash, self.server))
         except hashserv.client.HashConnectionError as e:
             bb.warn('Error contacting Hash Equivalence Server %s: %s' % (self.server, str(e)))
 
@@ -621,12 +615,12 @@ class SignatureGeneratorUniHashMixIn(object):
                 new_unihash = data['unihash']
 
                 if new_unihash != unihash:
-                    hashequiv_logger.debug(1, 'Task %s unihash changed %s -> %s by server %s' % (taskhash, unihash, new_unihash, self.server))
+                    hashequiv_logger.debug('Task %s unihash changed %s -> %s by server %s' % (taskhash, unihash, new_unihash, self.server))
                     bb.event.fire(bb.runqueue.taskUniHashUpdate(fn + ':do_' + task, new_unihash), d)
                     self.set_unihash(tid, new_unihash)
                     d.setVar('BB_UNIHASH', new_unihash)
                 else:
-                    hashequiv_logger.debug(1, 'Reported task %s as unihash %s to %s' % (taskhash, unihash, self.server))
+                    hashequiv_logger.debug('Reported task %s as unihash %s to %s' % (taskhash, unihash, self.server))
             except hashserv.client.HashConnectionError as e:
                 bb.warn('Error contacting Hash Equivalence Server %s: %s' % (self.server, str(e)))
         finally:
@@ -754,7 +748,7 @@ def clean_basepath(basepath):
     if basepath[0] == '/':
         return cleaned
 
-    if basepath.startswith("mc:"):
+    if basepath.startswith("mc:") and basepath.count(':') >= 2:
         mc, mc_name, basepath = basepath.split(":", 2)
         mc_suffix = ':mc:' + mc_name
     else:

bitbake/lib/bb/taskdata.py

@@ -131,7 +131,7 @@ class TaskData:
             for depend in dataCache.deps[fn]:
                 dependids.add(depend)
             self.depids[fn] = list(dependids)
-            logger.debug(2, "Added dependencies %s for %s", str(dataCache.deps[fn]), fn)
+            logger.debug2("Added dependencies %s for %s", str(dataCache.deps[fn]), fn)
 
         # Work out runtime dependencies
         if not fn in self.rdepids:
@@ -149,9 +149,9 @@ class TaskData:
                     rreclist.append(rdepend)
                     rdependids.add(rdepend)
             if rdependlist:
-                logger.debug(2, "Added runtime dependencies %s for %s", str(rdependlist), fn)
+                logger.debug2("Added runtime dependencies %s for %s", str(rdependlist), fn)
             if rreclist:
-                logger.debug(2, "Added runtime recommendations %s for %s", str(rreclist), fn)
+                logger.debug2("Added runtime recommendations %s for %s", str(rreclist), fn)
             self.rdepids[fn] = list(rdependids)
 
         for dep in self.depids[fn]:
@@ -378,7 +378,7 @@ class TaskData:
         for fn in eligible:
             if fn in self.failed_fns:
                 continue
-            logger.debug(2, "adding %s to satisfy %s", fn, item)
+            logger.debug2("adding %s to satisfy %s", fn, item)
             self.add_build_target(fn, item)
             self.add_tasks(fn, dataCache)
 
@@ -431,7 +431,7 @@ class TaskData:
         for fn in eligible:
             if fn in self.failed_fns:
                 continue
-            logger.debug(2, "adding '%s' to satisfy runtime '%s'", fn, item)
+            logger.debug2("adding '%s' to satisfy runtime '%s'", fn, item)
             self.add_runtime_target(fn, item)
             self.add_tasks(fn, dataCache)
 
@@ -446,7 +446,7 @@ class TaskData:
             return
         if not missing_list:
             missing_list = []
-        logger.debug(1, "File '%s' is unbuildable, removing...", fn)
+        logger.debug("File '%s' is unbuildable, removing...", fn)
         self.failed_fns.append(fn)
         for target in self.build_targets:
             if fn in self.build_targets[target]:
@@ -526,7 +526,7 @@ class TaskData:
                     added = added + 1
                 except (bb.providers.NoRProvider, bb.providers.MultipleRProvider):
                     self.remove_runtarget(target)
-            logger.debug(1, "Resolved " + str(added) + " extra dependencies")
+            logger.debug("Resolved " + str(added) + " extra dependencies")
             if added == 0:
                 break
         # self.dump_data()
@@ -549,38 +549,38 @@ class TaskData:
         """
         Dump some debug information on the internal data structures
         """
-        logger.debug(3, "build_names:")
-        logger.debug(3, ", ".join(self.build_targets))
+        logger.debug3("build_names:")
+        logger.debug3(", ".join(self.build_targets))
 
-        logger.debug(3, "run_names:")
-        logger.debug(3, ", ".join(self.run_targets))
+        logger.debug3("run_names:")
+        logger.debug3(", ".join(self.run_targets))
 
-        logger.debug(3, "build_targets:")
+        logger.debug3("build_targets:")
         for target in self.build_targets:
             targets = "None"
             if target in self.build_targets:
                 targets = self.build_targets[target]
-            logger.debug(3, " %s: %s", target, targets)
+            logger.debug3(" %s: %s", target, targets)
 
-        logger.debug(3, "run_targets:")
+        logger.debug3("run_targets:")
         for target in self.run_targets:
             targets = "None"
             if target in self.run_targets:
                 targets = self.run_targets[target]
-            logger.debug(3, " %s: %s", target, targets)
+            logger.debug3(" %s: %s", target, targets)
 
-        logger.debug(3, "tasks:")
+        logger.debug3("tasks:")
         for tid in self.taskentries:
-            logger.debug(3, " %s: %s %s %s",
+            logger.debug3(" %s: %s %s %s",
                        tid,
                        self.taskentries[tid].idepends,
                        self.taskentries[tid].irdepends,
                        self.taskentries[tid].tdepends)
 
-        logger.debug(3, "dependency ids (per fn):")
+        logger.debug3("dependency ids (per fn):")
         for fn in self.depids:
-            logger.debug(3, " %s: %s", fn, self.depids[fn])
+            logger.debug3(" %s: %s", fn, self.depids[fn])
 
-        logger.debug(3, "runtime dependency ids (per fn):")
+        logger.debug3("runtime dependency ids (per fn):")
         for fn in self.rdepids:
-            logger.debug(3, " %s: %s", fn, self.rdepids[fn])
+            logger.debug3(" %s: %s", fn, self.rdepids[fn])

bitbake/lib/bb/tests/codeparser.py

@@ -111,9 +111,9 @@ ${D}${libdir}/pkgconfig/*.pc
         self.assertExecs(set(["sed"]))
 
     def test_parameter_expansion_modifiers(self):
-        # - and + are also valid modifiers for parameter expansion, but are
+        # -,+ and : are also valid modifiers for parameter expansion, but are
         # valid characters in bitbake variable names, so are not included here
-        for i in ('=', ':-', ':=', '?', ':?', ':+', '#', '%', '##', '%%'):
+        for i in ('=', '?', '#', '%', '##', '%%'):
             name = "foo%sbar" % i
             self.parseExpression("${%s}" % name)
             self.assertNotIn(name, self.references)

bitbake/lib/bb/tests/color.py

@@ -31,7 +31,7 @@ class ColorCodeTests(unittest.TestCase):
     def setUp(self):
         self.d = bb.data.init()
         self._progress_watcher = ProgressWatcher()
-        bb.event.register("bb.build.TaskProgress", self._progress_watcher.handle_event)
+        bb.event.register("bb.build.TaskProgress", self._progress_watcher.handle_event, data=self.d)
 
     def tearDown(self):
         bb.event.remove("bb.build.TaskProgress", None)

bitbake/lib/bb/tests/fetch.py

@@ -87,6 +87,25 @@ class URITest(unittest.TestCase):
             },
             'relative': False
         },
+        # Check that trailing semicolons are handled correctly
+        "http://www.example.org/index.html?qparam1=qvalue1;param2=value2;" : {
+            'uri': 'http://www.example.org/index.html?qparam1=qvalue1;param2=value2',
+            'scheme': 'http',
+            'hostname': 'www.example.org',
+            'port': None,
+            'hostport': 'www.example.org',
+            'path': '/index.html',
+            'userinfo': '',
+            'username': '',
+            'password': '',
+            'params': {
+                'param2': 'value2'
+            },
+            'query': {
+                'qparam1': 'qvalue1'
+            },
+            'relative': False
+        },
         "http://www.example.com:8080/index.html" : {
             'uri': 'http://www.example.com:8080/index.html',
             'scheme': 'http',
@@ -371,6 +390,7 @@ class FetcherTest(unittest.TestCase):
         if os.environ.get("BB_TMPDIR_NOCLEAN") == "yes":
             print("Not cleaning up %s. Please remove manually." % self.tempdir)
         else:
+            bb.process.run('chmod u+rw -R %s' % self.tempdir)
             bb.utils.prunedir(self.tempdir)
 
 class MirrorUriTest(FetcherTest):
@@ -411,6 +431,10 @@ class MirrorUriTest(FetcherTest):
         ("git://someserver.org/bitbake;tag=1234567890123456789012345678901234567890;branch=master", "git://someserver.org/bitbake;branch=master", "git://git.openembedded.org/bitbake;protocol=http")
             : "git://git.openembedded.org/bitbake;tag=1234567890123456789012345678901234567890;branch=master;protocol=http",
 
+        ("git://user1@someserver.org/bitbake;tag=1234567890123456789012345678901234567890;branch=master", "git://someserver.org/bitbake;branch=master", "git://user2@git.openembedded.org/bitbake;protocol=http")
+            : "git://user2@git.openembedded.org/bitbake;tag=1234567890123456789012345678901234567890;branch=master;protocol=http",
+
+
         #Renaming files doesn't work
         #("http://somewhere.org/somedir1/somefile_1.2.3.tar.gz", "http://somewhere.org/somedir1/somefile_1.2.3.tar.gz", "http://somewhere2.org/somedir3/somefile_2.3.4.tar.gz") : "http://somewhere2.org/somedir3/somefile_2.3.4.tar.gz"
         #("file://sstate-xyz.tgz", "file://.*/.*", "file:///somewhere/1234/sstate-cache") : "file:///somewhere/1234/sstate-cache/sstate-xyz.tgz",
@@ -471,7 +495,7 @@ class GitDownloadDirectoryNamingTest(FetcherTest):
         super(GitDownloadDirectoryNamingTest, self).setUp()
         self.recipe_url = "git://git.openembedded.org/bitbake"
         self.recipe_dir = "git.openembedded.org.bitbake"
-        self.mirror_url = "git://github.com/openembedded/bitbake.git"
+        self.mirror_url = "git://github.com/openembedded/bitbake.git;protocol=https"
         self.mirror_dir = "github.com.openembedded.bitbake.git"
 
         self.d.setVar('SRCREV', '82ea737a0b42a8b53e11c9cde141e9e9c0bd8c40')
@@ -519,7 +543,7 @@ class TarballNamingTest(FetcherTest):
         super(TarballNamingTest, self).setUp()
         self.recipe_url = "git://git.openembedded.org/bitbake"
         self.recipe_tarball = "git2_git.openembedded.org.bitbake.tar.gz"
-        self.mirror_url = "git://github.com/openembedded/bitbake.git"
+        self.mirror_url = "git://github.com/openembedded/bitbake.git;protocol=https"
         self.mirror_tarball = "git2_github.com.openembedded.bitbake.git.tar.gz"
 
         self.d.setVar('BB_GENERATE_MIRROR_TARBALLS', '1')
@@ -553,7 +577,7 @@ class GitShallowTarballNamingTest(FetcherTest):
         super(GitShallowTarballNamingTest, self).setUp()
         self.recipe_url = "git://git.openembedded.org/bitbake"
         self.recipe_tarball = "gitshallow_git.openembedded.org.bitbake_82ea737-1_master.tar.gz"
-        self.mirror_url = "git://github.com/openembedded/bitbake.git"
+        self.mirror_url = "git://github.com/openembedded/bitbake.git;protocol=https"
         self.mirror_tarball = "gitshallow_github.com.openembedded.bitbake.git_82ea737-1_master.tar.gz"
 
         self.d.setVar('BB_GIT_SHALLOW', '1')
@@ -654,6 +678,62 @@ class FetcherLocalTest(FetcherTest):
         with self.assertRaises(bb.fetch2.UnpackError):
             self.fetchUnpack(['file://a;subdir=/bin/sh'])
 
+    def test_local_gitfetch_usehead(self):
+        # Create dummy local Git repo
+        src_dir = tempfile.mkdtemp(dir=self.tempdir,
+                                   prefix='gitfetch_localusehead_')
+        src_dir = os.path.abspath(src_dir)
+        bb.process.run("git init", cwd=src_dir)
+        bb.process.run("git config user.email 'you@example.com'", cwd=src_dir)
+        bb.process.run("git config user.name 'Your Name'", cwd=src_dir)
+        bb.process.run("git commit --allow-empty -m'Dummy commit'",
+                       cwd=src_dir)
+        # Use other branch than master
+        bb.process.run("git checkout -b my-devel", cwd=src_dir)
+        bb.process.run("git commit --allow-empty -m'Dummy commit 2'",
+                       cwd=src_dir)
+        stdout = bb.process.run("git rev-parse HEAD", cwd=src_dir)
+        orig_rev = stdout[0].strip()
+
+        # Fetch and check revision
+        self.d.setVar("SRCREV", "AUTOINC")
+        url = "git://" + src_dir + ";protocol=file;usehead=1"
+        fetcher = bb.fetch.Fetch([url], self.d)
+        fetcher.download()
+        fetcher.unpack(self.unpackdir)
+        stdout = bb.process.run("git rev-parse HEAD",
+                                cwd=os.path.join(self.unpackdir, 'git'))
+        unpack_rev = stdout[0].strip()
+        self.assertEqual(orig_rev, unpack_rev)
+
+    def test_local_gitfetch_usehead_withname(self):
+        # Create dummy local Git repo
+        src_dir = tempfile.mkdtemp(dir=self.tempdir,
+                                   prefix='gitfetch_localusehead_')
+        src_dir = os.path.abspath(src_dir)
+        bb.process.run("git init", cwd=src_dir)
+        bb.process.run("git config user.email 'you@example.com'", cwd=src_dir)
+        bb.process.run("git config user.name 'Your Name'", cwd=src_dir)
+        bb.process.run("git commit --allow-empty -m'Dummy commit'",
+                       cwd=src_dir)
+        # Use other branch than master
+        bb.process.run("git checkout -b my-devel", cwd=src_dir)
+        bb.process.run("git commit --allow-empty -m'Dummy commit 2'",
+                       cwd=src_dir)
+        stdout = bb.process.run("git rev-parse HEAD", cwd=src_dir)
+        orig_rev = stdout[0].strip()
+
+        # Fetch and check revision
+        self.d.setVar("SRCREV", "AUTOINC")
+        url = "git://" + src_dir + ";protocol=file;usehead=1;name=newName"
+        fetcher = bb.fetch.Fetch([url], self.d)
+        fetcher.download()
+        fetcher.unpack(self.unpackdir)
+        stdout = bb.process.run("git rev-parse HEAD",
+                                cwd=os.path.join(self.unpackdir, 'git'))
+        unpack_rev = stdout[0].strip()
+        self.assertEqual(orig_rev, unpack_rev)
+
 class FetcherNoNetworkTest(FetcherTest):
     def setUp(self):
         super().setUp()
@@ -844,35 +924,21 @@ class FetcherNetworkTest(FetcherTest):
         self.assertRaises(bb.fetch.FetchError, self.gitfetcher, url1, url2)
 
     @skipIfNoNetwork()
-    def test_gitfetch_localusehead(self):
-        # Create dummy local Git repo
-        src_dir = tempfile.mkdtemp(dir=self.tempdir,
-                                   prefix='gitfetch_localusehead_')
-        src_dir = os.path.abspath(src_dir)
-        bb.process.run("git init", cwd=src_dir)
-        bb.process.run("git commit --allow-empty -m'Dummy commit'",
-                       cwd=src_dir)
-        # Use other branch than master
-        bb.process.run("git checkout -b my-devel", cwd=src_dir)
-        bb.process.run("git commit --allow-empty -m'Dummy commit 2'",
-                       cwd=src_dir)
-        stdout = bb.process.run("git rev-parse HEAD", cwd=src_dir)
-        orig_rev = stdout[0].strip()
-
-        # Fetch and check revision
-        self.d.setVar("SRCREV", "AUTOINC")
-        url = "git://" + src_dir + ";protocol=file;usehead=1"
-        fetcher = bb.fetch.Fetch([url], self.d)
-        fetcher.download()
-        fetcher.unpack(self.unpackdir)
-        stdout = bb.process.run("git rev-parse HEAD",
-                                cwd=os.path.join(self.unpackdir, 'git'))
-        unpack_rev = stdout[0].strip()
-        self.assertEqual(orig_rev, unpack_rev)
+    def test_gitfetch_usehead(self):
+        # Since self.gitfetcher() sets SRCREV we expect this to override
+        # `usehead=1' and instead fetch the specified SRCREV. See
+        # test_local_gitfetch_usehead() for a positive use of the usehead
+        # feature.
+        url = "git://git.openembedded.org/bitbake;usehead=1"
+        self.assertRaises(bb.fetch.ParameterError, self.gitfetcher, url, url)
 
     @skipIfNoNetwork()
-    def test_gitfetch_remoteusehead(self):
-        url = "git://git.openembedded.org/bitbake;usehead=1"
+    def test_gitfetch_usehead_withname(self):
+        # Since self.gitfetcher() sets SRCREV we expect this to override
+        # `usehead=1' and instead fetch the specified SRCREV. See
+        # test_local_gitfetch_usehead() for a positive use of the usehead
+        # feature.
+        url = "git://git.openembedded.org/bitbake;usehead=1;name=newName"
         self.assertRaises(bb.fetch.ParameterError, self.gitfetcher, url, url)
 
     @skipIfNoNetwork()
@@ -923,7 +989,7 @@ class FetcherNetworkTest(FetcherTest):
     def test_git_submodule_dbus_broker(self):
         # The following external repositories have show failures in fetch and unpack operations
         # We want to avoid regressions!
-        url = "gitsm://github.com/bus1/dbus-broker;protocol=git;rev=fc874afa0992d0c75ec25acb43d344679f0ee7d2;branch=main"
+        url = "gitsm://github.com/bus1/dbus-broker;protocol=https;rev=fc874afa0992d0c75ec25acb43d344679f0ee7d2;branch=main"
         fetcher = bb.fetch.Fetch([url], self.d)
         fetcher.download()
         # Previous cwd has been deleted
@@ -939,7 +1005,7 @@ class FetcherNetworkTest(FetcherTest):
 
     @skipIfNoNetwork()
     def test_git_submodule_CLI11(self):
-        url = "gitsm://github.com/CLIUtils/CLI11;protocol=git;rev=bd4dc911847d0cde7a6b41dfa626a85aab213baf"
+        url = "gitsm://github.com/CLIUtils/CLI11;protocol=https;rev=bd4dc911847d0cde7a6b41dfa626a85aab213baf;branch=main"
         fetcher = bb.fetch.Fetch([url], self.d)
         fetcher.download()
         # Previous cwd has been deleted
@@ -954,12 +1020,12 @@ class FetcherNetworkTest(FetcherTest):
     @skipIfNoNetwork()
     def test_git_submodule_update_CLI11(self):
         """ Prevent regression on update detection not finding missing submodule, or modules without needed commits """
-        url = "gitsm://github.com/CLIUtils/CLI11;protocol=git;rev=cf6a99fa69aaefe477cc52e3ef4a7d2d7fa40714"
+        url = "gitsm://github.com/CLIUtils/CLI11;protocol=https;rev=cf6a99fa69aaefe477cc52e3ef4a7d2d7fa40714;branch=main"
         fetcher = bb.fetch.Fetch([url], self.d)
         fetcher.download()
 
         # CLI11 that pulls in a newer nlohmann-json
-        url = "gitsm://github.com/CLIUtils/CLI11;protocol=git;rev=49ac989a9527ee9bb496de9ded7b4872c2e0e5ca"
+        url = "gitsm://github.com/CLIUtils/CLI11;protocol=https;rev=49ac989a9527ee9bb496de9ded7b4872c2e0e5ca;branch=main"
         fetcher = bb.fetch.Fetch([url], self.d)
         fetcher.download()
         # Previous cwd has been deleted
@@ -973,7 +1039,7 @@ class FetcherNetworkTest(FetcherTest):
 
     @skipIfNoNetwork()
     def test_git_submodule_aktualizr(self):
-        url = "gitsm://github.com/advancedtelematic/aktualizr;branch=master;protocol=git;rev=d00d1a04cc2366d1a5f143b84b9f507f8bd32c44"
+        url = "gitsm://github.com/advancedtelematic/aktualizr;branch=master;protocol=https;rev=d00d1a04cc2366d1a5f143b84b9f507f8bd32c44"
         fetcher = bb.fetch.Fetch([url], self.d)
         fetcher.download()
         # Previous cwd has been deleted
@@ -993,7 +1059,7 @@ class FetcherNetworkTest(FetcherTest):
         """ Prevent regression on deeply nested submodules not being checked out properly, even though they were fetched. """
 
         # This repository also has submodules where the module (name), path and url do not align
-        url = "gitsm://github.com/azure/iotedge.git;protocol=git;rev=d76e0316c6f324345d77c48a83ce836d09392699"
+        url = "gitsm://github.com/azure/iotedge.git;protocol=https;rev=d76e0316c6f324345d77c48a83ce836d09392699;branch=main"
         fetcher = bb.fetch.Fetch([url], self.d)
         fetcher.download()
         # Previous cwd has been deleted
@@ -1051,7 +1117,7 @@ class SVNTest(FetcherTest):
 
         bb.process.run("svn co %s svnfetch_co" % self.repo_url, cwd=self.tempdir)
         # Github will emulate SVN.  Use this to check if we're downloding...
-        bb.process.run("svn propset svn:externals 'bitbake svn://vcs.pcre.org/pcre2/code' .",
+        bb.process.run("svn propset svn:externals 'bitbake https://github.com/PhilipHazel/pcre2.git' .",
                        cwd=os.path.join(self.tempdir, 'svnfetch_co', 'trunk'))
         bb.process.run("svn commit --non-interactive -m 'Add external'",
                        cwd=os.path.join(self.tempdir, 'svnfetch_co', 'trunk'))
@@ -1169,7 +1235,7 @@ class FetchLatestVersionTest(FetcherTest):
 
     test_git_uris = {
         # version pattern "X.Y.Z"
-        ("mx-1.0", "git://github.com/clutter-project/mx.git;branch=mx-1.4", "9b1db6b8060bd00b121a692f942404a24ae2960f", "")
+        ("mx-1.0", "git://github.com/clutter-project/mx.git;branch=mx-1.4;protocol=https", "9b1db6b8060bd00b121a692f942404a24ae2960f", "")
             : "1.99.4",
         # version pattern "vX.Y"
         # mirror of git.infradead.org since network issues interfered with testing
@@ -1180,7 +1246,7 @@ class FetchLatestVersionTest(FetcherTest):
         ("presentproto", "git://git.yoctoproject.org/bbfetchtests-presentproto", "24f3a56e541b0a9e6c6ee76081f441221a120ef9", "")
             : "1.0",
         # version pattern "pkg_name-vX.Y.Z"
-        ("dtc", "git://git.qemu.org/dtc.git", "65cc4d2748a2c2e6f27f1cf39e07a5dbabd80ebf", "")
+        ("dtc", "git://git.yoctoproject.org/bbfetchtests-dtc.git", "65cc4d2748a2c2e6f27f1cf39e07a5dbabd80ebf", "")
             : "1.4.0",
         # combination version pattern
         ("sysprof", "git://gitlab.gnome.org/GNOME/sysprof.git;protocol=https", "cd44ee6644c3641507fb53b8a2a69137f2971219", "")
@@ -1196,9 +1262,9 @@ class FetchLatestVersionTest(FetcherTest):
             : "0.4.3",
         ("build-appliance-image", "git://git.yoctoproject.org/poky", "b37dd451a52622d5b570183a81583cc34c2ff555", "(?P<pver>(([0-9][\.|_]?)+[0-9]))")
             : "11.0.0",
-        ("chkconfig-alternatives-native", "git://github.com/kergoth/chkconfig;branch=sysroot", "cd437ecbd8986c894442f8fce1e0061e20f04dee", "chkconfig\-(?P<pver>((\d+[\.\-_]*)+))")
+        ("chkconfig-alternatives-native", "git://github.com/kergoth/chkconfig;branch=sysroot;protocol=https", "cd437ecbd8986c894442f8fce1e0061e20f04dee", "chkconfig\-(?P<pver>((\d+[\.\-_]*)+))")
             : "1.3.59",
-        ("remake", "git://github.com/rocky/remake.git", "f05508e521987c8494c92d9c2871aec46307d51d", "(?P<pver>(\d+\.(\d+\.)*\d*(\+dbg\d+(\.\d+)*)*))")
+        ("remake", "git://github.com/rocky/remake.git;protocol=https", "f05508e521987c8494c92d9c2871aec46307d51d", "(?P<pver>(\d+\.(\d+\.)*\d*(\+dbg\d+(\.\d+)*)*))")
             : "3.82+dbg0.9",
     }
 
@@ -1288,13 +1354,10 @@ class FetchCheckStatusTest(FetcherTest):
                       "http://downloads.yoctoproject.org/releases/sato/sato-engine-0.2.tar.gz",
                       "http://downloads.yoctoproject.org/releases/sato/sato-engine-0.3.tar.gz",
                       "https://yoctoproject.org/",
-                      "https://yoctoproject.org/documentation",
+                      "https://docs.yoctoproject.org",
                       "http://downloads.yoctoproject.org/releases/opkg/opkg-0.1.7.tar.gz",
                       "http://downloads.yoctoproject.org/releases/opkg/opkg-0.3.0.tar.gz",
                       "ftp://sourceware.org/pub/libffi/libffi-1.20.tar.gz",
-                      "http://ftp.gnu.org/gnu/autoconf/autoconf-2.60.tar.gz",
-                      "https://ftp.gnu.org/gnu/chess/gnuchess-5.08.tar.gz",
-                      "https://ftp.gnu.org/gnu/gmp/gmp-4.0.tar.gz",
                       # GitHub releases are hosted on Amazon S3, which doesn't support HEAD
                       "https://github.com/kergoth/tslib/releases/download/1.1/tslib-1.1.tar.xz"
                       ]
@@ -1333,6 +1396,8 @@ class GitMakeShallowTest(FetcherTest):
         self.gitdir = os.path.join(self.tempdir, 'gitshallow')
         bb.utils.mkdirhier(self.gitdir)
         bb.process.run('git init', cwd=self.gitdir)
+        bb.process.run('git config user.email "you@example.com"', cwd=self.gitdir)
+        bb.process.run('git config user.name "Your Name"', cwd=self.gitdir)
 
     def assertRefs(self, expected_refs):
         actual_refs = self.git(['for-each-ref', '--format=%(refname)']).splitlines()
@@ -1456,6 +1521,8 @@ class GitShallowTest(FetcherTest):
 
         bb.utils.mkdirhier(self.srcdir)
         self.git('init', cwd=self.srcdir)
+        self.git('config user.email "you@example.com"', cwd=self.srcdir)
+        self.git('config user.name "Your Name"', cwd=self.srcdir)
         self.d.setVar('WORKDIR', self.tempdir)
         self.d.setVar('S', self.gitdir)
         self.d.delVar('PREMIRRORS')
@@ -1537,6 +1604,7 @@ class GitShallowTest(FetcherTest):
 
         # fetch and unpack, from the shallow tarball
         bb.utils.remove(self.gitdir, recurse=True)
+        bb.process.run('chmod u+w -R "%s"' % ud.clonedir)
         bb.utils.remove(ud.clonedir, recurse=True)
         bb.utils.remove(ud.clonedir.replace('gitsource', 'gitsubmodule'), recurse=True)
 
@@ -1689,6 +1757,8 @@ class GitShallowTest(FetcherTest):
         smdir = os.path.join(self.tempdir, 'gitsubmodule')
         bb.utils.mkdirhier(smdir)
         self.git('init', cwd=smdir)
+        self.git('config user.email "you@example.com"', cwd=smdir)
+        self.git('config user.name "Your Name"', cwd=smdir)
         # Make this look like it was cloned from a remote...
         self.git('config --add remote.origin.url "%s"' % smdir, cwd=smdir)
         self.git('config --add remote.origin.fetch "+refs/heads/*:refs/remotes/origin/*"', cwd=smdir)
@@ -1719,6 +1789,8 @@ class GitShallowTest(FetcherTest):
         smdir = os.path.join(self.tempdir, 'gitsubmodule')
         bb.utils.mkdirhier(smdir)
         self.git('init', cwd=smdir)
+        self.git('config user.email "you@example.com"', cwd=smdir)
+        self.git('config user.name "Your Name"', cwd=smdir)
         # Make this look like it was cloned from a remote...
         self.git('config --add remote.origin.url "%s"' % smdir, cwd=smdir)
         self.git('config --add remote.origin.fetch "+refs/heads/*:refs/remotes/origin/*"', cwd=smdir)
@@ -1761,8 +1833,8 @@ class GitShallowTest(FetcherTest):
             self.git('annex init', cwd=self.srcdir)
             open(os.path.join(self.srcdir, 'c'), 'w').close()
             self.git('annex add c', cwd=self.srcdir)
-            self.git('commit -m annex-c -a', cwd=self.srcdir)
-            bb.process.run('chmod u+w -R %s' % os.path.join(self.srcdir, '.git', 'annex'))
+            self.git('commit --author "Foo Bar <foo@bar>" -m annex-c -a', cwd=self.srcdir)
+            bb.process.run('chmod u+w -R %s' % self.srcdir)
 
             uri = 'gitannex://%s;protocol=file;subdir=${S}' % self.srcdir
             fetcher, ud = self.fetch_shallow(uri)
@@ -1976,7 +2048,7 @@ class GitShallowTest(FetcherTest):
 
     @skipIfNoNetwork()
     def test_bitbake(self):
-        self.git('remote add --mirror=fetch origin git://github.com/openembedded/bitbake', cwd=self.srcdir)
+        self.git('remote add --mirror=fetch origin https://github.com/openembedded/bitbake', cwd=self.srcdir)
         self.git('config core.bare true', cwd=self.srcdir)
         self.git('fetch', cwd=self.srcdir)
 
@@ -2037,6 +2109,8 @@ class GitLfsTest(FetcherTest):
 
         bb.utils.mkdirhier(self.srcdir)
         self.git('init', cwd=self.srcdir)
+        self.git('config user.email "you@example.com"', cwd=self.srcdir)
+        self.git('config user.name "Your Name"', cwd=self.srcdir)
         with open(os.path.join(self.srcdir, '.gitattributes'), 'wt') as attrs:
             attrs.write('*.mp3 filter=lfs -text')
         self.git(['add', '.gitattributes'], cwd=self.srcdir)
@@ -2051,13 +2125,14 @@ class GitLfsTest(FetcherTest):
             cwd = self.gitdir
         return bb.process.run(cmd, cwd=cwd)[0]
 
-    def fetch(self, uri=None):
+    def fetch(self, uri=None, download=True):
         uris = self.d.getVar('SRC_URI').split()
         uri = uris[0]
         d = self.d
 
         fetcher = bb.fetch2.Fetch(uris, d)
-        fetcher.download()
+        if download:
+            fetcher.download()
         ud = fetcher.ud[uri]
         return fetcher, ud
 
@@ -2067,16 +2142,21 @@ class GitLfsTest(FetcherTest):
         uri = 'git://%s;protocol=file;subdir=${S};lfs=1' % self.srcdir
         self.d.setVar('SRC_URI', uri)
 
-        fetcher, ud = self.fetch()
+        # Careful: suppress initial attempt at downloading until
+        # we know whether git-lfs is installed.
+        fetcher, ud = self.fetch(uri=None, download=False)
         self.assertIsNotNone(ud.method._find_git_lfs)
 
-        # If git-lfs can be found, the unpack should be successful
-        ud.method._find_git_lfs = lambda d: True
-        shutil.rmtree(self.gitdir, ignore_errors=True)
-        fetcher.unpack(self.d.getVar('WORKDIR'))
+        # If git-lfs can be found, the unpack should be successful. Only
+        # attempt this with the real live copy of git-lfs installed.
+        if ud.method._find_git_lfs(self.d):
+            fetcher.download()
+            shutil.rmtree(self.gitdir, ignore_errors=True)
+            fetcher.unpack(self.d.getVar('WORKDIR'))
 
         # If git-lfs cannot be found, the unpack should throw an error
         with self.assertRaises(bb.fetch2.FetchError):
+            fetcher.download()
             ud.method._find_git_lfs = lambda d: False
             shutil.rmtree(self.gitdir, ignore_errors=True)
             fetcher.unpack(self.d.getVar('WORKDIR'))
@@ -2087,10 +2167,16 @@ class GitLfsTest(FetcherTest):
         uri = 'git://%s;protocol=file;subdir=${S};lfs=0' % self.srcdir
         self.d.setVar('SRC_URI', uri)
 
+        # In contrast to test_lfs_enabled(), allow the implicit download
+        # done by self.fetch() to occur here. The point of this test case
+        # is to verify that the fetcher can survive even if the source
+        # repository has Git LFS usage configured.
         fetcher, ud = self.fetch()
         self.assertIsNotNone(ud.method._find_git_lfs)
 
-        # If git-lfs can be found, the unpack should be successful
+        # If git-lfs can be found, the unpack should be successful. A
+        # live copy of git-lfs is not required for this case, so
+        # unconditionally forge its presence.
         ud.method._find_git_lfs = lambda d: True
         shutil.rmtree(self.gitdir, ignore_errors=True)
         fetcher.unpack(self.d.getVar('WORKDIR'))

bitbake/lib/bb/tests/runqueue-tests/recipes/f1.bb

@@ -1 +1 @@
-do_install[mcdepends] = "mc:mc1:mc2:a1:do_build"
+do_install[mcdepends] = "mc:mc-1:mc_2:a1:do_build"

bitbake/lib/bb/tests/runqueue-tests/recipes/fails-mc/fails-mc1.bb

@@ -1,5 +1,5 @@
 python () {
-    if d.getVar("BB_CURRENT_MC") == "mc1":
-        bb.fatal("Multiconfig is mc1")
+    if d.getVar("BB_CURRENT_MC") == "mc-1":
+        bb.fatal("Multiconfig is mc-1")
 }
 

bitbake/lib/bb/tests/runqueue-tests/recipes/fails-mc/fails-mc2.bb

@@ -1,4 +1,4 @@
 python () {
-    if d.getVar("BB_CURRENT_MC") == "mc2":
-        bb.fatal("Multiconfig is mc2")
+    if d.getVar("BB_CURRENT_MC") == "mc_2":
+        bb.fatal("Multiconfig is mc_2")
 }

bitbake/lib/bb/tests/runqueue.py

@@ -216,66 +216,66 @@ class RunQueueTests(unittest.TestCase):
     def test_multiconfig_setscene_optimise(self):
         with tempfile.TemporaryDirectory(prefix="runqueuetest") as tempdir:
             extraenv = {
-                "BBMULTICONFIG" : "mc1 mc2",
+                "BBMULTICONFIG" : "mc-1 mc_2",
                 "BB_SIGNATURE_HANDLER" : "basic"
             }
-            cmd = ["bitbake", "b1", "mc:mc1:b1", "mc:mc2:b1"]
+            cmd = ["bitbake", "b1", "mc:mc-1:b1", "mc:mc_2:b1"]
             setscenetasks = ['package_write_ipk_setscene', 'package_write_rpm_setscene', 'packagedata_setscene',
                              'populate_sysroot_setscene', 'package_qa_setscene']
             sstatevalid = ""
             tasks = self.run_bitbakecmd(cmd, tempdir, sstatevalid, extraenv=extraenv)
             expected = ['a1:' + x for x in self.alltasks] + ['b1:' + x for x in self.alltasks] + \
-                       ['mc1:b1:' + x for x in setscenetasks] + ['mc1:a1:' + x for x in setscenetasks] + \
-                       ['mc2:b1:' + x for x in setscenetasks] + ['mc2:a1:' + x for x in setscenetasks] + \
-                       ['mc1:b1:build', 'mc2:b1:build']
-            for x in ['mc1:a1:package_qa_setscene', 'mc2:a1:package_qa_setscene', 'a1:build', 'a1:package_qa']:
+                       ['mc-1:b1:' + x for x in setscenetasks] + ['mc-1:a1:' + x for x in setscenetasks] + \
+                       ['mc_2:b1:' + x for x in setscenetasks] + ['mc_2:a1:' + x for x in setscenetasks] + \
+                       ['mc-1:b1:build', 'mc_2:b1:build']
+            for x in ['mc-1:a1:package_qa_setscene', 'mc_2:a1:package_qa_setscene', 'a1:build', 'a1:package_qa']:
                 expected.remove(x)
             self.assertEqual(set(tasks), set(expected))
 
     def test_multiconfig_bbmask(self):
         # This test validates that multiconfigs can independently mask off
         # recipes they do not want with BBMASK. It works by having recipes
-        # that will fail to parse for mc1 and mc2, then making each multiconfig
+        # that will fail to parse for mc-1 and mc_2, then making each multiconfig
         # build the one that does parse. This ensures that the recipes are in
         # each multiconfigs BBFILES, but each is masking only the one that
         # doesn't parse
         with tempfile.TemporaryDirectory(prefix="runqueuetest") as tempdir:
             extraenv = {
-                "BBMULTICONFIG" : "mc1 mc2",
+                "BBMULTICONFIG" : "mc-1 mc_2",
                 "BB_SIGNATURE_HANDLER" : "basic",
                 "EXTRA_BBFILES": "${COREBASE}/recipes/fails-mc/*.bb",
             }
-            cmd = ["bitbake", "mc:mc1:fails-mc2", "mc:mc2:fails-mc1"]
+            cmd = ["bitbake", "mc:mc-1:fails-mc2", "mc:mc_2:fails-mc1"]
             self.run_bitbakecmd(cmd, tempdir, "", extraenv=extraenv)
 
     def test_multiconfig_mcdepends(self):
         with tempfile.TemporaryDirectory(prefix="runqueuetest") as tempdir:
             extraenv = {
-                "BBMULTICONFIG" : "mc1 mc2",
+                "BBMULTICONFIG" : "mc-1 mc_2",
                 "BB_SIGNATURE_HANDLER" : "TestMulticonfigDepends",
                 "EXTRA_BBFILES": "${COREBASE}/recipes/fails-mc/*.bb",
             }
-            tasks = self.run_bitbakecmd(["bitbake", "mc:mc1:f1"], tempdir, "", extraenv=extraenv, cleanup=True)
-            expected = ["mc1:f1:%s" % t for t in self.alltasks] + \
-                       ["mc2:a1:%s" % t for t in self.alltasks]
+            tasks = self.run_bitbakecmd(["bitbake", "mc:mc-1:f1"], tempdir, "", extraenv=extraenv, cleanup=True)
+            expected = ["mc-1:f1:%s" % t for t in self.alltasks] + \
+                       ["mc_2:a1:%s" % t for t in self.alltasks]
             self.assertEqual(set(tasks), set(expected))
 
             # A rebuild does nothing
-            tasks = self.run_bitbakecmd(["bitbake", "mc:mc1:f1"], tempdir, "", extraenv=extraenv, cleanup=True)
+            tasks = self.run_bitbakecmd(["bitbake", "mc:mc-1:f1"], tempdir, "", extraenv=extraenv, cleanup=True)
             self.assertEqual(set(tasks), set())
 
             # Test that a signature change in the dependent task causes
             # mcdepends to rebuild
-            tasks = self.run_bitbakecmd(["bitbake", "mc:mc2:a1", "-c", "compile", "-f"], tempdir, "", extraenv=extraenv, cleanup=True)
-            expected = ["mc2:a1:compile"]
+            tasks = self.run_bitbakecmd(["bitbake", "mc:mc_2:a1", "-c", "compile", "-f"], tempdir, "", extraenv=extraenv, cleanup=True)
+            expected = ["mc_2:a1:compile"]
             self.assertEqual(set(tasks), set(expected))
 
             rerun_tasks = self.alltasks[:]
             for x in ("fetch", "unpack", "patch", "prepare_recipe_sysroot", "configure", "compile"):
                 rerun_tasks.remove(x)
-            tasks = self.run_bitbakecmd(["bitbake", "mc:mc1:f1"], tempdir, "", extraenv=extraenv, cleanup=True)
-            expected = ["mc1:f1:%s" % t for t in rerun_tasks] + \
-                       ["mc2:a1:%s" % t for t in rerun_tasks]
+            tasks = self.run_bitbakecmd(["bitbake", "mc:mc-1:f1"], tempdir, "", extraenv=extraenv, cleanup=True)
+            expected = ["mc-1:f1:%s" % t for t in rerun_tasks] + \
+                       ["mc_2:a1:%s" % t for t in rerun_tasks]
             self.assertEqual(set(tasks), set(expected))
 
     @unittest.skipIf(sys.version_info < (3, 5, 0), 'Python 3.5 or later required')
@@ -361,7 +361,7 @@ class RunQueueTests(unittest.TestCase):
 
     def shutdown(self, tempdir):
         # Wait for the hashserve socket to disappear else we'll see races with the tempdir cleanup
-        while os.path.exists(tempdir + "/hashserve.sock"):
+        while (os.path.exists(tempdir + "/hashserve.sock") or os.path.exists(tempdir + "cache/hashserv.db-wal")):
             time.sleep(0.5)
 
 

bitbake/lib/bb/tinfoil.py

@@ -440,7 +440,7 @@ class Tinfoil:
         to initialise Tinfoil and use it with config_only=True first and
         then conditionally call this function to parse recipes later.
         """
-        config_params = TinfoilConfigParameters(config_only=False)
+        config_params = TinfoilConfigParameters(config_only=False, quiet=self.quiet)
         self.run_actions(config_params)
         self.recipes_parsed = True
 

bitbake/lib/bb/ui/buildinfohelper.py

@@ -148,14 +148,14 @@ class ORMWrapper(object):
         buildrequest = None
         if brbe is not None:
             # Toaster-triggered build
-            logger.debug(1, "buildinfohelper: brbe is %s" % brbe)
+            logger.debug("buildinfohelper: brbe is %s" % brbe)
             br, _ = brbe.split(":")
             buildrequest = BuildRequest.objects.get(pk=br)
             prj = buildrequest.project
         else:
             # CLI build
             prj = Project.objects.get_or_create_default_project()
-            logger.debug(1, "buildinfohelper: project is not specified, defaulting to %s" % prj)
+            logger.debug("buildinfohelper: project is not specified, defaulting to %s" % prj)
 
         if buildrequest is not None:
             # reuse existing Build object
@@ -171,7 +171,7 @@ class ORMWrapper(object):
                 completed_on=now,
                 build_name='')
 
-        logger.debug(1, "buildinfohelper: build is created %s" % build)
+        logger.debug("buildinfohelper: build is created %s" % build)
 
         if buildrequest is not None:
             buildrequest.build = build
@@ -906,7 +906,7 @@ class BuildInfoHelper(object):
 
         self.project = None
 
-        logger.debug(1, "buildinfohelper: Build info helper inited %s" % vars(self))
+        logger.debug("buildinfohelper: Build info helper inited %s" % vars(self))
 
 
     ###################
@@ -1620,7 +1620,7 @@ class BuildInfoHelper(object):
             # if we have a backlog of events, do our best to save them here
             if len(self.internal_state['backlog']):
                 tempevent = self.internal_state['backlog'].pop()
-                logger.debug(1, "buildinfohelper: Saving stored event %s "
+                logger.debug("buildinfohelper: Saving stored event %s "
                              % tempevent)
                 self.store_log_event(tempevent,cli_backlog)
             else:

bitbake/lib/bb/ui/knotty.py

@@ -745,7 +745,7 @@ def main(server, eventHandler, params, tf = TerminalFilter):
                 continue
 
             if isinstance(event, bb.runqueue.sceneQueueTaskStarted):
-                logger.info("Running setscene task %d of %d (%s)" % (event.stats.completed + event.stats.active + event.stats.failed + 1, event.stats.total, event.taskstring))
+                logger.info("Running setscene task %d of %d (%s)" % (event.stats.setscene_covered + event.stats.setscene_active + event.stats.setscene_notcovered + 1, event.stats.setscene_total, event.taskstring))
                 continue
 
             if isinstance(event, bb.runqueue.runQueueTaskStarted):

bitbake/lib/bb/ui/uihelper.py

@@ -49,8 +49,8 @@ class BBUIHelper:
             tid = event._fn + ":" + event._task
             removetid(event.pid, tid)
             self.failed_tasks.append( { 'title' : "%s %s" % (event._package, event._task)})
-        elif isinstance(event, bb.runqueue.runQueueTaskStarted):
-            self.tasknumber_current = event.stats.completed + event.stats.active + event.stats.failed + 1
+        elif isinstance(event, bb.runqueue.runQueueTaskStarted) or isinstance(event, bb.runqueue.sceneQueueTaskStarted):
+            self.tasknumber_current = event.stats.completed + event.stats.active + event.stats.failed + event.stats.setscene_active + 1
             self.tasknumber_total = event.stats.total
             self.needUpdate = True
         elif isinstance(event, bb.build.TaskProgress):

bitbake/lib/bb/utils.py

@@ -16,7 +16,8 @@ import bb.msg
 import multiprocessing
 import fcntl
 import importlib
-from importlib import machinery
+import importlib.machinery
+import importlib.util
 import itertools
 import subprocess
 import glob
@@ -129,6 +130,7 @@ def vercmp(ta, tb):
     return r
 
 def vercmp_string(a, b):
+    """ Split version strings and compare them """
     ta = split_version(a)
     tb = split_version(b)
     return vercmp(ta, tb)
@@ -247,6 +249,12 @@ def explode_dep_versions2(s, *, sort=True):
     return r
 
 def explode_dep_versions(s):
+    """
+    Take an RDEPENDS style string of format:
+    "DEPEND1 (optional version) DEPEND2 (optional version) ..."
+    skip null value and items appeared in dependancy string multiple times
+    and return a dictionary of dependencies and versions.
+    """
     r = explode_dep_versions2(s)
     for d in r:
         if not r[d]:
@@ -444,6 +452,10 @@ def lockfile(name, shared=False, retry=True, block=False):
     consider the possibility of sending a signal to the process to break
     out - at which point you want block=True rather than retry=True.
     """
+    if len(name) > 255:
+        root, ext = os.path.splitext(name)
+        name = root[:255 - len(ext)] + ext
+
     dirname = os.path.dirname(name)
     mkdirhier(dirname)
 
@@ -480,7 +492,7 @@ def lockfile(name, shared=False, retry=True, block=False):
                     return lf
             lf.close()
         except OSError as e:
-            if e.errno == errno.EACCES:
+            if e.errno == errno.EACCES or e.errno == errno.ENAMETOOLONG:
                 logger.error("Unable to acquire lock '%s', %s",
                              e.strerror, name)
                 sys.exit(1)
@@ -602,7 +614,7 @@ def filter_environment(good_vars):
     os.environ["LC_ALL"] = "en_US.UTF-8"
 
     if removed_vars:
-        logger.debug(1, "Removed the following variables from the environment: %s", ", ".join(removed_vars.keys()))
+        logger.debug("Removed the following variables from the environment: %s", ", ".join(removed_vars.keys()))
 
     return removed_vars
 
@@ -692,7 +704,7 @@ def remove(path, recurse=False, ionice=False):
                 raise
 
 def prunedir(topdir, ionice=False):
-    # Delete everything reachable from the directory named in 'topdir'.
+    """ Delete everything reachable from the directory named in 'topdir'. """
     # CAUTION:  This is dangerous!
     if _check_unsafe_delete_path(topdir):
         raise Exception('bb.utils.prunedir: called with dangerous path "%s", refusing to delete!' % topdir)
@@ -703,8 +715,10 @@ def prunedir(topdir, ionice=False):
 # but thats possibly insane and suffixes is probably going to be small
 #
 def prune_suffix(var, suffixes, d):
-    # See if var ends with any of the suffixes listed and
-    # remove it if found
+    """ 
+    See if var ends with any of the suffixes listed and
+    remove it if found 
+    """
     for suffix in suffixes:
         if suffix and var.endswith(suffix):
             return var[:-len(suffix)]
@@ -956,6 +970,10 @@ def umask(new_mask):
         os.umask(current_mask)
 
 def to_boolean(string, default=None):
+    """ 
+    Check input string and return boolean value True/False/None
+    depending upon the checks 
+    """
     if not string:
         return default
 
@@ -999,6 +1017,23 @@ def contains(variable, checkvalues, truevalue, falsevalue, d):
     return falsevalue
 
 def contains_any(variable, checkvalues, truevalue, falsevalue, d):
+    """Check if a variable contains any values specified.
+
+    Arguments:
+
+    variable -- the variable name. This will be fetched and expanded (using
+    d.getVar(variable)) and then split into a set().
+
+    checkvalues -- if this is a string it is split on whitespace into a set(),
+    otherwise coerced directly into a set().
+
+    truevalue -- the value to return if checkvalues is a subset of variable.
+
+    falsevalue -- the value to return if variable is empty or if checkvalues is
+    not a subset of variable.
+
+    d -- the data store.
+    """
     val = d.getVar(variable)
     if not val:
         return falsevalue
@@ -1560,8 +1595,8 @@ def set_process_name(name):
     except:
         pass
 
-# export common proxies variables from datastore to environment
 def export_proxies(d):
+    """ export common proxies variables from datastore to environment """
     import os
 
     variables = ['http_proxy', 'HTTP_PROXY', 'https_proxy', 'HTTPS_PROXY',
@@ -1583,12 +1618,14 @@ def export_proxies(d):
 
 def load_plugins(logger, plugins, pluginpath):
     def load_plugin(name):
-        logger.debug(1, 'Loading plugin %s' % name)
+        logger.debug('Loading plugin %s' % name)
         spec = importlib.machinery.PathFinder.find_spec(name, path=[pluginpath] )
         if spec:
-            return spec.loader.load_module()
+            mod = importlib.util.module_from_spec(spec)
+            spec.loader.exec_module(mod)
+            return mod
 
-    logger.debug(1, 'Loading plugins from %s...' % pluginpath)
+    logger.debug('Loading plugins from %s...' % pluginpath)
 
     expanded = (glob.glob(os.path.join(pluginpath, '*' + ext))
                 for ext in python_extensions)

bitbake/lib/bblayers/action.py

@@ -50,10 +50,10 @@ class ActionPlugin(LayerPlugin):
             if not (args.force or notadded):
                 try:
                     self.tinfoil.run_command('parseConfiguration')
-                except bb.tinfoil.TinfoilUIException:
+                except (bb.tinfoil.TinfoilUIException, bb.BBHandledException):
                     # Restore the back up copy of bblayers.conf
                     shutil.copy2(backup, bblayers_conf)
-                    bb.fatal("Parse failure with the specified layer added")
+                    bb.fatal("Parse failure with the specified layer added, aborting.")
                 else:
                     for item in notadded:
                         sys.stderr.write("Specified layer %s is already in BBLAYERS\n" % item)

bitbake/lib/bblayers/layerindex.py

@@ -79,7 +79,7 @@ class LayerIndexPlugin(ActionPlugin):
             branches = [args.branch]
         else:
             branches = (self.tinfoil.config_data.getVar('LAYERSERIES_CORENAMES') or 'master').split()
-        logger.debug(1, 'Trying branches: %s' % branches)
+        logger.debug('Trying branches: %s' % branches)
 
         ignore_layers = []
         if args.ignore:

bitbake/lib/bblayers/query.py

@@ -128,7 +128,7 @@ skipped recipes will also be listed, with a " (skipped)" suffix.
                     sys.exit(1)
 
         pkg_pn = self.tinfoil.cooker.recipecaches[mc].pkg_pn
-        (latest_versions, preferred_versions) = self.tinfoil.find_providers(mc)
+        (latest_versions, preferred_versions, required_versions) = self.tinfoil.find_providers(mc)
         allproviders = self.tinfoil.get_all_providers(mc)
 
         # Ensure we list skipped recipes

bitbake/lib/hashserv/__init__.py

@@ -3,6 +3,7 @@
 # SPDX-License-Identifier: GPL-2.0-only
 #
 
+import asyncio
 from contextlib import closing
 import re
 import sqlite3
@@ -21,6 +22,24 @@ ADDR_TYPE_TCP = 1
 # is necessary
 DEFAULT_MAX_CHUNK = 32 * 1024
 
+TABLE_DEFINITION = (
+    ("method", "TEXT NOT NULL"),
+    ("outhash", "TEXT NOT NULL"),
+    ("taskhash", "TEXT NOT NULL"),
+    ("unihash", "TEXT NOT NULL"),
+    ("created", "DATETIME"),
+
+    # Optional fields
+    ("owner", "TEXT"),
+    ("PN", "TEXT"),
+    ("PV", "TEXT"),
+    ("PR", "TEXT"),
+    ("task", "TEXT"),
+    ("outhash_siginfo", "TEXT"),
+)
+
+TABLE_COLUMNS = tuple(name for name, _ in TABLE_DEFINITION)
+
 def setup_database(database, sync=True):
     db = sqlite3.connect(database)
     db.row_factory = sqlite3.Row
@@ -29,23 +48,10 @@ def setup_database(database, sync=True):
         cursor.execute('''
             CREATE TABLE IF NOT EXISTS tasks_v2 (
                 id INTEGER PRIMARY KEY AUTOINCREMENT,
-                method TEXT NOT NULL,
-                outhash TEXT NOT NULL,
-                taskhash TEXT NOT NULL,
-                unihash TEXT NOT NULL,
-                created DATETIME,
-
-                -- Optional fields
-                owner TEXT,
-                PN TEXT,
-                PV TEXT,
-                PR TEXT,
-                task TEXT,
-                outhash_siginfo TEXT,
-
+                %s
                 UNIQUE(method, outhash, taskhash)
                 )
-            ''')
+            ''' % " ".join("%s %s," % (name, typ) for name, typ in TABLE_DEFINITION))
         cursor.execute('PRAGMA journal_mode = WAL')
         cursor.execute('PRAGMA synchronous = %s' % ('NORMAL' if sync else 'OFF'))
 
@@ -88,10 +94,10 @@ def chunkify(msg, max_chunk):
         yield "\n"
 
 
-def create_server(addr, dbname, *, sync=True):
+def create_server(addr, dbname, *, sync=True, upstream=None, read_only=False):
     from . import server
     db = setup_database(dbname, sync=sync)
-    s = server.Server(db)
+    s = server.Server(db, upstream=upstream, read_only=read_only)
 
     (typ, a) = parse_address(addr)
     if typ == ADDR_TYPE_UNIX:
@@ -113,3 +119,15 @@ def create_client(addr):
         c.connect_tcp(*a)
 
     return c
+
+async def create_async_client(addr):
+    from . import client
+    c = client.AsyncClient()
+
+    (typ, a) = parse_address(addr)
+    if typ == ADDR_TYPE_UNIX:
+        await c.connect_unix(*a)
+    else:
+        await c.connect_tcp(*a)
+
+    return c

bitbake/lib/hashserv/client.py

@@ -3,189 +3,231 @@
 # SPDX-License-Identifier: GPL-2.0-only
 #
 
+import asyncio
 import json
 import logging
 import socket
 import os
-from . import chunkify, DEFAULT_MAX_CHUNK
+from . import chunkify, DEFAULT_MAX_CHUNK, create_async_client
 
 
-logger = logging.getLogger('hashserv.client')
+logger = logging.getLogger("hashserv.client")
 
 
 class HashConnectionError(Exception):
     pass
 
 
-class Client(object):
+class AsyncClient(object):
     MODE_NORMAL = 0
     MODE_GET_STREAM = 1
 
     def __init__(self):
-        self._socket = None
         self.reader = None
         self.writer = None
         self.mode = self.MODE_NORMAL
         self.max_chunk = DEFAULT_MAX_CHUNK
 
-    def connect_tcp(self, address, port):
-        def connect_sock():
-            s = socket.create_connection((address, port))
-
-            s.setsockopt(socket.SOL_TCP, socket.TCP_NODELAY, 1)
-            s.setsockopt(socket.SOL_TCP, socket.TCP_QUICKACK, 1)
-            s.setsockopt(socket.SOL_SOCKET, socket.SO_KEEPALIVE, 1)
-            return s
+    async def connect_tcp(self, address, port):
+        async def connect_sock():
+            return await asyncio.open_connection(address, port)
 
         self._connect_sock = connect_sock
 
-    def connect_unix(self, path):
-        def connect_sock():
-            s = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
-            # AF_UNIX has path length issues so chdir here to workaround
-            cwd = os.getcwd()
-            try:
-                os.chdir(os.path.dirname(path))
-                s.connect(os.path.basename(path))
-            finally:
-                os.chdir(cwd)
-            return s
+    async def connect_unix(self, path):
+        async def connect_sock():
+            return await asyncio.open_unix_connection(path)
 
         self._connect_sock = connect_sock
 
-    def connect(self):
-        if self._socket is None:
-            self._socket = self._connect_sock()
+    async def connect(self):
+        if self.reader is None or self.writer is None:
+            (self.reader, self.writer) = await self._connect_sock()
 
-            self.reader = self._socket.makefile('r', encoding='utf-8')
-            self.writer = self._socket.makefile('w', encoding='utf-8')
+            self.writer.write("OEHASHEQUIV 1.1\n\n".encode("utf-8"))
+            await self.writer.drain()
 
-            self.writer.write('OEHASHEQUIV 1.1\n\n')
-            self.writer.flush()
-
-            # Restore mode if the socket is being re-created
             cur_mode = self.mode
             self.mode = self.MODE_NORMAL
-            self._set_mode(cur_mode)
+            await self._set_mode(cur_mode)
 
-        return self._socket
+    async def close(self):
+        self.reader = None
 
-    def close(self):
-        if self._socket is not None:
-            self._socket.close()
-            self._socket = None
-            self.reader = None
+        if self.writer is not None:
+            self.writer.close()
             self.writer = None
 
-    def _send_wrapper(self, proc):
+    async def _send_wrapper(self, proc):
         count = 0
         while True:
             try:
-                self.connect()
-                return proc()
-            except (OSError, HashConnectionError, json.JSONDecodeError, UnicodeDecodeError) as e:
-                logger.warning('Error talking to server: %s' % e)
+                await self.connect()
+                return await proc()
+            except (
+                OSError,
+                HashConnectionError,
+                json.JSONDecodeError,
+                UnicodeDecodeError,
+            ) as e:
+                logger.warning("Error talking to server: %s" % e)
                 if count >= 3:
                     if not isinstance(e, HashConnectionError):
                         raise HashConnectionError(str(e))
                     raise e
-                self.close()
+                await self.close()
                 count += 1
 
-    def send_message(self, msg):
-        def get_line():
-            line = self.reader.readline()
+    async def send_message(self, msg):
+        async def get_line():
+            line = await self.reader.readline()
             if not line:
-                raise HashConnectionError('Connection closed')
+                raise HashConnectionError("Connection closed")
 
-            if not line.endswith('\n'):
-                raise HashConnectionError('Bad message %r' % message)
+            line = line.decode("utf-8")
+
+            if not line.endswith("\n"):
+                raise HashConnectionError("Bad message %r" % message)
 
             return line
 
-        def proc():
+        async def proc():
             for c in chunkify(json.dumps(msg), self.max_chunk):
-                self.writer.write(c)
-            self.writer.flush()
+                self.writer.write(c.encode("utf-8"))
+            await self.writer.drain()
 
-            l = get_line()
+            l = await get_line()
 
             m = json.loads(l)
-            if 'chunk-stream' in m:
+            if m and "chunk-stream" in m:
                 lines = []
                 while True:
-                    l = get_line().rstrip('\n')
+                    l = (await get_line()).rstrip("\n")
                     if not l:
                         break
                     lines.append(l)
 
-                m = json.loads(''.join(lines))
+                m = json.loads("".join(lines))
 
             return m
 
-        return self._send_wrapper(proc)
+        return await self._send_wrapper(proc)
 
-    def send_stream(self, msg):
-        def proc():
-            self.writer.write("%s\n" % msg)
-            self.writer.flush()
-            l = self.reader.readline()
+    async def send_stream(self, msg):
+        async def proc():
+            self.writer.write(("%s\n" % msg).encode("utf-8"))
+            await self.writer.drain()
+            l = await self.reader.readline()
             if not l:
-                raise HashConnectionError('Connection closed')
-            return l.rstrip()
+                raise HashConnectionError("Connection closed")
+            return l.decode("utf-8").rstrip()
 
-        return self._send_wrapper(proc)
+        return await self._send_wrapper(proc)
 
-    def _set_mode(self, new_mode):
+    async def _set_mode(self, new_mode):
         if new_mode == self.MODE_NORMAL and self.mode == self.MODE_GET_STREAM:
-            r = self.send_stream('END')
-            if r != 'ok':
-                raise HashConnectionError('Bad response from server %r' % r)
+            r = await self.send_stream("END")
+            if r != "ok":
+                raise HashConnectionError("Bad response from server %r" % r)
         elif new_mode == self.MODE_GET_STREAM and self.mode == self.MODE_NORMAL:
-            r = self.send_message({'get-stream': None})
-            if r != 'ok':
-                raise HashConnectionError('Bad response from server %r' % r)
+            r = await self.send_message({"get-stream": None})
+            if r != "ok":
+                raise HashConnectionError("Bad response from server %r" % r)
         elif new_mode != self.mode:
-            raise Exception('Undefined mode transition %r -> %r' % (self.mode, new_mode))
+            raise Exception(
+                "Undefined mode transition %r -> %r" % (self.mode, new_mode)
+            )
 
         self.mode = new_mode
 
-    def get_unihash(self, method, taskhash):
-        self._set_mode(self.MODE_GET_STREAM)
-        r = self.send_stream('%s %s' % (method, taskhash))
+    async def get_unihash(self, method, taskhash):
+        await self._set_mode(self.MODE_GET_STREAM)
+        r = await self.send_stream("%s %s" % (method, taskhash))
         if not r:
             return None
         return r
 
-    def report_unihash(self, taskhash, method, outhash, unihash, extra={}):
-        self._set_mode(self.MODE_NORMAL)
+    async def report_unihash(self, taskhash, method, outhash, unihash, extra={}):
+        await self._set_mode(self.MODE_NORMAL)
         m = extra.copy()
-        m['taskhash'] = taskhash
-        m['method'] = method
-        m['outhash'] = outhash
-        m['unihash'] = unihash
-        return self.send_message({'report': m})
+        m["taskhash"] = taskhash
+        m["method"] = method
+        m["outhash"] = outhash
+        m["unihash"] = unihash
+        return await self.send_message({"report": m})
 
-    def report_unihash_equiv(self, taskhash, method, unihash, extra={}):
-        self._set_mode(self.MODE_NORMAL)
+    async def report_unihash_equiv(self, taskhash, method, unihash, extra={}):
+        await self._set_mode(self.MODE_NORMAL)
         m = extra.copy()
-        m['taskhash'] = taskhash
-        m['method'] = method
-        m['unihash'] = unihash
-        return self.send_message({'report-equiv': m})
+        m["taskhash"] = taskhash
+        m["method"] = method
+        m["unihash"] = unihash
+        return await self.send_message({"report-equiv": m})
 
-    def get_taskhash(self, method, taskhash, all_properties=False):
-        self._set_mode(self.MODE_NORMAL)
-        return self.send_message({'get': {
-            'taskhash': taskhash,
-            'method': method,
-            'all': all_properties
-        }})
+    async def get_taskhash(self, method, taskhash, all_properties=False):
+        await self._set_mode(self.MODE_NORMAL)
+        return await self.send_message(
+            {"get": {"taskhash": taskhash, "method": method, "all": all_properties}}
+        )
 
-    def get_stats(self):
-        self._set_mode(self.MODE_NORMAL)
-        return self.send_message({'get-stats': None})
+    async def get_outhash(self, method, outhash, taskhash):
+        await self._set_mode(self.MODE_NORMAL)
+        return await self.send_message(
+            {"get-outhash": {"outhash": outhash, "taskhash": taskhash, "method": method}}
+        )
 
-    def reset_stats(self):
-        self._set_mode(self.MODE_NORMAL)
-        return self.send_message({'reset-stats': None})
+    async def get_stats(self):
+        await self._set_mode(self.MODE_NORMAL)
+        return await self.send_message({"get-stats": None})
+
+    async def reset_stats(self):
+        await self._set_mode(self.MODE_NORMAL)
+        return await self.send_message({"reset-stats": None})
+
+    async def backfill_wait(self):
+        await self._set_mode(self.MODE_NORMAL)
+        return (await self.send_message({"backfill-wait": None}))["tasks"]
+
+
+class Client(object):
+    def __init__(self):
+        self.client = AsyncClient()
+        self.loop = asyncio.new_event_loop()
+
+        for call in (
+            "connect_tcp",
+            "close",
+            "get_unihash",
+            "report_unihash",
+            "report_unihash_equiv",
+            "get_taskhash",
+            "get_stats",
+            "reset_stats",
+            "backfill_wait",
+        ):
+            downcall = getattr(self.client, call)
+            setattr(self, call, self._get_downcall_wrapper(downcall))
+
+    def _get_downcall_wrapper(self, downcall):
+        def wrapper(*args, **kwargs):
+            return self.loop.run_until_complete(downcall(*args, **kwargs))
+
+        return wrapper
+
+    def connect_unix(self, path):
+        # AF_UNIX has path length issues so chdir here to workaround
+        cwd = os.getcwd()
+        try:
+            os.chdir(os.path.dirname(path))
+            self.loop.run_until_complete(self.client.connect_unix(os.path.basename(path)))
+            self.loop.run_until_complete(self.client.connect())
+        finally:
+            os.chdir(cwd)
+
+    @property
+    def max_chunk(self):
+        return self.client.max_chunk
+
+    @max_chunk.setter
+    def max_chunk(self, value):
+        self.client.max_chunk = value

bitbake/lib/hashserv/server.py

@@ -3,7 +3,7 @@
 # SPDX-License-Identifier: GPL-2.0-only
 #
 
-from contextlib import closing
+from contextlib import closing, contextmanager
 from datetime import datetime
 import asyncio
 import json
@@ -12,8 +12,9 @@ import math
 import os
 import signal
 import socket
+import sys
 import time
-from . import chunkify, DEFAULT_MAX_CHUNK
+from . import chunkify, DEFAULT_MAX_CHUNK, create_async_client, TABLE_COLUMNS
 
 logger = logging.getLogger('hashserv.server')
 
@@ -111,29 +112,95 @@ class Stats(object):
 class ClientError(Exception):
     pass
 
+class ServerError(Exception):
+    pass
+
+def insert_task(cursor, data, ignore=False):
+    keys = sorted(data.keys())
+    query = '''INSERT%s INTO tasks_v2 (%s) VALUES (%s)''' % (
+        " OR IGNORE" if ignore else "",
+        ', '.join(keys),
+        ', '.join(':' + k for k in keys))
+    cursor.execute(query, data)
+
+async def copy_from_upstream(client, db, method, taskhash):
+    d = await client.get_taskhash(method, taskhash, True)
+    if d is not None:
+        # Filter out unknown columns
+        d = {k: v for k, v in d.items() if k in TABLE_COLUMNS}
+        keys = sorted(d.keys())
+
+        with closing(db.cursor()) as cursor:
+            insert_task(cursor, d)
+            db.commit()
+
+    return d
+
+async def copy_outhash_from_upstream(client, db, method, outhash, taskhash):
+    d = await client.get_outhash(method, outhash, taskhash)
+    if d is not None:
+        # Filter out unknown columns
+        d = {k: v for k, v in d.items() if k in TABLE_COLUMNS}
+        keys = sorted(d.keys())
+
+        with closing(db.cursor()) as cursor:
+            insert_task(cursor, d)
+            db.commit()
+
+    return d
+
 class ServerClient(object):
     FAST_QUERY = 'SELECT taskhash, method, unihash FROM tasks_v2 WHERE method=:method AND taskhash=:taskhash ORDER BY created ASC LIMIT 1'
     ALL_QUERY =  'SELECT *                         FROM tasks_v2 WHERE method=:method AND taskhash=:taskhash ORDER BY created ASC LIMIT 1'
+    OUTHASH_QUERY = '''
+        -- Find tasks with a matching outhash (that is, tasks that
+        -- are equivalent)
+        SELECT * FROM tasks_v2 WHERE method=:method AND outhash=:outhash
 
-    def __init__(self, reader, writer, db, request_stats):
+        -- If there is an exact match on the taskhash, return it.
+        -- Otherwise return the oldest matching outhash of any
+        -- taskhash
+        ORDER BY CASE WHEN taskhash=:taskhash THEN 1 ELSE 2 END,
+            created ASC
+
+        -- Only return one row
+        LIMIT 1
+        '''
+
+    def __init__(self, reader, writer, db, request_stats, backfill_queue, upstream, read_only):
         self.reader = reader
         self.writer = writer
         self.db = db
         self.request_stats = request_stats
         self.max_chunk = DEFAULT_MAX_CHUNK
+        self.backfill_queue = backfill_queue
+        self.upstream = upstream
 
         self.handlers = {
             'get': self.handle_get,
-            'report': self.handle_report,
-            'report-equiv': self.handle_equivreport,
+            'get-outhash': self.handle_get_outhash,
             'get-stream': self.handle_get_stream,
             'get-stats': self.handle_get_stats,
-            'reset-stats': self.handle_reset_stats,
             'chunk-stream': self.handle_chunk,
         }
 
+        if not read_only:
+            self.handlers.update({
+                'report': self.handle_report,
+                'report-equiv': self.handle_equivreport,
+                'reset-stats': self.handle_reset_stats,
+                'backfill-wait': self.handle_backfill_wait,
+            })
+
     async def process_requests(self):
+        if self.upstream is not None:
+            self.upstream_client = await create_async_client(self.upstream)
+        else:
+            self.upstream_client = None
+
         try:
+
+
             self.addr = self.writer.get_extra_info('peername')
             logger.debug('Client %r connected' % (self.addr,))
 
@@ -171,6 +238,9 @@ class ServerClient(object):
         except ClientError as e:
             logger.error(str(e))
         finally:
+            if self.upstream_client is not None:
+                await self.upstream_client.close()
+
             self.writer.close()
 
     async def dispatch_message(self, msg):
@@ -239,15 +309,34 @@ class ServerClient(object):
         if row is not None:
             logger.debug('Found equivalent task %s -> %s', (row['taskhash'], row['unihash']))
             d = {k: row[k] for k in row.keys()}
-
-            self.write_message(d)
+        elif self.upstream_client is not None:
+            d = await copy_from_upstream(self.upstream_client, self.db, method, taskhash)
         else:
-            self.write_message(None)
+            d = None
+
+        self.write_message(d)
+
+    async def handle_get_outhash(self, request):
+        with closing(self.db.cursor()) as cursor:
+            cursor.execute(self.OUTHASH_QUERY,
+                           {k: request[k] for k in ('method', 'outhash', 'taskhash')})
+
+            row = cursor.fetchone()
+
+        if row is not None:
+            logger.debug('Found equivalent outhash %s -> %s', (row['outhash'], row['unihash']))
+            d = {k: row[k] for k in row.keys()}
+        else:
+            d = None
+
+        self.write_message(d)
 
     async def handle_get_stream(self, request):
         self.write_message('ok')
 
         while True:
+            upstream = None
+
             l = await self.reader.readline()
             if not l:
                 return
@@ -272,6 +361,12 @@ class ServerClient(object):
                 if row is not None:
                     msg = ('%s\n' % row['unihash']).encode('utf-8')
                     #logger.debug('Found equivalent task %s -> %s', (row['taskhash'], row['unihash']))
+                elif self.upstream_client is not None:
+                    upstream = await self.upstream_client.get_unihash(method, taskhash)
+                    if upstream:
+                        msg = ("%s\n" % upstream).encode("utf-8")
+                    else:
+                        msg = "\n".encode("utf-8")
                 else:
                     msg = '\n'.encode('utf-8')
 
@@ -282,25 +377,26 @@ class ServerClient(object):
 
             await self.writer.drain()
 
+            # Post to the backfill queue after writing the result to minimize
+            # the turn around time on a request
+            if upstream is not None:
+                await self.backfill_queue.put((method, taskhash))
+
     async def handle_report(self, data):
         with closing(self.db.cursor()) as cursor:
-            cursor.execute('''
-                -- Find tasks with a matching outhash (that is, tasks that
-                -- are equivalent)
-                SELECT taskhash, method, unihash FROM tasks_v2 WHERE method=:method AND outhash=:outhash
-
-                -- If there is an exact match on the taskhash, return it.
-                -- Otherwise return the oldest matching outhash of any
-                -- taskhash
-                ORDER BY CASE WHEN taskhash=:taskhash THEN 1 ELSE 2 END,
-                    created ASC
-
-                -- Only return one row
-                LIMIT 1
-                ''', {k: data[k] for k in ('method', 'outhash', 'taskhash')})
+            cursor.execute(self.OUTHASH_QUERY,
+                           {k: data[k] for k in ('method', 'outhash', 'taskhash')})
 
             row = cursor.fetchone()
 
+            if row is None and self.upstream_client:
+                # Try upstream
+                row = await copy_outhash_from_upstream(self.upstream_client,
+                                                       self.db,
+                                                       data['method'],
+                                                       data['outhash'],
+                                                       data['taskhash'])
+
             # If no matching outhash was found, or one *was* found but it
             # wasn't an exact match on the taskhash, a new entry for this
             # taskhash should be added
@@ -324,11 +420,7 @@ class ServerClient(object):
                     if k in data:
                         insert_data[k] = data[k]
 
-                cursor.execute('''INSERT INTO tasks_v2 (%s) VALUES (%s)''' % (
-                    ', '.join(sorted(insert_data.keys())),
-                    ', '.join(':' + k for k in sorted(insert_data.keys()))),
-                    insert_data)
-
+                insert_task(cursor, insert_data)
                 self.db.commit()
 
                 logger.info('Adding taskhash %s with unihash %s',
@@ -358,11 +450,7 @@ class ServerClient(object):
                 if k in data:
                     insert_data[k] = data[k]
 
-            cursor.execute('''INSERT OR IGNORE INTO tasks_v2 (%s) VALUES (%s)''' % (
-                ', '.join(sorted(insert_data.keys())),
-                ', '.join(':' + k for k in sorted(insert_data.keys()))),
-                insert_data)
-
+            insert_task(cursor, insert_data, ignore=True)
             self.db.commit()
 
             # Fetch the unihash that will be reported for the taskhash. If the
@@ -394,6 +482,13 @@ class ServerClient(object):
         self.request_stats.reset()
         self.write_message(d)
 
+    async def handle_backfill_wait(self, request):
+        d = {
+            'tasks': self.backfill_queue.qsize(),
+        }
+        await self.backfill_queue.join()
+        self.write_message(d)
+
     def query_equivalent(self, method, taskhash, query):
         # This is part of the inner loop and must be as fast as possible
         try:
@@ -405,7 +500,10 @@ class ServerClient(object):
 
 
 class Server(object):
-    def __init__(self, db, loop=None):
+    def __init__(self, db, loop=None, upstream=None, read_only=False):
+        if upstream and read_only:
+            raise ServerError("Read-only hashserv cannot pull from an upstream server")
+
         self.request_stats = Stats()
         self.db = db
 
@@ -416,11 +514,14 @@ class Server(object):
             self.loop = loop
             self.close_loop = False
 
+        self.upstream = upstream
+        self.read_only = read_only
+
         self._cleanup_socket = None
 
     def start_tcp_server(self, host, port):
         self.server = self.loop.run_until_complete(
-            asyncio.start_server(self.handle_client, host, port, loop=self.loop)
+            asyncio.start_server(self.handle_client, host, port)
         )
 
         for s in self.server.sockets:
@@ -445,7 +546,7 @@ class Server(object):
             # Work around path length limits in AF_UNIX
             os.chdir(os.path.dirname(path))
             self.server = self.loop.run_until_complete(
-                asyncio.start_unix_server(self.handle_client, os.path.basename(path), loop=self.loop)
+                asyncio.start_unix_server(self.handle_client, os.path.basename(path))
             )
         finally:
             os.chdir(cwd)
@@ -458,7 +559,7 @@ class Server(object):
     async def handle_client(self, reader, writer):
         # writer.transport.set_write_buffer_limits(0)
         try:
-            client = ServerClient(reader, writer, self.db, self.request_stats)
+            client = ServerClient(reader, writer, self.db, self.request_stats, self.backfill_queue, self.upstream, self.read_only)
             await client.process_requests()
         except Exception as e:
             import traceback
@@ -467,23 +568,60 @@ class Server(object):
             writer.close()
         logger.info('Client disconnected')
 
+    @contextmanager
+    def _backfill_worker(self):
+        async def backfill_worker_task():
+            client = await create_async_client(self.upstream)
+            try:
+                while True:
+                    item = await self.backfill_queue.get()
+                    if item is None:
+                        self.backfill_queue.task_done()
+                        break
+                    method, taskhash = item
+                    await copy_from_upstream(client, self.db, method, taskhash)
+                    self.backfill_queue.task_done()
+            finally:
+                await client.close()
+
+        async def join_worker(worker):
+            await self.backfill_queue.put(None)
+            await worker
+
+        if self.upstream is not None:
+            worker = asyncio.ensure_future(backfill_worker_task())
+            try:
+                yield
+            finally:
+                self.loop.run_until_complete(join_worker(worker))
+        else:
+            yield
+
     def serve_forever(self):
         def signal_handler():
             self.loop.stop()
 
-        self.loop.add_signal_handler(signal.SIGTERM, signal_handler)
-
+        asyncio.set_event_loop(self.loop)
         try:
-            self.loop.run_forever()
-        except KeyboardInterrupt:
-            pass
+            self.backfill_queue = asyncio.Queue()
 
-        self.server.close()
-        self.loop.run_until_complete(self.server.wait_closed())
-        logger.info('Server shutting down')
+            self.loop.add_signal_handler(signal.SIGTERM, signal_handler)
 
-        if self.close_loop:
-            self.loop.close()
+            with self._backfill_worker():
+                try:
+                    self.loop.run_forever()
+                except KeyboardInterrupt:
+                    pass
 
-        if self._cleanup_socket is not None:
-            self._cleanup_socket()
+                self.server.close()
+
+            self.loop.run_until_complete(self.server.wait_closed())
+            logger.info('Server shutting down')
+        finally:
+            if self.close_loop:
+                if sys.version_info >= (3, 6):
+                    self.loop.run_until_complete(self.loop.shutdown_asyncgens())
+                self.loop.close()
+
+            if self._cleanup_socket is not None:
+                self._cleanup_socket()

bitbake/lib/hashserv/tests.py

@@ -6,6 +6,7 @@
 #
 
 from . import create_server, create_client
+from .client import HashConnectionError
 import hashlib
 import logging
 import multiprocessing
@@ -16,44 +17,68 @@ import threading
 import unittest
 import socket
 
+def _run_server(server, idx):
+    # logging.basicConfig(level=logging.DEBUG, filename='bbhashserv.log', filemode='w',
+    #                     format='%(levelname)s %(filename)s:%(lineno)d %(message)s')
+    sys.stdout = open('bbhashserv-%d.log' % idx, 'w')
+    sys.stderr = sys.stdout
+    server.serve_forever()
 
-class TestHashEquivalenceServer(object):
+
+class HashEquivalenceTestSetup(object):
     METHOD = 'TestMethod'
 
-    def _run_server(self):
-        # logging.basicConfig(level=logging.DEBUG, filename='bbhashserv.log', filemode='w',
-        #                     format='%(levelname)s %(filename)s:%(lineno)d %(message)s')
-        self.server.serve_forever()
+    server_index = 0
+
+    def start_server(self, dbpath=None, upstream=None, read_only=False):
+        self.server_index += 1
+        if dbpath is None:
+            dbpath = os.path.join(self.temp_dir.name, "db%d.sqlite" % self.server_index)
+
+        def cleanup_thread(thread):
+            thread.terminate()
+            thread.join()
+
+        server = create_server(self.get_server_addr(self.server_index),
+                               dbpath,
+                               upstream=upstream,
+                               read_only=read_only)
+        server.dbpath = dbpath
+
+        server.thread = multiprocessing.Process(target=_run_server, args=(server, self.server_index))
+        server.thread.start()
+        self.addCleanup(cleanup_thread, server.thread)
+
+        def cleanup_client(client):
+            client.close()
+
+        client = create_client(server.address)
+        self.addCleanup(cleanup_client, client)
+
+        return (client, server)
 
     def setUp(self):
         if sys.version_info < (3, 5, 0):
             self.skipTest('Python 3.5 or later required')
 
         self.temp_dir = tempfile.TemporaryDirectory(prefix='bb-hashserv')
-        self.dbfile = os.path.join(self.temp_dir.name, 'db.sqlite')
+        self.addCleanup(self.temp_dir.cleanup)
 
-        self.server = create_server(self.get_server_addr(), self.dbfile)
-        self.server_thread = multiprocessing.Process(target=self._run_server)
-        self.server_thread.start()
-        self.client = create_client(self.server.address)
+        (self.client, self.server) = self.start_server()
 
-    def tearDown(self):
-        # Shutdown server
-        s = getattr(self, 'server', None)
-        if s is not None:
-            self.server_thread.terminate()
-            self.server_thread.join()
-        self.client.close()
-        self.temp_dir.cleanup()
+    def assertClientGetHash(self, client, taskhash, unihash):
+        result = client.get_unihash(self.METHOD, taskhash)
+        self.assertEqual(result, unihash)
 
+
+class HashEquivalenceCommonTests(object):
     def test_create_hash(self):
         # Simple test that hashes can be created
         taskhash = '35788efcb8dfb0a02659d81cf2bfd695fb30faf9'
         outhash = '2765d4a5884be49b28601445c2760c5f21e7e5c0ee2b7e3fce98fd7e5970796f'
         unihash = 'f46d3fbb439bd9b921095da657a4de906510d2cd'
 
-        result = self.client.get_unihash(self.METHOD, taskhash)
-        self.assertIsNone(result, msg='Found unexpected task, %r' % result)
+        self.assertClientGetHash(self.client, taskhash, None)
 
         result = self.client.report_unihash(taskhash, self.METHOD, outhash, unihash)
         self.assertEqual(result['unihash'], unihash, 'Server returned bad unihash')
@@ -84,22 +109,19 @@ class TestHashEquivalenceServer(object):
         unihash = '218e57509998197d570e2c98512d0105985dffc9'
         self.client.report_unihash(taskhash, self.METHOD, outhash, unihash)
 
-        result = self.client.get_unihash(self.METHOD, taskhash)
-        self.assertEqual(result, unihash)
+        self.assertClientGetHash(self.client, taskhash, unihash)
 
         outhash2 = '0904a7fe3dc712d9fd8a74a616ddca2a825a8ee97adf0bd3fc86082c7639914d'
         unihash2 = 'ae9a7d252735f0dafcdb10e2e02561ca3a47314c'
         self.client.report_unihash(taskhash, self.METHOD, outhash2, unihash2)
 
-        result = self.client.get_unihash(self.METHOD, taskhash)
-        self.assertEqual(result, unihash)
+        self.assertClientGetHash(self.client, taskhash, unihash)
 
         outhash3 = '77623a549b5b1a31e3732dfa8fe61d7ce5d44b3370f253c5360e136b852967b4'
         unihash3 = '9217a7d6398518e5dc002ed58f2cbbbc78696603'
         self.client.report_unihash(taskhash, self.METHOD, outhash3, unihash3)
 
-        result = self.client.get_unihash(self.METHOD, taskhash)
-        self.assertEqual(result, unihash)
+        self.assertClientGetHash(self.client, taskhash, unihash)
 
     def test_huge_message(self):
         # Simple test that hashes can be created
@@ -107,8 +129,7 @@ class TestHashEquivalenceServer(object):
         outhash = '3c979c3db45c569f51ab7626a4651074be3a9d11a84b1db076f5b14f7d39db44'
         unihash = '90e9bc1d1f094c51824adca7f8ea79a048d68824'
 
-        result = self.client.get_unihash(self.METHOD, taskhash)
-        self.assertIsNone(result, msg='Found unexpected task, %r' % result)
+        self.assertClientGetHash(self.client, taskhash, None)
 
         siginfo = "0" * (self.client.max_chunk * 4)
 
@@ -156,16 +177,140 @@ class TestHashEquivalenceServer(object):
 
         self.assertFalse(failures)
 
+    def test_upstream_server(self):
+        # Tests upstream server support. This is done by creating two servers
+        # that share a database file. The downstream server has it upstream
+        # set to the test server, whereas the side server doesn't. This allows
+        # verification that the hash requests are being proxied to the upstream
+        # server by verifying that they appear on the downstream client, but not
+        # the side client. It also verifies that the results are pulled into
+        # the downstream database by checking that the downstream and side servers
+        # match after the downstream is done waiting for all backfill tasks
+        (down_client, down_server) = self.start_server(upstream=self.server.address)
+        (side_client, side_server) = self.start_server(dbpath=down_server.dbpath)
 
-class TestHashEquivalenceUnixServer(TestHashEquivalenceServer, unittest.TestCase):
-    def get_server_addr(self):
-        return "unix://" + os.path.join(self.temp_dir.name, 'sock')
+        def check_hash(taskhash, unihash, old_sidehash):
+            nonlocal down_client
+            nonlocal side_client
+
+            # check upstream server
+            self.assertClientGetHash(self.client, taskhash, unihash)
+
+            # Hash should *not* be present on the side server
+            self.assertClientGetHash(side_client, taskhash, old_sidehash)
+
+            # Hash should be present on the downstream server, since it
+            # will defer to the upstream server. This will trigger
+            # the backfill in the downstream server
+            self.assertClientGetHash(down_client, taskhash, unihash)
+
+            # After waiting for the downstream client to finish backfilling the
+            # task from the upstream server, it should appear in the side server
+            # since the database is populated
+            down_client.backfill_wait()
+            self.assertClientGetHash(side_client, taskhash, unihash)
+
+        # Basic report
+        taskhash = '8aa96fcffb5831b3c2c0cb75f0431e3f8b20554a'
+        outhash = 'afe240a439959ce86f5e322f8c208e1fedefea9e813f2140c81af866cc9edf7e'
+        unihash = '218e57509998197d570e2c98512d0105985dffc9'
+        self.client.report_unihash(taskhash, self.METHOD, outhash, unihash)
+
+        check_hash(taskhash, unihash, None)
+
+        # Duplicated taskhash with multiple output hashes and unihashes.
+        # All servers should agree with the originally reported hash
+        outhash2 = '0904a7fe3dc712d9fd8a74a616ddca2a825a8ee97adf0bd3fc86082c7639914d'
+        unihash2 = 'ae9a7d252735f0dafcdb10e2e02561ca3a47314c'
+        self.client.report_unihash(taskhash, self.METHOD, outhash2, unihash2)
+
+        check_hash(taskhash, unihash, unihash)
+
+        # Report an equivalent task. The sideload will originally report
+        # no unihash until backfilled
+        taskhash3 = "044c2ec8aaf480685a00ff6ff49e6162e6ad34e1"
+        unihash3 = "def64766090d28f627e816454ed46894bb3aab36"
+        self.client.report_unihash(taskhash3, self.METHOD, outhash, unihash3)
+
+        check_hash(taskhash3, unihash, None)
+
+        # Test that reporting a unihash in the downstream client isn't
+        # propagating to the upstream server
+        taskhash4 = "e3da00593d6a7fb435c7e2114976c59c5fd6d561"
+        outhash4 = "1cf8713e645f491eb9c959d20b5cae1c47133a292626dda9b10709857cbe688a"
+        unihash4 = "3b5d3d83f07f259e9086fcb422c855286e18a57d"
+        down_client.report_unihash(taskhash4, self.METHOD, outhash4, unihash4)
+        down_client.backfill_wait()
+
+        self.assertClientGetHash(down_client, taskhash4, unihash4)
+        self.assertClientGetHash(side_client, taskhash4, unihash4)
+        self.assertClientGetHash(self.client, taskhash4, None)
+
+        # Test that reporting a unihash in the downstream is able to find a
+        # match which was previously reported to the upstream server
+        taskhash5 = '35788efcb8dfb0a02659d81cf2bfd695fb30faf9'
+        outhash5 = '2765d4a5884be49b28601445c2760c5f21e7e5c0ee2b7e3fce98fd7e5970796f'
+        unihash5 = 'f46d3fbb439bd9b921095da657a4de906510d2cd'
+        result = self.client.report_unihash(taskhash5, self.METHOD, outhash5, unihash5)
+
+        taskhash6 = '35788efcb8dfb0a02659d81cf2bfd695fb30fafa'
+        unihash6 = 'f46d3fbb439bd9b921095da657a4de906510d2ce'
+        result = down_client.report_unihash(taskhash6, self.METHOD, outhash5, unihash6)
+        self.assertEqual(result['unihash'], unihash5, 'Server failed to copy unihash from upstream')
+
+    def test_ro_server(self):
+        (ro_client, ro_server) = self.start_server(dbpath=self.server.dbpath, read_only=True)
+
+        # Report a hash via the read-write server
+        taskhash = '35788efcb8dfb0a02659d81cf2bfd695fb30faf9'
+        outhash = '2765d4a5884be49b28601445c2760c5f21e7e5c0ee2b7e3fce98fd7e5970796f'
+        unihash = 'f46d3fbb439bd9b921095da657a4de906510d2cd'
+
+        result = self.client.report_unihash(taskhash, self.METHOD, outhash, unihash)
+        self.assertEqual(result['unihash'], unihash, 'Server returned bad unihash')
+
+        # Check the hash via the read-only server
+        self.assertClientGetHash(ro_client, taskhash, unihash)
+
+        # Ensure that reporting via the read-only server fails
+        taskhash2 = 'c665584ee6817aa99edfc77a44dd853828279370'
+        outhash2 = '3c979c3db45c569f51ab7626a4651074be3a9d11a84b1db076f5b14f7d39db44'
+        unihash2 = '90e9bc1d1f094c51824adca7f8ea79a048d68824'
+
+        with self.assertRaises(HashConnectionError):
+            ro_client.report_unihash(taskhash2, self.METHOD, outhash2, unihash2)
+
+        # Ensure that the database was not modified
+        self.assertClientGetHash(self.client, taskhash2, None)
 
 
-class TestHashEquivalenceTCPServer(TestHashEquivalenceServer, unittest.TestCase):
-    def get_server_addr(self):
+class TestHashEquivalenceUnixServer(HashEquivalenceTestSetup, HashEquivalenceCommonTests, unittest.TestCase):
+    def get_server_addr(self, server_idx):
+        return "unix://" + os.path.join(self.temp_dir.name, 'sock%d' % server_idx)
+
+
+class TestHashEquivalenceUnixServerLongPath(HashEquivalenceTestSetup, unittest.TestCase):
+    DEEP_DIRECTORY = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa/bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb/ccccccccccccccccccccccccccccccccccccccccccc"
+    def get_server_addr(self, server_idx):
+        os.makedirs(os.path.join(self.temp_dir.name, self.DEEP_DIRECTORY), exist_ok=True)
+        return "unix://" + os.path.join(self.temp_dir.name, self.DEEP_DIRECTORY, 'sock%d' % server_idx)
+
+
+    def test_long_sock_path(self):
+        # Simple test that hashes can be created
+        taskhash = '35788efcb8dfb0a02659d81cf2bfd695fb30faf9'
+        outhash = '2765d4a5884be49b28601445c2760c5f21e7e5c0ee2b7e3fce98fd7e5970796f'
+        unihash = 'f46d3fbb439bd9b921095da657a4de906510d2cd'
+
+        self.assertClientGetHash(self.client, taskhash, None)
+
+        result = self.client.report_unihash(taskhash, self.METHOD, outhash, unihash)
+        self.assertEqual(result['unihash'], unihash, 'Server returned bad unihash')
+
+
+class TestHashEquivalenceTCPServer(HashEquivalenceTestSetup, HashEquivalenceCommonTests, unittest.TestCase):
+    def get_server_addr(self, server_idx):
         # Some hosts cause asyncio module to misbehave, when IPv6 is not enabled.
         # If IPv6 is enabled, it should be safe to use localhost directly, in general
         # case it is more reliable to resolve the IP address explicitly.
         return socket.gethostbyname("localhost") + ":0"
-            

bitbake/lib/layerindexlib/__init__.py

@@ -94,7 +94,7 @@ class LayerIndex():
            if not param:
                continue
            item = param.split('=', 1)
-           logger.debug(1, item)
+           logger.debug(item)
            param_dict[item[0]] = item[1]
 
         return param_dict
@@ -123,7 +123,7 @@ class LayerIndex():
         up = urlparse(url)
 
         if username:
-            logger.debug(1, "Configuring authentication for %s..." % url)
+            logger.debug("Configuring authentication for %s..." % url)
             password_mgr = urllib.request.HTTPPasswordMgrWithDefaultRealm()
             password_mgr.add_password(None, "%s://%s" % (up.scheme, up.netloc), username, password)
             handler = urllib.request.HTTPBasicAuthHandler(password_mgr)
@@ -133,20 +133,20 @@ class LayerIndex():
 
         urllib.request.install_opener(opener)
 
-        logger.debug(1, "Fetching %s (%s)..." % (url, ["without authentication", "with authentication"][bool(username)]))
+        logger.debug("Fetching %s (%s)..." % (url, ["without authentication", "with authentication"][bool(username)]))
 
         try:
             res = urlopen(Request(url, headers={'User-Agent': 'Mozilla/5.0 (bitbake/lib/layerindex)'}, unverifiable=True))
         except urllib.error.HTTPError as e:
-            logger.debug(1, "HTTP Error: %s: %s" % (e.code, e.reason))
-            logger.debug(1, " Requested: %s" % (url))
-            logger.debug(1, " Actual:    %s" % (e.geturl()))
+            logger.debug("HTTP Error: %s: %s" % (e.code, e.reason))
+            logger.debug(" Requested: %s" % (url))
+            logger.debug(" Actual:    %s" % (e.geturl()))
 
             if e.code == 404:
-                logger.debug(1, "Request not found.")
+                logger.debug("Request not found.")
                 raise LayerIndexFetchError(url, e)
             else:
-                logger.debug(1, "Headers:\n%s" % (e.headers))
+                logger.debug("Headers:\n%s" % (e.headers))
                 raise LayerIndexFetchError(url, e)
         except OSError as e:
             error = 0
@@ -170,7 +170,7 @@ class LayerIndex():
                 raise LayerIndexFetchError(url, "Unable to fetch OSError exception: %s" % e)
 
         finally:
-            logger.debug(1, "...fetching %s (%s), done." % (url, ["without authentication", "with authentication"][bool(username)]))
+            logger.debug("...fetching %s (%s), done." % (url, ["without authentication", "with authentication"][bool(username)]))
 
         return res
 
@@ -205,14 +205,14 @@ The format of the indexURI:
         if reload:
             self.indexes = []
 
-        logger.debug(1, 'Loading: %s' % indexURI)
+        logger.debug('Loading: %s' % indexURI)
 
         if not self.plugins:
             raise LayerIndexException("No LayerIndex Plugins available")
 
         for plugin in self.plugins:
             # Check if the plugin was initialized
-            logger.debug(1, 'Trying %s' % plugin.__class__)
+            logger.debug('Trying %s' % plugin.__class__)
             if not hasattr(plugin, 'type') or not plugin.type:
                 continue
             try:
@@ -220,11 +220,11 @@ The format of the indexURI:
                 indexEnt = plugin.load_index(indexURI, load)
                 break
             except LayerIndexPluginUrlError as e:
-                logger.debug(1, "%s doesn't support %s" % (plugin.type, e.url))
+                logger.debug("%s doesn't support %s" % (plugin.type, e.url))
             except NotImplementedError:
                 pass
         else:
-            logger.debug(1, "No plugins support %s" % indexURI)
+            logger.debug("No plugins support %s" % indexURI)
             raise LayerIndexException("No plugins support %s" % indexURI)
 
         # Mark CONFIG data as something we've added...
@@ -255,19 +255,19 @@ will write out the individual elements split by layer and related components.
 
         for plugin in self.plugins:
             # Check if the plugin was initialized
-            logger.debug(1, 'Trying %s' % plugin.__class__)
+            logger.debug('Trying %s' % plugin.__class__)
             if not hasattr(plugin, 'type') or not plugin.type:
                 continue
             try:
                 plugin.store_index(indexURI, index)
                 break
             except LayerIndexPluginUrlError as e:
-                logger.debug(1, "%s doesn't support %s" % (plugin.type, e.url))
+                logger.debug("%s doesn't support %s" % (plugin.type, e.url))
             except NotImplementedError:
-                logger.debug(1, "Store not implemented in %s" % plugin.type)
+                logger.debug("Store not implemented in %s" % plugin.type)
                 pass
         else:
-            logger.debug(1, "No plugins support %s" % indexURI)
+            logger.debug("No plugins support %s" % indexURI)
             raise LayerIndexException("No plugins support %s" % indexURI)
 
 
@@ -292,7 +292,7 @@ layerBranches set.  If not, they are effectively blank.'''
            the default configuration until the first vcs_url/branch match.'''
 
         for index in self.indexes:
-            logger.debug(1, ' searching %s' % index.config['DESCRIPTION'])
+            logger.debug(' searching %s' % index.config['DESCRIPTION'])
             layerBranch = index.find_vcs_url(vcs_url, [branch])
             if layerBranch:
                 return layerBranch
@@ -304,7 +304,7 @@ layerBranches set.  If not, they are effectively blank.'''
            If a branch has not been specified, we will iterate over the branches in
            the default configuration until the first collection/branch match.'''
 
-        logger.debug(1, 'find_collection: %s (%s) %s' % (collection, version, branch))
+        logger.debug('find_collection: %s (%s) %s' % (collection, version, branch))
 
         if branch:
             branches = [branch]
@@ -312,12 +312,12 @@ layerBranches set.  If not, they are effectively blank.'''
             branches = None
 
         for index in self.indexes:
-            logger.debug(1, ' searching %s' % index.config['DESCRIPTION'])
+            logger.debug(' searching %s' % index.config['DESCRIPTION'])
             layerBranch = index.find_collection(collection, version, branches)
             if layerBranch:
                 return layerBranch
         else:
-            logger.debug(1, 'Collection %s (%s) not found for branch (%s)' % (collection, version, branch))
+            logger.debug('Collection %s (%s) not found for branch (%s)' % (collection, version, branch))
         return None
 
     def find_layerbranch(self, name, branch=None):
@@ -408,7 +408,7 @@ layerBranches set.  If not, they are effectively blank.'''
                                               version=deplayerbranch.version
                                           )
                         if rdeplayerbranch != deplayerbranch:
-                                logger.debug(1, 'Replaced %s:%s:%s with %s:%s:%s' % \
+                                logger.debug('Replaced %s:%s:%s with %s:%s:%s' % \
                                       (deplayerbranch.index.config['DESCRIPTION'],
                                        deplayerbranch.branch.name,
                                        deplayerbranch.layer.name,
@@ -1121,7 +1121,7 @@ class LayerBranch(LayerIndexItemObj):
     @property
     def branch(self):
         try:
-            logger.debug(1, "Get branch object from branches[%s]" % (self.branch_id))
+            logger.debug("Get branch object from branches[%s]" % (self.branch_id))
             return self.index.branches[self.branch_id]
         except KeyError:
             raise AttributeError('Unable to find branches in index to map branch_id %s' % self.branch_id)
@@ -1149,7 +1149,7 @@ class LayerBranch(LayerIndexItemObj):
 
     @actual_branch.setter
     def actual_branch(self, value):
-        logger.debug(1, "Set actual_branch to %s .. name is %s" % (value, self.branch.name))
+        logger.debug("Set actual_branch to %s .. name is %s" % (value, self.branch.name))
         if value != self.branch.name:
             self._setattr('actual_branch', value, prop=False)
         else:

bitbake/lib/layerindexlib/cooker.py

@@ -173,7 +173,7 @@ class CookerPlugin(layerindexlib.plugin.IndexPlugin):
         else:
             branches = ['HEAD']
 
-        logger.debug(1, "Loading cooker data branches %s" % branches)
+        logger.debug("Loading cooker data branches %s" % branches)
 
         index = self._load_bblayers(branches=branches)
 
@@ -220,7 +220,7 @@ class CookerPlugin(layerindexlib.plugin.IndexPlugin):
                                         required=required, layerbranch=layerBranchId,
                                         dependency=depLayerBranch.layer_id)
 
-                        logger.debug(1, '%s requires %s' % (layerDependency.layer.name, layerDependency.dependency.name))
+                        logger.debug('%s requires %s' % (layerDependency.layer.name, layerDependency.dependency.name))
                         index.add_element("layerDependencies", [layerDependency])
 
                     return layerDependencyId

bitbake/lib/layerindexlib/restapi.py

@@ -82,7 +82,7 @@ class RestApiPlugin(layerindexlib.plugin.IndexPlugin):
 
 
         def load_cache(path, index, branches=[]):
-            logger.debug(1, 'Loading json file %s' % path)
+            logger.debug('Loading json file %s' % path)
             with open(path, 'rt', encoding='utf-8') as f:
                 pindex = json.load(f)
 
@@ -102,7 +102,7 @@ class RestApiPlugin(layerindexlib.plugin.IndexPlugin):
             if newpBranch:
                 index.add_raw_element('branches', layerindexlib.Branch, newpBranch)
             else:
-                logger.debug(1, 'No matching branches (%s) in index file(s)' % branches)
+                logger.debug('No matching branches (%s) in index file(s)' % branches)
                 # No matching branches.. return nothing...
                 return
 
@@ -120,7 +120,7 @@ class RestApiPlugin(layerindexlib.plugin.IndexPlugin):
             load_cache(up.path, index, branches)
             return index
 
-        logger.debug(1, 'Loading from dir %s...' % (up.path))
+        logger.debug('Loading from dir %s...' % (up.path))
         for (dirpath, _, filenames) in os.walk(up.path):
             for filename in filenames:
                 if not filename.endswith('.json'):
@@ -144,7 +144,7 @@ class RestApiPlugin(layerindexlib.plugin.IndexPlugin):
         def _get_json_response(apiurl=None, username=None, password=None, retry=True):
             assert apiurl is not None
 
-            logger.debug(1, "fetching %s" % apiurl)
+            logger.debug("fetching %s" % apiurl)
 
             up = urlparse(apiurl)
 
@@ -163,9 +163,9 @@ class RestApiPlugin(layerindexlib.plugin.IndexPlugin):
                 parsed = json.loads(res.read().decode('utf-8'))
             except ConnectionResetError:
                 if retry:
-                    logger.debug(1, "%s: Connection reset by peer.  Retrying..." % url)
+                    logger.debug("%s: Connection reset by peer.  Retrying..." % url)
                     parsed = _get_json_response(apiurl=up_stripped.geturl(), username=username, password=password, retry=False)
-                    logger.debug(1, "%s: retry successful.")
+                    logger.debug("%s: retry successful.")
                 else:
                     raise layerindexlib.LayerIndexFetchError('%s: Connection reset by peer.  Is there a firewall blocking your connection?' % apiurl)
 
@@ -207,25 +207,25 @@ class RestApiPlugin(layerindexlib.plugin.IndexPlugin):
         if "*" not in branches:
             filter = "?filter=name:%s" % "OR".join(branches)
 
-        logger.debug(1, "Loading %s from %s" % (branches, index.apilinks['branches']))
+        logger.debug("Loading %s from %s" % (branches, index.apilinks['branches']))
 
         # The link won't include username/password, so pull it from the original url
         pindex['branches'] = _get_json_response(index.apilinks['branches'] + filter,
                                                     username=up.username, password=up.password)
         if not pindex['branches']:
-            logger.debug(1, "No valid branches (%s) found at url %s." % (branch, url))
+            logger.debug("No valid branches (%s) found at url %s." % (branch, url))
             return index
         index.add_raw_element("branches", layerindexlib.Branch, pindex['branches'])
 
         # Load all of the layerItems (these can not be easily filtered)
-        logger.debug(1, "Loading %s from %s" % ('layerItems', index.apilinks['layerItems']))
+        logger.debug("Loading %s from %s" % ('layerItems', index.apilinks['layerItems']))
 
 
         # The link won't include username/password, so pull it from the original url
         pindex['layerItems'] = _get_json_response(index.apilinks['layerItems'],
                                                   username=up.username, password=up.password)
         if not pindex['layerItems']:
-            logger.debug(1, "No layers were found at url %s." % (url))
+            logger.debug("No layers were found at url %s." % (url))
             return index
         index.add_raw_element("layerItems", layerindexlib.LayerItem, pindex['layerItems'])
 
@@ -235,13 +235,13 @@ class RestApiPlugin(layerindexlib.plugin.IndexPlugin):
         for branch in index.branches:
             filter = "?filter=branch__name:%s" % index.branches[branch].name
 
-            logger.debug(1, "Loading %s from %s" % ('layerBranches', index.apilinks['layerBranches']))
+            logger.debug("Loading %s from %s" % ('layerBranches', index.apilinks['layerBranches']))
 
             # The link won't include username/password, so pull it from the original url
             pindex['layerBranches'] = _get_json_response(index.apilinks['layerBranches'] + filter,
                                                   username=up.username, password=up.password)
             if not pindex['layerBranches']:
-                logger.debug(1, "No valid layer branches (%s) found at url %s." % (branches or "*", url))
+                logger.debug("No valid layer branches (%s) found at url %s." % (branches or "*", url))
                 return index
             index.add_raw_element("layerBranches", layerindexlib.LayerBranch, pindex['layerBranches'])
 
@@ -256,7 +256,7 @@ class RestApiPlugin(layerindexlib.plugin.IndexPlugin):
                                    ("distros", layerindexlib.Distro)]:
                 if lName not in load:
                     continue
-                logger.debug(1, "Loading %s from %s" % (lName, index.apilinks[lName]))
+                logger.debug("Loading %s from %s" % (lName, index.apilinks[lName]))
 
                 # The link won't include username/password, so pull it from the original url
                 pindex[lName] = _get_json_response(index.apilinks[lName] + filter,
@@ -283,7 +283,7 @@ class RestApiPlugin(layerindexlib.plugin.IndexPlugin):
         if up.scheme != 'file':
             raise layerindexlib.plugin.LayerIndexPluginUrlError(self.type, url)
 
-        logger.debug(1, "Storing to %s..." % up.path)
+        logger.debug("Storing to %s..." % up.path)
 
         try:
             layerbranches = index.layerBranches
@@ -299,12 +299,12 @@ class RestApiPlugin(layerindexlib.plugin.IndexPlugin):
                     if getattr(index, objects)[obj].layerbranch_id == layerbranchid:
                        filtered.append(getattr(index, objects)[obj]._data)
                 except AttributeError:
-                    logger.debug(1, 'No obj.layerbranch_id: %s' % objects)
+                    logger.debug('No obj.layerbranch_id: %s' % objects)
                     # No simple filter method, just include it...
                     try:
                         filtered.append(getattr(index, objects)[obj]._data)
                     except AttributeError:
-                        logger.debug(1, 'No obj._data: %s %s' % (objects, type(obj)))
+                        logger.debug('No obj._data: %s %s' % (objects, type(obj)))
                         filtered.append(obj)
             return filtered
 

bitbake/lib/layerindexlib/tests/cooker.py

@@ -72,7 +72,7 @@ class LayerIndexCookerTest(LayersTest):
 
     def test_find_collection(self):
         def _check(collection, expected):
-            self.logger.debug(1, "Looking for collection %s..." % collection)
+            self.logger.debug("Looking for collection %s..." % collection)
             result = self.layerindex.find_collection(collection)
             if expected:
                 self.assertIsNotNone(result, msg="Did not find %s when it shouldn't be there" % collection)
@@ -91,7 +91,7 @@ class LayerIndexCookerTest(LayersTest):
 
     def test_find_layerbranch(self):
         def _check(name, expected):
-            self.logger.debug(1, "Looking for layerbranch %s..." % name)
+            self.logger.debug("Looking for layerbranch %s..." % name)
             result = self.layerindex.find_layerbranch(name)
             if expected:
                 self.assertIsNotNone(result, msg="Did not find %s when it shouldn't be there" % collection)

bitbake/lib/layerindexlib/tests/restapi.py

@@ -57,11 +57,11 @@ class LayerIndexWebRestApiTest(LayersTest):
                type in self.layerindex.indexes[0].config['local']:
                 continue
             for id in getattr(self.layerindex.indexes[0], type):
-                self.logger.debug(1, "type %s" % (type))
+                self.logger.debug("type %s" % (type))
 
                 self.assertTrue(id in getattr(reload.indexes[0], type), msg="Id number not in reloaded index")
 
-                self.logger.debug(1, "%s ? %s" % (getattr(self.layerindex.indexes[0], type)[id], getattr(reload.indexes[0], type)[id]))
+                self.logger.debug("%s ? %s" % (getattr(self.layerindex.indexes[0], type)[id], getattr(reload.indexes[0], type)[id]))
 
                 self.assertEqual(getattr(self.layerindex.indexes[0], type)[id], getattr(reload.indexes[0], type)[id], msg="Reloaded contents different")
 
@@ -80,11 +80,11 @@ class LayerIndexWebRestApiTest(LayersTest):
                type in self.layerindex.indexes[0].config['local']:
                 continue
             for id in getattr(self.layerindex.indexes[0] ,type):
-                self.logger.debug(1, "type %s" % (type))
+                self.logger.debug("type %s" % (type))
 
                 self.assertTrue(id in getattr(reload.indexes[0], type), msg="Id number missing from reloaded data")
 
-                self.logger.debug(1, "%s ? %s" % (getattr(self.layerindex.indexes[0] ,type)[id], getattr(reload.indexes[0], type)[id]))
+                self.logger.debug("%s ? %s" % (getattr(self.layerindex.indexes[0] ,type)[id], getattr(reload.indexes[0], type)[id]))
 
                 self.assertEqual(getattr(self.layerindex.indexes[0] ,type)[id], getattr(reload.indexes[0], type)[id], msg="reloaded data does not match original")
 
@@ -111,14 +111,14 @@ class LayerIndexWebRestApiTest(LayersTest):
                 if dep.layer.name == 'meta-python':
                     break
             else:
-                self.logger.debug(1, "meta-python was not found")
+                self.logger.debug("meta-python was not found")
                 raise self.failureException
 
             # Only check the first element...
             break
         else:
             # Empty list, this is bad.
-            self.logger.debug(1, "Empty list of dependencies")
+            self.logger.debug("Empty list of dependencies")
             self.assertIsNotNone(first, msg="Empty list of dependencies")
 
             # Last dep should be the requested item
@@ -128,7 +128,7 @@ class LayerIndexWebRestApiTest(LayersTest):
     @skipIfNoNetwork()
     def test_find_collection(self):
         def _check(collection, expected):
-            self.logger.debug(1, "Looking for collection %s..." % collection)
+            self.logger.debug("Looking for collection %s..." % collection)
             result = self.layerindex.find_collection(collection)
             if expected:
                 self.assertIsNotNone(result, msg="Did not find %s when it should be there" % collection)
@@ -148,11 +148,11 @@ class LayerIndexWebRestApiTest(LayersTest):
     @skipIfNoNetwork()
     def test_find_layerbranch(self):
         def _check(name, expected):
-            self.logger.debug(1, "Looking for layerbranch %s..." % name)
+            self.logger.debug("Looking for layerbranch %s..." % name)
 
             for index in self.layerindex.indexes:
                 for layerbranchid in index.layerBranches:
-                    self.logger.debug(1, "Present: %s" % index.layerBranches[layerbranchid].layer.name)
+                    self.logger.debug("Present: %s" % index.layerBranches[layerbranchid].layer.name)
             result = self.layerindex.find_layerbranch(name)
             if expected:
                 self.assertIsNotNone(result, msg="Did not find %s when it should be there" % collection)

bitbake/lib/toaster/orm/fixtures/oe-core.xml

@@ -23,9 +23,9 @@
     <field type="CharField" name="branch">master</field>
   </object>
   <object model="orm.bitbakeversion" pk="4">
-    <field type="CharField" name="name">gatesgarth</field>
+    <field type="CharField" name="name">hardknott</field>
     <field type="CharField" name="giturl">git://git.openembedded.org/bitbake</field>
-    <field type="CharField" name="branch">1.48</field>
+    <field type="CharField" name="branch">1.50</field>
   </object>
 
   <!-- Releases available -->
@@ -51,11 +51,11 @@
     <field type="TextField" name="helptext">Toaster will run your builds using the tip of the &lt;a href=\"http://cgit.openembedded.org/openembedded-core/log/\"&gt;OpenEmbedded master&lt;/a&gt; branch.</field>
   </object>
   <object model="orm.release" pk="4">
-    <field type="CharField" name="name">gatesgarth</field>
-    <field type="CharField" name="description">Openembedded Gatesgarth</field>
+    <field type="CharField" name="name">hardknott</field>
+    <field type="CharField" name="description">Openembedded Hardknott</field>
     <field rel="ManyToOneRel" to="orm.bitbakeversion" name="bitbake_version">4</field>
-    <field type="CharField" name="branch_name">gatesgarth</field>
-    <field type="TextField" name="helptext">Toaster will run your builds using the tip of the &lt;a href=\"http://cgit.openembedded.org/openembedded-core/log/?h=gatesgarth\"&gt;OpenEmbedded Gatesgarth&lt;/a&gt; branch.</field>
+    <field type="CharField" name="branch_name">hardknott</field>
+    <field type="TextField" name="helptext">Toaster will run your builds using the tip of the &lt;a href=\"http://cgit.openembedded.org/openembedded-core/log/?h=hardknott\"&gt;OpenEmbedded Hardknott&lt;/a&gt; branch.</field>
   </object>
 
   <!-- Default layers for each release -->

bitbake/lib/toaster/orm/fixtures/poky.xml

@@ -26,9 +26,9 @@
     <field type="CharField" name="dirpath">bitbake</field>
   </object>
   <object model="orm.bitbakeversion" pk="4">
-    <field type="CharField" name="name">gatesgarth</field>
+    <field type="CharField" name="name">hardknott</field>
     <field type="CharField" name="giturl">git://git.yoctoproject.org/poky</field>
-    <field type="CharField" name="branch">gatesgarth</field>
+    <field type="CharField" name="branch">hardknott</field>
     <field type="CharField" name="dirpath">bitbake</field>
   </object>
 
@@ -56,11 +56,11 @@
     <field type="TextField" name="helptext">Toaster will run your builds using the tip of the &lt;a href="http://git.yoctoproject.org/cgit/cgit.cgi/poky/log/"&gt;Yocto Project Master branch&lt;/a&gt;.</field>
   </object>
   <object model="orm.release" pk="4">
-    <field type="CharField" name="name">gatesgarth</field>
-    <field type="CharField" name="description">Yocto Project 3.2 "Gatesgarth"</field>
+    <field type="CharField" name="name">hardknott</field>
+    <field type="CharField" name="description">Yocto Project 3.2 "Hardknott"</field>
     <field rel="ManyToOneRel" to="orm.bitbakeversion" name="bitbake_version">4</field>
-    <field type="CharField" name="branch_name">gatesgarth</field>
-    <field type="TextField" name="helptext">Toaster will run your builds using the tip of the &lt;a href="http://git.yoctoproject.org/cgit/cgit.cgi/poky/log/?h=gatesgarth"&gt;Yocto Project Gatesgarth branch&lt;/a&gt;.</field>
+    <field type="CharField" name="branch_name">hardknott</field>
+    <field type="TextField" name="helptext">Toaster will run your builds using the tip of the &lt;a href="http://git.yoctoproject.org/cgit/cgit.cgi/poky/log/?h=hardknott"&gt;Yocto Project Hardknott branch&lt;/a&gt;.</field>
   </object>
 
   <!-- Default project layers for each release -->
@@ -152,7 +152,7 @@
     <field rel="ManyToOneRel" to="orm.layer" name="layer">1</field>
     <field type="IntegerField" name="layer_source">0</field>
     <field rel="ManyToOneRel" to="orm.release" name="release">4</field>
-    <field type="CharField" name="branch">gatesgarth</field>
+    <field type="CharField" name="branch">hardknott</field>
     <field type="CharField" name="dirpath">meta</field>
   </object>
 
@@ -190,7 +190,7 @@
     <field rel="ManyToOneRel" to="orm.layer" name="layer">2</field>
     <field type="IntegerField" name="layer_source">0</field>
     <field rel="ManyToOneRel" to="orm.release" name="release">4</field>
-    <field type="CharField" name="branch">gatesgarth</field>
+    <field type="CharField" name="branch">hardknott</field>
     <field type="CharField" name="dirpath">meta-poky</field>
   </object>
 
@@ -228,7 +228,7 @@
     <field rel="ManyToOneRel" to="orm.layer" name="layer">3</field>
     <field type="IntegerField" name="layer_source">0</field>
     <field rel="ManyToOneRel" to="orm.release" name="release">4</field>
-    <field type="CharField" name="branch">gatesgarth</field>
+    <field type="CharField" name="branch">hardknott</field>
     <field type="CharField" name="dirpath">meta-yocto-bsp</field>
   </object>
 </django-objects>

bitbake/lib/toaster/orm/fixtures/settings.xml

@@ -19,7 +19,7 @@
     <field type="CharField" name="value">${TOPDIR}/../sstate-cache</field>
   </object>
   <object model="orm.toastersetting" pk="6">
-    <field type="CharField" name="name">DEFCONF_IMAGE_INSTALL_append</field>
+    <field type="CharField" name="name">DEFCONF_IMAGE_INSTALL:append</field>
     <field type="CharField" name="value"></field>
   </object>
   <object model="orm.toastersetting" pk="7">

bitbake/lib/toaster/orm/models.py

@@ -1717,7 +1717,7 @@ class CustomImageRecipe(Recipe):
 
     def generate_recipe_file_contents(self):
         """Generate the contents for the recipe file."""
-        # If we have no excluded packages we only need to _append
+        # If we have no excluded packages we only need to :append
         if self.excludes_set.count() == 0:
             packages_conf = "IMAGE_INSTALL_append = \" "
 

bitbake/lib/toaster/tests/builds/buildtest.py

@@ -119,7 +119,7 @@ class BuildTest(unittest.TestCase):
         if os.environ.get("TOASTER_TEST_USE_SSTATE_MIRROR"):
             ProjectVariable.objects.get_or_create(
                 name="SSTATE_MIRRORS",
-                value="file://.* http://autobuilder.yoctoproject.org/pub/sstate/PATH;downloadfilename=PATH",
+                value="file://.* http://sstate.yoctoproject.org/PATH;downloadfilename=PATH",
                 project=project)
 
         ProjectTarget.objects.create(project=project,

bitbake/lib/toaster/toastergui/templates/projectconf.html

@@ -73,7 +73,7 @@
 
     {% if image_install_append_defined %}
     <dt>
-    <span class="js-config-var-name js-config-var-managed-name">IMAGE_INSTALL_append</span>
+    <span class="js-config-var-name js-config-var-managed-name">IMAGE_INSTALL:append</span>
     <span class="glyphicon glyphicon-question-sign get-help" title="Specifies additional packages to install into an image. If your build creates more than one image, the packages will be installed in all of them"></span>
     </dt>
     <dd class="variable-list">
@@ -83,7 +83,7 @@
     <form id="change-image_install-form" class="form-inline" style="display:none;">
       <div class="row">
         <div class="col-md-4">
-          <span class="help-block">To set IMAGE_INSTALL_append to more than one package, type the package names separated by a space.</span>
+          <span class="help-block">To set IMAGE_INSTALL:append to more than one package, type the package names separated by a space.</span>
         </div>
       </div>
       <div class="form-group">
@@ -771,10 +771,10 @@ $(document).ready(function() {
 
   {% if image_install_append_defined %}
 
-  // init IMAGE_INSTALL_append trash icon
+  // init IMAGE_INSTALL:append trash icon
   setDeleteTooltip($('#delete-image_install-icon'));
 
-  // change IMAGE_INSTALL_append variable
+  // change IMAGE_INSTALL:append variable
   $('#change-image_install-icon').click(function() {
     // preset the edit value
     var current_val = $("span#image_install").text().trim();
@@ -814,7 +814,7 @@ $(document).ready(function() {
   $('#apply-change-image_install').click(function(){
     // insure these non-empty values have single space prefix
     var value = " " + $('#new-image_install').val().trim();
-    postEditAjaxRequest({"configvarChange" : 'IMAGE_INSTALL_append:'+value});
+    postEditAjaxRequest({"configvarChange" : 'IMAGE_INSTALL:append:'+value});
     $('#image_install').text(value);
     $('#image_install').removeClass('text-muted');
     $("#change-image_install-form").slideUp(function () {
@@ -826,10 +826,10 @@ $(document).ready(function() {
     });
   });
 
-  // delete IMAGE_INSTALL_append variable value
+  // delete IMAGE_INSTALL:append variable value
   $('#delete-image_install-icon').click(function(){
     $(this).tooltip('hide');
-    postEditAjaxRequest({"configvarChange" : 'IMAGE_INSTALL_append:'+''});
+    postEditAjaxRequest({"configvarChange" : 'IMAGE_INSTALL:append:'+''});
     $('#image_install').parent().fadeOut(1000, function(){
       $('#image_install').addClass('text-muted');
       $('#image_install').text('Not set');
@@ -1011,7 +1011,7 @@ $(document).ready(function() {
     $(".save").attr("disabled","disabled");
 
     // Reload page if admin-removed core managed value is manually added back in
-    if (0 <= " DISTRO DL_DIR IMAGE_FSTYPES IMAGE_INSTALL_append PACKAGE_CLASSES SSTATE_DIR ".indexOf( " "+variable+" " )) {
+    if (0 <= " DISTRO DL_DIR IMAGE_FSTYPES IMAGE_INSTALL:append PACKAGE_CLASSES SSTATE_DIR ".indexOf( " "+variable+" " )) {
       // delayed reload to avoid race condition with postEditAjaxRequest
       do_reload=true;
     }

contrib/git-hooks/sendemail-validate.sample

@@ -0,0 +1,78 @@
+#!/usr/bin/env python3
+
+# Copyright (C) 2020  Agilent Technologies, Inc.
+# Author: Chris Laplante <chris.laplante@agilent.com>
+
+# This sendemail-validate hook injects 'From: ' header lines into outgoing
+# emails sent via 'git send-email', to ensure that accurate commit authorship
+# information is present. It was created because some email servers
+# (notably Microsoft Exchange / Office 360) seem to butcher outgoing patches,
+# resulting in incorrect authorship.
+
+# Current limitations:
+#   1. Assumes one per patch per email
+#   2. Minimal error checking
+#
+# Installation:
+#   1. Copy to .git/hooks/sendemail-validate
+#   2. chmod +x .git/hooks/sendemail-validate
+
+
+import enum
+import re
+import subprocess
+import sys
+
+
+class Subject(enum.IntEnum):
+    NOT_SEEN = 0
+    CONSUMING = 1
+    SEEN = 2
+
+
+def make_from_line():
+    cmd = ["git", "var", "GIT_COMMITTER_IDENT"]
+    proc = subprocess.run(cmd, check=True, stdout=subprocess.PIPE, universal_newlines=True)
+    regex = re.compile(r"^(.*>).*$")
+    match = regex.match(proc.stdout)
+    assert match is not None
+    return "From: {0}".format(match.group(1))
+
+
+def main():
+    email = sys.argv[1]
+
+    with open(email, "r") as f:
+        email_lines = f.read().split("\n")
+
+    subject_seen = Subject.NOT_SEEN
+    first_body_line = None
+    for i, line in enumerate(email_lines):
+        if (subject_seen == Subject.NOT_SEEN) and line.startswith("Subject: "):
+            subject_seen = Subject.CONSUMING
+            continue
+        if subject_seen == Subject.CONSUMING:
+            if not line.strip():
+                subject_seen = Subject.SEEN
+                continue
+        if subject_seen == Subject.SEEN:
+            first_body_line = i
+            break
+
+    assert subject_seen == Subject.SEEN
+    assert first_body_line is not None
+
+    from_line = make_from_line()
+    # Only add FROM line if it is not already there
+    if email_lines[first_body_line] != from_line:
+        email_lines.insert(first_body_line, from_line)
+        email_lines.insert(first_body_line + 1, "")
+        with open(email, "w") as f:
+            f.write("\n".join(email_lines))
+
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())
+

documentation/.gitignore

@@ -1 +1,3 @@
 _build/
+Pipfile.lock
+.vscode/

documentation/Makefile

@@ -3,7 +3,7 @@
 
 # You can set these variables from the command line, and also
 # from the environment for the first two.
-SPHINXOPTS    ?=
+SPHINXOPTS    ?= -j auto
 SPHINXBUILD   ?= sphinx-build
 SOURCEDIR     = .
 BUILDDIR      = _build

documentation/Pipfile

@@ -0,0 +1,14 @@
+[[source]]
+name = "pypi"
+url = "https://pypi.org/simple"
+verify_ssl = true
+
+[dev-packages]
+
+[packages]
+sphinx = "*"
+sphinx-rtd-theme = "*"
+pyyaml = "*"
+
+[requires]
+python_version = "3"

documentation/README

@@ -2,7 +2,7 @@ documentation
 =============
 
 This is the directory that contains the Yocto Project documentation.  The Yocto
-Project source repositories at http://git.yoctoproject.org/cgit.cgi have two
+Project source repositories at https://git.yoctoproject.org/cgit.cgi have two
 instances of the "documentation" directory.  You should understand each of
 these instances.
 
@@ -47,12 +47,12 @@ Folders exist for individual manuals as follows:
 Each folder is self-contained regarding content and figures.
 
 If you want to find HTML versions of the Yocto Project manuals on the web,
-go to http://www.yoctoproject.org and click on the "Documentation" tab.  From
+go to https://www.yoctoproject.org and click on the "Docs" tab.  From
 there you have access to archived documentation from previous releases, current
 documentation for the latest release, and "Docs in Progress" for the release
 currently being developed.
 
-In general, the Yocto Project site (http://www.yoctoproject.org) is a great
+In general, the Yocto Project site (https://www.yoctoproject.org) is a great
 reference for both information and downloads.
 
 poky.yaml
@@ -91,13 +91,13 @@ Yocto Project documentation website
 A new website has been created to host the Yocto Project
 documentation, it can be found at: https://docs.yoctoproject.org/.
 
-The entire Yocto Project documentation, as well as the BitBake manual
+The entire Yocto Project documentation, as well as the BitBake manual,
 is published on this website, including all previously released
 versions. A version switcher was added, as a drop-down menu on the top
 of the page to switch back and forth between the various versions of
 the current active Yocto Project releases.
 
-Transition pages have been added (as rst file) to show links to old
+Transition pages have been added (as rst files) to show links to old
 versions of the Yocto Project documentation with links to each manual
 generated with DocBook.
 
@@ -109,7 +109,7 @@ obvious reasons, we will only support building the Yocto Project
 documentation with Python3.
 
 Sphinx might be available in your Linux distro packages repositories,
-however it is not recommend using distro packages, as they might be
+however it is not recommended to use distro packages, as they might be
 old versions, especially if you are using an LTS version of your
 distro. The recommended method to install Sphinx and all required
 dependencies is to use the Python Package Index (pip).
@@ -127,6 +127,13 @@ The resulting HTML index page will be _build/html/index.html, and you
 can browse your own copy of the locally generated documentation with
 your browser.
 
+Alternatively, you can use Pipenv to automatically install all required
+dependencies in a virtual environment:
+
+ $ cd documentation
+ $ pipenv install
+ $ pipenv run make html
+
 Sphinx theme and CSS customization
 ==================================
 
@@ -178,7 +185,7 @@ Sphinx has a glossary directive. From
 https://www.sphinx-doc.org/en/master/usage/restructuredtext/directives.html#glossary:
 
     This directive must contain a reST definition list with terms and
-    definitions. The definitions will then be referencable with the
+    definitions. It's then possible to refer to each definition through the
     [https://www.sphinx-doc.org/en/master/usage/restructuredtext/roles.html#role-term
     'term' role].
 
@@ -199,7 +206,7 @@ however there are important shortcomings. For example they cannot be
 used/nested inside code-block sections.
 
 A Sphinx extension was implemented to support variable substitutions
-to mimic the DocBook based documentation behavior. Variabes
+to mimic the DocBook based documentation behavior. Variable
 substitutions are done while reading/parsing the .rst files. The
 pattern for variables substitutions is the same as with DocBook,
 e.g. `&VAR;`.
@@ -215,13 +222,13 @@ For example, the following .rst content will produce the 'expected'
 content:
 
   .. code-block::
-      $ mkdir ~/poky-&DISTRO;
+      $ mkdir poky-&DISTRO;
       or
       $ git clone &YOCTO_GIT_URL;/git/poky -b &DISTRO_NAME_NO_CAP;
 
 Variables can be nested, like it was the case for DocBook:
 
-  YOCTO_HOME_URL : "http://www.yoctoproject.org"
+  YOCTO_HOME_URL : "https://www.yoctoproject.org"
   YOCTO_DOCS_URL : "&YOCTO_HOME_URL;/docs"
 
 Note directive
@@ -230,14 +237,14 @@ Note directive
 Sphinx has a builtin 'note' directive that produces clean Note section
 in the output file. There are various types of directives such as
 "attention", "caution", "danger", "error", "hint", "important", "tip",
-"warning", "admonition" that are supported, and additional directive
+"warning", "admonition" that are supported, and additional directives
 can be added as Sphinx extension if needed.
 
 Figures
 =======
 
 The Yocto Project documentation has many figures/images. Sphinx has a
-'figure' directive which is straight forward to use. To include a
+'figure' directive which is straightforward to use. To include a
 figure in the body of the documentation:
 
   .. image:: figures/YP-flow-diagram.png
@@ -252,10 +259,13 @@ websites.
 More information can be found here:
 https://sublime-and-sphinx-guide.readthedocs.io/en/latest/references.html.
 
+Anchor (<#link>) links are forbidden as they are not checked by Sphinx during
+the build and may be broken without knowing about it.
+
 References
 ==========
 
-The following extension is enabed by default:
+The following extension is enabled by default:
 sphinx.ext.autosectionlabel
 (https://www.sphinx-doc.org/en/master/usage/extensions/autosectionlabel.html).
 
@@ -264,7 +274,7 @@ autosectionlabel_prefix_document is enabled by default, so that we can
 insert references from any document.
 
 For example, to insert an HTML link to a section from
-documentaion/manual/intro.rst, use:
+documentation/manual/intro.rst, use:
 
   Please check this :ref:`manual/intro:Cross-References to Locations in the Same Document`
 
@@ -287,7 +297,8 @@ Extlinks
 
 The sphinx.ext.extlinks extension is enabled by default
 (https://sublime-and-sphinx-guide.readthedocs.io/en/latest/references.html#use-the-external-links-extension),
-and it is configured with:
+and it is configured with the 'extlinks' definitions in
+the 'documentation/conf.py' file:
 
   'yocto_home': ('https://yoctoproject.org%s', None),
   'yocto_wiki': ('https://wiki.yoctoproject.org%s', None),
@@ -299,6 +310,10 @@ and it is configured with:
   'yocto_git': ('https://git.yoctoproject.org%s', None),
   'oe_home': ('https://www.openembedded.org%s', None),
   'oe_lists': ('https://lists.openembedded.org%s', None),
+  'oe_git': ('https://git.openembedded.org%s', None),
+  'oe_wiki': ('https://www.openembedded.org/wiki%s', None),
+  'oe_layerindex': ('https://layers.openembedded.org%s', None),
+  'oe_layer': ('https://layers.openembedded.org/layerindex/branch/master/layer%s', None),
 
 It creates convenient shortcuts which can be used throughout the
 documentation rst files, as:
@@ -318,3 +333,9 @@ References to the bitbake manual can be done like this:
   See the ":ref:`-D <bitbake:bitbake-user-manual/bitbake-user-manual-intro:usage and syntax>`" option
 or
   :term:`bitbake:BB_NUMBER_PARSE_THREADS`
+
+Submitting documentation changes
+================================
+
+Please see the top level README file in this repository for details of where
+to send patches.

documentation/adt-manual/adt-command.rst

@@ -1,180 +0,0 @@
-.. SPDX-License-Identifier: CC-BY-SA-2.0-UK
-
-**********************
-Using the Command Line
-**********************
-
-Recall that earlier the manual discussed how to use an existing
-toolchain tarball that had been installed into the default installation
-directory, ``/opt/poky/DISTRO``, which is outside of the :term:`Build Directory`
-(see the section
-"`Using a Cross-Toolchain
-Tarball) <#using-an-existing-toolchain-tarball>`__". And, that sourcing
-your architecture-specific environment setup script initializes a
-suitable cross-toolchain development environment.
-
-During this setup, locations for the compiler, QEMU scripts, QEMU
-binary, a special version of ``pkgconfig`` and other useful utilities
-are added to the ``PATH`` variable. Also, variables to assist
-``pkgconfig`` and ``autotools`` are also defined so that, for example,
-``configure.sh`` can find pre-generated test results for tests that need
-target hardware on which to run. You can see the "`Setting Up the
-Cross-Development
-Environment <#setting-up-the-cross-development-environment>`__" section
-for the list of cross-toolchain environment variables established by the
-script.
-
-Collectively, these conditions allow you to easily use the toolchain
-outside of the OpenEmbedded build environment on both Autotools-based
-projects and Makefile-based projects. This chapter provides information
-for both these types of projects.
-
-Autotools-Based Projects
-========================
-
-Once you have a suitable cross-toolchain installed, it is very easy to
-develop a project outside of the OpenEmbedded build system. This section
-presents a simple "Helloworld" example that shows how to set up,
-compile, and run the project.
-
-Creating and Running a Project Based on GNU Autotools
------------------------------------------------------
-
-Follow these steps to create a simple Autotools-based project:
-
-1.  *Create your directory:* Create a clean directory for your project
-    and then make that directory your working location: $ mkdir
-    $HOME/helloworld $ cd $HOME/helloworld
-
-2.  *Populate the directory:* Create ``hello.c``, ``Makefile.am``, and
-    ``configure.in`` files as follows:
-
-    -  For ``hello.c``, include these lines: #include <stdio.h> main() {
-       printf("Hello World!\n"); }
-
-    -  For ``Makefile.am``, include these lines: bin_PROGRAMS = hello
-       hello_SOURCES = hello.c
-
-    -  For ``configure.in``, include these lines: AC_INIT(hello.c)
-       AM_INIT_AUTOMAKE(hello,0.1) AC_PROG_CC AC_PROG_INSTALL
-       AC_OUTPUT(Makefile)
-
-3.  *Source the cross-toolchain environment setup file:* Installation of
-    the cross-toolchain creates a cross-toolchain environment setup
-    script in the directory that the ADT was installed. Before you can
-    use the tools to develop your project, you must source this setup
-    script. The script begins with the string "environment-setup" and
-    contains the machine architecture, which is followed by the string
-    "poky-linux". Here is an example that sources a script from the
-    default ADT installation directory that uses the 32-bit Intel x86
-    Architecture and the DISTRO_NAME Yocto Project release: $ source
-    /opt/poky/DISTRO/environment-setup-i586-poky-linux
-
-4.  *Generate the local aclocal.m4 files and create the configure
-    script:* The following GNU Autotools generate the local
-    ``aclocal.m4`` files and create the configure script: $ aclocal $
-    autoconf
-
-5.  *Generate files needed by GNU coding standards:* GNU coding
-    standards require certain files in order for the project to be
-    compliant. This command creates those files: $ touch NEWS README
-    AUTHORS ChangeLog
-
-6.  *Generate the configure file:* This command generates the
-    ``configure``: $ automake -a
-
-7.  *Cross-compile the project:* This command compiles the project using
-    the cross-compiler. The
-    :term:`CONFIGURE_FLAGS`
-    environment variable provides the minimal arguments for GNU
-    configure: $ ./configure ${CONFIGURE_FLAGS}
-
-8.  *Make and install the project:* These two commands generate and
-    install the project into the destination directory: $ make $ make
-    install DESTDIR=./tmp
-
-9.  *Verify the installation:* This command is a simple way to verify
-    the installation of your project. Running the command prints the
-    architecture on which the binary file can run. This architecture
-    should be the same architecture that the installed cross-toolchain
-    supports. $ file ./tmp/usr/local/bin/hello
-
-10. *Execute your project:* To execute the project in the shell, simply
-    enter the name. You could also copy the binary to the actual target
-    hardware and run the project there as well: $ ./hello As expected,
-    the project displays the "Hello World!" message.
-
-Passing Host Options
---------------------
-
-For an Autotools-based project, you can use the cross-toolchain by just
-passing the appropriate host option to ``configure.sh``. The host option
-you use is derived from the name of the environment setup script found
-in the directory in which you installed the cross-toolchain. For
-example, the host option for an ARM-based target that uses the GNU EABI
-is ``armv5te-poky-linux-gnueabi``. You will notice that the name of the
-script is ``environment-setup-armv5te-poky-linux-gnueabi``. Thus, the
-following command works to update your project and rebuild it using the
-appropriate cross-toolchain tools: $ ./configure
---host=armv5te-poky-linux-gnueabi \\ --with-libtool-sysroot=sysroot_dir
-
-.. note::
-
-   If the
-   configure
-   script results in problems recognizing the
-   --with-libtool-sysroot=
-   sysroot-dir
-   option, regenerate the script to enable the support by doing the
-   following and then run the script again:
-   ::
-
-           $ libtoolize --automake
-           $ aclocal -I ${OECORE_NATIVE_SYSROOT}/usr/share/aclocal \
-              [-I dir_containing_your_project-specific_m4_macros]
-           $ autoconf
-           $ autoheader
-           $ automake -a
-                      
-
-Makefile-Based Projects
-=======================
-
-For Makefile-based projects, the cross-toolchain environment variables
-established by running the cross-toolchain environment setup script are
-subject to general ``make`` rules.
-
-To illustrate this, consider the following four cross-toolchain
-environment variables:
-:term:`CC`\ =i586-poky-linux-gcc -m32
--march=i586 --sysroot=/opt/poky/1.8/sysroots/i586-poky-linux
-:term:`LD`\ =i586-poky-linux-ld
---sysroot=/opt/poky/1.8/sysroots/i586-poky-linux
-:term:`CFLAGS`\ =-O2 -pipe -g
--feliminate-unused-debug-types
-:term:`CXXFLAGS`\ =-O2 -pipe -g
--feliminate-unused-debug-types Now, consider the following three cases:
-
--  *Case 1 - No Variables Set in the ``Makefile``:* Because these
-   variables are not specifically set in the ``Makefile``, the variables
-   retain their values based on the environment.
-
--  *Case 2 - Variables Set in the ``Makefile``:* Specifically setting
-   variables in the ``Makefile`` during the build results in the
-   environment settings of the variables being overwritten.
-
--  *Case 3 - Variables Set when the ``Makefile`` is Executed from the
-   Command Line:* Executing the ``Makefile`` from the command line
-   results in the variables being overwritten with command-line content
-   regardless of what is being set in the ``Makefile``. In this case,
-   environment variables are not considered unless you use the "-e" flag
-   during the build: $ make -e file If you use this flag, then the
-   environment values of the variables override any variables
-   specifically set in the ``Makefile``.
-
-.. note::
-
-   For the list of variables set up by the cross-toolchain environment
-   setup script, see the "
-   Setting Up the Cross-Development Environment
-   " section.

documentation/adt-manual/adt-intro.rst

@@ -1,138 +0,0 @@
-.. SPDX-License-Identifier: CC-BY-SA-2.0-UK
-
-*****************************************
-The Application Development Toolkit (ADT)
-*****************************************
-
-Part of the Yocto Project development solution is an Application
-Development Toolkit (ADT). The ADT provides you with a custom-built,
-cross-development platform suited for developing a user-targeted product
-application.
-
-Fundamentally, the ADT consists of the following:
-
--  An architecture-specific cross-toolchain and matching sysroot both
-   built by the :term:`OpenEmbedded Build System`.
-   The toolchain and
-   sysroot are based on a `Metadata <&YOCTO_DOCS_DEV_URL;#metadata>`__
-   configuration and extensions, which allows you to cross-develop on
-   the host machine for the target hardware.
-
--  The Eclipse IDE Yocto Plug-in.
-
--  The Quick EMUlator (QEMU), which lets you simulate target hardware.
-
--  Various user-space tools that greatly enhance your application
-   development experience.
-
-The Cross-Development Toolchain
-===============================
-
-The `Cross-Development
-Toolchain <&YOCTO_DOCS_DEV_URL;#cross-development-toolchain>`__ consists
-of a cross-compiler, cross-linker, and cross-debugger that are used to
-develop user-space applications for targeted hardware. This toolchain is
-created either by running the ADT Installer script, a toolchain
-installer script, or through a :term:`Build Directory`
-that is based on
-your Metadata configuration or extension for your targeted device. The
-cross-toolchain works with a matching target sysroot.
-
-Sysroot
-=======
-
-The matching target sysroot contains needed headers and libraries for
-generating binaries that run on the target architecture. The sysroot is
-based on the target root filesystem image that is built by the
-OpenEmbedded build system and uses the same Metadata configuration used
-to build the cross-toolchain.
-
-.. _eclipse-overview:
-
-Eclipse Yocto Plug-in
-=====================
-
-The Eclipse IDE is a popular development environment and it fully
-supports development using the Yocto Project. When you install and
-configure the Eclipse Yocto Project Plug-in into the Eclipse IDE, you
-maximize your Yocto Project experience. Installing and configuring the
-Plug-in results in an environment that has extensions specifically
-designed to let you more easily develop software. These extensions allow
-for cross-compilation, deployment, and execution of your output into a
-QEMU emulation session. You can also perform cross-debugging and
-profiling. The environment also supports a suite of tools that allows
-you to perform remote profiling, tracing, collection of power data,
-collection of latency data, and collection of performance data.
-
-For information about the application development workflow that uses the
-Eclipse IDE and for a detailed example of how to install and configure
-the Eclipse Yocto Project Plug-in, see the "`Working Within
-Eclipse <&YOCTO_DOCS_DEV_URL;#adt-eclipse>`__" section of the Yocto
-Project Development Manual.
-
-The QEMU Emulator
-=================
-
-The QEMU emulator allows you to simulate your hardware while running
-your application or image. QEMU is made available a number of ways:
-
--  If you use the ADT Installer script to install ADT, you can specify
-   whether or not to install QEMU.
-
--  If you have cloned the ``poky`` Git repository to create a
-   :term:`Source Directory` and you have
-   sourced the environment setup script, QEMU is installed and
-   automatically available.
-
--  If you have downloaded a Yocto Project release and unpacked it to
-   create a :term:`Source Directory`
-   and you have sourced the environment setup script, QEMU is installed
-   and automatically available.
-
--  If you have installed the cross-toolchain tarball and you have
-   sourced the toolchain's setup environment script, QEMU is also
-   installed and automatically available.
-
-User-Space Tools
-================
-
-User-space tools are included as part of the Yocto Project. You will
-find these tools helpful during development. The tools include
-LatencyTOP, PowerTOP, OProfile, Perf, SystemTap, and Lttng-ust. These
-tools are common development tools for the Linux platform.
-
--  *LatencyTOP:* LatencyTOP focuses on latency that causes skips in
-   audio, stutters in your desktop experience, or situations that
-   overload your server even when you have plenty of CPU power left.
-
--  *PowerTOP:* Helps you determine what software is using the most
-   power. You can find out more about PowerTOP at
-   https://01.org/powertop/.
-
--  *OProfile:* A system-wide profiler for Linux systems that is capable
-   of profiling all running code at low overhead. You can find out more
-   about OProfile at http://oprofile.sourceforge.net/about/. For
-   examples on how to setup and use this tool, see the
-   "`OProfile <&YOCTO_DOCS_PROF_URL;#profile-manual-oprofile>`__"
-   section in the Yocto Project Profiling and Tracing Manual.
-
--  *Perf:* Performance counters for Linux used to keep track of certain
-   types of hardware and software events. For more information on these
-   types of counters see https://perf.wiki.kernel.org/. For
-   examples on how to setup and use this tool, see the
-   "`perf <&YOCTO_DOCS_PROF_URL;#profile-manual-perf>`__" section in the
-   Yocto Project Profiling and Tracing Manual.
-
--  *SystemTap:* A free software infrastructure that simplifies
-   information gathering about a running Linux system. This information
-   helps you diagnose performance or functional problems. SystemTap is
-   not available as a user-space tool through the Eclipse IDE Yocto
-   Plug-in. See http://sourceware.org/systemtap for more
-   information on SystemTap. For examples on how to setup and use this
-   tool, see the
-   "`SystemTap <&YOCTO_DOCS_PROF_URL;#profile-manual-systemtap>`__"
-   section in the Yocto Project Profiling and Tracing Manual.
-
--  *Lttng-ust:* A User-space Tracer designed to provide detailed
-   information on user-space activity. See http://lttng.org/ust
-   for more information on Lttng-ust.

documentation/adt-manual/adt-manual-intro.rst

@@ -1,24 +0,0 @@
-.. SPDX-License-Identifier: CC-BY-SA-2.0-UK
-
-************
-Introduction
-************
-
-Welcome to the Yocto Project Application Developer's Guide. This manual
-provides information that lets you begin developing applications using
-the Yocto Project.
-
-The Yocto Project provides an application development environment based
-on an Application Development Toolkit (ADT) and the availability of
-stand-alone cross-development toolchains and other tools. This manual
-describes the ADT and how you can configure and install it, how to
-access and use the cross-development toolchains, how to customize the
-development packages installation, how to use command-line development
-for both Autotools-based and Makefile-based projects, and an
-introduction to the Eclipse IDE Yocto Plug-in.
-
-.. note::
-
-   The ADT is distribution-neutral and does not require the Yocto
-   Project reference distribution, which is called Poky. This manual,
-   however, uses examples that use the Poky distribution.

documentation/adt-manual/adt-manual.rst

@@ -1,17 +0,0 @@
-.. SPDX-License-Identifier: CC-BY-SA-2.0-UK
-
-===========================================
-Yocto Project Application Developer's Guide
-===========================================
-
-|
-
-.. toctree::
-   :caption: Table of Contents
-   :numbered:
-
-   adt-manual-intro
-   adt-intro
-   adt-prepare
-   adt-package
-   adt-command

documentation/adt-manual/adt-package.rst

@@ -1,70 +0,0 @@
-.. SPDX-License-Identifier: CC-BY-SA-2.0-UK
-
-************************************************************
-Optionally Customizing the Development Packages Installation
-************************************************************
-
-Because the Yocto Project is suited for embedded Linux development, it
-is likely that you will need to customize your development packages
-installation. For example, if you are developing a minimal image, then
-you might not need certain packages (e.g. graphics support packages).
-Thus, you would like to be able to remove those packages from your
-target sysroot.
-
-Package Management Systems
-==========================
-
-The OpenEmbedded build system supports the generation of sysroot files
-using three different Package Management Systems (PMS):
-
--  *OPKG:* A less well known PMS whose use originated in the
-   OpenEmbedded and OpenWrt embedded Linux projects. This PMS works with
-   files packaged in an ``.ipk`` format. See
-   http://en.wikipedia.org/wiki/Opkg for more information about
-   OPKG.
-
--  *RPM:* A more widely known PMS intended for GNU/Linux distributions.
-   This PMS works with files packaged in an ``.rpm`` format. The build
-   system currently installs through this PMS by default. See
-   http://en.wikipedia.org/wiki/RPM_Package_Manager for more
-   information about RPM.
-
--  *Debian:* The PMS for Debian-based systems is built on many PMS
-   tools. The lower-level PMS tool ``dpkg`` forms the base of the Debian
-   PMS. For information on dpkg see
-   http://en.wikipedia.org/wiki/Dpkg.
-
-Configuring the PMS
-===================
-
-Whichever PMS you are using, you need to be sure that the
-:term:`PACKAGE_CLASSES`
-variable in the ``conf/local.conf`` file is set to reflect that system.
-The first value you choose for the variable specifies the package file
-format for the root filesystem at sysroot. Additional values specify
-additional formats for convenience or testing. See the
-``conf/local.conf`` configuration file for details.
-
-.. note::
-
-   For build performance information related to the PMS, see the "
-   package.bbclass
-   " section in the Yocto Project Reference Manual.
-
-As an example, consider a scenario where you are using OPKG and you want
-to add the ``libglade`` package to the target sysroot.
-
-First, you should generate the IPK file for the ``libglade`` package and
-add it into a working ``opkg`` repository. Use these commands: $ bitbake
-libglade $ bitbake package-index
-
-Next, source the cross-toolchain environment setup script found in the
-:term:`Source Directory`. Follow
-that by setting up the installation destination to point to your sysroot
-as sysroot_dir. Finally, have an OPKG configuration file conf_file that
-corresponds to the ``opkg`` repository you have just created. The
-following command forms should now work: $ opkg-cl –f conf_file -o
-sysroot_dir update $ opkg-cl –f cconf_file -o sysroot_dir \\
---force-overwrite install libglade $ opkg-cl –f cconf_file -o
-sysroot_dir \\ --force-overwrite install libglade-dbg $ opkg-cl –f
-conf_file> -osysroot_dir> \\ --force-overwrite install libglade-dev

documentation/adt-manual/adt-prepare.rst

@@ -1,752 +0,0 @@
-.. SPDX-License-Identifier: CC-BY-SA-2.0-UK
-
-*************************************
-Preparing for Application Development
-*************************************
-
-In order to develop applications, you need set up your host development
-system. Several ways exist that allow you to install cross-development
-tools, QEMU, the Eclipse Yocto Plug-in, and other tools. This chapter
-describes how to prepare for application development.
-
-.. _installing-the-adt:
-
-Installing the ADT and Toolchains
-=================================
-
-The following list describes installation methods that set up varying
-degrees of tool availability on your system. Regardless of the
-installation method you choose, you must ``source`` the cross-toolchain
-environment setup script, which establishes several key environment
-variables, before you use a toolchain. See the "`Setting Up the
-Cross-Development
-Environment <#setting-up-the-cross-development-environment>`__" section
-for more information.
-
-.. note::
-
-   Avoid mixing installation methods when installing toolchains for
-   different architectures. For example, avoid using the ADT Installer
-   to install some toolchains and then hand-installing cross-development
-   toolchains by running the toolchain installer for different
-   architectures. Mixing installation methods can result in situations
-   where the ADT Installer becomes unreliable and might not install the
-   toolchain.
-
-   If you must mix installation methods, you might avoid problems by
-   deleting ``/var/lib/opkg``, thus purging the ``opkg`` package
-   metadata.
-
--  *Use the ADT installer script:* This method is the recommended way to
-   install the ADT because it automates much of the process for you. For
-   example, you can configure the installation to install the QEMU
-   emulator and the user-space NFS, specify which root filesystem
-   profiles to download, and define the target sysroot location.
-
--  *Use an existing toolchain:* Using this method, you select and
-   download an architecture-specific toolchain installer and then run
-   the script to hand-install the toolchain. If you use this method, you
-   just get the cross-toolchain and QEMU - you do not get any of the
-   other mentioned benefits had you run the ADT Installer script.
-
--  *Use the toolchain from within the Build Directory:* If you already
-   have a :term:`Build Directory`,
-   you can build the cross-toolchain within the directory. However, like
-   the previous method mentioned, you only get the cross-toolchain and
-   QEMU - you do not get any of the other benefits without taking
-   separate steps.
-
-Using the ADT Installer
------------------------
-
-To run the ADT Installer, you need to get the ADT Installer tarball, be
-sure you have the necessary host development packages that support the
-ADT Installer, and then run the ADT Installer Script.
-
-For a list of the host packages needed to support ADT installation and
-use, see the "ADT Installer Extras" lists in the "`Required Packages for
-the Host Development
-System <&YOCTO_DOCS_REF_URL;#required-packages-for-the-host-development-system>`__"
-section of the Yocto Project Reference Manual.
-
-Getting the ADT Installer Tarball
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-
-The ADT Installer is contained in the ADT Installer tarball. You can get
-the tarball using either of these methods:
-
--  *Download the Tarball:* You can download the tarball from
-   ` <&YOCTO_ADTINSTALLER_DL_URL;>`__ into any directory.
-
--  *Build the Tarball:* You can use
-   :term:`BitBake` to generate the
-   tarball inside an existing :term:`Build Directory`.
-
-   If you use BitBake to generate the ADT Installer tarball, you must
-   ``source`` the environment setup script
-   (````` <&YOCTO_DOCS_REF_URL;#structure-core-script>`__ or
-   ```oe-init-build-env-memres`` <&YOCTO_DOCS_REF_URL;#structure-memres-core-script>`__)
-   located in the Source Directory before running the ``bitbake``
-   command that creates the tarball.
-
-   The following example commands establish the
-   :term:`Source Directory`, check out the
-   current release branch, set up the build environment while also
-   creating the default Build Directory, and run the ``bitbake`` command
-   that results in the tarball
-   ``poky/build/tmp/deploy/sdk/adt_installer.tar.bz2``:
-
-   .. note::
-
-      Before using BitBake to build the ADT tarball, be sure to make
-      sure your
-      local.conf
-      file is properly configured. See the "
-      User Configuration
-      " section in the Yocto Project Reference Manual for general
-      configuration information.
-
-   $ cd ~ $ git clone git://git.yoctoproject.org/poky $ cd poky $ git
-   checkout -b DISTRO_NAME origin/DISTRO_NAME $ source oe-init-build-env $
-   bitbake adt-installer
-
-Configuring and Running the ADT Installer Script
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-
-Before running the ADT Installer script, you need to unpack the tarball.
-You can unpack the tarball in any directory you wish. For example, this
-command copies the ADT Installer tarball from where it was built into
-the home directory and then unpacks the tarball into a top-level
-directory named ``adt-installer``: $ cd ~ $ cp
-poky/build/tmp/deploy/sdk/adt_installer.tar.bz2 $HOME $ tar -xjf
-adt_installer.tar.bz2 Unpacking it creates the directory
-``adt-installer``, which contains the ADT Installer script
-(``adt_installer``) and its configuration file (``adt_installer.conf``).
-
-Before you run the script, however, you should examine the ADT Installer
-configuration file and be sure you are going to get what you want. Your
-configurations determine which kernel and filesystem image are
-downloaded.
-
-The following list describes the configurations you can define for the
-ADT Installer. For configuration values and restrictions, see the
-comments in the ``adt-installer.conf`` file:
-
--  ``YOCTOADT_REPO``: This area includes the IPKG-based packages and the
-   root filesystem upon which the installation is based. If you want to
-   set up your own IPKG repository pointed to by ``YOCTOADT_REPO``, you
-   need to be sure that the directory structure follows the same layout
-   as the reference directory set up at
-   http://adtrepo.yoctoproject.org. Also, your repository needs
-   to be accessible through HTTP.
-
--  ``YOCTOADT_TARGETS``: The machine target architectures for which you
-   want to set up cross-development environments.
-
--  ``YOCTOADT_QEMU``: Indicates whether or not to install the emulator
-   QEMU.
-
--  ``YOCTOADT_NFS_UTIL``: Indicates whether or not to install user-mode
-   NFS. If you plan to use the Eclipse IDE Yocto plug-in against QEMU,
-   you should install NFS.
-
-   .. note::
-
-      To boot QEMU images using our userspace NFS server, you need to be
-      running
-      portmap
-      or
-      rpcbind
-      . If you are running
-      rpcbind
-      , you will also need to add the
-      -i
-      option when
-      rpcbind
-      starts up. Please make sure you understand the security
-      implications of doing this. You might also have to modify your
-      firewall settings to allow NFS booting to work.
-
--  ``YOCTOADT_ROOTFS_``\ arch: The root filesystem images you want to
-   download from the ``YOCTOADT_IPKG_REPO`` repository.
-
--  ``YOCTOADT_TARGET_SYSROOT_IMAGE_``\ arch: The particular root
-   filesystem used to extract and create the target sysroot. The value
-   of this variable must have been specified with
-   ``YOCTOADT_ROOTFS_``\ arch. For example, if you downloaded both
-   ``minimal`` and ``sato-sdk`` images by setting
-   ``YOCTOADT_ROOTFS_``\ arch to "minimal sato-sdk", then
-   ``YOCTOADT_ROOTFS_``\ arch must be set to either "minimal" or
-   "sato-sdk".
-
--  ``YOCTOADT_TARGET_SYSROOT_LOC_``\ arch: The location on the
-   development host where the target sysroot is created.
-
-After you have configured the ``adt_installer.conf`` file, run the
-installer using the following command: $ cd adt-installer $
-./adt_installer Once the installer begins to run, you are asked to enter
-the location for cross-toolchain installation. The default location is
-``/opt/poky/``\ release. After either accepting the default location or
-selecting your own location, you are prompted to run the installation
-script interactively or in silent mode. If you want to closely monitor
-the installation, choose "I" for interactive mode rather than "S" for
-silent mode. Follow the prompts from the script to complete the
-installation.
-
-Once the installation completes, the ADT, which includes the
-cross-toolchain, is installed in the selected installation directory.
-You will notice environment setup files for the cross-toolchain in the
-installation directory, and image tarballs in the ``adt-installer``
-directory according to your installer configurations, and the target
-sysroot located according to the ``YOCTOADT_TARGET_SYSROOT_LOC_``\ arch
-variable also in your configuration file.
-
-.. _using-an-existing-toolchain-tarball:
-
-Using a Cross-Toolchain Tarball
--------------------------------
-
-If you want to simply install a cross-toolchain by hand, you can do so
-by running the toolchain installer. The installer includes the pre-built
-cross-toolchain, the ``runqemu`` script, and support files. If you use
-this method to install the cross-toolchain, you might still need to
-install the target sysroot by installing and extracting it separately.
-For information on how to install the sysroot, see the "`Extracting the
-Root Filesystem <#extracting-the-root-filesystem>`__" section.
-
-Follow these steps:
-
-1. *Get your toolchain installer using one of the following methods:*
-
-   -  Go to ` <&YOCTO_TOOLCHAIN_DL_URL;>`__ and find the folder that
-      matches your host development system (i.e. ``i686`` for 32-bit
-      machines or ``x86_64`` for 64-bit machines).
-
-      Go into that folder and download the toolchain installer whose
-      name includes the appropriate target architecture. The toolchains
-      provided by the Yocto Project are based off of the
-      ``core-image-sato`` image and contain libraries appropriate for
-      developing against that image. For example, if your host
-      development system is a 64-bit x86 system and you are going to use
-      your cross-toolchain for a 32-bit x86 target, go into the
-      ``x86_64`` folder and download the following installer:
-      poky-glibc-x86_64-core-image-sato-i586-toolchain-DISTRO.sh
-
-   -  Build your own toolchain installer. For cases where you cannot use
-      an installer from the download area, you can build your own as
-      described in the "`Optionally Building a Toolchain
-      Installer <#optionally-building-a-toolchain-installer>`__"
-      section.
-
-2. *Once you have the installer, run it to install the toolchain:*
-
-   .. note::
-
-      You must change the permissions on the toolchain installer script
-      so that it is executable.
-
-   The following command shows how to run the installer given a
-   toolchain tarball for a 64-bit x86 development host system and a
-   32-bit x86 target architecture. The example assumes the toolchain
-   installer is located in ``~/Downloads/``. $
-   ~/Downloads/poky-glibc-x86_64-core-image-sato-i586-toolchain-DISTRO.sh
-   The first thing the installer prompts you for is the directory into
-   which you want to install the toolchain. The default directory used
-   is ``/opt/poky/DISTRO``. If you do not have write permissions for the
-   directory into which you are installing the toolchain, the toolchain
-   installer notifies you and exits. Be sure you have write permissions
-   in the directory and run the installer again.
-
-   When the script finishes, the cross-toolchain is installed. You will
-   notice environment setup files for the cross-toolchain in the
-   installation directory.
-
-.. _using-the-toolchain-from-within-the-build-tree:
-
-Using BitBake and the Build Directory
--------------------------------------
-
-A final way of making the cross-toolchain available is to use BitBake to
-generate the toolchain within an existing :term:`Build Directory`.
-This method does
-not install the toolchain into the default ``/opt`` directory. As with
-the previous method, if you need to install the target sysroot, you must
-do that separately as well.
-
-Follow these steps to generate the toolchain into the Build Directory:
-
-1. *Set up the Build Environment:* Source the OpenEmbedded build
-   environment setup script (i.e.
-   ````` <&YOCTO_DOCS_REF_URL;#structure-core-script>`__ or
-   ```oe-init-build-env-memres`` <&YOCTO_DOCS_REF_URL;#structure-memres-core-script>`__)
-   located in the :term:`Source Directory`.
-
-2. *Check your Local Configuration File:* At this point, you should be
-   sure that the :term:`MACHINE`
-   variable in the ``local.conf`` file found in the ``conf`` directory
-   of the Build Directory is set for the target architecture. Comments
-   within the ``local.conf`` file list the values you can use for the
-   ``MACHINE`` variable. If you do not change the ``MACHINE`` variable,
-   the OpenEmbedded build system uses ``qemux86`` as the default target
-   machine when building the cross-toolchain.
-
-   .. note::
-
-      You can populate the Build Directory with the cross-toolchains for
-      more than a single architecture. You just need to edit the
-      MACHINE
-      variable in the
-      local.conf
-      file and re-run the
-      bitbake
-      command.
-
-3. *Make Sure Your Layers are Enabled:* Examine the
-   ``conf/bblayers.conf`` file and make sure that you have enabled all
-   the compatible layers for your target machine. The OpenEmbedded build
-   system needs to be aware of each layer you want included when
-   building images and cross-toolchains. For information on how to
-   enable a layer, see the "`Enabling Your
-   Layer <&YOCTO_DOCS_DEV_URL;#enabling-your-layer>`__" section in the
-   Yocto Project Development Manual.
-
-4. *Generate the Cross-Toolchain:* Run ``bitbake meta-ide-support`` to
-   complete the cross-toolchain generation. Once the ``bitbake`` command
-   finishes, the cross-toolchain is generated and populated within the
-   Build Directory. You will notice environment setup files for the
-   cross-toolchain that contain the string "``environment-setup``" in
-   the Build Directory's ``tmp`` folder.
-
-   Be aware that when you use this method to install the toolchain, you
-   still need to separately extract and install the sysroot filesystem.
-   For information on how to do this, see the "`Extracting the Root
-   Filesystem <#extracting-the-root-filesystem>`__" section.
-
-Setting Up the Cross-Development Environment
-============================================
-
-Before you can develop using the cross-toolchain, you need to set up the
-cross-development environment by sourcing the toolchain's environment
-setup script. If you used the ADT Installer or hand-installed
-cross-toolchain, then you can find this script in the directory you
-chose for installation. For this release, the default installation
-directory is ````. If you installed the toolchain in the
-:term:`Build Directory`, you can find the
-environment setup script for the toolchain in the Build Directory's
-``tmp`` directory.
-
-Be sure to run the environment setup script that matches the
-architecture for which you are developing. Environment setup scripts
-begin with the string "``environment-setup``" and include as part of
-their name the architecture. For example, the toolchain environment
-setup script for a 64-bit IA-based architecture installed in the default
-installation directory would be the following:
-YOCTO_ADTPATH_DIR/environment-setup-x86_64-poky-linux When you run the
-setup script, many environment variables are defined:
-:term:`SDKTARGETSYSROOT` -
-The path to the sysroot used for cross-compilation
-:term:`PKG_CONFIG_PATH` - The
-path to the target pkg-config files
-:term:`CONFIG_SITE` - A GNU
-autoconf site file preconfigured for the target
-:term:`CC` - The minimal command and
-arguments to run the C compiler
-:term:`CXX` - The minimal command and
-arguments to run the C++ compiler
-:term:`CPP` - The minimal command and
-arguments to run the C preprocessor
-:term:`AS` - The minimal command and
-arguments to run the assembler :term:`LD`
-- The minimal command and arguments to run the linker
-:term:`GDB` - The minimal command and
-arguments to run the GNU Debugger
-:term:`STRIP` - The minimal command and
-arguments to run 'strip', which strips symbols
-:term:`RANLIB` - The minimal command
-and arguments to run 'ranlib'
-:term:`OBJCOPY` - The minimal command
-and arguments to run 'objcopy'
-:term:`OBJDUMP` - The minimal command
-and arguments to run 'objdump' :term:`AR`
-- The minimal command and arguments to run 'ar'
-:term:`NM` - The minimal command and
-arguments to run 'nm'
-:term:`TARGET_PREFIX` - The
-toolchain binary prefix for the target tools
-:term:`CROSS_COMPILE` - The
-toolchain binary prefix for the target tools
-:term:`CONFIGURE_FLAGS` - The
-minimal arguments for GNU configure
-:term:`CFLAGS` - Suggested C flags
-:term:`CXXFLAGS` - Suggested C++
-flags :term:`LDFLAGS` - Suggested
-linker flags when you use CC to link
-:term:`CPPFLAGS` - Suggested
-preprocessor flags
-
-Securing Kernel and Filesystem Images
-=====================================
-
-You will need to have a kernel and filesystem image to boot using your
-hardware or the QEMU emulator. Furthermore, if you plan on booting your
-image using NFS or you want to use the root filesystem as the target
-sysroot, you need to extract the root filesystem.
-
-Getting the Images
-------------------
-
-To get the kernel and filesystem images, you either have to build them
-or download pre-built versions. For an example of how to build these
-images, see the "`Buiding
-Images <&YOCTO_DOCS_QS_URL;#qs-buiding-images>`__" section of the Yocto
-Project Quick Start. For an example of downloading pre-build versions,
-see the "`Example Using Pre-Built Binaries and
-QEMU <#using-pre-built>`__" section.
-
-The Yocto Project ships basic kernel and filesystem images for several
-architectures (``x86``, ``x86-64``, ``mips``, ``powerpc``, and ``arm``)
-that you can use unaltered in the QEMU emulator. These kernel images
-reside in the release area - ` <&YOCTO_MACHINES_DL_URL;>`__ and are
-ideal for experimentation using Yocto Project. For information on the
-image types you can build using the OpenEmbedded build system, see the
-":ref:`ref-manual/ref-images:Images`" chapter in the Yocto
-Project Reference Manual.
-
-If you are planning on developing against your image and you are not
-building or using one of the Yocto Project development images (e.g.
-``core-image-*-dev``), you must be sure to include the development
-packages as part of your image recipe.
-
-If you plan on remotely deploying and debugging your application from
-within the Eclipse IDE, you must have an image that contains the Yocto
-Target Communication Framework (TCF) agent (``tcf-agent``). You can do
-this by including the ``eclipse-debug`` image feature.
-
-.. note::
-
-   See the "
-   Image Features
-   " section in the Yocto Project Reference Manual for information on
-   image features.
-
-To include the ``eclipse-debug`` image feature, modify your
-``local.conf`` file in the :term:`Build Directory`
-so that the
-:term:`EXTRA_IMAGE_FEATURES`
-variable includes the "eclipse-debug" feature. After modifying the
-configuration file, you can rebuild the image. Once the image is
-rebuilt, the ``tcf-agent`` will be included in the image and is launched
-automatically after the boot.
-
-Extracting the Root Filesystem
-------------------------------
-
-If you install your toolchain by hand or build it using BitBake and you
-need a root filesystem, you need to extract it separately. If you use
-the ADT Installer to install the ADT, the root filesystem is
-automatically extracted and installed.
-
-Here are some cases where you need to extract the root filesystem:
-
--  You want to boot the image using NFS.
-
--  You want to use the root filesystem as the target sysroot. For
-   example, the Eclipse IDE environment with the Eclipse Yocto Plug-in
-   installed allows you to use QEMU to boot under NFS.
-
--  You want to develop your target application using the root filesystem
-   as the target sysroot.
-
-To extract the root filesystem, first ``source`` the cross-development
-environment setup script to establish necessary environment variables.
-If you built the toolchain in the Build Directory, you will find the
-toolchain environment script in the ``tmp`` directory. If you installed
-the toolchain by hand, the environment setup script is located in
-``/opt/poky/DISTRO``.
-
-After sourcing the environment script, use the ``runqemu-extract-sdk``
-command and provide the filesystem image.
-
-Following is an example. The second command sets up the environment. In
-this case, the setup script is located in the ``/opt/poky/DISTRO``
-directory. The third command extracts the root filesystem from a
-previously built filesystem that is located in the ``~/Downloads``
-directory. Furthermore, this command extracts the root filesystem into
-the ``qemux86-sato`` directory: $ cd ~ $ source
-/opt/poky/DISTRO/environment-setup-i586-poky-linux $ runqemu-extract-sdk
-\\ ~/Downloads/core-image-sato-sdk-qemux86-2011091411831.rootfs.tar.bz2
-\\ $HOME/qemux86-sato You could now point to the target sysroot at
-``qemux86-sato``.
-
-Optionally Building a Toolchain Installer
-=========================================
-
-As an alternative to locating and downloading a toolchain installer, you
-can build the toolchain installer if you have a :term:`Build Directory`.
-
-.. note::
-
-   Although not the preferred method, it is also possible to use
-   bitbake meta-toolchain
-   to build the toolchain installer. If you do use this method, you must
-   separately install and extract the target sysroot. For information on
-   how to install the sysroot, see the "
-   Extracting the Root Filesystem
-   " section.
-
-To build the toolchain installer and populate the SDK image, use the
-following command: $ bitbake image -c populate_sdk The command results
-in a toolchain installer that contains the sysroot that matches your
-target root filesystem.
-
-Another powerful feature is that the toolchain is completely
-self-contained. The binaries are linked against their own copy of
-``libc``, which results in no dependencies on the target system. To
-achieve this, the pointer to the dynamic loader is configured at install
-time since that path cannot be dynamically altered. This is the reason
-for a wrapper around the ``populate_sdk`` archive.
-
-Another feature is that only one set of cross-canadian toolchain
-binaries are produced per architecture. This feature takes advantage of
-the fact that the target hardware can be passed to ``gcc`` as a set of
-compiler options. Those options are set up by the environment script and
-contained in variables such as :term:`CC`
-and :term:`LD`. This reduces the space
-needed for the tools. Understand, however, that a sysroot is still
-needed for every target since those binaries are target-specific.
-
-Remember, before using any BitBake command, you must source the build
-environment setup script (i.e.
-````` <&YOCTO_DOCS_REF_URL;#structure-core-script>`__ or
-```oe-init-build-env-memres`` <&YOCTO_DOCS_REF_URL;#structure-memres-core-script>`__)
-located in the Source Directory and you must make sure your
-``conf/local.conf`` variables are correct. In particular, you need to be
-sure the :term:`MACHINE` variable
-matches the architecture for which you are building and that the
-:term:`SDKMACHINE` variable is
-correctly set if you are building a toolchain designed to run on an
-architecture that differs from your current development host machine
-(i.e. the build machine).
-
-When the ``bitbake`` command completes, the toolchain installer will be
-in ``tmp/deploy/sdk`` in the Build Directory.
-
-.. note::
-
-   By default, this toolchain does not build static binaries. If you
-   want to use the toolchain to build these types of libraries, you need
-   to be sure your image has the appropriate static development
-   libraries. Use the
-   IMAGE_INSTALL
-   variable inside your
-   local.conf
-   file to install the appropriate library packages. Following is an
-   example using
-   glibc
-   static development libraries:
-   ::
-
-           IMAGE_INSTALL_append = " glibc-staticdev"
-                  
-
-Optionally Using an External Toolchain
-======================================
-
-You might want to use an external toolchain as part of your development.
-If this is the case, the fundamental steps you need to accomplish are as
-follows:
-
--  Understand where the installed toolchain resides. For cases where you
-   need to build the external toolchain, you would need to take separate
-   steps to build and install the toolchain.
-
--  Make sure you add the layer that contains the toolchain to your
-   ``bblayers.conf`` file through the
-   :term:`BBLAYERS` variable.
-
--  Set the
-   :term:`EXTERNAL_TOOLCHAIN`
-   variable in your ``local.conf`` file to the location in which you
-   installed the toolchain.
-
-A good example of an external toolchain used with the Yocto Project is
-Mentor Graphics Sourcery G++ Toolchain. You can see information on how
-to use that particular layer in the ``README`` file at
-http://github.com/MentorEmbedded/meta-sourcery/. You can find
-further information by reading about the
-:term:`TCMODE` variable in the Yocto
-Project Reference Manual's variable glossary.
-
-.. _using-pre-built:
-
-Example Using Pre-Built Binaries and QEMU
-=========================================
-
-If hardware, libraries and services are stable, you can get started by
-using a pre-built binary of the filesystem image, kernel, and toolchain
-and run it using the QEMU emulator. This scenario is useful for
-developing application software.
-
-|Using a Pre-Built Image|
-
-For this scenario, you need to do several things:
-
--  Install the appropriate stand-alone toolchain tarball.
-
--  Download the pre-built image that will boot with QEMU. You need to be
-   sure to get the QEMU image that matches your target machine's
-   architecture (e.g. x86, ARM, etc.).
-
--  Download the filesystem image for your target machine's architecture.
-
--  Set up the environment to emulate the hardware and then start the
-   QEMU emulator.
-
-Installing the Toolchain
-------------------------
-
-You can download a tarball installer, which includes the pre-built
-toolchain, the ``runqemu`` script, and support files from the
-appropriate directory under ` <&YOCTO_TOOLCHAIN_DL_URL;>`__. Toolchains
-are available for 32-bit and 64-bit x86 development systems from the
-``i686`` and ``x86_64`` directories, respectively. The toolchains the
-Yocto Project provides are based off the ``core-image-sato`` image and
-contain libraries appropriate for developing against that image. Each
-type of development system supports five or more target architectures.
-
-The names of the tarball installer scripts are such that a string
-representing the host system appears first in the filename and then is
-immediately followed by a string representing the target architecture.
-
-::
-
-        poky-glibc-host_system-image_type-arch-toolchain-release_version.sh
-
-        Where:
-            host_system is a string representing your development system:
-
-                       i686 or x86_64.
-
-            image_type is a string representing the image you wish to
-                   develop a Software Development Toolkit (SDK) for use against.
-                   The Yocto Project builds toolchain installers using the
-                   following BitBake command:
-
-                       bitbake core-image-sato -c populate_sdk
-
-            arch is a string representing the tuned target architecture:
-
-                       i586, x86_64, powerpc, mips, armv7a or armv5te
-
-            release_version is a string representing the release number of the
-                   Yocto Project:
-
-                       DISTRO, DISTRO+snapshot
-               
-
-For example, the following toolchain installer is for a 64-bit
-development host system and a i586-tuned target architecture based off
-the SDK for ``core-image-sato``:
-poky-glibc-x86_64-core-image-sato-i586-toolchain-DISTRO.sh
-
-Toolchains are self-contained and by default are installed into
-``/opt/poky``. However, when you run the toolchain installer, you can
-choose an installation directory.
-
-The following command shows how to run the installer given a toolchain
-tarball for a 64-bit x86 development host system and a 32-bit x86 target
-architecture. You must change the permissions on the toolchain installer
-script so that it is executable.
-
-The example assumes the toolchain installer is located in
-``~/Downloads/``.
-
-.. note::
-
-   If you do not have write permissions for the directory into which you
-   are installing the toolchain, the toolchain installer notifies you
-   and exits. Be sure you have write permissions in the directory and
-   run the installer again.
-
-$ ~/Downloads/poky-glibc-x86_64-core-image-sato-i586-toolchain-DISTRO.sh
-
-For more information on how to install tarballs, see the "`Using a
-Cross-Toolchain
-Tarball <&YOCTO_DOCS_ADT_URL;#using-an-existing-toolchain-tarball>`__"
-and "`Using BitBake and the Build
-Directory <&YOCTO_DOCS_ADT_URL;#using-the-toolchain-from-within-the-build-tree>`__"
-sections in the Yocto Project Application Developer's Guide.
-
-Downloading the Pre-Built Linux Kernel
---------------------------------------
-
-You can download the pre-built Linux kernel suitable for running in the
-QEMU emulator from ` <&YOCTO_QEMU_DL_URL;>`__. Be sure to use the kernel
-that matches the architecture you want to simulate. Download areas exist
-for the five supported machine architectures: ``qemuarm``, ``qemumips``,
-``qemuppc``, ``qemux86``, and ``qemux86-64``.
-
-Most kernel files have one of the following forms: \*zImage-qemuarch.bin
-vmlinux-qemuarch.bin Where: arch is a string representing the target
-architecture: x86, x86-64, ppc, mips, or arm.
-
-You can learn more about downloading a Yocto Project kernel in the
-"`Yocto Project Kernel <&YOCTO_DOCS_DEV_URL;#local-kernel-files>`__"
-bulleted item in the Yocto Project Development Manual.
-
-Downloading the Filesystem
---------------------------
-
-You can also download the filesystem image suitable for your target
-architecture from ` <&YOCTO_QEMU_DL_URL;>`__. Again, be sure to use the
-filesystem that matches the architecture you want to simulate.
-
-The filesystem image has two tarball forms: ``ext3`` and ``tar``. You
-must use the ``ext3`` form when booting an image using the QEMU
-emulator. The ``tar`` form can be flattened out in your host development
-system and used for build purposes with the Yocto Project.
-core-image-profile-qemuarch.ext3 core-image-profile-qemuarch.tar.bz2
-Where: profile is the filesystem image's profile: lsb, lsb-dev, lsb-sdk,
-lsb-qt3, minimal, minimal-dev, sato, sato-dev, or sato-sdk. For
-information on these types of image profiles, see the
-":ref:`ref-manual/ref-images:Images`" chapter in the Yocto
-Project Reference Manual. arch is a string representing the target
-architecture: x86, x86-64, ppc, mips, or arm.
-
-Setting Up the Environment and Starting the QEMU Emulator
----------------------------------------------------------
-
-Before you start the QEMU emulator, you need to set up the emulation
-environment. The following command form sets up the emulation
-environment. $ source
-YOCTO_ADTPATH_DIR/environment-setup-arch-poky-linux-if Where: arch is a
-string representing the target architecture: i586, x86_64, ppc603e,
-mips, or armv5te. if is a string representing an embedded application
-binary interface. Not all setup scripts include this string.
-
-Finally, this command form invokes the QEMU emulator $ runqemu qemuarch
-kernel-image filesystem-image Where: qemuarch is a string representing
-the target architecture: qemux86, qemux86-64, qemuppc, qemumips, or
-qemuarm. kernel-image is the architecture-specific kernel image.
-filesystem-image is the .ext3 filesystem image.
-
-Continuing with the example, the following two commands setup the
-emulation environment and launch QEMU. This example assumes the root
-filesystem (``.ext3`` file) and the pre-built kernel image file both
-reside in your home directory. The kernel and filesystem are for a
-32-bit target architecture. $ cd $HOME $ source
-YOCTO_ADTPATH_DIR/environment-setup-i586-poky-linux $ runqemu qemux86
-bzImage-qemux86.bin \\ core-image-sato-qemux86.ext3
-
-The environment in which QEMU launches varies depending on the
-filesystem image and on the target architecture. For example, if you
-source the environment for the ARM target architecture and then boot the
-minimal QEMU image, the emulator comes up in a new shell in command-line
-mode. However, if you boot the SDK image, QEMU comes up with a GUI.
-
-.. note::
-
-   Booting the PPC image results in QEMU launching in the same shell in
-   command-line mode.
-
-.. |Using a Pre-Built Image| image:: figures/using-a-pre-built-image.png

documentation/bitbake.rst

@@ -0,0 +1,19 @@
+.. SPDX-License-Identifier: CC-BY-SA-2.0-UK
+
+=====================
+BitBake Documentation
+=====================
+
+|
+
+BitBake was originally a part of the OpenEmbedded project. It was inspired by
+the Portage package management system used by the Gentoo Linux distribution. In
+2004, the OpenEmbedded project was split the project into two distinct pieces:
+
+-  BitBake, a generic task executor
+-  OpenEmbedded, a metadata set utilized by BitBake
+
+Today, BitBake is the primary build tool of OpenEmbedded based projects, such as
+the Yocto Project.
+
+The BitBake documentation can be found :doc:`here <bitbake:index>`.

documentation/boilerplate.rst

@@ -8,11 +8,11 @@
 
 Permission is granted to copy, distribute and/or modify this document under the
 terms of the `Creative Commons Attribution-Share Alike 2.0 UK: England & Wales
-<http://creativecommons.org/licenses/by-sa/2.0/uk/>`_ as published by Creative
+<https://creativecommons.org/licenses/by-sa/2.0/uk/>`_ as published by Creative
 Commons.
 
 To report any inaccuracies or problems with this (or any other Yocto Project)
 manual, or to send additions or changes, please send email/patches to the Yocto
 Project documentation mailing list at ``docs@lists.yoctoproject.org`` or
-log into the freenode ``#yocto`` channel.
+log into the Freenode ``#yocto`` channel.