build-appliance-image: Update to gatesgarth head revision

(From OE-Core rev: ebaaee50cb3ac75112827f935c48affaf622ce7f)

Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>

.gitignore

@@ -8,6 +8,7 @@ pstage/
 scripts/oe-git-proxy-socks
 sources/
 meta-*/
+buildtools/
 !meta-skeleton
 !meta-selftest
 hob-image-*.bb

MEMORIAM

@@ -0,0 +1,5 @@
+Some project contributors who are sadly no longer with us:
+
+Greg Gilbert (treke) - Ahead of his time with licensing
+Thomas Wood (thos) - Creator of the original sato
+Scott Rifenbark (scottrif) - Our long standing techwriter whose words live on

Makefile

@@ -0,0 +1,35 @@
+# Minimal makefile for Sphinx documentation
+#
+
+# You can set these variables from the command line, and also
+# from the environment for the first two.
+SPHINXOPTS    ?=
+SPHINXBUILD   ?= sphinx-build
+SOURCEDIR     = .
+BUILDDIR      = _build
+DESTDIR       = final
+
+ifeq ($(shell if which $(SPHINXBUILD) >/dev/null 2>&1; then echo 1; else echo 0; fi),0)
+$(error "The '$(SPHINXBUILD)' command was not found. Make sure you have Sphinx installed")
+endif
+
+# Put it first so that "make" without argument is like "make help".
+help:
+	@$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)
+
+.PHONY: help Makefile.sphinx clean publish
+
+publish: Makefile.sphinx html singlehtml
+	rm -rf $(BUILDDIR)/$(DESTDIR)/
+	mkdir -p $(BUILDDIR)/$(DESTDIR)/
+	cp -r $(BUILDDIR)/html/* $(BUILDDIR)/$(DESTDIR)/
+	cp $(BUILDDIR)/singlehtml/index.html $(BUILDDIR)/$(DESTDIR)/singleindex.html
+	sed -i -e 's@index.html#@singleindex.html#@g' $(BUILDDIR)/$(DESTDIR)/singleindex.html
+
+clean:
+	@rm -rf $(BUILDDIR)
+
+# Catch-all target: route all unknown targets to Sphinx using the new
+# "make mode" option.  $(O) is meant as a shortcut for $(SPHINXOPTS).
+%: Makefile.sphinx
+	@$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)

bitbake/bin/bitbake

@@ -26,7 +26,7 @@ from bb.main import bitbake_main, BitBakeConfigParameters, BBMainException
 if sys.getfilesystemencoding() != "utf-8":
     sys.exit("Please use a locale setting which supports UTF-8 (such as LANG=en_US.UTF-8).\nPython can't change the filesystem locale after loading so we need a UTF-8 when Python starts or things won't work.")
 
-__version__ = "1.44.0"
+__version__ = "1.48.0"
 
 if __name__ == "__main__":
     if __version__ != bb.__version__:

bitbake/bin/bitbake-layers

@@ -14,7 +14,6 @@ import logging
 import os
 import sys
 import argparse
-import signal
 
 bindir = os.path.dirname(__file__)
 topdir = os.path.dirname(bindir)
@@ -26,7 +25,6 @@ import bb.msg
 logger = bb.msg.logger_create('bitbake-layers', sys.stdout)
 
 def main():
-    signal.signal(signal.SIGPIPE, signal.SIG_DFL)
     parser = argparse.ArgumentParser(
         description="BitBake layers utility",
         epilog="Use %(prog)s <subcommand> --help to get help on a specific command",
@@ -52,7 +50,9 @@ def main():
 
     # Need to re-run logger_create with color argument
     # (will be the same logger since it has the same name)
-    bb.msg.logger_create('bitbake-layers', output=sys.stdout, color=global_args.color)
+    bb.msg.logger_create('bitbake-layers', output=sys.stdout,
+                         color=global_args.color,
+                         level=logger.getEffectiveLevel())
 
     plugins = []
     tinfoil = bb.tinfoil.Tinfoil(tracking=True)

bitbake/bin/bitbake-selftest

@@ -18,6 +18,7 @@ except RuntimeError as exc:
     sys.exit(str(exc))
 
 tests = ["bb.tests.codeparser",
+         "bb.tests.color",
          "bb.tests.cooker",
          "bb.tests.cow",
          "bb.tests.data",
@@ -26,6 +27,7 @@ tests = ["bb.tests.codeparser",
          "bb.tests.parse",
          "bb.tests.persist_data",
          "bb.tests.runqueue",
+         "bb.tests.siggen",
          "bb.tests.utils",
          "hashserv.tests",
          "layerindexlib.tests.layerindexobj",

bitbake/bin/bitbake-server

@@ -0,0 +1,54 @@
+#!/usr/bin/env python3
+#
+# SPDX-License-Identifier: GPL-2.0-only
+#
+# Copyright (C) 2020        Richard Purdie
+#
+
+import os
+import sys
+import warnings
+import logging
+sys.path.insert(0, os.path.join(os.path.dirname(os.path.dirname(sys.argv[0])), 'lib'))
+
+if sys.getfilesystemencoding() != "utf-8":
+    sys.exit("Please use a locale setting which supports UTF-8 (such as LANG=en_US.UTF-8).\nPython can't change the filesystem locale after loading so we need a UTF-8 when Python starts or things won't work.")
+
+# Users shouldn't be running this code directly
+if len(sys.argv) != 10 or not sys.argv[1].startswith("decafbad"):
+    print("bitbake-server is meant for internal execution by bitbake itself, please don't use it standalone.")
+    sys.exit(1)
+
+import bb.server.process
+
+lockfd = int(sys.argv[2])
+readypipeinfd = int(sys.argv[3])
+logfile = sys.argv[4]
+lockname = sys.argv[5]
+sockname = sys.argv[6]
+timeout = sys.argv[7]
+xmlrpcinterface = (sys.argv[8], int(sys.argv[9]))
+if xmlrpcinterface[0] == "None":
+    xmlrpcinterface = (None, xmlrpcinterface[1])
+if timeout == "None":
+    timeout = None
+
+# Replace standard fds with our own
+with open('/dev/null', 'r') as si:
+    os.dup2(si.fileno(), sys.stdin.fileno())
+
+so = open(logfile, 'a+')
+os.dup2(so.fileno(), sys.stdout.fileno())
+os.dup2(so.fileno(), sys.stderr.fileno())
+
+# Have stdout and stderr be the same so log output matches chronologically
+# and there aren't two seperate buffers
+sys.stderr = sys.stdout
+
+logger = logging.getLogger("BitBake")
+# Ensure logging messages get sent to the UI as events
+handler = bb.event.LogHandler()
+logger.addHandler(handler)
+
+bb.server.process.execServer(lockfd, readypipeinfd, lockname, sockname, timeout, xmlrpcinterface)
+

bitbake/bin/bitbake-worker

@@ -65,7 +65,6 @@ if 0:
     format_str = "%(levelname)s: %(message)s"
     conlogformat = bb.msg.BBLogFormatter(format_str)
     consolelog = logging.FileHandler(logfilename)
-    bb.msg.addDefaultlogFilter(consolelog)
     consolelog.setFormatter(conlogformat)
     logger.addHandler(consolelog)
 
@@ -414,9 +413,9 @@ class BitbakeWorker(object):
 
     def handle_workerdata(self, data):
         self.workerdata = pickle.loads(data)
-        bb.msg.loggerDefaultDebugLevel = self.workerdata["logdefaultdebug"]
-        bb.msg.loggerDefaultVerbose = self.workerdata["logdefaultverbose"]
-        bb.msg.loggerVerboseLogs = self.workerdata["logdefaultverboselogs"]
+        bb.build.verboseShellLogging = self.workerdata["build_verbose_shell"]
+        bb.build.verboseStdoutLogging = self.workerdata["build_verbose_stdout"]
+        bb.msg.loggerDefaultLogLevel = self.workerdata["logdefaultlevel"]
         bb.msg.loggerDefaultDomains = self.workerdata["logdefaultdomain"]
         for mc in self.databuilder.mcdata:
             self.databuilder.mcdata[mc].setVar("PRSERV_HOST", self.workerdata["prhost"])

bitbake/bin/bitdoc

@@ -1,519 +0,0 @@
-#!/usr/bin/env python3
-#
-# Copyright (C) 2005 Holger Hans Peter Freyther
-#
-# SPDX-License-Identifier: GPL-2.0-only
-#
-
-import optparse, os, sys
-
-# bitbake
-sys.path.append(os.path.join(os.path.dirname(os.path.dirname(__file__), 'lib'))
-import bb
-import bb.parse
-from   string import split, join
-
-__version__ = "0.0.2"
-
-class HTMLFormatter:
-    """
-    Simple class to help to generate some sort of HTML files. It is
-    quite inferior solution compared to docbook, gtkdoc, doxygen but it
-    should work for now.
-    We've a global introduction site (index.html) and then one site for
-    the list of keys (alphabetical sorted) and one for the list of groups,
-    one site for each key with links to the relations and groups.
-
-        index.html
-        all_keys.html
-        all_groups.html
-        groupNAME.html
-        keyNAME.html
-    """
-
-    def replace(self, text, *pairs):
-        """
-        From pydoc... almost identical at least
-        """
-        while pairs:
-            (a, b) = pairs[0]
-            text = join(split(text, a), b)
-            pairs = pairs[1:]
-        return text
-    def escape(self, text):
-        """
-        Escape string to be conform HTML
-        """
-        return self.replace(text, 
-                            ('&', '&'), 
-                            ('<', '&lt;' ),
-                            ('>', '&gt;' ) )
-    def createNavigator(self):
-        """
-        Create the navgiator
-        """
-        return """<table class="navigation" width="100%" summary="Navigation header" cellpadding="2" cellspacing="2">
-<tr valign="middle">
-<td><a accesskey="g" href="index.html">Home</a></td>
-<td><a accesskey="n" href="all_groups.html">Groups</a></td>
-<td><a accesskey="u" href="all_keys.html">Keys</a></td>
-</tr></table>
-"""
-
-    def relatedKeys(self, item):
-        """
-        Create HTML to link to foreign keys
-        """
-
-        if len(item.related()) == 0:
-            return ""
-
-        txt = "<p><b>See also:</b><br>"
-        txts = []
-        for it in item.related():
-            txts.append("""<a href="key%(it)s.html">%(it)s</a>""" % vars() )
-
-        return txt + ",".join(txts)
-
-    def groups(self, item):
-        """
-        Create HTML to link to related groups
-        """
-
-        if len(item.groups()) == 0:
-            return ""
-
-
-        txt = "<p><b>See also:</b><br>"
-        txts = []
-        for group in item.groups():
-            txts.append( """<a href="group%s.html">%s</a> """ % (group, group) )
-
-        return txt + ",".join(txts)
-
-
-    def createKeySite(self, item):
-        """
-        Create a site for a key. It contains the header/navigator, a heading,
-        the description, links to related keys and to the groups.
-        """
-
-        return """<!doctype html PUBLIC "-//W3C//DTD HTML 4.0 Transitional//EN">
-<html><head><title>Key %s</title></head>
-<link rel="stylesheet" href="style.css" type="text/css">
-<body bgcolor="white" text="black" link="#0000FF" vlink="#840084" alink="#0000FF">
-%s
-<h2><span class="refentrytitle">%s</span></h2>
-
-<div class="refsynopsisdiv">
-<h2>Synopsis</h2>
-<p>
-%s
-</p>
-</div>
-
-<div class="refsynopsisdiv">
-<h2>Related Keys</h2>
-<p>
-%s
-</p>
-</div>
-
-<div class="refsynopsisdiv">
-<h2>Groups</h2>
-<p>
-%s
-</p>
-</div>
-
-
-</body>
-"""     % (item.name(), self.createNavigator(), item.name(), 
-           self.escape(item.description()), self.relatedKeys(item), self.groups(item))
-
-    def createGroupsSite(self, doc):
-        """
-        Create the Group Overview site
-        """
-
-        groups = ""
-        sorted_groups = sorted(doc.groups())
-        for group in sorted_groups:
-            groups += """<a href="group%s.html">%s</a><br>""" % (group, group)
-
-        return """<!doctype html PUBLIC "-//W3C//DTD HTML 4.0 Transitional//EN">
-<html><head><title>Group overview</title></head>
-<link rel="stylesheet" href="style.css" type="text/css">
-<body bgcolor="white" text="black" link="#0000FF" vlink="#840084" alink="#0000FF">
-%s
-<h2>Available Groups</h2>
-%s
-</body>
-""" % (self.createNavigator(), groups)
-
-    def createIndex(self):
-        """
-        Create the index file
-        """
-
-        return """<!doctype html PUBLIC "-//W3C//DTD HTML 4.0 Transitional//EN">
-<html><head><title>Bitbake Documentation</title></head>
-<link rel="stylesheet" href="style.css" type="text/css">
-<body bgcolor="white" text="black" link="#0000FF" vlink="#840084" alink="#0000FF">
-%s
-<h2>Documentation Entrance</h2>
-<a href="all_groups.html">All available groups</a><br>
-<a href="all_keys.html">All available keys</a><br>
-</body>
-""" % self.createNavigator()
-
-    def createKeysSite(self, doc):
-        """
-        Create Overview of all avilable keys
-        """
-        keys = ""
-        sorted_keys = sorted(doc.doc_keys())
-        for key in sorted_keys:
-            keys += """<a href="key%s.html">%s</a><br>""" % (key, key)
-
-        return """<!doctype html PUBLIC "-//W3C//DTD HTML 4.0 Transitional//EN">
-<html><head><title>Key overview</title></head>
-<link rel="stylesheet" href="style.css" type="text/css">
-<body bgcolor="white" text="black" link="#0000FF" vlink="#840084" alink="#0000FF">
-%s
-<h2>Available Keys</h2>
-%s
-</body>
-""" % (self.createNavigator(), keys)
-
-    def createGroupSite(self, gr, items, _description = None):
-        """
-        Create a site for a group:
-        Group the name of the group, items contain the name of the keys
-        inside this group
-        """
-        groups = ""
-        description = ""
-
-        # create a section with the group descriptions
-        if _description:
-            description  += "<h2 Description of Grozp %s</h2>" % gr
-            description  += _description
-
-        items.sort(lambda x, y:cmp(x.name(), y.name()))
-        for group in items:
-            groups += """<a href="key%s.html">%s</a><br>""" % (group.name(), group.name())
-
-        return """<!doctype html PUBLIC "-//W3C//DTD HTML 4.0 Transitional//EN">
-<html><head><title>Group %s</title></head>
-<link rel="stylesheet" href="style.css" type="text/css">
-<body bgcolor="white" text="black" link="#0000FF" vlink="#840084" alink="#0000FF">
-%s
-%s
-<div class="refsynopsisdiv">
-<h2>Keys in Group %s</h2>
-<pre class="synopsis">
-%s
-</pre>
-</div>
-</body>
-""" % (gr, self.createNavigator(), description, gr, groups)
-
-
-
-    def createCSS(self):
-        """
-        Create the CSS file
-        """
-        return """.synopsis, .classsynopsis
-{
-  background: #eeeeee;
-  border: solid 1px #aaaaaa;
-  padding: 0.5em;
-}
-.programlisting
-{
-  background: #eeeeff;
-  border: solid 1px #aaaaff;
-  padding: 0.5em;
-}
-.variablelist
-{
-  padding: 4px;
-  margin-left: 3em;
-}
-.variablelist td:first-child
-{
-  vertical-align: top;
-}
-table.navigation
-{
-  background: #ffeeee;
-  border: solid 1px #ffaaaa;
-  margin-top: 0.5em;
-  margin-bottom: 0.5em;
-}
-.navigation a
-{
-  color: #770000;
-}
-.navigation a:visited
-{
-  color: #550000;
-}
-.navigation .title
-{
-  font-size: 200%;
-}
-div.refnamediv
-{
-  margin-top: 2em;
-}
-div.gallery-float
-{
-  float: left;
-  padding: 10px;
-}
-div.gallery-float img
-{
-  border-style: none;
-}
-div.gallery-spacer
-{
-  clear: both;
-}
-a
-{
-  text-decoration: none;
-}
-a:hover
-{
-  text-decoration: underline;
-  color: #FF0000;
-}
-"""
-
-
-
-class DocumentationItem:
-    """
-    A class to hold information about a configuration
-    item. It contains the key name, description, a list of related names,
-    and the group this item is contained in.
-    """
-
-    def __init__(self):
-        self._groups  = []
-        self._related = []
-        self._name    = ""
-        self._desc    = ""
-
-    def groups(self):
-        return self._groups
-
-    def name(self):
-        return self._name
-
-    def description(self):
-        return self._desc
-
-    def related(self):
-        return self._related
-
-    def setName(self, name):
-        self._name = name
-
-    def setDescription(self, desc):
-        self._desc = desc
-
-    def addGroup(self, group):
-        self._groups.append(group)
-
-    def addRelation(self, relation):
-        self._related.append(relation)
-
-    def sort(self):
-        self._related.sort()
-        self._groups.sort()
-
-
-class Documentation:
-    """
-    Holds the documentation... with mappings from key to items...
-    """
-
-    def __init__(self):
-        self.__keys   = {}
-        self.__groups = {}
-
-    def insert_doc_item(self, item):
-        """
-        Insert the Doc Item into the internal list
-        of representation
-        """
-        item.sort()
-        self.__keys[item.name()] = item
-
-        for group in item.groups():
-            if not group in self.__groups:
-                self.__groups[group] = []
-            self.__groups[group].append(item)
-            self.__groups[group].sort()
-
-
-    def doc_item(self, key):
-        """
-        Return the DocumentationInstance describing the key
-        """
-        try:
-            return self.__keys[key]
-        except KeyError:
-            return None
-
-    def doc_keys(self):
-        """
-        Return the documented KEYS (names)
-        """
-        return self.__keys.keys()
-
-    def groups(self):
-        """
-        Return the names of available groups
-        """
-        return self.__groups.keys()
-
-    def group_content(self, group_name):
-        """
-        Return a list of keys/names that are in a specefic
-        group or the empty list
-        """
-        try:
-            return self.__groups[group_name]
-        except KeyError:
-            return []
-
-
-def parse_cmdline(args):
-    """
-    Parse the CMD line and return the result as a n-tuple
-    """
-
-    parser = optparse.OptionParser( version = "Bitbake Documentation Tool Core version %s, %%prog version %s" % (bb.__version__, __version__))
-    usage  = """%prog [options]
-
-Create a set of html pages (documentation) for a bitbake.conf....
-"""
-
-    # Add the needed options
-    parser.add_option( "-c", "--config", help = "Use the specified configuration file as source",
-                       action = "store", dest = "config", default = os.path.join("conf", "documentation.conf") )
-
-    parser.add_option( "-o", "--output", help = "Output directory for html files",
-                       action = "store", dest = "output", default = "html/" )
-
-    parser.add_option( "-D",  "--debug", help = "Increase the debug level",
-                       action = "count", dest = "debug", default = 0 )
-
-    parser.add_option( "-v", "--verbose", help = "output more chit-char to the terminal",
-                       action = "store_true", dest = "verbose", default = False )
-
-    options, args = parser.parse_args( sys.argv )
- 
-    bb.msg.init_msgconfig(options.verbose, options.debug)
-
-    return options.config, options.output
-
-def main():
-    """
-    The main Method
-    """
-
-    (config_file, output_dir) = parse_cmdline( sys.argv )
-
-    # right to let us load the file now
-    try:
-        documentation = bb.parse.handle( config_file, bb.data.init() )
-    except IOError:
-        bb.fatal( "Unable to open %s" % config_file )
-    except bb.parse.ParseError:
-        bb.fatal( "Unable to parse %s" % config_file )
-
-    if isinstance(documentation, dict):
-        documentation = documentation[""]
-
-    # Assuming we've the file loaded now, we will initialize the 'tree'
-    doc = Documentation()
-
-    # defined states
-    state_begin = 0
-    state_see   = 1
-    state_group = 2
-
-    for key in bb.data.keys(documentation):
-        data   = documentation.getVarFlag(key, "doc", False)
-        if not data:
-            continue
-
-        # The Documentation now starts
-        doc_ins = DocumentationItem()
-        doc_ins.setName(key)
-
-
-        tokens = data.split(' ')
-        state = state_begin
-        string= ""
-        for token in tokens:
-            token = token.strip(',')
-
-            if not state == state_see and token == "@see":
-                state = state_see
-                continue
-            elif not state == state_group and token  == "@group":
-                state = state_group
-                continue
-
-            if state == state_begin:
-                string += " %s" % token
-            elif state == state_see:
-                doc_ins.addRelation(token)
-            elif state == state_group:
-                doc_ins.addGroup(token)
-
-        # set the description
-        doc_ins.setDescription(string)
-        doc.insert_doc_item(doc_ins)
-
-    # let us create the HTML now
-    bb.utils.mkdirhier(output_dir)
-    os.chdir(output_dir)
-
-    # Let us create the sites now. We do it in the following order
-    # Start with the index.html. It will point to sites explaining all
-    # keys and groups
-    html_slave = HTMLFormatter()
-
-    f = file('style.css', 'w')
-    print >> f, html_slave.createCSS()
-
-    f = file('index.html', 'w')
-    print >> f, html_slave.createIndex()
-
-    f = file('all_groups.html', 'w')
-    print >> f, html_slave.createGroupsSite(doc)
-
-    f = file('all_keys.html', 'w')
-    print >> f, html_slave.createKeysSite(doc)
-
-    # now for each group create the site
-    for group in doc.groups():
-        f = file('group%s.html' % group, 'w')
-        print >> f, html_slave.createGroupSite(group, doc.group_content(group))
-
-    # now for the keys
-    for key in doc.doc_keys():
-        f = file('key%s.html' % doc.doc_item(key).name(), 'w')
-        print >> f, html_slave.createKeySite(doc.doc_item(key))
-
-
-if __name__ == "__main__":
-    main()

bitbake/bin/toaster

@@ -8,12 +8,13 @@
 #
 
 HELP="
-Usage: source toaster start|stop [webport=<address:port>] [noweb] [nobuild] [toasterdir]
+Usage 1: source toaster start|stop [webport=<address:port>] [noweb] [nobuild] [toasterdir]
     Optional arguments:
         [nobuild] Setup the environment for capturing builds with toaster but disable managed builds
         [noweb] Setup the environment for capturing builds with toaster but don't start the web server
         [webport] Set the development server (default: localhost:8000)
         [toasterdir] Set absolute path to be used as TOASTER_DIR (default: BUILDDIR/../)
+Usage 2: source toaster manage [createsuperuser|lsupdates|migrate|makemigrations|checksettings|collectstatic|...]
 "
 
 custom_extention()
@@ -208,13 +209,21 @@ for param in $*; do
     toasterdir=*)
             TOASTERDIR="${param#*=}"
     ;;
+    manage )
+            CMD=$param
+            manage_cmd=""
+    ;;
     --help)
             echo "$HELP"
             return 0
     ;;
     *)
-            echo "$HELP"
-            return 1
+            if [ "manage" == "$CMD" ] ; then
+                manage_cmd="$manage_cmd $param"
+            else
+                echo "$HELP"
+                exit 1
+            fi
     ;;
 
     esac
@@ -306,6 +315,10 @@ case $CMD in
         stop_system
         echo "Successful ${CMD}."
     ;;
+    manage )
+        cd $BBBASEDIR/lib/toaster
+        $MANAGE $manage_cmd
+    ;;
 esac
 custom_extention toaster_postpend $CMD $ADDR_PORT
 

bitbake/contrib/autobuilderlog.json

@@ -0,0 +1,13 @@
+{
+    "version": 1,
+    "loggers": {
+        "BitBake.SigGen.HashEquiv": {
+            "level": "VERBOSE",
+            "handlers": ["BitBake.verbconsole"]
+        },
+        "BitBake.RunQueue.HashEquiv": {
+            "level": "VERBOSE",
+            "handlers": ["BitBake.verbconsole"]
+        }
+    }
+}

bitbake/contrib/bbparse-torture.py

@@ -0,0 +1,89 @@
+#! /usr/bin/env python3
+#
+# Copyright (C) 2020 Joshua Watt <JPEWhacker@gmail.com>
+#
+# SPDX-License-Identifier: MIT
+
+import argparse
+import os
+import random
+import shutil
+import signal
+import subprocess
+import sys
+import time
+
+
+def try_unlink(path):
+    try:
+        os.unlink(path)
+    except:
+        pass
+
+
+def main():
+    def cleanup():
+        shutil.rmtree("tmp/cache", ignore_errors=True)
+        try_unlink("bitbake-cookerdaemon.log")
+        try_unlink("bitbake.sock")
+        try_unlink("bitbake.lock")
+
+    parser = argparse.ArgumentParser(
+        description="Bitbake parser torture test",
+        epilog="""
+        A torture test for bitbake's parser. Repeatedly interrupts parsing until
+        bitbake decides to deadlock.
+        """,
+    )
+
+    args = parser.parse_args()
+
+    if not "BUILDDIR" in os.environ:
+        print(
+            "'BUILDDIR' not found in the environment. Did you initialize the build environment?"
+        )
+        return 1
+
+    os.chdir(os.environ["BUILDDIR"])
+
+    run_num = 0
+    while True:
+        if run_num % 100 == 0:
+            print("Calibrating wait time...")
+            cleanup()
+
+            start_time = time.monotonic()
+            r = subprocess.run(["bitbake", "-p"])
+            max_wait_time = time.monotonic() - start_time
+
+            if r.returncode != 0:
+                print("Calibration run exited with %d" % r.returncode)
+                return 1
+
+            print("Maximum wait time is %f seconds" % max_wait_time)
+
+        run_num += 1
+        wait_time = random.random() * max_wait_time
+
+        print("Run #%d" % run_num)
+        print("Will sleep for %f seconds" % wait_time)
+
+        cleanup()
+        with subprocess.Popen(["bitbake", "-p"]) as proc:
+            time.sleep(wait_time)
+            proc.send_signal(signal.SIGINT)
+            try:
+                proc.wait(45)
+            except subprocess.TimeoutExpired:
+                print("Run #%d: Waited too long. Possible deadlock!" % run_num)
+                proc.wait()
+                return 1
+
+            if proc.returncode == 0:
+                print("Exited successfully. Timeout too long?")
+            else:
+                print("Exited with %d" % proc.returncode)
+
+
+if __name__ == "__main__":
+    sys.exit(main())

bitbake/contrib/vim/LICENSE.txt

@@ -0,0 +1,18 @@
+The MIT License (MIT)
+
+Permission is hereby granted, free of charge, to any person obtaining a copy of
+this software and associated documentation files (the "Software"), to deal in
+the Software without restriction, including without limitation the rights to
+use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of
+the Software, and to permit persons to whom the Software is furnished to do so,
+subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
+FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

bitbake/contrib/vim/ftdetect/bitbake.vim

@@ -6,12 +6,12 @@
 "
 " This sets up the syntax highlighting for BitBake files, like .bb, .bbclass and .inc
 
-if &compatible || version < 600
+if &compatible || version < 600 || exists("b:loaded_bitbake_plugin")
     finish
 endif
 
 " .bb, .bbappend and .bbclass
-au BufNewFile,BufRead *.{bb,bbappend,bbclass}	set filetype=bitbake
+au BufNewFile,BufRead *.{bb,bbappend,bbclass}  set filetype=bitbake
 
 " .inc
 au BufNewFile,BufRead *.inc		set filetype=bitbake

bitbake/contrib/vim/ftplugin/bitbake.vim

@@ -1,2 +1,13 @@
-set sts=4 sw=4 et
-set cms=#%s
+" Only do this when not done yet for this buffer
+if exists("b:did_ftplugin")
+  finish
+endif
+
+" Don't load another plugin for this buffer
+let b:did_ftplugin = 1
+
+let b:undo_ftplugin = "setl cms< sts< sw< et< sua<"
+
+setlocal commentstring=#\ %s
+setlocal softtabstop=4 shiftwidth=4 expandtab
+setlocal suffixesadd+=.bb,.bbclass

bitbake/contrib/vim/indent/bitbake.vim

@@ -0,0 +1,343 @@
+" Vim indent file
+" Language:             BitBake
+" Copyright:            Copyright (C) 2019 Agilent Technologies, Inc.
+" Maintainer:           Chris Laplante <chris.laplante@agilent.com>
+" License:              You may redistribute this under the same terms as Vim itself
+
+
+if exists("b:did_indent")
+    finish
+endif
+
+if exists("*BitbakeIndent")
+    finish
+endif
+
+runtime! indent/sh.vim
+unlet b:did_indent
+
+setlocal indentexpr=BitbakeIndent(v:lnum)
+setlocal autoindent nolisp
+
+function s:is_bb_python_func_def(lnum)
+    let stack = synstack(a:lnum, 1)
+    if len(stack) == 0
+        return 0
+    endif
+
+    let top = synIDattr(stack[0], "name")
+    echo top
+
+    return synIDattr(stack[0], "name") == "bbPyFuncDef"
+endfunction
+
+"""" begin modified from indent/python.vim, upstream commit 7a9bd7c1e0ce1baf5a02daf36eeae3638aa315c7
+"""" This copied code is licensed the same as Vim itself.
+setlocal indentkeys+=<:>,=elif,=except
+
+let s:keepcpo= &cpo
+set cpo&vim
+
+let s:maxoff = 50	" maximum number of lines to look backwards for ()
+
+function GetPythonIndent(lnum)
+
+  " If this line is explicitly joined: If the previous line was also joined,
+  " line it up with that one, otherwise add two 'shiftwidth'
+  if getline(a:lnum - 1) =~ '\\$'
+    if a:lnum > 1 && getline(a:lnum - 2) =~ '\\$'
+      return indent(a:lnum - 1)
+    endif
+    return indent(a:lnum - 1) + (exists("g:pyindent_continue") ? eval(g:pyindent_continue) : (shiftwidth() * 2))
+  endif
+
+  " If the start of the line is in a string don't change the indent.
+  if has('syntax_items')
+	\ && synIDattr(synID(a:lnum, 1, 1), "name") =~ "String$"
+    return -1
+  endif
+
+  " Search backwards for the previous non-empty line.
+  let plnum = prevnonblank(v:lnum - 1)
+
+  if plnum == 0
+    " This is the first non-empty line, use zero indent.
+    return 0
+  endif
+
+  call cursor(plnum, 1)
+
+  " Identing inside parentheses can be very slow, regardless of the searchpair()
+  " timeout, so let the user disable this feature if he doesn't need it
+  let disable_parentheses_indenting = get(g:, "pyindent_disable_parentheses_indenting", 0)
+
+  if disable_parentheses_indenting == 1
+    let plindent = indent(plnum)
+    let plnumstart = plnum
+  else
+    " searchpair() can be slow sometimes, limit the time to 150 msec or what is
+    " put in g:pyindent_searchpair_timeout
+    let searchpair_stopline = 0
+    let searchpair_timeout = get(g:, 'pyindent_searchpair_timeout', 150)
+
+    " If the previous line is inside parenthesis, use the indent of the starting
+    " line.
+    " Trick: use the non-existing "dummy" variable to break out of the loop when
+    " going too far back.
+    let parlnum = searchpair('(\|{\|\[', '', ')\|}\|\]', 'nbW',
+            \ "line('.') < " . (plnum - s:maxoff) . " ? dummy :"
+            \ . " synIDattr(synID(line('.'), col('.'), 1), 'name')"
+            \ . " =~ '\\(Comment\\|Todo\\|String\\)$'",
+            \ searchpair_stopline, searchpair_timeout)
+    if parlnum > 0
+      " We may have found the opening brace of a BitBake Python task, e.g. 'python do_task {'
+      " If so, ignore it here - it will be handled later.
+      if s:is_bb_python_func_def(parlnum)
+        let parlnum = 0
+        let plindent = indent(plnum)
+        let plnumstart = plnum
+      else
+        let plindent = indent(parlnum)
+        let plnumstart = parlnum
+      endif
+    else
+      let plindent = indent(plnum)
+      let plnumstart = plnum
+    endif
+
+    " When inside parenthesis: If at the first line below the parenthesis add
+    " two 'shiftwidth', otherwise same as previous line.
+    " i = (a
+    "       + b
+    "       + c)
+    call cursor(a:lnum, 1)
+    let p = searchpair('(\|{\|\[', '', ')\|}\|\]', 'bW',
+            \ "line('.') < " . (a:lnum - s:maxoff) . " ? dummy :"
+            \ . " synIDattr(synID(line('.'), col('.'), 1), 'name')"
+            \ . " =~ '\\(Comment\\|Todo\\|String\\)$'",
+            \ searchpair_stopline, searchpair_timeout)
+    if p > 0
+        if s:is_bb_python_func_def(p)
+          " Handle first non-empty line inside a BB Python task
+          if p == plnum
+              return shiftwidth()
+          endif
+
+          " Handle the user actually trying to close a BitBake Python task
+          let line = getline(a:lnum)
+          if line =~ '^\s*}'
+              return -2
+          endif
+
+          " Otherwise ignore the brace
+          let p = 0
+        else
+          if p == plnum
+            " When the start is inside parenthesis, only indent one 'shiftwidth'.
+            let pp = searchpair('(\|{\|\[', '', ')\|}\|\]', 'bW',
+                \ "line('.') < " . (a:lnum - s:maxoff) . " ? dummy :"
+                \ . " synIDattr(synID(line('.'), col('.'), 1), 'name')"
+                \ . " =~ '\\(Comment\\|Todo\\|String\\)$'",
+                \ searchpair_stopline, searchpair_timeout)
+            if pp > 0
+              return indent(plnum) + (exists("g:pyindent_nested_paren") ? eval(g:pyindent_nested_paren) : shiftwidth())
+            endif
+            return indent(plnum) + (exists("g:pyindent_open_paren") ? eval(g:pyindent_open_paren) : (shiftwidth() * 2))
+          endif
+          if plnumstart == p
+            return indent(plnum)
+          endif
+          return plindent
+        endif
+    endif
+
+  endif
+
+
+  " Get the line and remove a trailing comment.
+  " Use syntax highlighting attributes when possible.
+  let pline = getline(plnum)
+  let pline_len = strlen(pline)
+  if has('syntax_items')
+    " If the last character in the line is a comment, do a binary search for
+    " the start of the comment.  synID() is slow, a linear search would take
+    " too long on a long line.
+    if synIDattr(synID(plnum, pline_len, 1), "name") =~ "\\(Comment\\|Todo\\)$"
+      let min = 1
+      let max = pline_len
+      while min < max
+	let col = (min + max) / 2
+	if synIDattr(synID(plnum, col, 1), "name") =~ "\\(Comment\\|Todo\\)$"
+	  let max = col
+	else
+	  let min = col + 1
+	endif
+      endwhile
+      let pline = strpart(pline, 0, min - 1)
+    endif
+  else
+    let col = 0
+    while col < pline_len
+      if pline[col] == '#'
+	let pline = strpart(pline, 0, col)
+	break
+      endif
+      let col = col + 1
+    endwhile
+  endif
+
+  " If the previous line ended with a colon, indent this line
+  if pline =~ ':\s*$'
+    return plindent + shiftwidth()
+  endif
+
+  " If the previous line was a stop-execution statement...
+  " TODO: utilize this logic to deindent when ending a bbPyDefRegion
+  if getline(plnum) =~ '^\s*\(break\|continue\|raise\|return\|pass\|bb\.fatal\)\>'
+    " See if the user has already dedented
+    if indent(a:lnum) > indent(plnum) - shiftwidth()
+      " If not, recommend one dedent
+      return indent(plnum) - shiftwidth()
+    endif
+    " Otherwise, trust the user
+    return -1
+  endif
+
+  " If the current line begins with a keyword that lines up with "try"
+  if getline(a:lnum) =~ '^\s*\(except\|finally\)\>'
+    let lnum = a:lnum - 1
+    while lnum >= 1
+      if getline(lnum) =~ '^\s*\(try\|except\)\>'
+	let ind = indent(lnum)
+	if ind >= indent(a:lnum)
+	  return -1	" indent is already less than this
+	endif
+	return ind	" line up with previous try or except
+      endif
+      let lnum = lnum - 1
+    endwhile
+    return -1		" no matching "try"!
+  endif
+
+  " If the current line begins with a header keyword, dedent
+  if getline(a:lnum) =~ '^\s*\(elif\|else\)\>'
+
+    " Unless the previous line was a one-liner
+    if getline(plnumstart) =~ '^\s*\(for\|if\|try\)\>'
+      return plindent
+    endif
+
+    " Or the user has already dedented
+    if indent(a:lnum) <= plindent - shiftwidth()
+      return -1
+    endif
+
+    return plindent - shiftwidth()
+  endif
+
+  " When after a () construct we probably want to go back to the start line.
+  " a = (b
+  "       + c)
+  " here
+  if parlnum > 0
+    return plindent
+  endif
+
+  return -1
+
+endfunction
+
+let &cpo = s:keepcpo
+unlet s:keepcpo
+
+""" end of stuff from indent/python.vim
+
+
+let b:did_indent = 1
+setlocal indentkeys+=0\"
+
+
+function BitbakeIndent(lnum)
+    if !has('syntax_items')
+        return -1
+    endif
+
+    let stack = synstack(a:lnum, 1)
+    if len(stack) == 0
+        return -1
+    endif
+
+    let name = synIDattr(stack[0], "name")
+
+    " TODO: support different styles of indentation for assignments. For now,
+    " we only support like this:
+    " VAR = " \
+    "     value1 \
+    "     value2 \
+    " "
+    "
+    " i.e. each value indented by shiftwidth(), with the final quote " completely unindented.
+    if name == "bbVarValue"
+        " Quote handling is tricky. kernel.bbclass has this line for instance:
+        "     EXTRA_OEMAKE = " HOSTCC="${BUILD_CC} ${BUILD_CFLAGS} ${BUILD_LDFLAGS}" " HOSTCPP="${BUILD_CPP}""
+        " Instead of trying to handle crazy cases like that, just assume that a
+        " double-quote on a line by itself (following an assignment) means the
+        " user is closing the assignment, and de-dent.
+        if getline(a:lnum) =~ '^\s*"$'
+            return 0
+        endif
+
+        let prevstack = synstack(a:lnum - 1, 1)
+        if len(prevstack) == 0
+            return -1
+        endif
+
+        let prevname = synIDattr(prevstack[0], "name")
+
+        " Only indent if there was actually a continuation character on
+        " the previous line, to avoid misleading indentation.
+        let prevlinelastchar = synIDattr(synID(a:lnum - 1, col([a:lnum - 1, "$"]) - 1, 1), "name")
+        let prev_continued = prevlinelastchar == "bbContinue"
+
+        " Did the previous line introduce an assignment?
+        if index(["bbVarDef", "bbVarFlagDef"], prevname) != -1
+            if prev_continued
+                return shiftwidth()
+            endif
+        endif
+
+        if !prev_continued
+            return 0
+        endif
+
+        " Autoindent can take it from here
+        return -1
+    endif
+
+    if index(["bbPyDefRegion", "bbPyFuncRegion"], name) != -1
+        let ret = GetPythonIndent(a:lnum)
+        " Should normally always be indented by at least one shiftwidth; but allow
+        " return of -1 (defer to autoindent) or -2 (force indent to 0)
+        if ret == 0
+            return shiftwidth()
+        elseif ret == -2
+            return 0
+        endif
+        return ret
+    endif
+
+    " TODO: GetShIndent doesn't detect tasks prepended with 'fakeroot'
+    " Need to submit a patch upstream to Vim to provide an extension point.
+    " Unlike the Python indenter, the Sh indenter is way too large to copy and
+    " modify here.
+    if name == "bbShFuncRegion"
+        return GetShIndent()
+    endif
+
+    " TODO:
+    "   + heuristics for de-denting out of a bbPyDefRegion? e.g. when the user
+    "       types an obvious BB keyword like addhandler or addtask, or starts
+    "       writing a shell task. Maybe too hard to implement...
+
+    return -1
+endfunction

bitbake/contrib/vim/plugin/newbb.vim

@@ -10,7 +10,7 @@
 "
 " Will try to use git to find the user name and email
 
-if &compatible || v:version < 600
+if &compatible || v:version < 600 || exists("b:loaded_bitbake_plugin")
     finish
 endif
 
@@ -25,7 +25,7 @@ endfun
 fun! <SID>GetUserEmail()
     let l:user_email = system("git config --get user.email")
     if v:shell_error
-        return "unknow@user.org"
+        return "unknown@user.org"
     else
         return substitute(l:user_email, "\n", "", "")
 endfun
@@ -41,6 +41,10 @@ fun! BBHeader()
 endfun
 
 fun! NewBBTemplate()
+    if line2byte(line('$') + 1) != -1
+        return
+    endif
+
     let l:paste = &paste
     set nopaste
     
@@ -48,7 +52,7 @@ fun! NewBBTemplate()
     call BBHeader()
 
     " New the bb template
-    put ='DESCRIPTION = \"\"'
+    put ='SUMMARY = \"\"'
     put ='HOMEPAGE = \"\"'
     put ='LICENSE = \"\"' 
     put ='SECTION = \"\"'
@@ -58,7 +62,7 @@ fun! NewBBTemplate()
 
     " Go to the first place to edit
     0
-    /^DESCRIPTION =/
+    /^SUMMARY =/
     exec "normal 2f\""
 
     if paste == 1
@@ -76,7 +80,7 @@ if v:progname =~ "vimdiff"
 endif
 
 augroup NewBB
-    au BufNewFile *.bb
+    au BufNewFile,BufReadPost *.bb
                 \ if g:bb_create_on_empty |
                 \    call NewBBTemplate() |
                 \ endif

bitbake/contrib/vim/plugin/newbbappend.vim

@@ -0,0 +1,46 @@
+" Vim plugin file
+" Purpose:	Create a template for new bbappend file
+" Author:	Joshua Watt <JPEWhacker@gmail.com>
+" Copyright:	Copyright (C) 2017 Joshua Watt <JPEWhacker@gmail.com>
+"
+" This file is licensed under the MIT license, see COPYING.MIT in
+" this source distribution for the terms.
+"
+
+if &compatible || v:version < 600 || exists("b:loaded_bitbake_plugin")
+    finish
+endif
+
+fun! NewBBAppendTemplate()
+    if line2byte(line('$') + 1) != -1
+        return
+    endif
+
+    let l:paste = &paste
+    set nopaste
+
+    " New bbappend template
+    0 put ='FILESEXTRAPATHS_prepend := \"${THISDIR}/${PN}:\"'
+    2
+
+    if paste == 1
+        set paste
+    endif
+endfun
+
+if !exists("g:bb_create_on_empty")
+    let g:bb_create_on_empty = 1
+endif
+
+" disable in case of vimdiff
+if v:progname =~ "vimdiff"
+    let g:bb_create_on_empty = 0
+endif
+
+augroup NewBBAppend
+    au BufNewFile,BufReadPost *.bbappend
+                \ if g:bb_create_on_empty |
+                \    call NewBBAppendTemplate() |
+                \ endif
+augroup END
+

bitbake/contrib/vim/syntax/bitbake.vim

@@ -12,7 +12,7 @@
 "
 " It's an entirely new type, just has specific syntax in shell and python code
 
-if &compatible || v:version < 600
+if &compatible || v:version < 600 || exists("b:loaded_bitbake_plugin")
     finish
 endif
 if exists("b:current_syntax")
@@ -58,8 +58,8 @@ syn match bbVarValue            ".*$" contained contains=bbString,bbVarDeref,bbV
 syn region bbVarPyValue         start=+${@+ skip=+\\$+ end=+}+ contained contains=@python
 
 " Vars metadata flags
-syn match bbVarFlagDef          "^\([a-zA-Z0-9\-_\.]\+\)\(\[[a-zA-Z0-9\-_\.]\+\]\)\@=" contains=bbIdentifier nextgroup=bbVarFlagFlag
-syn region bbVarFlagFlag        matchgroup=bbArrayBrackets start="\[" end="\]\s*\(=\|+=\|=+\|?=\)\@=" contained contains=bbIdentifier nextgroup=bbVarEq
+syn match bbVarFlagDef          "^\([a-zA-Z0-9\-_\.]\+\)\(\[[a-zA-Z0-9\-_\.+]\+\]\)\@=" contains=bbIdentifier nextgroup=bbVarFlagFlag
+syn region bbVarFlagFlag        matchgroup=bbArrayBrackets start="\[" end="\]\s*\(:=\|=\|.=\|=.|+=\|=+\|?=\)\@=" contained contains=bbIdentifier nextgroup=bbVarEq
 
 " Includes and requires
 syn keyword bbInclude           inherit include require contained 
@@ -67,15 +67,15 @@ syn match bbIncludeRest         ".*$" contained contains=bbString,bbVarDeref
 syn match bbIncludeLine         "^\(inherit\|include\|require\)\s\+" contains=bbInclude nextgroup=bbIncludeRest
 
 " Add taks and similar
-syn keyword bbStatement         addtask addhandler after before EXPORT_FUNCTIONS contained
+syn keyword bbStatement         addtask deltask addhandler after before EXPORT_FUNCTIONS contained
 syn match bbStatementRest       ".*$" skipwhite contained contains=bbStatement
-syn match bbStatementLine       "^\(addtask\|addhandler\|after\|before\|EXPORT_FUNCTIONS\)\s\+" contains=bbStatement nextgroup=bbStatementRest
+syn match bbStatementLine       "^\(addtask\|deltask\|addhandler\|after\|before\|EXPORT_FUNCTIONS\)\s\+" contains=bbStatement nextgroup=bbStatementRest
 
 " OE Important Functions
 syn keyword bbOEFunctions       do_fetch do_unpack do_patch do_configure do_compile do_stage do_install do_package contained
 
 " Generic Functions
-syn match bbFunction            "\h[0-9A-Za-z_-]*" display contained contains=bbOEFunctions
+syn match bbFunction            "\h[0-9A-Za-z_\-\.]*" display contained contains=bbOEFunctions
 
 " BitBake shell metadata
 syn include @shell syntax/sh.vim
@@ -83,7 +83,7 @@ if exists("b:current_syntax")
   unlet b:current_syntax
 endif
 syn keyword bbShFakeRootFlag    fakeroot contained
-syn match bbShFuncDef           "^\(fakeroot\s*\)\?\([0-9A-Za-z_${}-]\+\)\(python\)\@<!\(\s*()\s*\)\({\)\@=" contains=bbShFakeRootFlag,bbFunction,bbVarDeref,bbDelimiter nextgroup=bbShFuncRegion skipwhite
+syn match bbShFuncDef           "^\(fakeroot\s*\)\?\([\.0-9A-Za-z_${}\-\.]\+\)\(python\)\@<!\(\s*()\s*\)\({\)\@=" contains=bbShFakeRootFlag,bbFunction,bbVarDeref,bbDelimiter nextgroup=bbShFuncRegion skipwhite
 syn region bbShFuncRegion       matchgroup=bbDelimiter start="{\s*$" end="^}\s*$" contained contains=@shell
 
 " Python value inside shell functions
@@ -91,7 +91,7 @@ syn region shDeref         start=+${@+ skip=+\\$+ excludenl end=+}+ contained co
 
 " BitBake python metadata
 syn keyword bbPyFlag            python contained
-syn match bbPyFuncDef           "^\(python\s\+\)\([0-9A-Za-z_${}-]\+\)\?\(\s*()\s*\)\({\)\@=" contains=bbPyFlag,bbFunction,bbVarDeref,bbDelimiter nextgroup=bbPyFuncRegion skipwhite
+syn match bbPyFuncDef           "^\(fakeroot\s*\)\?\(python\)\(\s\+[0-9A-Za-z_${}\-\.]\+\)\?\(\s*()\s*\)\({\)\@=" contains=bbShFakeRootFlag,bbPyFlag,bbFunction,bbVarDeref,bbDelimiter nextgroup=bbPyFuncRegion skipwhite
 syn region bbPyFuncRegion       matchgroup=bbDelimiter start="{\s*$" end="^}\s*$" contained contains=@python
 
 " BitBake 'def'd python functions

bitbake/doc/.gitignore

@@ -0,0 +1 @@
+_build/

bitbake/doc/Makefile

@@ -1,91 +1,35 @@
-# This is a single Makefile to handle all generated BitBake documents.
-# The Makefile needs to live in the documentation directory and all figures used
-# in any manuals must be .PNG files and live in the individual book's figures
-# directory.
-#
-# The Makefile has these targets:
-#
-#    pdf:      generates a PDF version of a manual.
-#    html:     generates an HTML version of a manual.
-#    tarball:  creates a tarball for the doc files.
-#    validate: validates
-#    clean:    removes files
-#
-# The Makefile generates an HTML version of every document.  The
-# variable DOC indicates the folder name for a given manual.
-#
-# To build a manual, you must invoke 'make' with the DOC argument.
-#
-# Examples:
-#
-#     make DOC=bitbake-user-manual
-#     make pdf DOC=bitbake-user-manual
-#
-# The first example generates the HTML version of the User Manual.
-# The second example generates the PDF version of the User Manual.
+# Minimal makefile for Sphinx documentation
 #
 
-ifeq ($(DOC),bitbake-user-manual)
-XSLTOPTS = --stringparam html.stylesheet bitbake-user-manual-style.css \
-           --stringparam  chapter.autolabel 1 \
-           --stringparam  section.autolabel 1 \
-           --stringparam  section.label.includes.component.label 1 \
-           --xinclude
-ALLPREQ = html tarball
-TARFILES = bitbake-user-manual-style.css bitbake-user-manual.html figures/bitbake-title.png
-MANUALS = $(DOC)/$(DOC).html
-FIGURES = figures
-STYLESHEET = $(DOC)/*.css
+# You can set these variables from the command line, and also
+# from the environment for the first two.
+SPHINXOPTS    ?=
+SPHINXBUILD   ?= sphinx-build
+SOURCEDIR     = .
+BUILDDIR      = _build
+DESTDIR       = final
 
+ifeq ($(shell if which $(SPHINXBUILD) >/dev/null 2>&1; then echo 1; else echo 0; fi),0)
+$(error "The '$(SPHINXBUILD)' command was not found. Make sure you have Sphinx installed")
 endif
 
-##
-# These URI should be rewritten by your distribution's xml catalog to
-# match your localy installed XSL stylesheets.
-XSL_BASE_URI  = http://docbook.sourceforge.net/release/xsl/current
-XSL_XHTML_URI = $(XSL_BASE_URI)/xhtml/docbook.xsl
+# Put it first so that "make" without argument is like "make help".
+help:
+	@$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)
 
-all: $(ALLPREQ)
+.PHONY: help Makefile clean publish
 
-pdf:
-ifeq ($(DOC),bitbake-user-manual)
-	@echo " "
-	@echo "********** Building."$(DOC)
-	@echo " "
-	cd $(DOC); ../tools/docbook-to-pdf $(DOC).xml ../template; cd ..
-endif
-
-html:
-ifeq ($(DOC),bitbake-user-manual)
-#       See http://www.sagehill.net/docbookxsl/HtmlOutput.html
-	@echo " "
-	@echo "******** Building "$(DOC)
-	@echo " "
-	cd $(DOC); xsltproc $(XSLTOPTS) -o $(DOC).html $(DOC)-customization.xsl $(DOC).xml; cd ..
-endif
-
-tarball: html
-	@echo " "
-	@echo "******** Creating Tarball of document files"
-	@echo " "
-	cd $(DOC); tar -cvzf $(DOC).tgz $(TARFILES); cd ..
-
-validate:
-	cd $(DOC); xmllint --postvalid --xinclude --noout $(DOC).xml; cd ..
-
-publish:
-	@if test -f $(DOC)/$(DOC).html; \
-	  then \
-            echo " "; \
-            echo "******** Publishing "$(DOC)".html"; \
-            echo " "; \
-            scp -r $(MANUALS) $(STYLESHEET) docs.yp:/var/www/www.yoctoproject.org-docs/$(VER)/$(DOC); \
-            cd $(DOC); scp -r $(FIGURES) docs.yp:/var/www/www.yoctoproject.org-docs/$(VER)/$(DOC); \
-	else \
-          echo " "; \
-          echo $(DOC)".html missing.  Generate the file first then try again."; \
-          echo " "; \
-	fi
+publish: Makefile html singlehtml
+	rm -rf $(BUILDDIR)/$(DESTDIR)/
+	mkdir -p $(BUILDDIR)/$(DESTDIR)/
+	cp -r $(BUILDDIR)/html/* $(BUILDDIR)/$(DESTDIR)/
+	cp $(BUILDDIR)/singlehtml/index.html $(BUILDDIR)/$(DESTDIR)/singleindex.html
+	sed -i -e 's@index.html#@singleindex.html#@g' $(BUILDDIR)/$(DESTDIR)/singleindex.html
 
 clean:
-	rm -rf $(MANUALS); rm $(DOC)/$(DOC).tgz;
+	@rm -rf $(BUILDDIR)
+
+# Catch-all target: route all unknown targets to Sphinx using the new
+# "make mode" option.  $(O) is meant as a shortcut for $(SPHINXOPTS).
+%: Makefile
+	@$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)

bitbake/doc/README

@@ -15,25 +15,41 @@ Each folder is self-contained regarding content and figures.
 If you want to find HTML versions of the BitBake manuals on the web, 
 go to http://www.openembedded.org/wiki/Documentation. 
 
-Makefile
-========
+Sphinx
+======
 
-The Makefile processes manual directories to create HTML, PDF,
-tarballs, etc.  Details on how the Makefile work are documented
-inside the Makefile.  See that file for more information.
+The BitBake documentation was migrated from the original DocBook
+format to Sphinx based documentation for the Yocto Project 3.2
+release.
 
-To build a manual, you run the make command and pass it the name
-of the folder containing the manual's contents. 
-For example, the following command run from the documentation directory 
-creates an HTML and a PDF version of the BitBake User Manual.
-The DOC variable specifies the manual you are making:
+Additional information related to the Sphinx migration, and guidelines
+for developers willing to contribute to the BitBake documentation can
+be found in the Yocto Project Documentation README file:
 
-     $ make DOC=bitbake-user-manual
+https://git.yoctoproject.org/cgit/cgit.cgi/yocto-docs/tree/documentation/README
 
-template
-========
-Contains various templates, fonts, and some old PNG files.
+How to build the Yocto Project documentation
+============================================
 
-tools
-=====
-Contains a tool to convert the DocBook files to PDF format.
+Sphinx is written in Python. While it might work with Python2, for
+obvious reasons, we will only support building the BitBake
+documentation with Python3.
+
+Sphinx might be available in your Linux distro packages repositories,
+however it is not recommend using distro packages, as they might be
+old versions, especially if you are using an LTS version of your
+distro. The recommended method to install Sphinx and all required
+dependencies is to use the Python Package Index (pip).
+
+To install all required packages run:
+
+ $ pip3 install sphinx sphinx_rtd_theme pyyaml
+
+To build the documentation locally, run:
+
+ $ cd documentation
+ $ make -f Makefile.sphinx html
+
+The resulting HTML index page will be _build/html/index.html, and you
+can browse your own copy of the locally generated documentation with
+your browser.

bitbake/doc/_templates/breadcrumbs.html

@@ -0,0 +1,14 @@
+{% extends "!breadcrumbs.html" %}
+
+{% block breadcrumbs %}
+    <li>
+      <span class="doctype_switcher_placeholder">{{ doctype or 'single' }}</span>
+      <span class="version_switcher_placeholder">{{ release }}</span>
+    </li>
+    <li> &raquo;</li>
+    {% for doc in parents %}
+      <li><a href="{{ doc.link|e }}">{{ doc.title }}</a> &raquo;</li>
+    {% endfor %}
+    <li>{{ title }}</li>
+{% endblock %}
+

bitbake/doc/_templates/layout.html

@@ -0,0 +1,7 @@
+{% extends "!layout.html" %}
+
+{% block extrabody %}
+<div id="outdated-warning" style="text-align: center; background-color: #FFBABA; color: #6A0E0E;">
+</div>
+{% endblock %}
+

bitbake/doc/bitbake-user-manual/bitbake-user-manual-customization.xsl

@@ -1,29 +0,0 @@
-<?xml version='1.0'?>
-<xsl:stylesheet xmlns:xsl="http://www.w3.org/1999/XSL/Transform" xmlns="http://www.w3.org/1999/xhtml" xmlns:fo="http://www.w3.org/1999/XSL/Format" version="1.0">
-
-  <xsl:import href="http://downloads.yoctoproject.org/mirror/docbook-mirror/docbook-xsl-1.76.1/xhtml/docbook.xsl" />
-
-<!--
-
-  <xsl:import href="../template/1.76.1/docbook-xsl-1.76.1/xhtml/docbook.xsl" />
-
-  <xsl:import href="http://docbook.sourceforge.net/release/xsl/1.76.1/xhtml/docbook.xsl" />
-
--->
-
-  <xsl:include href="../template/permalinks.xsl"/>
-  <xsl:include href="../template/section.title.xsl"/>
-  <xsl:include href="../template/component.title.xsl"/>
-  <xsl:include href="../template/division.title.xsl"/>
-  <xsl:include href="../template/formal.object.heading.xsl"/>
-  <xsl:include href="../template/gloss-permalinks.xsl"/>
-
-  <xsl:param name="html.stylesheet" select="'user-manual-style.css'" />
-  <xsl:param name="chapter.autolabel" select="1" />
-  <xsl:param name="section.autolabel" select="1" />
-  <xsl:param name="section.label.includes.component.label" select="1" />
-  <xsl:param name="appendix.autolabel">A</xsl:param>
-
-<!--  <xsl:param name="generate.toc" select="'article nop'"></xsl:param>  -->
-
-</xsl:stylesheet>

bitbake/doc/bitbake-user-manual/bitbake-user-manual-execution.rst

@@ -0,0 +1,733 @@
+.. SPDX-License-Identifier: CC-BY-2.5
+
+=========
+Execution
+=========
+
+|
+
+The primary purpose for running BitBake is to produce some kind of
+output such as a single installable package, a kernel, a software
+development kit, or even a full, board-specific bootable Linux image,
+complete with bootloader, kernel, and root filesystem. Of course, you
+can execute the ``bitbake`` command with options that cause it to
+execute single tasks, compile single recipe files, capture or clear
+data, or simply return information about the execution environment.
+
+This chapter describes BitBake's execution process from start to finish
+when you use it to create an image. The execution process is launched
+using the following command form: ::
+
+  $ bitbake target
+
+For information on
+the BitBake command and its options, see ":ref:`The BitBake Command
+<bitbake-user-manual-command>`" section.
+
+.. note::
+
+   Prior to executing BitBake, you should take advantage of available
+   parallel thread execution on your build host by setting the
+   :term:`BB_NUMBER_THREADS` variable in
+   your project's ``local.conf`` configuration file.
+
+   A common method to determine this value for your build host is to run
+   the following: ::
+
+     $ grep processor /proc/cpuinfo
+
+   This command returns
+   the number of processors, which takes into account hyper-threading.
+   Thus, a quad-core build host with hyper-threading most likely shows
+   eight processors, which is the value you would then assign to
+   ``BB_NUMBER_THREADS``.
+
+   A possibly simpler solution is that some Linux distributions (e.g.
+   Debian and Ubuntu) provide the ``ncpus`` command.
+
+Parsing the Base Configuration Metadata
+=======================================
+
+The first thing BitBake does is parse base configuration metadata. Base
+configuration metadata consists of your project's ``bblayers.conf`` file
+to determine what layers BitBake needs to recognize, all necessary
+``layer.conf`` files (one from each layer), and ``bitbake.conf``. The
+data itself is of various types:
+
+-  **Recipes:** Details about particular pieces of software.
+
+-  **Class Data:** An abstraction of common build information (e.g. how to
+   build a Linux kernel).
+
+-  **Configuration Data:** Machine-specific settings, policy decisions,
+   and so forth. Configuration data acts as the glue to bind everything
+   together.
+
+The ``layer.conf`` files are used to construct key variables such as
+:term:`BBPATH` and :term:`BBFILES`.
+``BBPATH`` is used to search for configuration and class files under the
+``conf`` and ``classes`` directories, respectively. ``BBFILES`` is used
+to locate both recipe and recipe append files (``.bb`` and
+``.bbappend``). If there is no ``bblayers.conf`` file, it is assumed the
+user has set the ``BBPATH`` and ``BBFILES`` directly in the environment.
+
+Next, the ``bitbake.conf`` file is located using the ``BBPATH`` variable
+that was just constructed. The ``bitbake.conf`` file may also include
+other configuration files using the ``include`` or ``require``
+directives.
+
+Prior to parsing configuration files, BitBake looks at certain
+variables, including:
+
+-  :term:`BB_ENV_WHITELIST`
+-  :term:`BB_ENV_EXTRAWHITE`
+-  :term:`BB_PRESERVE_ENV`
+-  :term:`BB_ORIGENV`
+-  :term:`BITBAKE_UI`
+
+The first four variables in this list relate to how BitBake treats shell
+environment variables during task execution. By default, BitBake cleans
+the environment variables and provides tight control over the shell
+execution environment. However, through the use of these first four
+variables, you can apply your control regarding the environment
+variables allowed to be used by BitBake in the shell during execution of
+tasks. See the
+":ref:`bitbake-user-manual/bitbake-user-manual-metadata:Passing Information Into the Build Task Environment`"
+section and the information about these variables in the variable
+glossary for more information on how they work and on how to use them.
+
+The base configuration metadata is global and therefore affects all
+recipes and tasks that are executed.
+
+BitBake first searches the current working directory for an optional
+``conf/bblayers.conf`` configuration file. This file is expected to
+contain a :term:`BBLAYERS` variable that is a
+space-delimited list of 'layer' directories. Recall that if BitBake
+cannot find a ``bblayers.conf`` file, then it is assumed the user has
+set the ``BBPATH`` and ``BBFILES`` variables directly in the
+environment.
+
+For each directory (layer) in this list, a ``conf/layer.conf`` file is
+located and parsed with the :term:`LAYERDIR` variable
+being set to the directory where the layer was found. The idea is these
+files automatically set up :term:`BBPATH` and other
+variables correctly for a given build directory.
+
+BitBake then expects to find the ``conf/bitbake.conf`` file somewhere in
+the user-specified ``BBPATH``. That configuration file generally has
+include directives to pull in any other metadata such as files specific
+to the architecture, the machine, the local environment, and so forth.
+
+Only variable definitions and include directives are allowed in BitBake
+``.conf`` files. Some variables directly influence BitBake's behavior.
+These variables might have been set from the environment depending on
+the environment variables previously mentioned or set in the
+configuration files. The ":ref:`bitbake-user-manual/bitbake-user-manual-ref-variables:Variables Glossary`"
+chapter presents a full list of
+variables.
+
+After parsing configuration files, BitBake uses its rudimentary
+inheritance mechanism, which is through class files, to inherit some
+standard classes. BitBake parses a class when the inherit directive
+responsible for getting that class is encountered.
+
+The ``base.bbclass`` file is always included. Other classes that are
+specified in the configuration using the
+:term:`INHERIT` variable are also included. BitBake
+searches for class files in a ``classes`` subdirectory under the paths
+in ``BBPATH`` in the same way as configuration files.
+
+A good way to get an idea of the configuration files and the class files
+used in your execution environment is to run the following BitBake
+command: ::
+
+  $ bitbake -e > mybb.log
+
+Examining the top of the ``mybb.log``
+shows you the many configuration files and class files used in your
+execution environment.
+
+.. note::
+
+   You need to be aware of how BitBake parses curly braces. If a recipe
+   uses a closing curly brace within the function and the character has
+   no leading spaces, BitBake produces a parsing error. If you use a
+   pair of curly braces in a shell function, the closing curly brace
+   must not be located at the start of the line without leading spaces.
+
+   Here is an example that causes BitBake to produce a parsing error: ::
+
+      fakeroot create_shar() {
+         cat << "EOF" > ${SDK_DEPLOY}/${TOOLCHAIN_OUTPUTNAME}.sh
+      usage()
+      {
+         echo "test"
+         ######  The following "}" at the start of the line causes a parsing error ######
+      }
+      EOF
+      }
+
+      Writing the recipe this way avoids the error:
+      fakeroot create_shar() {
+         cat << "EOF" > ${SDK_DEPLOY}/${TOOLCHAIN_OUTPUTNAME}.sh
+      usage()
+      {
+         echo "test"
+         ###### The following "}" with a leading space at the start of the line avoids the error ######
+       }
+      EOF
+      }
+
+Locating and Parsing Recipes
+============================
+
+During the configuration phase, BitBake will have set
+:term:`BBFILES`. BitBake now uses it to construct a
+list of recipes to parse, along with any append files (``.bbappend``) to
+apply. ``BBFILES`` is a space-separated list of available files and
+supports wildcards. An example would be: ::
+
+  BBFILES = "/path/to/bbfiles/*.bb /path/to/appends/*.bbappend"
+
+BitBake parses each
+recipe and append file located with ``BBFILES`` and stores the values of
+various variables into the datastore.
+
+.. note::
+
+   Append files are applied in the order they are encountered in BBFILES.
+
+For each file, a fresh copy of the base configuration is made, then the
+recipe is parsed line by line. Any inherit statements cause BitBake to
+find and then parse class files (``.bbclass``) using
+:term:`BBPATH` as the search path. Finally, BitBake
+parses in order any append files found in ``BBFILES``.
+
+One common convention is to use the recipe filename to define pieces of
+metadata. For example, in ``bitbake.conf`` the recipe name and version
+are used to set the variables :term:`PN` and
+:term:`PV`: ::
+
+   PN = "${@bb.parse.BBHandler.vars_from_file(d.getVar('FILE', False),d)[0] or 'defaultpkgname'}"
+   PV = "${@bb.parse.BBHandler.vars_from_file(d.getVar('FILE', False),d)[1] or '1.0'}"
+
+In this example, a recipe called "something_1.2.3.bb" would set
+``PN`` to "something" and ``PV`` to "1.2.3".
+
+By the time parsing is complete for a recipe, BitBake has a list of
+tasks that the recipe defines and a set of data consisting of keys and
+values as well as dependency information about the tasks.
+
+BitBake does not need all of this information. It only needs a small
+subset of the information to make decisions about the recipe.
+Consequently, BitBake caches the values in which it is interested and
+does not store the rest of the information. Experience has shown it is
+faster to re-parse the metadata than to try and write it out to the disk
+and then reload it.
+
+Where possible, subsequent BitBake commands reuse this cache of recipe
+information. The validity of this cache is determined by first computing
+a checksum of the base configuration data (see
+:term:`BB_HASHCONFIG_WHITELIST`) and
+then checking if the checksum matches. If that checksum matches what is
+in the cache and the recipe and class files have not changed, BitBake is
+able to use the cache. BitBake then reloads the cached information about
+the recipe instead of reparsing it from scratch.
+
+Recipe file collections exist to allow the user to have multiple
+repositories of ``.bb`` files that contain the same exact package. For
+example, one could easily use them to make one's own local copy of an
+upstream repository, but with custom modifications that one does not
+want upstream. Here is an example: ::
+
+  BBFILES = "/stuff/openembedded/*/*.bb /stuff/openembedded.modified/*/*.bb"
+  BBFILE_COLLECTIONS = "upstream local"
+  BBFILE_PATTERN_upstream = "^/stuff/openembedded/"
+  BBFILE_PATTERN_local = "^/stuff/openembedded.modified/"
+  BBFILE_PRIORITY_upstream = "5" BBFILE_PRIORITY_local = "10"
+
+.. note::
+
+   The layers mechanism is now the preferred method of collecting code.
+   While the collections code remains, its main use is to set layer
+   priorities and to deal with overlap (conflicts) between layers.
+
+.. _bb-bitbake-providers:
+
+Providers
+=========
+
+Assuming BitBake has been instructed to execute a target and that all
+the recipe files have been parsed, BitBake starts to figure out how to
+build the target. BitBake looks through the ``PROVIDES`` list for each
+of the recipes. A ``PROVIDES`` list is the list of names by which the
+recipe can be known. Each recipe's ``PROVIDES`` list is created
+implicitly through the recipe's :term:`PN` variable and
+explicitly through the recipe's :term:`PROVIDES`
+variable, which is optional.
+
+When a recipe uses ``PROVIDES``, that recipe's functionality can be
+found under an alternative name or names other than the implicit ``PN``
+name. As an example, suppose a recipe named ``keyboard_1.0.bb``
+contained the following: ::
+
+  PROVIDES += "fullkeyboard"
+
+The ``PROVIDES``
+list for this recipe becomes "keyboard", which is implicit, and
+"fullkeyboard", which is explicit. Consequently, the functionality found
+in ``keyboard_1.0.bb`` can be found under two different names.
+
+.. _bb-bitbake-preferences:
+
+Preferences
+===========
+
+The ``PROVIDES`` list is only part of the solution for figuring out a
+target's recipes. Because targets might have multiple providers, BitBake
+needs to prioritize providers by determining provider preferences.
+
+A common example in which a target has multiple providers is
+"virtual/kernel", which is on the ``PROVIDES`` list for each kernel
+recipe. Each machine often selects the best kernel provider by using a
+line similar to the following in the machine configuration file: ::
+
+  PREFERRED_PROVIDER_virtual/kernel = "linux-yocto"
+
+The default :term:`PREFERRED_PROVIDER` is the provider
+with the same name as the target. BitBake iterates through each target
+it needs to build and resolves them and their dependencies using this
+process.
+
+Understanding how providers are chosen is made complicated by the fact
+that multiple versions might exist for a given provider. BitBake
+defaults to the highest version of a provider. Version comparisons are
+made using the same method as Debian. You can use the
+:term:`PREFERRED_VERSION` variable to
+specify a particular version. You can influence the order by using the
+:term:`DEFAULT_PREFERENCE` variable.
+
+By default, files have a preference of "0". Setting
+``DEFAULT_PREFERENCE`` to "-1" makes the recipe unlikely to be used
+unless it is explicitly referenced. Setting ``DEFAULT_PREFERENCE`` to
+"1" makes it likely the recipe is used. ``PREFERRED_VERSION`` overrides
+any ``DEFAULT_PREFERENCE`` setting. ``DEFAULT_PREFERENCE`` is often used
+to mark newer and more experimental recipe versions until they have
+undergone sufficient testing to be considered stable.
+
+When there are multiple "versions" of a given recipe, BitBake defaults
+to selecting the most recent version, unless otherwise specified. If the
+recipe in question has a
+:term:`DEFAULT_PREFERENCE` set lower than
+the other recipes (default is 0), then it will not be selected. This
+allows the person or persons maintaining the repository of recipe files
+to specify their preference for the default selected version.
+Additionally, the user can specify their preferred version.
+
+If the first recipe is named ``a_1.1.bb``, then the
+:term:`PN` variable will be set to "a", and the
+:term:`PV` variable will be set to 1.1.
+
+Thus, if a recipe named ``a_1.2.bb`` exists, BitBake will choose 1.2 by
+default. However, if you define the following variable in a ``.conf``
+file that BitBake parses, you can change that preference: ::
+
+  PREFERRED_VERSION_a = "1.1"
+
+.. note::
+
+   It is common for a recipe to provide two versions -- a stable,
+   numbered (and preferred) version, and a version that is automatically
+   checked out from a source code repository that is considered more
+   "bleeding edge" but can be selected only explicitly.
+
+   For example, in the OpenEmbedded codebase, there is a standard,
+   versioned recipe file for BusyBox, ``busybox_1.22.1.bb``, but there
+   is also a Git-based version, ``busybox_git.bb``, which explicitly
+   contains the line ::
+
+     DEFAULT_PREFERENCE = "-1"
+
+   to ensure that the
+   numbered, stable version is always preferred unless the developer
+   selects otherwise.
+
+.. _bb-bitbake-dependencies:
+
+Dependencies
+============
+
+Each target BitBake builds consists of multiple tasks such as ``fetch``,
+``unpack``, ``patch``, ``configure``, and ``compile``. For best
+performance on multi-core systems, BitBake considers each task as an
+independent entity with its own set of dependencies.
+
+Dependencies are defined through several variables. You can find
+information about variables BitBake uses in the
+:doc:`bitbake-user-manual-ref-variables` near the end of this manual. At a
+basic level, it is sufficient to know that BitBake uses the
+:term:`DEPENDS` and
+:term:`RDEPENDS` variables when calculating
+dependencies.
+
+For more information on how BitBake handles dependencies, see the
+:ref:`bitbake-user-manual/bitbake-user-manual-metadata:Dependencies`
+section.
+
+.. _ref-bitbake-tasklist:
+
+The Task List
+=============
+
+Based on the generated list of providers and the dependency information,
+BitBake can now calculate exactly what tasks it needs to run and in what
+order it needs to run them. The
+:ref:`bitbake-user-manual/bitbake-user-manual-execution:executing tasks`
+section has more information on how BitBake chooses which task to
+execute next.
+
+The build now starts with BitBake forking off threads up to the limit
+set in the :term:`BB_NUMBER_THREADS`
+variable. BitBake continues to fork threads as long as there are tasks
+ready to run, those tasks have all their dependencies met, and the
+thread threshold has not been exceeded.
+
+It is worth noting that you can greatly speed up the build time by
+properly setting the ``BB_NUMBER_THREADS`` variable.
+
+As each task completes, a timestamp is written to the directory
+specified by the :term:`STAMP` variable. On subsequent
+runs, BitBake looks in the build directory within ``tmp/stamps`` and
+does not rerun tasks that are already completed unless a timestamp is
+found to be invalid. Currently, invalid timestamps are only considered
+on a per recipe file basis. So, for example, if the configure stamp has
+a timestamp greater than the compile timestamp for a given target, then
+the compile task would rerun. Running the compile task again, however,
+has no effect on other providers that depend on that target.
+
+The exact format of the stamps is partly configurable. In modern
+versions of BitBake, a hash is appended to the stamp so that if the
+configuration changes, the stamp becomes invalid and the task is
+automatically rerun. This hash, or signature used, is governed by the
+signature policy that is configured (see the
+:ref:`bitbake-user-manual/bitbake-user-manual-execution:checksums (signatures)`
+section for information). It is also
+possible to append extra metadata to the stamp using the
+``[stamp-extra-info]`` task flag. For example, OpenEmbedded uses this
+flag to make some tasks machine-specific.
+
+.. note::
+
+   Some tasks are marked as "nostamp" tasks. No timestamp file is
+   created when these tasks are run. Consequently, "nostamp" tasks are
+   always rerun.
+
+For more information on tasks, see the
+:ref:`bitbake-user-manual/bitbake-user-manual-metadata:tasks` section.
+
+Executing Tasks
+===============
+
+Tasks can be either a shell task or a Python task. For shell tasks,
+BitBake writes a shell script to
+``${``\ :term:`T`\ ``}/run.do_taskname.pid`` and then
+executes the script. The generated shell script contains all the
+exported variables, and the shell functions with all variables expanded.
+Output from the shell script goes to the file
+``${T}/log.do_taskname.pid``. Looking at the expanded shell functions in
+the run file and the output in the log files is a useful debugging
+technique.
+
+For Python tasks, BitBake executes the task internally and logs
+information to the controlling terminal. Future versions of BitBake will
+write the functions to files similar to the way shell tasks are handled.
+Logging will be handled in a way similar to shell tasks as well.
+
+The order in which BitBake runs the tasks is controlled by its task
+scheduler. It is possible to configure the scheduler and define custom
+implementations for specific use cases. For more information, see these
+variables that control the behavior:
+
+-  :term:`BB_SCHEDULER`
+
+-  :term:`BB_SCHEDULERS`
+
+It is possible to have functions run before and after a task's main
+function. This is done using the ``[prefuncs]`` and ``[postfuncs]``
+flags of the task that lists the functions to run.
+
+.. _checksums:
+
+Checksums (Signatures)
+======================
+
+A checksum is a unique signature of a task's inputs. The signature of a
+task can be used to determine if a task needs to be run. Because it is a
+change in a task's inputs that triggers running the task, BitBake needs
+to detect all the inputs to a given task. For shell tasks, this turns
+out to be fairly easy because BitBake generates a "run" shell script for
+each task and it is possible to create a checksum that gives you a good
+idea of when the task's data changes.
+
+To complicate the problem, some things should not be included in the
+checksum. First, there is the actual specific build path of a given task
+- the working directory. It does not matter if the working directory
+changes because it should not affect the output for target packages. The
+simplistic approach for excluding the working directory is to set it to
+some fixed value and create the checksum for the "run" script. BitBake
+goes one step better and uses the
+:term:`BB_HASHBASE_WHITELIST` variable
+to define a list of variables that should never be included when
+generating the signatures.
+
+Another problem results from the "run" scripts containing functions that
+might or might not get called. The incremental build solution contains
+code that figures out dependencies between shell functions. This code is
+used to prune the "run" scripts down to the minimum set, thereby
+alleviating this problem and making the "run" scripts much more readable
+as a bonus.
+
+So far we have solutions for shell scripts. What about Python tasks? The
+same approach applies even though these tasks are more difficult. The
+process needs to figure out what variables a Python function accesses
+and what functions it calls. Again, the incremental build solution
+contains code that first figures out the variable and function
+dependencies, and then creates a checksum for the data used as the input
+to the task.
+
+Like the working directory case, situations exist where dependencies
+should be ignored. For these cases, you can instruct the build process
+to ignore a dependency by using a line like the following: ::
+
+  PACKAGE_ARCHS[vardepsexclude] = "MACHINE"
+
+This example ensures that the
+``PACKAGE_ARCHS`` variable does not depend on the value of ``MACHINE``,
+even if it does reference it.
+
+Equally, there are cases where we need to add dependencies BitBake is
+not able to find. You can accomplish this by using a line like the
+following: ::
+
+  PACKAGE_ARCHS[vardeps] = "MACHINE"
+
+This example explicitly
+adds the ``MACHINE`` variable as a dependency for ``PACKAGE_ARCHS``.
+
+Consider a case with in-line Python, for example, where BitBake is not
+able to figure out dependencies. When running in debug mode (i.e. using
+``-DDD``), BitBake produces output when it discovers something for which
+it cannot figure out dependencies.
+
+Thus far, this section has limited discussion to the direct inputs into
+a task. Information based on direct inputs is referred to as the
+"basehash" in the code. However, there is still the question of a task's
+indirect inputs - the things that were already built and present in the
+build directory. The checksum (or signature) for a particular task needs
+to add the hashes of all the tasks on which the particular task depends.
+Choosing which dependencies to add is a policy decision. However, the
+effect is to generate a master checksum that combines the basehash and
+the hashes of the task's dependencies.
+
+At the code level, there are a variety of ways both the basehash and the
+dependent task hashes can be influenced. Within the BitBake
+configuration file, we can give BitBake some extra information to help
+it construct the basehash. The following statement effectively results
+in a list of global variable dependency excludes - variables never
+included in any checksum. This example uses variables from OpenEmbedded
+to help illustrate the concept: ::
+
+   BB_HASHBASE_WHITELIST ?= "TMPDIR FILE PATH PWD BB_TASKHASH BBPATH DL_DIR \
+       SSTATE_DIR THISDIR FILESEXTRAPATHS FILE_DIRNAME HOME LOGNAME SHELL \
+       USER FILESPATH STAGING_DIR_HOST STAGING_DIR_TARGET COREBASE PRSERV_HOST \
+       PRSERV_DUMPDIR PRSERV_DUMPFILE PRSERV_LOCKDOWN PARALLEL_MAKE \
+       CCACHE_DIR EXTERNAL_TOOLCHAIN CCACHE CCACHE_DISABLE LICENSE_PATH SDKPKGSUFFIX"
+
+The previous example excludes the work directory, which is part of
+``TMPDIR``.
+
+The rules for deciding which hashes of dependent tasks to include
+through dependency chains are more complex and are generally
+accomplished with a Python function. The code in
+``meta/lib/oe/sstatesig.py`` shows two examples of this and also
+illustrates how you can insert your own policy into the system if so
+desired. This file defines the two basic signature generators
+OpenEmbedded-Core uses: "OEBasic" and "OEBasicHash". By default, there
+is a dummy "noop" signature handler enabled in BitBake. This means that
+behavior is unchanged from previous versions. ``OE-Core`` uses the
+"OEBasicHash" signature handler by default through this setting in the
+``bitbake.conf`` file: ::
+
+  BB_SIGNATURE_HANDLER ?= "OEBasicHash"
+
+The "OEBasicHash" ``BB_SIGNATURE_HANDLER`` is the same as the "OEBasic"
+version but adds the task hash to the stamp files. This results in any
+metadata change that changes the task hash, automatically causing the
+task to be run again. This removes the need to bump
+:term:`PR` values, and changes to metadata automatically
+ripple across the build.
+
+It is also worth noting that the end result of these signature
+generators is to make some dependency and hash information available to
+the build. This information includes:
+
+-  ``BB_BASEHASH_task-``\ *taskname*: The base hashes for each task in the
+   recipe.
+
+-  ``BB_BASEHASH_``\ *filename:taskname*: The base hashes for each
+   dependent task.
+
+-  ``BBHASHDEPS_``\ *filename:taskname*: The task dependencies for
+   each task.
+
+-  ``BB_TASKHASH``: The hash of the currently running task.
+
+It is worth noting that BitBake's "-S" option lets you debug BitBake's
+processing of signatures. The options passed to -S allow different
+debugging modes to be used, either using BitBake's own debug functions
+or possibly those defined in the metadata/signature handler itself. The
+simplest parameter to pass is "none", which causes a set of signature
+information to be written out into ``STAMPS_DIR`` corresponding to the
+targets specified. The other currently available parameter is
+"printdiff", which causes BitBake to try to establish the closest
+signature match it can (e.g. in the sstate cache) and then run
+``bitbake-diffsigs`` over the matches to determine the stamps and delta
+where these two stamp trees diverge.
+
+.. note::
+
+   It is likely that future versions of BitBake will provide other
+   signature handlers triggered through additional "-S" parameters.
+
+You can find more information on checksum metadata in the
+:ref:`bitbake-user-manual/bitbake-user-manual-metadata:task checksums and setscene`
+section.
+
+Setscene
+========
+
+The setscene process enables BitBake to handle "pre-built" artifacts.
+The ability to handle and reuse these artifacts allows BitBake the
+luxury of not having to build something from scratch every time.
+Instead, BitBake can use, when possible, existing build artifacts.
+
+BitBake needs to have reliable data indicating whether or not an
+artifact is compatible. Signatures, described in the previous section,
+provide an ideal way of representing whether an artifact is compatible.
+If a signature is the same, an object can be reused.
+
+If an object can be reused, the problem then becomes how to replace a
+given task or set of tasks with the pre-built artifact. BitBake solves
+the problem with the "setscene" process.
+
+When BitBake is asked to build a given target, before building anything,
+it first asks whether cached information is available for any of the
+targets it's building, or any of the intermediate targets. If cached
+information is available, BitBake uses this information instead of
+running the main tasks.
+
+BitBake first calls the function defined by the
+:term:`BB_HASHCHECK_FUNCTION` variable
+with a list of tasks and corresponding hashes it wants to build. This
+function is designed to be fast and returns a list of the tasks for
+which it believes in can obtain artifacts.
+
+Next, for each of the tasks that were returned as possibilities, BitBake
+executes a setscene version of the task that the possible artifact
+covers. Setscene versions of a task have the string "_setscene" appended
+to the task name. So, for example, the task with the name ``xxx`` has a
+setscene task named ``xxx_setscene``. The setscene version of the task
+executes and provides the necessary artifacts returning either success
+or failure.
+
+As previously mentioned, an artifact can cover more than one task. For
+example, it is pointless to obtain a compiler if you already have the
+compiled binary. To handle this, BitBake calls the
+:term:`BB_SETSCENE_DEPVALID` function for
+each successful setscene task to know whether or not it needs to obtain
+the dependencies of that task.
+
+Finally, after all the setscene tasks have executed, BitBake calls the
+function listed in
+:term:`BB_SETSCENE_VERIFY_FUNCTION2`
+with the list of tasks BitBake thinks has been "covered". The metadata
+can then ensure that this list is correct and can inform BitBake that it
+wants specific tasks to be run regardless of the setscene result.
+
+You can find more information on setscene metadata in the
+:ref:`bitbake-user-manual/bitbake-user-manual-metadata:task checksums and setscene`
+section.
+
+Logging
+=======
+
+In addition to the standard command line option to control how verbose
+builds are when execute, bitbake also supports user defined
+configuration of the `Python
+logging <https://docs.python.org/3/library/logging.html>`__ facilities
+through the :term:`BB_LOGCONFIG` variable. This
+variable defines a json or yaml `logging
+configuration <https://docs.python.org/3/library/logging.config.html>`__
+that will be intelligently merged into the default configuration. The
+logging configuration is merged using the following rules:
+
+-  The user defined configuration will completely replace the default
+   configuration if top level key ``bitbake_merge`` is set to the value
+   ``False``. In this case, all other rules are ignored.
+
+-  The user configuration must have a top level ``version`` which must
+   match the value of the default configuration.
+
+-  Any keys defined in the ``handlers``, ``formatters``, or ``filters``,
+   will be merged into the same section in the default configuration,
+   with the user specified keys taking replacing a default one if there
+   is a conflict. In practice, this means that if both the default
+   configuration and user configuration specify a handler named
+   ``myhandler``, the user defined one will replace the default. To
+   prevent the user from inadvertently replacing a default handler,
+   formatter, or filter, all of the default ones are named with a prefix
+   of "``BitBake.``"
+
+-  If a logger is defined by the user with the key ``bitbake_merge`` set
+   to ``False``, that logger will be completely replaced by user
+   configuration. In this case, no other rules will apply to that
+   logger.
+
+-  All user defined ``filter`` and ``handlers`` properties for a given
+   logger will be merged with corresponding properties from the default
+   logger. For example, if the user configuration adds a filter called
+   ``myFilter`` to the ``BitBake.SigGen``, and the default configuration
+   adds a filter called ``BitBake.defaultFilter``, both filters will be
+   applied to the logger
+
+As an example, consider the following user logging configuration file
+which logs all Hash Equivalence related messages of VERBOSE or higher to
+a file called ``hashequiv.log`` ::
+
+   {
+       "version": 1,
+       "handlers": {
+           "autobuilderlog": {
+               "class": "logging.FileHandler",
+               "formatter": "logfileFormatter",
+               "level": "DEBUG",
+               "filename": "hashequiv.log",
+               "mode": "w"
+           }
+       },
+       "formatters": {
+               "logfileFormatter": {
+                   "format": "%(name)s: %(levelname)s: %(message)s"
+               }
+       },
+       "loggers": {
+           "BitBake.SigGen.HashEquiv": {
+               "level": "VERBOSE",
+               "handlers": ["autobuilderlog"]
+           },
+           "BitBake.RunQueue.HashEquiv": {
+               "level": "VERBOSE",
+               "handlers": ["autobuilderlog"]
+           }
+       }
+   }

bitbake/doc/bitbake-user-manual/bitbake-user-manual-execution.xml

@@ -1,932 +0,0 @@
-<!DOCTYPE chapter PUBLIC "-//OASIS//DTD DocBook XML V4.2//EN"
-"http://www.oasis-open.org/docbook/xml/4.2/docbookx.dtd">
-
-<chapter id="bitbake-user-manual-execution">
-    <title>Execution</title>
-
-    <para>
-        The primary purpose for running BitBake is to produce some kind
-        of output such as a single installable package, a kernel, a software
-        development kit, or even a full, board-specific bootable Linux image,
-        complete with bootloader, kernel, and root filesystem.
-        Of course, you can execute the <filename>bitbake</filename>
-        command with options that cause it to execute single tasks,
-        compile single recipe files, capture or clear data, or simply
-        return information about the execution environment.
-    </para>
-
-    <para>
-        This chapter describes BitBake's execution process from start
-        to finish when you use it to create an image.
-        The execution process is launched using the following command
-        form:
-        <literallayout class='monospaced'>
-     $ bitbake <replaceable>target</replaceable>
-        </literallayout>
-        For information on the BitBake command and its options,
-        see
-        "<link linkend='bitbake-user-manual-command'>The BitBake Command</link>"
-        section.
-        <note>
-            <para>
-                Prior to executing BitBake, you should take advantage of available
-                parallel thread execution on your build host by setting the
-                <link linkend='var-bb-BB_NUMBER_THREADS'><filename>BB_NUMBER_THREADS</filename></link>
-                variable in your project's <filename>local.conf</filename>
-                configuration file.
-            </para>
-
-            <para>
-                A common method to determine this value for your build host is to run
-                the following:
-                <literallayout class='monospaced'>
-     $ grep processor /proc/cpuinfo
-                </literallayout>
-                This command returns the number of processors, which takes into
-                account hyper-threading.
-                Thus, a quad-core build host with hyper-threading most likely
-                shows eight processors, which is the value you would then assign to
-                <filename>BB_NUMBER_THREADS</filename>.
-            </para>
-
-            <para>
-                A possibly simpler solution is that some Linux distributions
-                (e.g. Debian and Ubuntu) provide the <filename>ncpus</filename> command.
-            </para>
-        </note>
-    </para>
-
-    <section id='parsing-the-base-configuration-metadata'>
-        <title>Parsing the Base Configuration Metadata</title>
-
-        <para>
-            The first thing BitBake does is parse base configuration
-            metadata.
-            Base configuration metadata consists of your project's
-            <filename>bblayers.conf</filename> file to determine what
-            layers BitBake needs to recognize, all necessary
-            <filename>layer.conf</filename> files (one from each layer),
-            and <filename>bitbake.conf</filename>.
-            The data itself is of various types:
-            <itemizedlist>
-                <listitem><para><emphasis>Recipes:</emphasis>
-                    Details about particular pieces of software.
-                    </para></listitem>
-                <listitem><para><emphasis>Class Data:</emphasis>
-                    An abstraction of common build information
-                    (e.g. how to build a Linux kernel).
-                    </para></listitem>
-                <listitem><para><emphasis>Configuration Data:</emphasis>
-                    Machine-specific settings, policy decisions,
-                    and so forth.
-                    Configuration data acts as the glue to bind everything
-                    together.</para></listitem>
-            </itemizedlist>
-        </para>
-
-        <para>
-            The <filename>layer.conf</filename> files are used to
-            construct key variables such as
-            <link linkend='var-bb-BBPATH'><filename>BBPATH</filename></link>
-            and
-            <link linkend='var-bb-BBFILES'><filename>BBFILES</filename></link>.
-            <filename>BBPATH</filename> is used to search for
-            configuration and class files under the
-            <filename>conf</filename> and <filename>classes</filename>
-            directories, respectively.
-            <filename>BBFILES</filename> is used to locate both recipe
-            and recipe append files
-            (<filename>.bb</filename> and <filename>.bbappend</filename>).
-            If there is no <filename>bblayers.conf</filename> file,
-            it is assumed the user has set the <filename>BBPATH</filename>
-            and <filename>BBFILES</filename> directly in the environment.
-        </para>
-
-        <para>
-            Next, the <filename>bitbake.conf</filename> file is located
-            using the <filename>BBPATH</filename> variable that was
-            just constructed.
-            The <filename>bitbake.conf</filename> file may also include other
-            configuration files using the
-            <filename>include</filename> or
-            <filename>require</filename> directives.
-        </para>
-
-        <para>
-            Prior to parsing configuration files, Bitbake looks
-            at certain variables, including:
-            <itemizedlist>
-                <listitem><para>
-                    <link linkend='var-bb-BB_ENV_WHITELIST'><filename>BB_ENV_WHITELIST</filename></link>
-                    </para></listitem>
-                <listitem><para>
-                    <link linkend='var-bb-BB_ENV_EXTRAWHITE'><filename>BB_ENV_EXTRAWHITE</filename></link>
-                    </para></listitem>
-                <listitem><para>
-                    <link linkend='var-bb-BB_PRESERVE_ENV'><filename>BB_PRESERVE_ENV</filename></link>
-                    </para></listitem>
-                <listitem><para>
-                    <link linkend='var-bb-BB_ORIGENV'><filename>BB_ORIGENV</filename></link>
-                    </para></listitem>
-                <listitem><para>
-                    <link linkend='var-bb-BITBAKE_UI'><filename>BITBAKE_UI</filename></link>
-                    </para></listitem>
-            </itemizedlist>
-            The first four variables in this list relate to how BitBake treats shell
-            environment variables during task execution.
-            By default, BitBake cleans the environment variables and provides tight
-            control over the shell execution environment.
-            However, through the use of these first four variables, you can
-            apply your control regarding the
-            environment variables allowed to be used by BitBake in the shell
-            during execution of tasks.
-            See the
-            "<link linkend='passing-information-into-the-build-task-environment'>Passing Information Into the Build Task Environment</link>"
-            section and the information about these variables in the
-            variable glossary for more information on how they work and
-            on how to use them.
-        </para>
-
-        <para>
-            The base configuration metadata is global
-            and therefore affects all recipes and tasks that are executed.
-        </para>
-
-        <para>
-            BitBake first searches the current working directory for an
-            optional <filename>conf/bblayers.conf</filename> configuration file.
-            This file is expected to contain a
-            <link linkend='var-bb-BBLAYERS'><filename>BBLAYERS</filename></link>
-            variable that is a space-delimited list of 'layer' directories.
-            Recall that if BitBake cannot find a <filename>bblayers.conf</filename>
-            file, then it is assumed the user has set the <filename>BBPATH</filename>
-            and <filename>BBFILES</filename> variables directly in the environment.
-        </para>
-
-        <para>
-            For each directory (layer) in this list, a <filename>conf/layer.conf</filename>
-            file is located and parsed with the
-            <link linkend='var-bb-LAYERDIR'><filename>LAYERDIR</filename></link>
-            variable being set to the directory where the layer was found.
-            The idea is these files automatically set up
-            <link linkend='var-bb-BBPATH'><filename>BBPATH</filename></link>
-            and other variables correctly for a given build directory.
-        </para>
-
-        <para>
-            BitBake then expects to find the <filename>conf/bitbake.conf</filename>
-            file somewhere in the user-specified <filename>BBPATH</filename>.
-            That configuration file generally has include directives to pull
-            in any other metadata such as files specific to the architecture,
-            the machine, the local environment, and so forth.
-        </para>
-
-        <para>
-            Only variable definitions and include directives are allowed
-            in BitBake <filename>.conf</filename> files.
-            Some variables directly influence BitBake's behavior.
-            These variables might have been set from the environment
-            depending on the environment variables previously
-            mentioned or set in the configuration files.
-            The
-            "<link linkend='ref-bb-variables-glos'>Variables Glossary</link>"
-            chapter presents a full list of variables.
-        </para>
-
-        <para>
-            After parsing configuration files, BitBake uses its rudimentary
-            inheritance mechanism, which is through class files, to inherit
-            some standard classes.
-            BitBake parses a class when the inherit directive responsible
-            for getting that class is encountered.
-        </para>
-
-        <para>
-            The <filename>base.bbclass</filename> file is always included.
-            Other classes that are specified in the configuration using the
-            <link linkend='var-bb-INHERIT'><filename>INHERIT</filename></link>
-            variable are also included.
-            BitBake searches for class files in a
-            <filename>classes</filename> subdirectory under
-            the paths in <filename>BBPATH</filename> in the same way as
-            configuration files.
-        </para>
-
-        <para>
-            A good way to get an idea of the configuration files and
-            the class files used in your execution environment is to
-            run the following BitBake command:
-            <literallayout class='monospaced'>
-     $ bitbake -e > mybb.log
-            </literallayout>
-            Examining the top of the <filename>mybb.log</filename>
-            shows you the many configuration files and class files
-            used in your execution environment.
-        </para>
-
-        <note>
-            <para>
-                You need to be aware of how BitBake parses curly braces.
-                If a recipe uses a closing curly brace within the function and
-                the character has no leading spaces, BitBake produces a parsing
-                error.
-                If you use a pair of curly braces in a shell function, the
-                closing curly brace must not be located at the start of the line
-                without leading spaces.
-            </para>
-
-            <para>
-                Here is an example that causes BitBake to produce a parsing
-                error:
-                <literallayout class='monospaced'>
-     fakeroot create_shar() {
-         cat &lt;&lt; "EOF" &gt; ${SDK_DEPLOY}/${TOOLCHAIN_OUTPUTNAME}.sh
-     usage()
-     {
-       echo "test"
-       ###### The following "}" at the start of the line causes a parsing error ######
-     }
-     EOF
-     }
-                </literallayout>
-                Writing the recipe this way avoids the error:
-                <literallayout class='monospaced'>
-     fakeroot create_shar() {
-         cat &lt;&lt; "EOF" &gt; ${SDK_DEPLOY}/${TOOLCHAIN_OUTPUTNAME}.sh
-     usage()
-     {
-       echo "test"
-       ######The following "}" with a leading space at the start of the line avoids the error ######
-      }
-     EOF
-     }
-                </literallayout>
-            </para>
-        </note>
-    </section>
-
-    <section id='locating-and-parsing-recipes'>
-        <title>Locating and Parsing Recipes</title>
-
-        <para>
-            During the configuration phase, BitBake will have set
-            <link linkend='var-bb-BBFILES'><filename>BBFILES</filename></link>.
-            BitBake now uses it to construct a list of recipes to parse,
-            along with any append files (<filename>.bbappend</filename>)
-            to apply.
-            <filename>BBFILES</filename> is a space-separated list of
-            available files and supports wildcards.
-            An example would be:
-            <literallayout class='monospaced'>
-     BBFILES = "/path/to/bbfiles/*.bb /path/to/appends/*.bbappend"
-            </literallayout>
-            BitBake parses each recipe and append file located
-            with <filename>BBFILES</filename> and stores the values of
-            various variables into the datastore.
-            <note>
-                Append files are applied in the order they are encountered in
-                <filename>BBFILES</filename>.
-            </note>
-            For each file, a fresh copy of the base configuration is
-            made, then the recipe is parsed line by line.
-            Any inherit statements cause BitBake to find and
-            then parse class files (<filename>.bbclass</filename>)
-            using
-            <link linkend='var-bb-BBPATH'><filename>BBPATH</filename></link>
-            as the search path.
-            Finally, BitBake parses in order any append files found in
-            <filename>BBFILES</filename>.
-        </para>
-
-        <para>
-            One common convention is to use the recipe filename to define
-            pieces of metadata.
-            For example, in <filename>bitbake.conf</filename> the recipe
-            name and version are used to set the variables
-            <link linkend='var-bb-PN'><filename>PN</filename></link> and
-            <link linkend='var-bb-PV'><filename>PV</filename></link>:
-            <literallayout class='monospaced'>
-     PN = "${@bb.parse.BBHandler.vars_from_file(d.getVar('FILE', False),d)[0] or 'defaultpkgname'}"
-     PV = "${@bb.parse.BBHandler.vars_from_file(d.getVar('FILE', False),d)[1] or '1.0'}"
-            </literallayout>
-            In this example, a recipe called "something_1.2.3.bb" would set
-            <filename>PN</filename> to "something" and
-            <filename>PV</filename> to "1.2.3".
-        </para>
-
-        <para>
-            By the time parsing is complete for a recipe, BitBake
-            has a list of tasks that the recipe defines and a set of
-            data consisting of keys and values as well as
-            dependency information about the tasks.
-        </para>
-
-        <para>
-            BitBake does not need all of this information.
-            It only needs a small subset of the information to make
-            decisions about the recipe.
-            Consequently, BitBake caches the values in which it is
-            interested and does not store the rest of the information.
-            Experience has shown it is faster to re-parse the metadata than to
-            try and write it out to the disk and then reload it.
-        </para>
-
-        <para>
-            Where possible, subsequent BitBake commands reuse this cache of
-            recipe information.
-            The validity of this cache is determined by first computing a
-            checksum of the base configuration data (see
-            <link linkend='var-bb-BB_HASHCONFIG_WHITELIST'><filename>BB_HASHCONFIG_WHITELIST</filename></link>)
-            and then checking if the checksum matches.
-            If that checksum matches what is in the cache and the recipe
-            and class files have not changed, Bitbake is able to use
-            the cache.
-            BitBake then reloads the cached information about the recipe
-            instead of reparsing it from scratch.
-        </para>
-
-        <para>
-            Recipe file collections exist to allow the user to
-            have multiple repositories of
-            <filename>.bb</filename> files that contain the same
-            exact package.
-            For example, one could easily use them to make one's
-            own local copy of an upstream repository, but with
-            custom modifications that one does not want upstream.
-            Here is an example:
-            <literallayout class='monospaced'>
-    BBFILES = "/stuff/openembedded/*/*.bb /stuff/openembedded.modified/*/*.bb"
-    BBFILE_COLLECTIONS = "upstream local"
-    BBFILE_PATTERN_upstream = "^/stuff/openembedded/"
-    BBFILE_PATTERN_local = "^/stuff/openembedded.modified/"
-    BBFILE_PRIORITY_upstream = "5"
-    BBFILE_PRIORITY_local = "10"
-            </literallayout>
-            <note>
-                The layers mechanism is now the preferred method of collecting
-                code.
-                While the collections code remains, its main use is to set layer
-                priorities and to deal with overlap (conflicts) between layers.
-            </note>
-        </para>
-    </section>
-
-    <section id='bb-bitbake-providers'>
-        <title>Providers</title>
-
-        <para>
-            Assuming BitBake has been instructed to execute a target
-            and that all the recipe files have been parsed, BitBake
-            starts to figure out how to build the target.
-            BitBake looks through the <filename>PROVIDES</filename> list
-            for each of the recipes.
-            A <filename>PROVIDES</filename> list is the list of names by which
-            the recipe can be known.
-            Each recipe's <filename>PROVIDES</filename> list is created
-            implicitly through the recipe's
-            <link linkend='var-bb-PN'><filename>PN</filename></link> variable
-            and explicitly through the recipe's
-            <link linkend='var-bb-PROVIDES'><filename>PROVIDES</filename></link>
-            variable, which is optional.
-        </para>
-
-        <para>
-            When a recipe uses <filename>PROVIDES</filename>, that recipe's
-            functionality can be found under an alternative name or names other
-            than the implicit <filename>PN</filename> name.
-            As an example, suppose a recipe named <filename>keyboard_1.0.bb</filename>
-            contained the following:
-            <literallayout class='monospaced'>
-     PROVIDES += "fullkeyboard"
-            </literallayout>
-            The <filename>PROVIDES</filename> list for this recipe becomes
-            "keyboard", which is implicit, and "fullkeyboard", which is explicit.
-            Consequently, the functionality found in
-            <filename>keyboard_1.0.bb</filename> can be found under two
-            different names.
-        </para>
-    </section>
-
-    <section id='bb-bitbake-preferences'>
-        <title>Preferences</title>
-
-        <para>
-            The <filename>PROVIDES</filename> list is only part of the solution
-            for figuring out a target's recipes.
-            Because targets might have multiple providers, BitBake needs
-            to prioritize providers by determining provider preferences.
-        </para>
-
-        <para>
-            A common example in which a target has multiple providers
-            is "virtual/kernel", which is on the
-            <filename>PROVIDES</filename> list for each kernel recipe.
-            Each machine often selects the best kernel provider by using a
-            line similar to the following in the machine configuration file:
-            <literallayout class='monospaced'>
-     PREFERRED_PROVIDER_virtual/kernel = "linux-yocto"
-            </literallayout>
-            The default
-            <link linkend='var-bb-PREFERRED_PROVIDER'><filename>PREFERRED_PROVIDER</filename></link>
-            is the provider with the same name as the target.
-            Bitbake iterates through each target it needs to build and
-            resolves them and their dependencies using this process.
-        </para>
-
-        <para>
-            Understanding how providers are chosen is made complicated by the fact
-            that multiple versions might exist for a given provider.
-            BitBake defaults to the highest version of a provider.
-            Version comparisons are made using the same method as Debian.
-            You can use the
-            <link linkend='var-bb-PREFERRED_VERSION'><filename>PREFERRED_VERSION</filename></link>
-            variable to specify a particular version.
-            You can influence the order by using the
-            <link linkend='var-bb-DEFAULT_PREFERENCE'><filename>DEFAULT_PREFERENCE</filename></link>
-            variable.
-        </para>
-
-        <para>
-            By default, files have a preference of "0".
-            Setting <filename>DEFAULT_PREFERENCE</filename> to "-1" makes the
-            recipe unlikely to be used unless it is explicitly referenced.
-            Setting <filename>DEFAULT_PREFERENCE</filename> to "1" makes it
-            likely the recipe is used.
-            <filename>PREFERRED_VERSION</filename> overrides any
-            <filename>DEFAULT_PREFERENCE</filename> setting.
-            <filename>DEFAULT_PREFERENCE</filename> is often used to mark newer
-            and more experimental recipe versions until they have undergone
-            sufficient testing to be considered stable.
-        </para>
-
-        <para>
-            When there are multiple “versions” of a given recipe,
-            BitBake defaults to selecting the most recent
-            version, unless otherwise specified.
-            If the recipe in question has a
-            <link linkend='var-bb-DEFAULT_PREFERENCE'><filename>DEFAULT_PREFERENCE</filename></link>
-            set lower than the other recipes (default is 0), then
-            it will not be selected.
-            This allows the person or persons maintaining
-            the repository of recipe files to specify
-            their preference for the default selected version.
-            Additionally, the user can specify their preferred version.
-        </para>
-
-        <para>
-            If the first recipe is named <filename>a_1.1.bb</filename>, then the
-            <link linkend='var-bb-PN'><filename>PN</filename></link> variable
-            will be set to “a”, and the
-            <link linkend='var-bb-PV'><filename>PV</filename></link>
-            variable will be set to 1.1.
-        </para>
-
-        <para>
-            Thus, if a recipe named <filename>a_1.2.bb</filename> exists, BitBake
-            will choose 1.2 by default.
-            However, if you define the following variable in a
-            <filename>.conf</filename> file that BitBake parses, you
-            can change that preference:
-            <literallayout class='monospaced'>
-     PREFERRED_VERSION_a = "1.1"
-            </literallayout>
-        </para>
-
-        <note>
-            <para>
-                It is common for a recipe to provide two versions -- a stable,
-                numbered (and preferred) version, and a version that is
-                automatically checked out from a source code repository that
-                is considered more "bleeding edge" but can be selected only
-                explicitly.
-            </para>
-
-            <para>
-                For example, in the OpenEmbedded codebase, there is a standard,
-                versioned recipe file for BusyBox,
-                <filename>busybox_1.22.1.bb</filename>,
-                but there is also a Git-based version,
-                <filename>busybox_git.bb</filename>, which explicitly contains the line
-                <literallayout class='monospaced'>
-    DEFAULT_PREFERENCE = "-1"
-                </literallayout>
-                to ensure that the numbered, stable version is always preferred
-                unless the developer selects otherwise.
-            </para>
-        </note>
-    </section>
-
-    <section id='bb-bitbake-dependencies'>
-        <title>Dependencies</title>
-
-        <para>
-            Each target BitBake builds consists of multiple tasks such as
-            <filename>fetch</filename>, <filename>unpack</filename>,
-            <filename>patch</filename>, <filename>configure</filename>,
-            and <filename>compile</filename>.
-            For best performance on multi-core systems, BitBake considers each
-            task as an independent
-            entity with its own set of dependencies.
-        </para>
-
-        <para>
-            Dependencies are defined through several variables.
-            You can find information about variables BitBake uses in
-            the <link linkend='ref-bb-variables-glos'>Variables Glossary</link>
-            near the end of this manual.
-            At a basic level, it is sufficient to know that BitBake uses the
-            <link linkend='var-bb-DEPENDS'><filename>DEPENDS</filename></link> and
-            <link linkend='var-bb-RDEPENDS'><filename>RDEPENDS</filename></link> variables when
-            calculating dependencies.
-        </para>
-
-        <para>
-            For more information on how BitBake handles dependencies, see the
-            "<link linkend='dependencies'>Dependencies</link>" section.
-        </para>
-    </section>
-
-    <section id='ref-bitbake-tasklist'>
-        <title>The Task List</title>
-
-        <para>
-            Based on the generated list of providers and the dependency information,
-            BitBake can now calculate exactly what tasks it needs to run and in what
-            order it needs to run them.
-            The
-            "<link linkend='executing-tasks'>Executing Tasks</link>" section has more
-            information on how BitBake chooses which task to execute next.
-        </para>
-
-        <para>
-            The build now starts with BitBake forking off threads up to the limit set in the
-            <link linkend='var-bb-BB_NUMBER_THREADS'><filename>BB_NUMBER_THREADS</filename></link>
-            variable.
-            BitBake continues to fork threads as long as there are tasks ready to run,
-            those tasks have all their dependencies met, and the thread threshold has not been
-            exceeded.
-        </para>
-
-        <para>
-            It is worth noting that you can greatly speed up the build time by properly setting
-            the <filename>BB_NUMBER_THREADS</filename> variable.
-        </para>
-
-        <para>
-            As each task completes, a timestamp is written to the directory specified by the
-            <link linkend='var-bb-STAMP'><filename>STAMP</filename></link> variable.
-            On subsequent runs, BitBake looks in the build directory within
-            <filename>tmp/stamps</filename> and does not rerun
-            tasks that are already completed unless a timestamp is found to be invalid.
-            Currently, invalid timestamps are only considered on a per
-            recipe file basis.
-            So, for example, if the configure stamp has a timestamp greater than the
-            compile timestamp for a given target, then the compile task would rerun.
-            Running the compile task again, however, has no effect on other providers
-            that depend on that target.
-        </para>
-
-        <para>
-            The exact format of the stamps is partly configurable.
-            In modern versions of BitBake, a hash is appended to the
-            stamp so that if the configuration changes, the stamp becomes
-            invalid and the task is automatically rerun.
-            This hash, or signature used, is governed by the signature policy
-            that is configured (see the
-            "<link linkend='checksums'>Checksums (Signatures)</link>"
-            section for information).
-            It is also possible to append extra metadata to the stamp using
-            the <filename>[stamp-extra-info]</filename> task flag.
-            For example, OpenEmbedded uses this flag to make some tasks machine-specific.
-        </para>
-
-        <note>
-            Some tasks are marked as "nostamp" tasks.
-            No timestamp file is created when these tasks are run.
-            Consequently, "nostamp" tasks are always rerun.
-        </note>
-
-        <para>
-            For more information on tasks, see the
-            "<link linkend='tasks'>Tasks</link>" section.
-        </para>
-    </section>
-
-    <section id='executing-tasks'>
-        <title>Executing Tasks</title>
-
-        <para>
-            Tasks can be either a shell task or a Python task.
-            For shell tasks, BitBake writes a shell script to
-            <filename>${</filename><link linkend='var-bb-T'><filename>T</filename></link><filename>}/run.do_taskname.pid</filename>
-            and then executes the script.
-            The generated shell script contains all the exported variables,
-            and the shell functions with all variables expanded.
-            Output from the shell script goes to the file
-            <filename>${T}/log.do_taskname.pid</filename>.
-            Looking at the expanded shell functions in the run file and
-            the output in the log files is a useful debugging technique.
-        </para>
-
-        <para>
-            For Python tasks, BitBake executes the task internally and logs
-            information to the controlling terminal.
-            Future versions of BitBake will write the functions to files
-            similar to the way shell tasks are handled.
-            Logging will be handled in a way similar to shell tasks as well.
-        </para>
-
-        <para>
-            The order in which BitBake runs the tasks is controlled by its
-            task scheduler.
-            It is possible to configure the scheduler and define custom
-            implementations for specific use cases.
-            For more information, see these variables that control the
-            behavior:
-            <itemizedlist>
-                <listitem><para>
-                    <link linkend='var-bb-BB_SCHEDULER'><filename>BB_SCHEDULER</filename></link>
-                    </para></listitem>
-                <listitem><para>
-                    <link linkend='var-bb-BB_SCHEDULERS'><filename>BB_SCHEDULERS</filename></link>
-                    </para></listitem>
-            </itemizedlist>
-            It is possible to have functions run before and after a task's main
-            function.
-            This is done using the <filename>[prefuncs]</filename>
-            and <filename>[postfuncs]</filename> flags of the task
-            that lists the functions to run.
-        </para>
-    </section>
-
-    <section id='checksums'>
-        <title>Checksums (Signatures)</title>
-
-        <para>
-            A checksum is a unique signature of a task's inputs.
-            The signature of a task can be used to determine if a task
-            needs to be run.
-            Because it is a change in a task's inputs that triggers running
-            the task, BitBake needs to detect all the inputs to a given task.
-            For shell tasks, this turns out to be fairly easy because
-            BitBake generates a "run" shell script for each task and
-            it is possible to create a checksum that gives you a good idea of when
-            the task's data changes.
-        </para>
-
-        <para>
-            To complicate the problem, some things should not be included in
-            the checksum.
-            First, there is the actual specific build path of a given task -
-            the working directory.
-            It does not matter if the working directory changes because it should not
-            affect the output for target packages.
-            The simplistic approach for excluding the working directory is to set
-            it to some fixed value and create the checksum for the "run" script.
-            BitBake goes one step better and uses the
-            <link linkend='var-bb-BB_HASHBASE_WHITELIST'><filename>BB_HASHBASE_WHITELIST</filename></link>
-            variable to define a list of variables that should never be included
-            when generating the signatures.
-        </para>
-
-        <para>
-            Another problem results from the "run" scripts containing functions that
-            might or might not get called.
-            The incremental build solution contains code that figures out dependencies
-            between shell functions.
-            This code is used to prune the "run" scripts down to the minimum set,
-            thereby alleviating this problem and making the "run" scripts much more
-            readable as a bonus.
-        </para>
-
-        <para>
-            So far we have solutions for shell scripts.
-            What about Python tasks?
-            The same approach applies even though these tasks are more difficult.
-            The process needs to figure out what variables a Python function accesses
-            and what functions it calls.
-            Again, the incremental build solution contains code that first figures out
-            the variable and function dependencies, and then creates a checksum for the data
-            used as the input to the task.
-        </para>
-
-        <para>
-            Like the working directory case, situations exist where dependencies
-            should be ignored.
-            For these cases, you can instruct the build process to ignore a dependency
-            by using a line like the following:
-            <literallayout class='monospaced'>
-     PACKAGE_ARCHS[vardepsexclude] = "MACHINE"
-            </literallayout>
-            This example ensures that the <filename>PACKAGE_ARCHS</filename> variable does not
-            depend on the value of <filename>MACHINE</filename>, even if it does reference it.
-        </para>
-
-        <para>
-            Equally, there are cases where we need to add dependencies BitBake
-            is not able to find.
-            You can accomplish this by using a line like the following:
-            <literallayout class='monospaced'>
-      PACKAGE_ARCHS[vardeps] = "MACHINE"
-            </literallayout>
-            This example explicitly adds the <filename>MACHINE</filename> variable as a
-            dependency for <filename>PACKAGE_ARCHS</filename>.
-        </para>
-
-        <para>
-            Consider a case with in-line Python, for example, where BitBake is not
-            able to figure out dependencies.
-            When running in debug mode (i.e. using <filename>-DDD</filename>), BitBake
-            produces output when it discovers something for which it cannot figure out
-            dependencies.
-        </para>
-
-        <para>
-            Thus far, this section has limited discussion to the direct inputs into a task.
-            Information based on direct inputs is referred to as the "basehash" in the
-            code.
-            However, there is still the question of a task's indirect inputs - the
-            things that were already built and present in the build directory.
-            The checksum (or signature) for a particular task needs to add the hashes
-            of all the tasks on which the particular task depends.
-            Choosing which dependencies to add is a policy decision.
-            However, the effect is to generate a master checksum that combines the basehash
-            and the hashes of the task's dependencies.
-        </para>
-
-        <para>
-            At the code level, there are a variety of ways both the basehash and the
-            dependent task hashes can be influenced.
-            Within the BitBake configuration file, we can give BitBake some extra information
-            to help it construct the basehash.
-            The following statement effectively results in a list of global variable
-            dependency excludes - variables never included in any checksum.
-            This example uses variables from OpenEmbedded to help illustrate
-            the concept:
-            <literallayout class='monospaced'>
-     BB_HASHBASE_WHITELIST ?= "TMPDIR FILE PATH PWD BB_TASKHASH BBPATH DL_DIR \
-         SSTATE_DIR THISDIR FILESEXTRAPATHS FILE_DIRNAME HOME LOGNAME SHELL TERM \
-         USER FILESPATH STAGING_DIR_HOST STAGING_DIR_TARGET COREBASE PRSERV_HOST \
-         PRSERV_DUMPDIR PRSERV_DUMPFILE PRSERV_LOCKDOWN PARALLEL_MAKE \
-         CCACHE_DIR EXTERNAL_TOOLCHAIN CCACHE CCACHE_DISABLE LICENSE_PATH SDKPKGSUFFIX"
-            </literallayout>
-            The previous example excludes the work directory, which is part of
-            <filename>TMPDIR</filename>.
-        </para>
-
-        <para>
-            The rules for deciding which hashes of dependent tasks to include through
-            dependency chains are more complex and are generally accomplished with a
-            Python function.
-            The code in <filename>meta/lib/oe/sstatesig.py</filename> shows two examples
-            of this and also illustrates how you can insert your own policy into the system
-            if so desired.
-            This file defines the two basic signature generators OpenEmbedded-Core
-            uses:  "OEBasic" and "OEBasicHash".
-            By default, there is a dummy "noop" signature handler enabled in BitBake.
-            This means that behavior is unchanged from previous versions.
-            <filename>OE-Core</filename> uses the "OEBasicHash" signature handler by default
-            through this setting in the <filename>bitbake.conf</filename> file:
-            <literallayout class='monospaced'>
-     BB_SIGNATURE_HANDLER ?= "OEBasicHash"
-            </literallayout>
-            The "OEBasicHash" <filename>BB_SIGNATURE_HANDLER</filename> is the same as the
-            "OEBasic" version but adds the task hash to the stamp files.
-            This results in any metadata change that changes the task hash, automatically
-            causing the task to be run again.
-            This removes the need to bump
-            <link linkend='var-bb-PR'><filename>PR</filename></link>
-            values, and changes to metadata automatically ripple across the build.
-        </para>
-
-        <para>
-            It is also worth noting that the end result of these signature generators is to
-            make some dependency and hash information available to the build.
-            This information includes:
-            <itemizedlist>
-                <listitem><para><filename>BB_BASEHASH_task-</filename><replaceable>taskname</replaceable>:
-                    The base hashes for each task in the recipe.
-                    </para></listitem>
-                <listitem><para><filename>BB_BASEHASH_</filename><replaceable>filename</replaceable><filename>:</filename><replaceable>taskname</replaceable>:
-                    The base hashes for each dependent task.
-                    </para></listitem>
-                <listitem><para><filename>BBHASHDEPS_</filename><replaceable>filename</replaceable><filename>:</filename><replaceable>taskname</replaceable>:
-                    The task dependencies for each task.
-                    </para></listitem>
-                <listitem><para><filename>BB_TASKHASH</filename>:
-                    The hash of the currently running task.
-                    </para></listitem>
-            </itemizedlist>
-        </para>
-
-        <para>
-            It is worth noting that BitBake's "-S" option lets you
-            debug Bitbake's processing of signatures.
-            The options passed to -S allow different debugging modes
-            to be used, either using BitBake's own debug functions
-            or possibly those defined in the metadata/signature handler
-            itself.
-            The simplest parameter to pass is "none", which causes a
-            set of signature information to be written out into
-            <filename>STAMPS_DIR</filename>
-            corresponding to the targets specified.
-            The other currently available parameter is "printdiff",
-            which causes BitBake to try to establish the closest
-            signature match it can (e.g. in the sstate cache) and then
-            run <filename>bitbake-diffsigs</filename> over the matches
-            to determine the stamps and delta where these two
-            stamp trees diverge.
-            <note>
-                It is likely that future versions of BitBake will
-                provide other signature handlers triggered through
-                additional "-S" parameters.
-            </note>
-        </para>
-
-        <para>
-            You can find more information on checksum metadata in the
-            "<link linkend='task-checksums-and-setscene'>Task Checksums and Setscene</link>"
-            section.
-        </para>
-    </section>
-
-    <section id='setscene'>
-        <title>Setscene</title>
-
-        <para>
-            The setscene process enables BitBake to handle "pre-built" artifacts.
-            The ability to handle and reuse these artifacts allows BitBake
-            the luxury of not having to build something from scratch every time.
-            Instead, BitBake can use, when possible, existing build artifacts.
-        </para>
-
-        <para>
-            BitBake needs to have reliable data indicating whether or not an
-            artifact is compatible.
-            Signatures, described in the previous section, provide an ideal
-            way of representing whether an artifact is compatible.
-            If a signature is the same, an object can be reused.
-        </para>
-
-        <para>
-            If an object can be reused, the problem then becomes how to
-            replace a given task or set of tasks with the pre-built artifact.
-            BitBake solves the problem with the "setscene" process.
-        </para>
-
-        <para>
-            When BitBake is asked to build a given target, before building anything,
-            it first asks whether cached information is available for any of the
-            targets it's building, or any of the intermediate targets.
-            If cached information is available, BitBake uses this information instead of
-            running the main tasks.
-        </para>
-
-        <para>
-            BitBake first calls the function defined by the
-            <link linkend='var-bb-BB_HASHCHECK_FUNCTION'><filename>BB_HASHCHECK_FUNCTION</filename></link>
-            variable with a list of tasks and corresponding
-            hashes it wants to build.
-            This function is designed to be fast and returns a list
-            of the tasks for which it believes in can obtain artifacts.
-        </para>
-
-        <para>
-            Next, for each of the tasks that were returned as possibilities,
-            BitBake executes a setscene version of the task that the possible
-            artifact covers.
-            Setscene versions of a task have the string "_setscene" appended to the
-            task name.
-            So, for example, the task with the name <filename>xxx</filename> has
-            a setscene task named <filename>xxx_setscene</filename>.
-            The setscene version of the task executes and provides the necessary
-            artifacts returning either success or failure.
-        </para>
-
-        <para>
-            As previously mentioned, an artifact can cover more than one task.
-            For example, it is pointless to obtain a compiler if you
-            already have the compiled binary.
-            To handle this, BitBake calls the
-            <link linkend='var-bb-BB_SETSCENE_DEPVALID'><filename>BB_SETSCENE_DEPVALID</filename></link>
-            function for each successful setscene task to know whether or not it needs
-            to obtain the dependencies of that task.
-        </para>
-
-        <para>
-            Finally, after all the setscene tasks have executed, BitBake calls the
-            function listed in
-            <link linkend='var-bb-BB_SETSCENE_VERIFY_FUNCTION2'><filename>BB_SETSCENE_VERIFY_FUNCTION2</filename></link>
-            with the list of tasks BitBake thinks has been "covered".
-            The metadata can then ensure that this list is correct and can
-            inform BitBake that it wants specific tasks to be run regardless
-            of the setscene result.
-        </para>
-
-        <para>
-            You can find more information on setscene metadata in the
-            "<link linkend='task-checksums-and-setscene'>Task Checksums and Setscene</link>"
-            section.
-        </para>
-    </section>
-</chapter>

bitbake/doc/bitbake-user-manual/bitbake-user-manual-fetching.rst

@@ -0,0 +1,652 @@
+.. SPDX-License-Identifier: CC-BY-2.5
+
+=====================
+File Download Support
+=====================
+
+|
+
+BitBake's fetch module is a standalone piece of library code that deals
+with the intricacies of downloading source code and files from remote
+systems. Fetching source code is one of the cornerstones of building
+software. As such, this module forms an important part of BitBake.
+
+The current fetch module is called "fetch2" and refers to the fact that
+it is the second major version of the API. The original version is
+obsolete and has been removed from the codebase. Thus, in all cases,
+"fetch" refers to "fetch2" in this manual.
+
+The Download (Fetch)
+====================
+
+BitBake takes several steps when fetching source code or files. The
+fetcher codebase deals with two distinct processes in order: obtaining
+the files from somewhere (cached or otherwise) and then unpacking those
+files into a specific location and perhaps in a specific way. Getting
+and unpacking the files is often optionally followed by patching.
+Patching, however, is not covered by this module.
+
+The code to execute the first part of this process, a fetch, looks
+something like the following: ::
+
+   src_uri = (d.getVar('SRC_URI') or "").split()
+   fetcher = bb.fetch2.Fetch(src_uri, d)
+   fetcher.download()
+
+This code sets up an instance of the fetch class. The instance uses a
+space-separated list of URLs from the :term:`SRC_URI`
+variable and then calls the ``download`` method to download the files.
+
+The instantiation of the fetch class is usually followed by: ::
+
+   rootdir = l.getVar('WORKDIR')
+   fetcher.unpack(rootdir)
+
+This code unpacks the downloaded files to the specified by ``WORKDIR``.
+
+.. note::
+
+   For convenience, the naming in these examples matches the variables
+   used by OpenEmbedded. If you want to see the above code in action,
+   examine the OpenEmbedded class file ``base.bbclass``
+   .
+
+The ``SRC_URI`` and ``WORKDIR`` variables are not hardcoded into the
+fetcher, since those fetcher methods can be (and are) called with
+different variable names. In OpenEmbedded for example, the shared state
+(sstate) code uses the fetch module to fetch the sstate files.
+
+When the ``download()`` method is called, BitBake tries to resolve the
+URLs by looking for source files in a specific search order:
+
+-  *Pre-mirror Sites:* BitBake first uses pre-mirrors to try and find
+   source files. These locations are defined using the
+   :term:`PREMIRRORS` variable.
+
+-  *Source URI:* If pre-mirrors fail, BitBake uses the original URL (e.g
+   from ``SRC_URI``).
+
+-  *Mirror Sites:* If fetch failures occur, BitBake next uses mirror
+   locations as defined by the :term:`MIRRORS` variable.
+
+For each URL passed to the fetcher, the fetcher calls the submodule that
+handles that particular URL type. This behavior can be the source of
+some confusion when you are providing URLs for the ``SRC_URI`` variable.
+Consider the following two URLs: ::
+
+   http://git.yoctoproject.org/git/poky;protocol=git
+   git://git.yoctoproject.org/git/poky;protocol=http
+
+In the former case, the URL is passed to the ``wget`` fetcher, which does not
+understand "git". Therefore, the latter case is the correct form since the Git
+fetcher does know how to use HTTP as a transport.
+
+Here are some examples that show commonly used mirror definitions: ::
+
+   PREMIRRORS ?= "\
+      bzr://.*/.\*  http://somemirror.org/sources/ \\n \
+      cvs://.*/.\*  http://somemirror.org/sources/ \\n \
+      git://.*/.\*  http://somemirror.org/sources/ \\n \
+      hg://.*/.\*   http://somemirror.org/sources/ \\n \
+      osc://.*/.\*  http://somemirror.org/sources/ \\n \
+      p4://.*/.\*   http://somemirror.org/sources/ \\n \
+     svn://.*/.\*   http://somemirror.org/sources/ \\n"
+
+   MIRRORS =+ "\
+      ftp://.*/.\*   http://somemirror.org/sources/ \\n \
+      http://.*/.\*  http://somemirror.org/sources/ \\n \
+      https://.*/.\* http://somemirror.org/sources/ \\n"
+
+It is useful to note that BitBake
+supports cross-URLs. It is possible to mirror a Git repository on an
+HTTP server as a tarball. This is what the ``git://`` mapping in the
+previous example does.
+
+Since network accesses are slow, BitBake maintains a cache of files
+downloaded from the network. Any source files that are not local (i.e.
+downloaded from the Internet) are placed into the download directory,
+which is specified by the :term:`DL_DIR` variable.
+
+File integrity is of key importance for reproducing builds. For
+non-local archive downloads, the fetcher code can verify SHA-256 and MD5
+checksums to ensure the archives have been downloaded correctly. You can
+specify these checksums by using the ``SRC_URI`` variable with the
+appropriate varflags as follows: ::
+
+   SRC_URI[md5sum] = "value"
+   SRC_URI[sha256sum] = "value"
+
+You can also specify the checksums as
+parameters on the ``SRC_URI`` as shown below: ::
+
+  SRC_URI = "http://example.com/foobar.tar.bz2;md5sum=4a8e0f237e961fd7785d19d07fdb994d"
+
+If multiple URIs exist, you can specify the checksums either directly as
+in the previous example, or you can name the URLs. The following syntax
+shows how you name the URIs: ::
+
+   SRC_URI = "http://example.com/foobar.tar.bz2;name=foo"
+   SRC_URI[foo.md5sum] = 4a8e0f237e961fd7785d19d07fdb994d
+
+After a file has been downloaded and
+has had its checksum checked, a ".done" stamp is placed in ``DL_DIR``.
+BitBake uses this stamp during subsequent builds to avoid downloading or
+comparing a checksum for the file again.
+
+.. note::
+
+   It is assumed that local storage is safe from data corruption. If
+   this were not the case, there would be bigger issues to worry about.
+
+If :term:`BB_STRICT_CHECKSUM` is set, any
+download without a checksum triggers an error message. The
+:term:`BB_NO_NETWORK` variable can be used to
+make any attempted network access a fatal error, which is useful for
+checking that mirrors are complete as well as other things.
+
+.. _bb-the-unpack:
+
+The Unpack
+==========
+
+The unpack process usually immediately follows the download. For all
+URLs except Git URLs, BitBake uses the common ``unpack`` method.
+
+A number of parameters exist that you can specify within the URL to
+govern the behavior of the unpack stage:
+
+-  *unpack:* Controls whether the URL components are unpacked. If set to
+   "1", which is the default, the components are unpacked. If set to
+   "0", the unpack stage leaves the file alone. This parameter is useful
+   when you want an archive to be copied in and not be unpacked.
+
+-  *dos:* Applies to ``.zip`` and ``.jar`` files and specifies whether
+   to use DOS line ending conversion on text files.
+
+-  *basepath:* Instructs the unpack stage to strip the specified
+   directories from the source path when unpacking.
+
+-  *subdir:* Unpacks the specific URL to the specified subdirectory
+   within the root directory.
+
+The unpack call automatically decompresses and extracts files with ".Z",
+".z", ".gz", ".xz", ".zip", ".jar", ".ipk", ".rpm". ".srpm", ".deb" and
+".bz2" extensions as well as various combinations of tarball extensions.
+
+As mentioned, the Git fetcher has its own unpack method that is
+optimized to work with Git trees. Basically, this method works by
+cloning the tree into the final directory. The process is completed
+using references so that there is only one central copy of the Git
+metadata needed.
+
+.. _bb-fetchers:
+
+Fetchers
+========
+
+As mentioned earlier, the URL prefix determines which fetcher submodule
+BitBake uses. Each submodule can support different URL parameters, which
+are described in the following sections.
+
+.. _local-file-fetcher:
+
+Local file fetcher (``file://``)
+--------------------------------
+
+This submodule handles URLs that begin with ``file://``. The filename
+you specify within the URL can be either an absolute or relative path to
+a file. If the filename is relative, the contents of the
+:term:`FILESPATH` variable is used in the same way
+``PATH`` is used to find executables. If the file cannot be found, it is
+assumed that it is available in :term:`DL_DIR` by the
+time the ``download()`` method is called.
+
+If you specify a directory, the entire directory is unpacked.
+
+Here are a couple of example URLs, the first relative and the second
+absolute: ::
+
+   SRC_URI = "file://relativefile.patch"
+   SRC_URI = "file:///Users/ich/very_important_software"
+
+.. _http-ftp-fetcher:
+
+HTTP/FTP wget fetcher (``http://``, ``ftp://``, ``https://``)
+-------------------------------------------------------------
+
+This fetcher obtains files from web and FTP servers. Internally, the
+fetcher uses the wget utility.
+
+The executable and parameters used are specified by the
+``FETCHCMD_wget`` variable, which defaults to sensible values. The
+fetcher supports a parameter "downloadfilename" that allows the name of
+the downloaded file to be specified. Specifying the name of the
+downloaded file is useful for avoiding collisions in
+:term:`DL_DIR` when dealing with multiple files that
+have the same name.
+
+Some example URLs are as follows: ::
+
+   SRC_URI = "http://oe.handhelds.org/not_there.aac"
+   SRC_URI = "ftp://oe.handhelds.org/not_there_as_well.aac"
+   SRC_URI = "ftp://you@oe.handhelds.org/home/you/secret.plan"
+
+.. note::
+
+   Because URL parameters are delimited by semi-colons, this can
+   introduce ambiguity when parsing URLs that also contain semi-colons,
+   for example:
+   ::
+
+           SRC_URI = "http://abc123.org/git/?p=gcc/gcc.git;a=snapshot;h=a5dd47"
+
+
+   Such URLs should should be modified by replacing semi-colons with '&'
+   characters:
+   ::
+
+           SRC_URI = "http://abc123.org/git/?p=gcc/gcc.git&a=snapshot&h=a5dd47"
+
+
+   In most cases this should work. Treating semi-colons and '&' in
+   queries identically is recommended by the World Wide Web Consortium
+   (W3C). Note that due to the nature of the URL, you may have to
+   specify the name of the downloaded file as well:
+   ::
+
+           SRC_URI = "http://abc123.org/git/?p=gcc/gcc.git&a=snapshot&h=a5dd47;downloadfilename=myfile.bz2"
+
+
+.. _cvs-fetcher:
+
+CVS fetcher (``(cvs://``)
+-------------------------
+
+This submodule handles checking out files from the CVS version control
+system. You can configure it using a number of different variables:
+
+-  :term:`FETCHCMD_cvs <FETCHCMD>`: The name of the executable to use when running
+   the ``cvs`` command. This name is usually "cvs".
+
+-  :term:`SRCDATE`: The date to use when fetching the CVS source code. A
+   special value of "now" causes the checkout to be updated on every
+   build.
+
+-  :term:`CVSDIR`: Specifies where a temporary
+   checkout is saved. The location is often ``DL_DIR/cvs``.
+
+-  CVS_PROXY_HOST: The name to use as a "proxy=" parameter to the
+   ``cvs`` command.
+
+-  CVS_PROXY_PORT: The port number to use as a "proxyport="
+   parameter to the ``cvs`` command.
+
+As well as the standard username and password URL syntax, you can also
+configure the fetcher with various URL parameters:
+
+The supported parameters are as follows:
+
+-  *"method":* The protocol over which to communicate with the CVS
+   server. By default, this protocol is "pserver". If "method" is set to
+   "ext", BitBake examines the "rsh" parameter and sets ``CVS_RSH``. You
+   can use "dir" for local directories.
+
+-  *"module":* Specifies the module to check out. You must supply this
+   parameter.
+
+-  *"tag":* Describes which CVS TAG should be used for the checkout. By
+   default, the TAG is empty.
+
+-  *"date":* Specifies a date. If no "date" is specified, the
+   :term:`SRCDATE` of the configuration is used to
+   checkout a specific date. The special value of "now" causes the
+   checkout to be updated on every build.
+
+-  *"localdir":* Used to rename the module. Effectively, you are
+   renaming the output directory to which the module is unpacked. You
+   are forcing the module into a special directory relative to
+   :term:`CVSDIR`.
+
+-  *"rsh":* Used in conjunction with the "method" parameter.
+
+-  *"scmdata":* Causes the CVS metadata to be maintained in the tarball
+   the fetcher creates when set to "keep". The tarball is expanded into
+   the work directory. By default, the CVS metadata is removed.
+
+-  *"fullpath":* Controls whether the resulting checkout is at the
+   module level, which is the default, or is at deeper paths.
+
+-  *"norecurse":* Causes the fetcher to only checkout the specified
+   directory with no recurse into any subdirectories.
+
+-  *"port":* The port to which the CVS server connects.
+
+Some example URLs are as follows: ::
+
+   SRC_URI = "cvs://CVSROOT;module=mymodule;tag=some-version;method=ext"
+   SRC_URI = "cvs://CVSROOT;module=mymodule;date=20060126;localdir=usethat"
+
+.. _svn-fetcher:
+
+Subversion (SVN) Fetcher (``svn://``)
+-------------------------------------
+
+This fetcher submodule fetches code from the Subversion source control
+system. The executable used is specified by ``FETCHCMD_svn``, which
+defaults to "svn". The fetcher's temporary working directory is set by
+:term:`SVNDIR`, which is usually ``DL_DIR/svn``.
+
+The supported parameters are as follows:
+
+-  *"module":* The name of the svn module to checkout. You must provide
+   this parameter. You can think of this parameter as the top-level
+   directory of the repository data you want.
+
+-  *"path_spec":* A specific directory in which to checkout the
+   specified svn module.
+
+-  *"protocol":* The protocol to use, which defaults to "svn". If
+   "protocol" is set to "svn+ssh", the "ssh" parameter is also used.
+
+-  *"rev":* The revision of the source code to checkout.
+
+-  *"scmdata":* Causes the ".svn" directories to be available during
+   compile-time when set to "keep". By default, these directories are
+   removed.
+
+-  *"ssh":* An optional parameter used when "protocol" is set to
+   "svn+ssh". You can use this parameter to specify the ssh program used
+   by svn.
+
+-  *"transportuser":* When required, sets the username for the
+   transport. By default, this parameter is empty. The transport
+   username is different than the username used in the main URL, which
+   is passed to the subversion command.
+
+Following are three examples using svn: ::
+
+   SRC_URI = "svn://myrepos/proj1;module=vip;protocol=http;rev=667"
+   SRC_URI = "svn://myrepos/proj1;module=opie;protocol=svn+ssh"
+   SRC_URI = "svn://myrepos/proj1;module=trunk;protocol=http;path_spec=${MY_DIR}/proj1"
+
+.. _git-fetcher:
+
+Git Fetcher (``git://``)
+------------------------
+
+This fetcher submodule fetches code from the Git source control system.
+The fetcher works by creating a bare clone of the remote into
+:term:`GITDIR`, which is usually ``DL_DIR/git2``. This
+bare clone is then cloned into the work directory during the unpack
+stage when a specific tree is checked out. This is done using alternates
+and by reference to minimize the amount of duplicate data on the disk
+and make the unpack process fast. The executable used can be set with
+``FETCHCMD_git``.
+
+This fetcher supports the following parameters:
+
+-  *"protocol":* The protocol used to fetch the files. The default is
+   "git" when a hostname is set. If a hostname is not set, the Git
+   protocol is "file". You can also use "http", "https", "ssh" and
+   "rsync".
+
+-  *"nocheckout":* Tells the fetcher to not checkout source code when
+   unpacking when set to "1". Set this option for the URL where there is
+   a custom routine to checkout code. The default is "0".
+
+-  *"rebaseable":* Indicates that the upstream Git repository can be
+   rebased. You should set this parameter to "1" if revisions can become
+   detached from branches. In this case, the source mirror tarball is
+   done per revision, which has a loss of efficiency. Rebasing the
+   upstream Git repository could cause the current revision to disappear
+   from the upstream repository. This option reminds the fetcher to
+   preserve the local cache carefully for future use. The default value
+   for this parameter is "0".
+
+-  *"nobranch":* Tells the fetcher to not check the SHA validation for
+   the branch when set to "1". The default is "0". Set this option for
+   the recipe that refers to the commit that is valid for a tag instead
+   of the branch.
+
+-  *"bareclone":* Tells the fetcher to clone a bare clone into the
+   destination directory without checking out a working tree. Only the
+   raw Git metadata is provided. This parameter implies the "nocheckout"
+   parameter as well.
+
+-  *"branch":* The branch(es) of the Git tree to clone. If unset, this
+   is assumed to be "master". The number of branch parameters much match
+   the number of name parameters.
+
+-  *"rev":* The revision to use for the checkout. The default is
+   "master".
+
+-  *"tag":* Specifies a tag to use for the checkout. To correctly
+   resolve tags, BitBake must access the network. For that reason, tags
+   are often not used. As far as Git is concerned, the "tag" parameter
+   behaves effectively the same as the "rev" parameter.
+
+-  *"subpath":* Limits the checkout to a specific subpath of the tree.
+   By default, the whole tree is checked out.
+
+-  *"destsuffix":* The name of the path in which to place the checkout.
+   By default, the path is ``git/``.
+
+-  *"usehead":* Enables local ``git://`` URLs to use the current branch
+   HEAD as the revision for use with ``AUTOREV``. The "usehead"
+   parameter implies no branch and only works when the transfer protocol
+   is ``file://``.
+
+Here are some example URLs: ::
+
+   SRC_URI = "git://git.oe.handhelds.org/git/vip.git;tag=version-1"
+   SRC_URI = "git://git.oe.handhelds.org/git/vip.git;protocol=http"
+
+.. _gitsm-fetcher:
+
+Git Submodule Fetcher (``gitsm://``)
+------------------------------------
+
+This fetcher submodule inherits from the :ref:`Git
+fetcher<bitbake-user-manual/bitbake-user-manual-fetching:git fetcher
+(\`\`git://\`\`)>` and extends that fetcher's behavior by fetching a
+repository's submodules. :term:`SRC_URI` is passed to the Git fetcher as
+described in the :ref:`bitbake-user-manual/bitbake-user-manual-fetching:git
+fetcher (\`\`git://\`\`)` section.
+
+.. note::
+
+   You must clean a recipe when switching between '``git://``' and
+   '``gitsm://``' URLs.
+
+   The Git Submodules fetcher is not a complete fetcher implementation.
+   The fetcher has known issues where it does not use the normal source
+   mirroring infrastructure properly. Further, the submodule sources it
+   fetches are not visible to the licensing and source archiving
+   infrastructures.
+
+.. _clearcase-fetcher:
+
+ClearCase Fetcher (``ccrc://``)
+-------------------------------
+
+This fetcher submodule fetches code from a
+`ClearCase <http://en.wikipedia.org/wiki/Rational_ClearCase>`__
+repository.
+
+To use this fetcher, make sure your recipe has proper
+:term:`SRC_URI`, :term:`SRCREV`, and
+:term:`PV` settings. Here is an example: ::
+
+   SRC_URI = "ccrc://cc.example.org/ccrc;vob=/example_vob;module=/example_module"
+   SRCREV = "EXAMPLE_CLEARCASE_TAG"
+   PV = "${@d.getVar("SRCREV", False).replace("/", "+")}"
+
+The fetcher uses the ``rcleartool`` or
+``cleartool`` remote client, depending on which one is available.
+
+Following are options for the ``SRC_URI`` statement:
+
+-  *vob*: The name, which must include the prepending "/" character,
+   of the ClearCase VOB. This option is required.
+
+-  *module*: The module, which must include the prepending "/"
+   character, in the selected VOB.
+
+   .. note::
+
+      The module and vob options are combined to create the load rule in the
+      view config spec. As an example, consider the vob and module values from
+      the SRC_URI statement at the start of this section. Combining those values
+      results in the following: ::
+
+         load /example_vob/example_module
+
+-  *proto*: The protocol, which can be either ``http`` or ``https``.
+
+By default, the fetcher creates a configuration specification. If you
+want this specification written to an area other than the default, use
+the ``CCASE_CUSTOM_CONFIG_SPEC`` variable in your recipe to define where
+the specification is written.
+
+.. note::
+
+   the SRCREV loses its functionality if you specify this variable. However,
+   SRCREV is still used to label the archive after a fetch even though it does
+   not define what is fetched.
+
+Here are a couple of other behaviors worth mentioning:
+
+-  When using ``cleartool``, the login of ``cleartool`` is handled by
+   the system. The login require no special steps.
+
+-  In order to use ``rcleartool`` with authenticated users, an
+   "rcleartool login" is necessary before using the fetcher.
+
+.. _perforce-fetcher:
+
+Perforce Fetcher (``p4://``)
+----------------------------
+
+This fetcher submodule fetches code from the
+`Perforce <https://www.perforce.com/>`__ source control system. The
+executable used is specified by ``FETCHCMD_p4``, which defaults to "p4".
+The fetcher's temporary working directory is set by
+:term:`P4DIR`, which defaults to "DL_DIR/p4".
+The fetcher does not make use of a perforce client, instead it
+relies on ``p4 files`` to retrieve a list of
+files and ``p4 print`` to transfer the content
+of those files locally.
+
+To use this fetcher, make sure your recipe has proper
+:term:`SRC_URI`, :term:`SRCREV`, and
+:term:`PV` values. The p4 executable is able to use the
+config file defined by your system's ``P4CONFIG`` environment variable
+in order to define the Perforce server URL and port, username, and
+password if you do not wish to keep those values in a recipe itself. If
+you choose not to use ``P4CONFIG``, or to explicitly set variables that
+``P4CONFIG`` can contain, you can specify the ``P4PORT`` value, which is
+the server's URL and port number, and you can specify a username and
+password directly in your recipe within ``SRC_URI``.
+
+Here is an example that relies on ``P4CONFIG`` to specify the server URL
+and port, username, and password, and fetches the Head Revision: ::
+
+   SRC_URI = "p4://example-depot/main/source/..."
+   SRCREV = "${AUTOREV}"
+   PV = "p4-${SRCPV}"
+   S = "${WORKDIR}/p4"
+
+Here is an example that specifies the server URL and port, username, and
+password, and fetches a Revision based on a Label: ::
+
+   P4PORT = "tcp:p4server.example.net:1666"
+   SRC_URI = "p4://user:passwd@example-depot/main/source/..."
+   SRCREV = "release-1.0"
+   PV = "p4-${SRCPV}"
+   S = "${WORKDIR}/p4"
+
+.. note::
+
+   You should always set S to "${WORKDIR}/p4" in your recipe.
+
+By default, the fetcher strips the depot location from the local file paths. In
+the above example, the content of ``example-depot/main/source/`` will be placed
+in ``${WORKDIR}/p4``.  For situations where preserving parts of the remote depot
+paths locally is desirable, the fetcher supports two parameters:
+
+- *"module":*
+    The top-level depot location or directory to fetch. The value of this
+    parameter can also point to a single file within the depot, in which case
+    the local file path will include the module path.
+- *"remotepath":*
+    When used with the value "``keep``", the fetcher will mirror the full depot
+    paths locally for the specified location, even in combination with the
+    ``module`` parameter.
+
+Here is an example use of the the ``module`` parameter: ::
+
+   SRC_URI = "p4://user:passwd@example-depot/main;module=source/..."
+
+In this case, the content of the top-level directory ``source/`` will be fetched
+to ``${P4DIR}``, including the directory itself.  The top-level directory will
+be accesible at ``${P4DIR}/source/``.
+
+Here is an example use of the the ``remotepath`` parameter: ::
+
+   SRC_URI = "p4://user:passwd@example-depot/main;module=source/...;remotepath=keep"
+
+In this case, the content of the top-level directory ``source/`` will be fetched
+to ``${P4DIR}``, but the complete depot paths will be mirrored locally. The
+top-level directory will be accessible at
+``${P4DIR}/example-depot/main/source/``.
+
+.. _repo-fetcher:
+
+Repo Fetcher (``repo://``)
+--------------------------
+
+This fetcher submodule fetches code from ``google-repo`` source control
+system. The fetcher works by initiating and syncing sources of the
+repository into :term:`REPODIR`, which is usually
+``${DL_DIR}/repo``.
+
+This fetcher supports the following parameters:
+
+-  *"protocol":* Protocol to fetch the repository manifest (default:
+   git).
+
+-  *"branch":* Branch or tag of repository to get (default: master).
+
+-  *"manifest":* Name of the manifest file (default: ``default.xml``).
+
+Here are some example URLs: ::
+
+   SRC_URI = "repo://REPOROOT;protocol=git;branch=some_branch;manifest=my_manifest.xml"
+   SRC_URI = "repo://REPOROOT;protocol=file;branch=some_branch;manifest=my_manifest.xml"
+
+Other Fetchers
+--------------
+
+Fetch submodules also exist for the following:
+
+-  Bazaar (``bzr://``)
+
+-  Mercurial (``hg://``)
+
+-  npm (``npm://``)
+
+-  OSC (``osc://``)
+
+-  Secure FTP (``sftp://``)
+
+-  Secure Shell (``ssh://``)
+
+-  Trees using Git Annex (``gitannex://``)
+
+No documentation currently exists for these lesser used fetcher
+submodules. However, you might find the code helpful and readable.
+
+Auto Revisions
+==============
+
+We need to document ``AUTOREV`` and ``SRCREV_FORMAT`` here.

bitbake/doc/bitbake-user-manual/bitbake-user-manual-fetching.xml

@@ -1,868 +0,0 @@
-<!DOCTYPE chapter PUBLIC "-//OASIS//DTD DocBook XML V4.2//EN"
-"http://www.oasis-open.org/docbook/xml/4.2/docbookx.dtd">
-
-<chapter>
-<title>File Download Support</title>
-
-    <para>
-        BitBake's fetch module is a standalone piece of library code
-        that deals with the intricacies of downloading source code
-        and files from remote systems.
-        Fetching source code is one of the cornerstones of building software.
-        As such, this module forms an important part of BitBake.
-    </para>
-
-    <para>
-        The current fetch module is called "fetch2" and refers to the
-        fact that it is the second major version of the API.
-        The original version is obsolete and has been removed from the codebase.
-        Thus, in all cases, "fetch" refers to "fetch2" in this
-        manual.
-    </para>
-
-    <section id='the-download-fetch'>
-        <title>The Download (Fetch)</title>
-
-        <para>
-            BitBake takes several steps when fetching source code or files.
-            The fetcher codebase deals with two distinct processes in order:
-            obtaining the files from somewhere (cached or otherwise)
-            and then unpacking those files into a specific location and
-            perhaps in a specific way.
-            Getting and unpacking the files is often optionally followed
-            by patching.
-            Patching, however, is not covered by this module.
-        </para>
-
-        <para>
-            The code to execute the first part of this process, a fetch,
-            looks something like the following:
-            <literallayout class='monospaced'>
-     src_uri = (d.getVar('SRC_URI') or "").split()
-     fetcher = bb.fetch2.Fetch(src_uri, d)
-     fetcher.download()
-            </literallayout>
-            This code sets up an instance of the fetch class.
-            The instance uses a space-separated list of URLs from the
-            <link linkend='var-bb-SRC_URI'><filename>SRC_URI</filename></link>
-            variable and then calls the <filename>download</filename>
-            method to download the files.
-        </para>
-
-        <para>
-            The instantiation of the fetch class is usually followed by:
-            <literallayout class='monospaced'>
-     rootdir = l.getVar('WORKDIR')
-     fetcher.unpack(rootdir)
-            </literallayout>
-            This code unpacks the downloaded files to the
-            specified by <filename>WORKDIR</filename>.
-            <note>
-                For convenience, the naming in these examples matches
-                the variables used by OpenEmbedded.
-                If you want to see the above code in action, examine
-                the OpenEmbedded class file <filename>base.bbclass</filename>.
-            </note>
-            The <filename>SRC_URI</filename> and <filename>WORKDIR</filename>
-            variables are not hardcoded into the fetcher, since those fetcher
-            methods can be (and are) called with different variable names.
-            In OpenEmbedded for example, the shared state (sstate) code uses
-            the fetch module to fetch the sstate files.
-        </para>
-
-        <para>
-            When the <filename>download()</filename> method is called,
-            BitBake tries to resolve the URLs by looking for source files
-            in a specific search order:
-            <itemizedlist>
-                <listitem><para><emphasis>Pre-mirror Sites:</emphasis>
-                    BitBake first uses pre-mirrors to try and find source files.
-                    These locations are defined using the
-                    <link linkend='var-bb-PREMIRRORS'><filename>PREMIRRORS</filename></link>
-                    variable.
-                    </para></listitem>
-                <listitem><para><emphasis>Source URI:</emphasis>
-                    If pre-mirrors fail, BitBake uses the original URL (e.g from
-                    <filename>SRC_URI</filename>).
-                    </para></listitem>
-                <listitem><para><emphasis>Mirror Sites:</emphasis>
-                    If fetch failures occur, BitBake next uses mirror locations as
-                    defined by the
-                    <link linkend='var-bb-MIRRORS'><filename>MIRRORS</filename></link>
-                    variable.
-                    </para></listitem>
-            </itemizedlist>
-        </para>
-
-        <para>
-            For each URL passed to the fetcher, the fetcher
-            calls the submodule that handles that particular URL type.
-            This behavior can be the source of some confusion when you
-            are providing URLs for the <filename>SRC_URI</filename>
-            variable.
-            Consider the following two URLs:
-            <literallayout class='monospaced'>
-     http://git.yoctoproject.org/git/poky;protocol=git
-     git://git.yoctoproject.org/git/poky;protocol=http
-            </literallayout>
-            In the former case, the URL is passed to the
-            <filename>wget</filename> fetcher, which does not
-            understand "git".
-            Therefore, the latter case is the correct form since the
-            Git fetcher does know how to use HTTP as a transport.
-        </para>
-
-        <para>
-            Here are some examples that show commonly used mirror
-            definitions:
-            <literallayout class='monospaced'>
-     PREMIRRORS ?= "\
-         bzr://.*/.*   http://somemirror.org/sources/ \n \
-         cvs://.*/.*   http://somemirror.org/sources/ \n \
-         git://.*/.*   http://somemirror.org/sources/ \n \
-         hg://.*/.*    http://somemirror.org/sources/ \n \
-         osc://.*/.*   http://somemirror.org/sources/ \n \
-         p4://.*/.*    http://somemirror.org/sources/ \n \
-         svn://.*/.*   http://somemirror.org/sources/ \n"
-
-     MIRRORS =+ "\
-         ftp://.*/.*      http://somemirror.org/sources/ \n \
-         http://.*/.*     http://somemirror.org/sources/ \n \
-         https://.*/.*    http://somemirror.org/sources/ \n"
-            </literallayout>
-            It is useful to note that BitBake supports
-            cross-URLs.
-            It is possible to mirror a Git repository on an HTTP
-            server as a tarball.
-            This is what the <filename>git://</filename> mapping in
-            the previous example does.
-        </para>
-
-        <para>
-            Since network accesses are slow, Bitbake maintains a
-            cache of files downloaded from the network.
-            Any source files that are not local (i.e.
-            downloaded from the Internet) are placed into the download
-            directory, which is specified by the
-            <link linkend='var-bb-DL_DIR'><filename>DL_DIR</filename></link>
-            variable.
-        </para>
-
-        <para>
-            File integrity is of key importance for reproducing builds.
-            For non-local archive downloads, the fetcher code can verify
-            SHA-256 and MD5 checksums to ensure the archives have been
-            downloaded correctly.
-            You can specify these checksums by using the
-            <filename>SRC_URI</filename> variable with the appropriate
-            varflags as follows:
-            <literallayout class='monospaced'>
-     SRC_URI[md5sum] = "<replaceable>value</replaceable>"
-     SRC_URI[sha256sum] = "<replaceable>value</replaceable>"
-            </literallayout>
-            You can also specify the checksums as parameters on the
-            <filename>SRC_URI</filename> as shown below:
-            <literallayout class='monospaced'>
-     SRC_URI = "http://example.com/foobar.tar.bz2;md5sum=4a8e0f237e961fd7785d19d07fdb994d"
-            </literallayout>
-            If multiple URIs exist, you can specify the checksums either
-            directly as in the previous example, or you can name the URLs.
-            The following syntax shows how you name the URIs:
-            <literallayout class='monospaced'>
-     SRC_URI = "http://example.com/foobar.tar.bz2;name=foo"
-     SRC_URI[foo.md5sum] = 4a8e0f237e961fd7785d19d07fdb994d
-            </literallayout>
-            After a file has been downloaded and has had its checksum checked,
-            a ".done" stamp is placed in <filename>DL_DIR</filename>.
-            BitBake uses this stamp during subsequent builds to avoid
-            downloading or comparing a checksum for the file again.
-            <note>
-                It is assumed that local storage is safe from data corruption.
-                If this were not the case, there would be bigger issues to worry about.
-            </note>
-        </para>
-
-        <para>
-            If
-            <link linkend='var-bb-BB_STRICT_CHECKSUM'><filename>BB_STRICT_CHECKSUM</filename></link>
-            is set, any download without a checksum triggers an
-            error message.
-            The
-            <link linkend='var-bb-BB_NO_NETWORK'><filename>BB_NO_NETWORK</filename></link>
-            variable can be used to make any attempted network access a fatal
-            error, which is useful for checking that mirrors are complete
-            as well as other things.
-        </para>
-    </section>
-
-    <section id='bb-the-unpack'>
-        <title>The Unpack</title>
-
-        <para>
-            The unpack process usually immediately follows the download.
-            For all URLs except Git URLs, BitBake uses the common
-            <filename>unpack</filename> method.
-        </para>
-
-        <para>
-            A number of parameters exist that you can specify within the
-            URL to govern the behavior of the unpack stage:
-            <itemizedlist>
-                <listitem><para><emphasis>unpack:</emphasis>
-                    Controls whether the URL components are unpacked.
-                    If set to "1", which is the default, the components
-                    are unpacked.
-                    If set to "0", the unpack stage leaves the file alone.
-                    This parameter is useful when you want an archive to be
-                    copied in and not be unpacked.
-                    </para></listitem>
-                <listitem><para><emphasis>dos:</emphasis>
-                    Applies to <filename>.zip</filename> and
-                    <filename>.jar</filename> files and specifies whether to
-                    use DOS line ending conversion on text files.
-                    </para></listitem>
-                <listitem><para><emphasis>basepath:</emphasis>
-                    Instructs the unpack stage to strip the specified
-                    directories from the source path when unpacking.
-                    </para></listitem>
-                <listitem><para><emphasis>subdir:</emphasis>
-                    Unpacks the specific URL to the specified subdirectory
-                    within the root directory.
-                    </para></listitem>
-            </itemizedlist>
-            The unpack call automatically decompresses and extracts files
-            with ".Z", ".z", ".gz", ".xz", ".zip", ".jar", ".ipk", ".rpm".
-            ".srpm", ".deb" and ".bz2" extensions as well as various combinations
-            of tarball extensions.
-        </para>
-
-        <para>
-            As mentioned, the Git fetcher has its own unpack method that
-            is optimized to work with Git trees.
-            Basically, this method works by cloning the tree into the final
-            directory.
-            The process is completed using references so that there is
-            only one central copy of the Git metadata needed.
-        </para>
-    </section>
-
-    <section id='bb-fetchers'>
-        <title>Fetchers</title>
-
-        <para>
-            As mentioned earlier, the URL prefix determines which
-            fetcher submodule BitBake uses.
-            Each submodule can support different URL parameters,
-            which are described in the following sections.
-        </para>
-
-        <section id='local-file-fetcher'>
-            <title>Local file fetcher (<filename>file://</filename>)</title>
-
-            <para>
-                This submodule handles URLs that begin with
-                <filename>file://</filename>.
-                The filename you specify within the URL can be
-                either an absolute or relative path to a file.
-                If the filename is relative, the contents of the
-                <link linkend='var-bb-FILESPATH'><filename>FILESPATH</filename></link>
-                variable is used in the same way
-                <filename>PATH</filename> is used to find executables.
-                If the file cannot be found, it is assumed that it is available in
-                <link linkend='var-bb-DL_DIR'><filename>DL_DIR</filename></link>
-                by the time the <filename>download()</filename> method is called.
-            </para>
-
-            <para>
-                If you specify a directory, the entire directory is
-                unpacked.
-            </para>
-
-            <para>
-                Here are a couple of example URLs, the first relative and
-                the second absolute:
-                <literallayout class='monospaced'>
-     SRC_URI = "file://relativefile.patch"
-     SRC_URI = "file:///Users/ich/very_important_software"
-                </literallayout>
-            </para>
-        </section>
-
-        <section id='http-ftp-fetcher'>
-            <title>HTTP/FTP wget fetcher (<filename>http://</filename>, <filename>ftp://</filename>, <filename>https://</filename>)</title>
-
-            <para>
-                This fetcher obtains files from web and FTP servers.
-                Internally, the fetcher uses the wget utility.
-            </para>
-
-            <para>
-                The executable and parameters used are specified by the
-                <filename>FETCHCMD_wget</filename> variable, which defaults
-                to sensible values.
-                The fetcher supports a parameter "downloadfilename" that
-                allows the name of the downloaded file to be specified.
-                Specifying the name of the downloaded file is useful
-                for avoiding collisions in
-                <link linkend='var-bb-DL_DIR'><filename>DL_DIR</filename></link>
-                when dealing with multiple files that have the same name.
-            </para>
-
-            <para>
-                Some example URLs are as follows:
-                <literallayout class='monospaced'>
-     SRC_URI = "http://oe.handhelds.org/not_there.aac"
-     SRC_URI = "ftp://oe.handhelds.org/not_there_as_well.aac"
-     SRC_URI = "ftp://you@oe.handhelds.org/home/you/secret.plan"
-                </literallayout>
-            </para>
-            <note>
-               Because URL parameters are delimited by semi-colons, this can
-               introduce ambiguity when parsing URLs that also contain semi-colons,
-               for example:
-                <literallayout class='monospaced'>
-     SRC_URI = "http://abc123.org/git/?p=gcc/gcc.git;a=snapshot;h=a5dd47"
-                </literallayout>
-               Such URLs should should be modified by replacing semi-colons with '&amp;' characters:
-               <literallayout class='monospaced'>
-     SRC_URI = "http://abc123.org/git/?p=gcc/gcc.git&amp;a=snapshot&amp;h=a5dd47"
-                </literallayout>
-                In most cases this should work. Treating semi-colons and '&amp;' in queries
-                identically is recommended by the World Wide Web Consortium (W3C).
-                Note that due to the nature of the URL, you may have to specify the name
-                of the downloaded file as well:
-              <literallayout class='monospaced'>
-     SRC_URI = "http://abc123.org/git/?p=gcc/gcc.git&amp;a=snapshot&amp;h=a5dd47;downloadfilename=myfile.bz2"
-                </literallayout>
-            </note>
-        </section>
-
-        <section id='cvs-fetcher'>
-            <title>CVS fetcher (<filename>(cvs://</filename>)</title>
-
-            <para>
-                This submodule handles checking out files from the
-                CVS version control system.
-                You can configure it using a number of different variables:
-                <itemizedlist>
-                    <listitem><para><emphasis><filename>FETCHCMD_cvs</filename>:</emphasis>
-                        The name of the executable to use when running
-                        the <filename>cvs</filename> command.
-                        This name is usually "cvs".
-                        </para></listitem>
-                    <listitem><para><emphasis><filename>SRCDATE</filename>:</emphasis>
-                        The date to use when fetching the CVS source code.
-                        A special value of "now" causes the checkout to
-                        be updated on every build.
-                        </para></listitem>
-                    <listitem><para><emphasis><link linkend='var-bb-CVSDIR'><filename>CVSDIR</filename></link>:</emphasis>
-                        Specifies where a temporary checkout is saved.
-                        The location is often <filename>DL_DIR/cvs</filename>.
-                        </para></listitem>
-                    <listitem><para><emphasis><filename>CVS_PROXY_HOST</filename>:</emphasis>
-                        The name to use as a "proxy=" parameter to the
-                        <filename>cvs</filename> command.
-                        </para></listitem>
-                    <listitem><para><emphasis><filename>CVS_PROXY_PORT</filename>:</emphasis>
-                        The port number to use as a "proxyport=" parameter to
-                        the <filename>cvs</filename> command.
-                        </para></listitem>
-                </itemizedlist>
-                As well as the standard username and password URL syntax,
-                you can also configure the fetcher with various URL parameters:
-            </para>
-
-            <para>
-                The supported parameters are as follows:
-                <itemizedlist>
-                    <listitem><para><emphasis>"method":</emphasis>
-                        The protocol over which to communicate with the CVS
-                        server.
-                        By default, this protocol is "pserver".
-                        If "method" is set to "ext", BitBake examines the
-                        "rsh" parameter and sets <filename>CVS_RSH</filename>.
-                        You can use "dir" for local directories.
-                        </para></listitem>
-                    <listitem><para><emphasis>"module":</emphasis>
-                        Specifies the module to check out.
-                        You must supply this parameter.
-                        </para></listitem>
-                    <listitem><para><emphasis>"tag":</emphasis>
-                        Describes which CVS TAG should be used for
-                        the checkout.
-                        By default, the TAG is empty.
-                        </para></listitem>
-                    <listitem><para><emphasis>"date":</emphasis>
-                        Specifies a date.
-                        If no "date" is specified, the
-                        <link linkend='var-bb-SRCDATE'><filename>SRCDATE</filename></link>
-                        of the configuration is used to checkout a specific date.
-                        The special value of "now" causes the checkout to be
-                        updated on every build.
-                        </para></listitem>
-                    <listitem><para><emphasis>"localdir":</emphasis>
-                        Used to rename the module.
-                        Effectively, you are renaming the output directory
-                        to which the module is unpacked.
-                        You are forcing the module into a special
-                        directory relative to
-                        <link linkend='var-bb-CVSDIR'><filename>CVSDIR</filename></link>.
-                        </para></listitem>
-                    <listitem><para><emphasis>"rsh"</emphasis>
-                        Used in conjunction with the "method" parameter.
-                        </para></listitem>
-                    <listitem><para><emphasis>"scmdata":</emphasis>
-                        Causes the CVS metadata to be maintained in the tarball
-                        the fetcher creates when set to "keep".
-                        The tarball is expanded into the work directory.
-                        By default, the CVS metadata is removed.
-                        </para></listitem>
-                    <listitem><para><emphasis>"fullpath":</emphasis>
-                        Controls whether the resulting checkout is at the
-                        module level, which is the default, or is at deeper
-                        paths.
-                        </para></listitem>
-                    <listitem><para><emphasis>"norecurse":</emphasis>
-                        Causes the fetcher to only checkout the specified
-                        directory with no recurse into any subdirectories.
-                        </para></listitem>
-                    <listitem><para><emphasis>"port":</emphasis>
-                        The port to which the CVS server connects.
-                        </para></listitem>
-                </itemizedlist>
-                Some example URLs are as follows:
-                <literallayout class='monospaced'>
-     SRC_URI = "cvs://CVSROOT;module=mymodule;tag=some-version;method=ext"
-     SRC_URI = "cvs://CVSROOT;module=mymodule;date=20060126;localdir=usethat"
-                </literallayout>
-            </para>
-        </section>
-
-        <section id='svn-fetcher'>
-            <title>Subversion (SVN) Fetcher (<filename>svn://</filename>)</title>
-
-            <para>
-                This fetcher submodule fetches code from the
-                Subversion source control system.
-                The executable used is specified by
-                <filename>FETCHCMD_svn</filename>, which defaults
-                to "svn".
-                The fetcher's temporary working directory is set by
-                <link linkend='var-bb-SVNDIR'><filename>SVNDIR</filename></link>,
-                which is usually <filename>DL_DIR/svn</filename>.
-            </para>
-
-            <para>
-                The supported parameters are as follows:
-                <itemizedlist>
-                    <listitem><para><emphasis>"module":</emphasis>
-                        The name of the svn module to checkout.
-                        You must provide this parameter.
-                        You can think of this parameter as the top-level
-                        directory of the repository data you want.
-                        </para></listitem>
-                    <listitem><para><emphasis>"path_spec":</emphasis>
-                        A specific directory in which to checkout the
-                        specified svn module.
-                        </para></listitem>
-                    <listitem><para><emphasis>"protocol":</emphasis>
-                        The protocol to use, which defaults to "svn".
-                        If "protocol" is set to "svn+ssh", the "ssh"
-                        parameter is also used.
-                        </para></listitem>
-                    <listitem><para><emphasis>"rev":</emphasis>
-                        The revision of the source code to checkout.
-                        </para></listitem>
-                    <listitem><para><emphasis>"scmdata":</emphasis>
-                        Causes the “.svn” directories to be available during
-                        compile-time when set to "keep".
-                        By default, these directories are removed.
-                        </para></listitem>
-                    <listitem><para><emphasis>"ssh":</emphasis>
-                        An optional parameter used when "protocol" is set
-                        to "svn+ssh".
-                        You can use this parameter to specify the ssh
-                        program used by svn.
-                        </para></listitem>
-                    <listitem><para><emphasis>"transportuser":</emphasis>
-                        When required, sets the username for the transport.
-                        By default, this parameter is empty.
-                        The transport username is different than the username
-                        used in the main URL, which is passed to the subversion
-                        command.
-                        </para></listitem>
-                </itemizedlist>
-                Following are three examples using svn:
-                <literallayout class='monospaced'>
-     SRC_URI = "svn://myrepos/proj1;module=vip;protocol=http;rev=667"
-     SRC_URI = "svn://myrepos/proj1;module=opie;protocol=svn+ssh"
-     SRC_URI = "svn://myrepos/proj1;module=trunk;protocol=http;path_spec=${MY_DIR}/proj1"
-                </literallayout>
-            </para>
-        </section>
-
-        <section id='git-fetcher'>
-            <title>Git Fetcher (<filename>git://</filename>)</title>
-
-            <para>
-                This fetcher submodule fetches code from the Git
-                source control system.
-                The fetcher works by creating a bare clone of the
-                remote into
-                <link linkend='var-bb-GITDIR'><filename>GITDIR</filename></link>,
-                which is usually <filename>DL_DIR/git2</filename>.
-                This bare clone is then cloned into the work directory during the
-                unpack stage when a specific tree is checked out.
-                This is done using alternates and by reference to
-                minimize the amount of duplicate data on the disk and
-                make the unpack process fast.
-                The executable used can be set with
-                <filename>FETCHCMD_git</filename>.
-            </para>
-
-            <para>
-                This fetcher supports the following parameters:
-                <itemizedlist>
-                    <listitem><para><emphasis>"protocol":</emphasis>
-                        The protocol used to fetch the files.
-                        The default is "git" when a hostname is set.
-                        If a hostname is not set, the Git protocol is "file".
-                        You can also use "http", "https", "ssh" and "rsync".
-                        </para></listitem>
-                    <listitem><para><emphasis>"nocheckout":</emphasis>
-                        Tells the fetcher to not checkout source code when
-                        unpacking when set to "1".
-                        Set this option for the URL where there is a custom
-                        routine to checkout code.
-                        The default is "0".
-                        </para></listitem>
-                    <listitem><para><emphasis>"rebaseable":</emphasis>
-                        Indicates that the upstream Git repository can be rebased.
-                        You should set this parameter to "1" if
-                        revisions can become detached from branches.
-                        In this case, the source mirror tarball is done per
-                        revision, which has a loss of efficiency.
-                        Rebasing the upstream Git repository could cause the
-                        current revision to disappear from the upstream repository.
-                        This option reminds the fetcher to preserve the local cache
-                        carefully for future use.
-                        The default value for this parameter is "0".
-                        </para></listitem>
-                    <listitem><para><emphasis>"nobranch":</emphasis>
-                        Tells the fetcher to not check the SHA validation
-                        for the branch when set to "1".
-                        The default is "0".
-                        Set this option for the recipe that refers to
-                        the commit that is valid for a tag instead of
-                        the branch.
-                        </para></listitem>
-                    <listitem><para><emphasis>"bareclone":</emphasis>
-                        Tells the fetcher to clone a bare clone into the
-                        destination directory without checking out a working tree.
-                        Only the raw Git metadata is provided.
-                        This parameter implies the "nocheckout" parameter as well.
-                        </para></listitem>
-                    <listitem><para><emphasis>"branch":</emphasis>
-                        The branch(es) of the Git tree to clone.
-                        If unset, this is assumed to be "master".
-                        The number of branch parameters much match the number of
-                        name parameters.
-                        </para></listitem>
-                    <listitem><para><emphasis>"rev":</emphasis>
-                        The revision to use for the checkout.
-                        The default is "master".
-                        </para></listitem>
-                    <listitem><para><emphasis>"tag":</emphasis>
-                        Specifies a tag to use for the checkout.
-                        To correctly resolve tags, BitBake must access the
-                        network.
-                        For that reason, tags are often not used.
-                        As far as Git is concerned, the "tag" parameter behaves
-                        effectively the same as the "rev" parameter.
-                        </para></listitem>
-                    <listitem><para><emphasis>"subpath":</emphasis>
-                        Limits the checkout to a specific subpath of the tree.
-                        By default, the whole tree is checked out.
-                        </para></listitem>
-                    <listitem><para><emphasis>"destsuffix":</emphasis>
-                        The name of the path in which to place the checkout.
-                        By default, the path is <filename>git/</filename>.
-                        </para></listitem>
-                    <listitem><para><emphasis>"usehead":</emphasis>
-                        Enables local <filename>git://</filename> URLs to use the
-                        current branch HEAD as the revision for use with
-                        <filename>AUTOREV</filename>.
-                        The "usehead" parameter implies no branch and only works
-                        when the transfer protocol is
-                        <filename>file://</filename>.
-                        </para></listitem>
-                </itemizedlist>
-                Here are some example URLs:
-                <literallayout class='monospaced'>
-     SRC_URI = "git://git.oe.handhelds.org/git/vip.git;tag=version-1"
-     SRC_URI = "git://git.oe.handhelds.org/git/vip.git;protocol=http"
-                </literallayout>
-            </para>
-        </section>
-
-        <section id='gitsm-fetcher'>
-            <title>Git Submodule Fetcher (<filename>gitsm://</filename>)</title>
-
-            <para>
-                This fetcher submodule inherits from the
-                <link linkend='git-fetcher'>Git fetcher</link> and extends
-                that fetcher's behavior by fetching a repository's submodules.
-                <link linkend='var-bb-SRC_URI'><filename>SRC_URI</filename></link>
-                is passed to the Git fetcher as described in the
-                "<link linkend='git-fetcher'>Git Fetcher (<filename>git://</filename>)</link>"
-                section.
-                <note>
-                    <title>Notes and Warnings</title>
-                    <para>
-                        You must clean a recipe when switching between
-                        '<filename>git://</filename>' and
-                        '<filename>gitsm://</filename>' URLs.
-                    </para>
-
-                    <para>
-                        The Git Submodules fetcher is not a complete fetcher
-                        implementation.
-                        The fetcher has known issues where it does not use the
-                        normal source mirroring infrastructure properly. Further,
-                        the submodule sources it fetches are not visible to the
-                        licensing and source archiving infrastructures.
-                    </para>
-                </note>
-            </para>
-        </section>
-
-        <section id='clearcase-fetcher'>
-            <title>ClearCase Fetcher (<filename>ccrc://</filename>)</title>
-
-            <para>
-                This fetcher submodule fetches code from a
-                <ulink url='http://en.wikipedia.org/wiki/Rational_ClearCase'>ClearCase</ulink>
-                repository.
-            </para>
-
-            <para>
-                To use this fetcher, make sure your recipe has proper
-                <link linkend='var-bb-SRC_URI'><filename>SRC_URI</filename></link>,
-                <link linkend='var-bb-SRCREV'><filename>SRCREV</filename></link>, and
-                <link linkend='var-bb-PV'><filename>PV</filename></link> settings.
-                Here is an example:
-                <literallayout class='monospaced'>
-     SRC_URI = "ccrc://cc.example.org/ccrc;vob=/example_vob;module=/example_module"
-     SRCREV = "EXAMPLE_CLEARCASE_TAG"
-     PV = "${@d.getVar("SRCREV", False).replace("/", "+")}"
-                </literallayout>
-                The fetcher uses the <filename>rcleartool</filename> or
-                <filename>cleartool</filename> remote client, depending on
-                which one is available.
-            </para>
-
-            <para>
-                Following are options for the <filename>SRC_URI</filename>
-                statement:
-                <itemizedlist>
-                    <listitem><para><emphasis><filename>vob</filename></emphasis>:
-                        The name, which must include the
-                        prepending "/" character, of the ClearCase VOB.
-                        This option is required.
-                        </para></listitem>
-                    <listitem><para><emphasis><filename>module</filename></emphasis>:
-                        The module, which must include the
-                        prepending "/" character, in the selected VOB.
-                        <note>
-                            The <filename>module</filename> and <filename>vob</filename>
-                            options are combined to create the <filename>load</filename> rule in
-                            the view config spec.
-                            As an example, consider the <filename>vob</filename> and
-                            <filename>module</filename> values from the
-                            <filename>SRC_URI</filename> statement at the start of this section.
-                            Combining those values results in the following:
-                            <literallayout class='monospaced'>
-     load /example_vob/example_module
-                            </literallayout>
-                        </note>
-                        </para></listitem>
-                    <listitem><para><emphasis><filename>proto</filename></emphasis>:
-                        The protocol, which can be either <filename>http</filename> or
-                        <filename>https</filename>.
-                        </para></listitem>
-                </itemizedlist>
-            </para>
-
-            <para>
-                By default, the fetcher creates a configuration specification.
-                If you want this specification written to an area other than the default,
-                use the <filename>CCASE_CUSTOM_CONFIG_SPEC</filename> variable
-                in your recipe to define where the specification is written.
-                <note>
-                    the <filename>SRCREV</filename> loses its functionality if you
-                    specify this variable.
-                    However, <filename>SRCREV</filename> is still used to label the
-                    archive after a fetch even though it does not define what is
-                    fetched.
-                </note>
-            </para>
-
-            <para>
-                Here are a couple of other behaviors worth mentioning:
-                <itemizedlist>
-                    <listitem><para>
-                        When using <filename>cleartool</filename>, the login of
-                        <filename>cleartool</filename> is handled by the system.
-                        The login require no special steps.
-                        </para></listitem>
-                    <listitem><para>
-                        In order to use <filename>rcleartool</filename> with authenticated
-                        users, an "rcleartool login" is necessary before using the fetcher.
-                        </para></listitem>
-                </itemizedlist>
-            </para>
-        </section>
-
-        <section id='perforce-fetcher'>
-            <title>Perforce Fetcher (<filename>p4://</filename>)</title>
-
-            <para>
-                This fetcher submodule fetches code from the
-                <ulink url='https://www.perforce.com/'>Perforce</ulink>
-                source control system.
-                The executable used is specified by
-                <filename>FETCHCMD_p4</filename>, which defaults
-                to "p4".
-                The fetcher's temporary working directory is set by
-                <link linkend='var-bb-P4DIR'><filename>P4DIR</filename></link>,
-                which defaults to "DL_DIR/p4".
-            </para>
-
-            <para>
-                To use this fetcher, make sure your recipe has proper
-                <link linkend='var-bb-SRC_URI'><filename>SRC_URI</filename></link>,
-                <link linkend='var-bb-SRCREV'><filename>SRCREV</filename></link>, and
-                <link linkend='var-bb-PV'><filename>PV</filename></link> values.
-                The p4 executable is able to use the config file defined by your
-                system's <filename>P4CONFIG</filename> environment variable in
-                order to define the Perforce server URL and port, username, and
-                password if you do not wish to keep those values in a recipe
-                itself.
-                If you choose not to use <filename>P4CONFIG</filename>,
-                or to explicitly set variables that <filename>P4CONFIG</filename>
-                can contain, you can specify the <filename>P4PORT</filename> value,
-                which is the server's URL and port number, and you can
-                specify a username and password directly in your recipe within
-                <filename>SRC_URI</filename>.
-            </para>
-
-            <para>
-                Here is an example that relies on <filename>P4CONFIG</filename>
-                to specify the server URL and port, username, and password, and
-                fetches the Head Revision:
-                <literallayout class='monospaced'>
-    SRC_URI = "p4://example-depot/main/source/..."
-    SRCREV = "${AUTOREV}"
-    PV = "p4-${SRCPV}"
-    S = "${WORKDIR}/p4"
-                </literallayout>
-            </para>
-
-            <para>
-                Here is an example that specifies the server URL and port,
-                username, and password, and fetches a Revision based on a Label:
-                <literallayout class='monospaced'>
-    P4PORT = "tcp:p4server.example.net:1666"
-    SRC_URI = "p4://user:passwd@example-depot/main/source/..."
-    SRCREV = "release-1.0"
-    PV = "p4-${SRCPV}"
-    S = "${WORKDIR}/p4"
-                </literallayout>
-                <note>
-                    You should always set <filename>S</filename>
-                    to <filename>"${WORKDIR}/p4"</filename> in your recipe.
-                </note>
-            </para>
-        </section>
-
-        <section id='repo-fetcher'>
-            <title>Repo Fetcher (<filename>repo://</filename>)</title>
-
-            <para>
-                This fetcher submodule fetches code from
-                <filename>google-repo</filename> source control system.
-                The fetcher works by initiating and syncing sources of the
-                repository into
-                <link linkend='var-bb-REPODIR'><filename>REPODIR</filename></link>,
-                which is usually
-                <link linkend='var-bb-DL_DIR'><filename>DL_DIR</filename></link><filename>/repo</filename>.
-            </para>
-
-            <para>
-                This fetcher supports the following parameters:
-                <itemizedlist>
-                    <listitem><para>
-                        <emphasis>"protocol":</emphasis>
-                        Protocol to fetch the repository manifest (default: git).
-                        </para></listitem>
-                    <listitem><para>
-                        <emphasis>"branch":</emphasis>
-                        Branch or tag of repository to get (default: master).
-                        </para></listitem>
-                    <listitem><para>
-                        <emphasis>"manifest":</emphasis>
-                        Name of the manifest file (default: <filename>default.xml</filename>).
-                        </para></listitem>
-                </itemizedlist>
-                Here are some example URLs:
-                <literallayout class='monospaced'>
-    SRC_URI = "repo://REPOROOT;protocol=git;branch=some_branch;manifest=my_manifest.xml"
-    SRC_URI = "repo://REPOROOT;protocol=file;branch=some_branch;manifest=my_manifest.xml"
-                </literallayout>
-            </para>
-        </section>
-
-        <section id='other-fetchers'>
-            <title>Other Fetchers</title>
-
-            <para>
-                Fetch submodules also exist for the following:
-                <itemizedlist>
-                    <listitem><para>
-                        Bazaar (<filename>bzr://</filename>)
-                        </para></listitem>
-                    <listitem><para>
-                        Mercurial (<filename>hg://</filename>)
-                        </para></listitem>
-                    <listitem><para>
-                        npm (<filename>npm://</filename>)
-                        </para></listitem>
-                    <listitem><para>
-                        OSC (<filename>osc://</filename>)
-                        </para></listitem>
-                    <listitem><para>
-                        Secure FTP (<filename>sftp://</filename>)
-                        </para></listitem>
-                    <listitem><para>
-                        Secure Shell (<filename>ssh://</filename>)
-                        </para></listitem>
-                    <listitem><para>
-                        Trees using Git Annex (<filename>gitannex://</filename>)
-                        </para></listitem>
-                </itemizedlist>
-                No documentation currently exists for these lesser used
-                fetcher submodules.
-                However, you might find the code helpful and readable.
-            </para>
-        </section>
-    </section>
-
-    <section id='auto-revisions'>
-        <title>Auto Revisions</title>
-
-        <para>
-            We need to document <filename>AUTOREV</filename> and
-            <filename>SRCREV_FORMAT</filename> here.
-        </para>
-    </section>
-</chapter>

bitbake/doc/bitbake-user-manual/bitbake-user-manual-hello.rst

@@ -0,0 +1,415 @@
+.. SPDX-License-Identifier: CC-BY-2.5
+
+===================
+Hello World Example
+===================
+
+BitBake Hello World
+===================
+
+The simplest example commonly used to demonstrate any new programming
+language or tool is the "`Hello
+World <http://en.wikipedia.org/wiki/Hello_world_program>`__" example.
+This appendix demonstrates, in tutorial form, Hello World within the
+context of BitBake. The tutorial describes how to create a new project
+and the applicable metadata files necessary to allow BitBake to build
+it.
+
+Obtaining BitBake
+=================
+
+See the :ref:`bitbake-user-manual/bitbake-user-manual-hello:obtaining bitbake` section for
+information on how to obtain BitBake. Once you have the source code on
+your machine, the BitBake directory appears as follows: ::
+
+   $ ls -al
+   total 100
+   drwxrwxr-x. 9 wmat wmat  4096 Jan 31 13:44 .
+   drwxrwxr-x. 3 wmat wmat  4096 Feb  4 10:45 ..
+   -rw-rw-r--. 1 wmat wmat   365 Nov 26 04:55 AUTHORS
+   drwxrwxr-x. 2 wmat wmat  4096 Nov 26 04:55 bin
+   drwxrwxr-x. 4 wmat wmat  4096 Jan 31 13:44 build
+   -rw-rw-r--. 1 wmat wmat 16501 Nov 26 04:55 ChangeLog
+   drwxrwxr-x. 2 wmat wmat  4096 Nov 26 04:55 classes
+   drwxrwxr-x. 2 wmat wmat  4096 Nov 26 04:55 conf
+   drwxrwxr-x. 3 wmat wmat  4096 Nov 26 04:55 contrib
+   -rw-rw-r--. 1 wmat wmat 17987 Nov 26 04:55 COPYING
+   drwxrwxr-x. 3 wmat wmat  4096 Nov 26 04:55 doc
+   -rw-rw-r--. 1 wmat wmat    69 Nov 26 04:55 .gitignore
+   -rw-rw-r--. 1 wmat wmat   849 Nov 26 04:55 HEADER
+   drwxrwxr-x. 5 wmat wmat  4096 Jan 31 13:44 lib
+   -rw-rw-r--. 1 wmat wmat   195 Nov 26 04:55 MANIFEST.in
+   -rw-rw-r--. 1 wmat wmat  2887 Nov 26 04:55 TODO
+
+At this point, you should have BitBake cloned to a directory that
+matches the previous listing except for dates and user names.
+
+Setting Up the BitBake Environment
+==================================
+
+First, you need to be sure that you can run BitBake. Set your working
+directory to where your local BitBake files are and run the following
+command: ::
+
+  $ ./bin/bitbake --version
+  BitBake Build Tool Core version 1.23.0, bitbake version 1.23.0
+
+The console output tells you what version
+you are running.
+
+The recommended method to run BitBake is from a directory of your
+choice. To be able to run BitBake from any directory, you need to add
+the executable binary to your binary to your shell's environment
+``PATH`` variable. First, look at your current ``PATH`` variable by
+entering the following: ::
+
+  $ echo $PATH
+
+Next, add the directory location
+for the BitBake binary to the ``PATH``. Here is an example that adds the
+``/home/scott-lenovo/bitbake/bin`` directory to the front of the
+``PATH`` variable: ::
+
+  $ export PATH=/home/scott-lenovo/bitbake/bin:$PATH
+
+You should now be able to enter the ``bitbake`` command from the command
+line while working from any directory.
+
+The Hello World Example
+=======================
+
+The overall goal of this exercise is to build a complete "Hello World"
+example utilizing task and layer concepts. Because this is how modern
+projects such as OpenEmbedded and the Yocto Project utilize BitBake, the
+example provides an excellent starting point for understanding BitBake.
+
+To help you understand how to use BitBake to build targets, the example
+starts with nothing but the ``bitbake`` command, which causes BitBake to
+fail and report problems. The example progresses by adding pieces to the
+build to eventually conclude with a working, minimal "Hello World"
+example.
+
+While every attempt is made to explain what is happening during the
+example, the descriptions cannot cover everything. You can find further
+information throughout this manual. Also, you can actively participate
+in the :oe_lists:`/g/bitbake-devel`
+discussion mailing list about the BitBake build tool.
+
+.. note::
+
+   This example was inspired by and drew heavily from
+   `Mailing List post - The BitBake equivalent of "Hello, World!"
+   <http://www.mail-archive.com/yocto@yoctoproject.org/msg09379.html>`_.
+
+As stated earlier, the goal of this example is to eventually compile
+"Hello World". However, it is unknown what BitBake needs and what you
+have to provide in order to achieve that goal. Recall that BitBake
+utilizes three types of metadata files:
+:ref:`bitbake-user-manual/bitbake-user-manual-intro:configuration files`,
+:ref:`bitbake-user-manual/bitbake-user-manual-intro:classes`, and
+:ref:`bitbake-user-manual/bitbake-user-manual-intro:recipes`.
+But where do they go? How does BitBake find
+them? BitBake's error messaging helps you answer these types of
+questions and helps you better understand exactly what is going on.
+
+Following is the complete "Hello World" example.
+
+#.  **Create a Project Directory:** First, set up a directory for the
+    "Hello World" project. Here is how you can do so in your home
+    directory: ::
+
+      $ mkdir ~/hello
+      $ cd ~/hello
+
+    This is the directory that
+    BitBake will use to do all of its work. You can use this directory
+    to keep all the metafiles needed by BitBake. Having a project
+    directory is a good way to isolate your project.
+
+#.  **Run BitBake:** At this point, you have nothing but a project
+    directory. Run the ``bitbake`` command and see what it does: ::
+
+       $ bitbake
+       The BBPATH variable is not set and bitbake did not
+       find a conf/bblayers.conf file in the expected location.
+       Maybe you accidentally invoked bitbake from the wrong directory?
+       DEBUG: Removed the following variables from the environment:
+       GNOME_DESKTOP_SESSION_ID, XDG_CURRENT_DESKTOP,
+       GNOME_KEYRING_CONTROL, DISPLAY, SSH_AGENT_PID, LANG, no_proxy,
+       XDG_SESSION_PATH, XAUTHORITY, SESSION_MANAGER, SHLVL,
+       MANDATORY_PATH, COMPIZ_CONFIG_PROFILE, WINDOWID, EDITOR,
+       GPG_AGENT_INFO, SSH_AUTH_SOCK, GDMSESSION, GNOME_KEYRING_PID,
+       XDG_SEAT_PATH, XDG_CONFIG_DIRS, LESSOPEN, DBUS_SESSION_BUS_ADDRESS,
+       _, XDG_SESSION_COOKIE, DESKTOP_SESSION, LESSCLOSE, DEFAULTS_PATH,
+       UBUNTU_MENUPROXY, OLDPWD, XDG_DATA_DIRS, COLORTERM, LS_COLORS
+
+    The majority of this output is specific to environment variables that
+    are not directly relevant to BitBake. However, the very first
+    message regarding the ``BBPATH`` variable and the
+    ``conf/bblayers.conf`` file is relevant.
+
+    When you run BitBake, it begins looking for metadata files. The
+    :term:`BBPATH` variable is what tells BitBake where
+    to look for those files. ``BBPATH`` is not set and you need to set
+    it. Without ``BBPATH``, BitBake cannot find any configuration files
+    (``.conf``) or recipe files (``.bb``) at all. BitBake also cannot
+    find the ``bitbake.conf`` file.
+
+#.  **Setting BBPATH:** For this example, you can set ``BBPATH`` in
+    the same manner that you set ``PATH`` earlier in the appendix. You
+    should realize, though, that it is much more flexible to set the
+    ``BBPATH`` variable up in a configuration file for each project.
+
+    From your shell, enter the following commands to set and export the
+    ``BBPATH`` variable: ::
+
+      $ BBPATH="projectdirectory"
+      $ export BBPATH
+
+    Use your actual project directory in the command. BitBake uses that
+    directory to find the metadata it needs for your project.
+
+    .. note::
+
+       When specifying your project directory, do not use the tilde
+       ("~") character as BitBake does not expand that character as the
+       shell would.
+
+#.  **Run BitBake:** Now that you have ``BBPATH`` defined, run the
+    ``bitbake`` command again: ::
+
+       $ bitbake
+       ERROR: Traceback (most recent call last):
+         File "/home/scott-lenovo/bitbake/lib/bb/cookerdata.py", line 163, in wrapped
+           return func(fn, *args)
+         File "/home/scott-lenovo/bitbake/lib/bb/cookerdata.py", line 173, in parse_config_file
+           return bb.parse.handle(fn, data, include)
+         File "/home/scott-lenovo/bitbake/lib/bb/parse/__init__.py", line 99, in handle
+           return h['handle'](fn, data, include)
+         File "/home/scott-lenovo/bitbake/lib/bb/parse/parse_py/ConfHandler.py", line 120, in handle
+           abs_fn = resolve_file(fn, data)
+         File "/home/scott-lenovo/bitbake/lib/bb/parse/__init__.py", line 117, in resolve_file
+           raise IOError("file %s not found in %s" % (fn, bbpath))
+       IOError: file conf/bitbake.conf not found in /home/scott-lenovo/hello
+
+       ERROR: Unable to parse conf/bitbake.conf: file conf/bitbake.conf not found in /home/scott-lenovo/hello
+
+    This sample output shows that BitBake could not find the
+    ``conf/bitbake.conf`` file in the project directory. This file is
+    the first thing BitBake must find in order to build a target. And,
+    since the project directory for this example is empty, you need to
+    provide a ``conf/bitbake.conf`` file.
+
+#.  **Creating conf/bitbake.conf:** The ``conf/bitbake.conf`` includes
+    a number of configuration variables BitBake uses for metadata and
+    recipe files. For this example, you need to create the file in your
+    project directory and define some key BitBake variables. For more
+    information on the ``bitbake.conf`` file, see
+    http://git.openembedded.org/bitbake/tree/conf/bitbake.conf.
+
+    Use the following commands to create the ``conf`` directory in the
+    project directory: ::
+
+      $ mkdir conf
+
+    From within the ``conf`` directory,
+    use some editor to create the ``bitbake.conf`` so that it contains
+    the following: ::
+
+       PN  = "${@bb.parse.BBHandler.vars_from_file(d.getVar('FILE', False),d)[0] or 'defaultpkgname'}"
+
+       TMPDIR  = "${TOPDIR}/tmp"
+       CACHE   = "${TMPDIR}/cache"
+       STAMP   = "${TMPDIR}/${PN}/stamps"
+       T       = "${TMPDIR}/${PN}/work"
+       B       = "${TMPDIR}/${PN}"
+
+    .. note::
+
+       Without a value for PN , the variables STAMP , T , and B , prevent more
+       than one recipe from working. You can fix this by either setting PN to
+       have a value similar to what OpenEmbedded and BitBake use in the default
+       bitbake.conf file (see previous example). Or, by manually updating each
+       recipe to set PN . You will also need to include PN as part of the STAMP
+       , T , and B variable definitions in the local.conf file.
+
+    The ``TMPDIR`` variable establishes a directory that BitBake uses
+    for build output and intermediate files other than the cached
+    information used by the
+    :ref:`bitbake-user-manual/bitbake-user-manual-execution:setscene`
+    process. Here, the ``TMPDIR`` directory is set to ``hello/tmp``.
+
+    .. tip::
+
+       You can always safely delete the tmp directory in order to rebuild a
+       BitBake target. The build process creates the directory for you when you
+       run BitBake.
+
+    For information about each of the other variables defined in this
+    example, check :term:`PN`, :term:`TOPDIR`, :term:`CACHE`, :term:`STAMP`,
+    :term:`T` or :term:`B` to take you to the definitions in the
+    glossary.
+
+#.  **Run BitBake:** After making sure that the ``conf/bitbake.conf`` file
+    exists, you can run the ``bitbake`` command again: ::
+
+       $ bitbake
+       ERROR: Traceback (most recent call last):
+         File "/home/scott-lenovo/bitbake/lib/bb/cookerdata.py", line 163, in wrapped
+           return func(fn, *args)
+         File "/home/scott-lenovo/bitbake/lib/bb/cookerdata.py", line 177, in _inherit
+           bb.parse.BBHandler.inherit(bbclass, "configuration INHERITs", 0, data)
+         File "/home/scott-lenovo/bitbake/lib/bb/parse/parse_py/BBHandler.py", line 92, in inherit
+           include(fn, file, lineno, d, "inherit")
+         File "/home/scott-lenovo/bitbake/lib/bb/parse/parse_py/ConfHandler.py", line 100, in include
+           raise ParseError("Could not %(error_out)s file %(fn)s" % vars(), oldfn, lineno)
+       ParseError: ParseError in configuration INHERITs: Could not inherit file classes/base.bbclass
+
+       ERROR: Unable to parse base: ParseError in configuration INHERITs: Could not inherit file classes/base.bbclass
+
+    In the sample output,
+    BitBake could not find the ``classes/base.bbclass`` file. You need
+    to create that file next.
+
+#.  **Creating classes/base.bbclass:** BitBake uses class files to
+    provide common code and functionality. The minimally required class
+    for BitBake is the ``classes/base.bbclass`` file. The ``base`` class
+    is implicitly inherited by every recipe. BitBake looks for the class
+    in the ``classes`` directory of the project (i.e ``hello/classes``
+    in this example).
+
+    Create the ``classes`` directory as follows: ::
+
+      $ cd $HOME/hello
+      $ mkdir classes
+
+    Move to the ``classes`` directory and then create the
+    ``base.bbclass`` file by inserting this single line: addtask build
+    The minimal task that BitBake runs is the ``do_build`` task. This is
+    all the example needs in order to build the project. Of course, the
+    ``base.bbclass`` can have much more depending on which build
+    environments BitBake is supporting.
+
+#.  **Run BitBake:** After making sure that the ``classes/base.bbclass``
+    file exists, you can run the ``bitbake`` command again: ::
+
+       $ bitbake
+       Nothing to do. Use 'bitbake world' to build everything, or run 'bitbake --help' for usage information.
+
+    BitBake is finally reporting
+    no errors. However, you can see that it really does not have
+    anything to do. You need to create a recipe that gives BitBake
+    something to do.
+
+#.  **Creating a Layer:** While it is not really necessary for such a
+    small example, it is good practice to create a layer in which to
+    keep your code separate from the general metadata used by BitBake.
+    Thus, this example creates and uses a layer called "mylayer".
+
+    .. note::
+
+       You can find additional information on layers in the
+       ":ref:`bitbake-user-manual/bitbake-user-manual-intro:Layers`" section.
+
+    Minimally, you need a recipe file and a layer configuration file in
+    your layer. The configuration file needs to be in the ``conf``
+    directory inside the layer. Use these commands to set up the layer
+    and the ``conf`` directory: ::
+
+       $ cd $HOME
+       $ mkdir mylayer
+       $ cd mylayer
+       $ mkdir conf
+
+    Move to the ``conf`` directory and create a ``layer.conf`` file that has the
+    following: ::
+
+      BBPATH .= ":${LAYERDIR}"
+      BBFILES += "${LAYERDIR}/\*.bb"
+      BBFILE_COLLECTIONS += "mylayer"
+     `BBFILE_PATTERN_mylayer := "^${LAYERDIR_RE}/"
+
+    For information on these variables, click on :term:`BBFILES`,
+    :term:`LAYERDIR`, :term:`BBFILE_COLLECTIONS` or :term:`BBFILE_PATTERN_mylayer <BBFILE_PATTERN>`
+    to go to the definitions in the glossary.
+
+    You need to create the recipe file next. Inside your layer at the
+    top-level, use an editor and create a recipe file named
+    ``printhello.bb`` that has the following: ::
+
+       DESCRIPTION = "Prints Hello World"
+       PN = 'printhello'
+       PV = '1'
+
+       python do_build() {
+          bb.plain("********************");
+          bb.plain("*                  *");
+          bb.plain("*  Hello, World!   *");
+          bb.plain("*                  *");
+          bb.plain("********************");
+       }
+
+    The recipe file simply provides
+    a description of the recipe, the name, version, and the ``do_build``
+    task, which prints out "Hello World" to the console. For more
+    information on :term:`DESCRIPTION`, :term:`PN` or :term:`PV`
+    follow the links to the glossary.
+
+#. **Run BitBake With a Target:** Now that a BitBake target exists, run
+    the command and provide that target: ::
+
+      $ cd $HOME/hello
+      $ bitbake printhello
+      ERROR: no recipe files to build, check your BBPATH and BBFILES?
+
+      Summary: There was 1 ERROR message shown, returning a non-zero exit code.
+
+    We have created the layer with the recipe and
+    the layer configuration file but it still seems that BitBake cannot
+    find the recipe. BitBake needs a ``conf/bblayers.conf`` that lists
+    the layers for the project. Without this file, BitBake cannot find
+    the recipe.
+
+#. **Creating conf/bblayers.conf:** BitBake uses the
+    ``conf/bblayers.conf`` file to locate layers needed for the project.
+    This file must reside in the ``conf`` directory of the project (i.e.
+    ``hello/conf`` for this example).
+
+    Set your working directory to the ``hello/conf`` directory and then
+    create the ``bblayers.conf`` file so that it contains the following: ::
+
+       BBLAYERS ?= " \
+           /home/<you>/mylayer \
+       "
+
+    You need to provide your own information for ``you`` in the file.
+
+#. **Run BitBake With a Target:** Now that you have supplied the
+    ``bblayers.conf`` file, run the ``bitbake`` command and provide the
+    target: ::
+
+       $ bitbake printhello
+       Parsing recipes: 100% |##################################################################################|
+       Time: 00:00:00
+       Parsing of 1 .bb files complete (0 cached, 1 parsed). 1 targets, 0 skipped, 0 masked, 0 errors.
+       NOTE: Resolving any missing task queue dependencies
+       NOTE: Preparing RunQueue
+       NOTE: Executing RunQueue Tasks
+       ********************
+       *                  *
+       *  Hello, World!   *
+       *                  *
+       ********************
+       NOTE: Tasks Summary: Attempted 1 tasks of which 0 didn't need to be rerun and all succeeded.
+
+    .. note::
+
+       After the first execution, re-running bitbake printhello again will not
+       result in a BitBake run that prints the same console output. The reason
+       for this is that the first time the printhello.bb recipe's do_build task
+       executes successfully, BitBake writes a stamp file for the task. Thus,
+       the next time you attempt to run the task using that same bitbake
+       command, BitBake notices the stamp and therefore determines that the task
+       does not need to be re-run. If you delete the tmp directory or run
+       bitbake -c clean printhello and then re-run the build, the "Hello,
+       World!" message will be printed again.

bitbake/doc/bitbake-user-manual/bitbake-user-manual-hello.xml

@@ -1,513 +0,0 @@
-<!DOCTYPE chapter PUBLIC "-//OASIS//DTD DocBook XML V4.2//EN"
-"http://www.oasis-open.org/docbook/xml/4.2/docbookx.dtd">
-
-<appendix id='hello-world-example'>
-    <title>Hello World Example</title>
-
-    <section id='bitbake-hello-world'>
-        <title>BitBake Hello World</title>
-
-        <para>
-            The simplest example commonly used to demonstrate any new
-            programming language or tool is the
-            "<ulink url="http://en.wikipedia.org/wiki/Hello_world_program">Hello World</ulink>"
-            example.
-            This appendix demonstrates, in tutorial form, Hello
-            World within the context of BitBake.
-            The tutorial describes how to create a new project
-            and the applicable metadata files necessary to allow
-            BitBake to build it.
-        </para>
-    </section>
-
-    <section id='example-obtaining-bitbake'>
-        <title>Obtaining BitBake</title>
-
-        <para>
-            See the
-            "<link linkend='obtaining-bitbake'>Obtaining BitBake</link>"
-            section for information on how to obtain BitBake.
-            Once you have the source code on your machine, the BitBake directory
-            appears as follows:
-            <literallayout class='monospaced'>
-     $ ls -al
-     total 100
-     drwxrwxr-x. 9 wmat wmat  4096 Jan 31 13:44 .
-     drwxrwxr-x. 3 wmat wmat  4096 Feb  4 10:45 ..
-     -rw-rw-r--. 1 wmat wmat   365 Nov 26 04:55 AUTHORS
-     drwxrwxr-x. 2 wmat wmat  4096 Nov 26 04:55 bin
-     drwxrwxr-x. 4 wmat wmat  4096 Jan 31 13:44 build
-     -rw-rw-r--. 1 wmat wmat 16501 Nov 26 04:55 ChangeLog
-     drwxrwxr-x. 2 wmat wmat  4096 Nov 26 04:55 classes
-     drwxrwxr-x. 2 wmat wmat  4096 Nov 26 04:55 conf
-     drwxrwxr-x. 3 wmat wmat  4096 Nov 26 04:55 contrib
-     -rw-rw-r--. 1 wmat wmat 17987 Nov 26 04:55 COPYING
-     drwxrwxr-x. 3 wmat wmat  4096 Nov 26 04:55 doc
-     -rw-rw-r--. 1 wmat wmat    69 Nov 26 04:55 .gitignore
-     -rw-rw-r--. 1 wmat wmat   849 Nov 26 04:55 HEADER
-     drwxrwxr-x. 5 wmat wmat  4096 Jan 31 13:44 lib
-     -rw-rw-r--. 1 wmat wmat   195 Nov 26 04:55 MANIFEST.in
-     -rw-rw-r--. 1 wmat wmat  2887 Nov 26 04:55 TODO
-            </literallayout>
-        </para>
-
-        <para>
-            At this point, you should have BitBake cloned to
-            a directory that matches the previous listing except for
-            dates and user names.
-        </para>
-    </section>
-
-    <section id='setting-up-the-bitbake-environment'>
-        <title>Setting Up the BitBake Environment</title>
-
-        <para>
-            First, you need to be sure that you can run BitBake.
-            Set your working directory to where your local BitBake
-            files are and run the following command:
-            <literallayout class='monospaced'>
-     $ ./bin/bitbake --version
-     BitBake Build Tool Core version 1.23.0, bitbake version 1.23.0
-            </literallayout>
-            The console output tells you what version you are running.
-        </para>
-
-        <para>
-            The recommended method to run BitBake is from a directory of your
-            choice.
-            To be able to run BitBake from any directory, you need to add the
-            executable binary to your binary to your shell's environment
-            <filename>PATH</filename> variable.
-            First, look at your current <filename>PATH</filename> variable
-            by entering the following:
-            <literallayout class='monospaced'>
-     $ echo $PATH
-            </literallayout>
-            Next, add the directory location for the BitBake binary to the
-            <filename>PATH</filename>.
-            Here is an example that adds the
-            <filename>/home/scott-lenovo/bitbake/bin</filename> directory
-            to the front of the <filename>PATH</filename> variable:
-            <literallayout class='monospaced'>
-     $ export PATH=/home/scott-lenovo/bitbake/bin:$PATH
-            </literallayout>
-            You should now be able to enter the <filename>bitbake</filename>
-            command from the command line while working from any directory.
-        </para>
-    </section>
-
-    <section id='the-hello-world-example'>
-        <title>The Hello World Example</title>
-
-        <para>
-            The overall goal of this exercise is to build a
-            complete "Hello World" example utilizing task and layer
-            concepts.
-            Because this is how modern projects such as OpenEmbedded and
-            the Yocto Project utilize BitBake, the example
-            provides an excellent starting point for understanding
-            BitBake.
-        </para>
-
-        <para>
-            To help you understand how to use BitBake to build targets,
-            the example starts with nothing but the <filename>bitbake</filename>
-            command, which causes BitBake to fail and report problems.
-            The example progresses by adding pieces to the build to
-            eventually conclude with a working, minimal "Hello World"
-            example.
-        </para>
-
-        <para>
-            While every attempt is made to explain what is happening during
-            the example, the descriptions cannot cover everything.
-            You can find further information throughout this manual.
-            Also, you can actively participate in the
-            <ulink url='http://lists.openembedded.org/mailman/listinfo/bitbake-devel'></ulink>
-            discussion mailing list about the BitBake build tool.
-        </para>
-
-        <note>
-            This example was inspired by and drew heavily from
-            <ulink url="http://www.mail-archive.com/yocto@yoctoproject.org/msg09379.html">Mailing List post - The BitBake equivalent of "Hello, World!"</ulink>.
-        </note>
-
-        <para>
-            As stated earlier, the goal of this example
-            is to eventually compile "Hello World".
-            However, it is unknown what BitBake needs and what you have
-            to provide in order to achieve that goal.
-            Recall that BitBake utilizes three types of metadata files:
-            <link linkend='configuration-files'>Configuration Files</link>,
-            <link linkend='classes'>Classes</link>, and
-            <link linkend='recipes'>Recipes</link>.
-            But where do they go?
-            How does BitBake find them?
-            BitBake's error messaging helps you answer these types of questions
-            and helps you better understand exactly what is going on.
-        </para>
-
-        <para>
-            Following is the complete "Hello World" example.
-        </para>
-
-        <orderedlist>
-            <listitem><para><emphasis>Create a Project Directory:</emphasis>
-                First, set up a directory for the "Hello World" project.
-                Here is how you can do so in your home directory:
-                <literallayout class='monospaced'>
-     $ mkdir ~/hello
-     $ cd ~/hello
-                </literallayout>
-                This is the directory that BitBake will use to do all of
-                its work.
-                You can use this directory to keep all the metafiles needed
-                by BitBake.
-                Having a project directory is a good way to isolate your
-                project.
-                </para></listitem>
-            <listitem><para><emphasis>Run Bitbake:</emphasis>
-                At this point, you have nothing but a project directory.
-                Run the <filename>bitbake</filename> command and see what
-                it does:
-                <literallayout class='monospaced'>
-     $ bitbake
-     The BBPATH variable is not set and bitbake did not
-     find a conf/bblayers.conf file in the expected location.
-     Maybe you accidentally invoked bitbake from the wrong directory?
-     DEBUG: Removed the following variables from the environment:
-     GNOME_DESKTOP_SESSION_ID, XDG_CURRENT_DESKTOP,
-     GNOME_KEYRING_CONTROL, DISPLAY, SSH_AGENT_PID, LANG, no_proxy,
-     XDG_SESSION_PATH, XAUTHORITY, SESSION_MANAGER, SHLVL,
-     MANDATORY_PATH, COMPIZ_CONFIG_PROFILE, WINDOWID, EDITOR,
-     GPG_AGENT_INFO, SSH_AUTH_SOCK, GDMSESSION, GNOME_KEYRING_PID,
-     XDG_SEAT_PATH, XDG_CONFIG_DIRS, LESSOPEN, DBUS_SESSION_BUS_ADDRESS,
-     _, XDG_SESSION_COOKIE, DESKTOP_SESSION, LESSCLOSE, DEFAULTS_PATH,
-     UBUNTU_MENUPROXY, OLDPWD, XDG_DATA_DIRS, COLORTERM, LS_COLORS
-                </literallayout>
-                The majority of this output is specific to environment variables
-                that are not directly relevant to BitBake.
-                However, the very first message regarding the
-                <filename>BBPATH</filename> variable and the
-                <filename>conf/bblayers.conf</filename> file
-                is relevant.</para>
-                <para>
-                When you run BitBake, it begins looking for metadata files.
-                The
-                <link linkend='var-bb-BBPATH'><filename>BBPATH</filename></link>
-                variable is what tells BitBake where to look for those files.
-                <filename>BBPATH</filename> is not set and you need to set it.
-                Without <filename>BBPATH</filename>, Bitbake cannot
-                find any configuration files (<filename>.conf</filename>)
-                or recipe files (<filename>.bb</filename>) at all.
-                BitBake also cannot find the <filename>bitbake.conf</filename>
-                file.
-                </para></listitem>
-            <listitem><para><emphasis>Setting <filename>BBPATH</filename>:</emphasis>
-                For this example, you can set <filename>BBPATH</filename>
-                in the same manner that you set <filename>PATH</filename>
-                earlier in the appendix.
-                You should realize, though, that it is much more flexible to set the
-                <filename>BBPATH</filename> variable up in a configuration
-                file for each project.</para>
-                <para>From your shell, enter the following commands to set and
-                export the <filename>BBPATH</filename> variable:
-                <literallayout class='monospaced'>
-     $ BBPATH="<replaceable>projectdirectory</replaceable>"
-     $ export BBPATH
-                </literallayout>
-                Use your actual project directory in the command.
-                BitBake uses that directory to find the metadata it needs for
-                your project.
-                <note>
-                    When specifying your project directory, do not use the
-                    tilde ("~") character as BitBake does not expand that character
-                    as the shell would.
-                </note>
-                </para></listitem>
-            <listitem><para><emphasis>Run Bitbake:</emphasis>
-                Now that you have <filename>BBPATH</filename> defined, run
-                the <filename>bitbake</filename> command again:
-                <literallayout class='monospaced'>
-     $ bitbake
-     ERROR: Traceback (most recent call last):
-       File "/home/scott-lenovo/bitbake/lib/bb/cookerdata.py", line 163, in wrapped
-         return func(fn, *args)
-       File "/home/scott-lenovo/bitbake/lib/bb/cookerdata.py", line 173, in parse_config_file
-         return bb.parse.handle(fn, data, include)
-       File "/home/scott-lenovo/bitbake/lib/bb/parse/__init__.py", line 99, in handle
-         return h['handle'](fn, data, include)
-       File "/home/scott-lenovo/bitbake/lib/bb/parse/parse_py/ConfHandler.py", line 120, in handle
-         abs_fn = resolve_file(fn, data)
-       File "/home/scott-lenovo/bitbake/lib/bb/parse/__init__.py", line 117, in resolve_file
-         raise IOError("file %s not found in %s" % (fn, bbpath))
-     IOError: file conf/bitbake.conf not found in /home/scott-lenovo/hello
-
-     ERROR: Unable to parse conf/bitbake.conf: file conf/bitbake.conf not found in /home/scott-lenovo/hello
-                </literallayout>
-                This sample output shows that BitBake could not find the
-                <filename>conf/bitbake.conf</filename> file in the project
-                directory.
-                This file is the first thing BitBake must find in order
-                to build a target.
-                And, since the project directory for this example is
-                empty, you need to provide a <filename>conf/bitbake.conf</filename>
-                file.
-                </para></listitem>
-            <listitem><para><emphasis>Creating <filename>conf/bitbake.conf</filename>:</emphasis>
-                The <filename>conf/bitbake.conf</filename> includes a number of
-                configuration variables BitBake uses for metadata and recipe
-                files.
-                For this example, you need to create the file in your project directory
-                and define some key BitBake variables.
-                For more information on the <filename>bitbake.conf</filename> file,
-                see
-                <ulink url='http://git.openembedded.org/bitbake/tree/conf/bitbake.conf'></ulink>.
-                </para>
-                <para>Use the following commands to create the <filename>conf</filename>
-                directory in the project directory:
-                <literallayout class='monospaced'>
-     $ mkdir conf
-                </literallayout>
-                From within the <filename>conf</filename> directory, use
-                some editor to create the <filename>bitbake.conf</filename>
-                so that it contains the following:
-                <literallayout class='monospaced'>
-     <link linkend='var-bb-PN'>PN</link>  = "${@bb.parse.BBHandler.vars_from_file(d.getVar('FILE', False),d)[0] or 'defaultpkgname'}"
-                </literallayout>
-                <literallayout class='monospaced'>
-     TMPDIR  = "${<link linkend='var-bb-TOPDIR'>TOPDIR</link>}/tmp"
-     <link linkend='var-bb-CACHE'>CACHE</link>   = "${TMPDIR}/cache"
-     <link linkend='var-bb-STAMP'>STAMP</link>   = "${TMPDIR}/${PN}/stamps"
-     <link linkend='var-bb-T'>T</link>       = "${TMPDIR}/${PN}/work"
-     <link linkend='var-bb-B'>B</link>       = "${TMPDIR}/${PN}"
-                </literallayout>
-                <note>
-                    Without a value for <filename>PN</filename>, the
-                    variables <filename>STAMP</filename>,
-                    <filename>T</filename>, and <filename>B</filename>,
-                    prevent more than one recipe from working. You can fix
-                    this by either setting <filename>PN</filename> to have
-                    a value similar to what OpenEmbedded and BitBake use
-                    in the default <filename>bitbake.conf</filename> file
-                    (see previous example). Or, by manually updating each
-                    recipe to set <filename>PN</filename>. You will also
-                    need to include <filename>PN</filename> as part of the
-                    <filename>STAMP</filename>, <filename>T</filename>, and
-                    <filename>B</filename> variable definitions in the
-                    <filename>local.conf</filename> file.
-                </note>
-                The <filename>TMPDIR</filename> variable establishes a directory
-                that BitBake uses for build output and intermediate files other
-                than the cached information used by the
-                <link linkend='setscene'>Setscene</link> process.
-                Here, the <filename>TMPDIR</filename> directory is set to
-                <filename>hello/tmp</filename>.
-                <note><title>Tip</title>
-                    You can always safely delete the <filename>tmp</filename>
-                    directory in order to rebuild a BitBake target.
-                    The build process creates the directory for you
-                    when you run BitBake.
-                </note></para>
-                <para>For information about each of the other variables defined in this
-                example, click on the links to take you to the definitions in
-                the glossary.
-                </para></listitem>
-            <listitem><para><emphasis>Run Bitbake:</emphasis>
-                After making sure that the <filename>conf/bitbake.conf</filename>
-                file exists, you can run the <filename>bitbake</filename>
-                command again:
-                <literallayout class='monospaced'>
-     $ bitbake
-     ERROR: Traceback (most recent call last):
-       File "/home/scott-lenovo/bitbake/lib/bb/cookerdata.py", line 163, in wrapped
-         return func(fn, *args)
-       File "/home/scott-lenovo/bitbake/lib/bb/cookerdata.py", line 177, in _inherit
-         bb.parse.BBHandler.inherit(bbclass, "configuration INHERITs", 0, data)
-       File "/home/scott-lenovo/bitbake/lib/bb/parse/parse_py/BBHandler.py", line 92, in inherit
-         include(fn, file, lineno, d, "inherit")
-       File "/home/scott-lenovo/bitbake/lib/bb/parse/parse_py/ConfHandler.py", line 100, in include
-         raise ParseError("Could not %(error_out)s file %(fn)s" % vars(), oldfn, lineno)
-     ParseError: ParseError in configuration INHERITs: Could not inherit file classes/base.bbclass
-
-     ERROR: Unable to parse base: ParseError in configuration INHERITs: Could not inherit file classes/base.bbclass
-                </literallayout>
-                In the sample output, BitBake could not find the
-                <filename>classes/base.bbclass</filename> file.
-                You need to create that file next.
-                </para></listitem>
-            <listitem><para><emphasis>Creating <filename>classes/base.bbclass</filename>:</emphasis>
-                BitBake uses class files to provide common code and functionality.
-                The minimally required class for BitBake is the
-                <filename>classes/base.bbclass</filename> file.
-                The <filename>base</filename> class is implicitly inherited by
-                every recipe.
-                BitBake looks for the class in the <filename>classes</filename>
-                directory of the project (i.e <filename>hello/classes</filename>
-                in this example).
-                </para>
-                <para>Create the <filename>classes</filename> directory as follows:
-                <literallayout class='monospaced'>
-     $ cd $HOME/hello
-     $ mkdir classes
-                </literallayout>
-                Move to the <filename>classes</filename> directory and then
-                create the <filename>base.bbclass</filename> file by inserting
-                this single line:
-                <literallayout class='monospaced'>
-     addtask build
-                </literallayout>
-                The minimal task that BitBake runs is the
-                <filename>do_build</filename> task.
-                This is all the example needs in order to build the project.
-                Of course, the <filename>base.bbclass</filename> can have much
-                more depending on which build environments BitBake is
-                supporting.
-                </para></listitem>
-            <listitem><para><emphasis>Run Bitbake:</emphasis>
-                After making sure that the <filename>classes/base.bbclass</filename>
-                file exists, you can run the <filename>bitbake</filename>
-                command again:
-                <literallayout class='monospaced'>
-     $ bitbake
-     Nothing to do.  Use 'bitbake world' to build everything, or run 'bitbake --help' for usage information.
-                </literallayout>
-                BitBake is finally reporting no errors.
-                However, you can see that it really does not have anything
-                to do.
-                You need to create a recipe that gives BitBake something to do.
-                </para></listitem>
-            <listitem><para><emphasis>Creating a Layer:</emphasis>
-                While it is not really necessary for such a small example,
-                it is good practice to create a layer in which to keep your
-                code separate from the general metadata used by BitBake.
-                Thus, this example creates and uses a layer called "mylayer".
-                <note>
-                    You can find additional information on layers in the
-                    "<link linkend='layers'>Layers</link>" section.
-                </note></para>
-
-                <para>Minimally, you need a recipe file and a layer configuration
-                file in your layer.
-                The configuration file needs to be in the <filename>conf</filename>
-                directory inside the layer.
-                Use these commands to set up the layer and the <filename>conf</filename>
-                directory:
-                <literallayout class='monospaced'>
-     $ cd $HOME
-     $ mkdir mylayer
-     $ cd mylayer
-     $ mkdir conf
-                </literallayout>
-                Move to the <filename>conf</filename> directory and create a
-                <filename>layer.conf</filename> file that has the following:
-                <literallayout class='monospaced'>
-     BBPATH .= ":${<link linkend='var-bb-LAYERDIR'>LAYERDIR</link>}"
-
-     <link linkend='var-bb-BBFILES'>BBFILES</link> += "${LAYERDIR}/*.bb"
-
-     <link linkend='var-bb-BBFILE_COLLECTIONS'>BBFILE_COLLECTIONS</link> += "mylayer"
-     <link linkend='var-bb-BBFILE_PATTERN'>BBFILE_PATTERN_mylayer</link> := "^${LAYERDIR_RE}/"
-                </literallayout>
-                For information on these variables, click the links
-                to go to the definitions in the glossary.</para>
-                <para>You need to create the recipe file next.
-                Inside your layer at the top-level, use an editor and create
-                a recipe file named <filename>printhello.bb</filename> that
-                has the following:
-                <literallayout class='monospaced'>
-     <link linkend='var-bb-DESCRIPTION'>DESCRIPTION</link> = "Prints Hello World"
-     <link linkend='var-bb-PN'>PN</link> = 'printhello'
-     <link linkend='var-bb-PV'>PV</link> = '1'
-
-     python do_build() {
-        bb.plain("********************");
-        bb.plain("*                  *");
-        bb.plain("*  Hello, World!   *");
-        bb.plain("*                  *");
-        bb.plain("********************");
-     }
-                </literallayout>
-                The recipe file simply provides a description of the
-                recipe, the name, version, and the <filename>do_build</filename>
-                task, which prints out "Hello World" to the console.
-                For more information on these variables, follow the links
-                to the glossary.
-                </para></listitem>
-            <listitem><para><emphasis>Run Bitbake With a Target:</emphasis>
-                Now that a BitBake target exists, run the command and provide
-                that target:
-                <literallayout class='monospaced'>
-     $ cd $HOME/hello
-     $ bitbake printhello
-     ERROR: no recipe files to build, check your BBPATH and BBFILES?
-
-     Summary: There was 1 ERROR message shown, returning a non-zero exit code.
-                </literallayout>
-                We have created the layer with the recipe and the layer
-                configuration file but it still seems that BitBake cannot
-                find the recipe.
-                BitBake needs a <filename>conf/bblayers.conf</filename> that
-                lists the layers for the project.
-                Without this file, BitBake cannot find the recipe.
-                </para></listitem>
-            <listitem><para><emphasis>Creating <filename>conf/bblayers.conf</filename>:</emphasis>
-                BitBake uses the <filename>conf/bblayers.conf</filename> file
-                to locate layers needed for the project.
-                This file must reside in the <filename>conf</filename> directory
-                of the project (i.e. <filename>hello/conf</filename> for this
-                example).</para>
-                <para>Set your working directory to the <filename>hello/conf</filename>
-                directory and then create the <filename>bblayers.conf</filename>
-                file so that it contains the following:
-                <literallayout class='monospaced'>
-     BBLAYERS ?= " \
-       /home/&lt;you&gt;/mylayer \
-       "
-                </literallayout>
-                You need to provide your own information for
-                <filename>you</filename> in the file.
-                </para></listitem>
-            <listitem><para><emphasis>Run Bitbake With a Target:</emphasis>
-                Now that you have supplied the <filename>bblayers.conf</filename>
-                file, run the <filename>bitbake</filename> command and provide
-                the target:
-                <literallayout class='monospaced'>
-     $ bitbake printhello
-     Parsing recipes: 100% |##################################################################################|
-     Time: 00:00:00
-     Parsing of 1 .bb files complete (0 cached, 1 parsed). 1 targets, 0 skipped, 0 masked, 0 errors.
-     NOTE: Resolving any missing task queue dependencies
-     NOTE: Preparing RunQueue
-     NOTE: Executing RunQueue Tasks
-     ********************
-     *                  *
-     *  Hello, World!   *
-     *                  *
-     ********************
-     NOTE: Tasks Summary: Attempted 1 tasks of which 0 didn't need to be rerun and all succeeded.
-                </literallayout>
-                BitBake finds the <filename>printhello</filename> recipe and
-                successfully runs the task.
-                <note>
-                    After the first execution, re-running
-                    <filename>bitbake printhello</filename> again will not
-                    result in a BitBake run that prints the same console
-                    output.
-                    The reason for this is that the first time the
-                    <filename>printhello.bb</filename> recipe's
-                    <filename>do_build</filename> task executes
-                    successfully, BitBake writes a stamp file for the task.
-                    Thus, the next time you attempt to run the task
-                    using that same <filename>bitbake</filename> command,
-                    BitBake notices the stamp and therefore determines
-                    that the task does not need to be re-run.
-                    If you delete the <filename>tmp</filename> directory
-                    or run <filename>bitbake -c clean printhello</filename>
-                    and then re-run the build, the "Hello, World!" message will
-                    be printed again.
-                </note>
-                </para></listitem>
-        </orderedlist>
-    </section>
-</appendix>

bitbake/doc/bitbake-user-manual/bitbake-user-manual-intro.rst

@@ -0,0 +1,651 @@
+.. SPDX-License-Identifier: CC-BY-2.5
+
+========
+Overview
+========
+
+|
+
+Welcome to the BitBake User Manual. This manual provides information on
+the BitBake tool. The information attempts to be as independent as
+possible regarding systems that use BitBake, such as OpenEmbedded and
+the Yocto Project. In some cases, scenarios or examples within the
+context of a build system are used in the manual to help with
+understanding. For these cases, the manual clearly states the context.
+
+.. _intro:
+
+Introduction
+============
+
+Fundamentally, BitBake is a generic task execution engine that allows
+shell and Python tasks to be run efficiently and in parallel while
+working within complex inter-task dependency constraints. One of
+BitBake's main users, OpenEmbedded, takes this core and builds embedded
+Linux software stacks using a task-oriented approach.
+
+Conceptually, BitBake is similar to GNU Make in some regards but has
+significant differences:
+
+-  BitBake executes tasks according to provided metadata that builds up
+   the tasks. Metadata is stored in recipe (``.bb``) and related recipe
+   "append" (``.bbappend``) files, configuration (``.conf``) and
+   underlying include (``.inc``) files, and in class (``.bbclass``)
+   files. The metadata provides BitBake with instructions on what tasks
+   to run and the dependencies between those tasks.
+
+-  BitBake includes a fetcher library for obtaining source code from
+   various places such as local files, source control systems, or
+   websites.
+
+-  The instructions for each unit to be built (e.g. a piece of software)
+   are known as "recipe" files and contain all the information about the
+   unit (dependencies, source file locations, checksums, description and
+   so on).
+
+-  BitBake includes a client/server abstraction and can be used from a
+   command line or used as a service over XML-RPC and has several
+   different user interfaces.
+
+History and Goals
+=================
+
+BitBake was originally a part of the OpenEmbedded project. It was
+inspired by the Portage package management system used by the Gentoo
+Linux distribution. On December 7, 2004, OpenEmbedded project team
+member Chris Larson split the project into two distinct pieces:
+
+-  BitBake, a generic task executor
+
+-  OpenEmbedded, a metadata set utilized by BitBake
+
+Today, BitBake is the primary basis of the
+`OpenEmbedded <http://www.openembedded.org/>`__ project, which is being
+used to build and maintain Linux distributions such as the `Angstrom
+Distribution <http://www.angstrom-distribution.org/>`__, and which is
+also being used as the build tool for Linux projects such as the `Yocto
+Project <http://www.yoctoproject.org>`__.
+
+Prior to BitBake, no other build tool adequately met the needs of an
+aspiring embedded Linux distribution. All of the build systems used by
+traditional desktop Linux distributions lacked important functionality,
+and none of the ad hoc Buildroot-based systems, prevalent in the
+embedded space, were scalable or maintainable.
+
+Some important original goals for BitBake were:
+
+-  Handle cross-compilation.
+
+-  Handle inter-package dependencies (build time on target architecture,
+   build time on native architecture, and runtime).
+
+-  Support running any number of tasks within a given package,
+   including, but not limited to, fetching upstream sources, unpacking
+   them, patching them, configuring them, and so forth.
+
+-  Be Linux distribution agnostic for both build and target systems.
+
+-  Be architecture agnostic.
+
+-  Support multiple build and target operating systems (e.g. Cygwin, the
+   BSDs, and so forth).
+
+-  Be self-contained, rather than tightly integrated into the build
+   machine's root filesystem.
+
+-  Handle conditional metadata on the target architecture, operating
+   system, distribution, and machine.
+
+-  Be easy to use the tools to supply local metadata and packages
+   against which to operate.
+
+-  Be easy to use BitBake to collaborate between multiple projects for
+   their builds.
+
+-  Provide an inheritance mechanism to share common metadata between
+   many packages.
+
+Over time it became apparent that some further requirements were
+necessary:
+
+-  Handle variants of a base recipe (e.g. native, sdk, and multilib).
+
+-  Split metadata into layers and allow layers to enhance or override
+   other layers.
+
+-  Allow representation of a given set of input variables to a task as a
+   checksum. Based on that checksum, allow acceleration of builds with
+   prebuilt components.
+
+BitBake satisfies all the original requirements and many more with
+extensions being made to the basic functionality to reflect the
+additional requirements. Flexibility and power have always been the
+priorities. BitBake is highly extensible and supports embedded Python
+code and execution of any arbitrary tasks.
+
+.. _Concepts:
+
+Concepts
+========
+
+BitBake is a program written in the Python language. At the highest
+level, BitBake interprets metadata, decides what tasks are required to
+run, and executes those tasks. Similar to GNU Make, BitBake controls how
+software is built. GNU Make achieves its control through "makefiles",
+while BitBake uses "recipes".
+
+BitBake extends the capabilities of a simple tool like GNU Make by
+allowing for the definition of much more complex tasks, such as
+assembling entire embedded Linux distributions.
+
+The remainder of this section introduces several concepts that should be
+understood in order to better leverage the power of BitBake.
+
+Recipes
+-------
+
+BitBake Recipes, which are denoted by the file extension ``.bb``, are
+the most basic metadata files. These recipe files provide BitBake with
+the following:
+
+-  Descriptive information about the package (author, homepage, license,
+   and so on)
+
+-  The version of the recipe
+
+-  Existing dependencies (both build and runtime dependencies)
+
+-  Where the source code resides and how to fetch it
+
+-  Whether the source code requires any patches, where to find them, and
+   how to apply them
+
+-  How to configure and compile the source code
+
+-  How to assemble the generated artifacts into one or more installable
+   packages
+
+-  Where on the target machine to install the package or packages
+   created
+
+Within the context of BitBake, or any project utilizing BitBake as its
+build system, files with the ``.bb`` extension are referred to as
+recipes.
+
+.. note::
+
+   The term "package" is also commonly used to describe recipes.
+   However, since the same word is used to describe packaged output from
+   a project, it is best to maintain a single descriptive term -
+   "recipes". Put another way, a single "recipe" file is quite capable
+   of generating a number of related but separately installable
+   "packages". In fact, that ability is fairly common.
+
+Configuration Files
+-------------------
+
+Configuration files, which are denoted by the ``.conf`` extension,
+define various configuration variables that govern the project's build
+process. These files fall into several areas that define machine
+configuration, distribution configuration, possible compiler tuning,
+general common configuration, and user configuration. The main
+configuration file is the sample ``bitbake.conf`` file, which is located
+within the BitBake source tree ``conf`` directory.
+
+Classes
+-------
+
+Class files, which are denoted by the ``.bbclass`` extension, contain
+information that is useful to share between metadata files. The BitBake
+source tree currently comes with one class metadata file called
+``base.bbclass``. You can find this file in the ``classes`` directory.
+The ``base.bbclass`` class files is special since it is always included
+automatically for all recipes and classes. This class contains
+definitions for standard basic tasks such as fetching, unpacking,
+configuring (empty by default), compiling (runs any Makefile present),
+installing (empty by default) and packaging (empty by default). These
+tasks are often overridden or extended by other classes added during the
+project development process.
+
+Layers
+------
+
+Layers allow you to isolate different types of customizations from each
+other. While you might find it tempting to keep everything in one layer
+when working on a single project, the more modular your metadata, the
+easier it is to cope with future changes.
+
+To illustrate how you can use layers to keep things modular, consider
+customizations you might make to support a specific target machine.
+These types of customizations typically reside in a special layer,
+rather than a general layer, called a Board Support Package (BSP) layer.
+Furthermore, the machine customizations should be isolated from recipes
+and metadata that support a new GUI environment, for example. This
+situation gives you a couple of layers: one for the machine
+configurations and one for the GUI environment. It is important to
+understand, however, that the BSP layer can still make machine-specific
+additions to recipes within the GUI environment layer without polluting
+the GUI layer itself with those machine-specific changes. You can
+accomplish this through a recipe that is a BitBake append
+(``.bbappend``) file.
+
+.. _append-bbappend-files:
+
+Append Files
+------------
+
+Append files, which are files that have the ``.bbappend`` file
+extension, extend or override information in an existing recipe file.
+
+BitBake expects every append file to have a corresponding recipe file.
+Furthermore, the append file and corresponding recipe file must use the
+same root filename. The filenames can differ only in the file type
+suffix used (e.g. ``formfactor_0.0.bb`` and
+``formfactor_0.0.bbappend``).
+
+Information in append files extends or overrides the information in the
+underlying, similarly-named recipe files.
+
+When you name an append file, you can use the "``%``" wildcard character
+to allow for matching recipe names. For example, suppose you have an
+append file named as follows: ::
+
+  busybox_1.21.%.bbappend
+
+That append file
+would match any ``busybox_1.21.``\ x\ ``.bb`` version of the recipe. So,
+the append file would match the following recipe names: ::
+
+  busybox_1.21.1.bb
+  busybox_1.21.2.bb
+  busybox_1.21.3.bb
+
+.. note::
+
+   The use of the " % " character is limited in that it only works directly in
+   front of the .bbappend portion of the append file's name. You cannot use the
+   wildcard character in any other location of the name.
+
+If the ``busybox`` recipe was updated to ``busybox_1.3.0.bb``, the
+append name would not match. However, if you named the append file
+``busybox_1.%.bbappend``, then you would have a match.
+
+In the most general case, you could name the append file something as
+simple as ``busybox_%.bbappend`` to be entirely version independent.
+
+Obtaining BitBake
+=================
+
+You can obtain BitBake several different ways:
+
+-  **Cloning BitBake:** Using Git to clone the BitBake source code
+   repository is the recommended method for obtaining BitBake. Cloning
+   the repository makes it easy to get bug fixes and have access to
+   stable branches and the master branch. Once you have cloned BitBake,
+   you should use the latest stable branch for development since the
+   master branch is for BitBake development and might contain less
+   stable changes.
+
+   You usually need a version of BitBake that matches the metadata you
+   are using. The metadata is generally backwards compatible but not
+   forward compatible.
+
+   Here is an example that clones the BitBake repository: ::
+
+     $ git clone git://git.openembedded.org/bitbake
+
+   This command clones the BitBake
+   Git repository into a directory called ``bitbake``. Alternatively,
+   you can designate a directory after the ``git clone`` command if you
+   want to call the new directory something other than ``bitbake``. Here
+   is an example that names the directory ``bbdev``: ::
+
+     $ git clone git://git.openembedded.org/bitbake bbdev
+
+-  **Installation using your Distribution Package Management System:**
+   This method is not recommended because the BitBake version that is
+   provided by your distribution, in most cases, is several releases
+   behind a snapshot of the BitBake repository.
+
+-  **Taking a snapshot of BitBake:** Downloading a snapshot of BitBake
+   from the source code repository gives you access to a known branch or
+   release of BitBake.
+
+      .. note::
+
+         Cloning the Git repository, as described earlier, is the preferred
+         method for getting BitBake. Cloning the repository makes it easier
+         to update as patches are added to the stable branches.
+
+   The following example downloads a snapshot of BitBake version 1.17.0: ::
+
+     $ wget http://git.openembedded.org/bitbake/snapshot/bitbake-1.17.0.tar.gz
+     $ tar zxpvf bitbake-1.17.0.tar.gz
+
+   After extraction of the tarball using
+   the tar utility, you have a directory entitled ``bitbake-1.17.0``.
+
+-  **Using the BitBake that Comes With Your Build Checkout:** A final
+   possibility for getting a copy of BitBake is that it already comes
+   with your checkout of a larger BitBake-based build system, such as
+   Poky. Rather than manually checking out individual layers and gluing
+   them together yourself, you can check out an entire build system. The
+   checkout will already include a version of BitBake that has been
+   thoroughly tested for compatibility with the other components. For
+   information on how to check out a particular BitBake-based build
+   system, consult that build system's supporting documentation.
+
+.. _bitbake-user-manual-command:
+
+The BitBake Command
+===================
+
+The ``bitbake`` command is the primary interface to the BitBake tool.
+This section presents the BitBake command syntax and provides several
+execution examples.
+
+Usage and syntax
+----------------
+
+Following is the usage and syntax for BitBake: ::
+
+   $ bitbake -h
+   Usage: bitbake [options] [recipename/target recipe:do_task ...]
+
+       Executes the specified task (default is 'build') for a given set of target recipes (.bb files).
+       It is assumed there is a conf/bblayers.conf available in cwd or in BBPATH which
+       will provide the layer, BBFILES and other configuration information.
+
+   Options:
+     --version             show program's version number and exit
+     -h, --help            show this help message and exit
+     -b BUILDFILE, --buildfile=BUILDFILE
+                           Execute tasks from a specific .bb recipe directly.
+                           WARNING: Does not handle any dependencies from other
+                           recipes.
+     -k, --continue        Continue as much as possible after an error. While the
+                           target that failed and anything depending on it cannot
+                           be built, as much as possible will be built before
+                           stopping.
+     -f, --force           Force the specified targets/task to run (invalidating
+                           any existing stamp file).
+     -c CMD, --cmd=CMD     Specify the task to execute. The exact options
+                           available depend on the metadata. Some examples might
+                           be 'compile' or 'populate_sysroot' or 'listtasks' may
+                           give a list of the tasks available.
+     -C INVALIDATE_STAMP, --clear-stamp=INVALIDATE_STAMP
+                           Invalidate the stamp for the specified task such as
+                           'compile' and then run the default task for the
+                           specified target(s).
+     -r PREFILE, --read=PREFILE
+                           Read the specified file before bitbake.conf.
+     -R POSTFILE, --postread=POSTFILE
+                           Read the specified file after bitbake.conf.
+     -v, --verbose         Enable tracing of shell tasks (with 'set -x'). Also
+                           print bb.note(...) messages to stdout (in addition to
+                           writing them to ${T}/log.do_&lt;task&gt;).
+     -D, --debug           Increase the debug level. You can specify this more
+                           than once. -D sets the debug level to 1, where only
+                           bb.debug(1, ...) messages are printed to stdout; -DD
+                           sets the debug level to 2, where both bb.debug(1, ...)
+                           and bb.debug(2, ...) messages are printed; etc.
+                           Without -D, no debug messages are printed. Note that
+                           -D only affects output to stdout. All debug messages
+                           are written to ${T}/log.do_taskname, regardless of the
+                           debug level.
+     -q, --quiet           Output less log message data to the terminal. You can
+                           specify this more than once.
+     -n, --dry-run         Don't execute, just go through the motions.
+     -S SIGNATURE_HANDLER, --dump-signatures=SIGNATURE_HANDLER
+                           Dump out the signature construction information, with
+                           no task execution. The SIGNATURE_HANDLER parameter is
+                           passed to the handler. Two common values are none and
+                           printdiff but the handler may define more/less. none
+                           means only dump the signature, printdiff means compare
+                           the dumped signature with the cached one.
+     -p, --parse-only      Quit after parsing the BB recipes.
+     -s, --show-versions   Show current and preferred versions of all recipes.
+     -e, --environment     Show the global or per-recipe environment complete
+                           with information about where variables were
+                           set/changed.
+     -g, --graphviz        Save dependency tree information for the specified
+                           targets in the dot syntax.
+     -I EXTRA_ASSUME_PROVIDED, --ignore-deps=EXTRA_ASSUME_PROVIDED
+                           Assume these dependencies don't exist and are already
+                           provided (equivalent to ASSUME_PROVIDED). Useful to
+                           make dependency graphs more appealing
+     -l DEBUG_DOMAINS, --log-domains=DEBUG_DOMAINS
+                           Show debug logging for the specified logging domains
+     -P, --profile         Profile the command and save reports.
+     -u UI, --ui=UI        The user interface to use (knotty, ncurses or taskexp
+                           - default knotty).
+     --token=XMLRPCTOKEN   Specify the connection token to be used when
+                           connecting to a remote server.
+     --revisions-changed   Set the exit code depending on whether upstream
+                           floating revisions have changed or not.
+     --server-only         Run bitbake without a UI, only starting a server
+                           (cooker) process.
+     -B BIND, --bind=BIND  The name/address for the bitbake xmlrpc server to bind
+                           to.
+     -T SERVER_TIMEOUT, --idle-timeout=SERVER_TIMEOUT
+                           Set timeout to unload bitbake server due to
+                           inactivity, set to -1 means no unload, default:
+                           Environment variable BB_SERVER_TIMEOUT.
+     --no-setscene         Do not run any setscene tasks. sstate will be ignored
+                           and everything needed, built.
+     --setscene-only       Only run setscene tasks, don't run any real tasks.
+     --remote-server=REMOTE_SERVER
+                           Connect to the specified server.
+     -m, --kill-server     Terminate any running bitbake server.
+     --observe-only        Connect to a server as an observing-only client.
+     --status-only         Check the status of the remote bitbake server.
+     -w WRITEEVENTLOG, --write-log=WRITEEVENTLOG
+                           Writes the event log of the build to a bitbake event
+                           json file. Use '' (empty string) to assign the name
+                           automatically.
+     --runall=RUNALL       Run the specified task for any recipe in the taskgraph
+                           of the specified target (even if it wouldn't otherwise
+                           have run).
+     --runonly=RUNONLY     Run only the specified task within the taskgraph of
+                           the specified targets (and any task dependencies those
+                           tasks may have).
+
+.. _bitbake-examples:
+
+Examples
+--------
+
+This section presents some examples showing how to use BitBake.
+
+.. _example-executing-a-task-against-a-single-recipe:
+
+Executing a Task Against a Single Recipe
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Executing tasks for a single recipe file is relatively simple. You
+specify the file in question, and BitBake parses it and executes the
+specified task. If you do not specify a task, BitBake executes the
+default task, which is "build". BitBake obeys inter-task dependencies
+when doing so.
+
+The following command runs the build task, which is the default task, on
+the ``foo_1.0.bb`` recipe file: ::
+
+  $ bitbake -b foo_1.0.bb
+
+The following command runs the clean task on the ``foo.bb`` recipe file: ::
+
+  $ bitbake -b foo.bb -c clean
+
+.. note::
+
+   The "-b" option explicitly does not handle recipe dependencies. Other
+   than for debugging purposes, it is instead recommended that you use
+   the syntax presented in the next section.
+
+Executing Tasks Against a Set of Recipe Files
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+There are a number of additional complexities introduced when one wants
+to manage multiple ``.bb`` files. Clearly there needs to be a way to
+tell BitBake what files are available and, of those, which you want to
+execute. There also needs to be a way for each recipe to express its
+dependencies, both for build-time and runtime. There must be a way for
+you to express recipe preferences when multiple recipes provide the same
+functionality, or when there are multiple versions of a recipe.
+
+The ``bitbake`` command, when not using "--buildfile" or "-b" only
+accepts a "PROVIDES". You cannot provide anything else. By default, a
+recipe file generally "PROVIDES" its "packagename" as shown in the
+following example: ::
+
+  $ bitbake foo
+
+This next example "PROVIDES" the
+package name and also uses the "-c" option to tell BitBake to just
+execute the ``do_clean`` task: ::
+
+  $ bitbake -c clean foo
+
+Executing a List of Task and Recipe Combinations
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+The BitBake command line supports specifying different tasks for
+individual targets when you specify multiple targets. For example,
+suppose you had two targets (or recipes) ``myfirstrecipe`` and
+``mysecondrecipe`` and you needed BitBake to run ``taskA`` for the first
+recipe and ``taskB`` for the second recipe: ::
+
+  $ bitbake myfirstrecipe:do_taskA mysecondrecipe:do_taskB
+
+Generating Dependency Graphs
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+BitBake is able to generate dependency graphs using the ``dot`` syntax.
+You can convert these graphs into images using the ``dot`` tool from
+`Graphviz <http://www.graphviz.org>`__.
+
+When you generate a dependency graph, BitBake writes two files to the
+current working directory:
+
+-  ``task-depends.dot``: Shows dependencies between tasks. These
+   dependencies match BitBake's internal task execution list.
+
+-  ``pn-buildlist``: Shows a simple list of targets that are to be
+   built.
+
+To stop depending on common depends, use the "-I" depend option and
+BitBake omits them from the graph. Leaving this information out can
+produce more readable graphs. This way, you can remove from the graph
+``DEPENDS`` from inherited classes such as ``base.bbclass``.
+
+Here are two examples that create dependency graphs. The second example
+omits depends common in OpenEmbedded from the graph: ::
+
+  $ bitbake -g foo
+
+  $ bitbake -g -I virtual/kernel -I eglibc foo
+
+Executing a Multiple Configuration Build
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+BitBake is able to build multiple images or packages using a single
+command where the different targets require different configurations
+(multiple configuration builds). Each target, in this scenario, is
+referred to as a "multiconfig".
+
+To accomplish a multiple configuration build, you must define each
+target's configuration separately using a parallel configuration file in
+the build directory. The location for these multiconfig configuration
+files is specific. They must reside in the current build directory in a
+sub-directory of ``conf`` named ``multiconfig``. Following is an example
+for two separate targets:
+
+.. image:: figures/bb_multiconfig_files.png
+   :align: center
+
+The reason for this required file hierarchy is because the ``BBPATH``
+variable is not constructed until the layers are parsed. Consequently,
+using the configuration file as a pre-configuration file is not possible
+unless it is located in the current working directory.
+
+Minimally, each configuration file must define the machine and the
+temporary directory BitBake uses for the build. Suggested practice
+dictates that you do not overlap the temporary directories used during
+the builds.
+
+Aside from separate configuration files for each target, you must also
+enable BitBake to perform multiple configuration builds. Enabling is
+accomplished by setting the
+:term:`BBMULTICONFIG` variable in the
+``local.conf`` configuration file. As an example, suppose you had
+configuration files for ``target1`` and ``target2`` defined in the build
+directory. The following statement in the ``local.conf`` file both
+enables BitBake to perform multiple configuration builds and specifies
+the two extra multiconfigs: ::
+
+  BBMULTICONFIG = "target1 target2"
+
+Once the target configuration files are in place and BitBake has been
+enabled to perform multiple configuration builds, use the following
+command form to start the builds: ::
+
+  $ bitbake [mc:multiconfigname:]target [[[mc:multiconfigname:]target] ... ]
+
+Here is an example for two extra multiconfigs: ``target1`` and ``target2``: ::
+
+  $ bitbake mc::target mc:target1:target mc:target2:target
+
+.. _bb-enabling-multiple-configuration-build-dependencies:
+
+Enabling Multiple Configuration Build Dependencies
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Sometimes dependencies can exist between targets (multiconfigs) in a
+multiple configuration build. For example, suppose that in order to
+build an image for a particular architecture, the root filesystem of
+another build for a different architecture needs to exist. In other
+words, the image for the first multiconfig depends on the root
+filesystem of the second multiconfig. This dependency is essentially
+that the task in the recipe that builds one multiconfig is dependent on
+the completion of the task in the recipe that builds another
+multiconfig.
+
+To enable dependencies in a multiple configuration build, you must
+declare the dependencies in the recipe using the following statement
+form: ::
+
+  task_or_package[mcdepends] = "mc:from_multiconfig:to_multiconfig:recipe_name:task_on_which_to_depend"
+
+To better show how to use this statement, consider an example with two
+multiconfigs: ``target1`` and ``target2``: ::
+
+  image_task[mcdepends] = "mc:target1:target2:image2:rootfs_task"
+
+In this example, the
+``from_multiconfig`` is "target1" and the ``to_multiconfig`` is "target2". The
+task on which the image whose recipe contains image_task depends on the
+completion of the rootfs_task used to build out image2, which is
+associated with the "target2" multiconfig.
+
+Once you set up this dependency, you can build the "target1" multiconfig
+using a BitBake command as follows: ::
+
+  $ bitbake mc:target1:image1
+
+This command executes all the tasks needed to create ``image1`` for the "target1"
+multiconfig. Because of the dependency, BitBake also executes through
+the ``rootfs_task`` for the "target2" multiconfig build.
+
+Having a recipe depend on the root filesystem of another build might not
+seem that useful. Consider this change to the statement in the image1
+recipe: ::
+
+  image_task[mcdepends] = "mc:target1:target2:image2:image_task"
+
+In this case, BitBake must create ``image2`` for the "target2" build since
+the "target1" build depends on it.
+
+Because "target1" and "target2" are enabled for multiple configuration
+builds and have separate configuration files, BitBake places the
+artifacts for each build in the respective temporary build directories.

bitbake/doc/bitbake-user-manual/bitbake-user-manual-intro.xml

@@ -1,894 +0,0 @@
-<!DOCTYPE chapter PUBLIC "-//OASIS//DTD DocBook XML V4.2//EN"
-    "http://www.oasis-open.org/docbook/xml/4.2/docbookx.dtd">
-
-<chapter id="bitbake-user-manual-intro">
-    <title>Overview</title>
-
-    <para>
-        Welcome to the BitBake User Manual.
-        This manual provides information on the BitBake tool.
-        The information attempts to be as independent as possible regarding
-        systems that use BitBake, such as OpenEmbedded and the
-        Yocto Project.
-        In some cases, scenarios or examples within the context of
-        a build system are used in the manual to help with understanding.
-        For these cases, the manual clearly states the context.
-    </para>
-
-    <section id="intro">
-        <title>Introduction</title>
-
-        <para>
-            Fundamentally, BitBake is a generic task execution
-            engine that allows shell and Python tasks to be run
-            efficiently and in parallel while working within
-            complex inter-task dependency constraints.
-            One of BitBake's main users, OpenEmbedded, takes this core
-            and builds embedded Linux software stacks using
-            a task-oriented approach.
-        </para>
-
-        <para>
-            Conceptually, BitBake is similar to GNU Make in
-            some regards but has significant differences:
-            <itemizedlist>
-                <listitem><para>
-                    BitBake executes tasks according to provided
-                    metadata that builds up the tasks.
-                    Metadata is stored in recipe (<filename>.bb</filename>)
-                    and related recipe "append" (<filename>.bbappend</filename>)
-                    files, configuration (<filename>.conf</filename>) and
-                    underlying include (<filename>.inc</filename>) files, and
-                    in class (<filename>.bbclass</filename>) files.
-                    The metadata provides
-                    BitBake with instructions on what tasks to run and
-                    the dependencies between those tasks.
-                    </para></listitem>
-                <listitem><para>
-                    BitBake includes a fetcher library for obtaining source
-                    code from various places such as local files, source control
-                    systems, or websites.
-                    </para></listitem>
-                <listitem><para>
-                    The instructions for each unit to be built (e.g. a piece
-                    of software) are known as "recipe" files and
-                    contain all the information about the unit
-                    (dependencies, source file locations, checksums, description
-                    and so on).
-                    </para></listitem>
-                <listitem><para>
-                    BitBake includes a client/server abstraction and can
-                    be used from a command line or used as a service over
-                    XML-RPC and has several different user interfaces.
-                    </para></listitem>
-            </itemizedlist>
-        </para>
-    </section>
-
-    <section id="history-and-goals">
-        <title>History and Goals</title>
-
-        <para>
-            BitBake was originally a part of the OpenEmbedded project.
-            It was inspired by the Portage package management system
-            used by the Gentoo Linux distribution.
-            On December 7, 2004, OpenEmbedded project team member
-            Chris Larson split the project into two distinct pieces:
-            <itemizedlist>
-                <listitem><para>BitBake, a generic task executor</para></listitem>
-                <listitem><para>OpenEmbedded, a metadata set utilized by
-                    BitBake</para></listitem>
-            </itemizedlist>
-            Today, BitBake is the primary basis of the
-            <ulink url="http://www.openembedded.org/">OpenEmbedded</ulink>
-            project, which is being used to build and maintain Linux
-            distributions such as the
-            <ulink url='http://www.angstrom-distribution.org/'>Angstrom Distribution</ulink>,
-            and which is also being used as the build tool for Linux projects
-            such as the
-            <ulink url='http://www.yoctoproject.org'>Yocto Project</ulink>.
-        </para>
-
-        <para>
-            Prior to BitBake, no other build tool adequately met the needs of
-            an aspiring embedded Linux distribution.
-            All of the build systems used by traditional desktop Linux
-            distributions lacked important functionality, and none of the
-            ad hoc Buildroot-based systems, prevalent in the
-            embedded space, were scalable or maintainable.
-        </para>
-
-        <para>
-            Some important original goals for BitBake were:
-            <itemizedlist>
-                <listitem><para>
-                    Handle cross-compilation.
-                    </para></listitem>
-                <listitem><para>
-                    Handle inter-package dependencies (build time on
-                    target architecture, build time on native
-                    architecture, and runtime).
-                    </para></listitem>
-                <listitem><para>
-                    Support running any number of tasks within a given
-                    package, including, but not limited to, fetching
-                    upstream sources, unpacking them, patching them,
-                    configuring them, and so forth.
-                    </para></listitem>
-                <listitem><para>
-                    Be Linux distribution agnostic for both build and
-                    target systems.
-                    </para></listitem>
-                <listitem><para>
-                    Be architecture agnostic.
-                    </para></listitem>
-                <listitem><para>
-                    Support multiple build and target operating systems
-                    (e.g. Cygwin, the BSDs, and so forth).
-                    </para></listitem>
-                <listitem><para>
-                    Be self contained, rather than tightly
-                    integrated into the build machine's root
-                    filesystem.
-                    </para></listitem>
-                <listitem><para>
-                    Handle conditional metadata on the target architecture,
-                    operating system, distribution, and machine.
-                    </para></listitem>
-                <listitem><para>
-                    Be easy to use the tools to supply local metadata and packages
-                    against which to operate.
-                    </para></listitem>
-                <listitem><para>
-                    Be easy to use BitBake to collaborate between multiple
-                    projects for their builds.
-                    </para></listitem>
-                <listitem><para>
-                    Provide an inheritance mechanism to share
-                    common metadata between many packages.
-                    </para></listitem>
-            </itemizedlist>
-            Over time it became apparent that some further requirements
-            were necessary:
-            <itemizedlist>
-                <listitem><para>
-                    Handle variants of a base recipe (e.g. native, sdk,
-                    and multilib).
-                    </para></listitem>
-                <listitem><para>
-                    Split metadata into layers and allow layers
-                    to enhance or override other layers.
-                    </para></listitem>
-                <listitem><para>
-                    Allow representation of a given set of input variables
-                    to a task as a checksum.
-                    Based on that checksum, allow acceleration of builds
-                    with prebuilt components.
-                    </para></listitem>
-            </itemizedlist>
-            BitBake satisfies all the original requirements and many more
-            with extensions being made to the basic functionality to
-            reflect the additional requirements.
-            Flexibility and power have always been the priorities.
-            BitBake is highly extensible and supports embedded Python code and
-            execution of any arbitrary tasks.
-        </para>
-    </section>
-
-    <section id="Concepts">
-        <title>Concepts</title>
-
-        <para>
-            BitBake is a program written in the Python language.
-            At the highest level, BitBake interprets metadata, decides
-            what tasks are required to run, and executes those tasks.
-            Similar to GNU Make, BitBake controls how software is
-            built.
-            GNU Make achieves its control through "makefiles", while
-            BitBake uses "recipes".
-        </para>
-
-        <para>
-            BitBake extends the capabilities of a simple
-            tool like GNU Make by allowing for the definition of much more
-            complex tasks, such as assembling entire embedded Linux
-            distributions.
-        </para>
-
-        <para>
-            The remainder of this section introduces several concepts
-            that should be understood in order to better leverage
-            the power of BitBake.
-        </para>
-
-        <section id='recipes'>
-            <title>Recipes</title>
-
-            <para>
-                BitBake Recipes, which are denoted by the file extension
-                <filename>.bb</filename>, are the most basic metadata files.
-                These recipe files provide BitBake with the following:
-                <itemizedlist>
-                    <listitem><para>Descriptive information about the
-                        package (author, homepage, license, and so on)</para></listitem>
-                    <listitem><para>The version of the recipe</para></listitem>
-                    <listitem><para>Existing dependencies (both build
-                        and runtime dependencies)</para></listitem>
-                    <listitem><para>Where the source code resides and
-                        how to fetch it</para></listitem>
-                    <listitem><para>Whether the source code requires
-                        any patches, where to find them, and how to apply
-                        them</para></listitem>
-                    <listitem><para>How to configure and compile the
-                        source code</para></listitem>
-                    <listitem><para>Where on the target machine to install the
-                        package or packages created</para></listitem>
-                </itemizedlist>
-            </para>
-
-            <para>
-                Within the context of BitBake, or any project utilizing BitBake
-                as its build system, files with the <filename>.bb</filename>
-                extension are referred to as recipes.
-                <note>
-                    The term "package" is also commonly used to describe recipes.
-                    However, since the same word is used to describe packaged
-                    output from a project, it is best to maintain a single
-                    descriptive term - "recipes".
-                    Put another way, a single "recipe" file is quite capable
-                    of generating a number of related but separately installable
-                    "packages".
-                    In fact, that ability is fairly common.
-                </note>
-            </para>
-        </section>
-
-        <section id='configuration-files'>
-            <title>Configuration Files</title>
-
-            <para>
-                Configuration files, which are denoted by the
-                <filename>.conf</filename> extension, define
-                various configuration variables that govern the project's build
-                process.
-                These files fall into several areas that define
-                machine configuration options, distribution configuration
-                options, compiler tuning options, general common
-                configuration options, and user configuration options.
-                The main configuration file is the sample
-                <filename>bitbake.conf</filename> file, which is
-                located within the BitBake source tree
-                <filename>conf</filename> directory.
-            </para>
-        </section>
-
-        <section id='classes'>
-            <title>Classes</title>
-
-            <para>
-                Class files, which are denoted by the
-                <filename>.bbclass</filename> extension, contain
-                information that is useful to share between metadata files.
-                The BitBake source tree currently comes with one class metadata file
-                called <filename>base.bbclass</filename>.
-                You can find this file in the
-                <filename>classes</filename> directory.
-                The <filename>base.bbclass</filename> class files is special since it
-                is always included automatically for all recipes
-                and classes.
-                This class contains definitions for standard basic tasks such
-                as fetching, unpacking, configuring (empty by default),
-                compiling (runs any Makefile present), installing (empty by
-                default) and packaging (empty by default).
-                These tasks are often overridden or extended by other classes
-                added during the project development process.
-            </para>
-        </section>
-
-        <section id='layers'>
-            <title>Layers</title>
-
-            <para>
-                Layers allow you to isolate different types of
-                customizations from each other.
-                While you might find it tempting to keep everything in one layer
-                when working on a single project, the more modular you organize
-                your metadata, the easier it is to cope with future changes.
-            </para>
-
-            <para>
-                To illustrate how you can use layers to keep things modular,
-                consider customizations you might make to support a specific target machine.
-                These types of customizations typically reside in a special layer,
-                rather than a general layer, called a Board Support Package (BSP)
-                Layer.
-                Furthermore, the machine customizations should be isolated from
-                recipes and metadata that support a new GUI environment, for
-                example.
-                This situation gives you a couple of layers: one for the machine
-                configurations and one for the GUI environment.
-                It is important to understand, however, that the BSP layer can still
-                make machine-specific additions to recipes within
-                the GUI environment layer without polluting the GUI layer itself
-                with those machine-specific changes.
-                You can accomplish this through a recipe that is a BitBake append
-                (<filename>.bbappend</filename>) file.
-            </para>
-        </section>
-
-        <section id='append-bbappend-files'>
-            <title>Append Files</title>
-
-            <para>
-                Append files, which are files that have the
-                <filename>.bbappend</filename> file extension, extend or
-                override information in an existing recipe file.
-            </para>
-
-            <para>
-                BitBake expects every append file to have a corresponding recipe file.
-                Furthermore, the append file and corresponding recipe file
-                must use the same root filename.
-                The filenames can differ only in the file type suffix used
-                (e.g. <filename>formfactor_0.0.bb</filename> and
-                <filename>formfactor_0.0.bbappend</filename>).
-            </para>
-
-            <para>
-                Information in append files extends or
-                overrides the information in the underlying,
-                similarly-named recipe files.
-            </para>
-
-            <para>
-                When you name an append file, you can use the
-                "<filename>%</filename>" wildcard character to allow for matching
-                recipe names.
-                For example, suppose you have an append file named
-                as follows:
-                <literallayout class='monospaced'>
-     busybox_1.21.%.bbappend
-                </literallayout>
-                That append file would match any <filename>busybox_1.21.</filename><replaceable>x</replaceable><filename>.bb</filename>
-                version of the recipe.
-                So, the append file would match the following recipe names:
-                <literallayout class='monospaced'>
-     busybox_1.21.1.bb
-     busybox_1.21.2.bb
-     busybox_1.21.3.bb
-                </literallayout>
-                <note><title>Important</title>
-                    The use of the "<filename>%</filename>" character
-                    is limited in that it only works directly in front of the
-                    <filename>.bbappend</filename> portion of the append file's
-                    name.
-                    You cannot use the wildcard character in any other
-                    location of the name.
-                </note>
-                If the <filename>busybox</filename> recipe was updated to
-                <filename>busybox_1.3.0.bb</filename>, the append name would not
-                match.
-                However, if you named the append file
-                <filename>busybox_1.%.bbappend</filename>, then you would have a match.
-            </para>
-
-            <para>
-                In the most general case, you could name the append file something as
-                simple as <filename>busybox_%.bbappend</filename> to be entirely
-                version independent.
-            </para>
-        </section>
-    </section>
-
-    <section id='obtaining-bitbake'>
-        <title>Obtaining BitBake</title>
-
-        <para>
-            You can obtain BitBake several different ways:
-            <itemizedlist>
-                <listitem><para><emphasis>Cloning BitBake:</emphasis>
-                    Using Git to clone the BitBake source code repository
-                    is the recommended method for obtaining BitBake.
-                    Cloning the repository makes it easy to get bug fixes
-                    and have access to stable branches and the master
-                    branch.
-                    Once you have cloned BitBake, you should use
-                    the latest stable
-                    branch for development since the master branch is for
-                    BitBake development and might contain less stable changes.
-                    </para>
-                    <para>You usually need a version of BitBake
-                    that matches the metadata you are using.
-                    The metadata is generally backwards compatible but
-                    not forward compatible.</para>
-                    <para>Here is an example that clones the BitBake repository:
-                    <literallayout class='monospaced'>
-     $ git clone git://git.openembedded.org/bitbake
-                    </literallayout>
-                    This command clones the BitBake Git repository into a
-                    directory called <filename>bitbake</filename>.
-                    Alternatively, you can
-                    designate a directory after the
-                    <filename>git clone</filename> command
-                    if you want to call the new directory something
-                    other than <filename>bitbake</filename>.
-                    Here is an example that names the directory
-                    <filename>bbdev</filename>:
-                    <literallayout class='monospaced'>
-     $ git clone git://git.openembedded.org/bitbake bbdev
-                    </literallayout></para></listitem>
-                <listitem><para><emphasis>Installation using your Distribution
-                    Package Management System:</emphasis>
-                    This method is not
-                    recommended because the BitBake version that is
-                    provided by your distribution, in most cases,
-                    is several
-                    releases behind a snapshot of the BitBake repository.
-                    </para></listitem>
-                <listitem><para><emphasis>Taking a snapshot of BitBake:</emphasis>
-                    Downloading a snapshot of BitBake from the
-                    source code repository gives you access to a known
-                    branch or release of BitBake.
-                    <note>
-                         Cloning the Git repository, as described earlier,
-                         is the preferred method for getting BitBake.
-                         Cloning the repository makes it easier to update as
-                         patches are added to the stable branches.
-                    </note></para>
-                    <para>The following example downloads a snapshot of
-                    BitBake version 1.17.0:
-                    <literallayout class='monospaced'>
-     $ wget http://git.openembedded.org/bitbake/snapshot/bitbake-1.17.0.tar.gz
-     $ tar zxpvf bitbake-1.17.0.tar.gz
-                    </literallayout>
-                    After extraction of the tarball using the tar utility,
-                    you have a directory entitled
-                    <filename>bitbake-1.17.0</filename>.
-                    </para></listitem>
-                <listitem><para><emphasis>Using the BitBake that Comes With Your
-                    Build Checkout:</emphasis>
-                    A final possibility for getting a copy of BitBake is that it
-                    already comes with your checkout of a larger Bitbake-based build
-                    system, such as Poky.
-                    Rather than manually checking out individual layers and
-                    gluing them together yourself, you can check
-                    out an entire build system.
-                    The checkout will already include a version of BitBake that
-                    has been thoroughly tested for compatibility with the other
-                    components.
-                    For information on how to check out a particular BitBake-based
-                    build system, consult that build system's supporting documentation.
-                    </para></listitem>
-            </itemizedlist>
-        </para>
-    </section>
-
-    <section id="bitbake-user-manual-command">
-        <title>The BitBake Command</title>
-
-        <para>
-            The <filename>bitbake</filename> command is the primary interface
-            to the BitBake tool.
-            This section presents the BitBake command syntax and provides
-            several execution examples.
-        </para>
-
-        <section id='usage-and-syntax'>
-            <title>Usage and syntax</title>
-
-            <para>
-                Following is the usage and syntax for BitBake:
-                <literallayout class='monospaced'>
-     $ bitbake -h
-     Usage: bitbake [options] [recipename/target recipe:do_task ...]
-
-         Executes the specified task (default is 'build') for a given set of target recipes (.bb files).
-         It is assumed there is a conf/bblayers.conf available in cwd or in BBPATH which
-         will provide the layer, BBFILES and other configuration information.
-
-     Options:
-       --version             show program's version number and exit
-       -h, --help            show this help message and exit
-       -b BUILDFILE, --buildfile=BUILDFILE
-                             Execute tasks from a specific .bb recipe directly.
-                             WARNING: Does not handle any dependencies from other
-                             recipes.
-       -k, --continue        Continue as much as possible after an error. While the
-                             target that failed and anything depending on it cannot
-                             be built, as much as possible will be built before
-                             stopping.
-       -f, --force           Force the specified targets/task to run (invalidating
-                             any existing stamp file).
-       -c CMD, --cmd=CMD     Specify the task to execute. The exact options
-                             available depend on the metadata. Some examples might
-                             be 'compile' or 'populate_sysroot' or 'listtasks' may
-                             give a list of the tasks available.
-       -C INVALIDATE_STAMP, --clear-stamp=INVALIDATE_STAMP
-                             Invalidate the stamp for the specified task such as
-                             'compile' and then run the default task for the
-                             specified target(s).
-       -r PREFILE, --read=PREFILE
-                             Read the specified file before bitbake.conf.
-       -R POSTFILE, --postread=POSTFILE
-                             Read the specified file after bitbake.conf.
-       -v, --verbose         Enable tracing of shell tasks (with 'set -x'). Also
-                             print bb.note(...) messages to stdout (in addition to
-                             writing them to ${T}/log.do_&lt;task&gt;).
-       -D, --debug           Increase the debug level. You can specify this more
-                             than once. -D sets the debug level to 1, where only
-                             bb.debug(1, ...) messages are printed to stdout; -DD
-                             sets the debug level to 2, where both bb.debug(1, ...)
-                             and bb.debug(2, ...) messages are printed; etc.
-                             Without -D, no debug messages are printed. Note that
-                             -D only affects output to stdout. All debug messages
-                             are written to ${T}/log.do_taskname, regardless of the
-                             debug level.
-       -q, --quiet           Output less log message data to the terminal. You can
-                             specify this more than once.
-       -n, --dry-run         Don't execute, just go through the motions.
-       -S SIGNATURE_HANDLER, --dump-signatures=SIGNATURE_HANDLER
-                             Dump out the signature construction information, with
-                             no task execution. The SIGNATURE_HANDLER parameter is
-                             passed to the handler. Two common values are none and
-                             printdiff but the handler may define more/less. none
-                             means only dump the signature, printdiff means compare
-                             the dumped signature with the cached one.
-       -p, --parse-only      Quit after parsing the BB recipes.
-       -s, --show-versions   Show current and preferred versions of all recipes.
-       -e, --environment     Show the global or per-recipe environment complete
-                             with information about where variables were
-                             set/changed.
-       -g, --graphviz        Save dependency tree information for the specified
-                             targets in the dot syntax.
-       -I EXTRA_ASSUME_PROVIDED, --ignore-deps=EXTRA_ASSUME_PROVIDED
-                             Assume these dependencies don't exist and are already
-                             provided (equivalent to ASSUME_PROVIDED). Useful to
-                             make dependency graphs more appealing
-       -l DEBUG_DOMAINS, --log-domains=DEBUG_DOMAINS
-                             Show debug logging for the specified logging domains
-       -P, --profile         Profile the command and save reports.
-       -u UI, --ui=UI        The user interface to use (knotty, ncurses or taskexp
-                             - default knotty).
-       --token=XMLRPCTOKEN   Specify the connection token to be used when
-                             connecting to a remote server.
-       --revisions-changed   Set the exit code depending on whether upstream
-                             floating revisions have changed or not.
-       --server-only         Run bitbake without a UI, only starting a server
-                             (cooker) process.
-       -B BIND, --bind=BIND  The name/address for the bitbake xmlrpc server to bind
-                             to.
-       -T SERVER_TIMEOUT, --idle-timeout=SERVER_TIMEOUT
-                             Set timeout to unload bitbake server due to
-                             inactivity, set to -1 means no unload, default:
-                             Environment variable BB_SERVER_TIMEOUT.
-       --no-setscene         Do not run any setscene tasks. sstate will be ignored
-                             and everything needed, built.
-       --setscene-only       Only run setscene tasks, don't run any real tasks.
-       --remote-server=REMOTE_SERVER
-                             Connect to the specified server.
-       -m, --kill-server     Terminate any running bitbake server.
-       --observe-only        Connect to a server as an observing-only client.
-       --status-only         Check the status of the remote bitbake server.
-       -w WRITEEVENTLOG, --write-log=WRITEEVENTLOG
-                             Writes the event log of the build to a bitbake event
-                             json file. Use '' (empty string) to assign the name
-                             automatically.
-       --runall=RUNALL       Run the specified task for any recipe in the taskgraph
-                             of the specified target (even if it wouldn't otherwise
-                             have run).
-       --runonly=RUNONLY     Run only the specified task within the taskgraph of
-                             the specified targets (and any task dependencies those
-                             tasks may have).
-                </literallayout>
-            </para>
-        </section>
-
-        <section id='bitbake-examples'>
-            <title>Examples</title>
-
-            <para>
-                This section presents some examples showing how to use BitBake.
-            </para>
-
-            <section id='example-executing-a-task-against-a-single-recipe'>
-                <title>Executing a Task Against a Single Recipe</title>
-
-                <para>
-                    Executing tasks for a single recipe file is relatively simple.
-                    You specify the file in question, and BitBake parses
-                    it and executes the specified task.
-                    If you do not specify a task, BitBake executes the default
-                    task, which is "build”.
-                    BitBake obeys inter-task dependencies when doing
-                    so.
-                </para>
-
-                <para>
-                    The following command runs the build task, which is
-                    the default task, on the <filename>foo_1.0.bb</filename>
-                    recipe file:
-                    <literallayout class='monospaced'>
-     $ bitbake -b foo_1.0.bb
-                    </literallayout>
-                    The following command runs the clean task on the
-                    <filename>foo.bb</filename> recipe file:
-                    <literallayout class='monospaced'>
-     $ bitbake -b foo.bb -c clean
-                    </literallayout>
-                    <note>
-                        The "-b" option explicitly does not handle recipe
-                        dependencies.
-                        Other than for debugging purposes, it is instead
-                        recommended that you use the syntax presented in the
-                        next section.
-                    </note>
-                </para>
-            </section>
-
-            <section id='executing-tasks-against-a-set-of-recipe-files'>
-                <title>Executing Tasks Against a Set of Recipe Files</title>
-
-                <para>
-                    There are a number of additional complexities introduced
-                    when one wants to manage multiple <filename>.bb</filename>
-                    files.
-                    Clearly there needs to be a way to tell BitBake what
-                    files are available and, of those, which you
-                    want to execute.
-                    There also needs to be a way for each recipe
-                    to express its dependencies, both for build-time and
-                    runtime.
-                    There must be a way for you to express recipe preferences
-                    when multiple recipes provide the same functionality, or when
-                    there are multiple versions of a recipe.
-                </para>
-
-                <para>
-                    The <filename>bitbake</filename> command, when not using
-                    "--buildfile" or "-b" only accepts a "PROVIDES".
-                    You cannot provide anything else.
-                    By default, a recipe file generally "PROVIDES" its
-                    "packagename" as shown in the following example:
-                    <literallayout class='monospaced'>
-     $ bitbake foo
-                    </literallayout>
-                    This next example "PROVIDES" the package name and also uses
-                    the "-c" option to tell BitBake to just execute the
-                    <filename>do_clean</filename> task:
-                    <literallayout class='monospaced'>
-     $ bitbake -c clean foo
-                    </literallayout>
-                </para>
-            </section>
-
-            <section id='executing-a-list-of-task-and-recipe-combinations'>
-                <title>Executing a List of Task and Recipe Combinations</title>
-
-                <para>
-                    The BitBake command line supports specifying different
-                    tasks for individual targets when you specify multiple
-                    targets.
-                    For example, suppose you had two targets (or recipes)
-                    <filename>myfirstrecipe</filename> and
-                    <filename>mysecondrecipe</filename> and you needed
-                    BitBake to run <filename>taskA</filename> for the first
-                    recipe and <filename>taskB</filename> for the second
-                    recipe:
-                    <literallayout class='monospaced'>
-     $ bitbake myfirstrecipe:do_taskA mysecondrecipe:do_taskB
-                    </literallayout>
-                </para>
-            </section>
-
-            <section id='generating-dependency-graphs'>
-                <title>Generating Dependency Graphs</title>
-
-                <para>
-                    BitBake is able to generate dependency graphs using
-                    the <filename>dot</filename> syntax.
-                    You can convert these graphs into images using the
-                    <filename>dot</filename> tool from
-                    <ulink url='http://www.graphviz.org'>Graphviz</ulink>.
-                </para>
-
-                <para>
-                    When you generate a dependency graph, BitBake writes three files
-                    to the current working directory:
-                    <itemizedlist>
-                        <listitem><para>
-                            <emphasis><filename>recipe-depends.dot</filename>:</emphasis>
-                            Shows dependencies between recipes (i.e. a collapsed version of
-                            <filename>task-depends.dot</filename>).
-                            </para></listitem>
-                        <listitem><para>
-                            <emphasis><filename>task-depends.dot</filename>:</emphasis>
-                            Shows dependencies between tasks.
-                            These dependencies match BitBake's internal task execution list.
-                            </para></listitem>
-                        <listitem><para>
-                            <emphasis><filename>pn-buildlist</filename>:</emphasis>
-                            Shows a simple list of targets that are to be built.
-                            </para></listitem>
-                    </itemizedlist>
-                </para>
-
-                <para>
-                    To stop depending on common depends, use the "-I" depend
-                    option and BitBake omits them from the graph.
-                    Leaving this information out can produce more readable graphs.
-                    This way, you can remove from the graph
-                    <filename>DEPENDS</filename> from inherited classes
-                    such as <filename>base.bbclass</filename>.
-                </para>
-
-                <para>
-                    Here are two examples that create dependency graphs.
-                    The second example omits depends common in OpenEmbedded from
-                    the graph:
-                    <literallayout class='monospaced'>
-     $ bitbake -g foo
-
-     $ bitbake -g -I virtual/kernel -I eglibc foo
-                    </literallayout>
-                </para>
-            </section>
-
-            <section id='executing-a-multiple-configuration-build'>
-                <title>Executing a Multiple Configuration Build</title>
-
-                <para>
-                    BitBake is able to build multiple images or packages
-                    using a single command where the different targets
-                    require different configurations (multiple configuration
-                    builds).
-                    Each target, in this scenario, is referred to as a
-                    "multiconfig".
-                </para>
-
-                <para>
-                    To accomplish a multiple configuration build, you must
-                    define each target's configuration separately using
-                    a parallel configuration file in the build directory.
-                    The location for these multiconfig configuration files
-                    is specific.
-                    They must reside in the current build directory in
-                    a sub-directory of <filename>conf</filename> named
-                    <filename>multiconfig</filename>.
-                    Following is an example for two separate targets:
-                    <imagedata fileref="figures/bb_multiconfig_files.png" align="center" width="4in" depth="3in" />
-                </para>
-
-                <para>
-                    The reason for this required file hierarchy
-                    is because the <filename>BBPATH</filename> variable
-                    is not constructed until the layers are parsed.
-                    Consequently, using the configuration file as a
-                    pre-configuration file is not possible unless it is
-                    located in the current working directory.
-                </para>
-
-                <para>
-                    Minimally, each configuration file must define the
-                    machine and the temporary directory BitBake uses
-                    for the build.
-                    Suggested practice dictates that you do not
-                    overlap the temporary directories used during the
-                    builds.
-                </para>
-
-                <para>
-                    Aside from separate configuration files for each
-                    target, you must also enable BitBake to perform multiple
-                    configuration builds.
-                    Enabling is accomplished by setting the
-                    <link linkend='var-bb-BBMULTICONFIG'><filename>BBMULTICONFIG</filename></link>
-                    variable in the <filename>local.conf</filename>
-                    configuration file.
-                    As an example, suppose you had configuration files
-                    for <filename>target1</filename> and
-                    <filename>target2</filename> defined in the build
-                    directory.
-                    The following statement in the
-                    <filename>local.conf</filename> file both enables
-                    BitBake to perform multiple configuration builds and
-                    specifies the two extra multiconfigs:
-                    <literallayout class='monospaced'>
-     BBMULTICONFIG = "target1 target2"
-                    </literallayout>
-                </para>
-
-                <para>
-                    Once the target configuration files are in place and
-                    BitBake has been enabled to perform multiple configuration
-                    builds, use the following command form to start the
-                    builds:
-                    <literallayout class='monospaced'>
-     $ bitbake [mc:<replaceable>multiconfigname</replaceable>:]<replaceable>target</replaceable> [[[mc:<replaceable>multiconfigname</replaceable>:]<replaceable>target</replaceable>] ... ]
-                    </literallayout>
-                    Here is an example for two extra multiconfigs:
-                    <filename>target1</filename> and
-                    <filename>target2</filename>:
-                    <literallayout class='monospaced'>
-     $ bitbake mc::<replaceable>target</replaceable> mc:target1:<replaceable>target</replaceable> mc:target2:<replaceable>target</replaceable>
-                    </literallayout>
-                </para>
-            </section>
-
-            <section id='bb-enabling-multiple-configuration-build-dependencies'>
-                <title>Enabling Multiple Configuration Build Dependencies</title>
-
-                <para>
-                    Sometimes dependencies can exist between targets
-                    (multiconfigs) in a multiple configuration build.
-                    For example, suppose that in order to build an image
-                    for a particular architecture, the root filesystem of
-                    another build for a different architecture needs to
-                    exist.
-                    In other words, the image for the first multiconfig depends
-                    on the root filesystem of the second multiconfig.
-                    This dependency is essentially that the task in the recipe
-                    that builds one multiconfig is dependent on the
-                    completion of the task in the recipe that builds
-                    another multiconfig.
-                </para>
-
-                <para>
-                    To enable dependencies in a multiple configuration
-                    build, you must declare the dependencies in the recipe
-                    using the following statement form:
-                    <literallayout class='monospaced'>
-     <replaceable>task_or_package</replaceable>[mcdepends] = "mc:<replaceable>from_multiconfig</replaceable>:<replaceable>to_multiconfig</replaceable>:<replaceable>recipe_name</replaceable>:<replaceable>task_on_which_to_depend</replaceable>"
-                    </literallayout>
-                    To better show how to use this statement, consider an
-                    example with two multiconfigs: <filename>target1</filename>
-                    and <filename>target2</filename>:
-                    <literallayout class='monospaced'>
-     <replaceable>image_task</replaceable>[mcdepends] = "mc:target1:target2:<replaceable>image2</replaceable>:<replaceable>rootfs_task</replaceable>"
-                    </literallayout>
-                    In this example, the
-                    <replaceable>from_multiconfig</replaceable> is "target1" and
-                    the <replaceable>to_multiconfig</replaceable> is "target2".
-                    The task on which the image whose recipe contains
-                    <replaceable>image_task</replaceable> depends on the
-                    completion of the <replaceable>rootfs_task</replaceable>
-                    used to build out <replaceable>image2</replaceable>, which
-                    is associated with the "target2" multiconfig.
-               </para>
-
-               <para>
-                   Once you set up this dependency, you can build the
-                   "target1" multiconfig using a BitBake command as follows:
-                   <literallayout class='monospaced'>
-     $ bitbake mc:target1:<replaceable>image1</replaceable>
-                   </literallayout>
-                   This command executes all the tasks needed to create
-                   <replaceable>image1</replaceable> for the "target1"
-                   multiconfig.
-                   Because of the dependency, BitBake also executes through
-                   the <replaceable>rootfs_task</replaceable> for the "target2"
-                   multiconfig build.
-               </para>
-
-               <para>
-                   Having a recipe depend on the root filesystem of another
-                   build might not seem that useful.
-                   Consider this change to the statement in the
-                   <replaceable>image1</replaceable> recipe:
-                   <literallayout class='monospaced'>
-     <replaceable>image_task</replaceable>[mcdepends] = "mc:target1:target2:<replaceable>image2</replaceable>:<replaceable>image_task</replaceable>"
-                   </literallayout>
-                   In this case, BitBake must create
-                   <replaceable>image2</replaceable> for the "target2"
-                   build since the "target1" build depends on it.
-               </para>
-
-               <para>
-                   Because "target1" and "target2" are enabled for multiple
-                   configuration builds and have separate configuration
-                   files, BitBake places the artifacts for each build in the
-                   respective temporary build directories.
-               </para>
-            </section>
-        </section>
-    </section>
-</chapter>

bitbake/doc/bitbake-user-manual/bitbake-user-manual-style.css

@@ -1,984 +0,0 @@
-/*
-   Generic XHTML / DocBook XHTML CSS Stylesheet.
-
-   Browser wrangling and typographic design by
-      Oyvind Kolas / pippin@gimp.org
-
-   Customised for Poky by
-      Matthew Allum / mallum@o-hand.com
-
-   Thanks to:
-     Liam R. E. Quin
-     William Skaggs
-     Jakub Steiner
-
-   Structure
-   ---------
-
-   The stylesheet is divided into the following sections:
-
-       Positioning
-          Margins, paddings, width, font-size, clearing.
-       Decorations
-          Borders, style
-       Colors
-          Colors
-       Graphics
-          Graphical backgrounds
-       Nasty IE tweaks
-          Workarounds needed to make it work in internet explorer,
-          currently makes the stylesheet non validating, but up until
-          this point it is validating.
-       Mozilla extensions
-          Transparency for footer
-	  Rounded corners on boxes
-
-*/
-
-
-  /*************** /
- /  Positioning   /
-/ ***************/
-
-body {
-  font-family: Verdana, Sans, sans-serif;
-
-  min-width: 640px;
-  width: 80%;
-  margin:  0em auto;
-  padding: 2em 5em 5em 5em;
-  color: #333;
-}
-
-h1,h2,h3,h4,h5,h6,h7 {
-  font-family: Arial, Sans;
-  color: #00557D;
-  clear: both;
-}
-
-h1 {
-  font-size: 2em;
-  text-align: left;
-  padding: 0em 0em 0em 0em;
-  margin: 2em 0em 0em 0em;
-}
-
-h2.subtitle {
-  margin: 0.10em 0em 3.0em 0em;
-  padding: 0em 0em 0em 0em;
-  font-size: 1.8em;
-  padding-left: 20%;
-  font-weight: normal;
-  font-style: italic;
-}
-
-h2 {
-  margin: 2em 0em 0.66em 0em;
-  padding: 0.5em 0em 0em 0em;
-  font-size: 1.5em;
-  font-weight: bold;
-}
-
-h3.subtitle {
-  margin: 0em 0em 1em 0em;
-  padding: 0em 0em 0em 0em;
-  font-size: 142.14%;
-  text-align: right;
-}
-
-h3 {
-  margin: 1em 0em 0.5em 0em;
-  padding: 1em 0em 0em 0em;
-  font-size: 140%;
-  font-weight: bold;
-}
-
-h4 {
-  margin: 1em 0em 0.5em 0em;
-  padding: 1em 0em 0em 0em;
-  font-size: 120%;
-  font-weight: bold;
-}
-
-h5 {
-  margin: 1em 0em 0.5em 0em;
-  padding: 1em 0em 0em 0em;
-  font-size: 110%;
-  font-weight: bold;
-}
-
-h6 {
-  margin: 1em 0em 0em 0em;
-  padding: 1em 0em 0em 0em;
-  font-size: 110%;
-  font-weight: bold;
-}
-
-.authorgroup {
-  background-color: transparent;
-  background-repeat: no-repeat;
-  padding-top: 256px;
-  background-image: url("figures/bitbake-title.png");
-  background-position: left top;
-  margin-top: -256px;
-  padding-right: 50px;
-  margin-left: 0px;
-  text-align: right;
-  width: 740px;
-}
-
-h3.author {
-  margin: 0em 0me 0em 0em;
-  padding: 0em 0em 0em 0em;
-  font-weight: normal;
-  font-size: 100%;
-  color: #333;
-  clear: both;
-}
-
-.author tt.email {
-  font-size: 66%;
-}
-
-.titlepage hr {
-  width: 0em;
-  clear: both;
-}
-
-.revhistory {
-  padding-top: 2em;
-  clear: both;
-}
-
-.toc,
-.list-of-tables,
-.list-of-examples,
-.list-of-figures {
-  padding: 1.33em 0em 2.5em 0em;
-  color: #00557D;
-}
-
-.toc p,
-.list-of-tables p,
-.list-of-figures p,
-.list-of-examples p {
-  padding: 0em 0em 0em 0em;
-  padding: 0em 0em 0.3em;
-  margin: 1.5em 0em 0em 0em;
-}
-
-.toc p b,
-.list-of-tables p b,
-.list-of-figures p b,
-.list-of-examples p b{
-  font-size: 100.0%;
-  font-weight: bold;
-}
-
-.toc dl,
-.list-of-tables dl,
-.list-of-figures dl,
-.list-of-examples dl {
-  margin: 0em 0em 0.5em 0em;
-  padding: 0em 0em 0em 0em;
-}
-
-.toc dt {
-  margin: 0em 0em 0em 0em;
-  padding: 0em 0em 0em 0em;
-}
-
-.toc dd {
-  margin: 0em 0em 0em 2.6em;
-  padding: 0em 0em 0em 0em;
-}
-
-div.glossary dl,
-div.variablelist dl {
-}
-
-.glossary dl dt,
-.variablelist dl dt,
-.variablelist dl dt span.term {
-  font-weight: normal;
-  width: 20em;
-  text-align: right;
-}
-
-.variablelist dl dt {
-  margin-top: 0.5em;
-}
-
-.glossary dl dd,
-.variablelist dl dd {
-  margin-top: -1em;
-  margin-left: 25.5em;
-}
-
-.glossary dd p,
-.variablelist dd p {
-  margin-top: 0em;
-  margin-bottom: 1em;
-}
-
-
-div.calloutlist table td {
-  padding: 0em 0em 0em 0em;
-  margin: 0em 0em 0em 0em;
-}
-
-div.calloutlist table td p {
-  margin-top: 0em;
-  margin-bottom: 1em;
-}
-
-div p.copyright {
-  text-align: left;
-}
-
-div.legalnotice p.legalnotice-title {
-  margin-bottom: 0em;
-}
-
-p {
-  line-height: 1.5em;
-  margin-top: 0em;
-
-}
-
-dl {
-  padding-top: 0em;
-}
-
-hr {
-  border: solid 1px;
-}
-
-
-.mediaobject,
-.mediaobjectco {
-  text-align: center;
-}
-
-img {
-  border: none;
-}
-
-ul {
-  padding: 0em 0em 0em 1.5em;
-}
-
-ul li {
-  padding: 0em 0em 0em 0em;
-}
-
-ul li p {
-  text-align: left;
-}
-
-table {
-  width :100%;
-}
-
-th {
-  padding: 0.25em;
-  text-align: left;
-  font-weight: normal;
-  vertical-align: top;
-}
-
-td {
-  padding: 0.25em;
-  vertical-align: top;
-}
-
-p a[id] {
-  margin: 0px;
-  padding: 0px;
-  display: inline;
-  background-image: none;
-}
-
-a {
-  text-decoration: underline;
-  color: #444;
-}
-
-pre {
-    overflow: auto;
-}
-
-a:hover {
-  text-decoration: underline;
-  /*font-weight: bold;*/
-}
-
-/* This style defines how the permalink character
-   appears by itself and when hovered over with
-   the mouse. */
-
-[alt='Permalink'] { color: #eee; }
-[alt='Permalink']:hover { color: black; }
-
-
-div.informalfigure,
-div.informalexample,
-div.informaltable,
-div.figure,
-div.table,
-div.example {
-  margin: 1em 0em;
-  padding: 1em;
-  page-break-inside: avoid;
-}
-
-
-div.informalfigure p.title b,
-div.informalexample p.title b,
-div.informaltable p.title b,
-div.figure p.title b,
-div.example p.title b,
-div.table p.title b{
-    padding-top: 0em;
-    margin-top: 0em;
-    font-size: 100%;
-    font-weight: normal;
-}
-
-.mediaobject .caption,
-.mediaobject .caption p  {
-  text-align: center;
-  font-size: 80%;
-  padding-top: 0.5em;
-  padding-bottom: 0.5em;
-}
-
-.epigraph {
-  padding-left: 55%;
-  margin-bottom: 1em;
-}
-
-.epigraph p {
-  text-align: left;
-}
-
-.epigraph .quote {
-  font-style: italic;
-}
-.epigraph .attribution {
-  font-style: normal;
-  text-align: right;
-}
-
-span.application {
-  font-style: italic;
-}
-
-.programlisting {
-  font-family: monospace;
-  font-size: 80%;
-  white-space: pre;
-  margin: 1.33em 0em;
-  padding: 1.33em;
-}
-
-.tip,
-.warning,
-.caution,
-.note {
-  margin-top: 1em;
-  margin-bottom: 1em;
-
-}
-
-/* force full width of table within div */
-.tip table,
-.warning table,
-.caution table,
-.note table {
-  border: none;
-  width: 100%;
-}
-
-
-.tip table th,
-.warning table th,
-.caution table th,
-.note table th {
-  padding: 0.8em 0.0em 0.0em 0.0em;
-  margin : 0em 0em 0em 0em;
-}
-
-.tip p,
-.warning p,
-.caution p,
-.note p {
-  margin-top: 0.5em;
-  margin-bottom: 0.5em;
-  padding-right: 1em;
-  text-align: left;
-}
-
-.acronym {
-  text-transform: uppercase;
-}
-
-b.keycap,
-.keycap {
-  padding: 0.09em 0.3em;
-  margin: 0em;
-}
-
-.itemizedlist li {
-  clear: none;
-}
-
-.filename {
-  font-size: medium;
-  font-family: Courier, monospace;
-}
-
-
-div.navheader, div.heading{
-  position: absolute;
-  left: 0em;
-  top: 0em;
-  width: 100%;
-  background-color: #cdf;
-  width: 100%;
-}
-
-div.navfooter, div.footing{
-  position: fixed;
-  left: 0em;
-  bottom: 0em;
-  background-color: #eee;
-  width: 100%;
-}
-
-
-div.navheader td,
-div.navfooter td {
-  font-size: 66%;
-}
-
-div.navheader table th {
-  /*font-family: Georgia, Times, serif;*/
-  /*font-size: x-large;*/
-  font-size: 80%;
-}
-
-div.navheader table {
-  border-left: 0em;
-  border-right: 0em;
-  border-top: 0em;
-  width: 100%;
-}
-
-div.navfooter table {
-  border-left: 0em;
-  border-right: 0em;
-  border-bottom: 0em;
-  width: 100%;
-}
-
-div.navheader table td a,
-div.navfooter table td a {
-  color: #777;
-  text-decoration: none;
-}
-
-/* normal text in the footer */
-div.navfooter table td {
-  color: black;
-}
-
-div.navheader table td a:visited,
-div.navfooter table td a:visited {
-  color: #444;
-}
-
-
-/* links in header and footer */
-div.navheader table td a:hover,
-div.navfooter table td a:hover {
-  text-decoration: underline;
-  background-color: transparent;
-  color: #33a;
-}
-
-div.navheader hr,
-div.navfooter hr {
-  display: none;
-}
-
-
-.qandaset tr.question td p {
-  margin: 0em 0em 1em 0em;
-  padding: 0em 0em 0em 0em;
-}
-
-.qandaset tr.answer td p {
-  margin: 0em 0em 1em 0em;
-  padding: 0em 0em 0em 0em;
-}
-.answer td {
-  padding-bottom: 1.5em;
-}
-
-.emphasis {
-  font-weight: bold;
-}
-
-
-  /************* /
- / decorations  /
-/ *************/
-
-.titlepage {
-}
-
-.part .title {
-}
-
-.subtitle {
-    border: none;
-}
-
-/*
-h1 {
-  border: none;
-}
-
-h2 {
-  border-top: solid 0.2em;
-  border-bottom: solid 0.06em;
-}
-
-h3 {
-  border-top: 0em;
-  border-bottom: solid 0.06em;
-}
-
-h4 {
-  border: 0em;
-  border-bottom: solid 0.06em;
-}
-
-h5 {
-  border: 0em;
-}
-*/
-
-.programlisting {
-  border: solid 1px;
-}
-
-div.figure,
-div.table,
-div.informalfigure,
-div.informaltable,
-div.informalexample,
-div.example {
-  border: 1px solid;
-}
-
-
-
-.tip,
-.warning,
-.caution,
-.note {
-  border: 1px solid;
-}
-
-.tip table th,
-.warning table th,
-.caution table th,
-.note table th {
-  border-bottom: 1px solid;
-}
-
-.question td {
-  border-top: 1px solid black;
-}
-
-.answer {
-}
-
-
-b.keycap,
-.keycap {
-  border: 1px solid;
-}
-
-
-div.navheader, div.heading{
-  border-bottom: 1px solid;
-}
-
-
-div.navfooter, div.footing{
-  border-top: 1px solid;
-}
-
-  /********* /
- /  colors  /
-/ *********/
-
-body {
-  color: #333;
-  background: white;
-}
-
-a {
-  background: transparent;
-}
-
-a:hover {
-  background-color: #dedede;
-}
-
-
-h1,
-h2,
-h3,
-h4,
-h5,
-h6,
-h7,
-h8 {
-  background-color: transparent;
-}
-
-hr {
-  border-color: #aaa;
-}
-
-
-.tip, .warning, .caution, .note {
-  border-color: #fff;
-}
-
-
-.tip table th,
-.warning table th,
-.caution table th,
-.note table th {
-  border-bottom-color: #fff;
-}
-
-
-.warning {
-  background-color: #f0f0f2;
-}
-
-.caution {
-  background-color: #f0f0f2;
-}
-
-.tip {
-  background-color: #f0f0f2;
-}
-
-.note {
-  background-color: #f0f0f2;
-}
-
-.glossary dl dt,
-.variablelist dl dt,
-.variablelist dl dt span.term {
-  color: #044;
-}
-
-div.figure,
-div.table,
-div.example,
-div.informalfigure,
-div.informaltable,
-div.informalexample {
-  border-color: #aaa;
-}
-
-pre.programlisting {
-  color: black;
-  background-color: #fff;
-  border-color: #aaa;
-  border-width: 2px;
-}
-
-.guimenu,
-.guilabel,
-.guimenuitem {
-  background-color: #eee;
-}
-
-
-b.keycap,
-.keycap {
-  background-color: #eee;
-  border-color: #999;
-}
-
-
-div.navheader {
-  border-color: black;
-}
-
-
-div.navfooter {
-  border-color: black;
-}
-
-
-  /*********** /
- /  graphics  /
-/ ***********/
-
-/*
-body {
-  background-image: url("images/body_bg.jpg");
-  background-attachment: fixed;
-}
-
-.navheader,
-.note,
-.tip {
-  background-image: url("images/note_bg.jpg");
-  background-attachment: fixed;
-}
-
-.warning,
-.caution {
-  background-image: url("images/warning_bg.jpg");
-  background-attachment: fixed;
-}
-
-.figure,
-.informalfigure,
-.example,
-.informalexample,
-.table,
-.informaltable {
-  background-image: url("images/figure_bg.jpg");
-  background-attachment: fixed;
-}
-
-*/
-h1,
-h2,
-h3,
-h4,
-h5,
-h6,
-h7{
-}
-
-/*
-Example of how to stick an image as part of the title.
-
-div.article .titlepage .title
-{
-  background-image: url("figures/white-on-black.png");
-  background-position: center;
-  background-repeat: repeat-x;
-}
-*/
-
-div.preface .titlepage .title,
-div.colophon .title,
-div.chapter .titlepage .title,
-div.article .titlepage .title
-{
-}
-
-div.section div.section .titlepage .title,
-div.sect2 .titlepage .title {
-    background: none;
-}
-
-
-h1.title {
-  background-color: transparent;
-  background-repeat: no-repeat;
-  height: 256px;
-  text-indent: -9000px;
-  overflow:hidden;
-}
-
-h2.subtitle {
-  background-color: transparent;
-  text-indent: -9000px;
-  overflow:hidden;
-  width: 0px;
-  display: none;
-}
-
-  /*************************************** /
- /  pippin.gimp.org specific alterations  /
-/ ***************************************/
-
-/*
-div.heading, div.navheader {
-  color: #777;
-  font-size: 80%;
-  padding: 0;
-  margin: 0;
-  text-align: left;
-  position: absolute;
-  top: 0px;
-  left: 0px;
-  width: 100%;
-  height: 50px;
-  background: url('/gfx/heading_bg.png') transparent;
-  background-repeat: repeat-x;
-  background-attachment: fixed;
-  border: none;
-}
-
-div.heading a {
-  color: #444;
-}
-
-div.footing, div.navfooter {
-  border: none;
-  color: #ddd;
-  font-size: 80%;
-  text-align:right;
-
-  width: 100%;
-  padding-top: 10px;
-  position: absolute;
-  bottom: 0px;
-  left: 0px;
-
-  background: url('/gfx/footing_bg.png') transparent;
-}
-*/
-
-
-
-  /****************** /
- /  nasty ie tweaks  /
-/ ******************/
-
-/*
-div.heading, div.navheader {
-  width:expression(document.body.clientWidth + "px");
-}
-
-div.footing, div.navfooter {
-  width:expression(document.body.clientWidth + "px");
-  margin-left:expression("-5em");
-}
-body {
-  padding:expression("4em 5em 0em 5em");
-}
-*/
-
-  /**************************************** /
- / mozilla vendor specific css extensions  /
-/ ****************************************/
-/*
-div.navfooter, div.footing{
-  -moz-opacity: 0.8em;
-}
-
-div.figure,
-div.table,
-div.informalfigure,
-div.informaltable,
-div.informalexample,
-div.example,
-.tip,
-.warning,
-.caution,
-.note {
-  -moz-border-radius: 0.5em;
-}
-
-b.keycap,
-.keycap {
-  -moz-border-radius: 0.3em;
-}
-*/
-
-table tr td table tr td {
-  display: none;
-}
-
-
-hr {
-  display: none;
-}
-
-table {
-  border: 0em;
-}
-
- .photo {
-  float: right;
-  margin-left:   1.5em;
-  margin-bottom: 1.5em;
-  margin-top: 0em;
-  max-width:      17em;
-  border:     1px solid gray;
-  padding:    3px;
-  background: white;
-}
- .seperator {
-   padding-top: 2em;
-   clear: both;
-  }
-
-  #validators {
-      margin-top: 5em;
-      text-align: right;
-      color: #777;
-  }
-  @media print {
-      body {
-          font-size: 8pt;
-      }
-      .noprint {
-          display: none;
-      }
-  }
-
-
-.tip,
-.note {
-   background: #f0f0f2;
-   color: #333;
-   padding: 20px;
-   margin: 20px;
-}
-
-.tip h3,
-.note h3 {
-   padding: 0em;
-   margin: 0em;
-   font-size: 2em;
-   font-weight: bold;
-   color: #333;
-}
-
-.tip a,
-.note a {
-   color: #333;
-   text-decoration: underline;
-}
-
-.footnote {
-   font-size: small;
-   color: #333;
-}
-
-/* Changes the announcement text */
-.tip h3,
-.warning h3,
-.caution h3,
-.note h3 {
-   font-size:large;
-   color: #00557D;
-}

bitbake/doc/bitbake-user-manual/bitbake-user-manual.xml

@@ -1,88 +0,0 @@
-<!DOCTYPE book PUBLIC "-//OASIS//DTD DocBook XML V4.2//EN"
-"http://www.oasis-open.org/docbook/xml/4.2/docbookx.dtd">
-
-<book id='bitbake-user-manual' lang='en'
-        xmlns:xi="http://www.w3.org/2003/XInclude"
-        xmlns="http://docbook.org/ns/docbook"
-        >
-    <bookinfo>
-
-        <mediaobject>
-            <imageobject>
-                <imagedata fileref='figures/bitbake-title.png'
-                    format='SVG'
-                    align='left' scalefit='1' width='100%'/>
-            </imageobject>
-        </mediaobject>
-
-        <title>
-            BitBake User Manual
-        </title>
-
-        <authorgroup>
-            <author>
-                <firstname>Richard Purdie, Chris Larson, and </firstname> <surname>Phil Blundell</surname>
-                <affiliation>
-                    <orgname>BitBake Community</orgname>
-                </affiliation>
-                <email>bitbake-devel@lists.openembedded.org</email>
-            </author>
-        </authorgroup>
-
-<!--
-# Add in some revision history if we want it here.
-        <revhistory>
-            <revision>
-                <revnumber>x.x</revnumber>
-                <date>dd month year</date>
-                <revremark>Some relevent comment</revremark>
-            </revision>
-            <revision>
-                <revnumber>x.x</revnumber>
-                <date>dd month year</date>
-                <revremark>Some relevent comment</revremark>
-            </revision>
-            <revision>
-                <revnumber>x.x</revnumber>
-                <date>dd month year</date>
-                <revremark>Some relevent comment</revremark>
-            </revision>
-            <revision>
-                <revnumber>x.x</revnumber>
-                <date>dd month year</date>
-                <revremark>Some relevent comment</revremark>
-            </revision>
-       </revhistory>
--->
-
-        <copyright>
-            <year>2004-2018</year>
-            <holder>Richard Purdie</holder>
-            <holder>Chris Larson</holder>
-            <holder>and Phil Blundell</holder>
-        </copyright>
-
-        <legalnotice>
-            <para>
-                This work is licensed under the Creative Commons Attribution License.
-                To view a copy of this license, visit
-                <ulink url="http://creativecommons.org/licenses/by/2.5/">http://creativecommons.org/licenses/by/2.5/</ulink>
-                or send a letter to Creative Commons, 444 Castro Street,
-                Suite 900, Mountain View, California 94041, USA.
-            </para>
-        </legalnotice>
-    </bookinfo>
-
-    <xi:include href="bitbake-user-manual-intro.xml"/>
-
-    <xi:include href="bitbake-user-manual-execution.xml"/>
-
-    <xi:include href="bitbake-user-manual-metadata.xml"/>
-
-    <xi:include href="bitbake-user-manual-fetching.xml"/>
-
-    <xi:include href="bitbake-user-manual-ref-variables.xml"/>
-
-    <xi:include href="bitbake-user-manual-hello.xml"/>
-
-</book>

bitbake/doc/bitbake-user-manual/html.css

@@ -1,281 +0,0 @@
-/* Feuille de style DocBook du projet Traduc.org                */
-/* DocBook CSS stylesheet of the Traduc.org project             */
-
-/* (c) Jean-Philippe Gu<47>rard - 14 ao<61>t 2004                     */
-/* (c) Jean-Philippe Gu<47>rard - 14 August 2004                   */
-
-/* Cette feuille de style est libre, vous pouvez la             */
-/* redistribuer et la modifier selon les termes de la Licence   */
-/* Art Libre. Vous trouverez un exemplaire de cette Licence sur */
-/* http://tigreraye.org/Petit-guide-du-traducteur.html#licence-art-libre */
-
-/* This work of art is free, you can redistribute it and/or     */
-/* modify it according to terms of the Free Art license. You    */
-/* will find a specimen of this license on the Copyleft         */
-/* Attitude web site: http://artlibre.org as well as on other   */
-/* sites.                                                       */
-/* Please note that the French version of this licence as shown */
-/* on http://tigreraye.org/Petit-guide-du-traducteur.html#licence-art-libre */
-/* is only official licence of this document. The English       */
-/* is only provided to help you understand this licence.        */
-
-/* La derni<6E>re version de cette feuille de style est toujours   */
-/* disponible sur<75>: http://tigreraye.org/style.css              */
-/* Elle est <20>galement disponible sur<75>:                          */
-/* http://www.traduc.org/docs/HOWTO/lecture/style.css           */
-
-/* The latest version of this stylesheet is available from:     */
-/* http://tigreraye.org/style.css                               */
-/* It is also available on:                                     */
-/* http://www.traduc.org/docs/HOWTO/lecture/style.css           */
-
-/* N'h<>sitez pas <20> envoyer vos commentaires et corrections <20>    */
-/* Jean-Philippe Gu<47>rard <jean-philippe.guerard@tigreraye.org>  */
-
-/* Please send feedback and bug reports to                      */
-/* Jean-Philippe Gu<47>rard <jean-philippe.guerard@tigreraye.org>  */
-
-/* $Id: style.css,v 1.14 2004/09/10 20:12:09 fevrier Exp fevrier $ */
-
-/* Pr<50>sentation g<>n<EFBFBD>rale du document */
-/* Overall document presentation */
-
-body {
-    /*
-    font-family: Apolline, "URW Palladio L", Garamond, jGaramond,
-                 "Bitstream Cyberbit", "Palatino Linotype", serif;
-     */
-    margin: 7%;
-    background-color: white;
-}
-
-/* Taille du texte */
-/* Text size */
-
-* { font-size: 100%; }
-
-/* Gestion des textes mis en relief imbriqu<71>s */
-/* Embedded emphasis */
-
-em { font-style: italic; }
-em em { font-style: normal; }
-em em em { font-style: italic; }
-
-/* Titres */
-/* Titles */
-
-h1 { font-size: 200%; font-weight: 900; }
-h2 { font-size: 160%; font-weight: 900; }
-h3 { font-size: 130%; font-weight: bold; }
-h4 { font-size: 115%; font-weight: bold; }
-h5 { font-size: 108%; font-weight: bold; }
-h6 {                  font-weight: bold; }
-
-/* Nom de famille en petites majuscules (uniquement en fran<61>ais) */
-/* Last names in small caps (for French only) */
-
-*[class~="surname"]:lang(fr) { font-variant: small-caps; }
-
-/* Blocs de citation */
-/* Quotation blocs */
-
-div[class~="blockquote"] {
-  border: solid 2px #AAA;
-  padding: 5px;
-  margin: 5px;
-}
-
-div[class~="blockquote"] > table {
-  border: none;
-}
-
-/* Blocs lit<69>raux<75>: fond gris clair */
-/* Literal blocs: light gray background */
-
-*[class~="literallayout"] {
-  background: #f0f0f0;
-  padding: 5px;
-  margin: 5px;
-}
-
-/* Programmes et captures texte<74>: fond bleu clair */
-/* Listing and text screen snapshots: light blue background */
-
-*[class~="programlisting"], *[class~="screen"] {
-  background: #f0f0ff;
-  padding: 5px;
-  margin: 5px;
-}
-
-/* Les textes <20> remplacer sont surlign<67>s en vert p<>le */
-/* Replaceable text in highlighted in pale green */
-
-*[class~="replaceable"] { 
-    background-color: #98fb98;
-    font-style: normal; }
-
-/* Tables<65>: fonds gris clair & bords simples */
-/* Tables: light gray background and solid borders */
-
-*[class~="table"] *[class~="title"] { width:100%; border: 0px; }
-
-table {
-    border: 1px solid #aaa;
-    border-collapse: collapse;
-    padding: 2px;
-    margin: 5px;
-}
-
-/* Listes simples en style table */
-/* Simples lists in table presentation */
-
-table[class~="simplelist"] {
-    background-color: #F0F0F0;
-    margin: 5px;
-    border: solid 1px #AAA;
-}
-
-table[class~="simplelist"] td {
-    border: solid 1px #AAA;
-}
-
-/* Les tables */
-/* Tables */
-
-*[class~="table"] table {
-    background-color: #F0F0F0;
-    border: solid 1px #AAA;
-}
-*[class~="informaltable"] table { background-color: #F0F0F0; }
-
-th,td {
-    vertical-align: baseline;
-    text-align: left;
-    padding: 0.1em 0.3em;
-    empty-cells: show; 
-}
-
-/* Alignement des colonnes */
-/* Colunms alignment */
-
-td[align=center] ,  th[align=center]  { text-align: center; }
-td[align=right] ,   th[align=right]   { text-align: right; }
-td[align=left] ,    th[align=left]    { text-align: left; }
-td[align=justify] , th[align=justify] { text-align: justify; }
-
-/* Pas de marge autour des images */
-/* No inside margins for images */
-
-img { border: 0; }
-
-/* Les liens ne sont pas soulign<67>s */
-/* No underlines for links */
-
-:link , :visited , :active { text-decoration: none; }
-
-/* Prudence<63>: cadre jaune et fond jaune clair */
-/* Caution: yellow border and light yellow background */
-
-*[class~="caution"] {
-    border: solid 2px yellow;
-    background-color: #ffffe0;
-    padding: 1em 6px 1em ;
-    margin: 5px;
-}
-
-*[class~="caution"] th {
-    vertical-align: middle
-}
-
-*[class~="caution"] table {
-    background-color: #ffffe0;
-    border: none;
-}
-
-/* Note importante<74>: cadre jaune et fond jaune clair */
-/* Important: yellow border and light yellow background */
-
-*[class~="important"] {
-    border: solid 2px yellow;
-    background-color: #ffffe0;
-    padding: 1em 6px 1em;
-    margin: 5px;
-}
-
-*[class~="important"] th {
-    vertical-align: middle
-}
-
-*[class~="important"] table  {
-    background-color: #ffffe0;
-    border: none;
-}
-
-/* Mise en <20>vidence<63>: texte l<>g<EFBFBD>rement plus grand */
-/* Highlights: slightly larger texts */
-
-*[class~="highlights"] {
-    font-size:  110%;
-}
-
-/* Note<74>: cadre bleu et fond bleu clair */
-/* Notes: blue border and light blue background */
-
-*[class~="note"]   {
-    border: solid 2px #7099C5;
-    background-color: #f0f0ff;
-    padding: 1em 6px 1em ;
-    margin: 5px;
-}
-
-*[class~="note"] th {
-    vertical-align: middle
-}
-
-*[class~="note"] table {
-    background-color: #f0f0ff;
-    border: none;
-}
-
-/* Astuce<63>: cadre vert et fond vert clair */
-/* Tip: green border and light green background */
-
-*[class~="tip"] {
-    border: solid 2px #00ff00;
-    background-color: #f0ffff;
-    padding: 1em 6px 1em ;
-    margin: 5px;
-}
-
-*[class~="tip"] th {
-    vertical-align: middle;
-}
-
-*[class~="tip"] table {
-    background-color: #f0ffff;
-    border: none;
-}
-
-/* Avertissement<6E>: cadre rouge et fond rouge clair */
-/* Warning: red border and light red background */
-
-*[class~="warning"] {
-    border: solid 2px #ff0000;
-    background-color: #fff0f0; 
-    padding: 1em 6px 1em ;
-    margin: 5px;
-}
-
-*[class~="warning"] th {
-    vertical-align: middle;
-}
-                    
-
-*[class~="warning"] table {
-    background-color: #fff0f0;
-    border: none;
-}
-
-/* Fin */
-/* The End */
-

bitbake/doc/conf.py

@@ -0,0 +1,100 @@
+# Configuration file for the Sphinx documentation builder.
+#
+# This file only contains a selection of the most common options. For a full
+# list see the documentation:
+# https://www.sphinx-doc.org/en/master/usage/configuration.html
+
+# -- Path setup --------------------------------------------------------------
+
+# If extensions (or modules to document with autodoc) are in another directory,
+# add these directories to sys.path here. If the directory is relative to the
+# documentation root, use os.path.abspath to make it absolute, like shown here.
+#
+# import os
+# import sys
+# sys.path.insert(0, os.path.abspath('.'))
+
+import datetime
+
+current_version = "dev"
+
+# String used in sidebar
+version = 'Version: ' + current_version
+if current_version == 'dev':
+    version = 'Version: Current Development'
+# Version seen in documentation_options.js and hence in js switchers code
+release = current_version
+
+# -- Project information -----------------------------------------------------
+
+project = 'Bitbake'
+copyright = '2004-%s, Richard Purdie, Chris Larson, and Phil Blundell' \
+    % datetime.datetime.now().year
+author = 'Richard Purdie, Chris Larson, and Phil Blundell'
+
+# external links and substitutions
+extlinks = {
+    'yocto_docs': ('https://docs.yoctoproject.org%s', None),
+    'oe_lists': ('https://lists.openembedded.org%s', None),
+}
+
+# -- General configuration ---------------------------------------------------
+
+# Add any Sphinx extension module names here, as strings. They can be
+# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom
+# ones.
+extensions = [
+    'sphinx.ext.autosectionlabel',
+    'sphinx.ext.extlinks',
+]
+autosectionlabel_prefix_document = True
+
+# Add any paths that contain templates here, relative to this directory.
+templates_path = ['_templates']
+
+# List of patterns, relative to source directory, that match files and
+# directories to ignore when looking for source files.
+# This pattern also affects html_static_path and html_extra_path.
+exclude_patterns = ['_build', 'Thumbs.db', '.DS_Store']
+
+# master document name. The default changed from contents to index. so better
+# set it ourselves.
+master_doc = 'index'
+
+# create substitution for project configuration variables
+rst_prolog = """
+.. |project_name| replace:: %s
+.. |copyright| replace:: %s
+.. |author| replace:: %s
+""" % (project, copyright, author)
+
+# -- Options for HTML output -------------------------------------------------
+
+# The theme to use for HTML and HTML Help pages.  See the documentation for
+# a list of builtin themes.
+#
+try:
+    import sphinx_rtd_theme
+    html_theme = 'sphinx_rtd_theme'
+except ImportError:
+    sys.stderr.write("The Sphinx sphinx_rtd_theme HTML theme was not found.\
+    \nPlease make sure to install the sphinx_rtd_theme python package.\n")
+    sys.exit(1)
+
+# Add any paths that contain custom static files (such as style sheets) here,
+# relative to this directory. They are copied after the builtin static files,
+# so a file named "default.css" will overwrite the builtin "default.css".
+html_static_path = ['sphinx-static']
+
+# Add customm CSS and JS files
+html_css_files = ['theme_overrides.css']
+html_js_files = ['switchers.js']
+
+# Hide 'Created using Sphinx' text
+html_show_sphinx = False
+
+# Add 'Last updated' on each page
+html_last_updated_fmt = '%b %d, %Y'
+
+# Remove the trailing 'dot' in section numbers
+html_secnumber_suffix = " "

bitbake/doc/genindex.rst

@@ -0,0 +1,3 @@
+=====
+Index
+=====

bitbake/doc/index.rst

@@ -0,0 +1,38 @@
+.. SPDX-License-Identifier: CC-BY-2.5
+
+===================
+BitBake User Manual
+===================
+
+|
+
+.. toctree::
+   :caption: Table of Contents
+   :numbered:
+
+   bitbake-user-manual/bitbake-user-manual-intro
+   bitbake-user-manual/bitbake-user-manual-execution
+   bitbake-user-manual/bitbake-user-manual-metadata
+   bitbake-user-manual/bitbake-user-manual-fetching
+   bitbake-user-manual/bitbake-user-manual-ref-variables
+   bitbake-user-manual/bitbake-user-manual-hello
+
+.. toctree::
+   :maxdepth: 1
+   :hidden:
+
+   genindex
+   releases
+
+----
+
+.. include:: <xhtml1-lat1.txt>
+
+| BitBake Community
+| Copyright |copy| |copyright|
+| <bitbake-devel@lists.openembedded.org>
+
+This work is licensed under the Creative Commons Attribution License.  To view a
+copy of this license, visit http://creativecommons.org/licenses/by/2.5/ or send
+a letter to Creative Commons, 444 Castro Street, Suite 900, Mountain View,
+California 94041, USA.

bitbake/doc/poky.ent

@@ -1,51 +0,0 @@
-<!ENTITY DISTRO "1.4">
-<!ENTITY DISTRO_NAME "tbd">
-<!ENTITY YOCTO_DOC_VERSION "1.4">
-<!ENTITY POKYVERSION "8.0">
-<!ENTITY YOCTO_POKY "poky-&DISTRO_NAME;-&POKYVERSION;">
-<!ENTITY COPYRIGHT_YEAR "2010-2013">
-<!ENTITY YOCTO_DL_URL "http://downloads.yoctoproject.org">
-<!ENTITY YOCTO_HOME_URL "http://www.yoctoproject.org">
-<!ENTITY YOCTO_LISTS_URL "http://lists.yoctoproject.org">
-<!ENTITY YOCTO_BUGZILLA_URL "http://bugzilla.yoctoproject.org">
-<!ENTITY YOCTO_WIKI_URL "https://wiki.yoctoproject.org">
-<!ENTITY YOCTO_AB_URL "http://autobuilder.yoctoproject.org">
-<!ENTITY YOCTO_GIT_URL "http://git.yoctoproject.org">
-<!ENTITY YOCTO_ADTREPO_URL "http://adtrepo.yoctoproject.org">
-<!ENTITY OE_HOME_URL "http://www.openembedded.org">
-<!ENTITY OE_LISTS_URL "http://lists.linuxtogo.org/cgi-bin/mailman">
-<!ENTITY OE_DOCS_URL "http://docs.openembedded.org">
-<!ENTITY OH_HOME_URL "http://o-hand.com">
-<!ENTITY BITBAKE_HOME_URL "http://developer.berlios.de/projects/bitbake/">
-<!ENTITY YOCTO_DOCS_URL "&YOCTO_HOME_URL;/docs">
-<!ENTITY YOCTO_SOURCES_URL "&YOCTO_HOME_URL;/sources/">
-<!ENTITY YOCTO_AB_PORT_URL "&YOCTO_AB_URL;:8010">
-<!ENTITY YOCTO_AB_NIGHTLY_URL "&YOCTO_AB_URL;/nightly/">
-<!ENTITY YOCTO_POKY_URL "&YOCTO_DL_URL;/releases/poky/">
-<!ENTITY YOCTO_RELEASE_DL_URL "&YOCTO_DL_URL;/releases/yocto/yocto-&DISTRO;">
-<!ENTITY YOCTO_TOOLCHAIN_DL_URL "&YOCTO_RELEASE_DL_URL;/toolchain/">
-<!ENTITY YOCTO_ADTINSTALLER_DL_URL "&YOCTO_RELEASE_DL_URL;/adt_installer">
-<!ENTITY YOCTO_POKY_DL_URL "&YOCTO_RELEASE_DL_URL;/&YOCTO_POKY;.tar.bz2">
-<!ENTITY YOCTO_MACHINES_DL_URL "&YOCTO_RELEASE_DL_URL;/machines">
-<!ENTITY YOCTO_QEMU_DL_URL "&YOCTO_MACHINES_DL_URL;/qemu">
-<!ENTITY YOCTO_PYTHON-i686_DL_URL "&YOCTO_DL_URL;/releases/miscsupport/python-nativesdk-standalone-i686.tar.bz2">
-<!ENTITY YOCTO_PYTHON-x86_64_DL_URL "&YOCTO_DL_URL;/releases/miscsupport/python-nativesdk-standalone-x86_64.tar.bz2">
-<!ENTITY YOCTO_DOCS_QS_URL "&YOCTO_DOCS_URL;/&YOCTO_DOC_VERSION;/yocto-project-qs/yocto-project-qs.html">
-<!ENTITY YOCTO_DOCS_ADT_URL "&YOCTO_DOCS_URL;/&YOCTO_DOC_VERSION;/adt-manual/adt-manual.html">
-<!ENTITY YOCTO_DOCS_REF_URL "&YOCTO_DOCS_URL;/&YOCTO_DOC_VERSION;/ref-manual/ref-manual.html">
-<!ENTITY YOCTO_DOCS_BSP_URL "&YOCTO_DOCS_URL;/&YOCTO_DOC_VERSION;/bsp-guide/bsp-guide.html">
-<!ENTITY YOCTO_DOCS_DEV_URL "&YOCTO_DOCS_URL;/&YOCTO_DOC_VERSION;/dev-manual/dev-manual.html">
-<!ENTITY YOCTO_DOCS_KERNEL_URL "&YOCTO_DOCS_URL;/&YOCTO_DOC_VERSION;/kernel-manual/kernel-manual.html">
-<!ENTITY YOCTO_ADTPATH_DIR "/opt/poky/&DISTRO;">
-<!ENTITY YOCTO_POKY_TARBALL "&YOCTO_POKY;.tar.bz2">
-<!ENTITY OE_INIT_PATH "&YOCTO_POKY;/oe-init-build-env">
-<!ENTITY OE_INIT_FILE "oe-init-build-env">
-<!ENTITY UBUNTU_HOST_PACKAGES_ESSENTIAL "gawk wget git-core diffstat unzip texinfo \
-     build-essential chrpath">
-<!ENTITY FEDORA_HOST_PACKAGES_ESSENTIAL "gawk make wget tar bzip2 gzip python unzip perl patch \
-     diffutils diffstat git cpp gcc gcc-c++ eglibc-devel texinfo chrpath \
-     ccache">
-<!ENTITY OPENSUSE_HOST_PACKAGES_ESSENTIAL "python gcc gcc-c++ git chrpath make wget python-xml \
-     diffstat texinfo python-curses">
-<!ENTITY CENTOS_HOST_PACKAGES_ESSENTIAL "gawk make wget tar bzip2 gzip python unzip perl patch \
-     diffutils diffstat git cpp gcc gcc-c++ glibc-devel texinfo chrpath">

bitbake/doc/releases.rst

@@ -0,0 +1,130 @@
+.. SPDX-License-Identifier: CC-BY-2.5
+
+=========================
+ Current Release Manuals
+=========================
+
+****************************
+3.1 'dunfell' Release Series
+****************************
+
+- :yocto_docs:`3.1 BitBake User Manual </3.1/bitbake-user-manual/bitbake-user-manual.html>`
+- :yocto_docs:`3.1.1 BitBake User Manual </3.1.1/bitbake-user-manual/bitbake-user-manual.html>`
+- :yocto_docs:`3.1.2 BitBake User Manual </3.1.2/bitbake-user-manual/bitbake-user-manual.html>`
+
+==========================
+ Previous Release Manuals
+==========================
+
+*************************
+3.0 'zeus' Release Series
+*************************
+
+- :yocto_docs:`3.0 BitBake User Manual </3.0/bitbake-user-manual/bitbake-user-manual.html>`
+- :yocto_docs:`3.0.1 BitBake User Manual </3.0.1/bitbake-user-manual/bitbake-user-manual.html>`
+- :yocto_docs:`3.0.2 BitBake User Manual </3.0.2/bitbake-user-manual/bitbake-user-manual.html>`
+- :yocto_docs:`3.0.3 BitBake User Manual </3.0.3/bitbake-user-manual/bitbake-user-manual.html>`
+
+****************************
+2.7 'warrior' Release Series
+****************************
+
+- :yocto_docs:`2.7 BitBake User Manual </2.7/bitbake-user-manual/bitbake-user-manual.html>`
+- :yocto_docs:`2.7.1 BitBake User Manual </2.7.1/bitbake-user-manual/bitbake-user-manual.html>`
+- :yocto_docs:`2.7.2 BitBake User Manual </2.7.2/bitbake-user-manual/bitbake-user-manual.html>`
+- :yocto_docs:`2.7.3 BitBake User Manual </2.7.3/bitbake-user-manual/bitbake-user-manual.html>`
+- :yocto_docs:`2.7.4 BitBake User Manual </2.7.4/bitbake-user-manual/bitbake-user-manual.html>`
+
+*************************
+2.6 'thud' Release Series
+*************************
+
+- :yocto_docs:`2.6 BitBake User Manual </2.6/bitbake-user-manual/bitbake-user-manual.html>`
+- :yocto_docs:`2.6.1 BitBake User Manual </2.6.1/bitbake-user-manual/bitbake-user-manual.html>`
+- :yocto_docs:`2.6.2 BitBake User Manual </2.6.2/bitbake-user-manual/bitbake-user-manual.html>`
+- :yocto_docs:`2.6.3 BitBake User Manual </2.6.3/bitbake-user-manual/bitbake-user-manual.html>`
+- :yocto_docs:`2.6.4 BitBake User Manual </2.6.4/bitbake-user-manual/bitbake-user-manual.html>`
+
+*************************
+2.5 'sumo' Release Series
+*************************
+
+- :yocto_docs:`2.5 BitBake User Manual </2.5/bitbake-user-manual/bitbake-user-manual.html>`
+- :yocto_docs:`2.5.1 BitBake User Manual </2.5.1/bitbake-user-manual/bitbake-user-manual.html>`
+- :yocto_docs:`2.5.2 BitBake User Manual </2.5.2/bitbake-user-manual/bitbake-user-manual.html>`
+- :yocto_docs:`2.5.3 BitBake User Manual </2.5.3/bitbake-user-manual/bitbake-user-manual.html>`
+
+**************************
+2.4 'rocko' Release Series
+**************************
+
+- :yocto_docs:`2.4 BitBake User Manual </2.4/bitbake-user-manual/bitbake-user-manual.html>`
+- :yocto_docs:`2.4.1 BitBake User Manual </2.4.1/bitbake-user-manual/bitbake-user-manual.html>`
+- :yocto_docs:`2.4.2 BitBake User Manual </2.4.2/bitbake-user-manual/bitbake-user-manual.html>`
+- :yocto_docs:`2.4.3 BitBake User Manual </2.4.3/bitbake-user-manual/bitbake-user-manual.html>`
+- :yocto_docs:`2.4.4 BitBake User Manual </2.4.4/bitbake-user-manual/bitbake-user-manual.html>`
+
+*************************
+2.3 'pyro' Release Series
+*************************
+
+- :yocto_docs:`2.3 BitBake User Manual </2.3/bitbake-user-manual/bitbake-user-manual.html>`
+- :yocto_docs:`2.3.1 BitBake User Manual </2.3.1/bitbake-user-manual/bitbake-user-manual.html>`
+- :yocto_docs:`2.3.2 BitBake User Manual </2.3.2/bitbake-user-manual/bitbake-user-manual.html>`
+- :yocto_docs:`2.3.3 BitBake User Manual </2.3.3/bitbake-user-manual/bitbake-user-manual.html>`
+- :yocto_docs:`2.3.4 BitBake User Manual </2.3.4/bitbake-user-manual/bitbake-user-manual.html>`
+
+**************************
+2.2 'morty' Release Series
+**************************
+
+- :yocto_docs:`2.2 BitBake User Manual </2.2/bitbake-user-manual/bitbake-user-manual.html>`
+- :yocto_docs:`2.2.1 BitBake User Manual </2.2.1/bitbake-user-manual/bitbake-user-manual.html>`
+- :yocto_docs:`2.2.2 BitBake User Manual </2.2.2/bitbake-user-manual/bitbake-user-manual.html>`
+- :yocto_docs:`2.2.3 BitBake User Manual </2.2.3/bitbake-user-manual/bitbake-user-manual.html>`
+
+****************************
+2.1 'krogoth' Release Series
+****************************
+
+- :yocto_docs:`2.1 BitBake User Manual </2.1/bitbake-user-manual/bitbake-user-manual.html>`
+- :yocto_docs:`2.1.1 BitBake User Manual </2.1.1/bitbake-user-manual/bitbake-user-manual.html>`
+- :yocto_docs:`2.1.2 BitBake User Manual </2.1.2/bitbake-user-manual/bitbake-user-manual.html>`
+- :yocto_docs:`2.1.3 BitBake User Manual </2.1.3/bitbake-user-manual/bitbake-user-manual.html>`
+
+***************************
+2.0 'jethro' Release Series
+***************************
+
+- :yocto_docs:`1.9 BitBake User Manual </1.9/bitbake-user-manual/bitbake-user-manual.html>`
+- :yocto_docs:`2.0 BitBake User Manual </2.0/bitbake-user-manual/bitbake-user-manual.html>`
+- :yocto_docs:`2.0.1 BitBake User Manual </2.0.1/bitbake-user-manual/bitbake-user-manual.html>`
+- :yocto_docs:`2.0.2 BitBake User Manual </2.0.2/bitbake-user-manual/bitbake-user-manual.html>`
+- :yocto_docs:`2.0.3 BitBake User Manual </2.0.3/bitbake-user-manual/bitbake-user-manual.html>`
+
+*************************
+1.8 'fido' Release Series
+*************************
+
+- :yocto_docs:`1.8 BitBake User Manual </1.8/bitbake-user-manual/bitbake-user-manual.html>`
+- :yocto_docs:`1.8.1 BitBake User Manual </1.8.1/bitbake-user-manual/bitbake-user-manual.html>`
+- :yocto_docs:`1.8.2 BitBake User Manual </1.8.2/bitbake-user-manual/bitbake-user-manual.html>`
+
+**************************
+1.7 'dizzy' Release Series
+**************************
+
+- :yocto_docs:`1.7 BitBake User Manual </1.7/bitbake-user-manual/bitbake-user-manual.html>`
+- :yocto_docs:`1.7.1 BitBake User Manual </1.7.1/bitbake-user-manual/bitbake-user-manual.html>`
+- :yocto_docs:`1.7.2 BitBake User Manual </1.7.2/bitbake-user-manual/bitbake-user-manual.html>`
+- :yocto_docs:`1.7.3 BitBake User Manual </1.7.3/bitbake-user-manual/bitbake-user-manual.html>`
+
+**************************
+1.6 'daisy' Release Series
+**************************
+
+- :yocto_docs:`1.6 BitBake User Manual </1.6/bitbake-user-manual/bitbake-user-manual.html>`
+- :yocto_docs:`1.6.1 BitBake User Manual </1.6.1/bitbake-user-manual/bitbake-user-manual.html>`
+- :yocto_docs:`1.6.2 BitBake User Manual </1.6.2/bitbake-user-manual/bitbake-user-manual.html>`
+- :yocto_docs:`1.6.3 BitBake User Manual </1.6.3/bitbake-user-manual/bitbake-user-manual.html>`
+

bitbake/doc/sphinx-static/switchers.js

@@ -0,0 +1,233 @@
+(function() {
+  'use strict';
+
+  var all_versions = {
+    'dev': 'dev (3.2)',
+    '3.1.2': '3.1.2',
+    '3.0.3': '3.0.3',
+    '2.7.4': '2.7.4',
+  };
+
+  var all_doctypes = {
+      'single': 'Individual Webpages',
+      'mega': "All-in-one 'Mega' Manual",
+  };
+
+  // Simple version comparision
+  // Return 1 if a > b
+  // Return -1 if a < b
+  // Return 0 if a == b
+  function ver_compare(a, b) {
+    if (a == "dev") {
+       return 1;
+    }
+
+    if (a === b) {
+       return 0;
+    }
+
+    var a_components = a.split(".");
+    var b_components = b.split(".");
+
+    var len = Math.min(a_components.length, b_components.length);
+
+    // loop while the components are equal
+    for (var i = 0; i < len; i++) {
+        // A bigger than B
+        if (parseInt(a_components[i]) > parseInt(b_components[i])) {
+            return 1;
+        }
+
+        // B bigger than A
+        if (parseInt(a_components[i]) < parseInt(b_components[i])) {
+            return -1;
+        }
+    }
+
+    // If one's a prefix of the other, the longer one is greater.
+    if (a_components.length > b_components.length) {
+        return 1;
+    }
+
+    if (a_components.length < b_components.length) {
+        return -1;
+    }
+
+    // Otherwise they are the same.
+    return 0;
+  }
+
+  function build_version_select(current_series, current_version) {
+    var buf = ['<select>'];
+
+    $.each(all_versions, function(version, title) {
+      var series = version.substr(0, 3);
+      if (series == current_series) {
+        if (version == current_version)
+            buf.push('<option value="' + version + '" selected="selected">' + title + '</option>');
+        else
+            buf.push('<option value="' + version + '">' + title + '</option>');
+
+        if (version != current_version)
+            buf.push('<option value="' + current_version + '" selected="selected">' + current_version + '</option>');
+      } else {
+        buf.push('<option value="' + version + '">' + title + '</option>');
+      }
+    });
+
+    buf.push('</select>');
+    return buf.join('');
+  }
+
+  function build_doctype_select(current_doctype) {
+    var buf = ['<select>'];
+
+    $.each(all_doctypes, function(doctype, title) {
+      if (doctype == current_doctype)
+        buf.push('<option value="' + doctype + '" selected="selected">' +
+                 all_doctypes[current_doctype] + '</option>');
+      else
+        buf.push('<option value="' + doctype + '">' + title + '</option>');
+    });
+    if (!(current_doctype in all_doctypes)) {
+        // In case we're browsing a doctype that is not yet in all_doctypes.
+        buf.push('<option value="' + current_doctype + '" selected="selected">' +
+                 current_doctype + '</option>');
+        all_doctypes[current_doctype] = current_doctype;
+    }
+    buf.push('</select>');
+    return buf.join('');
+  }
+
+  function navigate_to_first_existing(urls) {
+    // Navigate to the first existing URL in urls.
+    var url = urls.shift();
+
+    // Web browsers won't redirect file:// urls to file urls using ajax but
+    // its useful for local testing
+    if (url.startsWith("file://")) {
+      window.location.href = url;
+      return;
+    }
+
+    if (urls.length == 0) {
+      window.location.href = url;
+      return;
+    }
+    $.ajax({
+      url: url,
+      success: function() {
+        window.location.href = url;
+      },
+      error: function() {
+        navigate_to_first_existing(urls);
+      }
+    });
+  }
+
+  function get_docroot_url() {
+    var url = window.location.href;
+    var root = DOCUMENTATION_OPTIONS.URL_ROOT;
+
+    var urlarray = url.split('/');
+    // Trim off anything after '/'
+    urlarray.pop();
+    var depth = (root.match(/\.\.\//g) || []).length;
+    for (var i = 0; i < depth; i++) {
+      urlarray.pop();
+    }
+
+    return urlarray.join('/') + '/';
+  }
+
+  function on_version_switch() {
+    var selected_version = $(this).children('option:selected').attr('value');
+    var url = window.location.href;
+    var current_version = DOCUMENTATION_OPTIONS.VERSION;
+    var docroot = get_docroot_url()
+
+    var new_versionpath = selected_version + '/';
+    if (selected_version == "dev")
+        new_versionpath = '';
+
+    // dev versions have no version prefix
+    if (current_version == "dev") {
+        var new_url = docroot + new_versionpath + url.replace(docroot, "");
+        var fallback_url = docroot + new_versionpath;
+    } else {
+        var new_url = url.replace('/' + current_version + '/', '/' + new_versionpath);
+        var fallback_url = new_url.replace(url.replace(docroot, ""), "");
+    }
+
+    console.log(get_docroot_url())
+    console.log(url + " to url " + new_url);
+    console.log(url + " to fallback " + fallback_url);
+
+    if (new_url != url) {
+      navigate_to_first_existing([
+        new_url,
+        fallback_url,
+        'https://www.yoctoproject.org/docs/',
+      ]);
+    }
+  }
+
+  function on_doctype_switch() {
+    var selected_doctype = $(this).children('option:selected').attr('value');
+    var url = window.location.href;
+    if (selected_doctype == 'mega') {
+      var docroot = get_docroot_url()
+      var current_version = DOCUMENTATION_OPTIONS.VERSION;
+      // Assume manuals before 3.2 are using old docbook mega-manual
+      if (ver_compare(current_version, "3.2") < 0) {
+        var new_url = docroot + "mega-manual/mega-manual.html";
+      } else {
+        var new_url = docroot + "singleindex.html";
+      }
+    } else {
+      var new_url = url.replace("singleindex.html", "index.html")
+    }
+
+    if (new_url != url) {
+      navigate_to_first_existing([
+        new_url,
+        'https://www.yoctoproject.org/docs/',
+      ]);
+    }
+  }
+
+  // Returns the current doctype based upon the url
+  function doctype_segment_from_url(url) {
+    if (url.includes("singleindex") || url.includes("mega-manual"))
+      return "mega";
+    return "single";
+  }
+
+  $(document).ready(function() {
+    var release = DOCUMENTATION_OPTIONS.VERSION;
+    var current_doctype = doctype_segment_from_url(window.location.href);
+    var current_series = release.substr(0, 3);
+    var version_select = build_version_select(current_series, release);
+
+    $('.version_switcher_placeholder').html(version_select);
+    $('.version_switcher_placeholder select').bind('change', on_version_switch);
+
+    var doctype_select = build_doctype_select(current_doctype);
+
+    $('.doctype_switcher_placeholder').html(doctype_select);
+    $('.doctype_switcher_placeholder select').bind('change', on_doctype_switch);
+
+    if (ver_compare(release, "3.1") < 0) {
+      $('#outdated-warning').html('Version ' + release + ' of the project is now considered obsolete, please select and use a more recent version');
+      $('#outdated-warning').css('padding', '.5em');
+    } else if (release != "dev") {
+      $.each(all_versions, function(version, title) {
+        var series = version.substr(0, 3);
+        if (series == current_series && version != release) {
+          $('#outdated-warning').html('This document is for outdated version ' + release + ', you should select the latest release version in this series, ' + version + '.');
+          $('#outdated-warning').css('padding', '.5em');
+        }
+      });
+    }
+  });
+})();

bitbake/doc/sphinx-static/theme_overrides.css

@@ -0,0 +1,162 @@
+/*
+    SPDX-License-Identifier: CC-BY-2.0-UK
+*/
+
+body {
+  font-family: Verdana, Sans, sans-serif;
+  margin:  0em auto;
+  color: #333;
+}
+
+h1,h2,h3,h4,h5,h6,h7 {
+  font-family: Arial, Sans;
+  color: #00557D;
+  clear: both;
+}
+
+h1 {
+  font-size: 2em;
+  text-align: left;
+  padding: 0em 0em 0em 0em;
+  margin: 2em 0em 0em 0em;
+}
+
+h2.subtitle {
+  margin: 0.10em 0em 3.0em 0em;
+  padding: 0em 0em 0em 0em;
+  font-size: 1.8em;
+  padding-left: 20%;
+  font-weight: normal;
+  font-style: italic;
+}
+
+h2 {
+  margin: 2em 0em 0.66em 0em;
+  padding: 0.5em 0em 0em 0em;
+  font-size: 1.5em;
+  font-weight: bold;
+}
+
+h3.subtitle {
+  margin: 0em 0em 1em 0em;
+  padding: 0em 0em 0em 0em;
+  font-size: 142.14%;
+  text-align: right;
+}
+
+h3 {
+  margin: 1em 0em 0.5em 0em;
+  padding: 1em 0em 0em 0em;
+  font-size: 140%;
+  font-weight: bold;
+}
+
+h4 {
+  margin: 1em 0em 0.5em 0em;
+  padding: 1em 0em 0em 0em;
+  font-size: 120%;
+  font-weight: bold;
+}
+
+h5 {
+  margin: 1em 0em 0.5em 0em;
+  padding: 1em 0em 0em 0em;
+  font-size: 110%;
+  font-weight: bold;
+}
+
+h6 {
+  margin: 1em 0em 0em 0em;
+  padding: 1em 0em 0em 0em;
+  font-size: 110%;
+  font-weight: bold;
+}
+
+em {
+  font-weight: bold;
+}
+
+.pre {
+  font-size: medium;
+  font-family: Courier, monospace;
+}
+
+.wy-nav-content a {
+  text-decoration: underline;
+  color: #444;
+  background: transparent;
+}
+
+.wy-nav-content a:hover {
+  text-decoration: underline;
+  background-color: #dedede;
+}
+
+.wy-nav-content a:visited {
+  color: #444;
+}
+
+[alt='Permalink'] { color: #eee; }
+[alt='Permalink']:hover { color: black; }
+
+@media screen {
+    /* content column
+     *
+     * RTD theme's default is 800px as max width for the content, but we have
+     * tables with tons of columns, which need the full width of the view-port.
+     */
+
+    .wy-nav-content{max-width: none; }
+
+    /* inline literal: drop the borderbox, padding and red color */
+    code, .rst-content tt, .rst-content code {
+        color: inherit;
+        border: none;
+        padding: unset;
+        background: inherit;
+        font-size: 85%;
+    }
+
+    .rst-content tt.literal,.rst-content tt.literal,.rst-content code.literal {
+        color: inherit;
+    }
+
+    /* Admonition should be gray, not blue or green */
+    .rst-content .note .admonition-title,
+    .rst-content .tip .admonition-title,
+    .rst-content .warning .admonition-title,
+    .rst-content .caution .admonition-title,
+    .rst-content .important .admonition-title {
+        background: #f0f0f2;
+        color: #00557D;
+
+    }
+
+    .rst-content .note,
+    .rst-content .tip,
+    .rst-content .important,
+    .rst-content .warning,
+    .rst-content .caution  {
+        background: #f0f0f2;
+    }
+
+    /* Remove the icon in front of note/tip element, and before the logo */
+    .icon-home:before, .rst-content .admonition-title:before {
+        display: none
+    }
+
+    /* a custom informalexample container is used in some doc */
+    .informalexample {
+        border: 1px solid;
+        border-color: #aaa;
+        margin: 1em 0em;
+        padding: 1em;
+        page-break-inside: avoid;
+    }
+
+    /* Remove the blue background in the top left corner, around the logo */
+    .wy-side-nav-search {
+        background: inherit;
+    }
+
+}

bitbake/doc/template/component.title.xsl

@@ -1,39 +0,0 @@
-<xsl:stylesheet version="1.0"
-  xmlns:xsl="http://www.w3.org/1999/XSL/Transform"
-  xmlns:d="http://docbook.org/ns/docbook"
-  xmlns="http://www.w3.org/1999/xhtml"
-  exclude-result-prefixes="d">
-  
-  <xsl:template name="component.title">
-    <xsl:param name="node" select="."/>
-    
-    <xsl:variable name="level">
-      <xsl:choose>
-        <xsl:when test="ancestor::d:section">
-          <xsl:value-of select="count(ancestor::d:section)+1"/>
-        </xsl:when>
-        <xsl:when test="ancestor::d:sect5">6</xsl:when>
-        <xsl:when test="ancestor::d:sect4">5</xsl:when>
-        <xsl:when test="ancestor::d:sect3">4</xsl:when>
-        <xsl:when test="ancestor::d:sect2">3</xsl:when>
-        <xsl:when test="ancestor::d:sect1">2</xsl:when>
-        <xsl:otherwise>1</xsl:otherwise>
-      </xsl:choose>
-    </xsl:variable>
-    <xsl:element name="h{$level+1}" namespace="http://www.w3.org/1999/xhtml">
-      <xsl:attribute name="class">title</xsl:attribute>
-      <xsl:if test="$generate.id.attributes = 0">
-        <xsl:call-template name="anchor">
-          <xsl:with-param name="node" select="$node"/>
-          <xsl:with-param name="conditional" select="0"/>
-        </xsl:call-template>
-      </xsl:if>
-      <xsl:apply-templates select="$node" mode="object.title.markup">
-        <xsl:with-param name="allow-anchors" select="1"/>
-      </xsl:apply-templates>
-      <xsl:call-template name="permalink">
-        <xsl:with-param name="node" select="$node"/>
-      </xsl:call-template>
-    </xsl:element>
-  </xsl:template>
-</xsl:stylesheet>

bitbake/doc/template/db-pdf.xsl

@@ -1,64 +0,0 @@
-<?xml version='1.0'?>
-<xsl:stylesheet xmlns:xsl="http://www.w3.org/1999/XSL/Transform" xmlns="http://www.w3.org/1999/xhtml" xmlns:fo="http://www.w3.org/1999/XSL/Format" version="1.0">
-  
-  <xsl:import href="http://docbook.sourceforge.net/release/xsl/current/fo/docbook.xsl" />
-
-  <!-- check project-plan.sh for how this is generated, needed to tweak 
-       the cover page     
-    -->
-  <xsl:include href="/tmp/titlepage.xsl"/> 
-
-  <!-- To force a page break in document, i.e per section add a 
-      <?hard-pagebreak?> tag.
-  -->
- <xsl:template match="processing-instruction('hard-pagebreak')">
-   <fo:block break-before='page' />
- </xsl:template>
-
-  <!--Fix for defualt indent getting TOC all wierd..
-      See http://sources.redhat.com/ml/docbook-apps/2005-q1/msg00455.html 
-      FIXME: must be a better fix
-    -->
-  <xsl:param name="body.start.indent" select="'0'"/>
-  <!--<xsl:param name="title.margin.left" select="'0'"/>-->
-
-  <!-- stop long-ish header titles getting wrapped -->
-  <xsl:param name="header.column.widths">1 10 1</xsl:param>
-
-  <!-- customise headers and footers a little --> 
-
-  <xsl:template name="head.sep.rule">
-   <xsl:if test="$header.rule != 0">
-     <xsl:attribute name="border-bottom-width">0.5pt</xsl:attribute>
-     <xsl:attribute name="border-bottom-style">solid</xsl:attribute>
-     <xsl:attribute name="border-bottom-color">#cccccc</xsl:attribute>
-   </xsl:if>
-  </xsl:template>
-
-  <xsl:template name="foot.sep.rule">
-    <xsl:if test="$footer.rule != 0">
-     <xsl:attribute name="border-top-width">0.5pt</xsl:attribute>
-     <xsl:attribute name="border-top-style">solid</xsl:attribute>
-     <xsl:attribute name="border-top-color">#cccccc</xsl:attribute>
-    </xsl:if>
-  </xsl:template>
-
-  <xsl:attribute-set name="header.content.properties">
-    <xsl:attribute name="color">#cccccc</xsl:attribute>
-  </xsl:attribute-set>
-
-  <xsl:attribute-set name="footer.content.properties">
-    <xsl:attribute name="color">#cccccc</xsl:attribute>
-  </xsl:attribute-set>
-
- 
-  <!-- general settings -->
-
-  <xsl:param name="fop1.extensions" select="1"></xsl:param>
-  <xsl:param name="paper.type" select="'A4'"></xsl:param>
-  <xsl:param name="section.autolabel" select="1"></xsl:param>
-  <xsl:param name="body.font.family" select="'verasans'"></xsl:param>
-  <xsl:param name="title.font.family" select="'verasans'"></xsl:param>
-  <xsl:param name="monospace.font.family" select="'veramono'"></xsl:param>
-
-</xsl:stylesheet>

bitbake/doc/template/division.title.xsl

@@ -1,25 +0,0 @@
-<xsl:stylesheet version="1.0"
-  xmlns:xsl="http://www.w3.org/1999/XSL/Transform"
-  xmlns:d="http://docbook.org/ns/docbook"
-  xmlns="http://www.w3.org/1999/xhtml"
-  exclude-result-prefixes="d">
-  
-  <xsl:template name="division.title">
-    <xsl:param name="node" select="."/>
-    
-    <h1>
-      <xsl:attribute name="class">title</xsl:attribute>
-      <xsl:call-template name="anchor">
-        <xsl:with-param name="node" select="$node"/>
-        <xsl:with-param name="conditional" select="0"/>
-      </xsl:call-template>
-      <xsl:apply-templates select="$node" mode="object.title.markup">
-        <xsl:with-param name="allow-anchors" select="1"/>
-      </xsl:apply-templates>
-      <xsl:call-template name="permalink">
-        <xsl:with-param name="node" select="$node"/>
-      </xsl:call-template>
-    </h1>
-  </xsl:template>
-</xsl:stylesheet>
-

bitbake/doc/template/fop-config.xml

@@ -1,58 +0,0 @@
-<fop version="1.0">
-
-  <!-- Strict user configuration -->
-  <strict-configuration>true</strict-configuration>
-
-  <!-- Strict FO validation -->
-  <strict-validation>true</strict-validation>
-
-   <!--
-    Set the baseDir so common/openedhand.svg references in plans still
-    work ok. Note, relative file references to current dir should still work.
-    -->	
-  <base>../template</base>
-  <font-base>../template</font-base>
- 
-  <!-- Source resolution in dpi (dots/pixels per inch) for determining the
-       size of pixels in SVG and bitmap images, default: 72dpi -->
-  <!-- <source-resolution>72</source-resolution> -->
-  <!-- Target resolution in dpi (dots/pixels per inch) for specifying the
-       target resolution for generated bitmaps, default: 72dpi -->
-  <!-- <target-resolution>72</target-resolution> -->
- 
-  <!-- default page-height and page-width, in case
-       value is specified as auto -->
-  <default-page-settings height="11in" width="8.26in"/> 
- 
-  <!-- <use-cache>false</use-cache> -->
- 
-  <renderers>
-    <renderer mime="application/pdf">
-      <fonts>
-        <font  metrics-file="VeraMono.xml"
-               kerning="yes" 
-               embed-url="VeraMono.ttf">
-          <font-triplet name="veramono" style="normal" weight="normal"/>
-        </font>
-
-        <font  metrics-file="VeraMoBd.xml"
-               kerning="yes" 
-               embed-url="VeraMoBd.ttf">
-          <font-triplet name="veramono" style="normal" weight="bold"/>
-        </font>
-
-        <font  metrics-file="Vera.xml"
-               kerning="yes" 
-               embed-url="Vera.ttf">
-          <font-triplet name="verasans" style="normal" weight="normal"/>
-          <font-triplet name="verasans" style="normal" weight="bold"/>
-          <font-triplet name="verasans" style="italic" weight="normal"/>
-          <font-triplet name="verasans" style="italic" weight="bold"/>
-        </font>
-        
-        <auto-detect/>
-      </fonts>
-    </renderer>
-  </renderers>
-</fop>
-

bitbake/doc/template/formal.object.heading.xsl

@@ -1,21 +0,0 @@
-<xsl:stylesheet version="1.0"
-  xmlns:xsl="http://www.w3.org/1999/XSL/Transform"
-  xmlns:d="http://docbook.org/ns/docbook"
-  xmlns="http://www.w3.org/1999/xhtml"
-  exclude-result-prefixes="d">
-  
-  <xsl:template name="formal.object.heading">
-    <xsl:param name="object" select="."/>
-    <xsl:param name="title">
-      <xsl:apply-templates select="$object" mode="object.title.markup">
-        <xsl:with-param name="allow-anchors" select="1"/>
-      </xsl:apply-templates>
-    </xsl:param>
-    <p class="title">
-      <b><xsl:copy-of select="$title"/></b>
-      <xsl:call-template name="permalink">
-        <xsl:with-param name="node" select="$object"/>
-      </xsl:call-template>
-    </p>
-  </xsl:template>
-</xsl:stylesheet>

bitbake/doc/template/gloss-permalinks.xsl

@@ -1,14 +0,0 @@
-<xsl:stylesheet version="1.0"
-  xmlns:xsl="http://www.w3.org/1999/XSL/Transform"
-  xmlns:d="http://docbook.org/ns/docbook"
-  xmlns="http://www.w3.org/1999/xhtml">
-
-  <xsl:template match="glossentry/glossterm">
-    <xsl:apply-imports/>
-    <xsl:if test="$generate.permalink != 0">
-      <xsl:call-template name="permalink">
-        <xsl:with-param name="node" select=".."/>
-      </xsl:call-template>
-    </xsl:if>
-  </xsl:template>
-</xsl:stylesheet>

bitbake/doc/template/permalinks.xsl

@@ -1,25 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<xsl:stylesheet version="1.0"
-  xmlns="http://www.w3.org/1999/xhtml"
-  xmlns:xsl="http://www.w3.org/1999/XSL/Transform">
-
-  <xsl:param name="generate.permalink" select="1"/>
-  <xsl:param name="permalink.text">¶</xsl:param>
-
-  <xsl:template name="permalink">
-    <xsl:param name="node"/>
-
-    <xsl:if test="$generate.permalink != '0'">
-      <span class="permalink">
-        <a alt="Permalink" title="Permalink">
-          <xsl:attribute name="href">
-            <xsl:call-template name="href.target">
-              <xsl:with-param name="object"  select="$node"/>
-            </xsl:call-template>
-          </xsl:attribute>
-          <xsl:copy-of select="$permalink.text"/>
-        </a>
-      </span>
-    </xsl:if>
-  </xsl:template>
-</xsl:stylesheet>

bitbake/doc/template/section.title.xsl

@@ -1,55 +0,0 @@
-<xsl:stylesheet version="1.0"
-  xmlns:xsl="http://www.w3.org/1999/XSL/Transform"
-  xmlns:d="http://docbook.org/ns/docbook"
-  xmlns="http://www.w3.org/1999/xhtml" exclude-result-prefixes="d">
-
-  <xsl:template name="section.title">
-    <xsl:variable name="section"
-      select="(ancestor::section |
-               ancestor::simplesect|
-               ancestor::sect1|
-               ancestor::sect2|
-               ancestor::sect3|
-               ancestor::sect4|
-               ancestor::sect5)[last()]"/>
-
-    <xsl:variable name="renderas">
-      <xsl:choose>
-        <xsl:when test="$section/@renderas = 'sect1'">1</xsl:when>
-        <xsl:when test="$section/@renderas = 'sect2'">2</xsl:when>
-        <xsl:when test="$section/@renderas = 'sect3'">3</xsl:when>
-        <xsl:when test="$section/@renderas = 'sect4'">4</xsl:when>
-        <xsl:when test="$section/@renderas = 'sect5'">5</xsl:when>
-        <xsl:otherwise><xsl:value-of select="''"/></xsl:otherwise>
-      </xsl:choose>
-    </xsl:variable>
-
-    <xsl:variable name="level">
-      <xsl:choose>
-        <xsl:when test="$renderas != ''">
-          <xsl:value-of select="$renderas"/>
-        </xsl:when>
-        <xsl:otherwise>
-          <xsl:call-template name="section.level">
-            <xsl:with-param name="node" select="$section"/>
-          </xsl:call-template>
-        </xsl:otherwise>
-      </xsl:choose>
-    </xsl:variable>
-
-    <xsl:call-template name="section.heading">
-      <xsl:with-param name="section" select="$section"/>
-      <xsl:with-param name="level" select="$level"/>
-      <xsl:with-param name="title">
-        <xsl:apply-templates select="$section" mode="object.title.markup">
-          <xsl:with-param name="allow-anchors" select="1"/>
-        </xsl:apply-templates>
-        <xsl:if test="$level &gt; 0">
-          <xsl:call-template name="permalink">
-            <xsl:with-param name="node" select="$section"/>
-          </xsl:call-template>
-        </xsl:if>
-      </xsl:with-param>
-    </xsl:call-template>
-  </xsl:template>
-</xsl:stylesheet>

bitbake/doc/tools/docbook-to-pdf

@@ -1,51 +0,0 @@
-#!/bin/sh
-	
-if [ -z "$1" -o -z "$2" ]; then
-   echo "usage: [-v] $0 <docbook file> <templatedir>"
-   echo
-   echo "*NOTE* you need xsltproc, fop and nwalsh docbook stylesheets" 
-   echo "       installed for this to work!"
-   echo
-   exit 0
-fi
-
-FO=`echo $1 | sed s/.xml/.fo/` || exit 1
-PDF=`echo $1 | sed s/.xml/.pdf/` || exit 1
-TEMPLATEDIR=$2
-
-##
-# These URI should be rewritten by your distribution's xml catalog to
-# match your localy installed XSL stylesheets.
-XSL_BASE_URI="http://docbook.sourceforge.net/release/xsl/current"
-
-# Creates a temporary XSL stylesheet based on titlepage.xsl
-xsltproc -o /tmp/titlepage.xsl                                           \
-	 --xinclude                                                      \
-         $XSL_BASE_URI/template/titlepage.xsl \
-         $TEMPLATEDIR/titlepage.templates.xml || exit 1
-
-# Creates the file needed for FOP
-xsltproc --xinclude                    \
-	 --stringparam hyphenate false \
-	 --stringparam formal.title.placement "figure after" \
-	 --stringparam ulink.show 1 \
-         --stringparam  body.font.master  9 \
-         --stringparam  title.font.master  11 \
-         --stringparam draft.watermark.image "$TEMPLATEDIR/draft.png" \
-         --stringparam  chapter.autolabel 1 \
-         --stringparam  appendix.autolabel A \
-         --stringparam  section.autolabel 1 \
-         --stringparam  section.label.includes.component.label 1 \
-         --output $FO               \
-         $TEMPLATEDIR/db-pdf.xsl    \
-	 $1                 || exit 1
-
-# Invokes the Java version of FOP.  Uses the additional configuration file common/fop-config.xml
-fop -c $TEMPLATEDIR/fop-config.xml -fo $FO -pdf $PDF       || exit 1
-
-rm -f $FO
-rm -f  /tmp/titlepage.xsl
-
-echo
-echo " #### Success! $PDF ready. ####"
-echo

bitbake/lib/bb/COW.py

@@ -3,14 +3,14 @@
 #
 # Copyright (C) 2006 Tim Ansell
 #
-#Please Note:
+# Please Note:
 # Be careful when using mutable types (ie Dict and Lists) - operations involving these are SLOW.
 # Assign a file to __warn__ to get warnings about slow operations.
 #
 
 
 import copy
-import types
+
 ImmutableTypes = (
     bool,
     complex,
@@ -23,9 +23,11 @@ ImmutableTypes = (
 
 MUTABLE = "__mutable__"
 
+
 class COWMeta(type):
     pass
 
+
 class COWDictMeta(COWMeta):
     __warn__ = False
     __hasmutable__ = False
@@ -34,12 +36,15 @@ class COWDictMeta(COWMeta):
     def __str__(cls):
         # FIXME: I have magic numbers!
         return "<COWDict Level: %i Current Keys: %i>" % (cls.__count__, len(cls.__dict__) - 3)
+
     __repr__ = __str__
 
     def cow(cls):
         class C(cls):
             __count__ = cls.__count__ + 1
+
         return C
+
     copy = cow
     __call__ = cow
 
@@ -71,8 +76,9 @@ class COWDictMeta(COWMeta):
         return value
 
     __getmarker__ = []
+
     def __getreadonly__(cls, key, default=__getmarker__):
-        """\
+        """
         Get a value (even if mutable) which you promise not to change.
         """
         return cls.__getitem__(key, default, True)
@@ -139,24 +145,29 @@ class COWDictMeta(COWMeta):
 
     def iterkeys(cls):
         return cls.iter("keys")
+
     def itervalues(cls, readonly=False):
         if not cls.__warn__ is False and cls.__hasmutable__ and readonly is False:
-            print("Warning: If you arn't going to change any of the values call with True.", file=cls.__warn__)
+            print("Warning: If you aren't going to change any of the values call with True.", file=cls.__warn__)
         return cls.iter("values", readonly)
+
     def iteritems(cls, readonly=False):
         if not cls.__warn__ is False and cls.__hasmutable__ and readonly is False:
-            print("Warning: If you arn't going to change any of the values call with True.", file=cls.__warn__)
+            print("Warning: If you aren't going to change any of the values call with True.", file=cls.__warn__)
         return cls.iter("items", readonly)
 
+
 class COWSetMeta(COWDictMeta):
     def __str__(cls):
         # FIXME: I have magic numbers!
-        return "<COWSet Level: %i Current Keys: %i>" % (cls.__count__, len(cls.__dict__) -3)
+        return "<COWSet Level: %i Current Keys: %i>" % (cls.__count__, len(cls.__dict__) - 3)
+
     __repr__ = __str__
 
     def cow(cls):
         class C(cls):
             __count__ = cls.__count__ + 1
+
         return C
 
     def add(cls, value):
@@ -174,131 +185,11 @@ class COWSetMeta(COWDictMeta):
     def iteritems(cls):
         raise TypeError("sets don't have 'items'")
 
+
 # These are the actual classes you use!
-class COWDictBase(object, metaclass = COWDictMeta):
+class COWDictBase(metaclass=COWDictMeta):
     __count__ = 0
 
-class COWSetBase(object, metaclass = COWSetMeta):
+
+class COWSetBase(metaclass=COWSetMeta):
     __count__ = 0
-
-if __name__ == "__main__":
-    import sys
-    COWDictBase.__warn__ = sys.stderr
-    a = COWDictBase()
-    print("a", a)
-
-    a['a'] = 'a'
-    a['b'] = 'b'
-    a['dict'] = {}
-
-    b = a.copy()
-    print("b", b)
-    b['c'] = 'b'
-
-    print()
-
-    print("a", a)
-    for x in a.iteritems():
-        print(x)
-    print("--")
-    print("b", b)
-    for x in b.iteritems():
-        print(x)
-    print()
-
-    b['dict']['a'] = 'b'
-    b['a'] = 'c'
-
-    print("a", a)
-    for x in a.iteritems():
-        print(x)
-    print("--")
-    print("b", b)
-    for x in b.iteritems():
-        print(x)
-    print()
-
-    try:
-        b['dict2']
-    except KeyError as e:
-        print("Okay!")
-
-    a['set'] = COWSetBase()
-    a['set'].add("o1")
-    a['set'].add("o1")
-    a['set'].add("o2")
-
-    print("a", a)
-    for x in a['set'].itervalues():
-        print(x)
-    print("--")
-    print("b", b)
-    for x in b['set'].itervalues():
-        print(x)
-    print()
-
-    b['set'].add('o3')
-
-    print("a", a)
-    for x in a['set'].itervalues():
-        print(x)
-    print("--")
-    print("b", b)
-    for x in b['set'].itervalues():
-        print(x)
-    print()
-
-    a['set2'] = set()
-    a['set2'].add("o1")
-    a['set2'].add("o1")
-    a['set2'].add("o2")
-
-    print("a", a)
-    for x in a.iteritems():
-        print(x)
-    print("--")
-    print("b", b)
-    for x in b.iteritems(readonly=True):
-        print(x)
-    print()
-
-    del b['b']
-    try:
-        print(b['b'])
-    except KeyError:
-        print("Yay! deleted key raises error")
-
-    if 'b' in b:
-        print("Boo!")
-    else:
-        print("Yay - has_key with delete works!")
-
-    print("a", a)
-    for x in a.iteritems():
-        print(x)
-    print("--")
-    print("b", b)
-    for x in b.iteritems(readonly=True):
-        print(x)
-    print()
-
-    b.__revertitem__('b')
-
-    print("a", a)
-    for x in a.iteritems():
-        print(x)
-    print("--")
-    print("b", b)
-    for x in b.iteritems(readonly=True):
-        print(x)
-    print()
-
-    b.__revertitem__('dict')
-    print("a", a)
-    for x in a.iteritems():
-        print(x)
-    print("--")
-    print("b", b)
-    for x in b.iteritems(readonly=True):
-        print(x)
-    print()

bitbake/lib/bb/__init__.py

@@ -9,11 +9,11 @@
 # SPDX-License-Identifier: GPL-2.0-only
 #
 
-__version__ = "1.44.0"
+__version__ = "1.48.0"
 
 import sys
-if sys.version_info < (3, 4, 0):
-    raise RuntimeError("Sorry, python 3.4.0 or later is required for this version of bitbake")
+if sys.version_info < (3, 5, 0):
+    raise RuntimeError("Sorry, python 3.5.0 or later is required for this version of bitbake")
 
 
 class BBHandledException(Exception):
@@ -35,20 +35,23 @@ class NullHandler(logging.Handler):
     def emit(self, record):
         pass
 
-Logger = logging.getLoggerClass()
-class BBLogger(Logger):
-    def __init__(self, name):
+class BBLoggerMixin(object):
+    def __init__(self, *args, **kwargs):
+        # Does nothing to allow calling super() from derived classes
+        pass
+
+    def setup_bblogger(self, name):
         if name.split(".")[0] == "BitBake":
             self.debug = self.bbdebug
-        Logger.__init__(self, name)
 
     def bbdebug(self, level, msg, *args, **kwargs):
+        loglevel = logging.DEBUG - level + 1
         if not bb.event.worker_pid:
-            if self.name in bb.msg.loggerDefaultDomains and level > (bb.msg.loggerDefaultDomains[self.name]):
+            if self.name in bb.msg.loggerDefaultDomains and loglevel > (bb.msg.loggerDefaultDomains[self.name]):
                 return
-            if level > (bb.msg.loggerDefaultDebugLevel):
+            if loglevel > bb.msg.loggerDefaultLogLevel:
                 return
-        return self.log(logging.DEBUG - level + 1, msg, *args, **kwargs)
+        return self.log(loglevel, msg, *args, **kwargs)
 
     def plain(self, msg, *args, **kwargs):
         return self.log(logging.INFO + 1, msg, *args, **kwargs)
@@ -59,16 +62,56 @@ class BBLogger(Logger):
     def verbnote(self, msg, *args, **kwargs):
         return self.log(logging.INFO + 2, msg, *args, **kwargs)
 
+Logger = logging.getLoggerClass()
+class BBLogger(Logger, BBLoggerMixin):
+    def __init__(self, name, *args, **kwargs):
+        self.setup_bblogger(name)
+        super().__init__(name, *args, **kwargs)
 
 logging.raiseExceptions = False
 logging.setLoggerClass(BBLogger)
 
+class BBLoggerAdapter(logging.LoggerAdapter, BBLoggerMixin):
+    def __init__(self, logger, *args, **kwargs):
+        self.setup_bblogger(logger.name)
+        super().__init__(logger, *args, **kwargs)
+
+    if sys.version_info < (3, 6):
+        # These properties were added in Python 3.6. Add them in older versions
+        # for compatibility
+        @property
+        def manager(self):
+            return self.logger.manager
+
+        @manager.setter
+        def manager(self, value):
+            self.logger.manager = value
+
+        @property
+        def name(self):
+            return self.logger.name
+
+        def __repr__(self):
+            logger = self.logger
+            level = logger.getLevelName(logger.getEffectiveLevel())
+            return '<%s %s (%s)>' % (self.__class__.__name__, logger.name, level)
+
+logging.LoggerAdapter = BBLoggerAdapter
+
 logger = logging.getLogger("BitBake")
 logger.addHandler(NullHandler())
 logger.setLevel(logging.DEBUG - 2)
 
 mainlogger = logging.getLogger("BitBake.Main")
 
+class PrefixLoggerAdapter(logging.LoggerAdapter):
+    def __init__(self, prefix, logger):
+        super().__init__(logger, {})
+        self.__msg_prefix = prefix
+
+    def process(self, msg, kwargs):
+        return "%s%s" %(self.__msg_prefix, msg), kwargs
+
 # This has to be imported after the setLoggerClass, as the import of bb.msg
 # can result in construction of the various loggers.
 import bb.msg

bitbake/lib/bb/build.py

@@ -15,9 +15,10 @@
 import os
 import sys
 import logging
-import shlex
 import glob
+import itertools
 import time
+import re
 import stat
 import bb
 import bb.msg
@@ -28,6 +29,9 @@ from bb import data, event, utils
 bblogger = logging.getLogger('BitBake')
 logger = logging.getLogger('BitBake.Build')
 
+verboseShellLogging = False
+verboseStdoutLogging = False
+
 __mtime_cache = {}
 
 def cached_mtime_noerror(f):
@@ -66,6 +70,8 @@ class TaskBase(event.Event):
         self.taskname = self._task
         self.logfile = logfile
         self.time = time.time()
+        self.pn = d.getVar("PN")
+        self.pv = d.getVar("PV")
         event.Event.__init__(self)
         self._message = "recipe %s: task %s: %s" % (d.getVar("PF"), t, self.getDisplayName())
 
@@ -302,20 +308,60 @@ def exec_func_python(func, d, runfile, cwd=None):
 
 def shell_trap_code():
     return '''#!/bin/sh\n
+__BITBAKE_LAST_LINE=0
+
 # Emit a useful diagnostic if something fails:
-bb_exit_handler() {
+bb_sh_exit_handler() {
     ret=$?
-    case $ret in
-    0)  ;;
-    *)  case $BASH_VERSION in
-        "") echo "WARNING: exit code $ret from a shell command.";;
-        *)  echo "WARNING: ${BASH_SOURCE[0]}:${BASH_LINENO[0]} exit $ret from '$BASH_COMMAND'";;
-        esac
-        exit $ret
-    esac
+    if [ "$ret" != 0 ]; then
+        echo "WARNING: exit code $ret from a shell command."
+    fi
+    exit $ret
 }
-trap 'bb_exit_handler' 0
-set -e
+
+bb_bash_exit_handler() {
+    ret=$?
+    { set +x; } > /dev/null
+    trap "" DEBUG
+    if [ "$ret" != 0 ]; then
+        echo "WARNING: ${BASH_SOURCE[0]}:${__BITBAKE_LAST_LINE} exit $ret from '$1'"
+
+        echo "WARNING: Backtrace (BB generated script): "
+        for i in $(seq 1 $((${#FUNCNAME[@]} - 1))); do
+            if [ "$i" -eq 1 ]; then
+                echo -e "\t#$((i)): ${FUNCNAME[$i]}, ${BASH_SOURCE[$((i-1))]}, line ${__BITBAKE_LAST_LINE}"
+            else
+                echo -e "\t#$((i)): ${FUNCNAME[$i]}, ${BASH_SOURCE[$((i-1))]}, line ${BASH_LINENO[$((i-1))]}"
+            fi
+        done
+    fi
+    exit $ret
+}
+
+bb_bash_debug_handler() {
+    local line=${BASH_LINENO[0]}
+    # For some reason the DEBUG trap trips with lineno=1 when scripts exit; ignore it
+    if [ "$line" -eq 1 ]; then
+        return
+    fi
+
+    # Track the line number of commands as they execute. This is so we can have access to the failing line number
+    # in the EXIT trap. See http://gnu-bash.2382.n7.nabble.com/trap-echo-quot-trap-exit-on-LINENO-quot-EXIT-gt-wrong-linenumber-td3666.html
+    if [ "${FUNCNAME[1]}" != "bb_bash_exit_handler" ]; then
+        __BITBAKE_LAST_LINE=$line
+    fi
+}
+
+case $BASH_VERSION in
+"") trap 'bb_sh_exit_handler' 0
+    set -e
+    ;;
+*)  trap 'bb_bash_exit_handler "$BASH_COMMAND"' 0
+    trap '{ bb_bash_debug_handler; } 2>/dev/null' DEBUG
+    set -e
+    shopt -s extdebug
+    ;;
+esac
 '''
 
 def create_progress_handler(func, progress, logfile, d):
@@ -345,7 +391,7 @@ def create_progress_handler(func, progress, logfile, d):
             cls_obj = functools.reduce(resolve, cls.split("."), bb.utils._context)
             if not cls_obj:
                 # Fall-back on __builtins__
-                cls_obj = functools.reduce(lambda x, y: x.get(y), cls.split("."), __builtins__)
+                cls_obj = functools.reduce(resolve, cls.split("."), __builtins__)
             if cls_obj:
                 return cls_obj(d, outfile=logfile, otherargs=otherargs)
             bb.warn('%s: unknown custom progress handler in task progress varflag value "%s", ignoring' % (func, cls))
@@ -370,7 +416,7 @@ def exec_func_shell(func, d, runfile, cwd=None):
 
         bb.data.emit_func(func, script, d)
 
-        if bb.msg.loggerVerboseLogs:
+        if verboseShellLogging or bb.utils.to_boolean(d.getVar("BB_VERBOSE_LOGS", False)):
             script.write("set -x\n")
         if cwd:
             script.write("cd '%s'\n" % cwd)
@@ -390,14 +436,20 @@ exit $ret
         if fakerootcmd:
             cmd = [fakerootcmd, runfile]
 
-    if bb.msg.loggerDefaultVerbose:
+    if verboseStdoutLogging:
         logfile = LogTee(logger, StdoutNoopContextManager())
     else:
         logfile = StdoutNoopContextManager()
 
     progress = d.getVarFlag(func, 'progress')
     if progress:
-        logfile = create_progress_handler(func, progress, logfile, d)
+        try:
+            logfile = create_progress_handler(func, progress, logfile, d)
+        except:
+            from traceback import format_exc
+            logger.error("Failed to create progress handler")
+            logger.error(format_exc())
+            raise
 
     fifobuffer = bytearray()
     def readfifo(data):
@@ -449,6 +501,62 @@ exit $ret
             bb.debug(2, "Executing shell function %s" % func)
             with open(os.devnull, 'r+') as stdin, logfile:
                 bb.process.run(cmd, shell=False, stdin=stdin, log=logfile, extrafiles=[(fifo,readfifo)])
+        except bb.process.ExecutionError as exe:
+            # Find the backtrace that the shell trap generated
+            backtrace_marker_regex = re.compile(r"WARNING: Backtrace \(BB generated script\)")
+            stdout_lines = (exe.stdout or "").split("\n")
+            backtrace_start_line = None
+            for i, line in enumerate(reversed(stdout_lines)):
+                if backtrace_marker_regex.search(line):
+                    backtrace_start_line = len(stdout_lines) - i
+                    break
+
+            # Read the backtrace frames, starting at the location we just found
+            backtrace_entry_regex = re.compile(r"#(?P<frameno>\d+): (?P<funcname>[^\s]+), (?P<file>.+?), line ("
+                                               r"?P<lineno>\d+)")
+            backtrace_frames = []
+            if backtrace_start_line:
+                for line in itertools.islice(stdout_lines, backtrace_start_line, None):
+                    match = backtrace_entry_regex.search(line)
+                    if match:
+                        backtrace_frames.append(match.groupdict())
+
+            with open(runfile, "r") as script:
+                script_lines = [line.rstrip() for line in script.readlines()]
+
+            # For each backtrace frame, search backwards in the script (from the line number called out by the frame),
+            # to find the comment that emit_vars injected when it wrote the script. This will give us the metadata
+            # filename (e.g. .bb or .bbclass) and line number where the shell function was originally defined.
+            script_metadata_comment_regex = re.compile(r"# line: (?P<lineno>\d+), file: (?P<file>.+)")
+            better_frames = []
+            # Skip the very last frame since it's just the call to the shell task in the body of the script
+            for frame in backtrace_frames[:-1]:
+                # Check whether the frame corresponds to a function defined in the script vs external script.
+                if os.path.samefile(frame["file"], runfile):
+                    # Search backwards from the frame lineno to locate the comment that BB injected
+                    i = int(frame["lineno"]) - 1
+                    while i >= 0:
+                        match = script_metadata_comment_regex.match(script_lines[i])
+                        if match:
+                            # Calculate the relative line in the function itself
+                            relative_line_in_function = int(frame["lineno"]) - i - 2
+                            # Calculate line in the function as declared in the metadata
+                            metadata_function_line = relative_line_in_function + int(match["lineno"])
+                            better_frames.append("#{frameno}: {funcname}, {file}, line {lineno}".format(
+                                frameno=frame["frameno"],
+                                funcname=frame["funcname"],
+                                file=match["file"],
+                                lineno=metadata_function_line
+                            ))
+                            break
+                        i -= 1
+                else:
+                    better_frames.append("#{frameno}: {funcname}, {file}, line {lineno}".format(**frame))
+
+            if better_frames:
+                better_frames = ("\t{0}".format(frame) for frame in better_frames)
+                exe.extra_message = "\nBacktrace (metadata-relative locations):\n{0}".format("\n".join(better_frames))
+            raise
         finally:
             os.unlink(fifopath)
 

bitbake/lib/bb/cache.py

@@ -17,18 +17,22 @@
 #
 
 import os
-import sys
 import logging
 import pickle
-from collections import defaultdict
+from collections import defaultdict, Mapping
 import bb.utils
+from bb import PrefixLoggerAdapter
+import re
 
 logger = logging.getLogger("BitBake.Cache")
 
-__cache_version__ = "152"
+__cache_version__ = "153"
 
-def getCacheFile(path, filename, data_hash):
-    return os.path.join(path, filename + "." + data_hash)
+def getCacheFile(path, filename, mc, data_hash):
+    mcspec = ''
+    if mc:
+        mcspec = ".%s" % mc
+    return os.path.join(path, filename + mcspec + "." + data_hash)
 
 # RecipeInfoCommon defines common data retrieving methods
 # from meta data for caches. CoreRecipeInfo as well as other
@@ -324,7 +328,7 @@ class NoCache(object):
         bb_data = self.load_bbfile(virtualfn, appends, virtonly=True)
         return bb_data[virtual]
 
-    def load_bbfile(self, bbfile, appends, virtonly = False):
+    def load_bbfile(self, bbfile, appends, virtonly = False, mc=None):
         """
         Load and parse one .bb build file
         Return the data and whether parsing resulted in the file being skipped
@@ -337,6 +341,10 @@ class NoCache(object):
             datastores = parse_recipe(bb_data, bbfile, appends, mc)
             return datastores
 
+        if mc is not None:
+            bb_data = self.databuilder.mcdata[mc].createCopy()
+            return parse_recipe(bb_data, bbfile, appends, mc)
+
         bb_data = self.data.createCopy()
         datastores = parse_recipe(bb_data, bbfile, appends)
 
@@ -354,14 +362,15 @@ class Cache(NoCache):
     """
     BitBake Cache implementation
     """
-
-    def __init__(self, databuilder, data_hash, caches_array):
+    def __init__(self, databuilder, mc, data_hash, caches_array):
         super().__init__(databuilder)
         data = databuilder.data
 
         # Pass caches_array information into Cache Constructor
         # It will be used later for deciding whether we
         # need extra cache file dump/load support
+        self.mc = mc
+        self.logger = PrefixLoggerAdapter("Cache: %s: " % (mc if mc else "default"), logger)
         self.caches_array = caches_array
         self.cachedir = data.getVar("CACHE")
         self.clean = set()
@@ -370,34 +379,51 @@ class Cache(NoCache):
         self.data_fn = None
         self.cacheclean = True
         self.data_hash = data_hash
+        self.filelist_regex = re.compile(r'(?:(?<=:True)|(?<=:False))\s+')
 
         if self.cachedir in [None, '']:
             self.has_cache = False
-            logger.info("Not using a cache. "
-                        "Set CACHE = <directory> to enable.")
+            self.logger.info("Not using a cache. "
+                             "Set CACHE = <directory> to enable.")
             return
 
         self.has_cache = True
-        self.cachefile = getCacheFile(self.cachedir, "bb_cache.dat", self.data_hash)
 
-        logger.debug(1, "Cache dir: %s", self.cachedir)
+    def getCacheFile(self, cachefile):
+        return getCacheFile(self.cachedir, cachefile, self.mc, self.data_hash)
+
+    def prepare_cache(self, progress):
+        if not self.has_cache:
+            return 0
+
+        loaded = 0
+
+        self.cachefile = self.getCacheFile("bb_cache.dat")
+
+        self.logger.debug(1, "Cache dir: %s", self.cachedir)
         bb.utils.mkdirhier(self.cachedir)
 
         cache_ok = True
         if self.caches_array:
             for cache_class in self.caches_array:
-                cachefile = getCacheFile(self.cachedir, cache_class.cachefile, self.data_hash)
-                cache_ok = cache_ok and os.path.exists(cachefile)
+                cachefile = self.getCacheFile(cache_class.cachefile)
+                cache_exists = os.path.exists(cachefile)
+                self.logger.debug(2, "Checking if %s exists: %r", cachefile, cache_exists)
+                cache_ok = cache_ok and cache_exists
                 cache_class.init_cacheData(self)
         if cache_ok:
-            self.load_cachefile()
+            loaded = self.load_cachefile(progress)
         elif os.path.isfile(self.cachefile):
-            logger.info("Out of date cache found, rebuilding...")
+            self.logger.info("Out of date cache found, rebuilding...")
         else:
-            logger.debug(1, "Cache file %s not found, building..." % self.cachefile)
+            self.logger.debug(1, "Cache file %s not found, building..." % self.cachefile)
 
         # We don't use the symlink, its just for debugging convinience
-        symlink = os.path.join(self.cachedir, "bb_cache.dat")
+        if self.mc:
+            symlink = os.path.join(self.cachedir, "bb_cache.dat.%s" % self.mc)
+        else:
+            symlink = os.path.join(self.cachedir, "bb_cache.dat")
+
         if os.path.exists(symlink):
             bb.utils.remove(symlink)
         try:
@@ -405,22 +431,31 @@ class Cache(NoCache):
         except OSError:
             pass
 
-    def load_cachefile(self):
+        return loaded
+
+    def cachesize(self):
+        if not self.has_cache:
+            return 0
+
         cachesize = 0
+        for cache_class in self.caches_array:
+            cachefile = self.getCacheFile(cache_class.cachefile)
+            try:
+                with open(cachefile, "rb") as cachefile:
+                    cachesize += os.fstat(cachefile.fileno()).st_size
+            except FileNotFoundError:
+                pass
+
+        return cachesize
+
+    def load_cachefile(self, progress):
+        cachesize = self.cachesize()
         previous_progress = 0
         previous_percent = 0
 
-        # Calculate the correct cachesize of all those cache files
         for cache_class in self.caches_array:
-            cachefile = getCacheFile(self.cachedir, cache_class.cachefile, self.data_hash)
-            with open(cachefile, "rb") as cachefile:
-                cachesize += os.fstat(cachefile.fileno()).st_size
-
-        bb.event.fire(bb.event.CacheLoadStarted(cachesize), self.data)
-
-        for cache_class in self.caches_array:
-            cachefile = getCacheFile(self.cachedir, cache_class.cachefile, self.data_hash)
-            logger.debug(1, 'Loading cache file: %s' % cachefile)
+            cachefile = self.getCacheFile(cache_class.cachefile)
+            self.logger.debug(1, 'Loading cache file: %s' % cachefile)
             with open(cachefile, "rb") as cachefile:
                 pickled = pickle.Unpickler(cachefile)
                 # Check cache version information
@@ -428,15 +463,15 @@ class Cache(NoCache):
                     cache_ver = pickled.load()
                     bitbake_ver = pickled.load()
                 except Exception:
-                    logger.info('Invalid cache, rebuilding...')
-                    return
+                    self.logger.info('Invalid cache, rebuilding...')
+                    return 0
 
                 if cache_ver != __cache_version__:
-                    logger.info('Cache version mismatch, rebuilding...')
-                    return
+                    self.logger.info('Cache version mismatch, rebuilding...')
+                    return 0
                 elif bitbake_ver != bb.__version__:
-                    logger.info('Bitbake version mismatch, rebuilding...')
-                    return
+                    self.logger.info('Bitbake version mismatch, rebuilding...')
+                    return 0
 
                 # Load the rest of the cache file
                 current_progress = 0
@@ -459,29 +494,17 @@ class Cache(NoCache):
                         self.depends_cache[key] = [value]
                     # only fire events on even percentage boundaries
                     current_progress = cachefile.tell() + previous_progress
-                    if current_progress > cachesize:
-                        # we might have calculated incorrect total size because a file
-                        # might've been written out just after we checked its size
-                        cachesize = current_progress
-                    current_percent = 100 * current_progress / cachesize
-                    if current_percent > previous_percent:
-                        previous_percent = current_percent
-                        bb.event.fire(bb.event.CacheLoadProgress(current_progress, cachesize),
-                                      self.data)
+                    progress(cachefile.tell() + previous_progress)
 
                 previous_progress += current_progress
 
-        # Note: depends cache number is corresponding to the parsing file numbers.
-        # The same file has several caches, still regarded as one item in the cache
-        bb.event.fire(bb.event.CacheLoadCompleted(cachesize,
-                                                  len(self.depends_cache)),
-                      self.data)
+        return len(self.depends_cache)
 
     def parse(self, filename, appends):
         """Parse the specified filename, returning the recipe information"""
-        logger.debug(1, "Parsing %s", filename)
+        self.logger.debug(1, "Parsing %s", filename)
         infos = []
-        datastores = self.load_bbfile(filename, appends)
+        datastores = self.load_bbfile(filename, appends, mc=self.mc)
         depends = []
         variants = []
         # Process the "real" fn last so we can store variants list
@@ -533,7 +556,7 @@ class Cache(NoCache):
         cached, infos = self.load(fn, appends)
         for virtualfn, info_array in infos:
             if info_array[0].skipped:
-                logger.debug(1, "Skipping %s: %s", virtualfn, info_array[0].skipreason)
+                self.logger.debug(1, "Skipping %s: %s", virtualfn, info_array[0].skipreason)
                 skipped += 1
             else:
                 self.add_info(virtualfn, info_array, cacheData, not cached)
@@ -569,21 +592,21 @@ class Cache(NoCache):
 
         # File isn't in depends_cache
         if not fn in self.depends_cache:
-            logger.debug(2, "Cache: %s is not cached", fn)
+            self.logger.debug(2, "%s is not cached", fn)
             return False
 
         mtime = bb.parse.cached_mtime_noerror(fn)
 
         # Check file still exists
         if mtime == 0:
-            logger.debug(2, "Cache: %s no longer exists", fn)
+            self.logger.debug(2, "%s no longer exists", fn)
             self.remove(fn)
             return False
 
         info_array = self.depends_cache[fn]
         # Check the file's timestamp
         if mtime != info_array[0].timestamp:
-            logger.debug(2, "Cache: %s changed", fn)
+            self.logger.debug(2, "%s changed", fn)
             self.remove(fn)
             return False
 
@@ -594,45 +617,37 @@ class Cache(NoCache):
                 fmtime = bb.parse.cached_mtime_noerror(f)
                 # Check if file still exists
                 if old_mtime != 0 and fmtime == 0:
-                    logger.debug(2, "Cache: %s's dependency %s was removed",
-                                    fn, f)
+                    self.logger.debug(2, "%s's dependency %s was removed",
+                                         fn, f)
                     self.remove(fn)
                     return False
 
                 if (fmtime != old_mtime):
-                    logger.debug(2, "Cache: %s's dependency %s changed",
-                                    fn, f)
+                    self.logger.debug(2, "%s's dependency %s changed",
+                                         fn, f)
                     self.remove(fn)
                     return False
 
         if hasattr(info_array[0], 'file_checksums'):
             for _, fl in info_array[0].file_checksums.items():
                 fl = fl.strip()
-                while fl:
-                    # A .split() would be simpler but means spaces or colons in filenames would break
-                    a = fl.find(":True")
-                    b = fl.find(":False")
-                    if ((a < 0) and b) or ((b > 0) and (b < a)):
-                        f = fl[:b+6]
-                        fl = fl[b+7:]
-                    elif ((b < 0) and a) or ((a > 0) and (a < b)):
-                        f = fl[:a+5]
-                        fl = fl[a+6:]
-                    else:
-                        break
-                    fl = fl.strip()
-                    if "*" in f:
+                if not fl:
+                    continue
+                # Have to be careful about spaces and colons in filenames
+                flist = self.filelist_regex.split(fl)
+                for f in flist:
+                    if not f:
                         continue
                     f, exist = f.split(":")
                     if (exist == "True" and not os.path.exists(f)) or (exist == "False" and os.path.exists(f)):
-                        logger.debug(2, "Cache: %s's file checksum list file %s changed",
-                                        fn, f)
+                        self.logger.debug(2, "%s's file checksum list file %s changed",
+                                             fn, f)
                         self.remove(fn)
                         return False
 
-        if appends != info_array[0].appends:
-            logger.debug(2, "Cache: appends for %s changed", fn)
-            logger.debug(2, "%s to %s" % (str(appends), str(info_array[0].appends)))
+        if tuple(appends) != tuple(info_array[0].appends):
+            self.logger.debug(2, "appends for %s changed", fn)
+            self.logger.debug(2, "%s to %s" % (str(appends), str(info_array[0].appends)))
             self.remove(fn)
             return False
 
@@ -641,10 +656,10 @@ class Cache(NoCache):
             virtualfn = variant2virtual(fn, cls)
             self.clean.add(virtualfn)
             if virtualfn not in self.depends_cache:
-                logger.debug(2, "Cache: %s is not cached", virtualfn)
+                self.logger.debug(2, "%s is not cached", virtualfn)
                 invalid = True
             elif len(self.depends_cache[virtualfn]) != len(self.caches_array):
-                logger.debug(2, "Cache: Extra caches missing for %s?" % virtualfn)
+                self.logger.debug(2, "Extra caches missing for %s?" % virtualfn)
                 invalid = True
 
         # If any one of the variants is not present, mark as invalid for all
@@ -652,10 +667,10 @@ class Cache(NoCache):
             for cls in info_array[0].variants:
                 virtualfn = variant2virtual(fn, cls)
                 if virtualfn in self.clean:
-                    logger.debug(2, "Cache: Removing %s from cache", virtualfn)
+                    self.logger.debug(2, "Removing %s from cache", virtualfn)
                     self.clean.remove(virtualfn)
             if fn in self.clean:
-                logger.debug(2, "Cache: Marking %s as not clean", fn)
+                self.logger.debug(2, "Marking %s as not clean", fn)
                 self.clean.remove(fn)
             return False
 
@@ -668,10 +683,10 @@ class Cache(NoCache):
         Called from the parser in error cases
         """
         if fn in self.depends_cache:
-            logger.debug(1, "Removing %s from cache", fn)
+            self.logger.debug(1, "Removing %s from cache", fn)
             del self.depends_cache[fn]
         if fn in self.clean:
-            logger.debug(1, "Marking %s as unclean", fn)
+            self.logger.debug(1, "Marking %s as unclean", fn)
             self.clean.remove(fn)
 
     def sync(self):
@@ -684,12 +699,13 @@ class Cache(NoCache):
             return
 
         if self.cacheclean:
-            logger.debug(2, "Cache is clean, not saving.")
+            self.logger.debug(2, "Cache is clean, not saving.")
             return
 
         for cache_class in self.caches_array:
             cache_class_name = cache_class.__name__
-            cachefile = getCacheFile(self.cachedir, cache_class.cachefile, self.data_hash)
+            cachefile = self.getCacheFile(cache_class.cachefile)
+            self.logger.debug(2, "Writing %s", cachefile)
             with open(cachefile, "wb") as f:
                 p = pickle.Pickler(f, pickle.HIGHEST_PROTOCOL)
                 p.dump(__cache_version__)
@@ -708,8 +724,18 @@ class Cache(NoCache):
         return bb.parse.cached_mtime_noerror(cachefile)
 
     def add_info(self, filename, info_array, cacheData, parsed=None, watcher=None):
+        if self.mc is not None:
+            (fn, cls, mc) = virtualfn2realfn(filename)
+            if mc:
+                self.logger.error("Unexpected multiconfig %s", filename)
+                return
+
+            vfn = realfn2virtual(fn, cls, self.mc)
+        else:
+            vfn = filename
+
         if isinstance(info_array[0], CoreRecipeInfo) and (not info_array[0].skipped):
-            cacheData.add_from_recipeinfo(filename, info_array)
+            cacheData.add_from_recipeinfo(vfn, info_array)
 
             if watcher:
                 watcher(info_array[0].file_depends)
@@ -734,6 +760,65 @@ class Cache(NoCache):
             info_array.append(cache_class(realfn, data))
         self.add_info(file_name, info_array, cacheData, parsed)
 
+class MulticonfigCache(Mapping):
+    def __init__(self, databuilder, data_hash, caches_array):
+        def progress(p):
+            nonlocal current_progress
+            nonlocal previous_progress
+            nonlocal previous_percent
+            nonlocal cachesize
+
+            current_progress = previous_progress + p
+
+            if current_progress > cachesize:
+                # we might have calculated incorrect total size because a file
+                # might've been written out just after we checked its size
+                cachesize = current_progress
+            current_percent = 100 * current_progress / cachesize
+            if current_percent > previous_percent:
+                previous_percent = current_percent
+                bb.event.fire(bb.event.CacheLoadProgress(current_progress, cachesize),
+                                databuilder.data)
+
+
+        cachesize = 0
+        current_progress = 0
+        previous_progress = 0
+        previous_percent = 0
+        self.__caches = {}
+
+        for mc, mcdata in databuilder.mcdata.items():
+            self.__caches[mc] = Cache(databuilder, mc, data_hash, caches_array)
+
+            cachesize += self.__caches[mc].cachesize()
+
+        bb.event.fire(bb.event.CacheLoadStarted(cachesize), databuilder.data)
+        loaded = 0
+
+        for c in self.__caches.values():
+            loaded += c.prepare_cache(progress)
+            previous_progress = current_progress
+
+        # Note: depends cache number is corresponding to the parsing file numbers.
+        # The same file has several caches, still regarded as one item in the cache
+        bb.event.fire(bb.event.CacheLoadCompleted(cachesize, loaded), databuilder.data)
+
+    def __len__(self):
+        return len(self.__caches)
+
+    def __getitem__(self, key):
+        return self.__caches[key]
+
+    def __contains__(self, key):
+        return key in self.__caches
+
+    def __iter__(self):
+        for k in self.__caches:
+            yield k
+
+    def keys(self):
+        return self.__caches[key]
+
 
 def init(cooker):
     """

bitbake/lib/bb/checksum.py

@@ -9,7 +9,6 @@ import glob
 import operator
 import os
 import stat
-import pickle
 import bb.utils
 import logging
 from bb.cache import MultiProcessCache

bitbake/lib/bb/codeparser.py

@@ -25,13 +25,11 @@ import ast
 import sys
 import codegen
 import logging
-import pickle
 import bb.pysh as pysh
-import os.path
 import bb.utils, bb.data
 import hashlib
 from itertools import chain
-from bb.pysh import pyshyacc, pyshlex, sherrors
+from bb.pysh import pyshyacc, pyshlex
 from bb.cache import MultiProcessCache
 
 logger = logging.getLogger('BitBake.CodeParser')
@@ -58,30 +56,10 @@ def check_indent(codestr):
 
     return codestr
 
-
-# Basically pickle, in python 2.7.3 at least, does badly with data duplication 
-# upon pickling and unpickling. Combine this with duplicate objects and things
-# are a mess.
-#
-# When the sets are originally created, python calls intern() on the set keys
-# which significantly improves memory usage. Sadly the pickle/unpickle process
-# doesn't call intern() on the keys and results in the same strings being duplicated
-# in memory. This also means pickle will save the same string multiple times in
-# the cache file.
-#
-# By having shell and python cacheline objects with setstate/getstate, we force
-# the object creation through our own routine where we can call intern (via internSet).
-#
-# We also use hashable frozensets and ensure we use references to these so that
-# duplicates can be removed, both in memory and in the resulting pickled data.
-#
-# By playing these games, the size of the cache file shrinks dramatically
-# meaning faster load times and the reloaded cache files also consume much less
-# memory. Smaller cache files, faster load times and lower memory usage is good.
-#
 # A custom getstate/setstate using tuples is actually worth 15% cachesize by
 # avoiding duplication of the attribute names!
 
+
 class SetCache(object):
     def __init__(self):
         self.setcache = {}

bitbake/lib/bb/command.py

@@ -54,18 +54,25 @@ class Command:
         self.cooker = cooker
         self.cmds_sync = CommandsSync()
         self.cmds_async = CommandsAsync()
-        self.remotedatastores = bb.remotedata.RemoteDatastores(cooker)
+        self.remotedatastores = None
 
         # FIXME Add lock for this
         self.currentAsyncCommand = None
 
     def runCommand(self, commandline, ro_only = False):
         command = commandline.pop(0)
+
+        # Ensure cooker is ready for commands
+        if command != "updateConfig" and command != "setFeatures":
+            self.cooker.init_configdata()
+            if not self.remotedatastores:
+                self.remotedatastores = bb.remotedata.RemoteDatastores(self.cooker)
+
         if hasattr(CommandsSync, command):
             # Can run synchronous commands straight away
             command_method = getattr(self.cmds_sync, command)
             if ro_only:
-                if not hasattr(command_method, 'readonly') or False == getattr(command_method, 'readonly'):
+                if not hasattr(command_method, 'readonly') or not getattr(command_method, 'readonly'):
                     return None, "Not able to execute not readonly commands in readonly mode"
             try:
                 self.cooker.process_inotify_updates()
@@ -74,8 +81,12 @@ class Command:
                 result = command_method(self, commandline)
             except CommandError as exc:
                 return None, exc.args[0]
-            except (Exception, SystemExit):
+            except (Exception, SystemExit) as exc:
                 import traceback
+                if isinstance(exc, bb.BBHandledException):
+                    # We need to start returning real exceptions here. Until we do, we can't
+                    # tell if an exception is an instance of bb.BBHandledException
+                    return None, "bb.BBHandledException()\n" + traceback.format_exc()
                 return None, traceback.format_exc()
             else:
                 return result, None
@@ -84,7 +95,7 @@ class Command:
         if command not in CommandsAsync.__dict__:
             return None, "No such command"
         self.currentAsyncCommand = (command, commandline)
-        self.cooker.configuration.server_register_idlecallback(self.cooker.runCommands, self.cooker)
+        self.cooker.idleCallBackRegister(self.cooker.runCommands, self.cooker)
         return True, None
 
     def runAsyncCommand(self):
@@ -136,13 +147,8 @@ class Command:
         self.cooker.finishcommand()
 
     def reset(self):
-        self.remotedatastores = bb.remotedata.RemoteDatastores(self.cooker)
-
-def split_mc_pn(pn):
-    if pn.startswith("multiconfig:"):
-        _, mc, pn = pn.split(":", 2)
-        return (mc, pn)
-    return ('', pn)
+        if self.remotedatastores:
+           self.remotedatastores = bb.remotedata.RemoteDatastores(self.cooker)
 
 class CommandsSync:
     """
@@ -232,7 +238,11 @@ class CommandsSync:
 
     def matchFile(self, command, params):
         fMatch = params[0]
-        return command.cooker.matchFile(fMatch)
+        try:
+            mc = params[0]
+        except IndexError:
+            mc = ''
+        return command.cooker.matchFile(fMatch, mc)
     matchFile.needconfig = False
 
     def getUIHandlerNum(self, command, params):
@@ -395,30 +405,50 @@ class CommandsSync:
     def getSkippedRecipes(self, command, params):
         # Return list sorted by reverse priority order
         import bb.cache
-        skipdict = OrderedDict(sorted(command.cooker.skiplist.items(),
-                                      key=lambda x: (-command.cooker.collection.calc_bbfile_priority(bb.cache.virtualfn2realfn(x[0])[0]), x[0])))
+        def sortkey(x):
+            vfn, _ = x
+            realfn, _, mc = bb.cache.virtualfn2realfn(vfn)
+            return (-command.cooker.collections[mc].calc_bbfile_priority(realfn)[0], vfn)
+
+        skipdict = OrderedDict(sorted(command.cooker.skiplist.items(), key=sortkey))
         return list(skipdict.items())
     getSkippedRecipes.readonly = True
 
     def getOverlayedRecipes(self, command, params):
-        return list(command.cooker.collection.overlayed.items())
+        try:
+            mc = params[0]
+        except IndexError:
+            mc = ''
+        return list(command.cooker.collections[mc].overlayed.items())
     getOverlayedRecipes.readonly = True
 
     def getFileAppends(self, command, params):
         fn = params[0]
-        return command.cooker.collection.get_file_appends(fn)
+        try:
+            mc = params[1]
+        except IndexError:
+            mc = ''
+        return command.cooker.collections[mc].get_file_appends(fn)
     getFileAppends.readonly = True
 
     def getAllAppends(self, command, params):
-        return command.cooker.collection.bbappends
+        try:
+            mc = params[0]
+        except IndexError:
+            mc = ''
+        return command.cooker.collections[mc].bbappends
     getAllAppends.readonly = True
 
     def findProviders(self, command, params):
-        return command.cooker.findProviders()
+        try:
+            mc = params[0]
+        except IndexError:
+            mc = ''
+        return command.cooker.findProviders(mc)
     findProviders.readonly = True
 
     def findBestProvider(self, command, params):
-        (mc, pn) = split_mc_pn(params[0])
+        (mc, pn) = bb.runqueue.split_mc(params[0])
         return command.cooker.findBestProvider(pn, mc)
     findBestProvider.readonly = True
 
@@ -446,54 +476,38 @@ class CommandsSync:
         return all_p, best
     getRuntimeProviders.readonly = True
 
-    def dataStoreConnectorFindVar(self, command, params):
+    def dataStoreConnectorCmd(self, command, params):
         dsindex = params[0]
-        name = params[1]
-        datastore = command.remotedatastores[dsindex]
-        value, overridedata = datastore._findVar(name)
+        method = params[1]
+        args = params[2]
+        kwargs = params[3]
 
-        if value:
-            content = value.get('_content', None)
-            if isinstance(content, bb.data_smart.DataSmart):
-                # Value is a datastore (e.g. BB_ORIGENV) - need to handle this carefully
-                idx = command.remotedatastores.check_store(content, True)
-                return {'_content': DataStoreConnectionHandle(idx),
-                        '_connector_origtype': 'DataStoreConnectionHandle',
-                        '_connector_overrides': overridedata}
-            elif isinstance(content, set):
-                return {'_content': list(content),
-                        '_connector_origtype': 'set',
-                        '_connector_overrides': overridedata}
-            else:
-                value['_connector_overrides'] = overridedata
-        else:
-            value = {}
-            value['_connector_overrides'] = overridedata
-        return value
-    dataStoreConnectorFindVar.readonly = True
+        d = command.remotedatastores[dsindex]
+        ret = getattr(d, method)(*args, **kwargs)
 
-    def dataStoreConnectorGetKeys(self, command, params):
+        if isinstance(ret, bb.data_smart.DataSmart):
+            idx = command.remotedatastores.store(ret)
+            return DataStoreConnectionHandle(idx)
+
+        return ret
+
+    def dataStoreConnectorVarHistCmd(self, command, params):
         dsindex = params[0]
-        datastore = command.remotedatastores[dsindex]
-        return list(datastore.keys())
-    dataStoreConnectorGetKeys.readonly = True
+        method = params[1]
+        args = params[2]
+        kwargs = params[3]
 
-    def dataStoreConnectorGetVarHistory(self, command, params):
+        d = command.remotedatastores[dsindex].varhistory
+        return getattr(d, method)(*args, **kwargs)
+
+    def dataStoreConnectorIncHistCmd(self, command, params):
         dsindex = params[0]
-        name = params[1]
-        datastore = command.remotedatastores[dsindex]
-        return datastore.varhistory.variable(name)
-    dataStoreConnectorGetVarHistory.readonly = True
+        method = params[1]
+        args = params[2]
+        kwargs = params[3]
 
-    def dataStoreConnectorExpandPythonRef(self, command, params):
-        config_data_dict = params[0]
-        varname = params[1]
-        expr = params[2]
-
-        config_data = command.remotedatastores.receive_datastore(config_data_dict)
-
-        varparse = bb.data_smart.VariableParse(varname, config_data)
-        return varparse.python_sub(expr)
+        d = command.remotedatastores[dsindex].inchistory
+        return getattr(d, method)(*args, **kwargs)
 
     def dataStoreConnectorRelease(self, command, params):
         dsindex = params[0]
@@ -501,31 +515,6 @@ class CommandsSync:
             raise CommandError('dataStoreConnectorRelease: invalid index %d' % dsindex)
         command.remotedatastores.release(dsindex)
 
-    def dataStoreConnectorSetVarFlag(self, command, params):
-        dsindex = params[0]
-        name = params[1]
-        flag = params[2]
-        value = params[3]
-        datastore = command.remotedatastores[dsindex]
-        datastore.setVarFlag(name, flag, value)
-
-    def dataStoreConnectorDelVar(self, command, params):
-        dsindex = params[0]
-        name = params[1]
-        datastore = command.remotedatastores[dsindex]
-        if len(params) > 2:
-            flag = params[2]
-            datastore.delVarFlag(name, flag)
-        else:
-            datastore.delVar(name)
-
-    def dataStoreConnectorRenameVar(self, command, params):
-        dsindex = params[0]
-        name = params[1]
-        newname = params[2]
-        datastore = command.remotedatastores[dsindex]
-        datastore.renameVar(name, newname)
-
     def parseRecipeFile(self, command, params):
         """
         Parse the specified recipe file (with or without bbappends)
@@ -533,11 +522,11 @@ class CommandsSync:
         for the recipe.
         """
         fn = params[0]
+        mc = bb.runqueue.mc_from_tid(fn)
         appends = params[1]
         appendlist = params[2]
         if len(params) > 3:
-            config_data_dict = params[3]
-            config_data = command.remotedatastores.receive_datastore(config_data_dict)
+            config_data = command.remotedatastores[params[3]]
         else:
             config_data = None
 
@@ -545,7 +534,7 @@ class CommandsSync:
             if appendlist is not None:
                 appendfiles = appendlist
             else:
-                appendfiles = command.cooker.collection.get_file_appends(fn)
+                appendfiles = command.cooker.collections[mc].get_file_appends(fn)
         else:
             appendfiles = []
         # We are calling bb.cache locally here rather than on the server,
@@ -555,7 +544,7 @@ class CommandsSync:
         if config_data:
             # We have to use a different function here if we're passing in a datastore
             # NOTE: we took a copy above, so we don't do it here again
-            envdata = bb.cache.parse_recipe(config_data, fn, appendfiles)['']
+            envdata = bb.cache.parse_recipe(config_data, fn, appendfiles, mc)['']
         else:
             # Use the standard path
             parser = bb.cache.NoCache(command.cooker.databuilder)
@@ -746,10 +735,10 @@ class CommandsAsync:
         """
         Find signature info files via the signature generator
         """
-        pn = params[0]
+        (mc, pn) = bb.runqueue.split_mc(params[0])
         taskname = params[1]
         sigs = params[2]
-        res = bb.siggen.find_siginfo(pn, taskname, sigs, command.cooker.data)
-        bb.event.fire(bb.event.FindSigInfoResult(res), command.cooker.data)
+        res = bb.siggen.find_siginfo(pn, taskname, sigs, command.cooker.databuilder.mcdata[mc])
+        bb.event.fire(bb.event.FindSigInfoResult(res), command.cooker.databuilder.mcdata[mc])
         command.finishAsyncCommand()
     findSigInfo.needcache = False

bitbake/lib/bb/compat.py

@@ -1,10 +0,0 @@
-#
-# SPDX-License-Identifier: GPL-2.0-only
-#
-
-"""Code pulled from future python versions, here for compatibility"""
-
-from collections import MutableMapping, KeysView, ValuesView, ItemsView, OrderedDict
-from functools import total_ordering
-
-

bitbake/lib/bb/cooker.py

@@ -10,7 +10,6 @@
 #
 
 import sys, os, glob, os.path, re, time
-import atexit
 import itertools
 import logging
 import multiprocessing
@@ -18,14 +17,11 @@ import sre_constants
 import threading
 from io import StringIO, UnsupportedOperation
 from contextlib import closing
-from functools import wraps
 from collections import defaultdict, namedtuple
 import bb, bb.exceptions, bb.command
 from bb import utils, data, parse, event, cache, providers, taskdata, runqueue, build
 import queue
 import signal
-import subprocess
-import errno
 import prserv.serv
 import pyinotify
 import json
@@ -152,15 +148,18 @@ class BBCooker:
     Manages one bitbake build run
     """
 
-    def __init__(self, configuration, featureSet=None):
+    def __init__(self, featureSet=None, idleCallBackRegister=None):
         self.recipecaches = None
+        self.eventlog = None
         self.skiplist = {}
         self.featureset = CookerFeatures()
         if featureSet:
             for f in featureSet:
                 self.featureset.setFeature(f)
 
-        self.configuration = configuration
+        self.configuration = bb.cookerdata.CookerConfiguration()
+
+        self.idleCallBackRegister = idleCallBackRegister
 
         bb.debug(1, "BBCooker starting %s" % time.time())
         sys.stdout.flush()
@@ -196,25 +195,13 @@ class BBCooker:
         self.hashserv = None
         self.hashservaddr = None
 
-        self.initConfigurationData()
-
-        bb.debug(1, "BBCooker parsed base configuration %s" % time.time())
-        sys.stdout.flush()
-
-        # we log all events to a file if so directed
-        if self.configuration.writeeventlog:
-            # register the log file writer as UI Handler
-            writer = EventWriter(self, self.configuration.writeeventlog)
-            EventLogWriteHandler = namedtuple('EventLogWriteHandler', ['event'])
-            bb.event.register_UIHhandler(EventLogWriteHandler(writer))
-
         self.inotify_modified_files = []
 
         def _process_inotify_updates(server, cooker, abort):
             cooker.process_inotify_updates()
             return 1.0
 
-        self.configuration.server_register_idlecallback(_process_inotify_updates, self)
+        self.idleCallBackRegister(_process_inotify_updates, self)
 
         # TOSTOP must not be set or our children will hang when they output
         try:
@@ -241,6 +228,13 @@ class BBCooker:
         bb.debug(1, "BBCooker startup complete %s" % time.time())
         sys.stdout.flush()
 
+    def init_configdata(self):
+        if not hasattr(self, "data"):
+            self.initConfigurationData()
+            bb.debug(1, "BBCooker parsed base configuration %s" % time.time())
+            sys.stdout.flush()
+            self.handlePRServ()
+
     def process_inotify_updates(self):
         for n in [self.confignotifier, self.notifier]:
             if n.check_events(timeout=0):
@@ -326,7 +320,7 @@ class BBCooker:
         for feature in features:
             self.featureset.setFeature(feature)
         bb.debug(1, "Features set %s (was %s)" % (original_featureset, list(self.featureset)))
-        if (original_featureset != list(self.featureset)) and self.state != state.error:
+        if (original_featureset != list(self.featureset)) and self.state != state.error and hasattr(self, "data"):
             self.reset()
 
     def initConfigurationData(self):
@@ -358,7 +352,7 @@ class BBCooker:
                 self.caches_array.append(getattr(module, cache_name))
             except ImportError as exc:
                 logger.critical("Unable to import extra RecipeInfo '%s' from '%s': %s" % (cache_name, module_name, exc))
-                sys.exit("FATAL: Failed to import extra cache class '%s'." % cache_name)
+                raise bb.BBHandledException()
 
         self.databuilder = bb.cookerdata.CookerDataBuilder(self.configuration, False)
         self.databuilder.parseBaseConfiguration()
@@ -415,11 +409,6 @@ class BBCooker:
             self.data.disableTracking()
 
     def parseConfiguration(self):
-        # Set log file verbosity
-        verboselogs = bb.utils.to_boolean(self.data.getVar("BB_VERBOSE_LOGS", False))
-        if verboselogs:
-            bb.msg.loggerVerboseLogs = True
-
         # Change nice level if we're asked to
         nice = self.data.getVar("BB_NICE_LEVEL")
         if nice:
@@ -453,7 +442,28 @@ class BBCooker:
                 logger.debug(1, "Marking as dirty due to '%s' option change to '%s'" % (o, options[o]))
                 print("Marking as dirty due to '%s' option change to '%s'" % (o, options[o]))
                 clean = False
-            setattr(self.configuration, o, options[o])
+            if hasattr(self.configuration, o):
+                setattr(self.configuration, o, options[o])
+
+        if self.configuration.writeeventlog:
+            if self.eventlog and self.eventlog[0] != self.configuration.writeeventlog:
+                bb.event.unregister_UIHhandler(self.eventlog[1])
+            if not self.eventlog or self.eventlog[0] != self.configuration.writeeventlog:
+                # we log all events to a file if so directed
+                # register the log file writer as UI Handler
+                writer = EventWriter(self, self.configuration.writeeventlog)
+                EventLogWriteHandler = namedtuple('EventLogWriteHandler', ['event'])
+                self.eventlog = (self.configuration.writeeventlog, bb.event.register_UIHhandler(EventLogWriteHandler(writer)))
+
+        bb.msg.loggerDefaultLogLevel = self.configuration.default_loglevel
+        bb.msg.loggerDefaultDomains = self.configuration.debug_domains
+
+        if hasattr(self, "data"):
+            origenv = bb.data.init()
+            for k in environment:
+                origenv.setVar(k, environment[k])
+            self.data.setVar("BB_ORIGENV", origenv)
+
         for k in bb.utils.approved_variables():
             if k in environment and k not in self.configuration.env:
                 logger.debug(1, "Updating new environment variable %s to %s" % (k, environment[k]))
@@ -469,6 +479,10 @@ class BBCooker:
                 logger.debug(1, "Updating environment variable %s from %s to %s" % (k, self.configuration.env[k], environment[k]))
                 self.configuration.env[k] = environment[k]
                 clean = False
+
+        # Now update all the variables not in the datastore to match
+        self.configuration.env = environment
+
         if not clean:
             logger.debug(1, "Base environment change, triggering reparse")
             self.reset()
@@ -529,7 +543,7 @@ class BBCooker:
             self.parseConfiguration()
 
             fn, cls, mc = bb.cache.virtualfn2realfn(buildfile)
-            fn = self.matchFile(fn)
+            fn = self.matchFile(fn, mc)
             fn = bb.cache.realfn2virtual(fn, cls, mc)
         elif len(pkgs_to_build) == 1:
             mc = mc_base(pkgs_to_build[0])
@@ -545,8 +559,8 @@ class BBCooker:
 
         if fn:
             try:
-                bb_cache = bb.cache.Cache(self.databuilder, self.data_hash, self.caches_array)
-                envdata = bb_cache.loadDataFull(fn, self.collection.get_file_appends(fn))
+                bb_caches = bb.cache.MulticonfigCache(self.databuilder, self.data_hash, self.caches_array)
+                envdata = bb_caches[mc].loadDataFull(fn, self.collections[mc].get_file_appends(fn))
             except Exception as e:
                 parselog.exception("Unable to read %s", fn)
                 raise
@@ -630,6 +644,7 @@ class BBCooker:
         current = 0
         runlist = []
         for k in fulltargetlist:
+            origk = k
             mc = ""
             if k.startswith("mc:"):
                 mc = k.split(":")[1]
@@ -639,6 +654,10 @@ class BBCooker:
                 k2 = k.split(":do_")
                 k = k2[0]
                 ktask = k2[1]
+
+            if mc not in self.multiconfigs:
+                 bb.fatal("Multiconfig dependency %s depends on nonexistent multiconfig configuration named %s" % (origk, mc))
+
             taskdata[mc].add_provider(localdata[mc], self.recipecaches[mc], k)
             current += 1
             if not ktask.startswith("do_"):
@@ -674,7 +693,7 @@ class BBCooker:
                         l = k.split(':')
                         depmc = l[2]
                         if depmc not in self.multiconfigs:
-                            bb.fatal("Multiconfig dependency %s depends on nonexistent mc configuration %s" % (k,depmc))
+                            bb.fatal("Multiconfig dependency %s depends on nonexistent multiconfig configuration named configuration %s" % (k,depmc))
                         else:
                             logger.debug(1, "Adding providers for multiconfig dependency %s" % l[3])
                             taskdata[depmc].add_provider(localdata[depmc], self.recipecaches[depmc], l[3])
@@ -933,26 +952,33 @@ class BBCooker:
         logger.info("Task dependencies saved to 'task-depends.dot'")
 
     def show_appends_with_no_recipes(self):
+        appends_without_recipes = {}
         # Determine which bbappends haven't been applied
+        for mc in self.multiconfigs:
+            # First get list of recipes, including skipped
+            recipefns = list(self.recipecaches[mc].pkg_fn.keys())
+            recipefns.extend(self.skiplist.keys())
 
-        # First get list of recipes, including skipped
-        recipefns = list(self.recipecaches[''].pkg_fn.keys())
-        recipefns.extend(self.skiplist.keys())
+            # Work out list of bbappends that have been applied
+            applied_appends = []
+            for fn in recipefns:
+                applied_appends.extend(self.collections[mc].get_file_appends(fn))
 
-        # Work out list of bbappends that have been applied
-        applied_appends = []
-        for fn in recipefns:
-            applied_appends.extend(self.collection.get_file_appends(fn))
+            appends_without_recipes[mc] = []
+            for _, appendfn in self.collections[mc].bbappends:
+                if not appendfn in applied_appends:
+                    appends_without_recipes[mc].append(appendfn)
 
-        appends_without_recipes = []
-        for _, appendfn in self.collection.bbappends:
-            if not appendfn in applied_appends:
-                appends_without_recipes.append(appendfn)
+        msgs = []
+        for mc in sorted(appends_without_recipes.keys()):
+            if appends_without_recipes[mc]:
+                msgs.append('No recipes in %s available for:\n  %s' % (mc if mc else 'default',
+                                                                        '\n  '.join(appends_without_recipes[mc])))
 
-        if appends_without_recipes:
-            msg = 'No recipes available for:\n  %s' % '\n  '.join(appends_without_recipes)
-            warn_only = self.data.getVar("BB_DANGLINGAPPENDS_WARNONLY", \
-                 False) or "no"
+        if msgs:
+            msg = "\n".join(msgs)
+            warn_only = self.databuilder.mcdata[mc].getVar("BB_DANGLINGAPPENDS_WARNONLY", \
+                False) or "no"
             if warn_only.lower() in ("1", "yes", "true"):
                 bb.warn(msg)
             else:
@@ -1027,16 +1053,16 @@ class BBCooker:
             bb.event.fire(bb.event.FilesMatchingFound(filepattern, matches), self.data)
 
     def findProviders(self, mc=''):
-        return bb.providers.findProviders(self.data, self.recipecaches[mc], self.recipecaches[mc].pkg_pn)
+        return bb.providers.findProviders(self.databuilder.mcdata[mc], self.recipecaches[mc], self.recipecaches[mc].pkg_pn)
 
     def findBestProvider(self, pn, mc=''):
         if pn in self.recipecaches[mc].providers:
             filenames = self.recipecaches[mc].providers[pn]
-            eligible, foundUnique = bb.providers.filterProviders(filenames, pn, self.data, self.recipecaches[mc])
+            eligible, foundUnique = bb.providers.filterProviders(filenames, pn, self.databuilder.mcdata[mc], self.recipecaches[mc])
             filename = eligible[0]
             return None, None, None, filename
         elif pn in self.recipecaches[mc].pkg_pn:
-            return bb.providers.findBestProvider(pn, self.data, self.recipecaches[mc], self.recipecaches[mc].pkg_pn)
+            return bb.providers.findBestProvider(pn, self.databuilder.mcdata[mc], self.recipecaches[mc], self.recipecaches[mc].pkg_pn)
         else:
             return None, None, None, None
 
@@ -1101,7 +1127,7 @@ class BBCooker:
             from bb import shell
         except ImportError:
             parselog.exception("Interactive mode not available")
-            sys.exit(1)
+            raise bb.BBHandledException()
         else:
             shell.start( self )
 
@@ -1208,7 +1234,7 @@ class BBCooker:
             for c in collection_list:
                 calc_layer_priority(c)
                 regex = self.data.getVar("BBFILE_PATTERN_%s" % c)
-                if regex == None:
+                if regex is None:
                     parselog.error("BBFILE_PATTERN_%s not defined" % c)
                     errors = True
                     continue
@@ -1253,15 +1279,15 @@ class BBCooker:
         if siggen_cache:
             bb.parse.siggen.checksum_cache.mtime_cache.clear()
 
-    def matchFiles(self, bf):
+    def matchFiles(self, bf, mc=''):
         """
         Find the .bb files which match the expression in 'buildfile'.
         """
         if bf.startswith("/") or bf.startswith("../"):
             bf = os.path.abspath(bf)
 
-        self.collection = CookerCollectFiles(self.bbfile_config_priorities)
-        filelist, masked, searchdirs = self.collection.collect_bbfiles(self.data, self.data)
+        self.collections = {mc: CookerCollectFiles(self.bbfile_config_priorities, mc)}
+        filelist, masked, searchdirs = self.collections[mc].collect_bbfiles(self.databuilder.mcdata[mc], self.databuilder.mcdata[mc])
         try:
             os.stat(bf)
             bf = os.path.abspath(bf)
@@ -1274,12 +1300,12 @@ class BBCooker:
                     matches.append(f)
             return matches
 
-    def matchFile(self, buildfile):
+    def matchFile(self, buildfile, mc=''):
         """
         Find the .bb file which matches the expression in 'buildfile'.
         Raise an error if multiple files
         """
-        matches = self.matchFiles(buildfile)
+        matches = self.matchFiles(buildfile, mc)
         if len(matches) != 1:
             if matches:
                 msg = "Unable to match '%s' to a specific recipe file - %s matches found:" % (buildfile, len(matches))
@@ -1314,20 +1340,20 @@ class BBCooker:
         self.parseConfiguration()
 
         # If we are told to do the None task then query the default task
-        if (task == None):
+        if task is None:
             task = self.configuration.cmd
         if not task.startswith("do_"):
             task = "do_%s" % task
 
         fn, cls, mc = bb.cache.virtualfn2realfn(buildfile)
-        fn = self.matchFile(fn)
+        fn = self.matchFile(fn, mc)
 
         self.buildSetVars()
         self.reset_mtime_caches()
 
-        bb_cache = bb.cache.Cache(self.databuilder, self.data_hash, self.caches_array)
+        bb_caches = bb.cache.MulticonfigCache(self.databuilder, self.data_hash, self.caches_array)
 
-        infos = bb_cache.parse(fn, self.collection.get_file_appends(fn))
+        infos = bb_caches[mc].parse(fn, self.collections[mc].get_file_appends(fn))
         infos = dict(infos)
 
         fn = bb.cache.realfn2virtual(fn, cls, mc)
@@ -1415,7 +1441,7 @@ class BBCooker:
                 return True
             return retval
 
-        self.configuration.server_register_idlecallback(buildFileIdle, rq)
+        self.idleCallBackRegister(buildFileIdle, rq)
 
     def buildTargets(self, targets, task):
         """
@@ -1458,7 +1484,7 @@ class BBCooker:
         self.buildSetVars()
 
         # If we are told to do the None task then query the default task
-        if (task == None):
+        if task is None:
             task = self.configuration.cmd
 
         if not task.startswith("do_"):
@@ -1486,7 +1512,7 @@ class BBCooker:
         if 'universe' in targets:
             rq.rqdata.warn_multi_bb = True
 
-        self.configuration.server_register_idlecallback(buildTargetsIdle, rq)
+        self.idleCallBackRegister(buildTargetsIdle, rq)
 
 
     def getAllKeysWithFlags(self, flaglist):
@@ -1537,6 +1563,7 @@ class BBCooker:
         if self.state in (state.shutdown, state.forceshutdown, state.error):
             if hasattr(self.parser, 'shutdown'):
                 self.parser.shutdown(clean=False, force = True)
+                self.parser.final_cleanup()
             raise bb.BBHandledException()
 
         if self.state != state.parsing:
@@ -1556,14 +1583,24 @@ class BBCooker:
                 for dep in self.configuration.extra_assume_provided:
                     self.recipecaches[mc].ignored_dependencies.add(dep)
 
-            self.collection = CookerCollectFiles(self.bbfile_config_priorities)
-            (filelist, masked, searchdirs) = self.collection.collect_bbfiles(self.data, self.data)
+            self.collections = {}
+
+            mcfilelist = {}
+            total_masked = 0
+            searchdirs = set()
+            for mc in self.multiconfigs:
+                self.collections[mc] = CookerCollectFiles(self.bbfile_config_priorities, mc)
+                (filelist, masked, search) = self.collections[mc].collect_bbfiles(self.databuilder.mcdata[mc], self.databuilder.mcdata[mc])
+
+                mcfilelist[mc] = filelist
+                total_masked += masked
+                searchdirs |= set(search)
 
             # Add inotify watches for directories searched for bb/bbappend files
             for dirent in searchdirs:
                 self.add_filewatch([[dirent]], dirs=True)
 
-            self.parser = CookerParser(self, filelist, masked)
+            self.parser = CookerParser(self, mcfilelist, total_masked)
             self.parsecache_valid = True
 
         self.state = state.parsing
@@ -1575,7 +1612,7 @@ class BBCooker:
             self.show_appends_with_no_recipes()
             self.handlePrefProviders()
             for mc in self.multiconfigs:
-                self.recipecaches[mc].bbfile_priority = self.collection.collection_priorities(self.recipecaches[mc].pkg_fn, self.data)
+                self.recipecaches[mc].bbfile_priority = self.collections[mc].collection_priorities(self.recipecaches[mc].pkg_fn, self.parser.mcfilelist[mc], self.data)
             self.state = state.running
 
             # Send an event listing all stamps reachable after parsing
@@ -1596,7 +1633,7 @@ class BBCooker:
             raise NothingToBuild
 
         ignore = (self.data.getVar("ASSUME_PROVIDED") or "").split()
-        for pkg in pkgs_to_build:
+        for pkg in pkgs_to_build.copy():
             if pkg in ignore:
                 parselog.warning("Explicit target \"%s\" is in ASSUME_PROVIDED, ignoring" % pkg)
             if pkg.startswith("multiconfig:"):
@@ -1634,17 +1671,16 @@ class BBCooker:
         return pkgs_to_build
 
     def pre_serve(self):
-        # We now are in our own process so we can call this here.
-        # PRServ exits if its parent process exits
-        self.handlePRServ()
         return
 
     def post_serve(self):
+        self.shutdown(force=True)
         prserv.serv.auto_shutdown()
         if self.hashserv:
             self.hashserv.process.terminate()
             self.hashserv.process.join()
-        bb.event.fire(CookerExit(), self.data)
+        if hasattr(self, "data"):
+            bb.event.fire(CookerExit(), self.data)
 
     def shutdown(self, force = False):
         if force:
@@ -1654,6 +1690,7 @@ class BBCooker:
 
         if self.parser:
             self.parser.shutdown(clean=not force, force=force)
+            self.parser.final_cleanup()
 
     def finishcommand(self):
         self.state = state.initial
@@ -1667,8 +1704,11 @@ class BBCooker:
         self.finishcommand()
         self.extraconfigdata = {}
         self.command.reset()
-        self.databuilder.reset()
-        self.data = self.databuilder.data
+        if hasattr(self, "data"):
+           self.databuilder.reset()
+           self.data = self.databuilder.data
+        self.parsecache_valid = False
+        self.baseconfig_valid = False
 
 
 class CookerExit(bb.event.Event):
@@ -1681,21 +1721,19 @@ class CookerExit(bb.event.Event):
 
 
 class CookerCollectFiles(object):
-    def __init__(self, priorities):
+    def __init__(self, priorities, mc=''):
+        self.mc = mc
         self.bbappends = []
         # Priorities is a list of tupples, with the second element as the pattern.
         # We need to sort the list with the longest pattern first, and so on to
         # the shortest.  This allows nested layers to be properly evaluated.
         self.bbfile_config_priorities = sorted(priorities, key=lambda tup: tup[1], reverse=True)
 
-    def calc_bbfile_priority( self, filename, matched = None ):
+    def calc_bbfile_priority(self, filename):
         for _, _, regex, pri in self.bbfile_config_priorities:
             if regex.match(filename):
-                if matched != None:
-                    if not regex in matched:
-                        matched.add(regex)
-                return pri
-        return 0
+                return pri, regex
+        return 0, None
 
     def get_bbfiles(self):
         """Get list of default .bb files by reading out the current directory"""
@@ -1725,10 +1763,10 @@ class CookerCollectFiles(object):
         collectlog.debug(1, "collecting .bb files")
 
         files = (config.getVar( "BBFILES") or "").split()
-        config.setVar("BBFILES", " ".join(files))
 
         # Sort files by priority
-        files.sort( key=lambda fileitem: self.calc_bbfile_priority(fileitem) )
+        files.sort( key=lambda fileitem: self.calc_bbfile_priority(fileitem)[0] )
+        config.setVar("BBFILES_PRIORITIZED", " ".join(files))
 
         if not len(files):
             files = self.get_bbfiles()
@@ -1848,43 +1886,67 @@ class CookerCollectFiles(object):
             (bbappend, filename) = b
             if (bbappend == f) or ('%' in bbappend and bbappend.startswith(f[:bbappend.index('%')])):
                 filelist.append(filename)
-        return filelist
+        return tuple(filelist)
 
-    def collection_priorities(self, pkgfns, d):
+    def collection_priorities(self, pkgfns, fns, d):
+        # Return the priorities of the entries in pkgfns
+        # Also check that all the regexes in self.bbfile_config_priorities are used
+        # (but to do that we need to ensure skipped recipes aren't counted, nor
+        # collections in BBFILE_PATTERN_IGNORE_EMPTY)
 
         priorities = {}
+        seen = set()
+        matched = set()
+
+        matched_regex = set()
+        unmatched_regex = set()
+        for _, _, regex, _ in self.bbfile_config_priorities:
+            unmatched_regex.add(regex)
 
         # Calculate priorities for each file
-        matched = set()
         for p in pkgfns:
             realfn, cls, mc = bb.cache.virtualfn2realfn(p)
-            priorities[p] = self.calc_bbfile_priority(realfn, matched)
+            priorities[p], regex = self.calc_bbfile_priority(realfn)
+            if regex in unmatched_regex:
+                matched_regex.add(regex)
+                unmatched_regex.remove(regex)
+            seen.add(realfn)
+            if regex:
+                matched.add(realfn)
 
-        unmatched = set()
-        for _, _, regex, pri in self.bbfile_config_priorities:
-            if not regex in matched:
-                unmatched.add(regex)
-
-        # Don't show the warning if the BBFILE_PATTERN did match .bbappend files
-        def find_bbappend_match(regex):
+        if unmatched_regex:
+            # Account for bbappend files
             for b in self.bbappends:
                 (bbfile, append) = b
-                if regex.match(append):
-                    # If the bbappend is matched by already "matched set", return False
-                    for matched_regex in matched:
-                        if matched_regex.match(append):
-                            return False
-                    return True
-            return False
+                seen.add(append)
 
-        for unmatch in unmatched.copy():
-            if find_bbappend_match(unmatch):
-                unmatched.remove(unmatch)
+            # Account for skipped recipes
+            seen.update(fns)
+
+            seen.difference_update(matched)
+
+            def already_matched(fn):
+                for regex in matched_regex:
+                    if regex.match(fn):
+                        return True
+                return False
+
+            for unmatch in unmatched_regex.copy():
+                for fn in seen:
+                    if unmatch.match(fn):
+                        # If the bbappend or file was already matched by another regex, skip it
+                        # e.g. for a layer within a layer, the outer regex could match, the inner
+                        # regex may match nothing and we should warn about that
+                        if already_matched(fn):
+                            continue
+                        unmatched_regex.remove(unmatch)
+                        break
 
         for collection, pattern, regex, _ in self.bbfile_config_priorities:
-            if regex in unmatched:
+            if regex in unmatched_regex:
                 if d.getVar('BBFILE_PATTERN_IGNORE_EMPTY_%s' % collection) != '1':
-                    collectlog.warning("No bb files matched BBFILE_PATTERN_%s '%s'" % (collection, pattern))
+                    collectlog.warning("No bb files in %s matched BBFILE_PATTERN_%s '%s'" % (self.mc if self.mc else 'default',
+                                                                                             collection, pattern))
 
         return priorities
 
@@ -1933,7 +1995,8 @@ class Parser(multiprocessing.Process):
             except queue.Empty:
                 pass
             else:
-                self.results.cancel_join_thread()
+                self.results.close()
+                self.results.join_thread()
                 break
 
             if pending:
@@ -1942,6 +2005,8 @@ class Parser(multiprocessing.Process):
                 try:
                     job = self.jobs.pop()
                 except IndexError:
+                    self.results.close()
+                    self.results.join_thread()
                     break
                 result = self.parse(*job)
                 # Clear the siggen cache after parsing to control memory usage, its huge
@@ -1951,8 +2016,9 @@ class Parser(multiprocessing.Process):
             except queue.Full:
                 pending.append(result)
 
-    def parse(self, filename, appends):
+    def parse(self, mc, cache, filename, appends):
         try:
+            origfilter = bb.event.LogHandler.filter
             # Record the filename we're parsing into any events generated
             def parse_filter(self, record):
                 record.taskpid = bb.event.worker_pid
@@ -1964,7 +2030,7 @@ class Parser(multiprocessing.Process):
             bb.event.set_class_handlers(self.handlers.copy())
             bb.event.LogHandler.filter = parse_filter
 
-            return True, self.bb_cache.parse(filename, appends)
+            return True, mc, cache.parse(filename, appends)
         except Exception as exc:
             tb = sys.exc_info()[2]
             exc.recipe = filename
@@ -1975,10 +2041,12 @@ class Parser(multiprocessing.Process):
         # a SystemExit event for example.
         except BaseException as exc:
             return True, ParsingFailure(exc, filename)
+        finally:
+            bb.event.LogHandler.filter = origfilter
 
 class CookerParser(object):
-    def __init__(self, cooker, filelist, masked):
-        self.filelist = filelist
+    def __init__(self, cooker, mcfilelist, masked):
+        self.mcfilelist = mcfilelist
         self.cooker = cooker
         self.cfgdata = cooker.data
         self.cfghash = cooker.data_hash
@@ -1992,28 +2060,31 @@ class CookerParser(object):
 
         self.skipped = 0
         self.virtuals = 0
-        self.total = len(filelist)
 
         self.current = 0
         self.process_names = []
 
-        self.bb_cache = bb.cache.Cache(self.cfgbuilder, self.cfghash, cooker.caches_array)
-        self.fromcache = []
-        self.willparse = []
-        for filename in self.filelist:
-            appends = self.cooker.collection.get_file_appends(filename)
-            if not self.bb_cache.cacheValid(filename, appends):
-                self.willparse.append((filename, appends))
-            else:
-                self.fromcache.append((filename, appends))
-        self.toparse = self.total - len(self.fromcache)
+        self.bb_caches = bb.cache.MulticonfigCache(self.cfgbuilder, self.cfghash, cooker.caches_array)
+        self.fromcache = set()
+        self.willparse = set()
+        for mc in self.cooker.multiconfigs:
+            for filename in self.mcfilelist[mc]:
+                appends = self.cooker.collections[mc].get_file_appends(filename)
+                if not self.bb_caches[mc].cacheValid(filename, appends):
+                    self.willparse.add((mc, self.bb_caches[mc], filename, appends))
+                else:
+                    self.fromcache.add((mc, self.bb_caches[mc], filename, appends))
+
+        self.total = len(self.fromcache) + len(self.willparse)
+        self.toparse = len(self.willparse)
         self.progress_chunk = int(max(self.toparse / 100, 1))
 
         self.num_processes = min(int(self.cfgdata.getVar("BB_NUMBER_PARSE_THREADS") or
-                                 multiprocessing.cpu_count()), len(self.willparse))
+                                 multiprocessing.cpu_count()), self.toparse)
 
         self.start()
         self.haveshutdown = False
+        self.syncthread = None
 
     def start(self):
         self.results = self.load_cached()
@@ -2021,7 +2092,9 @@ class CookerParser(object):
         if self.toparse:
             bb.event.fire(bb.event.ParseStarted(self.toparse), self.cfgdata)
             def init():
-                Parser.bb_cache = self.bb_cache
+                signal.signal(signal.SIGTERM, signal.SIG_DFL)
+                signal.signal(signal.SIGHUP, signal.SIG_DFL)
+                signal.signal(signal.SIGINT, signal.SIG_IGN)
                 bb.utils.set_process_name(multiprocessing.current_process().name)
                 multiprocessing.util.Finalize(None, bb.codeparser.parser_cache_save, exitpriority=1)
                 multiprocessing.util.Finalize(None, bb.fetch.fetcher_parse_save, exitpriority=1)
@@ -2031,7 +2104,7 @@ class CookerParser(object):
 
             def chunkify(lst,n):
                 return [lst[i::n] for i in range(n)]
-            self.jobs = chunkify(self.willparse, self.num_processes)
+            self.jobs = chunkify(list(self.willparse), self.num_processes)
 
             for i in range(0, self.num_processes):
                 parser = Parser(self.jobs[i], self.result_queue, self.parser_quit, init, self.cooker.configuration.profile)
@@ -2055,12 +2128,9 @@ class CookerParser(object):
                                             self.total)
 
             bb.event.fire(event, self.cfgdata)
-            for process in self.processes:
-                self.parser_quit.put(None)
-        else:
-            self.parser_quit.cancel_join_thread()
-            for process in self.processes:
-                self.parser_quit.put(None)
+
+        for process in self.processes:
+            self.parser_quit.put(None)
 
         # Cleanup the queue before call process.join(), otherwise there might be
         # deadlocks.
@@ -2077,9 +2147,17 @@ class CookerParser(object):
             else:
                 process.join()
 
-        sync = threading.Thread(target=self.bb_cache.sync)
+        self.parser_quit.close()
+        # Allow data left in the cancel queue to be discarded
+        self.parser_quit.cancel_join_thread()
+
+        def sync_caches():
+            for c in self.bb_caches.values():
+                c.sync()
+
+        sync = threading.Thread(target=sync_caches, name="SyncThread")
+        self.syncthread = sync
         sync.start()
-        multiprocessing.util.Finalize(None, sync.join, exitpriority=-100)
         bb.codeparser.parser_cache_savemerge()
         bb.fetch.fetcher_parse_done()
         if self.cooker.configuration.profile:
@@ -2093,10 +2171,14 @@ class CookerParser(object):
             bb.utils.process_profilelog(profiles, pout = pout)
             print("Processed parsing statistics saved to %s" % (pout))
 
+    def final_cleanup(self):
+        if self.syncthread:
+            self.syncthread.join()
+
     def load_cached(self):
-        for filename, appends in self.fromcache:
-            cached, infos = self.bb_cache.load(filename, appends)
-            yield not cached, infos
+        for mc, cache, filename, appends in self.fromcache:
+            cached, infos = cache.load(filename, appends)
+            yield not cached, mc, infos
 
     def parse_generator(self):
         while True:
@@ -2118,7 +2200,7 @@ class CookerParser(object):
         result = []
         parsed = None
         try:
-            parsed, result = next(self.results)
+            parsed, mc, result = next(self.results)
         except StopIteration:
             self.shutdown()
             return False
@@ -2174,13 +2256,16 @@ class CookerParser(object):
             if info_array[0].skipped:
                 self.skipped += 1
                 self.cooker.skiplist[virtualfn] = SkippedPackage(info_array[0])
-            (fn, cls, mc) = bb.cache.virtualfn2realfn(virtualfn)
-            self.bb_cache.add_info(virtualfn, info_array, self.cooker.recipecaches[mc],
+            self.bb_caches[mc].add_info(virtualfn, info_array, self.cooker.recipecaches[mc],
                                         parsed=parsed, watcher = self.cooker.add_filewatch)
         return True
 
     def reparse(self, filename):
-        infos = self.bb_cache.parse(filename, self.cooker.collection.get_file_appends(filename))
-        for vfn, info_array in infos:
-            (fn, cls, mc) = bb.cache.virtualfn2realfn(vfn)
-            self.cooker.recipecaches[mc].add_from_recipeinfo(vfn, info_array)
+        to_reparse = set()
+        for mc in self.cooker.multiconfigs:
+            to_reparse.add((mc, filename, self.cooker.collections[mc].get_file_appends(filename)))
+
+        for mc, filename, appends in to_reparse:
+            infos = self.bb_caches[mc].parse(filename, appends)
+            for vfn, info_array in infos:
+                self.cooker.recipecaches[mc].add_from_recipeinfo(vfn, info_array)

bitbake/lib/bb/cookerdata.py

@@ -58,11 +58,18 @@ class ConfigParameters(object):
     def updateToServer(self, server, environment):
         options = {}
         for o in ["abort", "force", "invalidate_stamp",
-                  "verbose", "debug", "dry_run", "dump_signatures",
-                  "debug_domains", "extra_assume_provided", "profile",
-                  "prefile", "postfile", "server_timeout"]:
+                  "dry_run", "dump_signatures",
+                  "extra_assume_provided", "profile",
+                  "prefile", "postfile", "server_timeout",
+                  "nosetscene", "setsceneonly", "skipsetscene",
+                  "runall", "runonly", "writeeventlog"]:
             options[o] = getattr(self.options, o)
 
+        options['build_verbose_shell'] = self.options.verbose
+        options['build_verbose_stdout'] = self.options.verbose
+        options['default_loglevel'] = bb.msg.loggerDefaultLogLevel
+        options['debug_domains'] = bb.msg.loggerDefaultDomains
+
         ret, error = server.runCommand(["updateConfig", options, environment, sys.argv])
         if error:
             raise Exception("Unable to update the server configuration with local parameters: %s" % error)
@@ -111,11 +118,11 @@ class CookerConfiguration(object):
     """
 
     def __init__(self):
-        self.debug_domains = []
+        self.debug_domains = bb.msg.loggerDefaultDomains
+        self.default_loglevel = bb.msg.loggerDefaultLogLevel
         self.extra_assume_provided = []
         self.prefile = []
         self.postfile = []
-        self.debug = 0
         self.cmd = None
         self.abort = True
         self.force = False
@@ -125,34 +132,21 @@ class CookerConfiguration(object):
         self.skipsetscene = False
         self.invalidate_stamp = False
         self.dump_signatures = []
+        self.build_verbose_shell = False
+        self.build_verbose_stdout = False
         self.dry_run = False
         self.tracking = False
-        self.xmlrpcinterface = []
-        self.server_timeout = None
         self.writeeventlog = False
-        self.server_only = False
         self.limited_deps = False
         self.runall = []
         self.runonly = []
 
         self.env = {}
 
-    def setConfigParameters(self, parameters):
-        for key in self.__dict__.keys():
-            if key in parameters.options.__dict__:
-                setattr(self, key, parameters.options.__dict__[key])
-        self.env = parameters.environment.copy()
-
-    def setServerRegIdleCallback(self, srcb):
-        self.server_register_idlecallback = srcb
-
     def __getstate__(self):
         state = {}
         for key in self.__dict__.keys():
-            if key == "server_register_idlecallback":
-                state[key] = None
-            else:
-                state[key] = getattr(self, key)
+            state[key] = getattr(self, key)
         return state
 
     def __setstate__(self,state):
@@ -170,7 +164,7 @@ def catch_parse_error(func):
             import traceback
             parselog.critical(traceback.format_exc())
             parselog.critical("Unable to parse %s: %s" % (fn, exc))
-            sys.exit(1)
+            raise bb.BBHandledException()
         except bb.data_smart.ExpansionError as exc:
             import traceback
 
@@ -182,10 +176,10 @@ def catch_parse_error(func):
                 if not fn.startswith(bbdir):
                     break
             parselog.critical("Unable to parse %s" % fn, exc_info=(exc_class, exc, tb))
-            sys.exit(1)
+            raise bb.BBHandledException()
         except bb.parse.ParseError as exc:
             parselog.critical(str(exc))
-            sys.exit(1)
+            raise bb.BBHandledException()
     return wrapped
 
 @catch_parse_error
@@ -306,13 +300,13 @@ class CookerDataBuilder(object):
 
             self.data_hash = data_hash.hexdigest()
         except (SyntaxError, bb.BBHandledException):
-            raise bb.BBHandledException
+            raise bb.BBHandledException()
         except bb.data_smart.ExpansionError as e:
             logger.error(str(e))
-            raise bb.BBHandledException
+            raise bb.BBHandledException()
         except Exception:
             logger.exception("Error parsing configuration files")
-            raise bb.BBHandledException
+            raise bb.BBHandledException()
 
         # Create a copy so we can reset at a later date when UIs disconnect
         self.origdata = self.data
@@ -361,7 +355,7 @@ class CookerDataBuilder(object):
                 for layer in broken_layers:
                     parselog.critical("   %s", layer)
                 parselog.critical("Please check BBLAYERS in %s" % (layerconf))
-                sys.exit(1)
+                raise bb.BBHandledException()
 
             for layer in layers:
                 parselog.debug(2, "Adding layer %s", layer)
@@ -387,10 +381,13 @@ class CookerDataBuilder(object):
                     invalid.append(entry)
                     continue
                 l, f = parts
-                if l in collections:
+                invert = l[0] == "!"
+                if invert:
+                    l = l[1:]
+                if (l in collections and not invert) or (l not in collections and invert):
                     data.appendVar("BBFILES", " " + f)
             if invalid:
-                bb.fatal("BBFILES_DYNAMIC entries must be of the form <collection name>:<filename pattern>, not:\n    %s" % "\n    ".join(invalid))
+                bb.fatal("BBFILES_DYNAMIC entries must be of the form {!}<collection name>:<filename pattern>, not:\n    %s" % "\n    ".join(invalid))
 
             layerseries = set((data.getVar("LAYERSERIES_CORENAMES") or "").split())
             collections_tmp = collections[:]
@@ -430,7 +427,7 @@ class CookerDataBuilder(object):
             handlerfn = data.getVarFlag(var, "filename", False)
             if not handlerfn:
                 parselog.critical("Undefined event handler function '%s'" % var)
-                sys.exit(1)
+                raise bb.BBHandledException()
             handlerln = int(data.getVarFlag(var, "lineno", False))
             bb.event.register(var, data.getVar(var, False),  (data.getVarFlag(var, "eventmask") or "").split(), handlerfn, handlerln)
 

bitbake/lib/bb/daemonize.py

@@ -14,6 +14,8 @@ import sys
 import io
 import traceback
 
+import bb
+
 def createDaemon(function, logfile):
     """
     Detach a process from the controlling terminal and run it in the

bitbake/lib/bb/data.py

@@ -79,7 +79,7 @@ def expand(s, d, varname = None):
     return d.expand(s, varname)
 
 def expandKeys(alterdata, readdata = None):
-    if readdata == None:
+    if readdata is None:
         readdata = alterdata
 
     todolist = {}
@@ -161,6 +161,12 @@ def emit_var(var, o=sys.__stdout__, d = init(), all=False):
         return True
 
     if func:
+        # Write a comment indicating where the shell function came from (line number and filename) to make it easier
+        # for the user to diagnose task failures. This comment is also used by build.py to determine the metadata
+        # location of shell functions.
+        o.write("# line: {0}, file: {1}\n".format(
+            d.getVarFlag(var, "lineno", False),
+            d.getVarFlag(var, "filename", False)))
         # NOTE: should probably check for unbalanced {} within the var
         val = val.rstrip('\n')
         o.write("%s() {\n%s\n}\n" % (varExpanded, val))
@@ -365,7 +371,7 @@ def build_dependencies(key, keys, shelldeps, varflagsexcl, d):
     #bb.note("Variable %s references %s and calls %s" % (key, str(deps), str(execs)))
     #d.setVarFlag(key, "vardeps", deps)
 
-def generate_dependencies(d):
+def generate_dependencies(d, whitelist):
 
     keys = set(key for key in d if not key.startswith("__"))
     shelldeps = set(key for key in d.getVar("__exportlist", False) if d.getVarFlag(key, "export", False) and not d.getVarFlag(key, "unexport", False))
@@ -380,7 +386,7 @@ def generate_dependencies(d):
         newdeps = deps[task]
         seen = set()
         while newdeps:
-            nextdeps = newdeps
+            nextdeps = newdeps - whitelist
             seen |= nextdeps
             newdeps = set()
             for dep in nextdeps:

bitbake/lib/bb/data_smart.py

@@ -107,10 +107,6 @@ class VariableParse:
             else:
                 code = match.group()[3:-1]
 
-            if "_remote_data" in self.d:
-                connector = self.d["_remote_data"]
-                return connector.expandPythonRef(self.varname, code, self.d)
-
             if self.varname:
                 varname = 'Var <%s>' % self.varname
             else:
@@ -193,7 +189,7 @@ class IncludeHistory(object):
         if self.current.parent:
             self.current = self.current.parent
         else:
-            bb.warn("Include log: Tried to finish '%s' at top level." % filename)
+            bb.warn("Include log: Tried to finish '%s' at top level." % self.filename)
         return False
 
     def emit(self, o, level = 0):
@@ -268,12 +264,7 @@ class VariableHistory(object):
             self.variables[newvar].append(i.copy())
 
     def variable(self, var):
-        remote_connector = self.dataroot.getVar('_remote_data', False)
-        if remote_connector:
-            varhistory = remote_connector.getVarHistory(var)
-        else:
-            varhistory = []
-
+        varhistory = []
         if var in self.variables:
             varhistory.extend(self.variables[var])
         return varhistory
@@ -338,11 +329,12 @@ class VariableHistory(object):
                 lines.append(line)
         return lines
 
-    def get_variable_items_files(self, var, d):
+    def get_variable_items_files(self, var):
         """
         Use variable history to map items added to a list variable and
         the files in which they were added.
         """
+        d = self.dataroot
         history = self.variable(var)
         finalitems = (d.getVar(var) or '').split()
         filemap = {}
@@ -470,10 +462,6 @@ class DataSmart(MutableMapping):
             if var in dest:
                 return dest[var], self.overridedata.get(var, None)
 
-            if "_remote_data" in dest:
-                connector = dest["_remote_data"]["_content"]
-                return connector.getVar(var)
-
             if "_data" not in dest:
                 break
             dest = dest["_data"]
@@ -498,12 +486,6 @@ class DataSmart(MutableMapping):
         if 'parsing' in loginfo:
             parsing=True
 
-        if '_remote_data' in self.dict:
-            connector = self.dict["_remote_data"]["_content"]
-            res = connector.setVar(var, value)
-            if not res:
-                return
-
         if 'op' not in loginfo:
             loginfo['op'] = "set"
 
@@ -607,11 +589,9 @@ class DataSmart(MutableMapping):
         """
         Rename the variable key to newkey
         """
-        if '_remote_data' in self.dict:
-            connector = self.dict["_remote_data"]["_content"]
-            res = connector.renameVar(key, newkey)
-            if not res:
-                return
+        if key == newkey:
+            bb.warn("Calling renameVar with equivalent keys (%s) is invalid" % key)
+            return
 
         val = self.getVar(key, 0, parsing=True)
         if val is not None:
@@ -658,11 +638,6 @@ class DataSmart(MutableMapping):
 
     def delVar(self, var, **loginfo):
         self.expand_cache = {}
-        if '_remote_data' in self.dict:
-            connector = self.dict["_remote_data"]["_content"]
-            res = connector.delVar(var)
-            if not res:
-                return
 
         loginfo['detail'] = ""
         loginfo['op'] = 'del'
@@ -690,11 +665,6 @@ class DataSmart(MutableMapping):
 
     def setVarFlag(self, var, flag, value, **loginfo):
         self.expand_cache = {}
-        if '_remote_data' in self.dict:
-            connector = self.dict["_remote_data"]["_content"]
-            res = connector.setVarFlag(var, flag, value)
-            if not res:
-                return
 
         if 'op' not in loginfo:
             loginfo['op'] = "set"
@@ -845,11 +815,6 @@ class DataSmart(MutableMapping):
 
     def delVarFlag(self, var, flag, **loginfo):
         self.expand_cache = {}
-        if '_remote_data' in self.dict:
-            connector = self.dict["_remote_data"]["_content"]
-            res = connector.delVarFlag(var, flag)
-            if not res:
-                return
 
         local_var, _ = self._findVar(var)
         if not local_var:
@@ -967,7 +932,7 @@ class DataSmart(MutableMapping):
 
     def localkeys(self):
         for key in self.dict:
-            if key not in ['_data', '_remote_data']:
+            if key not in ['_data']:
                 yield key
 
     def __iter__(self):
@@ -976,7 +941,7 @@ class DataSmart(MutableMapping):
         def keylist(d):        
             klist = set()
             for key in d:
-                if key in ["_data", "_remote_data"]:
+                if key in ["_data"]:
                     continue
                 if key in deleted:
                     continue
@@ -990,13 +955,6 @@ class DataSmart(MutableMapping):
             if "_data" in d:
                 klist |= keylist(d["_data"])
 
-            if "_remote_data" in d:
-                connector = d["_remote_data"]["_content"]
-                for key in connector.getKeys():
-                    if key in deleted:
-                        continue
-                    klist.add(key)
-
             return klist
 
         self.need_overrides()
@@ -1042,9 +1000,12 @@ class DataSmart(MutableMapping):
                 continue
 
             value = d.getVar(key, False) or ""
-            data.update({key:value})
+            if type(value) is type(self):
+                data.update({key:value.get_hash()})
+            else:
+                data.update({key:value})
 
-            varflags = d.getVarFlags(key, internalflags = True)
+            varflags = d.getVarFlags(key, internalflags = True, expand=["vardepvalue"])
             if not varflags:
                 continue
             for f in varflags:

bitbake/lib/bb/event.py

@@ -10,18 +10,17 @@ BitBake build tools.
 # SPDX-License-Identifier: GPL-2.0-only
 #
 
-import os, sys
-import warnings
-import pickle
-import logging
-import atexit
-import traceback
 import ast
+import atexit
+import collections
+import logging
+import pickle
+import sys
 import threading
+import traceback
 
-import bb.utils
-import bb.compat
 import bb.exceptions
+import bb.utils
 
 # This is the pid for which we should generate the event. This is set when
 # the runqueue forks off.
@@ -57,7 +56,7 @@ def set_class_handlers(h):
     _handlers = h
 
 def clean_class_handlers():
-    return bb.compat.OrderedDict()
+    return collections.OrderedDict()
 
 # Internal
 _handlers = clean_class_handlers()
@@ -347,7 +346,7 @@ def set_UIHmask(handlerNum, level, debug_domains, mask):
 
 def getName(e):
     """Returns the name of a class or class instance"""
-    if getattr(e, "__name__", None) == None:
+    if getattr(e, "__name__", None) is None:
         return e.__class__.__name__
     else:
         return e.__name__
@@ -390,6 +389,10 @@ class RecipeEvent(Event):
 class RecipePreFinalise(RecipeEvent):
     """ Recipe Parsing Complete but not yet finalised"""
 
+class RecipePostKeyExpansion(RecipeEvent):
+    """ Recipe Parsing Complete but not yet finalised"""
+
+
 class RecipeTaskPreProcess(RecipeEvent):
     """
     Recipe Tasks about to be finalised
@@ -509,7 +512,7 @@ class NoProvider(Event):
         extra = ''
         if not self._reasons:
             if self._close_matches:
-                extra = ". Close matches:\n  %s" % '\n  '.join(self._close_matches)
+                extra = ". Close matches:\n  %s" % '\n  '.join(sorted(set(self._close_matches)))
 
         if self._dependees:
             msg = "Nothing %sPROVIDES '%s' (but %s %sDEPENDS on or otherwise requires it)%s" % (r, self._item, ", ".join(self._dependees), r, extra)

bitbake/lib/bb/fetch2/__init__.py

@@ -33,6 +33,9 @@ _checksum_cache = bb.checksum.FileChecksumCache()
 
 logger = logging.getLogger("BitBake.Fetcher")
 
+CHECKSUM_LIST = [ "md5", "sha256", "sha1", "sha384", "sha512" ]
+SHOWN_CHECKSUM_LIST = ["sha256"]
+
 class BBFetchException(Exception):
     """Class all fetch exceptions inherit from"""
     def __init__(self, message):
@@ -131,10 +134,9 @@ class NonLocalMethod(Exception):
         Exception.__init__(self)
 
 class MissingChecksumEvent(bb.event.Event):
-    def __init__(self, url, md5sum, sha256sum):
+    def __init__(self, url, **checksums):
         self.url = url
-        self.checksums = {'md5sum': md5sum,
-                          'sha256sum': sha256sum}
+        self.checksums = checksums
         bb.event.Event.__init__(self)
 
 
@@ -484,17 +486,22 @@ def fetcher_init(d):
     Called to initialize the fetchers once the configuration data is known.
     Calls before this must not hit the cache.
     """
+
+    revs = bb.persist_data.persist('BB_URI_HEADREVS', d)
+    try:
+        # fetcher_init is called multiple times, so make sure we only save the
+        # revs the first time it is called.
+        if not bb.fetch2.saved_headrevs:
+            bb.fetch2.saved_headrevs = dict(revs)
+    except:
+        pass
+
     # When to drop SCM head revisions controlled by user policy
     srcrev_policy = d.getVar('BB_SRCREV_POLICY') or "clear"
     if srcrev_policy == "cache":
         logger.debug(1, "Keeping SRCREV cache due to cache policy of: %s", srcrev_policy)
     elif srcrev_policy == "clear":
         logger.debug(1, "Clearing SRCREV cache due to cache policy of: %s", srcrev_policy)
-        revs = bb.persist_data.persist('BB_URI_HEADREVS', d)
-        try:
-            bb.fetch2.saved_headrevs = revs.items()
-        except:
-            pass
         revs.clear()
     else:
         raise FetchError("Invalid SRCREV cache policy of: %s" % srcrev_policy)
@@ -513,22 +520,12 @@ def fetcher_parse_done():
 
 def fetcher_compare_revisions(d):
     """
-    Compare the revisions in the persistant cache with current values and
-    return true/false on whether they've changed.
+    Compare the revisions in the persistent cache with the saved values from
+    when bitbake was started and return true if they have changed.
     """
 
-    data = bb.persist_data.persist('BB_URI_HEADREVS', d).items()
-    data2 = bb.fetch2.saved_headrevs
-
-    changed = False
-    for key in data:
-        if key not in data2 or data2[key] != data[key]:
-            logger.debug(1, "%s changed", key)
-            changed = True
-            return True
-        else:
-            logger.debug(2, "%s did not change", key)
-    return False
+    headrevs = dict(bb.persist_data.persist('BB_URI_HEADREVS', d))
+    return headrevs != bb.fetch2.saved_headrevs
 
 def mirror_from_string(data):
     mirrors = (data or "").replace('\\n',' ').split()
@@ -552,71 +549,84 @@ def verify_checksum(ud, d, precomputed={}):
     downloading. See https://bugzilla.yoctoproject.org/show_bug.cgi?id=5571.
     """
 
-    _MD5_KEY = "md5"
-    _SHA256_KEY = "sha256"
-
     if ud.ignore_checksums or not ud.method.supports_checksum(ud):
         return {}
 
-    if _MD5_KEY in precomputed:
-        md5data = precomputed[_MD5_KEY]
-    else:
-        md5data = bb.utils.md5_file(ud.localpath)
+    def compute_checksum_info(checksum_id):
+        checksum_name = getattr(ud, "%s_name" % checksum_id)
 
-    if _SHA256_KEY in precomputed:
-        sha256data = precomputed[_SHA256_KEY]
-    else:
-        sha256data = bb.utils.sha256_file(ud.localpath)
+        if checksum_id in precomputed:
+            checksum_data = precomputed[checksum_id]
+        else:
+            checksum_data = getattr(bb.utils, "%s_file" % checksum_id)(ud.localpath)
 
-    if ud.method.recommends_checksum(ud) and not ud.md5_expected and not ud.sha256_expected:
-        # If strict checking enabled and neither sum defined, raise error
+        checksum_expected = getattr(ud, "%s_expected" % checksum_id)
+
+        return {
+            "id": checksum_id,
+            "name": checksum_name,
+            "data": checksum_data,
+            "expected": checksum_expected
+        }
+
+    checksum_infos = []
+    for checksum_id in CHECKSUM_LIST:
+        checksum_infos.append(compute_checksum_info(checksum_id))
+
+    checksum_dict = {ci["id"] : ci["data"] for ci in checksum_infos}
+    checksum_event = {"%ssum" % ci["id"] : ci["data"] for ci in checksum_infos}
+
+    for ci in checksum_infos:
+        if ci["id"] in SHOWN_CHECKSUM_LIST:
+            checksum_lines = ["SRC_URI[%s] = \"%s\"" % (ci["name"], ci["data"])]
+
+    # If no checksum has been provided
+    if ud.method.recommends_checksum(ud) and all(ci["expected"] is None for ci in checksum_infos):
+        messages = []
         strict = d.getVar("BB_STRICT_CHECKSUM") or "0"
-        if strict == "1":
-            logger.error('No checksum specified for %s, please add at least one to the recipe:\n'
-                             'SRC_URI[%s] = "%s"\nSRC_URI[%s] = "%s"' %
-                             (ud.localpath, ud.md5_name, md5data,
-                              ud.sha256_name, sha256data))
-            raise NoChecksumError('Missing SRC_URI checksum', ud.url)
 
-        bb.event.fire(MissingChecksumEvent(ud.url, md5data, sha256data), d)
+        # If strict checking enabled and neither sum defined, raise error
+        if strict == "1":
+            messages.append("No checksum specified for '%s', please add at " \
+                            "least one to the recipe:" % ud.localpath)
+            messages.extend(checksum_lines)
+            logger.error("\n".join(messages))
+            raise NoChecksumError("Missing SRC_URI checksum", ud.url)
+
+        bb.event.fire(MissingChecksumEvent(ud.url, **checksum_event), d)
 
         if strict == "ignore":
-            return {
-                _MD5_KEY: md5data,
-                _SHA256_KEY: sha256data
-            }
+            return checksum_dict
 
         # Log missing sums so user can more easily add them
-        logger.warning('Missing md5 SRC_URI checksum for %s, consider adding to the recipe:\n'
-                       'SRC_URI[%s] = "%s"',
-                       ud.localpath, ud.md5_name, md5data)
-        logger.warning('Missing sha256 SRC_URI checksum for %s, consider adding to the recipe:\n'
-                       'SRC_URI[%s] = "%s"',
-                       ud.localpath, ud.sha256_name, sha256data)
+        messages.append("Missing checksum for '%s', consider adding at " \
+                        "least one to the recipe:" % ud.localpath)
+        messages.extend(checksum_lines)
+        logger.warning("\n".join(messages))
 
     # We want to alert the user if a checksum is defined in the recipe but
     # it does not match.
-    msg = ""
-    mismatch = False
-    if ud.md5_expected and ud.md5_expected != md5data:
-        msg = msg + "\nFile: '%s' has %s checksum %s when %s was expected" % (ud.localpath, 'md5', md5data, ud.md5_expected)
-        mismatch = True;
+    messages = []
+    messages.append("Checksum mismatch!")
+    bad_checksum = None
 
-    if ud.sha256_expected and ud.sha256_expected != sha256data:
-        msg = msg + "\nFile: '%s' has %s checksum %s when %s was expected" % (ud.localpath, 'sha256', sha256data, ud.sha256_expected)
-        mismatch = True;
+    for ci in checksum_infos:
+        if ci["expected"] and ci["expected"] != ci["data"]:
+            messages.append("File: '%s' has %s checksum %s when %s was " \
+                            "expected" % (ud.localpath, ci["id"], ci["data"], ci["expected"]))
+            bad_checksum = ci["data"]
 
-    if mismatch:
-        msg = msg + '\nIf this change is expected (e.g. you have upgraded to a new version without updating the checksums) then you can use these lines within the recipe:\nSRC_URI[%s] = "%s"\nSRC_URI[%s] = "%s"\nOtherwise you should retry the download and/or check with upstream to determine if the file has become corrupted or otherwise unexpectedly modified.\n' % (ud.md5_name, md5data, ud.sha256_name, sha256data)
-
-    if len(msg):
-        raise ChecksumError('Checksum mismatch!%s' % msg, ud.url, md5data)
-
-    return {
-        _MD5_KEY: md5data,
-        _SHA256_KEY: sha256data
-    }
+    if bad_checksum:
+        messages.append("If this change is expected (e.g. you have upgraded " \
+                        "to a new version without updating the checksums) " \
+                        "then you can use these lines within the recipe:")
+        messages.extend(checksum_lines)
+        messages.append("Otherwise you should retry the download and/or " \
+                        "check with upstream to determine if the file has " \
+                        "become corrupted or otherwise unexpectedly modified.")
+        raise ChecksumError("\n".join(messages), ud.url, bad_checksum)
 
+    return checksum_dict
 
 def verify_donestamp(ud, d, origud=None):
     """
@@ -843,11 +853,6 @@ def runfetchcmd(cmd, d, quiet=False, cleanup=None, log=None, workdir=None):
         if val:
             cmd = 'export ' + var + '=\"%s\"; %s' % (val, cmd)
 
-    # Ensure that a _PYTHON_SYSCONFIGDATA_NAME value set by a recipe
-    # (for example via python3native.bbclass since warrior) is not set for
-    # host Python (otherwise tools like git-make-shallow will fail)
-    cmd = 'unset _PYTHON_SYSCONFIGDATA_NAME; ' + cmd
-
     # Disable pseudo as it may affect ssh, potentially causing it to hang.
     cmd = 'export PSEUDO_DISABLED=1; ' + cmd
 
@@ -1081,7 +1086,7 @@ def try_mirrors(fetch, d, origud, mirrors, check = False):
 
     for index, uri in enumerate(uris):
         ret = try_mirror_url(fetch, origud, uds[index], ld, check)
-        if ret != False:
+        if ret:
             return ret
     return None
 
@@ -1185,8 +1190,6 @@ def get_checksum_file_list(d):
             paths = ud.method.localpaths(ud, d)
             for f in paths:
                 pth = ud.decodedurl
-                if '*' in pth:
-                    f = os.path.join(os.path.abspath(f), pth)
                 if f.startswith(dl_dir):
                     # The local fetcher's behaviour is to return a path under DL_DIR if it couldn't find the file anywhere else
                     if os.path.exists(f):
@@ -1230,24 +1233,26 @@ class FetchData(object):
             self.pswd = self.parm["pswd"]
         self.setup = False
 
-        if "name" in self.parm:
-            self.md5_name = "%s.md5sum" % self.parm["name"]
-            self.sha256_name = "%s.sha256sum" % self.parm["name"]
-        else:
-            self.md5_name = "md5sum"
-            self.sha256_name = "sha256sum"
-        if self.md5_name in self.parm:
-            self.md5_expected = self.parm[self.md5_name]
-        elif self.type not in ["http", "https", "ftp", "ftps", "sftp", "s3"]:
-            self.md5_expected = None
-        else:
-            self.md5_expected = d.getVarFlag("SRC_URI", self.md5_name)
-        if self.sha256_name in self.parm:
-            self.sha256_expected = self.parm[self.sha256_name]
-        elif self.type not in ["http", "https", "ftp", "ftps", "sftp", "s3"]:
-            self.sha256_expected = None
-        else:
-            self.sha256_expected = d.getVarFlag("SRC_URI", self.sha256_name)
+        def configure_checksum(checksum_id):
+            if "name" in self.parm:
+                checksum_name = "%s.%ssum" % (self.parm["name"], checksum_id)
+            else:
+                checksum_name = "%ssum" % checksum_id
+
+            setattr(self, "%s_name" % checksum_id, checksum_name)
+
+            if checksum_name in self.parm:
+                checksum_expected = self.parm[checksum_name]
+            elif self.type not in ["http", "https", "ftp", "ftps", "sftp", "s3"]:
+                checksum_expected = None
+            else:
+                checksum_expected = d.getVarFlag("SRC_URI", checksum_name)
+
+            setattr(self, "%s_expected" % checksum_id, checksum_expected)
+
+        for checksum_id in CHECKSUM_LIST:
+            configure_checksum(checksum_id)
+
         self.ignore_checksums = False
 
         self.names = self.parm.get("name",'default').split(',')
@@ -1351,11 +1356,8 @@ class FetchMethod(object):
         """
 
         # We cannot compute checksums for directories
-        if os.path.isdir(urldata.localpath) == True:
+        if os.path.isdir(urldata.localpath):
             return False
-        if urldata.localpath.find("*") != -1:
-            return False
-
         return True
 
     def recommends_checksum(self, urldata):
@@ -1365,6 +1367,18 @@ class FetchMethod(object):
         """
         return False
 
+    def verify_donestamp(self, ud, d):
+        """
+        Verify the donestamp file
+        """
+        return verify_donestamp(ud, d)
+
+    def update_donestamp(self, ud, d):
+        """
+        Update the donestamp file
+        """
+        update_stamp(ud, d)
+
     def _strip_leading_slashes(self, relpath):
         """
         Remove leading slash as os.path.join can't cope
@@ -1406,11 +1420,6 @@ class FetchMethod(object):
         iterate = False
         file = urldata.localpath
 
-        # Localpath can't deal with 'dir/*' entries, so it converts them to '.',
-        # but it must be corrected back for local files copying
-        if urldata.basename == '*' and file.endswith('/.'):
-            file = '%s/%s' % (file.rstrip('/.'), urldata.path)
-
         try:
             unpack = bb.utils.to_boolean(urldata.parm.get('unpack'), True)
         except ValueError as exc:
@@ -1506,7 +1515,7 @@ class FetchMethod(object):
                     if urlpath.find("/") != -1:
                         destdir = urlpath.rsplit("/", 1)[0] + '/'
                         bb.utils.mkdirhier("%s/%s" % (unpackdir, destdir))
-                cmd = 'cp -fpPRH %s %s' % (file, destdir)
+                cmd = 'cp -fpPRH "%s" "%s"' % (file, destdir)
 
         if not cmd:
             return
@@ -1539,6 +1548,12 @@ class FetchMethod(object):
         """
         return True
 
+    def try_mirrors(self, fetch, urldata, d, mirrors, check=False):
+        """
+        Try to use a mirror
+        """
+        return bool(try_mirrors(fetch, d, urldata, mirrors, check))
+
     def checkstatus(self, fetch, urldata, d):
         """
         Check the status of a URL
@@ -1577,6 +1592,21 @@ class FetchMethod(object):
         """
         return ('', '')
 
+    def done(self, ud, d):
+        """
+        Is the download done ?
+        """
+        if os.path.exists(ud.localpath):
+            return True
+        return False
+
+    def implicit_urldata(self, ud, d):
+        """
+        Get a list of FetchData objects for any implicit URLs that will also
+        be downloaded when we fetch the given URL.
+        """
+        return []
+
 class Fetch(object):
     def __init__(self, urls, d, cache = True, localonly = False, connection_cache = None):
         if localonly and cache:
@@ -1591,8 +1621,11 @@ class Fetch(object):
 
         fn = d.getVar('FILE')
         mc = d.getVar('__BBMULTICONFIG') or ""
-        if cache and fn and mc + fn in urldata_cache:
-            self.ud = urldata_cache[mc + fn + str(id(d))]
+        key = None
+        if cache and fn:
+            key = mc + fn + str(id(d))
+        if key in urldata_cache:
+            self.ud = urldata_cache[key]
 
         for url in urls:
             if url not in self.ud:
@@ -1603,8 +1636,8 @@ class Fetch(object):
                         self.ud[url] = None
                         pass
 
-        if fn and cache:
-            urldata_cache[mc + fn + str(id(d))] = self.ud
+        if key:
+            urldata_cache[key] = self.ud
 
     def localpath(self, url):
         if url not in self.urls:
@@ -1640,7 +1673,7 @@ class Fetch(object):
             ud = self.ud[u]
             ud.setup_localpath(self.d)
             m = ud.method
-            localpath = ""
+            done = False
 
             if ud.lockfile:
                 lf = bb.utils.lockfile(ud.lockfile)
@@ -1648,28 +1681,28 @@ class Fetch(object):
             try:
                 self.d.setVar("BB_NO_NETWORK", network)
 
-                if verify_donestamp(ud, self.d) and not m.need_update(ud, self.d):
-                    localpath = ud.localpath
+                if m.verify_donestamp(ud, self.d) and not m.need_update(ud, self.d):
+                    done = True
                 elif m.try_premirror(ud, self.d):
                     logger.debug(1, "Trying PREMIRRORS")
                     mirrors = mirror_from_string(self.d.getVar('PREMIRRORS'))
-                    localpath = try_mirrors(self, self.d, ud, mirrors, False)
-                    if localpath:
+                    done = m.try_mirrors(self, ud, self.d, mirrors)
+                    if done:
                         try:
                             # early checksum verification so that if the checksum of the premirror
                             # contents mismatch the fetcher can still try upstream and mirrors
-                            update_stamp(ud, self.d)
+                            m.update_donestamp(ud, self.d)
                         except ChecksumError as e:
                             logger.warning("Checksum failure encountered with premirror download of %s - will attempt other sources." % u)
                             logger.debug(1, str(e))
-                            localpath = ""
+                            done = False
 
                 if premirroronly:
                     self.d.setVar("BB_NO_NETWORK", "1")
 
                 firsterr = None
-                verified_stamp = verify_donestamp(ud, self.d)
-                if not localpath and (not verified_stamp or m.need_update(ud, self.d)):
+                verified_stamp = m.verify_donestamp(ud, self.d)
+                if not done and (not verified_stamp or m.need_update(ud, self.d)):
                     try:
                         if not trusted_network(self.d, ud.url):
                             raise UntrustedUrl(ud.url)
@@ -1677,10 +1710,10 @@ class Fetch(object):
                         m.download(ud, self.d)
                         if hasattr(m, "build_mirror_data"):
                             m.build_mirror_data(ud, self.d)
-                        localpath = ud.localpath
+                        done = True
                         # early checksum verify, so that if checksum mismatched,
                         # fetcher still have chance to fetch from mirror
-                        update_stamp(ud, self.d)
+                        m.update_donestamp(ud, self.d)
 
                     except bb.fetch2.NetworkAccess:
                         raise
@@ -1702,14 +1735,14 @@ class Fetch(object):
                             m.clean(ud, self.d)
                         logger.debug(1, "Trying MIRRORS")
                         mirrors = mirror_from_string(self.d.getVar('MIRRORS'))
-                        localpath = try_mirrors(self, self.d, ud, mirrors)
+                        done = m.try_mirrors(self, ud, self.d, mirrors)
 
-                if not localpath or ((not os.path.exists(localpath)) and localpath.find("*") == -1):
+                if not done or not m.done(ud, self.d):
                     if firsterr:
                         logger.error(str(firsterr))
                     raise FetchError("Unable to fetch URL from any source.", u)
 
-                update_stamp(ud, self.d)
+                m.update_donestamp(ud, self.d)
 
             except IOError as e:
                 if e.errno in [errno.ESTALE]:
@@ -1740,14 +1773,14 @@ class Fetch(object):
             logger.debug(1, "Testing URL %s", u)
             # First try checking uri, u, from PREMIRRORS
             mirrors = mirror_from_string(self.d.getVar('PREMIRRORS'))
-            ret = try_mirrors(self, self.d, ud, mirrors, True)
+            ret = m.try_mirrors(self, ud, self.d, mirrors, True)
             if not ret:
                 # Next try checking from the original uri, u
                 ret = m.checkstatus(self, ud, self.d)
                 if not ret:
                     # Finally, try checking uri, u, from MIRRORS
                     mirrors = mirror_from_string(self.d.getVar('MIRRORS'))
-                    ret = try_mirrors(self, self.d, ud, mirrors, True)
+                    ret = m.try_mirrors(self, ud, self.d, mirrors, True)
 
             if not ret:
                 raise FetchError("URL %s doesn't work" % u, u)
@@ -1799,6 +1832,24 @@ class Fetch(object):
             if ud.lockfile:
                 bb.utils.unlockfile(lf)
 
+    def expanded_urldata(self, urls=None):
+        """
+        Get an expanded list of FetchData objects covering both the given
+        URLS and any additional implicit URLs that are added automatically by
+        the appropriate FetchMethod.
+        """
+
+        if not urls:
+            urls = self.urls
+
+        urldata = []
+        for url in urls:
+            ud = self.ud[url]
+            urldata.append(ud)
+            urldata += ud.method.implicit_urldata(ud, self.d)
+
+        return urldata
+
 class FetchConnectionCache(object):
     """
         A class which represents an container for socket connections.
@@ -1852,6 +1903,7 @@ from . import osc
 from . import repo
 from . import clearcase
 from . import npm
+from . import npmsw
 
 methods.append(local.Local())
 methods.append(wget.Wget())
@@ -1870,3 +1922,4 @@ methods.append(osc.Osc())
 methods.append(repo.Repo())
 methods.append(clearcase.ClearCase())
 methods.append(npm.Npm())
+methods.append(npmsw.NpmShrinkWrap())

bitbake/lib/bb/fetch2/bzr.py

@@ -14,8 +14,6 @@ BitBake 'Fetch' implementation for bzr.
 #
 
 import os
-import sys
-import logging
 import bb
 from bb.fetch2 import FetchMethod
 from bb.fetch2 import FetchError

bitbake/lib/bb/fetch2/clearcase.py

@@ -49,7 +49,6 @@ User credentials:
 #
 
 import os
-import sys
 import shutil
 import bb
 from   bb.fetch2 import FetchMethod
@@ -238,7 +237,7 @@ class ClearCase(FetchMethod):
 
         # Clean clearcase meta-data before tar
 
-        runfetchcmd('tar -czf "%s" .' % (ud.localpath), d, cleanup = [ud.localpath])
+        runfetchcmd('tar -czf "%s" .' % (ud.localpath), d, cleanup = [ud.localpath], workdir = ud.viewdir)
 
         # Clean up so we can create a new view next time
         self.clean(ud, d);

bitbake/lib/bb/fetch2/cvs.py

@@ -14,7 +14,6 @@ BitBake build tools.
 #
 
 import os
-import logging
 import bb
 from bb.fetch2 import FetchMethod, FetchError, MissingParameterError, logger
 from bb.fetch2 import runfetchcmd
@@ -52,6 +51,10 @@ class Cvs(FetchMethod):
 
         ud.localfile = d.expand('%s_%s_%s_%s%s%s.tar.gz' % (ud.module.replace('/', '.'), ud.host, ud.tag, ud.date, norecurse, fullpath))
 
+        pkg = d.getVar('PN')
+        cvsdir = d.getVar("CVSDIR") or (d.getVar("DL_DIR") + "/cvs")
+        ud.pkgdir = os.path.join(cvsdir, pkg)
+
     def need_update(self, ud, d):
         if (ud.date == "now"):
             return True
@@ -107,10 +110,7 @@ class Cvs(FetchMethod):
 
         # create module directory
         logger.debug(2, "Fetch: checking for module directory")
-        pkg = d.getVar('PN')
-        cvsdir = d.getVar("CVSDIR") or (d.getVar("DL_DIR") + "/cvs")
-        pkgdir = os.path.join(cvsdir, pkg)
-        moddir = os.path.join(pkgdir, localdir)
+        moddir = os.path.join(ud.pkgdir, localdir)
         workdir = None
         if os.access(os.path.join(moddir, 'CVS'), os.R_OK):
             logger.info("Update " + ud.url)
@@ -121,8 +121,8 @@ class Cvs(FetchMethod):
         else:
             logger.info("Fetch " + ud.url)
             # check out sources there
-            bb.utils.mkdirhier(pkgdir)
-            workdir = pkgdir
+            bb.utils.mkdirhier(ud.pkgdir)
+            workdir = ud.pkgdir
             logger.debug(1, "Running %s", cvscmd)
             bb.fetch2.check_network_access(d, cvscmd, ud.url)
             cmd = cvscmd
@@ -141,7 +141,7 @@ class Cvs(FetchMethod):
         # tar them up to a defined filename
         workdir = None
         if 'fullpath' in ud.parm:
-            workdir = pkgdir
+            workdir = ud.pkgdir
             cmd = "tar %s -czf %s %s" % (tar_flags, ud.localpath, localdir)
         else:
             workdir = os.path.dirname(os.path.realpath(moddir))
@@ -152,9 +152,6 @@ class Cvs(FetchMethod):
     def clean(self, ud, d):
         """ Clean CVS Files and tarballs """
 
-        pkg = d.getVar('PN')
-        pkgdir = os.path.join(d.getVar("CVSDIR"), pkg)
-
-        bb.utils.remove(pkgdir, True)
+        bb.utils.remove(ud.pkgdir, True)
         bb.utils.remove(ud.localpath)
 

bitbake/lib/bb/fetch2/git.py

@@ -63,6 +63,7 @@ import errno
 import fnmatch
 import os
 import re
+import shlex
 import subprocess
 import tempfile
 import bb
@@ -236,7 +237,7 @@ class Git(FetchMethod):
                     ud.unresolvedrev[name] = ud.revisions[name]
                 ud.revisions[name] = self.latest_revision(ud, d, name)
 
-        gitsrcname = '%s%s' % (ud.host.replace(':', '.'), ud.path.replace('/', '.').replace('*', '.'))
+        gitsrcname = '%s%s' % (ud.host.replace(':', '.'), ud.path.replace('/', '.').replace('*', '.').replace(' ','_'))
         if gitsrcname.startswith('.'):
             gitsrcname = gitsrcname[1:]
 
@@ -342,7 +343,7 @@ class Git(FetchMethod):
             # We do this since git will use a "-l" option automatically for local urls where possible
             if repourl.startswith("file://"):
                 repourl = repourl[7:]
-            clone_cmd = "LANG=C %s clone --bare --mirror %s %s --progress" % (ud.basecmd, repourl, ud.clonedir)
+            clone_cmd = "LANG=C %s clone --bare --mirror %s %s --progress" % (ud.basecmd, shlex.quote(repourl), ud.clonedir)
             if ud.proto.lower() != 'file':
                 bb.fetch2.check_network_access(d, clone_cmd, ud.url)
             progresshandler = GitProgressHandler(d)
@@ -354,8 +355,8 @@ class Git(FetchMethod):
             if "origin" in output:
               runfetchcmd("%s remote rm origin" % ud.basecmd, d, workdir=ud.clonedir)
 
-            runfetchcmd("%s remote add --mirror=fetch origin %s" % (ud.basecmd, repourl), d, workdir=ud.clonedir)
-            fetch_cmd = "LANG=C %s fetch -f --prune --progress %s refs/*:refs/*" % (ud.basecmd, repourl)
+            runfetchcmd("%s remote add --mirror=fetch origin %s" % (ud.basecmd, shlex.quote(repourl)), d, workdir=ud.clonedir)
+            fetch_cmd = "LANG=C %s fetch -f --progress %s refs/*:refs/*" % (ud.basecmd, shlex.quote(repourl))
             if ud.proto.lower() != 'file':
                 bb.fetch2.check_network_access(d, fetch_cmd, ud.url)
             progresshandler = GitProgressHandler(d)
@@ -378,6 +379,35 @@ class Git(FetchMethod):
             if missing_rev:
                 raise bb.fetch2.FetchError("Unable to find revision %s even from upstream" % missing_rev)
 
+        if self._contains_lfs(ud, d, ud.clonedir) and self._need_lfs(ud):
+            # Unpack temporary working copy, use it to run 'git checkout' to force pre-fetching
+            # of all LFS blobs needed at the the srcrev.
+            #
+            # It would be nice to just do this inline here by running 'git-lfs fetch'
+            # on the bare clonedir, but that operation requires a working copy on some
+            # releases of Git LFS.
+            tmpdir = tempfile.mkdtemp(dir=d.getVar('DL_DIR'))
+            try:
+                # Do the checkout. This implicitly involves a Git LFS fetch.
+                self.unpack(ud, tmpdir, d)
+
+                # Scoop up a copy of any stuff that Git LFS downloaded. Merge them into
+                # the bare clonedir.
+                #
+                # As this procedure is invoked repeatedly on incremental fetches as
+                # a recipe's SRCREV is bumped throughout its lifetime, this will
+                # result in a gradual accumulation of LFS blobs in <ud.clonedir>/lfs
+                # corresponding to all the blobs reachable from the different revs
+                # fetched across time.
+                #
+                # Only do this if the unpack resulted in a .git/lfs directory being
+                # created; this only happens if at least one blob needed to be
+                # downloaded.
+                if os.path.exists(os.path.join(tmpdir, "git", ".git", "lfs")):
+                    runfetchcmd("tar -cf - lfs | tar -xf - -C %s" % ud.clonedir, d, workdir="%s/git/.git" % tmpdir)
+            finally:
+                bb.utils.remove(tmpdir, recurse=True)
+
     def build_mirror_data(self, ud, d):
         if ud.shallow and ud.write_shallow_tarballs:
             if not os.path.exists(ud.fullshallow):
@@ -473,7 +503,10 @@ class Git(FetchMethod):
         if os.path.exists(destdir):
             bb.utils.prunedir(destdir)
 
-        need_lfs = ud.parm.get("lfs", "1") == "1"
+        need_lfs = self._need_lfs(ud)
+
+        if not need_lfs:
+            ud.basecmd = "GIT_LFS_SKIP_SMUDGE=1 " + ud.basecmd
 
         source_found = False
         source_error = []
@@ -501,12 +534,12 @@ class Git(FetchMethod):
             raise bb.fetch2.UnpackError("No up to date source found: " + "; ".join(source_error), ud.url)
 
         repourl = self._get_repo_url(ud)
-        runfetchcmd("%s remote set-url origin %s" % (ud.basecmd, repourl), d, workdir=destdir)
+        runfetchcmd("%s remote set-url origin %s" % (ud.basecmd, shlex.quote(repourl)), d, workdir=destdir)
 
         if self._contains_lfs(ud, d, destdir):
             if need_lfs and not self._find_git_lfs(d):
                 raise bb.fetch2.FetchError("Repository %s has LFS content, install git-lfs on host to download (or set lfs=0 to ignore it)" % (repourl))
-            else:
+            elif not need_lfs:
                 bb.note("Repository %s has LFS content but it is not being fetched" % (repourl))
 
         if not ud.nocheckout:
@@ -559,12 +592,28 @@ class Git(FetchMethod):
             raise bb.fetch2.FetchError("The command '%s' gave output with more then 1 line unexpectedly, output: '%s'" % (cmd, output))
         return output.split()[0] != "0"
 
+    def _need_lfs(self, ud):
+        return ud.parm.get("lfs", "1") == "1"
+
     def _contains_lfs(self, ud, d, wd):
         """
         Check if the repository has 'lfs' (large file) content
         """
-        cmd = "%s grep lfs HEAD:.gitattributes | wc -l" % (
-                ud.basecmd)
+
+        if not ud.nobranch:
+            branchname = ud.branches[ud.names[0]]
+        else:
+            branchname = "master"
+
+        # The bare clonedir doesn't use the remote names; it has the branch immediately.
+        if wd == ud.clonedir:
+            refname = ud.branches[ud.names[0]]
+        else:
+            refname = "origin/%s" % ud.branches[ud.names[0]]
+
+        cmd = "%s grep lfs %s:.gitattributes | wc -l" % (
+            ud.basecmd, refname)
+
         try:
             output = runfetchcmd(cmd, d, quiet=True, workdir=wd)
             if int(output) > 0:
@@ -614,7 +663,7 @@ class Git(FetchMethod):
         try:
             repourl = self._get_repo_url(ud)
             cmd = "%s ls-remote %s %s" % \
-                (ud.basecmd, repourl, search)
+                (ud.basecmd, shlex.quote(repourl), search)
             if ud.proto.lower() != 'file':
                 bb.fetch2.check_network_access(d, cmd, repourl)
             output = runfetchcmd(cmd, d, True)
@@ -673,7 +722,7 @@ class Git(FetchMethod):
 
             # search for version in the line
             tag = tagregex.search(tag_head)
-            if tag == None:
+            if tag is None:
                 continue
 
             tag = tag.group('pver')

bitbake/lib/bb/fetch2/gitannex.py

@@ -8,11 +8,9 @@ BitBake 'Fetch' git annex implementation
 # SPDX-License-Identifier: GPL-2.0-only
 #
 
-import os
 import bb
 from   bb.fetch2.git import Git
 from   bb.fetch2 import runfetchcmd
-from   bb.fetch2 import logger
 
 class GitANNEX(Git):
     def supports(self, ud, d):

bitbake/lib/bb/fetch2/gitsm.py

@@ -20,11 +20,12 @@ NOTE: Switching a SRC_URI from "git://" to "gitsm://" requires a clean of your r
 import os
 import bb
 import copy
+import shutil
+import tempfile
 from   bb.fetch2.git import Git
 from   bb.fetch2 import runfetchcmd
 from   bb.fetch2 import logger
 from   bb.fetch2 import Fetch
-from   bb.fetch2 import BBFetchException
 
 class GitSM(Git):
     def supports(self, ud, d):
@@ -131,7 +132,7 @@ class GitSM(Git):
             ld.setVar('SRCPV', d.getVar('SRCPV'))
             ld.setVar('SRCREV_FORMAT', module)
 
-            function(ud, url, module, paths[module], ld)
+            function(ud, url, module, paths[module], workdir, ld)
 
         return submodules != []
 
@@ -142,18 +143,49 @@ class GitSM(Git):
         try:
             # Check for the nugget dropped by the download operation
             known_srcrevs = runfetchcmd("%s config --get-all bitbake.srcrev" % \
-                            (ud.basecmd), d, workdir=ud.clonedir)
+                                        (ud.basecmd), d, workdir=ud.clonedir)
 
-            if ud.revisions[ud.names[0]] not in known_srcrevs.split():
-                return True
+            if ud.revisions[ud.names[0]] in known_srcrevs.split():
+                return False
         except bb.fetch2.FetchError:
-            # No srcrev nuggets, so this is new and needs to be updated
+            pass
+
+        need_update_list = []
+        def need_update_submodule(ud, url, module, modpath, workdir, d):
+            url += ";bareclone=1;nobranch=1"
+
+            try:
+                newfetch = Fetch([url], d, cache=False)
+                new_ud = newfetch.ud[url]
+                if new_ud.method.need_update(new_ud, d):
+                    need_update_list.append(modpath)
+            except Exception as e:
+                logger.error('gitsm: submodule update check failed: %s %s' % (type(e).__name__, str(e)))
+                need_update_result = True
+
+        # If we're using a shallow mirror tarball it needs to be unpacked
+        # temporarily so that we can examine the .gitmodules file
+        if ud.shallow and os.path.exists(ud.fullshallow) and not os.path.exists(ud.clonedir):
+            tmpdir = tempfile.mkdtemp(dir=d.getVar("DL_DIR"))
+            runfetchcmd("tar -xzf %s" % ud.fullshallow, d, workdir=tmpdir)
+            self.process_submodules(ud, tmpdir, need_update_submodule, d)
+            shutil.rmtree(tmpdir)
+        else:
+            self.process_submodules(ud, ud.clonedir, need_update_submodule, d)
+            if len(need_update_list) == 0:
+                # We already have the required commits of all submodules. Drop
+                # a nugget so we don't need to check again.
+                runfetchcmd("%s config --add bitbake.srcrev %s" % \
+                            (ud.basecmd, ud.revisions[ud.names[0]]), d, workdir=ud.clonedir)
+
+        if len(need_update_list) > 0:
+            logger.debug(1, 'gitsm: Submodules requiring update: %s' % (' '.join(need_update_list)))
             return True
 
         return False
 
     def download(self, ud, d):
-        def download_submodule(ud, url, module, modpath, d):
+        def download_submodule(ud, url, module, modpath, workdir, d):
             url += ";bareclone=1;nobranch=1"
 
             # Is the following still needed?
@@ -162,18 +194,27 @@ class GitSM(Git):
             try:
                 newfetch = Fetch([url], d, cache=False)
                 newfetch.download()
-                # Drop a nugget to add each of the srcrevs we've fetched (used by need_update)
-                runfetchcmd("%s config --add bitbake.srcrev %s" % \
-                            (ud.basecmd, ud.revisions[ud.names[0]]), d, workdir=ud.clonedir)
             except Exception as e:
                 logger.error('gitsm: submodule download failed: %s %s' % (type(e).__name__, str(e)))
                 raise
 
         Git.download(self, ud, d)
-        self.process_submodules(ud, ud.clonedir, download_submodule, d)
+
+        # If we're using a shallow mirror tarball it needs to be unpacked
+        # temporarily so that we can examine the .gitmodules file
+        if ud.shallow and os.path.exists(ud.fullshallow) and self.need_update(ud, d):
+            tmpdir = tempfile.mkdtemp(dir=d.getVar("DL_DIR"))
+            runfetchcmd("tar -xzf %s" % ud.fullshallow, d, workdir=tmpdir)
+            self.process_submodules(ud, tmpdir, download_submodule, d)
+            shutil.rmtree(tmpdir)
+        else:
+            self.process_submodules(ud, ud.clonedir, download_submodule, d)
+            # Drop a nugget for the srcrev we've fetched (used by need_update)
+            runfetchcmd("%s config --add bitbake.srcrev %s" % \
+                        (ud.basecmd, ud.revisions[ud.names[0]]), d, workdir=ud.clonedir)
 
     def unpack(self, ud, destdir, d):
-        def unpack_submodules(ud, url, module, modpath, d):
+        def unpack_submodules(ud, url, module, modpath, workdir, d):
             url += ";bareclone=1;nobranch=1"
 
             # Figure out where we clone over the bare submodules...
@@ -213,3 +254,24 @@ class GitSM(Git):
             # up the configuration and checks out the files.  The main project config should remain
             # unmodified, and no download from the internet should occur.
             runfetchcmd("%s submodule update --recursive --no-fetch" % (ud.basecmd), d, quiet=True, workdir=ud.destdir)
+
+    def implicit_urldata(self, ud, d):
+        import shutil, subprocess, tempfile
+
+        urldata = []
+        def add_submodule(ud, url, module, modpath, workdir, d):
+            url += ";bareclone=1;nobranch=1"
+            newfetch = Fetch([url], d, cache=False)
+            urldata.extend(newfetch.expanded_urldata())
+
+        # If we're using a shallow mirror tarball it needs to be unpacked
+        # temporarily so that we can examine the .gitmodules file
+        if ud.shallow and os.path.exists(ud.fullshallow) and ud.method.need_update(ud, d):
+            tmpdir = tempfile.mkdtemp(dir=d.getVar("DL_DIR"))
+            subprocess.check_call("tar -xzf %s" % ud.fullshallow, cwd=tmpdir, shell=True)
+            self.process_submodules(ud, tmpdir, add_submodule, d)
+            shutil.rmtree(tmpdir)
+        else:
+            self.process_submodules(ud, ud.clonedir, add_submodule, d)
+
+        return urldata

bitbake/lib/bb/fetch2/hg.py

@@ -13,8 +13,6 @@ BitBake 'Fetch' implementation for mercurial DRCS (hg).
 #
 
 import os
-import sys
-import logging
 import bb
 import errno
 from bb.fetch2 import FetchMethod

bitbake/lib/bb/fetch2/local.py

@@ -17,7 +17,7 @@ import os
 import urllib.request, urllib.parse, urllib.error
 import bb
 import bb.utils
-from   bb.fetch2 import FetchMethod, FetchError
+from   bb.fetch2 import FetchMethod, FetchError, ParameterError
 from   bb.fetch2 import logger
 
 class Local(FetchMethod):
@@ -33,6 +33,8 @@ class Local(FetchMethod):
         ud.basename = os.path.basename(ud.decodedurl)
         ud.basepath = ud.decodedurl
         ud.needdonestamp = False
+        if "*" in ud.decodedurl:
+            raise bb.fetch2.ParameterError("file:// urls using globbing are no longer supported. Please place the files in a directory and reference that instead.", ud.url)
         return
 
     def localpath(self, urldata, d):
@@ -55,12 +57,6 @@ class Local(FetchMethod):
             logger.debug(2, "Searching for %s in paths:\n    %s" % (path, "\n    ".join(filespath.split(":"))))
             newpath, hist = bb.utils.which(filespath, path, history=True)
             searched.extend(hist)
-        if (not newpath or not os.path.exists(newpath)) and path.find("*") != -1:
-            # For expressions using '*', best we can do is take the first directory in FILESPATH that exists
-            newpath, hist = bb.utils.which(filespath, ".", history=True)
-            searched.extend(hist)
-            logger.debug(2, "Searching for %s in path: %s" % (path, newpath))
-            return searched
         if not os.path.exists(newpath):
             dldirfile = os.path.join(d.getVar("DL_DIR"), path)
             logger.debug(2, "Defaulting to %s for %s" % (dldirfile, path))
@@ -70,8 +66,6 @@ class Local(FetchMethod):
         return searched
 
     def need_update(self, ud, d):
-        if ud.url.find("*") != -1:
-            return False
         if os.path.exists(ud.localpath):
             return False
         return True
@@ -95,9 +89,6 @@ class Local(FetchMethod):
         """
         Check the status of the url
         """
-        if urldata.localpath.find("*") != -1:
-            logger.info("URL %s looks like a glob and was therefore not checked.", urldata.url)
-            return True
         if os.path.exists(urldata.localpath):
             return True
         return False

bitbake/lib/bb/fetch2/npm.py

@@ -1,301 +1,296 @@
+# Copyright (C) 2020 Savoir-Faire Linux
 #
 # SPDX-License-Identifier: GPL-2.0-only
 #
 """
-BitBake 'Fetch' NPM implementation
+BitBake 'Fetch' npm implementation
 
-The NPM fetcher is used to retrieve files from the npmjs repository
+npm fetcher support the SRC_URI with format of:
+SRC_URI = "npm://some.registry.url;OptionA=xxx;OptionB=xxx;..."
 
-Usage in the recipe:
+Supported SRC_URI options are:
 
-    SRC_URI = "npm://registry.npmjs.org/;name=${PN};version=${PV}"
-    Suported SRC_URI options are:
+- package
+   The npm package name. This is a mandatory parameter.
 
-    - name
-    - version
+- version
+    The npm package version. This is a mandatory parameter.
 
-    npm://registry.npmjs.org/${PN}/-/${PN}-${PV}.tgz  would become npm://registry.npmjs.org;name=${PN};version=${PV}
-    The fetcher all triggers off the existence of ud.localpath. If that exists and has the ".done" stamp, its assumed the fetch is good/done
+- downloadfilename
+    Specifies the filename used when storing the downloaded file.
 
+- destsuffix
+    Specifies the directory to use to unpack the package (default: npm).
 """
 
-import os
-import sys
-import urllib.request, urllib.parse, urllib.error
+import base64
 import json
-import subprocess
-import signal
+import os
+import re
+import shlex
+import tempfile
 import bb
-from   bb.fetch2 import FetchMethod
-from   bb.fetch2 import FetchError
-from   bb.fetch2 import ChecksumError
-from   bb.fetch2 import runfetchcmd
-from   bb.fetch2 import logger
-from   bb.fetch2 import UnpackError
-from   bb.fetch2 import ParameterError
+from bb.fetch2 import Fetch
+from bb.fetch2 import FetchError
+from bb.fetch2 import FetchMethod
+from bb.fetch2 import MissingParameterError
+from bb.fetch2 import ParameterError
+from bb.fetch2 import URI
+from bb.fetch2 import check_network_access
+from bb.fetch2 import runfetchcmd
+from bb.utils import is_semver
 
-def subprocess_setup():
-    # Python installs a SIGPIPE handler by default. This is usually not what
-    # non-Python subprocesses expect.
-    # SIGPIPE errors are known issues with gzip/bash
-    signal.signal(signal.SIGPIPE, signal.SIG_DFL)
+def npm_package(package):
+    """Convert the npm package name to remove unsupported character"""
+    # Scoped package names (with the @) use the same naming convention
+    # as the 'npm pack' command.
+    if package.startswith("@"):
+        return re.sub("/", "-", package[1:])
+    return package
+
+def npm_filename(package, version):
+    """Get the filename of a npm package"""
+    return npm_package(package) + "-" + version + ".tgz"
+
+def npm_localfile(package, version):
+    """Get the local filename of a npm package"""
+    return os.path.join("npm2", npm_filename(package, version))
+
+def npm_integrity(integrity):
+    """
+    Get the checksum name and expected value from the subresource integrity
+        https://www.w3.org/TR/SRI/
+    """
+    algo, value = integrity.split("-", maxsplit=1)
+    return "%ssum" % algo, base64.b64decode(value).hex()
+
+def npm_unpack(tarball, destdir, d):
+    """Unpack a npm tarball"""
+    bb.utils.mkdirhier(destdir)
+    cmd = "tar --extract --gzip --file=%s" % shlex.quote(tarball)
+    cmd += " --no-same-owner"
+    cmd += " --strip-components=1"
+    runfetchcmd(cmd, d, workdir=destdir)
+
+class NpmEnvironment(object):
+    """
+    Using a npm config file seems more reliable than using cli arguments.
+    This class allows to create a controlled environment for npm commands.
+    """
+    def __init__(self, d, configs=None):
+        self.d = d
+        self.configs = configs
+
+    def run(self, cmd, args=None, configs=None, workdir=None):
+        """Run npm command in a controlled environment"""
+        with tempfile.TemporaryDirectory() as tmpdir:
+            d = bb.data.createCopy(self.d)
+            d.setVar("HOME", tmpdir)
+
+            cfgfile = os.path.join(tmpdir, "npmrc")
+
+            if not workdir:
+                workdir = tmpdir
+
+            def _run(cmd):
+                cmd = "NPM_CONFIG_USERCONFIG=%s " % cfgfile + cmd
+                cmd = "NPM_CONFIG_GLOBALCONFIG=%s " % cfgfile + cmd
+                return runfetchcmd(cmd, d, workdir=workdir)
+
+            if self.configs:
+                for key, value in self.configs:
+                    _run("npm config set %s %s" % (key, shlex.quote(value)))
+
+            if configs:
+                for key, value in configs:
+                    _run("npm config set %s %s" % (key, shlex.quote(value)))
+
+            if args:
+                for key, value in args:
+                    cmd += " --%s=%s" % (key, shlex.quote(value))
+
+            return _run(cmd)
 
 class Npm(FetchMethod):
-
-    """Class to fetch urls via 'npm'"""
-    def init(self, d):
-        pass
+    """Class to fetch a package from a npm registry"""
 
     def supports(self, ud, d):
-        """
-        Check to see if a given url can be fetched with npm
-        """
-        return ud.type in ['npm']
-
-    def debug(self, msg):
-        logger.debug(1, "NpmFetch: %s", msg)
-
-    def clean(self, ud, d):
-        logger.debug(2, "Calling cleanup %s" % ud.pkgname)
-        bb.utils.remove(ud.localpath, False)
-        bb.utils.remove(ud.pkgdatadir, True)
-        bb.utils.remove(ud.fullmirror, False)
+        """Check if a given url can be fetched with npm"""
+        return ud.type in ["npm"]
 
     def urldata_init(self, ud, d):
-        """
-        init NPM specific variable within url data
-        """
-        if 'downloadfilename' in ud.parm:
-            ud.basename = ud.parm['downloadfilename']
-        else:
-            ud.basename = os.path.basename(ud.path)
+        """Init npm specific variables within url data"""
+        ud.package = None
+        ud.version = None
+        ud.registry = None
+
+        # Get the 'package' parameter
+        if "package" in ud.parm:
+            ud.package = ud.parm.get("package")
+
+        if not ud.package:
+            raise MissingParameterError("Parameter 'package' required", ud.url)
+
+        # Get the 'version' parameter
+        if "version" in ud.parm:
+            ud.version = ud.parm.get("version")
 
-        # can't call it ud.name otherwise fetcher base class will start doing sha1stuff
-        # TODO: find a way to get an sha1/sha256 manifest of pkg & all deps
-        ud.pkgname = ud.parm.get("name", None)
-        if not ud.pkgname:
-            raise ParameterError("NPM fetcher requires a name parameter", ud.url)
-        ud.version = ud.parm.get("version", None)
         if not ud.version:
-            raise ParameterError("NPM fetcher requires a version parameter", ud.url)
-        ud.bbnpmmanifest = "%s-%s.deps.json" % (ud.pkgname, ud.version)
-        ud.bbnpmmanifest = ud.bbnpmmanifest.replace('/', '-')
-        ud.registry = "http://%s" % (ud.url.replace('npm://', '', 1).split(';'))[0]
-        prefixdir = "npm/%s" % ud.pkgname
-        ud.pkgdatadir = d.expand("${DL_DIR}/%s" % prefixdir)
-        if not os.path.exists(ud.pkgdatadir):
-            bb.utils.mkdirhier(ud.pkgdatadir)
-        ud.localpath = d.expand("${DL_DIR}/npm/%s" % ud.bbnpmmanifest)
+            raise MissingParameterError("Parameter 'version' required", ud.url)
 
-        self.basecmd = d.getVar("FETCHCMD_wget") or "/usr/bin/env wget -O -t 2 -T 30 -nv --passive-ftp --no-check-certificate "
-        ud.prefixdir = prefixdir
+        if not is_semver(ud.version) and not ud.version == "latest":
+            raise ParameterError("Invalid 'version' parameter", ud.url)
 
-        ud.write_tarballs = ((d.getVar("BB_GENERATE_MIRROR_TARBALLS") or "0") != "0")
-        mirrortarball = 'npm_%s-%s.tar.xz' % (ud.pkgname, ud.version)
-        mirrortarball = mirrortarball.replace('/', '-')
-        ud.fullmirror = os.path.join(d.getVar("DL_DIR"), mirrortarball)
-        ud.mirrortarballs = [mirrortarball]
+        # Extract the 'registry' part of the url
+        ud.registry = re.sub(r"^npm://", "http://", ud.url.split(";")[0])
+
+        # Using the 'downloadfilename' parameter as local filename
+        # or the npm package name.
+        if "downloadfilename" in ud.parm:
+            ud.localfile = d.expand(ud.parm["downloadfilename"])
+        else:
+            ud.localfile = npm_localfile(ud.package, ud.version)
+
+        # Get the base 'npm' command
+        ud.basecmd = d.getVar("FETCHCMD_npm") or "npm"
+
+        # This fetcher resolves a URI from a npm package name and version and
+        # then forwards it to a proxy fetcher. A resolve file containing the
+        # resolved URI is created to avoid unwanted network access (if the file
+        # already exists). The management of the donestamp file, the lockfile
+        # and the checksums are forwarded to the proxy fetcher.
+        ud.proxy = None
+        ud.needdonestamp = False
+        ud.resolvefile = self.localpath(ud, d) + ".resolved"
+
+    def _resolve_proxy_url(self, ud, d):
+        def _npm_view():
+            configs = []
+            configs.append(("json", "true"))
+            configs.append(("registry", ud.registry))
+            pkgver = shlex.quote(ud.package + "@" + ud.version)
+            cmd = ud.basecmd + " view %s" % pkgver
+            env = NpmEnvironment(d)
+            check_network_access(d, cmd, ud.registry)
+            view_string = env.run(cmd, configs=configs)
+
+            if not view_string:
+                raise FetchError("Unavailable package %s" % pkgver, ud.url)
+
+            try:
+                view = json.loads(view_string)
+
+                error = view.get("error")
+                if error is not None:
+                    raise FetchError(error.get("summary"), ud.url)
+
+                if ud.version == "latest":
+                    bb.warn("The npm package %s is using the latest " \
+                            "version available. This could lead to " \
+                            "non-reproducible builds." % pkgver)
+                elif ud.version != view.get("version"):
+                    raise ParameterError("Invalid 'version' parameter", ud.url)
+
+                return view
+
+            except Exception as e:
+                raise FetchError("Invalid view from npm: %s" % str(e), ud.url)
+
+        def _get_url(view):
+            tarball_url = view.get("dist", {}).get("tarball")
+
+            if tarball_url is None:
+                raise FetchError("Invalid 'dist.tarball' in view", ud.url)
+
+            uri = URI(tarball_url)
+            uri.params["downloadfilename"] = ud.localfile
+
+            integrity = view.get("dist", {}).get("integrity")
+            shasum = view.get("dist", {}).get("shasum")
+
+            if integrity is not None:
+                checksum_name, checksum_expected = npm_integrity(integrity)
+                uri.params[checksum_name] = checksum_expected
+            elif shasum is not None:
+                uri.params["sha1sum"] = shasum
+            else:
+                raise FetchError("Invalid 'dist.integrity' in view", ud.url)
+
+            return str(uri)
+
+        url = _get_url(_npm_view())
+
+        bb.utils.mkdirhier(os.path.dirname(ud.resolvefile))
+        with open(ud.resolvefile, "w") as f:
+            f.write(url)
+
+    def _setup_proxy(self, ud, d):
+        if ud.proxy is None:
+            if not os.path.exists(ud.resolvefile):
+                self._resolve_proxy_url(ud, d)
+
+            with open(ud.resolvefile, "r") as f:
+                url = f.read()
+
+            # Avoid conflicts between the environment data and:
+            # - the proxy url checksum
+            data = bb.data.createCopy(d)
+            data.delVarFlags("SRC_URI")
+            ud.proxy = Fetch([url], data)
+
+    def _get_proxy_method(self, ud, d):
+        self._setup_proxy(ud, d)
+        proxy_url = ud.proxy.urls[0]
+        proxy_ud = ud.proxy.ud[proxy_url]
+        proxy_d = ud.proxy.d
+        proxy_ud.setup_localpath(proxy_d)
+        return proxy_ud.method, proxy_ud, proxy_d
+
+    def verify_donestamp(self, ud, d):
+        """Verify the donestamp file"""
+        proxy_m, proxy_ud, proxy_d = self._get_proxy_method(ud, d)
+        return proxy_m.verify_donestamp(proxy_ud, proxy_d)
+
+    def update_donestamp(self, ud, d):
+        """Update the donestamp file"""
+        proxy_m, proxy_ud, proxy_d = self._get_proxy_method(ud, d)
+        proxy_m.update_donestamp(proxy_ud, proxy_d)
 
     def need_update(self, ud, d):
-        if os.path.exists(ud.localpath):
-            return False
-        return True
+        """Force a fetch, even if localpath exists ?"""
+        if not os.path.exists(ud.resolvefile):
+            return True
+        if ud.version == "latest":
+            return True
+        proxy_m, proxy_ud, proxy_d = self._get_proxy_method(ud, d)
+        return proxy_m.need_update(proxy_ud, proxy_d)
 
-    def _runpack(self, ud, d, pkgfullname: str, quiet=False) -> str:
-        """
-        Runs npm pack on a full package name.
-        Returns the filename of the downloaded package
-        """
-        bb.fetch2.check_network_access(d, pkgfullname, ud.registry)
-        dldir = d.getVar("DL_DIR")
-        dldir = os.path.join(dldir, ud.prefixdir)
-
-        command = "npm pack {} --registry {}".format(pkgfullname, ud.registry)
-        logger.debug(2, "Fetching {} using command '{}' in {}".format(pkgfullname, command, dldir))
-        filename = runfetchcmd(command, d, quiet, workdir=dldir)
-        return filename.rstrip()
-
-    def _unpackdep(self, ud, pkg, data, destdir, dldir, d):
-        file = data[pkg]['tgz']
-        logger.debug(2, "file to extract is %s" % file)
-        if file.endswith('.tgz') or file.endswith('.tar.gz') or file.endswith('.tar.Z'):
-            cmd = 'tar xz --strip 1 --no-same-owner --warning=no-unknown-keyword -f %s/%s' % (dldir, file)
-        else:
-            bb.fatal("NPM package %s downloaded not a tarball!" % file)
-
-        # Change to subdir before executing command
-        if not os.path.exists(destdir):
-            os.makedirs(destdir)
-        path = d.getVar('PATH')
-        if path:
-            cmd = "PATH=\"%s\" %s" % (path, cmd)
-        bb.note("Unpacking %s to %s/" % (file, destdir))
-        ret = subprocess.call(cmd, preexec_fn=subprocess_setup, shell=True, cwd=destdir)
-
-        if ret != 0:
-            raise UnpackError("Unpack command %s failed with return value %s" % (cmd, ret), ud.url)
-
-        if 'deps' not in data[pkg]:
-            return
-        for dep in data[pkg]['deps']:
-            self._unpackdep(ud, dep, data[pkg]['deps'], "%s/node_modules/%s" % (destdir, dep), dldir, d)
-
-
-    def unpack(self, ud, destdir, d):
-        dldir = d.getVar("DL_DIR")
-        with open("%s/npm/%s" % (dldir, ud.bbnpmmanifest)) as datafile:
-            workobj = json.load(datafile)
-        dldir = "%s/%s" % (os.path.dirname(ud.localpath), ud.pkgname)
-
-        if 'subdir' in ud.parm:
-            unpackdir = '%s/%s' % (destdir, ud.parm.get('subdir'))
-        else:
-            unpackdir = '%s/npmpkg' % destdir
-
-        self._unpackdep(ud, ud.pkgname, workobj, unpackdir, dldir, d)
-
-    def _parse_view(self, output):
-        '''
-        Parse the output of npm view --json; the last JSON result
-        is assumed to be the one that we're interested in.
-        '''
-        pdata = json.loads(output);
-        try:
-            return pdata[-1]
-        except:
-            return pdata
-
-    def _getdependencies(self, pkg, data, version, d, ud, optional=False, fetchedlist=None):
-        if fetchedlist is None:
-            fetchedlist = []
-        pkgfullname = pkg
-        if version != '*' and not '/' in version:
-            pkgfullname += "@'%s'" % version
-        if pkgfullname in fetchedlist:
-            return
-
-        logger.debug(2, "Calling getdeps on %s" % pkg)
-        fetchcmd = "npm view %s --json --registry %s" % (pkgfullname, ud.registry)
-        output = runfetchcmd(fetchcmd, d, True)
-        pdata = self._parse_view(output)
-        if not pdata:
-            raise FetchError("The command '%s' returned no output" % fetchcmd)
-        if optional:
-            pkg_os = pdata.get('os', None)
-            if pkg_os:
-                if not isinstance(pkg_os, list):
-                    pkg_os = [pkg_os]
-                blacklist = False
-                for item in pkg_os:
-                    if item.startswith('!'):
-                        blacklist = True
-                        break
-                if (not blacklist and 'linux' not in pkg_os) or '!linux' in pkg_os:
-                    logger.debug(2, "Skipping %s since it's incompatible with Linux" % pkg)
-                    return
-        filename = self._runpack(ud, d, pkgfullname)
-        data[pkg] = {}
-        data[pkg]['tgz'] = filename
-        fetchedlist.append(pkgfullname)
-
-        dependencies = pdata.get('dependencies', {})
-        optionalDependencies = pdata.get('optionalDependencies', {})
-        dependencies.update(optionalDependencies)
-        depsfound = {}
-        optdepsfound = {}
-        data[pkg]['deps'] = {}
-        for dep in dependencies:
-            if dep in optionalDependencies:
-                optdepsfound[dep] = dependencies[dep]
-            else:
-                depsfound[dep] = dependencies[dep]
-        for dep, version in optdepsfound.items():
-            self._getdependencies(dep, data[pkg]['deps'], version, d, ud, optional=True, fetchedlist=fetchedlist)
-        for dep, version in depsfound.items():
-            self._getdependencies(dep, data[pkg]['deps'], version, d, ud, fetchedlist=fetchedlist)
-
-    def _getshrinkeddependencies(self, pkg, data, version, d, ud, lockdown, manifest, toplevel=True):
-        logger.debug(2, "NPM shrinkwrap file is %s" % data)
-        if toplevel:
-            name = data.get('name', None)
-            if name and name != pkg:
-                for obj in data.get('dependencies', []):
-                    if obj == pkg:
-                        self._getshrinkeddependencies(obj, data['dependencies'][obj], data['dependencies'][obj]['version'], d, ud, lockdown, manifest, False)
-                        return
-
-        pkgnameWithVersion = "{}@{}".format(pkg, version)
-        logger.debug(2, "Get dependencies for {}".format(pkgnameWithVersion))
-        filename = self._runpack(ud, d, pkgnameWithVersion)
-        manifest[pkg] = {}
-        manifest[pkg]['tgz'] = filename
-        manifest[pkg]['deps'] = {}
-
-        if pkg in lockdown:
-            sha1_expected = lockdown[pkg][version]
-            sha1_data = bb.utils.sha1_file("npm/%s/%s" % (ud.pkgname, manifest[pkg]['tgz']))
-            if sha1_expected != sha1_data:
-                msg = "\nFile: '%s' has %s checksum %s when %s was expected" % (manifest[pkg]['tgz'], 'sha1', sha1_data, sha1_expected)
-                raise ChecksumError('Checksum mismatch!%s' % msg)
-        else:
-            logger.debug(2, "No lockdown data for %s@%s" % (pkg, version))
-
-        if 'dependencies' in data:
-            for obj in data['dependencies']:
-                logger.debug(2, "Found dep is %s" % str(obj))
-                self._getshrinkeddependencies(obj, data['dependencies'][obj], data['dependencies'][obj]['version'], d, ud, lockdown, manifest[pkg]['deps'], False)
+    def try_mirrors(self, fetch, ud, d, mirrors):
+        """Try to use a mirror"""
+        proxy_m, proxy_ud, proxy_d = self._get_proxy_method(ud, d)
+        return proxy_m.try_mirrors(fetch, proxy_ud, proxy_d, mirrors)
 
     def download(self, ud, d):
         """Fetch url"""
-        jsondepobj = {}
-        shrinkobj = {}
-        lockdown = {}
+        self._setup_proxy(ud, d)
+        ud.proxy.download()
 
-        if not os.listdir(ud.pkgdatadir) and os.path.exists(ud.fullmirror):
-            dest = d.getVar("DL_DIR")
-            bb.utils.mkdirhier(dest)
-            runfetchcmd("tar -xJf %s" % (ud.fullmirror), d, workdir=dest)
-            return
+    def unpack(self, ud, rootdir, d):
+        """Unpack the downloaded archive"""
+        destsuffix = ud.parm.get("destsuffix", "npm")
+        destdir = os.path.join(rootdir, destsuffix)
+        npm_unpack(ud.localpath, destdir, d)
 
-        if ud.parm.get("noverify", None) != '1':
-            shwrf = d.getVar('NPM_SHRINKWRAP')
-            logger.debug(2, "NPM shrinkwrap file is %s" % shwrf)
-            if shwrf:
-                try:
-                    with open(shwrf) as datafile:
-                        shrinkobj = json.load(datafile)
-                except Exception as e:
-                    raise FetchError('Error loading NPM_SHRINKWRAP file "%s" for %s: %s' % (shwrf, ud.pkgname, str(e)))
-            elif not ud.ignore_checksums:
-                logger.warning('Missing shrinkwrap file in NPM_SHRINKWRAP for %s, this will lead to unreliable builds!' % ud.pkgname)
-            lckdf = d.getVar('NPM_LOCKDOWN')
-            logger.debug(2, "NPM lockdown file is %s" % lckdf)
-            if lckdf:
-                try:
-                    with open(lckdf) as datafile:
-                        lockdown = json.load(datafile)
-                except Exception as e:
-                    raise FetchError('Error loading NPM_LOCKDOWN file "%s" for %s: %s' % (lckdf, ud.pkgname, str(e)))
-            elif not ud.ignore_checksums:
-                logger.warning('Missing lockdown file in NPM_LOCKDOWN for %s, this will lead to unreproducible builds!' % ud.pkgname)
+    def clean(self, ud, d):
+        """Clean any existing full or partial download"""
+        if os.path.exists(ud.resolvefile):
+            self._setup_proxy(ud, d)
+            ud.proxy.clean()
+            bb.utils.remove(ud.resolvefile)
 
-        if ('name' not in shrinkobj):
-            self._getdependencies(ud.pkgname, jsondepobj, ud.version, d, ud)
-        else:
-            self._getshrinkeddependencies(ud.pkgname, shrinkobj, ud.version, d, ud, lockdown, jsondepobj)
-
-        with open(ud.localpath, 'w') as outfile:
-            json.dump(jsondepobj, outfile)
-
-    def build_mirror_data(self, ud, d):
-        # Generate a mirror tarball if needed
-        if ud.write_tarballs and not os.path.exists(ud.fullmirror):
-            # it's possible that this symlink points to read-only filesystem with PREMIRROR
-            if os.path.islink(ud.fullmirror):
-                os.unlink(ud.fullmirror)
-
-            dldir = d.getVar("DL_DIR")
-            logger.info("Creating tarball of npm data")
-            runfetchcmd("tar -cJf %s npm/%s npm/%s" % (ud.fullmirror, ud.bbnpmmanifest, ud.pkgname), d,
-                        workdir=dldir)
-            runfetchcmd("touch %s.done" % (ud.fullmirror), d, workdir=dldir)
+    def done(self, ud, d):
+        """Is the download done ?"""
+        if not os.path.exists(ud.resolvefile):
+            return False
+        proxy_m, proxy_ud, proxy_d = self._get_proxy_method(ud, d)
+        return proxy_m.done(proxy_ud, proxy_d)

bitbake/lib/bb/fetch2/npmsw.py

@@ -0,0 +1,255 @@
+# Copyright (C) 2020 Savoir-Faire Linux
+#
+# SPDX-License-Identifier: GPL-2.0-only
+#
+"""
+BitBake 'Fetch' npm shrinkwrap implementation
+
+npm fetcher support the SRC_URI with format of:
+SRC_URI = "npmsw://some.registry.url;OptionA=xxx;OptionB=xxx;..."
+
+Supported SRC_URI options are:
+
+- dev
+   Set to 1 to also install devDependencies.
+
+- destsuffix
+    Specifies the directory to use to unpack the dependencies (default: ${S}).
+"""
+
+import json
+import os
+import re
+import bb
+from bb.fetch2 import Fetch
+from bb.fetch2 import FetchMethod
+from bb.fetch2 import ParameterError
+from bb.fetch2 import URI
+from bb.fetch2.npm import npm_integrity
+from bb.fetch2.npm import npm_localfile
+from bb.fetch2.npm import npm_unpack
+from bb.utils import is_semver
+
+def foreach_dependencies(shrinkwrap, callback=None, dev=False):
+    """
+        Run a callback for each dependencies of a shrinkwrap file.
+        The callback is using the format:
+            callback(name, params, deptree)
+        with:
+            name = the package name (string)
+            params = the package parameters (dictionary)
+            deptree = the package dependency tree (array of strings)
+    """
+    def _walk_deps(deps, deptree):
+        for name in deps:
+            subtree = [*deptree, name]
+            _walk_deps(deps[name].get("dependencies", {}), subtree)
+            if callback is not None:
+                if deps[name].get("dev", False) and not dev:
+                    continue
+                elif deps[name].get("bundled", False):
+                    continue
+                callback(name, deps[name], subtree)
+
+    _walk_deps(shrinkwrap.get("dependencies", {}), [])
+
+class NpmShrinkWrap(FetchMethod):
+    """Class to fetch all package from a shrinkwrap file"""
+
+    def supports(self, ud, d):
+        """Check if a given url can be fetched with npmsw"""
+        return ud.type in ["npmsw"]
+
+    def urldata_init(self, ud, d):
+        """Init npmsw specific variables within url data"""
+
+        # Get the 'shrinkwrap' parameter
+        ud.shrinkwrap_file = re.sub(r"^npmsw://", "", ud.url.split(";")[0])
+
+        # Get the 'dev' parameter
+        ud.dev = bb.utils.to_boolean(ud.parm.get("dev"), False)
+
+        # Resolve the dependencies
+        ud.deps = []
+
+        def _resolve_dependency(name, params, deptree):
+            url = None
+            localpath = None
+            extrapaths = []
+            destsubdirs = [os.path.join("node_modules", dep) for dep in deptree]
+            destsuffix = os.path.join(*destsubdirs)
+
+            integrity = params.get("integrity", None)
+            resolved = params.get("resolved", None)
+            version = params.get("version", None)
+
+            # Handle registry sources
+            if is_semver(version) and resolved and integrity:
+                localfile = npm_localfile(name, version)
+
+                uri = URI(resolved)
+                uri.params["downloadfilename"] = localfile
+
+                checksum_name, checksum_expected = npm_integrity(integrity)
+                uri.params[checksum_name] = checksum_expected
+
+                url = str(uri)
+
+                localpath = os.path.join(d.getVar("DL_DIR"), localfile)
+
+                # Create a resolve file to mimic the npm fetcher and allow
+                # re-usability of the downloaded file.
+                resolvefile = localpath + ".resolved"
+
+                bb.utils.mkdirhier(os.path.dirname(resolvefile))
+                with open(resolvefile, "w") as f:
+                    f.write(url)
+
+                extrapaths.append(resolvefile)
+
+            # Handle http tarball sources
+            elif version.startswith("http") and integrity:
+                localfile = os.path.join("npm2", os.path.basename(version))
+
+                uri = URI(version)
+                uri.params["downloadfilename"] = localfile
+
+                checksum_name, checksum_expected = npm_integrity(integrity)
+                uri.params[checksum_name] = checksum_expected
+
+                url = str(uri)
+
+                localpath = os.path.join(d.getVar("DL_DIR"), localfile)
+
+            # Handle git sources
+            elif version.startswith("git"):
+                regex = re.compile(r"""
+                    ^
+                    git\+
+                    (?P<protocol>[a-z]+)
+                    ://
+                    (?P<url>[^#]+)
+                    \#
+                    (?P<rev>[0-9a-f]+)
+                    $
+                    """, re.VERBOSE)
+
+                match = regex.match(version)
+
+                if not match:
+                    raise ParameterError("Invalid git url: %s" % version, ud.url)
+
+                groups = match.groupdict()
+
+                uri = URI("git://" + str(groups["url"]))
+                uri.params["protocol"] = str(groups["protocol"])
+                uri.params["rev"] = str(groups["rev"])
+                uri.params["destsuffix"] = destsuffix
+
+                url = str(uri)
+
+            # local tarball sources and local link sources are unsupported
+            else:
+                raise ParameterError("Unsupported dependency: %s" % name, ud.url)
+
+            ud.deps.append({
+                "url": url,
+                "localpath": localpath,
+                "extrapaths": extrapaths,
+                "destsuffix": destsuffix,
+            })
+
+        try:
+            with open(ud.shrinkwrap_file, "r") as f:
+                shrinkwrap = json.load(f)
+        except Exception as e:
+            raise ParameterError("Invalid shrinkwrap file: %s" % str(e), ud.url)
+
+        foreach_dependencies(shrinkwrap, _resolve_dependency, ud.dev)
+
+        # Avoid conflicts between the environment data and:
+        # - the proxy url revision
+        # - the proxy url checksum
+        data = bb.data.createCopy(d)
+        data.delVar("SRCREV")
+        data.delVarFlags("SRC_URI")
+
+        # This fetcher resolves multiple URIs from a shrinkwrap file and then
+        # forwards it to a proxy fetcher. The management of the donestamp file,
+        # the lockfile and the checksums are forwarded to the proxy fetcher.
+        ud.proxy = Fetch([dep["url"] for dep in ud.deps], data)
+        ud.needdonestamp = False
+
+    @staticmethod
+    def _foreach_proxy_method(ud, handle):
+        returns = []
+        for proxy_url in ud.proxy.urls:
+            proxy_ud = ud.proxy.ud[proxy_url]
+            proxy_d = ud.proxy.d
+            proxy_ud.setup_localpath(proxy_d)
+            returns.append(handle(proxy_ud.method, proxy_ud, proxy_d))
+        return returns
+
+    def verify_donestamp(self, ud, d):
+        """Verify the donestamp file"""
+        def _handle(m, ud, d):
+            return m.verify_donestamp(ud, d)
+        return all(self._foreach_proxy_method(ud, _handle))
+
+    def update_donestamp(self, ud, d):
+        """Update the donestamp file"""
+        def _handle(m, ud, d):
+            m.update_donestamp(ud, d)
+        self._foreach_proxy_method(ud, _handle)
+
+    def need_update(self, ud, d):
+        """Force a fetch, even if localpath exists ?"""
+        def _handle(m, ud, d):
+            return m.need_update(ud, d)
+        return all(self._foreach_proxy_method(ud, _handle))
+
+    def try_mirrors(self, fetch, ud, d, mirrors):
+        """Try to use a mirror"""
+        def _handle(m, ud, d):
+            return m.try_mirrors(fetch, ud, d, mirrors)
+        return all(self._foreach_proxy_method(ud, _handle))
+
+    def download(self, ud, d):
+        """Fetch url"""
+        ud.proxy.download()
+
+    def unpack(self, ud, rootdir, d):
+        """Unpack the downloaded dependencies"""
+        destdir = d.getVar("S")
+        destsuffix = ud.parm.get("destsuffix")
+        if destsuffix:
+            destdir = os.path.join(rootdir, destsuffix)
+
+        bb.utils.mkdirhier(destdir)
+        bb.utils.copyfile(ud.shrinkwrap_file,
+                          os.path.join(destdir, "npm-shrinkwrap.json"))
+
+        auto = [dep["url"] for dep in ud.deps if not dep["localpath"]]
+        manual = [dep for dep in ud.deps if dep["localpath"]]
+
+        if auto:
+            ud.proxy.unpack(destdir, auto)
+
+        for dep in manual:
+            depdestdir = os.path.join(destdir, dep["destsuffix"])
+            npm_unpack(dep["localpath"], depdestdir, d)
+
+    def clean(self, ud, d):
+        """Clean any existing full or partial download"""
+        ud.proxy.clean()
+
+        # Clean extra files
+        for dep in ud.deps:
+            for path in dep["extrapaths"]:
+                bb.utils.remove(path)
+
+    def done(self, ud, d):
+        """Is the download done ?"""
+        def _handle(m, ud, d):
+            return m.done(ud, d)
+        return all(self._foreach_proxy_method(ud, _handle))

bitbake/lib/bb/fetch2/osc.py

@@ -7,15 +7,16 @@ Based on the svn "Fetch" implementation.
 
 """
 
-import  os
-import  sys
 import logging
+import os
 import  bb
 from    bb.fetch2 import FetchMethod
 from    bb.fetch2 import FetchError
 from    bb.fetch2 import MissingParameterError
 from    bb.fetch2 import runfetchcmd
 
+logger = logging.getLogger(__name__)
+
 class Osc(FetchMethod):
     """Class to fetch a module or modules from Opensuse build server
        repositories."""
@@ -43,7 +44,7 @@ class Osc(FetchMethod):
         else:
             pv = d.getVar("PV", False)
             rev = bb.fetch2.srcrev_internal_helper(ud, d)
-            if rev and rev != True:
+            if rev:
                 ud.revision = rev
             else:
                 ud.revision = ""

bitbake/lib/bb/fetch2/perforce.py

@@ -1,6 +1,20 @@
 """
 BitBake 'Fetch' implementation for perforce
 
+Supported SRC_URI options are:
+
+- module
+   The top-level location to fetch while preserving the remote paths
+
+   The value of module can point to either a directory or a file. The result,
+   in both cases, is that the fetcher will preserve all file paths starting
+   from the module path. That is, the top-level directory in the module value
+   will also be the top-level directory in P4DIR.
+
+- remotepath
+   If the value "keep" is given, the full depot location of each file is
+   preserved in P4DIR. This option overrides the effect of the module option.
+
 """
 
 # Copyright (C) 2003, 2004  Chris Larson
@@ -11,13 +25,42 @@ BitBake 'Fetch' implementation for perforce
 # Based on functions from the base bb module, Copyright 2003 Holger Schurig
 
 import os
-import logging
 import bb
 from   bb.fetch2 import FetchMethod
 from   bb.fetch2 import FetchError
 from   bb.fetch2 import logger
 from   bb.fetch2 import runfetchcmd
 
+class PerforceProgressHandler (bb.progress.BasicProgressHandler):
+    """
+    Implements basic progress information for perforce, based on the number of
+    files to be downloaded.
+
+    The p4 print command will print one line per file, therefore it can be used
+    to "count" the number of files already completed and give an indication of
+    the progress.
+    """
+    def __init__(self, d, num_files):
+        self._num_files = num_files
+        self._count = 0
+        super(PerforceProgressHandler, self).__init__(d)
+
+        # Send an initial progress event so the bar gets shown
+        self._fire_progress(-1)
+
+    def write(self, string):
+        self._count = self._count + 1
+
+        percent = int(100.0 * float(self._count) / float(self._num_files))
+
+        # In case something goes wrong, we try to preserve our sanity
+        if percent > 100:
+            percent = 100
+
+        self.update(percent)
+
+        super(PerforceProgressHandler, self).write(string)
+
 class Perforce(FetchMethod):
     """ Class to fetch from perforce repositories """
     def supports(self, ud, d):
@@ -59,14 +102,33 @@ class Perforce(FetchMethod):
             logger.debug(1, 'Determined P4PORT to be: %s' % ud.host)
             if not ud.host:
                 raise FetchError('Could not determine P4PORT from P4CONFIG')
-	
+
+        # Fetcher options
+        ud.module = ud.parm.get('module')
+        ud.keepremotepath = (ud.parm.get('remotepath', '') == 'keep')
+
         if ud.path.find('/...') >= 0:
             ud.pathisdir = True
         else:
             ud.pathisdir = False
 
+        # Avoid using the "/..." syntax in SRC_URI when a module value is given
+        if ud.pathisdir and ud.module:
+            raise FetchError('SRC_URI depot path cannot not end in /... when a module value is given')
+
         cleanedpath = ud.path.replace('/...', '').replace('/', '.')
         cleanedhost = ud.host.replace(':', '.')
+
+        # Merge the path and module into the final depot location
+        if ud.module:
+            if ud.module.find('/') == 0:
+                raise FetchError('module cannot begin with /')
+            ud.path = os.path.join(ud.path, ud.module)
+
+            # Append the module path to the local pkg name
+            cleanedmodule = ud.module.replace('/...', '').replace('/', '.')
+            cleanedpath += '--%s' % cleanedmodule
+
         ud.pkgdir = os.path.join(ud.dldir, cleanedhost, cleanedpath)
 
         ud.setup_revisions(d)
@@ -96,16 +158,26 @@ class Perforce(FetchMethod):
             pathnrev = '%s' % (ud.path)
 
         if depot_filename:
-            if ud.pathisdir: # Remove leading path to obtain filename
+            if ud.keepremotepath:
+                # preserve everything, remove the leading //
+                filename = depot_filename.lstrip('/')
+            elif ud.module:
+                # remove everything up to the module path
+                modulepath = ud.module.rstrip('/...')
+                filename = depot_filename[depot_filename.rfind(modulepath):]
+            elif ud.pathisdir:
+                # Remove leading (visible) path to obtain the filepath
                 filename = depot_filename[len(ud.path)-1:]
             else:
+                # Remove everything, except the filename
                 filename = depot_filename[depot_filename.rfind('/'):]
+
             filename = filename[:filename.find('#')] # Remove trailing '#rev'
 
         if command == 'changes':
             p4cmd = '%s%s changes -m 1 //%s' % (ud.basecmd, p4opt, pathnrev)
         elif command == 'print':
-            if depot_filename != None:
+            if depot_filename is not None:
                 p4cmd = '%s%s print -o "p4/%s" "%s"' % (ud.basecmd, p4opt, filename, depot_filename)
             else:
                 raise FetchError('No depot file name provided to p4 %s' % command, ud.url)
@@ -151,10 +223,12 @@ class Perforce(FetchMethod):
         bb.utils.remove(ud.pkgdir, True)
         bb.utils.mkdirhier(ud.pkgdir)
 
+        progresshandler = PerforceProgressHandler(d, len(filelist))
+
         for afile in filelist:
             p4fetchcmd = self._buildp4command(ud, d, 'print', afile)
             bb.fetch2.check_network_access(d, p4fetchcmd, ud.url)
-            runfetchcmd(p4fetchcmd, d, workdir=ud.pkgdir)
+            runfetchcmd(p4fetchcmd, d, workdir=ud.pkgdir, log=progresshandler)
 
         runfetchcmd('tar -czf %s p4' % (ud.localpath), d, cleanup=[ud.localpath], workdir=ud.pkgdir)
 

bitbake/lib/bb/fetch2/ssh.py

@@ -31,10 +31,7 @@ IETF secsh internet draft:
 #
 
 import re, os
-from   bb.fetch2 import FetchMethod
-from   bb.fetch2 import FetchError
-from   bb.fetch2 import logger
-from   bb.fetch2 import runfetchcmd
+from bb.fetch2 import check_network_access, FetchMethod, ParameterError, runfetchcmd
 
 
 __pattern__ = re.compile(r'''
@@ -60,14 +57,14 @@ class SSH(FetchMethod):
     '''Class to fetch a module or modules via Secure Shell'''
 
     def supports(self, urldata, d):
-        return __pattern__.match(urldata.url) != None
+        return __pattern__.match(urldata.url) is not None
 
     def supports_checksum(self, urldata):
         return False
 
     def urldata_init(self, urldata, d):
         if 'protocol' in urldata.parm and urldata.parm['protocol'] == 'git':
-            raise bb.fetch2.ParameterError(
+            raise ParameterError(
                 "Invalid protocol - if you wish to fetch from a git " +
                 "repository using ssh, you need to use " +
                 "git:// prefix with protocol=ssh", urldata.url)
@@ -107,7 +104,7 @@ class SSH(FetchMethod):
             dldir
         )
 
-        bb.fetch2.check_network_access(d, cmd, urldata.url)
+        check_network_access(d, cmd, urldata.url)
 
         runfetchcmd(cmd, d)
 

bitbake/lib/bb/fetch2/svn.py

@@ -11,8 +11,6 @@ BitBake 'Fetch' implementation for svn.
 # Based on functions from the base bb module, Copyright 2003 Holger Schurig
 
 import os
-import sys
-import logging
 import bb
 import re
 from   bb.fetch2 import FetchMethod
@@ -49,7 +47,7 @@ class Svn(FetchMethod):
         svndir = d.getVar("SVNDIR") or (d.getVar("DL_DIR") + "/svn")
         relpath = self._strip_leading_slashes(ud.path)
         ud.pkgdir = os.path.join(svndir, ud.host, relpath)
-        ud.moddir = os.path.join(ud.pkgdir, ud.module)
+        ud.moddir = os.path.join(ud.pkgdir, ud.path_spec)
         # Protects the repository from concurrent updates, e.g. from two
         # recipes fetching different revisions at the same time
         ud.svnlock = os.path.join(ud.pkgdir, "svn.lock")
@@ -124,30 +122,30 @@ class Svn(FetchMethod):
 
         try:
             if os.access(os.path.join(ud.moddir, '.svn'), os.R_OK):
-                svnupdatecmd = self._buildsvncommand(ud, d, "update")
+                svncmd = self._buildsvncommand(ud, d, "update")
                 logger.info("Update " + ud.url)
                 # We need to attempt to run svn upgrade first in case its an older working format
                 try:
                     runfetchcmd(ud.basecmd + " upgrade", d, workdir=ud.moddir)
                 except FetchError:
                     pass
-                logger.debug(1, "Running %s", svnupdatecmd)
-                bb.fetch2.check_network_access(d, svnupdatecmd, ud.url)
-                runfetchcmd(svnupdatecmd, d, workdir=ud.moddir)
+                logger.debug(1, "Running %s", svncmd)
+                bb.fetch2.check_network_access(d, svncmd, ud.url)
+                runfetchcmd(svncmd, d, workdir=ud.moddir)
             else:
-                svnfetchcmd = self._buildsvncommand(ud, d, "fetch")
+                svncmd = self._buildsvncommand(ud, d, "fetch")
                 logger.info("Fetch " + ud.url)
                 # check out sources there
                 bb.utils.mkdirhier(ud.pkgdir)
-                logger.debug(1, "Running %s", svnfetchcmd)
-                bb.fetch2.check_network_access(d, svnfetchcmd, ud.url)
-                runfetchcmd(svnfetchcmd, d, workdir=ud.pkgdir)
+                logger.debug(1, "Running %s", svncmd)
+                bb.fetch2.check_network_access(d, svncmd, ud.url)
+                runfetchcmd(svncmd, d, workdir=ud.pkgdir)
 
             if not ("externals" in ud.parm and ud.parm["externals"] == "nowarn"):
                 # Warn the user if this had externals (won't catch them all)
                 output = runfetchcmd("svn propget svn:externals || true", d, workdir=ud.moddir)
                 if output:
-                    if "--ignore-externals" in svnfetchcmd.split():
+                    if "--ignore-externals" in svncmd.split():
                         bb.warn("%s contains svn:externals." % ud.url)
                         bb.warn("These should be added to the recipe SRC_URI as necessary.")
                         bb.warn("svn fetch has ignored externals:\n%s" % output)

bitbake/lib/bb/fetch2/wget.py

@@ -12,11 +12,10 @@ BitBake build tools.
 #
 # Based on functions from the base bb module, Copyright 2003 Holger Schurig
 
+import shlex
 import re
 import tempfile
-import subprocess
 import os
-import logging
 import errno
 import bb
 import bb.progress
@@ -27,7 +26,6 @@ from   bb.fetch2 import FetchMethod
 from   bb.fetch2 import FetchError
 from   bb.fetch2 import logger
 from   bb.fetch2 import runfetchcmd
-from   bb.fetch2 import FetchConnectionCache
 from   bb.utils import export_proxies
 from   bs4 import BeautifulSoup
 from   bs4 import SoupStrainer
@@ -94,9 +92,9 @@ class Wget(FetchMethod):
         fetchcmd = self.basecmd
 
         if 'downloadfilename' in ud.parm:
-            dldir = d.getVar("DL_DIR")
-            bb.utils.mkdirhier(os.path.dirname(dldir + os.sep + ud.localfile))
-            fetchcmd += " -O " + dldir + os.sep + ud.localfile
+            localpath = os.path.join(d.getVar("DL_DIR"), ud.localfile)
+            bb.utils.mkdirhier(os.path.dirname(localpath))
+            fetchcmd += " -O %s" % shlex.quote(localpath)
 
         if ud.user and ud.pswd:
             fetchcmd += " --user=%s --password=%s --auth-no-challenge" % (ud.user, ud.pswd)
@@ -210,10 +208,7 @@ class Wget(FetchMethod):
                         fetch.connection_cache.remove_connection(h.host, h.port)
                     raise urllib.error.URLError(err)
                 else:
-                    try:
-                        r = h.getresponse(buffering=True)
-                    except TypeError: # buffering kw not supported
-                        r = h.getresponse()
+                    r = h.getresponse()
 
                 # Pick apart the HTTPResponse object to get the addinfourl
                 # object initialized properly.
@@ -302,6 +297,7 @@ class Wget(FetchMethod):
             # Some servers (FusionForge, as used on Alioth) require that the
             # optional Accept header is set.
             r.add_header("Accept", "*/*")
+            r.add_header("User-Agent", "Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.9.2.12) Gecko/20101027 Ubuntu/9.10 (karmic) Firefox/3.6.12")
             def add_basic_auth(login_str, request):
                 '''Adds Basic auth to http request, pass in login:password as string'''
                 import base64

bitbake/lib/bb/main.py

@@ -344,8 +344,6 @@ def bitbake_main(configParams, configuration):
     except:
         pass
 
-    configuration.setConfigParameters(configParams)
-
     if configParams.server_only and configParams.remote_server:
             raise BBMainException("FATAL: The '--server-only' option conflicts with %s.\n" %
                                   ("the BBSERVER environment variable" if "BBSERVER" in os.environ \
@@ -357,13 +355,13 @@ def bitbake_main(configParams, configuration):
 
     if "BBDEBUG" in os.environ:
         level = int(os.environ["BBDEBUG"])
-        if level > configuration.debug:
-            configuration.debug = level
+        if level > configParams.debug:
+            configParams.debug = level
 
-    bb.msg.init_msgconfig(configParams.verbose, configuration.debug,
-                          configuration.debug_domains)
+    bb.msg.init_msgconfig(configParams.verbose, configParams.debug,
+                          configParams.debug_domains)
 
-    server_connection, ui_module = setup_bitbake(configParams, configuration)
+    server_connection, ui_module = setup_bitbake(configParams)
     # No server connection
     if server_connection is None:
         if configParams.status_only:
@@ -390,7 +388,7 @@ def bitbake_main(configParams, configuration):
 
     return 1
 
-def setup_bitbake(configParams, configuration, extrafeatures=None):
+def setup_bitbake(configParams, extrafeatures=None):
     # Ensure logging messages get sent to the UI as events
     handler = bb.event.LogHandler()
     if not configParams.status_only:
@@ -431,11 +429,11 @@ def setup_bitbake(configParams, configuration, extrafeatures=None):
                         logger.info("bitbake server is not running.")
                         lock.close()
                         return None, None
-                    # we start a server with a given configuration
+                    # we start a server with a given featureset
                     logger.info("Starting bitbake server...")
                     # Clear the event queue since we already displayed messages
                     bb.event.ui_queue = []
-                    server = bb.server.process.BitBakeServer(lock, sockname, configuration, featureset)
+                    server = bb.server.process.BitBakeServer(lock, sockname, featureset, configParams.server_timeout, configParams.xmlrpcinterface)
 
                 else:
                     logger.info("Reconnecting to bitbake server...")
@@ -458,15 +456,17 @@ def setup_bitbake(configParams, configuration, extrafeatures=None):
                     break
             except BBMainFatal:
                 raise
-            except (Exception, bb.server.process.ProcessTimeout) as e:
+            except (Exception, bb.server.process.ProcessTimeout, SystemExit) as e:
+                # SystemExit does not inherit from the Exception class, needs to be included explicitly
                 if not retries:
                     raise
                 retries -= 1
                 tryno = 8 - retries
-                if isinstance(e, (bb.server.process.ProcessTimeout, BrokenPipeError, EOFError)):
+                if isinstance(e, (bb.server.process.ProcessTimeout, BrokenPipeError, EOFError, SystemExit)):
                     logger.info("Retrying server connection (#%d)..." % tryno)
                 else:
                     logger.info("Retrying server connection (#%d)... (%s)" % (tryno, traceback.format_exc()))
+                
             if not retries:
                 bb.fatal("Unable to connect to bitbake server, or start one (server startup failures would be in bitbake-cookerdaemon.log).")
             bb.event.print_ui_queue()

bitbake/lib/bb/monitordisk.py

@@ -4,7 +4,7 @@
 # SPDX-License-Identifier: GPL-2.0-only
 #
 
-import os, logging, re, sys
+import os, logging, re
 import bb
 logger = logging.getLogger("BitBake.Monitor")
 

bitbake/lib/bb/msg.py

@@ -13,9 +13,9 @@ Message handling infrastructure for bitbake
 import sys
 import copy
 import logging
-import collections
+import logging.config
+import os
 from itertools import groupby
-import warnings
 import bb
 import bb.event
 
@@ -100,6 +100,9 @@ class BBLogFormatter(logging.Formatter):
     def enable_color(self):
         self.color_enabled = True
 
+    def __repr__(self):
+        return "%s fmt='%s' color=%s" % (self.__class__.__name__, self._fmt, "True" if self.color_enabled else "False")
+
 class BBLogFilter(object):
     def __init__(self, handler, level, debug_domains):
         self.stdlevel = level
@@ -118,60 +121,53 @@ class BBLogFilter(object):
             return True
         return False
 
-class BBLogFilterStdErr(BBLogFilter):
-    def filter(self, record):
-        if not BBLogFilter.filter(self, record):
-            return False
-        if record.levelno >= logging.ERROR:
-            return True
-        return False
+class LogFilterGEQLevel(logging.Filter):
+    def __init__(self, level):
+        self.strlevel = str(level)
+        self.level = stringToLevel(level)
+
+    def __repr__(self):
+        return "%s level >= %s (%d)" % (self.__class__.__name__, self.strlevel, self.level)
 
-class BBLogFilterStdOut(BBLogFilter):
     def filter(self, record):
-        if not BBLogFilter.filter(self, record):
-            return False
-        if record.levelno < logging.ERROR:
-            return True
-        return False
+        return (record.levelno >= self.level)
+
+class LogFilterLTLevel(logging.Filter):
+    def __init__(self, level):
+        self.strlevel = str(level)
+        self.level = stringToLevel(level)
+
+    def __repr__(self):
+        return "%s level < %s (%d)" % (self.__class__.__name__, self.strlevel, self.level)
+
+    def filter(self, record):
+        return (record.levelno < self.level)
 
 # Message control functions
 #
 
-loggerDefaultDebugLevel = 0
-loggerDefaultVerbose = False
-loggerVerboseLogs = False
-loggerDefaultDomains = []
+loggerDefaultLogLevel = BBLogFormatter.NOTE
+loggerDefaultDomains = {}
 
 def init_msgconfig(verbose, debug, debug_domains=None):
     """
     Set default verbosity and debug levels config the logger
     """
-    bb.msg.loggerDefaultDebugLevel = debug
-    bb.msg.loggerDefaultVerbose = verbose
-    if verbose:
-        bb.msg.loggerVerboseLogs = True
-    if debug_domains:
-        bb.msg.loggerDefaultDomains = debug_domains
+    if debug:
+        bb.msg.loggerDefaultLogLevel = BBLogFormatter.DEBUG - debug + 1
+    elif verbose:
+        bb.msg.loggerDefaultLogLevel = BBLogFormatter.VERBOSE
     else:
-        bb.msg.loggerDefaultDomains = []
+        bb.msg.loggerDefaultLogLevel = BBLogFormatter.NOTE
+
+    bb.msg.loggerDefaultDomains = {}
+    if debug_domains:
+        for (domainarg, iterator) in groupby(debug_domains):
+            dlevel = len(tuple(iterator))
+            bb.msg.loggerDefaultDomains["BitBake.%s" % domainarg] = logging.DEBUG - dlevel + 1
 
 def constructLogOptions():
-    debug = loggerDefaultDebugLevel
-    verbose = loggerDefaultVerbose
-    domains = loggerDefaultDomains
-
-    if debug:
-        level = BBLogFormatter.DEBUG - debug + 1
-    elif verbose:
-        level = BBLogFormatter.VERBOSE
-    else:
-        level = BBLogFormatter.NOTE
-
-    debug_domains = {}
-    for (domainarg, iterator) in groupby(domains):
-        dlevel = len(tuple(iterator))
-        debug_domains["BitBake.%s" % domainarg] = logging.DEBUG - dlevel + 1
-    return level, debug_domains
+    return loggerDefaultLogLevel, loggerDefaultDomains
 
 def addDefaultlogFilter(handler, cls = BBLogFilter, forcelevel=None):
     level, debug_domains = constructLogOptions()
@@ -181,6 +177,19 @@ def addDefaultlogFilter(handler, cls = BBLogFilter, forcelevel=None):
 
     cls(handler, level, debug_domains)
 
+def stringToLevel(level):
+    try:
+        return int(level)
+    except ValueError:
+        pass
+
+    try:
+        return getattr(logging, level)
+    except AttributeError:
+        pass
+
+    return getattr(BBLogFormatter, level)
+
 #
 # Message handling functions
 #
@@ -214,3 +223,105 @@ def has_console_handler(logger):
             if handler.stream in [sys.stderr, sys.stdout]:
                 return True
     return False
+
+def mergeLoggingConfig(logconfig, userconfig):
+    logconfig = copy.deepcopy(logconfig)
+    userconfig = copy.deepcopy(userconfig)
+
+    # Merge config with the default config
+    if userconfig.get('version') != logconfig['version']:
+        raise BaseException("Bad user configuration version. Expected %r, got %r" % (logconfig['version'], userconfig.get('version')))
+
+    # Set some defaults to make merging easier
+    userconfig.setdefault("loggers", {})
+
+    # If a handler, formatter, or filter is defined in the user
+    # config, it will replace an existing one in the default config
+    for k in ("handlers", "formatters", "filters"):
+        logconfig.setdefault(k, {}).update(userconfig.get(k, {}))
+
+    seen_loggers = set()
+    for name, l in logconfig["loggers"].items():
+        # If the merge option is set, merge the handlers and
+        # filters. Otherwise, if it is False, this logger won't get
+        # add to the set of seen loggers and will replace the
+        # existing one
+        if l.get('bitbake_merge', True):
+            ulogger = userconfig["loggers"].setdefault(name, {})
+            ulogger.setdefault("handlers", [])
+            ulogger.setdefault("filters", [])
+
+            # Merge lists
+            l.setdefault("handlers", []).extend(ulogger["handlers"])
+            l.setdefault("filters", []).extend(ulogger["filters"])
+
+            # Replace other properties if present
+            if "level" in ulogger:
+                l["level"] = ulogger["level"]
+
+            if "propagate" in ulogger:
+                l["propagate"] = ulogger["propagate"]
+
+            seen_loggers.add(name)
+
+    # Add all loggers present in the user config, but not any that
+    # have already been processed
+    for name in set(userconfig["loggers"].keys()) - seen_loggers:
+        logconfig["loggers"][name] = userconfig["loggers"][name]
+
+    return logconfig
+
+def setLoggingConfig(defaultconfig, userconfigfile=None):
+    logconfig = copy.deepcopy(defaultconfig)
+
+    if userconfigfile:
+        with open(os.path.normpath(userconfigfile), 'r') as f:
+            if userconfigfile.endswith('.yml') or userconfigfile.endswith('.yaml'):
+                import yaml
+                userconfig = yaml.load(f)
+            elif userconfigfile.endswith('.json') or userconfigfile.endswith('.cfg'):
+                import json
+                userconfig = json.load(f)
+            else:
+                raise BaseException("Unrecognized file format: %s" % userconfigfile)
+
+            if userconfig.get('bitbake_merge', True):
+                logconfig = mergeLoggingConfig(logconfig, userconfig)
+            else:
+                # Replace the entire default config
+                logconfig = userconfig
+
+    # Convert all level parameters to integers in case users want to use the
+    # bitbake defined level names
+    for h in logconfig["handlers"].values():
+        if "level" in h:
+            h["level"] = bb.msg.stringToLevel(h["level"])
+
+    for l in logconfig["loggers"].values():
+        if "level" in l:
+            l["level"] = bb.msg.stringToLevel(l["level"])
+
+    conf = logging.config.dictConfigClass(logconfig)
+    conf.configure()
+
+    # The user may have specified logging domains they want at a higher debug
+    # level than the standard.
+    for name, l in logconfig["loggers"].items():
+        if not name.startswith("BitBake."):
+            continue
+
+        if not "level" in l:
+            continue
+
+        curlevel = bb.msg.loggerDefaultDomains.get(name)
+        # Note: level parameter should already be a int because of conversion
+        # above
+        newlevel = int(l["level"])
+        if curlevel is None or newlevel < curlevel:
+            bb.msg.loggerDefaultDomains[name] = newlevel
+
+        # TODO: I don't think that setting the global log level should be necessary
+        #if newlevel < bb.msg.loggerDefaultLogLevel:
+        #    bb.msg.loggerDefaultLogLevel = newlevel
+
+    return conf

bitbake/lib/bb/namedtuple_with_abc.py

@@ -61,17 +61,9 @@ class _NamedTupleABCMeta(ABCMeta):
         return ABCMeta.__new__(mcls, name, bases, namespace)
 
 
-exec(
-    # Python 2.x metaclass declaration syntax
-    """class _NamedTupleABC(object):
-        '''The abstract base class + mix-in for named tuples.'''
-        __metaclass__ = _NamedTupleABCMeta
-        _fields = abstractproperty()""" if version_info[0] < 3 else
-    # Python 3.x metaclass declaration syntax
-    """class _NamedTupleABC(metaclass=_NamedTupleABCMeta):
-        '''The abstract base class + mix-in for named tuples.'''
-        _fields = abstractproperty()"""
-)
+class _NamedTupleABC(metaclass=_NamedTupleABCMeta):
+    '''The abstract base class + mix-in for named tuples.'''
+    _fields = abstractproperty()
 
 
 _namedtuple.abc = _NamedTupleABC

bitbake/lib/bb/parse/ast.py

@@ -9,11 +9,7 @@
 # SPDX-License-Identifier: GPL-2.0-only
 #
 
-import re
-import string
-import logging
 import bb
-import itertools
 from bb import methodpool
 from bb.parse import logger
 
@@ -93,7 +89,7 @@ class DataNode(AstNode):
         self.groupd = groupd
 
     def getFunc(self, key, data):
-        if 'flag' in self.groupd and self.groupd['flag'] != None:
+        if 'flag' in self.groupd and self.groupd['flag'] is not None:
             return data.getVarFlag(key, self.groupd['flag'], expand=False, noweakdefault=True)
         else:
             return data.getVar(key, False, noweakdefault=True, parsing=True)
@@ -106,36 +102,36 @@ class DataNode(AstNode):
             'file': self.filename,
             'line': self.lineno,
         }
-        if "exp" in groupd and groupd["exp"] != None:
+        if "exp" in groupd and groupd["exp"] is not None:
             data.setVarFlag(key, "export", 1, op = 'exported', **loginfo)
 
         op = "set"
-        if "ques" in groupd and groupd["ques"] != None:
+        if "ques" in groupd and groupd["ques"] is not None:
             val = self.getFunc(key, data)
             op = "set?"
-            if val == None:
+            if val is None:
                 val = groupd["value"]
-        elif "colon" in groupd and groupd["colon"] != None:
+        elif "colon" in groupd and groupd["colon"] is not None:
             e = data.createCopy()
             op = "immediate"
             val = e.expand(groupd["value"], key + "[:=]")
-        elif "append" in groupd and groupd["append"] != None:
+        elif "append" in groupd and groupd["append"] is not None:
             op = "append"
             val = "%s %s" % ((self.getFunc(key, data) or ""), groupd["value"])
-        elif "prepend" in groupd and groupd["prepend"] != None:
+        elif "prepend" in groupd and groupd["prepend"] is not None:
             op = "prepend"
             val = "%s %s" % (groupd["value"], (self.getFunc(key, data) or ""))
-        elif "postdot" in groupd and groupd["postdot"] != None:
+        elif "postdot" in groupd and groupd["postdot"] is not None:
             op = "postdot"
             val = "%s%s" % ((self.getFunc(key, data) or ""), groupd["value"])
-        elif "predot" in groupd and groupd["predot"] != None:
+        elif "predot" in groupd and groupd["predot"] is not None:
             op = "predot"
             val = "%s%s" % (groupd["value"], (self.getFunc(key, data) or ""))
         else:
             val = groupd["value"]
 
         flag = None
-        if 'flag' in groupd and groupd['flag'] != None:
+        if 'flag' in groupd and groupd['flag'] is not None:
             flag = groupd['flag']
         elif groupd["lazyques"]:
             flag = "_defaultval"
@@ -248,12 +244,14 @@ class AddTaskNode(AstNode):
         bb.build.addtask(self.func, self.before, self.after, data)
 
 class DelTaskNode(AstNode):
-    def __init__(self, filename, lineno, func):
+    def __init__(self, filename, lineno, tasks):
         AstNode.__init__(self, filename, lineno)
-        self.func = func
+        self.tasks = tasks
 
     def eval(self, data):
-        bb.build.deltask(self.func, data)
+        tasks = data.expand(self.tasks).split()
+        for task in tasks:
+            bb.build.deltask(task, data)
 
 class BBHandlerNode(AstNode):
     def __init__(self, filename, lineno, fns):
@@ -309,7 +307,7 @@ def handleAddTask(statements, filename, lineno, m):
     statements.append(AddTaskNode(filename, lineno, func, before, after))
 
 def handleDelTask(statements, filename, lineno, m):
-    func = m.group("func")
+    func = m.group(1)
     if func is None:
         return
 
@@ -342,6 +340,9 @@ def finalize(fn, d, variant = None):
         bb.event.fire(bb.event.RecipePreFinalise(fn), d)
 
         bb.data.expandKeys(d)
+
+        bb.event.fire(bb.event.RecipePostKeyExpansion(fn), d)
+
         runAnonFuncs(d)
 
         tasklist = d.getVar('__BBTASKS', False) or []

bitbake/lib/bb/parse/parse_py/BBHandler.py

@@ -13,9 +13,7 @@
 #
 
 import re, bb, os
-import logging
 import bb.build, bb.utils
-from bb import data
 
 from . import ConfHandler
 from .. import resolve_file, ast, logger, ParseError
@@ -28,7 +26,7 @@ __func_start_regexp__    = re.compile(r"(((?P<py>python)|(?P<fr>fakeroot))\s*)*(
 __inherit_regexp__       = re.compile(r"inherit\s+(.+)" )
 __export_func_regexp__   = re.compile(r"EXPORT_FUNCTIONS\s+(.+)" )
 __addtask_regexp__       = re.compile(r"addtask\s+(?P<func>\w+)\s*((before\s*(?P<before>((.*(?=after))|(.*))))|(after\s*(?P<after>((.*(?=before))|(.*)))))*")
-__deltask_regexp__       = re.compile(r"deltask\s+(?P<func>\w+)(?P<ignores>.*)")
+__deltask_regexp__       = re.compile(r"deltask\s+(.+)")
 __addhandler_regexp__    = re.compile(r"addhandler\s+(.+)" )
 __def_regexp__           = re.compile(r"def\s+(\w+).*:" )
 __python_func_regexp__   = re.compile(r"(\s+.*)|(^$)|(^#)" )
@@ -240,9 +238,6 @@ def feeder(lineno, s, fn, root, statements, eof=False):
 
     m = __deltask_regexp__.match(s)
     if m:
-        # Check and warn "for deltask task1 task2"
-        if m.group('ignores'):
-            logger.warning('deltask ignored: "%s"' % m.group('ignores'))
         ast.handleDelTask(statements, fn, lineno, m)
         return
 

bitbake/lib/bb/parse/parse_py/ConfHandler.py

@@ -119,30 +119,30 @@ def handle(fn, data, include):
         oldfile = data.getVar('FILE', False)
 
     abs_fn = resolve_file(fn, data)
-    f = open(abs_fn, 'r')
+    with open(abs_fn, 'r') as f:
 
-    statements = ast.StatementGroup()
-    lineno = 0
-    while True:
-        lineno = lineno + 1
-        s = f.readline()
-        if not s:
-            break
-        w = s.strip()
-        # skip empty lines
-        if not w:
-            continue
-        s = s.rstrip()
-        while s[-1] == '\\':
-            s2 = f.readline().rstrip()
+        statements = ast.StatementGroup()
+        lineno = 0
+        while True:
             lineno = lineno + 1
-            if (not s2 or s2 and s2[0] != "#") and s[0] == "#" :
-                bb.fatal("There is a confusing multiline, partially commented expression on line %s of file %s (%s).\nPlease clarify whether this is all a comment or should be parsed." % (lineno, fn, s))
-            s = s[:-1] + s2
-        # skip comments
-        if s[0] == '#':
-            continue
-        feeder(lineno, s, abs_fn, statements)
+            s = f.readline()
+            if not s:
+                break
+            w = s.strip()
+            # skip empty lines
+            if not w:
+                continue
+            s = s.rstrip()
+            while s[-1] == '\\':
+                s2 = f.readline().rstrip()
+                lineno = lineno + 1
+                if (not s2 or s2 and s2[0] != "#") and s[0] == "#" :
+                    bb.fatal("There is a confusing multiline, partially commented expression on line %s of file %s (%s).\nPlease clarify whether this is all a comment or should be parsed." % (lineno, fn, s))
+                s = s[:-1] + s2
+            # skip comments
+            if s[0] == '#':
+                continue
+            feeder(lineno, s, abs_fn, statements)
 
     # DONE WITH PARSING... time to evaluate
     data.setVar('FILE', abs_fn)

bitbake/lib/bb/persist_data.py

@@ -12,14 +12,14 @@ currently, providing a key/value store accessed by 'domain'.
 #
 
 import collections
+import contextlib
+import functools
 import logging
 import os.path
+import sqlite3
 import sys
 import warnings
-from bb.compat import total_ordering
 from collections import Mapping
-import sqlite3
-import contextlib
 
 sqlversion = sqlite3.sqlite_version_info
 if sqlversion[0] < 3 or (sqlversion[0] == 3 and sqlversion[1] < 3):
@@ -28,7 +28,7 @@ if sqlversion[0] < 3 or (sqlversion[0] == 3 and sqlversion[1] < 3):
 
 logger = logging.getLogger("BitBake.PersistData")
 
-@total_ordering
+@functools.total_ordering
 class SQLTable(collections.MutableMapping):
     class _Decorators(object):
         @staticmethod

bitbake/lib/bb/process.py

@@ -7,6 +7,7 @@ import signal
 import subprocess
 import errno
 import select
+import bb
 
 logger = logging.getLogger('BitBake.Process')
 
@@ -41,6 +42,7 @@ class ExecutionError(CmdError):
         self.exitcode = exitcode
         self.stdout = stdout
         self.stderr = stderr
+        self.extra_message = None
 
     def __str__(self):
         message = ""
@@ -51,7 +53,7 @@ class ExecutionError(CmdError):
         if message:
             message = ":\n" + message
         return (CmdError.__str__(self) +
-                " with exit code %s" % self.exitcode + message)
+                " with exit code %s" % self.exitcode + message + (self.extra_message or ""))
 
 class Popen(subprocess.Popen):
     defaults = {

bitbake/lib/bb/progress.py

@@ -7,7 +7,6 @@ BitBake progress handling code
 # SPDX-License-Identifier: GPL-2.0-only
 #
 
-import sys
 import re
 import time
 import inspect
@@ -15,7 +14,27 @@ import bb.event
 import bb.build
 from bb.build import StdoutNoopContextManager
 
-class ProgressHandler(object):
+
+# from https://stackoverflow.com/a/14693789/221061
+ANSI_ESCAPE_REGEX = re.compile(r'\x1B\[[0-?]*[ -/]*[@-~]')
+
+
+def filter_color(string):
+    """
+    Filter ANSI escape codes out of |string|, return new string
+    """
+    return ANSI_ESCAPE_REGEX.sub('', string)
+
+
+def filter_color_n(string):
+    """
+    Filter ANSI escape codes out of |string|, returns tuple of
+    (new string, # of ANSI codes removed)
+    """
+    return ANSI_ESCAPE_REGEX.subn('', string)
+
+
+class ProgressHandler:
     """
     Base class that can pretend to be a file object well enough to be
     used to build objects to intercept console output and determine the
@@ -56,6 +75,7 @@ class ProgressHandler(object):
             self._lastevent = ts
             self._progress = progress
 
+
 class LineFilterProgressHandler(ProgressHandler):
     """
     A ProgressHandler variant that provides the ability to filter out
@@ -67,7 +87,7 @@ class LineFilterProgressHandler(ProgressHandler):
     """
     def __init__(self, d, outfile=None):
         self._linebuffer = ''
-        super(LineFilterProgressHandler, self).__init__(d, outfile)
+        super().__init__(d, outfile)
 
     def write(self, string):
         self._linebuffer += string
@@ -81,41 +101,44 @@ class LineFilterProgressHandler(ProgressHandler):
             lbreakpos = line.rfind('\r') + 1
             if lbreakpos:
                 line = line[lbreakpos:]
-            if self.writeline(line):
-                super(LineFilterProgressHandler, self).write(line)
+            if self.writeline(filter_color(line)):
+                super().write(line)
 
     def writeline(self, line):
         return True
 
+
 class BasicProgressHandler(ProgressHandler):
     def __init__(self, d, regex=r'(\d+)%', outfile=None):
-        super(BasicProgressHandler, self).__init__(d, outfile)
+        super().__init__(d, outfile)
         self._regex = re.compile(regex)
         # Send an initial progress event so the bar gets shown
         self._fire_progress(0)
 
     def write(self, string):
-        percs = self._regex.findall(string)
+        percs = self._regex.findall(filter_color(string))
         if percs:
             progress = int(percs[-1])
             self.update(progress)
-        super(BasicProgressHandler, self).write(string)
+        super().write(string)
+
 
 class OutOfProgressHandler(ProgressHandler):
     def __init__(self, d, regex, outfile=None):
-        super(OutOfProgressHandler, self).__init__(d, outfile)
+        super().__init__(d, outfile)
         self._regex = re.compile(regex)
         # Send an initial progress event so the bar gets shown
         self._fire_progress(0)
 
     def write(self, string):
-        nums = self._regex.findall(string)
+        nums = self._regex.findall(filter_color(string))
         if nums:
             progress = (float(nums[-1][0]) / float(nums[-1][1])) * 100
             self.update(progress)
-        super(OutOfProgressHandler, self).write(string)
+        super().write(string)
 
-class MultiStageProgressReporter(object):
+
+class MultiStageProgressReporter:
     """
     Class which allows reporting progress without the caller
     having to know where they are in the overall sequence. Useful
@@ -200,6 +223,7 @@ class MultiStageProgressReporter(object):
           value is considered to be out of stage_total, otherwise it should
           be a percentage value from 0 to 100.
         """
+        progress = None
         if self._stage_total:
             stage_progress = (float(stage_progress) / self._stage_total) * 100
         if self._stage < 0:
@@ -208,9 +232,10 @@ class MultiStageProgressReporter(object):
             progress = self._base_progress + (stage_progress * self._stage_weights[self._stage])
         else:
             progress = self._base_progress
-        if progress > 100:
-            progress = 100
-        self._fire_progress(progress)
+        if progress:
+            if progress > 100:
+                progress = 100
+            self._fire_progress(progress)
 
     def finish(self):
         if self._finished:
@@ -231,6 +256,7 @@ class MultiStageProgressReporter(object):
                     out.append('Up to finish: %d' % stage_weight)
             bb.warn('Stage times:\n  %s' % '\n  '.join(out))
 
+
 class MultiStageProcessProgressReporter(MultiStageProgressReporter):
     """
     Version of MultiStageProgressReporter intended for use with
@@ -239,7 +265,7 @@ class MultiStageProcessProgressReporter(MultiStageProgressReporter):
     def __init__(self, d, processname, stage_weights, debug=False):
         self._processname = processname
         self._started = False
-        MultiStageProgressReporter.__init__(self, d, stage_weights, debug)
+        super().__init__(d, stage_weights, debug)
 
     def start(self):
         if not self._started:
@@ -256,13 +282,14 @@ class MultiStageProcessProgressReporter(MultiStageProgressReporter):
         MultiStageProgressReporter.finish(self)
         bb.event.fire(bb.event.ProcessFinished(self._processname), self._data)
 
+
 class DummyMultiStageProcessProgressReporter(MultiStageProgressReporter):
     """
     MultiStageProcessProgressReporter that takes the calls and does nothing
     with them (to avoid a bunch of "if progress_reporter:" checks)
     """
     def __init__(self):
-        MultiStageProcessProgressReporter.__init__(self, "", None, [])
+        super().__init__(None, [])
 
     def _fire_progress(self, taskprogress, rate=None):
         pass