bitbake: server/process: Always place the server logfile in the build directory

Currently the bitbake-cookerdaemon.log is placed into cwd. This seems like a
bad idea, we can place it in the build directory alongside the lockfile that
represents the server instead.

(Bitbake rev: 1620dbc48ffb2a882371cf9174a7b12648befc8a)

Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>

.gitignore

@@ -18,9 +18,13 @@ hob-image-*.bb
 !meta-yocto
 !meta-yocto-bsp
 !meta-yocto-imported
-documentation/user-manual/user-manual.html
-documentation/user-manual/user-manual.pdf
-documentation/user-manual/user-manual.tgz
+/documentation/*/eclipse/
+/documentation/*/*.html
+/documentation/*/*.pdf
+/documentation/*/*.tgz
+/bitbake/doc/bitbake-user-manual/bitbake-user-manual.html
+/bitbake/doc/bitbake-user-manual/bitbake-user-manual.pdf
+/bitbake/doc/bitbake-user-manual/bitbake-user-manual.tgz
 pull-*/
 bitbake/lib/toaster/contrib/tts/backlog.txt
 bitbake/lib/toaster/contrib/tts/log/*

README.LSB

@@ -23,3 +23,4 @@ therefore provides compatibility with the following caveats:
 * libpng 1.2 isn't provided; oe-core includes the latest release of libpng
   instead.
 
+* pax (POSIX standard archive) tool is not provided.

bitbake/bin/bitbake

@@ -38,7 +38,7 @@ from bb.main import bitbake_main, BitBakeConfigParameters, BBMainException
 if sys.getfilesystemencoding() != "utf-8":
     sys.exit("Please use a locale setting which supports UTF-8 (such as LANG=en_US.UTF-8).\nPython can't change the filesystem locale after loading so we need a UTF-8 when Python starts or things won't work.")
 
-__version__ = "1.37.0"
+__version__ = "1.39.0"
 
 if __name__ == "__main__":
     if __version__ != bb.__version__:

bitbake/bin/toaster

@@ -68,7 +68,7 @@ webserverKillAll()
         if [ -f ${pidfile} ]; then
             pid=`cat ${pidfile}`
             while kill -0 $pid 2>/dev/null; do
-                kill -SIGTERM -$pid 2>/dev/null
+                kill -SIGTERM $pid 2>/dev/null
                 sleep 1
             done
             rm  ${pidfile}
@@ -91,7 +91,7 @@ webserverStartAll()
 
     echo "Starting webserver..."
 
-    $MANAGE runserver "$ADDR_PORT" \
+    $MANAGE runserver --noreload "$ADDR_PORT" \
            </dev/null >>${BUILDDIR}/toaster_web.log 2>&1 \
            & echo $! >${BUILDDIR}/.toastermain.pid
 

bitbake/contrib/dump_cache.py

@@ -2,7 +2,7 @@
 # ex:ts=4:sw=4:sts=4:et
 # -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*-
 #
-# Copyright (C) 2012 Wind River Systems, Inc.
+# Copyright (C) 2012, 2018 Wind River Systems, Inc.
 #
 # This program is free software; you can redistribute it and/or modify
 # it under the terms of the GNU General Public License version 2 as
@@ -18,51 +18,68 @@
 # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
 
 #
-# This is used for dumping the bb_cache.dat, the output format is:
-# recipe_path PN PV PACKAGES
+# Used for dumping the bb_cache.dat
 #
 import os
 import sys
-import warnings
+import argparse
 
 # For importing bb.cache
 sys.path.insert(0, os.path.join(os.path.abspath(os.path.dirname(sys.argv[0])), '../lib'))
 from bb.cache import CoreRecipeInfo
 
-import pickle as pickle
+import pickle
 
-def main(argv=None):
-    """
-    Get the mapping for the target recipe.
-    """
-    if len(argv) != 1:
-        print("Error, need one argument!", file=sys.stderr)
-        return 2
+class DumpCache(object):
+    def __init__(self):
+        parser = argparse.ArgumentParser(
+            description="bb_cache.dat's dumper",
+            epilog="Use %(prog)s --help to get help")
+        parser.add_argument("-r", "--recipe",
+            help="specify the recipe, default: all recipes", action="store")
+        parser.add_argument("-m", "--members",
+            help = "specify the member, use comma as separator for multiple ones, default: all members", action="store", default="")
+        parser.add_argument("-s", "--skip",
+            help = "skip skipped recipes", action="store_true")
+        parser.add_argument("cachefile",
+            help = "specify bb_cache.dat", nargs = 1, action="store", default="")
 
-    cachefile = argv[0]
+        self.args = parser.parse_args()
 
-    with open(cachefile, "rb") as cachefile:
-        pickled = pickle.Unpickler(cachefile)
-        while cachefile:
-            try:
-                key = pickled.load()
-                val = pickled.load()
-            except Exception:
-                break
-            if isinstance(val, CoreRecipeInfo) and (not val.skipped):
-                pn = val.pn
-                # Filter out the native recipes.
-                if key.startswith('virtual:native:') or pn.endswith("-native"):
-                    continue
+    def main(self):
+        with open(self.args.cachefile[0], "rb") as cachefile:
+            pickled = pickle.Unpickler(cachefile)
+            while True:
+                try:
+                    key = pickled.load()
+                    val = pickled.load()
+                except Exception:
+                    break
+                if isinstance(val, CoreRecipeInfo):
+                    pn = val.pn
 
-                # 1.0 is the default version for a no PV recipe.
-                if "pv" in val.__dict__:
-                    pv = val.pv
-                else:
-                    pv = "1.0"
+                    if self.args.recipe and self.args.recipe != pn:
+                        continue
 
-                print("%s %s %s %s" % (key, pn, pv, ' '.join(val.packages)))
+                    if self.args.skip and val.skipped:
+                        continue
+
+                    if self.args.members:
+                        out = key
+                        for member in self.args.members.split(','):
+                            out += ": %s" % val.__dict__.get(member)
+                        print("%s" % out)
+                    else:
+                        print("%s: %s" % (key, val.__dict__))
+                elif not self.args.recipe:
+                    print("%s %s" % (key, val))
 
 if __name__ == "__main__":
-    sys.exit(main(sys.argv[1:]))
-
+    try:
+        dump = DumpCache()
+        ret = dump.main()
+    except Exception as esc:
+        ret = 1
+        import traceback
+        traceback.print_exc()
+    sys.exit(ret)

bitbake/doc/bitbake-user-manual/bitbake-user-manual-execution.xml

@@ -781,7 +781,7 @@
             The code in <filename>meta/lib/oe/sstatesig.py</filename> shows two examples
             of this and also illustrates how you can insert your own policy into the system
             if so desired.
-            This file defines the two basic signature generators OpenEmbedded Core
+            This file defines the two basic signature generators OpenEmbedded-Core
             uses:  "OEBasic" and "OEBasicHash".
             By default, there is a dummy "noop" signature handler enabled in BitBake.
             This means that behavior is unchanged from previous versions.

bitbake/doc/bitbake-user-manual/bitbake-user-manual-fetching.xml

@@ -777,6 +777,43 @@
             </para>
         </section>
 
+        <section id='repo-fetcher'>
+            <title>Repo Fetcher (<filename>repo://</filename>)</title>
+
+            <para>
+                This fetcher submodule fetches code from
+                <filename>google-repo</filename> source control system.
+                The fetcher works by initiating and syncing sources of the
+                repository into
+                <link linkend='var-REPODIR'><filename>REPODIR</filename></link>,
+                which is usually
+                <link linkend='var-DL_DIR'><filename>DL_DIR</filename></link><filename>/repo</filename>.
+            </para>
+
+            <para>
+                This fetcher supports the following parameters:
+                <itemizedlist>
+                    <listitem><para>
+                        <emphasis>"protocol":</emphasis>
+                        Protocol to fetch the repository manifest (default: git).
+                        </para></listitem>
+                    <listitem><para>
+                        <emphasis>"branch":</emphasis>
+                        Branch or tag of repository to get (default: master).
+                        </para></listitem>
+                    <listitem><para>
+                        <emphasis>"manifest":</emphasis>
+                        Name of the manifest file (default: <filename>default.xml</filename>).
+                        </para></listitem>
+                </itemizedlist>
+                Here are some example URLs:
+                <literallayout class='monospaced'>
+    SRC_URI = "repo://REPOROOT;protocol=git;branch=some_branch;manifest=my_manifest.xml"
+    SRC_URI = "repo://REPOROOT;protocol=file;branch=some_branch;manifest=my_manifest.xml"
+                </literallayout>
+            </para>
+        </section>
+
         <section id='other-fetchers'>
             <title>Other Fetchers</title>
 
@@ -795,9 +832,6 @@
                     <listitem><para>
                         Secure Shell (<filename>ssh://</filename>)
                         </para></listitem>
-                    <listitem><para>
-                        Repo (<filename>repo://</filename>)
-                        </para></listitem>
                     <listitem><para>
                         OSC (<filename>osc://</filename>)
                         </para></listitem>

bitbake/doc/bitbake-user-manual/bitbake-user-manual-hello.xml

@@ -260,7 +260,7 @@
                 files.
                 For this example, you need to create the file in your project directory
                 and define some key BitBake variables.
-                For more information on the <filename>bitbake.conf</filename>,
+                For more information on the <filename>bitbake.conf</filename> file,
                 see
                 <ulink url='http://git.openembedded.org/bitbake/tree/conf/bitbake.conf'></ulink>.
                 </para>
@@ -273,14 +273,32 @@
                 some editor to create the <filename>bitbake.conf</filename>
                 so that it contains the following:
                 <literallayout class='monospaced'>
+     <link linkend='var-PN'>PN</link>  = "${@bb.parse.BBHandler.vars_from_file(d.getVar('FILE', False),d)[0] or 'defaultpkgname'}"
+                </literallayout>
+                <literallayout class='monospaced'>
      TMPDIR  = "${<link linkend='var-TOPDIR'>TOPDIR</link>}/tmp"
      <link linkend='var-CACHE'>CACHE</link>   = "${TMPDIR}/cache"
-     <link linkend='var-STAMP'>STAMP</link>   = "${TMPDIR}/stamps"
-     <link linkend='var-T'>T</link>       = "${TMPDIR}/work"
-     <link linkend='var-B'>B</link>       = "${TMPDIR}"
+     <link linkend='var-STAMP'>STAMP</link>   = "${TMPDIR}/${PN}/stamps"
+     <link linkend='var-T'>T</link>       = "${TMPDIR}/${PN}/work"
+     <link linkend='var-B'>B</link>       = "${TMPDIR}/${PN}"
                 </literallayout>
+                <note>
+                    Without a value for <filename>PN</filename>, the
+                    variables <filename>STAMP</filename>,
+                    <filename>T</filename>, and <filename>B</filename>,
+                    prevent more than one recipe from working. You can fix
+                    this by either setting <filename>PN</filename> to have
+                    a value similar to what OpenEmbedded and BitBake use
+                    in the default <filename>bitbake.conf</filename> file
+                    (see previous example). Or, by manually updating each
+                    recipe to set <filename>PN</filename>. You will also
+                    need to include <filename>PN</filename> as part of the
+                    <filename>STAMP</filename>, <filename>T</filename>, and
+                    <filename>B</filename> variable definitions in the
+                    <filename>local.conf</filename> file.
+                </note>
                 The <filename>TMPDIR</filename> variable establishes a directory
-                that BitBake uses for build output and intermediate files (other
+                that BitBake uses for build output and intermediate files other
                 than the cached information used by the
                 <link linkend='setscene'>Setscene</link> process.
                 Here, the <filename>TMPDIR</filename> directory is set to
@@ -300,19 +318,19 @@
                 file exists, you can run the <filename>bitbake</filename>
                 command again:
                 <literallayout class='monospaced'>
-$ bitbake
-ERROR: Traceback (most recent call last):
-  File "/home/scott-lenovo/bitbake/lib/bb/cookerdata.py", line 163, in wrapped
-    return func(fn, *args)
-  File "/home/scott-lenovo/bitbake/lib/bb/cookerdata.py", line 177, in _inherit
-    bb.parse.BBHandler.inherit(bbclass, "configuration INHERITs", 0, data)
-  File "/home/scott-lenovo/bitbake/lib/bb/parse/parse_py/BBHandler.py", line 92, in inherit
-    include(fn, file, lineno, d, "inherit")
-  File "/home/scott-lenovo/bitbake/lib/bb/parse/parse_py/ConfHandler.py", line 100, in include
-    raise ParseError("Could not %(error_out)s file %(fn)s" % vars(), oldfn, lineno)
-ParseError: ParseError in configuration INHERITs: Could not inherit file classes/base.bbclass
+     $ bitbake
+     ERROR: Traceback (most recent call last):
+       File "/home/scott-lenovo/bitbake/lib/bb/cookerdata.py", line 163, in wrapped
+         return func(fn, *args)
+       File "/home/scott-lenovo/bitbake/lib/bb/cookerdata.py", line 177, in _inherit
+         bb.parse.BBHandler.inherit(bbclass, "configuration INHERITs", 0, data)
+       File "/home/scott-lenovo/bitbake/lib/bb/parse/parse_py/BBHandler.py", line 92, in inherit
+         include(fn, file, lineno, d, "inherit")
+       File "/home/scott-lenovo/bitbake/lib/bb/parse/parse_py/ConfHandler.py", line 100, in include
+         raise ParseError("Could not %(error_out)s file %(fn)s" % vars(), oldfn, lineno)
+     ParseError: ParseError in configuration INHERITs: Could not inherit file classes/base.bbclass
 
-ERROR: Unable to parse base: ParseError in configuration INHERITs: Could not inherit file classes/base.bbclass
+     ERROR: Unable to parse base: ParseError in configuration INHERITs: Could not inherit file classes/base.bbclass
                 </literallayout>
                 In the sample output, BitBake could not find the
                 <filename>classes/base.bbclass</filename> file.
@@ -365,10 +383,10 @@ ERROR: Unable to parse base: ParseError in configuration INHERITs: Could not inh
                 code separate from the general metadata used by BitBake.
                 Thus, this example creates and uses a layer called "mylayer".
                 <note>
-                    You can find additional information on layers at
-                    <ulink url='http://www.yoctoproject.org/docs/2.3/bitbake-user-manual/bitbake-user-manual.html#layers'></ulink>.
-                </note>
-                </para>
+                    You can find additional information on layers in the
+                    "<link linkend='layers'>Layers</link>" section.
+                </note></para>
+
                 <para>Minimally, you need a recipe file and a layer configuration
                 file in your layer.
                 The configuration file needs to be in the <filename>conf</filename>

bitbake/doc/bitbake-user-manual/bitbake-user-manual-intro.xml

@@ -488,8 +488,6 @@
                              target that failed and anything depending on it cannot
                              be built, as much as possible will be built before
                              stopping.
-       -a, --tryaltconfigs   Continue with builds by trying to use alternative
-                             providers where possible.
        -f, --force           Force the specified targets/task to run (invalidating
                              any existing stamp file).
        -c CMD, --cmd=CMD     Specify the task to execute. The exact options
@@ -504,19 +502,20 @@
                              Read the specified file before bitbake.conf.
        -R POSTFILE, --postread=POSTFILE
                              Read the specified file after bitbake.conf.
-       -v, --verbose         Enable tracing of shell tasks (with 'set -x').
-                             Also print bb.note(...) messages to stdout (in
-                             addition to writing them to ${T}/log.do_<task>).
-       -D, --debug           Increase the debug level. You can specify this
-                             more than once. -D sets the debug level to 1,
-                             where only bb.debug(1, ...) messages are printed
-                             to stdout; -DD sets the debug level to 2, where
-                             both bb.debug(1, ...) and bb.debug(2, ...)
-                             messages are printed; etc. Without -D, no debug
-                             messages are printed. Note that -D only affects
-                             output to stdout. All debug messages are written
-                             to ${T}/log.do_taskname, regardless of the debug
-                             level.
+       -v, --verbose         Enable tracing of shell tasks (with 'set -x'). Also
+                             print bb.note(...) messages to stdout (in addition to
+                             writing them to ${T}/log.do_<task>).
+       -D, --debug           Increase the debug level. You can specify this more
+                             than once. -D sets the debug level to 1, where only
+                             bb.debug(1, ...) messages are printed to stdout; -DD
+                             sets the debug level to 2, where both bb.debug(1, ...)
+                             and bb.debug(2, ...) messages are printed; etc.
+                             Without -D, no debug messages are printed. Note that
+                             -D only affects output to stdout. All debug messages
+                             are written to ${T}/log.do_taskname, regardless of the
+                             debug level.
+       -q, --quiet           Output less log message data to the terminal. You can
+                             specify this more than once.
        -n, --dry-run         Don't execute, just go through the motions.
        -S SIGNATURE_HANDLER, --dump-signatures=SIGNATURE_HANDLER
                              Dump out the signature construction information, with
@@ -539,30 +538,38 @@
        -l DEBUG_DOMAINS, --log-domains=DEBUG_DOMAINS
                              Show debug logging for the specified logging domains
        -P, --profile         Profile the command and save reports.
-       -u UI, --ui=UI        The user interface to use (taskexp, knotty or
-                             ncurses - default knotty).
-       -t SERVERTYPE, --servertype=SERVERTYPE
-                             Choose which server type to use (process or xmlrpc -
-                             default process).
+       -u UI, --ui=UI        The user interface to use (knotty, ncurses or taskexp
+                             - default knotty).
        --token=XMLRPCTOKEN   Specify the connection token to be used when
                              connecting to a remote server.
        --revisions-changed   Set the exit code depending on whether upstream
                              floating revisions have changed or not.
        --server-only         Run bitbake without a UI, only starting a server
                              (cooker) process.
-       -B BIND, --bind=BIND  The name/address for the bitbake server to bind to.
+       -B BIND, --bind=BIND  The name/address for the bitbake xmlrpc server to bind
+                             to.
+       -T SERVER_TIMEOUT, --idle-timeout=SERVER_TIMEOUT
+                             Set timeout to unload bitbake server due to
+                             inactivity, set to -1 means no unload, default:
+                             Environment variable BB_SERVER_TIMEOUT.
        --no-setscene         Do not run any setscene tasks. sstate will be ignored
                              and everything needed, built.
        --setscene-only       Only run setscene tasks, don't run any real tasks.
        --remote-server=REMOTE_SERVER
                              Connect to the specified server.
-       -m, --kill-server     Terminate the remote server.
+       -m, --kill-server     Terminate any running bitbake server.
        --observe-only        Connect to a server as an observing-only client.
        --status-only         Check the status of the remote bitbake server.
        -w WRITEEVENTLOG, --write-log=WRITEEVENTLOG
                              Writes the event log of the build to a bitbake event
                              json file. Use '' (empty string) to assign the name
                              automatically.
+       --runall=RUNALL       Run the specified task for any recipe in the taskgraph
+                             of the specified target (even if it wouldn't otherwise
+                             have run).
+       --runonly=RUNONLY     Run only the specified task within the taskgraph of
+                             the specified targets (and any task dependencies those
+                             tasks may have).
                 </literallayout>
             </para>
         </section>

bitbake/doc/bitbake-user-manual/bitbake-user-manual-metadata.xml

@@ -1929,6 +1929,38 @@
                         not careful.
                     </note>
                     </para></listitem>
+                <listitem><para><emphasis><filename>[number_threads]</filename>:</emphasis>
+                    Limits tasks to a specific number of simultaneous threads
+                    during execution.
+                    This varflag is useful when your build host has a large number
+                    of cores but certain tasks need to be rate-limited due to various
+                    kinds of resource constraints (e.g. to avoid network throttling).
+                    <filename>number_threads</filename> works similarly to the
+                    <link linkend='var-BB_NUMBER_THREADS'><filename>BB_NUMBER_THREADS</filename></link>
+                    variable but is task-specific.</para>
+
+                    <para>Set the value globally.
+                    For example, the following makes sure the
+                    <filename>do_fetch</filename> task uses no more than two
+                    simultaneous execution threads:
+                    <literallayout class='monospaced'>
+     do_fetch[number_threads] = "2"
+                    </literallayout>
+                    <note><title>Warnings</title>
+                        <itemizedlist>
+                            <listitem><para>
+                                Setting the varflag in individual recipes rather
+                                than globally can result in unpredictable behavior.
+                                </para></listitem>
+                            <listitem><para>
+                                Setting the varflag to a value greater than the
+                                value used in the <filename>BB_NUMBER_THREADS</filename>
+                                variable causes <filename>number_threads</filename>
+                                to have no effect.
+                                </para></listitem>
+                        </itemizedlist>
+                    </note>
+                    </para></listitem>
                 <listitem><para><emphasis><filename>[postfuncs]</filename>:</emphasis>
                     List of functions to call after the completion of the task.
                     </para></listitem>
@@ -2132,6 +2164,8 @@
                 <listitem><para>
                     <filename>bb.event.BuildStarted()</filename>:
                     Fired when a new build starts.
+                    BitBake fires multiple "BuildStarted" events (one per configuration)
+                    when multiple configuration (multiconfig) is enabled.
                     </para></listitem>
                 <listitem><para>
                     <filename>bb.build.TaskStarted()</filename>:
@@ -2650,47 +2684,70 @@
         </para>
 
         <para>
-            This list is a place holder of content existed from previous work
-            on the manual.
-            Some or all of it probably needs integrated into the subsections
-            that make up this section.
-            For now, I have just provided a short glossary-like description
-            for each variable.
-            Ultimately, this list goes away.
+            These checksums are stored in
+            <link linkend='var-STAMP'><filename>STAMP</filename></link>.
+            You can examine the checksums using the following BitBake command:
+            <literallayout class='monospaced'>
+     $ bitbake-dumpsigs
+            </literallayout>
+            This command returns the signature data in a readable format
+            that allows you to examine the inputs used when the
+            OpenEmbedded build system generates signatures.
+            For example, using <filename>bitbake-dumpsigs</filename>
+            allows you to examine the <filename>do_compile</filename>
+            task's “sigdata” for a C application (e.g.
+            <filename>bash</filename>).
+            Running the command also reveals that the “CC” variable is part of
+            the inputs that are hashed.
+            Any changes to this variable would invalidate the stamp and
+            cause the <filename>do_compile</filename> task to run.
+        </para>
+
+        <para>
+            The following list describes related variables:
             <itemizedlist>
-                <listitem><para><filename>STAMP</filename>:
-                    The base path to create stamp files.</para></listitem>
-                <listitem><para><filename>STAMPCLEAN</filename>
-                    Again, the base path to create stamp files but can use wildcards
-                    for matching a range of files for clean operations.
-                    </para></listitem>
-                <listitem><para><filename>BB_STAMP_WHITELIST</filename>
-                    Lists stamp files that are looked at when the stamp policy
-                    is "whitelist".
-                    </para></listitem>
-                <listitem><para><filename>BB_STAMP_POLICY</filename>
-                    Defines the mode for comparing timestamps of stamp files.
-                    </para></listitem>
-                <listitem><para><filename>BB_HASHCHECK_FUNCTION</filename>
+                <listitem><para>
+                    <link linkend='var-BB_HASHCHECK_FUNCTION'><filename>BB_HASHCHECK_FUNCTION</filename></link>:
                     Specifies the name of the function to call during
                     the "setscene" part of the task's execution in order
                     to validate the list of task hashes.
                     </para></listitem>
-                <listitem><para><filename>BB_SETSCENE_VERIFY_FUNCTION2</filename>
-                    Specifies a function to call that verifies the list of
-                    planned task execution before the main task execution
-                    happens.
-                    </para></listitem>
-                <listitem><para><filename>BB_SETSCENE_DEPVALID</filename>
+                <listitem><para>
+                    <link linkend='var-BB_SETSCENE_DEPVALID'><filename>BB_SETSCENE_DEPVALID</filename></link>:
                     Specifies a function BitBake calls that determines
                     whether BitBake requires a setscene dependency to
                     be met.
                     </para></listitem>
-                <listitem><para><filename>BB_TASKHASH</filename>
+                <listitem><para>
+                    <link linkend='var-BB_SETSCENE_VERIFY_FUNCTION2'><filename>BB_SETSCENE_VERIFY_FUNCTION2</filename></link>:
+                    Specifies a function to call that verifies the list of
+                    planned task execution before the main task execution
+                    happens.
+                    </para></listitem>
+                <listitem><para>
+                    <link linkend='var-BB_STAMP_POLICY'><filename>BB_STAMP_POLICY</filename></link>:
+                    Defines the mode for comparing timestamps of stamp files.
+                    </para></listitem>
+                <listitem><para>
+                    <link linkend='var-BB_STAMP_WHITELIST'><filename>BB_STAMP_WHITELIST</filename></link>:
+                    Lists stamp files that are looked at when the stamp policy
+                    is "whitelist".
+                    </para></listitem>
+                <listitem><para>
+                    <link linkend='var-BB_TASKHASH'><filename>BB_TASKHASH</filename></link>:
                     Within an executing task, this variable holds the hash
                     of the task as returned by the currently enabled
                     signature generator.
                     </para></listitem>
+                <listitem><para>
+                    <link linkend='var-STAMP'><filename>STAMP</filename></link>:
+                    The base path to create stamp files.
+                    </para></listitem>
+                <listitem><para>
+                    <link linkend='var-STAMPCLEAN'><filename>STAMPCLEAN</filename></link>:
+                    Again, the base path to create stamp files but can use wildcards
+                    for matching a range of files for clean operations.
+                    </para></listitem>
             </itemizedlist>
         </para>
     </section>

bitbake/doc/bitbake-user-manual/bitbake-user-manual-ref-variables.xml

@@ -78,7 +78,7 @@
                 </para>
 
                 <para>
-                    In OpenEmbedded Core, <filename>ASSUME_PROVIDED</filename>
+                    In OpenEmbedded-Core, <filename>ASSUME_PROVIDED</filename>
                     mostly specifies native tools that should not be built.
                     An example is <filename>git-native</filename>, which
                     when specified allows for the Git binary from the host to
@@ -964,7 +964,7 @@
                     Allows you to extend a recipe so that it builds variants
                     of the software.
                     Some examples of these variants for recipes from the
-                    OpenEmbedded Core metadata are "natives" such as
+                    OpenEmbedded-Core metadata are "natives" such as
                     <filename>quilt-native</filename>, which is a copy of
                     Quilt built to run on the build system; "crosses" such
                     as <filename>gcc-cross</filename>, which is a compiler
@@ -980,7 +980,7 @@
                     amount of code, it usually is as simple as adding the
                     variable to your recipe.
                     Here are two examples.
-                    The "native" variants are from the OpenEmbedded Core
+                    The "native" variants are from the OpenEmbedded-Core
                     metadata:
                     <literallayout class='monospaced'>
      BBCLASSEXTEND =+ "native nativesdk"
@@ -2089,6 +2089,16 @@
             </glossdef>
         </glossentry>
 
+        <glossentry id='var-REPODIR'><glossterm>REPODIR</glossterm>
+            <glossdef>
+                <para>
+                    The directory in which a local copy of a
+                    <filename>google-repo</filename> directory is stored
+                    when it is synced.
+                </para>
+            </glossdef>
+        </glossentry>
+
         <glossentry id='var-RPROVIDES'><glossterm>RPROVIDES</glossterm>
             <glossdef>
                 <para>

bitbake/doc/bitbake-user-manual/bitbake-user-manual.xml

@@ -56,7 +56,7 @@
 -->
 
         <copyright>
-            <year>2004-2016</year>
+            <year>2004-2018</year>
             <holder>Richard Purdie</holder>
             <holder>Chris Larson</holder>
             <holder>and Phil Blundell</holder>

bitbake/lib/bb/__init__.py

@@ -21,7 +21,7 @@
 # with this program; if not, write to the Free Software Foundation, Inc.,
 # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
 
-__version__ = "1.37.0"
+__version__ = "1.39.0"
 
 import sys
 if sys.version_info < (3, 4, 0):

bitbake/lib/bb/cache.py

@@ -395,7 +395,7 @@ class Cache(NoCache):
         self.has_cache = True
         self.cachefile = getCacheFile(self.cachedir, "bb_cache.dat", self.data_hash)
 
-        logger.debug(1, "Using cache in '%s'", self.cachedir)
+        logger.debug(1, "Cache dir: %s", self.cachedir)
         bb.utils.mkdirhier(self.cachedir)
 
         cache_ok = True
@@ -408,6 +408,8 @@ class Cache(NoCache):
             self.load_cachefile()
         elif os.path.isfile(self.cachefile):
             logger.info("Out of date cache found, rebuilding...")
+        else:
+            logger.debug(1, "Cache file %s not found, building..." % self.cachefile)
 
     def load_cachefile(self):
         cachesize = 0
@@ -424,6 +426,7 @@ class Cache(NoCache):
 
         for cache_class in self.caches_array:
             cachefile = getCacheFile(self.cachedir, cache_class.cachefile, self.data_hash)
+            logger.debug(1, 'Loading cache file: %s' % cachefile)
             with open(cachefile, "rb") as cachefile:
                 pickled = pickle.Unpickler(cachefile)
                 # Check cache version information

bitbake/lib/bb/cooker.py

@@ -516,6 +516,8 @@ class BBCooker:
             fn = runlist[0][3]
         else:
             envdata = self.data
+            data.expandKeys(envdata)
+            parse.ast.runAnonFuncs(envdata)
 
         if fn:
             try:
@@ -536,7 +538,6 @@ class BBCooker:
             logger.plain(env.getvalue())
 
         # emit the metadata which isnt valid shell
-        data.expandKeys(envdata)
         for e in sorted(envdata.keys()):
             if envdata.getVarFlag(e, 'func', False) and envdata.getVarFlag(e, 'python', False):
                 logger.plain("\npython %s () {\n%s}\n", e, envdata.getVar(e, False))
@@ -856,12 +857,12 @@ class BBCooker:
 
         with open('task-depends.dot', 'w') as f:
             f.write("digraph depends {\n")
-            for task in depgraph["tdepends"]:
+            for task in sorted(depgraph["tdepends"]):
                 (pn, taskname) = task.rsplit(".", 1)
                 fn = depgraph["pn"][pn]["filename"]
                 version = depgraph["pn"][pn]["version"]
                 f.write('"%s.%s" [label="%s %s\\n%s\\n%s"]\n' % (pn, taskname, pn, taskname, version, fn))
-                for dep in depgraph["tdepends"][task]:
+                for dep in sorted(depgraph["tdepends"][task]):
                     f.write('"%s" -> "%s"\n' % (task, dep))
             f.write("}\n")
         logger.info("Task dependencies saved to 'task-depends.dot'")
@@ -869,18 +870,18 @@ class BBCooker:
         with open('recipe-depends.dot', 'w') as f:
             f.write("digraph depends {\n")
             pndeps = {}
-            for task in depgraph["tdepends"]:
+            for task in sorted(depgraph["tdepends"]):
                 (pn, taskname) = task.rsplit(".", 1)
                 if pn not in pndeps:
                     pndeps[pn] = set()
-                for dep in depgraph["tdepends"][task]:
+                for dep in sorted(depgraph["tdepends"][task]):
                     (deppn, deptaskname) = dep.rsplit(".", 1)
                     pndeps[pn].add(deppn)
-            for pn in pndeps:
+            for pn in sorted(pndeps):
                 fn = depgraph["pn"][pn]["filename"]
                 version = depgraph["pn"][pn]["version"]
                 f.write('"%s" [label="%s\\n%s\\n%s"]\n' % (pn, pn, version, fn))
-                for dep in pndeps[pn]:
+                for dep in sorted(pndeps[pn]):
                     if dep == pn:
                         continue
                     f.write('"%s" -> "%s"\n' % (pn, dep))
@@ -1170,6 +1171,7 @@ class BBCooker:
                 elif regex == "":
                     parselog.debug(1, "BBFILE_PATTERN_%s is empty" % c)
                     errors = False
+                    continue
                 else:
                     try:
                         cre = re.compile(regex)
@@ -1603,8 +1605,6 @@ class BBCooker:
 
         if self.parser:
             self.parser.shutdown(clean=not force, force=force)
-        self.notifier.stop()
-        self.confignotifier.stop()
 
     def finishcommand(self):
         self.state = state.initial
@@ -1807,21 +1807,25 @@ class CookerCollectFiles(object):
             realfn, cls, mc = bb.cache.virtualfn2realfn(p)
             priorities[p] = self.calc_bbfile_priority(realfn, matched)
 
-        # Don't show the warning if the BBFILE_PATTERN did match .bbappend files
         unmatched = set()
         for _, _, regex, pri in self.bbfile_config_priorities:
             if not regex in matched:
                 unmatched.add(regex)
 
-        def findmatch(regex):
+        # Don't show the warning if the BBFILE_PATTERN did match .bbappend files
+        def find_bbappend_match(regex):
             for b in self.bbappends:
                 (bbfile, append) = b
                 if regex.match(append):
+                    # If the bbappend is matched by already "matched set", return False
+                    for matched_regex in matched:
+                        if matched_regex.match(append):
+                            return False
                     return True
             return False
 
         for unmatch in unmatched.copy():
-            if findmatch(unmatch):
+            if find_bbappend_match(unmatch):
                 unmatched.remove(unmatch)
 
         for collection, pattern, regex, _ in self.bbfile_config_priorities:

bitbake/lib/bb/cookerdata.py

@@ -143,7 +143,8 @@ class CookerConfiguration(object):
         self.writeeventlog = False
         self.server_only = False
         self.limited_deps = False
-        self.runall = None
+        self.runall = []
+        self.runonly = []
 
         self.env = {}
 
@@ -395,6 +396,8 @@ class CookerDataBuilder(object):
                 if compat and not (compat & layerseries):
                     bb.fatal("Layer %s is not compatible with the core layer which only supports these series: %s (layer is compatible with %s)"
                               % (c, " ".join(layerseries), " ".join(compat)))
+                elif not compat and not data.getVar("BB_WORKERCONTEXT"):
+                    bb.warn("Layer %s should set LAYERSERIES_COMPAT_%s in its conf/layer.conf file to list the core layer names it is compatible with." % (c, c))
 
         if not data.getVar("BBPATH"):
             msg = "The BBPATH variable is not set"

bitbake/lib/bb/event.py

@@ -449,12 +449,6 @@ class BuildBase(Event):
     def setName(self, name):
         self._name = name
 
-    def getCfg(self):
-        return self.data
-
-    def setCfg(self, cfg):
-        self.data = cfg
-
     def getFailures(self):
         """
         Return the number of failed packages
@@ -463,9 +457,6 @@ class BuildBase(Event):
 
     pkgs = property(getPkgs, setPkgs, None, "pkgs property")
     name = property(getName, setName, None, "name property")
-    cfg = property(getCfg, setCfg, None, "cfg property")
-
-
 
 class BuildInit(BuildBase):
     """buildFile or buildTargets was invoked"""

bitbake/lib/bb/fetch2/__init__.py

@@ -383,7 +383,7 @@ def decodeurl(url):
         path = location
     else:
         host = location
-        path = ""
+        path = "/"
     if user:
         m = re.compile('(?P<user>[^:]+)(:?(?P<pswd>.*))').match(user)
         if m:
@@ -452,8 +452,8 @@ def uri_replace(ud, uri_find, uri_replace, replacements, d, mirrortarball=None):
             # Handle URL parameters
             if i:
                 # Any specified URL parameters must match
-                for k in uri_replace_decoded[loc]:
-                    if uri_decoded[loc][k] != uri_replace_decoded[loc][k]:
+                for k in uri_find_decoded[loc]:
+                    if uri_decoded[loc][k] != uri_find_decoded[loc][k]:
                         return None
             # Overwrite any specified replacement parameters
             for k in uri_replace_decoded[loc]:
@@ -643,26 +643,25 @@ def verify_donestamp(ud, d, origud=None):
     if not ud.needdonestamp or (origud and not origud.needdonestamp):
         return True
 
-    if not os.path.exists(ud.donestamp):
+    if not os.path.exists(ud.localpath):
+        # local path does not exist
+        if os.path.exists(ud.donestamp):
+            # done stamp exists, but the downloaded file does not; the done stamp
+            # must be incorrect, re-trigger the download
+            bb.utils.remove(ud.donestamp)
         return False
 
     if (not ud.method.supports_checksum(ud) or
         (origud and not origud.method.supports_checksum(origud))):
-        # done stamp exists, checksums not supported; assume the local file is
-        # current
-        return True
-
-    if not os.path.exists(ud.localpath):
-        # done stamp exists, but the downloaded file does not; the done stamp
-        # must be incorrect, re-trigger the download
-        bb.utils.remove(ud.donestamp)
-        return False
+        # if done stamp exists and checksums not supported; assume the local
+        # file is current
+        return os.path.exists(ud.donestamp)
 
     precomputed_checksums = {}
     # Only re-use the precomputed checksums if the donestamp is newer than the
     # file. Do not rely on the mtime of directories, though. If ud.localpath is
     # a directory, there will probably not be any checksums anyway.
-    if (os.path.isdir(ud.localpath) or
+    if os.path.exists(ud.donestamp) and (os.path.isdir(ud.localpath) or
             os.path.getmtime(ud.localpath) < os.path.getmtime(ud.donestamp)):
         try:
             with open(ud.donestamp, "rb") as cachefile:
@@ -853,6 +852,9 @@ def runfetchcmd(cmd, d, quiet=False, cleanup=None, log=None, workdir=None):
         if val:
             cmd = 'export ' + var + '=\"%s\"; %s' % (val, cmd)
 
+    # Disable pseudo as it may affect ssh, potentially causing it to hang.
+    cmd = 'export PSEUDO_DISABLED=1; ' + cmd
+
     logger.debug(1, "Running %s", cmd)
 
     success = False

bitbake/lib/bb/fetch2/bzr.py

@@ -41,8 +41,9 @@ class Bzr(FetchMethod):
         init bzr specific variable within url data
         """
         # Create paths to bzr checkouts
+        bzrdir = d.getVar("BZRDIR") or (d.getVar("DL_DIR") + "/bzr")
         relpath = self._strip_leading_slashes(ud.path)
-        ud.pkgdir = os.path.join(d.expand('${BZRDIR}'), ud.host, relpath)
+        ud.pkgdir = os.path.join(bzrdir, ud.host, relpath)
 
         ud.setup_revisions(d)
 
@@ -57,7 +58,7 @@ class Bzr(FetchMethod):
         command is "fetch", "update", "revno"
         """
 
-        basecmd = d.expand('${FETCHCMD_bzr}')
+        basecmd = d.getVar("FETCHCMD_bzr") or "/usr/bin/env bzr"
 
         proto =  ud.parm.get('protocol', 'http')
 

bitbake/lib/bb/fetch2/clearcase.py

@@ -69,7 +69,6 @@ from   bb.fetch2 import FetchMethod
 from   bb.fetch2 import FetchError
 from   bb.fetch2 import runfetchcmd
 from   bb.fetch2 import logger
-from   distutils import spawn
 
 class ClearCase(FetchMethod):
     """Class to fetch urls via 'clearcase'"""
@@ -107,7 +106,7 @@ class ClearCase(FetchMethod):
         else:
             ud.module = ""
 
-        ud.basecmd = d.getVar("FETCHCMD_ccrc") or spawn.find_executable("cleartool") or spawn.find_executable("rcleartool")
+        ud.basecmd = d.getVar("FETCHCMD_ccrc") or "/usr/bin/env cleartool || rcleartool"
 
         if d.getVar("SRCREV") == "INVALID":
           raise FetchError("Set a valid SRCREV for the clearcase fetcher in your recipe, e.g. SRCREV = \"/main/LATEST\" or any other label of your choice.")

bitbake/lib/bb/fetch2/cvs.py

@@ -110,7 +110,7 @@ class Cvs(FetchMethod):
         if ud.tag:
             options.append("-r %s" % ud.tag)
 
-        cvsbasecmd = d.getVar("FETCHCMD_cvs")
+        cvsbasecmd = d.getVar("FETCHCMD_cvs") or "/usr/bin/env cvs"
         cvscmd = cvsbasecmd + " '-d" + cvsroot + "' co " + " ".join(options) + " " + ud.module
         cvsupdatecmd = cvsbasecmd + " '-d" + cvsroot + "' update -d -P " + " ".join(options)
 
@@ -121,7 +121,8 @@ class Cvs(FetchMethod):
         # create module directory
         logger.debug(2, "Fetch: checking for module directory")
         pkg = d.getVar('PN')
-        pkgdir = os.path.join(d.getVar('CVSDIR'), pkg)
+        cvsdir = d.getVar("CVSDIR") or (d.getVar("DL_DIR") + "/cvs")
+        pkgdir = os.path.join(cvsdir, pkg)
         moddir = os.path.join(pkgdir, localdir)
         workdir = None
         if os.access(os.path.join(moddir, 'CVS'), os.R_OK):

bitbake/lib/bb/fetch2/git.py

@@ -125,6 +125,9 @@ class GitProgressHandler(bb.progress.LineFilterProgressHandler):
 
 
 class Git(FetchMethod):
+    bitbake_dir = os.path.abspath(os.path.join(os.path.dirname(os.path.join(os.path.abspath(__file__))), '..', '..', '..'))
+    make_shallow_path = os.path.join(bitbake_dir, 'bin', 'git-make-shallow')
+
     """Class to fetch a module or modules from git repositories"""
     def init(self, d):
         pass
@@ -258,7 +261,7 @@ class Git(FetchMethod):
                 gitsrcname = gitsrcname + '_' + ud.revisions[name]
 
         dl_dir = d.getVar("DL_DIR")
-        gitdir = d.getVar("GITDIR") or (dl_dir + "/git2/")
+        gitdir = d.getVar("GITDIR") or (dl_dir + "/git2")
         ud.clonedir = os.path.join(gitdir, gitsrcname)
         ud.localfile = ud.clonedir
 
@@ -319,16 +322,13 @@ class Git(FetchMethod):
     def download(self, ud, d):
         """Fetch url"""
 
-        no_clone = not os.path.exists(ud.clonedir)
-        need_update = no_clone or self.need_update(ud, d)
-
         # A current clone is preferred to either tarball, a shallow tarball is
         # preferred to an out of date clone, and a missing clone will use
         # either tarball.
-        if ud.shallow and os.path.exists(ud.fullshallow) and need_update:
+        if ud.shallow and os.path.exists(ud.fullshallow) and self.need_update(ud, d):
             ud.localpath = ud.fullshallow
             return
-        elif os.path.exists(ud.fullmirror) and no_clone:
+        elif os.path.exists(ud.fullmirror) and not os.path.exists(ud.clonedir):
             bb.utils.mkdirhier(ud.clonedir)
             runfetchcmd("tar -xzf %s" % ud.fullmirror, d, workdir=ud.clonedir)
 
@@ -350,6 +350,8 @@ class Git(FetchMethod):
         for name in ud.names:
             if not self._contains_ref(ud, d, name, ud.clonedir):
                 needupdate = True
+                break
+
         if needupdate:
             try: 
                 runfetchcmd("%s remote rm origin" % ud.basecmd, d, workdir=ud.clonedir)
@@ -363,12 +365,14 @@ class Git(FetchMethod):
             progresshandler = GitProgressHandler(d)
             runfetchcmd(fetch_cmd, d, log=progresshandler, workdir=ud.clonedir)
             runfetchcmd("%s prune-packed" % ud.basecmd, d, workdir=ud.clonedir)
+            runfetchcmd("%s pack-refs --all" % ud.basecmd, d, workdir=ud.clonedir)
             runfetchcmd("%s pack-redundant --all | xargs -r rm" % ud.basecmd, d, workdir=ud.clonedir)
             try:
                 os.unlink(ud.fullmirror)
             except OSError as exc:
                 if exc.errno != errno.ENOENT:
                     raise
+
         for name in ud.names:
             if not self._contains_ref(ud, d, name, ud.clonedir):
                 raise bb.fetch2.FetchError("Unable to find revision %s in branch %s even from upstream" % (ud.revisions[name], ud.branches[name]))
@@ -445,7 +449,7 @@ class Git(FetchMethod):
                 shallow_branches.append(r)
 
         # Make the repository shallow
-        shallow_cmd = ['git', 'make-shallow', '-s']
+        shallow_cmd = [self.make_shallow_path, '-s']
         for b in shallow_branches:
             shallow_cmd.append('-r')
             shallow_cmd.append(b)
@@ -468,7 +472,7 @@ class Git(FetchMethod):
         if os.path.exists(destdir):
             bb.utils.prunedir(destdir)
 
-        if ud.shallow and (not os.path.exists(ud.clonedir) or self.need_update(ud, d)):
+        if ud.shallow and self.need_update(ud, d):
             bb.utils.mkdirhier(destdir)
             runfetchcmd("tar -xzf %s" % ud.fullshallow, d, workdir=destdir)
         else:
@@ -591,7 +595,8 @@ class Git(FetchMethod):
         tagregex = re.compile(d.getVar('UPSTREAM_CHECK_GITTAGREGEX') or "(?P<pver>([0-9][\.|_]?)+)")
         try:
             output = self._lsremote(ud, d, "refs/tags/*")
-        except bb.fetch2.FetchError or bb.fetch2.NetworkAccess:
+        except (bb.fetch2.FetchError, bb.fetch2.NetworkAccess) as e:
+            bb.note("Could not list remote: %s" % str(e))
             return pupver
 
         verstring = ""

bitbake/lib/bb/fetch2/gitsm.py

@@ -98,7 +98,7 @@ class GitSM(Git):
                         for line in lines:
                             f.write(line)
 
-    def update_submodules(self, ud, d):
+    def update_submodules(self, ud, d, allow_network):
         # We have to convert bare -> full repo, do the submodule bit, then convert back
         tmpclonedir = ud.clonedir + ".tmp"
         gitdir = tmpclonedir + os.sep + ".git"
@@ -108,11 +108,46 @@ class GitSM(Git):
         runfetchcmd("sed " + gitdir + "/config -i -e 's/bare.*=.*true/bare = false/'", d)
         runfetchcmd(ud.basecmd + " reset --hard", d, workdir=tmpclonedir)
         runfetchcmd(ud.basecmd + " checkout -f " + ud.revisions[ud.names[0]], d, workdir=tmpclonedir)
-        runfetchcmd(ud.basecmd + " submodule update --init --recursive", d, workdir=tmpclonedir)
-        self._set_relative_paths(tmpclonedir)
-        runfetchcmd("sed " + gitdir + "/config -i -e 's/bare.*=.*false/bare = true/'", d, workdir=tmpclonedir)
-        os.rename(gitdir, ud.clonedir,)
-        bb.utils.remove(tmpclonedir, True)
+
+        try:
+            if allow_network:
+                fetch_flags = ""
+            else:
+                fetch_flags = "--no-fetch"
+
+            # The 'git submodule sync' sandwiched between two successive 'git submodule update' commands is
+            # intentional. See the notes on the similar construction in download() for an explanation.
+            runfetchcmd("%(basecmd)s submodule update --init --recursive %(fetch_flags)s || (%(basecmd)s submodule sync --recursive && %(basecmd)s submodule update --init --recursive %(fetch_flags)s)" % {'basecmd': ud.basecmd, 'fetch_flags' : fetch_flags}, d, workdir=tmpclonedir)
+        except bb.fetch.FetchError:
+            if allow_network:
+                raise
+            else:
+                # This method was called as a probe to see whether the submodule history
+                # is complete enough to allow the current working copy to have its
+                # modules filled in. It's not, so swallow up the exception and report
+                # the negative result.
+                return False
+        finally:
+            self._set_relative_paths(tmpclonedir)
+            runfetchcmd("sed " + gitdir + "/config -i -e 's/bare.*=.*false/bare = true/'", d, workdir=tmpclonedir)
+            os.rename(gitdir, ud.clonedir,)
+            bb.utils.remove(tmpclonedir, True)
+
+        return True
+
+    def need_update(self, ud, d):
+        main_repo_needs_update = Git.need_update(self, ud, d)
+
+        # First check that the main repository has enough history fetched. If it doesn't, then we don't
+        # even have the .gitmodules and gitlinks for the submodules to attempt asking whether the
+        # submodules' histories are recent enough.
+        if main_repo_needs_update:
+            return True
+
+        # Now check that the submodule histories are new enough. The git-submodule command doesn't have
+        # any clean interface for doing this aside from just attempting the checkout (with network
+        # fetched disabled).
+        return not self.update_submodules(ud, d, allow_network=False)
 
     def download(self, ud, d):
         Git.download(self, ud, d)
@@ -120,7 +155,7 @@ class GitSM(Git):
         if not ud.shallow or ud.localpath != ud.fullshallow:
             submodules = self.uses_submodules(ud, d, ud.clonedir)
             if submodules:
-                self.update_submodules(ud, d)
+                self.update_submodules(ud, d, allow_network=True)
 
     def clone_shallow_local(self, ud, dest, d):
         super(GitSM, self).clone_shallow_local(ud, dest, d)
@@ -132,4 +167,36 @@ class GitSM(Git):
 
         if self.uses_submodules(ud, d, ud.destdir):
             runfetchcmd(ud.basecmd + " checkout " + ud.revisions[ud.names[0]], d, workdir=ud.destdir)
-            runfetchcmd(ud.basecmd + " submodule update --init --recursive", d, workdir=ud.destdir)
+
+            # Copy over the submodules' fetched histories too.
+            if ud.bareclone:
+                repo_conf = ud.destdir
+            else:
+                repo_conf = os.path.join(ud.destdir, '.git')
+
+            if os.path.exists(ud.clonedir):
+                # This is not a copy unpacked from a shallow mirror clone. So
+                # the manual intervention to populate the .git/modules done
+                # in clone_shallow_local() won't have been done yet.
+                runfetchcmd("cp -fpPRH %s %s" % (os.path.join(ud.clonedir, 'modules'), repo_conf), d)
+                fetch_flags = "--no-fetch"
+            elif os.path.exists(os.path.join(repo_conf, 'modules')):
+                # Unpacked from a shallow mirror clone. Manual population of
+                # .git/modules is already done.
+                fetch_flags = "--no-fetch"
+            else:
+                # This isn't fatal; git-submodule will just fetch it
+                # during do_unpack().
+                fetch_flags = ""
+                bb.error("submodule history not retrieved during do_fetch()")
+
+            # Careful not to hit the network during unpacking; all history should already
+            # be fetched.
+            #
+            # The repeated attempts to do the submodule initialization sandwiched around a sync to
+            # install the correct remote URLs into the submodules' .git/config metadata are deliberate.
+            # Bad remote URLs are leftover in the modules' .git/config files from the unpack of bare
+            # clone tarballs and an initial 'git submodule update' is necessary to prod them back to
+            # enough life so that the 'git submodule sync' realizes the existing module .git/config
+            # files exist to be updated.
+            runfetchcmd("%(basecmd)s submodule update --init --recursive %(fetch_flags)s || (%(basecmd)s submodule sync --recursive && %(basecmd)s submodule update --init --recursive %(fetch_flags)s)" % {'basecmd': ud.basecmd, 'fetch_flags': fetch_flags}, d, workdir=ud.destdir)

bitbake/lib/bb/fetch2/hg.py

@@ -80,7 +80,7 @@ class Hg(FetchMethod):
         ud.fullmirror = os.path.join(d.getVar("DL_DIR"), mirrortarball)
         ud.mirrortarballs = [mirrortarball]
 
-        hgdir = d.getVar("HGDIR") or (d.getVar("DL_DIR") + "/hg/")
+        hgdir = d.getVar("HGDIR") or (d.getVar("DL_DIR") + "/hg")
         ud.pkgdir = os.path.join(hgdir, hgsrcname)
         ud.moddir = os.path.join(ud.pkgdir, ud.module)
         ud.localfile = ud.moddir

bitbake/lib/bb/fetch2/npm.py

@@ -32,7 +32,6 @@ from   bb.fetch2 import runfetchcmd
 from   bb.fetch2 import logger
 from   bb.fetch2 import UnpackError
 from   bb.fetch2 import ParameterError
-from   distutils import spawn
 
 def subprocess_setup():
     # Python installs a SIGPIPE handler by default. This is usually not what
@@ -195,9 +194,11 @@ class Npm(FetchMethod):
         outputurl = pdata['dist']['tarball']
         data[pkg] = {}
         data[pkg]['tgz'] = os.path.basename(outputurl)
-        if not outputurl in fetchedlist:
-            self._runwget(ud, d, "%s --directory-prefix=%s %s" % (self.basecmd, ud.prefixdir, outputurl), False)
-            fetchedlist.append(outputurl)
+        if outputurl in fetchedlist:
+            return
+
+        self._runwget(ud, d, "%s --directory-prefix=%s %s" % (self.basecmd, ud.prefixdir, outputurl), False)
+        fetchedlist.append(outputurl)
 
         dependencies = pdata.get('dependencies', {})
         optionalDependencies = pdata.get('optionalDependencies', {})

bitbake/lib/bb/fetch2/osc.py

@@ -32,8 +32,9 @@ class Osc(FetchMethod):
         ud.module = ud.parm["module"]
 
         # Create paths to osc checkouts
+        oscdir = d.getVar("OSCDIR") or (d.getVar("DL_DIR") + "/osc")
         relpath = self._strip_leading_slashes(ud.path)
-        ud.pkgdir = os.path.join(d.getVar('OSCDIR'), ud.host)
+        ud.pkgdir = os.path.join(oscdir, ud.host)
         ud.moddir = os.path.join(ud.pkgdir, relpath, ud.module)
 
         if 'rev' in ud.parm:
@@ -54,7 +55,7 @@ class Osc(FetchMethod):
         command is "fetch", "update", "info"
         """
 
-        basecmd = d.expand('${FETCHCMD_osc}')
+        basecmd = d.getVar("FETCHCMD_osc") or "/usr/bin/env osc"
 
         proto = ud.parm.get('protocol', 'ocs')
 

bitbake/lib/bb/fetch2/perforce.py

@@ -43,13 +43,9 @@ class Perforce(FetchMethod):
         provided by the env, use it.  If P4PORT is specified by the recipe, use
         its values, which may override the settings in P4CONFIG.
         """
-        ud.basecmd = d.getVar('FETCHCMD_p4')
-        if not ud.basecmd:
-            ud.basecmd = "/usr/bin/env p4"
+        ud.basecmd = d.getVar("FETCHCMD_p4") or "/usr/bin/env p4"
 
-        ud.dldir = d.getVar('P4DIR')
-        if not ud.dldir:
-            ud.dldir = '%s/%s' % (d.getVar('DL_DIR'), 'p4')
+        ud.dldir = d.getVar("P4DIR") or (d.getVar("DL_DIR") + "/p4")
 
         path = ud.url.split('://')[1]
         path = path.split(';')[0]

bitbake/lib/bb/fetch2/repo.py

@@ -45,6 +45,8 @@ class Repo(FetchMethod):
         "master".
         """
 
+        ud.basecmd = d.getVar("FETCHCMD_repo") or "/usr/bin/env repo"
+
         ud.proto = ud.parm.get('protocol', 'git')
         ud.branch = ud.parm.get('branch', 'master')
         ud.manifest = ud.parm.get('manifest', 'default.xml')
@@ -60,8 +62,8 @@ class Repo(FetchMethod):
             logger.debug(1, "%s already exists (or was stashed). Skipping repo init / sync.", ud.localpath)
             return
 
+        repodir = d.getVar("REPODIR") or (d.getVar("DL_DIR") + "/repo")
         gitsrcname = "%s%s" % (ud.host, ud.path.replace("/", "."))
-        repodir = d.getVar("REPODIR") or os.path.join(d.getVar("DL_DIR"), "repo")
         codir = os.path.join(repodir, gitsrcname, ud.manifest)
 
         if ud.user:
@@ -72,11 +74,11 @@ class Repo(FetchMethod):
         repodir = os.path.join(codir, "repo")
         bb.utils.mkdirhier(repodir)
         if not os.path.exists(os.path.join(repodir, ".repo")):
-            bb.fetch2.check_network_access(d, "repo init -m %s -b %s -u %s://%s%s%s" % (ud.manifest, ud.branch, ud.proto, username, ud.host, ud.path), ud.url)
-            runfetchcmd("repo init -m %s -b %s -u %s://%s%s%s" % (ud.manifest, ud.branch, ud.proto, username, ud.host, ud.path), d, workdir=repodir)
+            bb.fetch2.check_network_access(d, "%s init -m %s -b %s -u %s://%s%s%s" % (ud.basecmd, ud.manifest, ud.branch, ud.proto, username, ud.host, ud.path), ud.url)
+            runfetchcmd("%s init -m %s -b %s -u %s://%s%s%s" % (ud.basecmd, ud.manifest, ud.branch, ud.proto, username, ud.host, ud.path), d, workdir=repodir)
 
-        bb.fetch2.check_network_access(d, "repo sync %s" % ud.url, ud.url)
-        runfetchcmd("repo sync", d, workdir=repodir)
+        bb.fetch2.check_network_access(d, "%s sync %s" % (ud.basecmd, ud.url), ud.url)
+        runfetchcmd("%s sync" % ud.basecmd, d, workdir=repodir)
 
         scmdata = ud.parm.get("scmdata", "")
         if scmdata == "keep":

bitbake/lib/bb/fetch2/svn.py

@@ -49,7 +49,7 @@ class Svn(FetchMethod):
         if not "module" in ud.parm:
             raise MissingParameterError('module', ud.url)
 
-        ud.basecmd = d.getVar('FETCHCMD_svn')
+        ud.basecmd = d.getVar("FETCHCMD_svn") or "/usr/bin/env svn --non-interactive --trust-server-cert"
 
         ud.module = ud.parm["module"]
 
@@ -59,8 +59,9 @@ class Svn(FetchMethod):
             ud.path_spec = ud.parm["path_spec"]
 
         # Create paths to svn checkouts
+        svndir = d.getVar("SVNDIR") or (d.getVar("DL_DIR") + "/svn")
         relpath = self._strip_leading_slashes(ud.path)
-        ud.pkgdir = os.path.join(d.expand('${SVNDIR}'), ud.host, relpath)
+        ud.pkgdir = os.path.join(svndir, ud.host, relpath)
         ud.moddir = os.path.join(ud.pkgdir, ud.module)
 
         ud.setup_revisions(d)

bitbake/lib/bb/main.py

@@ -292,8 +292,12 @@ class BitBakeConfigParameters(cookerdata.ConfigParameters):
                           help="Writes the event log of the build to a bitbake event json file. "
                                "Use '' (empty string) to assign the name automatically.")
 
-        parser.add_option("", "--runall", action="store", dest="runall",
-                          help="Run the specified task for all build targets and their dependencies.")
+        parser.add_option("", "--runall", action="append", dest="runall",
+                          help="Run the specified task for any recipe in the taskgraph of the specified target (even if it wouldn't otherwise have run).")
+
+        parser.add_option("", "--runonly", action="append", dest="runonly",
+                          help="Run only the specified task within the taskgraph of the specified targets (and any task dependencies those tasks may have).")
+
 
         options, targets = parser.parse_args(argv)
 

bitbake/lib/bb/parse/__init__.py

@@ -134,8 +134,9 @@ def resolve_file(fn, d):
         if not newfn:
             raise IOError(errno.ENOENT, "file %s not found in %s" % (fn, bbpath))
         fn = newfn
+    else:
+        mark_dependency(d, fn)
 
-    mark_dependency(d, fn)
     if not os.path.isfile(fn):
         raise IOError(errno.ENOENT, "file %s not found" % fn)
 

bitbake/lib/bb/parse/ast.py

@@ -335,6 +335,12 @@ def handleInherit(statements, filename, lineno, m):
     classes = m.group(1)
     statements.append(InheritNode(filename, lineno, classes))
 
+def runAnonFuncs(d):
+    code = []
+    for funcname in d.getVar("__BBANONFUNCS", False) or []:
+        code.append("%s(d)" % funcname)
+    bb.utils.better_exec("\n".join(code), {"d": d})
+
 def finalize(fn, d, variant = None):
     saved_handlers = bb.event.get_handlers().copy()
 
@@ -349,10 +355,7 @@ def finalize(fn, d, variant = None):
     bb.event.fire(bb.event.RecipePreFinalise(fn), d)
 
     bb.data.expandKeys(d)
-    code = []
-    for funcname in d.getVar("__BBANONFUNCS", False) or []:
-        code.append("%s(d)" % funcname)
-    bb.utils.better_exec("\n".join(code), {"d": d})
+    runAnonFuncs(d)
 
     tasklist = d.getVar('__BBTASKS', False) or []
     bb.event.fire(bb.event.RecipeTaskPreProcess(fn, list(tasklist)), d)

bitbake/lib/bb/parse/parse_py/BBHandler.py

@@ -131,9 +131,6 @@ def handle(fn, d, include):
 
     abs_fn = resolve_file(fn, d)
 
-    if include:
-        bb.parse.mark_dependency(d, abs_fn)
-
     # actual loading
     statements = get_statements(fn, abs_fn, base_name)
 

bitbake/lib/bb/parse/parse_py/ConfHandler.py

@@ -134,9 +134,6 @@ def handle(fn, data, include):
     abs_fn = resolve_file(fn, data)
     f = open(abs_fn, 'r')
 
-    if include:
-        bb.parse.mark_dependency(data, abs_fn)
-
     statements = ast.StatementGroup()
     lineno = 0
     while True:

bitbake/lib/bb/runqueue.py

@@ -94,13 +94,13 @@ class RunQueueStats:
         self.active = self.active - 1
         self.failed = self.failed + 1
 
-    def taskCompleted(self, number = 1):
-        self.active = self.active - number
-        self.completed = self.completed + number
+    def taskCompleted(self):
+        self.active = self.active - 1
+        self.completed = self.completed + 1
 
-    def taskSkipped(self, number = 1):
-        self.active = self.active + number
-        self.skipped = self.skipped + number
+    def taskSkipped(self):
+        self.active = self.active + 1
+        self.skipped = self.skipped + 1
 
     def taskActive(self):
         self.active = self.active + 1
@@ -134,6 +134,7 @@ class RunQueueScheduler(object):
         self.prio_map = [self.rqdata.runtaskentries.keys()]
 
         self.buildable = []
+        self.skip_maxthread = {}
         self.stamps = {}
         for tid in self.rqdata.runtaskentries:
             (mc, fn, taskname, taskfn) = split_tid_mcfn(tid)
@@ -150,8 +151,25 @@ class RunQueueScheduler(object):
         self.buildable = [x for x in self.buildable if x not in self.rq.runq_running]
         if not self.buildable:
             return None
+
+        # Filter out tasks that have a max number of threads that have been exceeded
+        skip_buildable = {}
+        for running in self.rq.runq_running.difference(self.rq.runq_complete):
+            rtaskname = taskname_from_tid(running)
+            if rtaskname not in self.skip_maxthread:
+                self.skip_maxthread[rtaskname] = self.rq.cfgData.getVarFlag(rtaskname, "number_threads")
+            if not self.skip_maxthread[rtaskname]:
+                continue
+            if rtaskname in skip_buildable:
+                skip_buildable[rtaskname] += 1
+            else:
+                skip_buildable[rtaskname] = 1
+
         if len(self.buildable) == 1:
             tid = self.buildable[0]
+            taskname = taskname_from_tid(tid)
+            if taskname in skip_buildable and skip_buildable[taskname] >= int(self.skip_maxthread[taskname]):
+                return None
             stamp = self.stamps[tid]
             if stamp not in self.rq.build_stamps.values():
                 return tid
@@ -164,6 +182,9 @@ class RunQueueScheduler(object):
         best = None
         bestprio = None
         for tid in self.buildable:
+            taskname = taskname_from_tid(tid)
+            if taskname in skip_buildable and skip_buildable[taskname] >= int(self.skip_maxthread[taskname]):
+                continue
             prio = self.rev_prio_map[tid]
             if bestprio is None or bestprio > prio:
                 stamp = self.stamps[tid]
@@ -581,12 +602,6 @@ class RunQueueData:
                     if t in taskData[mc].taskentries:
                         depends.add(t)
 
-        def add_resolved_dependencies(mc, fn, tasknames, depends):
-            for taskname in tasknames:
-                tid = build_tid(mc, fn, taskname)
-                if tid in self.runtaskentries:
-                    depends.add(tid)
-
         for mc in taskData:
             for tid in taskData[mc].taskentries:
 
@@ -673,57 +688,106 @@ class RunQueueData:
                             recursiveitasks[tid].append(newdep)
 
                 self.runtaskentries[tid].depends = depends
+                # Remove all self references
+                self.runtaskentries[tid].depends.discard(tid)
 
         #self.dump_data()
 
+        self.init_progress_reporter.next_stage()
+
         # Resolve recursive 'recrdeptask' dependencies (Part B)
         #
         # e.g. do_sometask[recrdeptask] = "do_someothertask"
         # (makes sure sometask runs after someothertask of all DEPENDS, RDEPENDS and intertask dependencies, recursively)
         # We need to do this separately since we need all of runtaskentries[*].depends to be complete before this is processed
-        self.init_progress_reporter.next_stage(len(recursivetasks))
-        extradeps = {}
-        for taskcounter, tid in enumerate(recursivetasks):
-            extradeps[tid] = set(self.runtaskentries[tid].depends)
 
-            tasknames = recursivetasks[tid]
-            seendeps = set()
+        # Generating/interating recursive lists of dependencies is painful and potentially slow
+        # Precompute recursive task dependencies here by:
+        #     a) create a temp list of reverse dependencies (revdeps)
+        #     b) walk up the ends of the chains (when a given task no longer has dependencies i.e. len(deps) == 0)
+        #     c) combine the total list of dependencies in cumulativedeps
+        #     d) optimise by pre-truncating 'task' off the items in cumulativedeps (keeps items in sets lower)
 
-            def generate_recdeps(t):
-                newdeps = set()
-                (mc, fn, taskname, _) = split_tid_mcfn(t)
-                add_resolved_dependencies(mc, fn, tasknames, newdeps)
-                extradeps[tid].update(newdeps)
-                seendeps.add(t)
-                newdeps.add(t)
-                for i in newdeps:
-                    if i not in self.runtaskentries:
-                        # Not all recipes might have the recrdeptask task as a task
-                        continue
-                    task = self.runtaskentries[i].task
-                    for n in self.runtaskentries[i].depends:
-                        if n not in seendeps:
-                             generate_recdeps(n)
-            generate_recdeps(tid)
-
-            if tid in recursiveitasks:
-                for dep in recursiveitasks[tid]:
-                    generate_recdeps(dep)
-            self.init_progress_reporter.update(taskcounter)
-
-        # Remove circular references so that do_a[recrdeptask] = "do_a do_b" can work
-        for tid in recursivetasks:
-            extradeps[tid].difference_update(recursivetasksselfref)
 
+        revdeps = {}
+        deps = {}
+        cumulativedeps = {}
         for tid in self.runtaskentries:
-            task = self.runtaskentries[tid].task
-            # Add in extra dependencies
-            if tid in extradeps:
-                 self.runtaskentries[tid].depends = extradeps[tid]
-            # Remove all self references
-            if tid in self.runtaskentries[tid].depends:
-                logger.debug(2, "Task %s contains self reference!", tid)
-                self.runtaskentries[tid].depends.remove(tid)
+            deps[tid] = set(self.runtaskentries[tid].depends)
+            revdeps[tid] = set()
+            cumulativedeps[tid] = set()
+        # Generate a temp list of reverse dependencies
+        for tid in self.runtaskentries:
+            for dep in self.runtaskentries[tid].depends:
+                revdeps[dep].add(tid)
+        # Find the dependency chain endpoints
+        endpoints = set()
+        for tid in self.runtaskentries:
+            if len(deps[tid]) == 0:
+                endpoints.add(tid)
+        # Iterate the chains collating dependencies
+        while endpoints:
+            next = set()
+            for tid in endpoints:
+                for dep in revdeps[tid]:
+                    cumulativedeps[dep].add(fn_from_tid(tid))
+                    cumulativedeps[dep].update(cumulativedeps[tid])
+                    if tid in deps[dep]:
+                        deps[dep].remove(tid)
+                    if len(deps[dep]) == 0:
+                        next.add(dep)
+            endpoints = next
+        #for tid in deps:
+        #    if len(deps[tid]) != 0:
+        #        bb.warn("Sanity test failure, dependencies left for %s (%s)" % (tid, deps[tid]))
+
+        # Loop here since recrdeptasks can depend upon other recrdeptasks and we have to
+        # resolve these recursively until we aren't adding any further extra dependencies
+        extradeps = True
+        while extradeps:
+            extradeps = 0
+            for tid in recursivetasks:
+                tasknames = recursivetasks[tid]
+
+                totaldeps = set(self.runtaskentries[tid].depends)
+                if tid in recursiveitasks:
+                    totaldeps.update(recursiveitasks[tid])
+                    for dep in recursiveitasks[tid]:
+                        if dep not in self.runtaskentries:
+                            continue
+                        totaldeps.update(self.runtaskentries[dep].depends)
+
+                deps = set()
+                for dep in totaldeps:
+                    if dep in cumulativedeps:
+                        deps.update(cumulativedeps[dep])
+
+                for t in deps:
+                    for taskname in tasknames:
+                        newtid = t + ":" + taskname
+                        if newtid == tid:
+                            continue
+                        if newtid in self.runtaskentries and newtid not in self.runtaskentries[tid].depends:
+                            extradeps += 1
+                            self.runtaskentries[tid].depends.add(newtid)
+
+                # Handle recursive tasks which depend upon other recursive tasks
+                deps = set()
+                for dep in self.runtaskentries[tid].depends.intersection(recursivetasks):
+                    deps.update(self.runtaskentries[dep].depends.difference(self.runtaskentries[tid].depends))
+                for newtid in deps:
+                    for taskname in tasknames:
+                        if not newtid.endswith(":" + taskname):
+                            continue
+                        if newtid in self.runtaskentries:
+                            extradeps += 1
+                            self.runtaskentries[tid].depends.add(newtid)
+
+            bb.debug(1, "Added %s recursive dependencies in this loop" % extradeps)
+
+        # Remove recrdeptask circular references so that do_a[recrdeptask] = "do_a do_b" can work
+        for tid in recursivetasksselfref:
+            self.runtaskentries[tid].depends.difference_update(recursivetasksselfref)
 
         self.init_progress_reporter.next_stage()
 
@@ -798,30 +862,57 @@ class RunQueueData:
         #
         # Once all active tasks are marked, prune the ones we don't need.
 
-        delcount = 0
+        delcount = {}
         for tid in list(self.runtaskentries.keys()):
             if tid not in runq_build:
+                delcount[tid] = self.runtaskentries[tid]
                 del self.runtaskentries[tid]
-                delcount += 1
-
-        self.init_progress_reporter.next_stage()
-
-        if self.cooker.configuration.runall is not None:
-            runall = "do_%s" % self.cooker.configuration.runall
-            runall_tids = { k: v for k, v in self.runtaskentries.items() if taskname_from_tid(k) == runall }
 
+        # Handle --runall
+        if self.cooker.configuration.runall:
             # re-run the mark_active and then drop unused tasks from new list
             runq_build = {}
-            for tid in list(runall_tids):
-                mark_active(tid,1)
+
+            for task in self.cooker.configuration.runall:
+                runall_tids = set()
+                for tid in list(self.runtaskentries):
+                    wanttid = fn_from_tid(tid) + ":do_%s" % task
+                    if wanttid in delcount:
+                        self.runtaskentries[wanttid] = delcount[wanttid]
+                    if wanttid in self.runtaskentries:
+                        runall_tids.add(wanttid)
+
+                for tid in list(runall_tids):
+                    mark_active(tid,1)
 
             for tid in list(self.runtaskentries.keys()):
                 if tid not in runq_build:
+                    delcount[tid] = self.runtaskentries[tid]
                     del self.runtaskentries[tid]
-                    delcount += 1
 
             if len(self.runtaskentries) == 0:
-                bb.msg.fatal("RunQueue", "No remaining tasks to run for build target %s with runall %s" % (target, runall))
+                bb.msg.fatal("RunQueue", "Could not find any tasks with the tasknames %s to run within the recipes of the taskgraphs of the targets %s" % (str(self.cooker.configuration.runall), str(self.targets)))
+
+        self.init_progress_reporter.next_stage()
+
+        # Handle runonly
+        if self.cooker.configuration.runonly:
+            # re-run the mark_active and then drop unused tasks from new list
+            runq_build = {}
+
+            for task in self.cooker.configuration.runonly:
+                runonly_tids = { k: v for k, v in self.runtaskentries.items() if taskname_from_tid(k) == "do_%s" % task }
+
+                for tid in list(runonly_tids):
+                    mark_active(tid,1)
+
+            for tid in list(self.runtaskentries.keys()):
+                if tid not in runq_build:
+                    delcount[tid] = self.runtaskentries[tid]
+                    del self.runtaskentries[tid]
+
+            if len(self.runtaskentries) == 0:
+                bb.msg.fatal("RunQueue", "Could not find any tasks with the tasknames %s to run within the taskgraphs of the targets %s" % (str(self.cooker.configuration.runonly), str(self.targets)))
 
         #
         # Step D - Sanity checks and computation
@@ -834,7 +925,7 @@ class RunQueueData:
             else:
                 bb.msg.fatal("RunQueue", "No active tasks and not in --continue mode?! Please report this bug.")
 
-        logger.verbose("Pruned %s inactive tasks, %s left", delcount, len(self.runtaskentries))
+        logger.verbose("Pruned %s inactive tasks, %s left", len(delcount), len(self.runtaskentries))
 
         logger.verbose("Assign Weightings")
 
@@ -970,7 +1061,7 @@ class RunQueueData:
 
         # Create a whitelist usable by the stamp checks
         self.stampfnwhitelist = {}
-        for mc in self.taskData: 
+        for mc in self.taskData:
             self.stampfnwhitelist[mc] = []
             for entry in self.stampwhitelist.split():
                 if entry not in self.taskData[mc].build_targets:
@@ -1002,7 +1093,7 @@ class RunQueueData:
                     bb.debug(1, "Task %s is marked nostamp, cannot invalidate this task" % taskname)
             else:
                 logger.verbose("Invalidate task %s, %s", taskname, fn)
-                bb.parse.siggen.invalidate_task(taskname, self.dataCaches[mc], fn)
+                bb.parse.siggen.invalidate_task(taskname, self.dataCaches[mc], taskfn)
 
         self.init_progress_reporter.next_stage()
 
@@ -1795,14 +1886,13 @@ class RunQueueExecuteTasks(RunQueueExecute):
                 continue
             if revdep in self.runq_buildable:
                 continue
-            alldeps = 1
+            alldeps = True
             for dep in self.rqdata.runtaskentries[revdep].depends:
                 if dep not in self.runq_complete:
-                    alldeps = 0
-            if alldeps == 1:
+                    alldeps = False
+                    break
+            if alldeps:
                 self.setbuildable(revdep)
-                fn = fn_from_tid(revdep)
-                taskname = taskname_from_tid(revdep)
                 logger.debug(1, "Marking task %s as buildable", revdep)
 
     def task_complete(self, task):
@@ -1826,8 +1916,8 @@ class RunQueueExecuteTasks(RunQueueExecute):
         self.setbuildable(task)
         bb.event.fire(runQueueTaskSkipped(task, self.stats, self.rq, reason), self.cfgData)
         self.task_completeoutright(task)
-        self.stats.taskCompleted()
         self.stats.taskSkipped()
+        self.stats.taskCompleted()
 
     def execute(self):
         """
@@ -2272,8 +2362,8 @@ class RunQueueExecuteScenequeue(RunQueueExecute):
     def task_failoutright(self, task):
         self.runq_running.add(task)
         self.runq_buildable.add(task)
-        self.stats.taskCompleted()
         self.stats.taskSkipped()
+        self.stats.taskCompleted()
         self.scenequeue_notcovered.add(task)
         self.scenequeue_updatecounters(task, True)
 
@@ -2281,8 +2371,8 @@ class RunQueueExecuteScenequeue(RunQueueExecute):
         self.runq_running.add(task)
         self.runq_buildable.add(task)
         self.task_completeoutright(task)
-        self.stats.taskCompleted()
         self.stats.taskSkipped()
+        self.stats.taskCompleted()
 
     def execute(self):
         """

bitbake/lib/bb/server/process.py

@@ -223,6 +223,8 @@ class ProcessServer(multiprocessing.Process):
 
         try: 
             self.cooker.shutdown(True)
+            self.cooker.notifier.stop()
+            self.cooker.confignotifier.stop()
         except:
             pass
 
@@ -375,11 +377,12 @@ class BitBakeServer(object):
         if os.path.exists(sockname):
             os.unlink(sockname)
 
+        # Place the log in the builddirectory alongside the lock file
+        logfile = os.path.join(os.path.dirname(self.bitbake_lock.name), "bitbake-cookerdaemon.log")
+
         self.sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
         # AF_UNIX has path length issues so chdir here to workaround
         cwd = os.getcwd()
-        logfile = os.path.join(cwd, "bitbake-cookerdaemon.log")
-
         try:
             os.chdir(os.path.dirname(sockname))
             self.sock.bind(os.path.basename(sockname))

bitbake/lib/bb/shell.py

@@ -1,820 +0,0 @@
-# ex:ts=4:sw=4:sts=4:et
-# -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*-
-##########################################################################
-#
-# Copyright (C) 2005-2006 Michael 'Mickey' Lauer <mickey@Vanille.de>
-# Copyright (C) 2005-2006 Vanille Media
-#
-# This program is free software; you can redistribute it and/or modify
-# it under the terms of the GNU General Public License version 2 as
-# published by the Free Software Foundation.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License along
-# with this program; if not, write to the Free Software Foundation, Inc.,
-# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
-#
-##########################################################################
-#
-# Thanks to:
-# * Holger Freyther <zecke@handhelds.org>
-# * Justin Patrin <papercrane@reversefold.com>
-#
-##########################################################################
-
-"""
-BitBake Shell
-
-IDEAS:
-    * list defined tasks per package
-    * list classes
-    * toggle force
-    * command to reparse just one (or more) bbfile(s)
-    * automatic check if reparsing is necessary (inotify?)
-    * frontend for bb file manipulation
-    * more shell-like features:
-        - output control, i.e. pipe output into grep, sort, etc.
-        - job control, i.e. bring running commands into background and foreground
-    * start parsing in background right after startup
-    * ncurses interface
-
-PROBLEMS:
-    * force doesn't always work
-    * readline completion for commands with more than one parameters
-
-"""
-
-##########################################################################
-# Import and setup global variables
-##########################################################################
-
-from __future__ import print_function
-from functools import reduce
-try:
-    set
-except NameError:
-    from sets import Set as set
-import sys, os, readline, socket, httplib, urllib, commands, popen2, shlex, Queue, fnmatch
-from bb import data, parse, build, cache, taskdata, runqueue, providers as Providers
-
-__version__ = "0.5.3.1"
-__credits__ = """BitBake Shell Version %s (C) 2005 Michael 'Mickey' Lauer <mickey@Vanille.de>
-Type 'help' for more information, press CTRL-D to exit.""" % __version__
-
-cmds = {}
-leave_mainloop = False
-last_exception = None
-cooker = None
-parsed = False
-debug = os.environ.get( "BBSHELL_DEBUG", "" )
-
-##########################################################################
-# Class BitBakeShellCommands
-##########################################################################
-
-class BitBakeShellCommands:
-    """This class contains the valid commands for the shell"""
-
-    def __init__( self, shell ):
-        """Register all the commands"""
-        self._shell = shell
-        for attr in BitBakeShellCommands.__dict__:
-            if not attr.startswith( "_" ):
-                if attr.endswith( "_" ):
-                    command = attr[:-1].lower()
-                else:
-                    command = attr[:].lower()
-                method = getattr( BitBakeShellCommands, attr )
-                debugOut( "registering command '%s'" % command )
-                # scan number of arguments
-                usage = getattr( method, "usage", "" )
-                if usage != "<...>":
-                    numArgs = len( usage.split() )
-                else:
-                    numArgs = -1
-                shell.registerCommand( command, method, numArgs, "%s %s" % ( command, usage ), method.__doc__ )
-
-    def _checkParsed( self ):
-        if not parsed:
-            print("SHELL: This command needs to parse bbfiles...")
-            self.parse( None )
-
-    def _findProvider( self, item ):
-        self._checkParsed()
-        # Need to use taskData for this information
-        preferred = data.getVar( "PREFERRED_PROVIDER_%s" % item, cooker.configuration.data, 1 )
-        if not preferred: preferred = item
-        try:
-            lv, lf, pv, pf = Providers.findBestProvider(preferred, cooker.configuration.data, cooker.status)
-        except KeyError:
-            if item in cooker.status.providers:
-                pf = cooker.status.providers[item][0]
-            else:
-                pf = None
-        return pf
-
-    def alias( self, params ):
-        """Register a new name for a command"""
-        new, old = params
-        if not old in cmds:
-            print("ERROR: Command '%s' not known" % old)
-        else:
-            cmds[new] = cmds[old]
-            print("OK")
-    alias.usage = "<alias> <command>"
-
-    def buffer( self, params ):
-        """Dump specified output buffer"""
-        index = params[0]
-        print(self._shell.myout.buffer( int( index ) ))
-    buffer.usage = "<index>"
-
-    def buffers( self, params ):
-        """Show the available output buffers"""
-        commands = self._shell.myout.bufferedCommands()
-        if not commands:
-            print("SHELL: No buffered commands available yet. Start doing something.")
-        else:
-            print("="*35, "Available Output Buffers", "="*27)
-            for index, cmd in enumerate( commands ):
-                print("| %s %s" % ( str( index ).ljust( 3 ), cmd ))
-            print("="*88)
-
-    def build( self, params, cmd = "build" ):
-        """Build a providee"""
-        global last_exception
-        globexpr = params[0]
-        self._checkParsed()
-        names = globfilter( cooker.status.pkg_pn, globexpr )
-        if len( names ) == 0: names = [ globexpr ]
-        print("SHELL: Building %s" % ' '.join( names ))
-
-        td = taskdata.TaskData(cooker.configuration.abort)
-        localdata = data.createCopy(cooker.configuration.data)
-        data.update_data(localdata)
-        data.expandKeys(localdata)
-
-        try:
-            tasks = []
-            for name in names:
-                td.add_provider(localdata, cooker.status, name)
-                providers = td.get_provider(name)
-
-                if len(providers) == 0:
-                    raise Providers.NoProvider
-
-                tasks.append([name, "do_%s" % cmd])
-
-            td.add_unresolved(localdata, cooker.status)
-
-            rq = runqueue.RunQueue(cooker, localdata, cooker.status, td, tasks)
-            rq.prepare_runqueue()
-            rq.execute_runqueue()
-
-        except Providers.NoProvider:
-            print("ERROR: No Provider")
-            last_exception = Providers.NoProvider
-
-        except runqueue.TaskFailure as fnids:
-            last_exception = runqueue.TaskFailure
-
-        except build.FuncFailed as e:
-            print("ERROR: Couldn't build '%s'" % names)
-            last_exception = e
-
-
-    build.usage = "<providee>"
-
-    def clean( self, params ):
-        """Clean a providee"""
-        self.build( params, "clean" )
-    clean.usage = "<providee>"
-
-    def compile( self, params ):
-        """Execute 'compile' on a providee"""
-        self.build( params, "compile" )
-    compile.usage = "<providee>"
-
-    def configure( self, params ):
-        """Execute 'configure' on a providee"""
-        self.build( params, "configure" )
-    configure.usage = "<providee>"
-
-    def install( self, params ):
-        """Execute 'install' on a providee"""
-        self.build( params, "install" )
-    install.usage = "<providee>"
-
-    def edit( self, params ):
-        """Call $EDITOR on a providee"""
-        name = params[0]
-        bbfile = self._findProvider( name )
-        if bbfile is not None:
-            os.system( "%s %s" % ( os.environ.get( "EDITOR", "vi" ), bbfile ) )
-        else:
-            print("ERROR: Nothing provides '%s'" % name)
-    edit.usage = "<providee>"
-
-    def environment( self, params ):
-        """Dump out the outer BitBake environment"""
-        cooker.showEnvironment()
-
-    def exit_( self, params ):
-        """Leave the BitBake Shell"""
-        debugOut( "setting leave_mainloop to true" )
-        global leave_mainloop
-        leave_mainloop = True
-
-    def fetch( self, params ):
-        """Fetch a providee"""
-        self.build( params, "fetch" )
-    fetch.usage = "<providee>"
-
-    def fileBuild( self, params, cmd = "build" ):
-        """Parse and build a .bb file"""
-        global last_exception
-        name = params[0]
-        bf = completeFilePath( name )
-        print("SHELL: Calling '%s' on '%s'" % ( cmd, bf ))
-
-        try:
-            cooker.buildFile(bf, cmd)
-        except parse.ParseError:
-            print("ERROR: Unable to open or parse '%s'" % bf)
-        except build.FuncFailed as e:
-            print("ERROR: Couldn't build '%s'" % name)
-            last_exception = e
-
-    fileBuild.usage = "<bbfile>"
-
-    def fileClean( self, params ):
-        """Clean a .bb file"""
-        self.fileBuild( params, "clean" )
-    fileClean.usage = "<bbfile>"
-
-    def fileEdit( self, params ):
-        """Call $EDITOR on a .bb file"""
-        name = params[0]
-        os.system( "%s %s" % ( os.environ.get( "EDITOR", "vi" ), completeFilePath( name ) ) )
-    fileEdit.usage = "<bbfile>"
-
-    def fileRebuild( self, params ):
-        """Rebuild (clean & build) a .bb file"""
-        self.fileBuild( params, "rebuild" )
-    fileRebuild.usage = "<bbfile>"
-
-    def fileReparse( self, params ):
-        """(re)Parse a bb file"""
-        bbfile = params[0]
-        print("SHELL: Parsing '%s'" % bbfile)
-        parse.update_mtime( bbfile )
-        cooker.parser.reparse(bbfile)
-        if False: #fromCache:
-            print("SHELL: File has not been updated, not reparsing")
-        else:
-            print("SHELL: Parsed")
-    fileReparse.usage = "<bbfile>"
-
-    def abort( self, params ):
-        """Toggle abort task execution flag (see bitbake -k)"""
-        cooker.configuration.abort = not cooker.configuration.abort
-        print("SHELL: Abort Flag is now '%s'" % repr( cooker.configuration.abort ))
-
-    def force( self, params ):
-        """Toggle force task execution flag (see bitbake -f)"""
-        cooker.configuration.force = not cooker.configuration.force
-        print("SHELL: Force Flag is now '%s'" % repr( cooker.configuration.force ))
-
-    def help( self, params ):
-        """Show a comprehensive list of commands and their purpose"""
-        print("="*30, "Available Commands", "="*30)
-        for cmd in sorted(cmds):
-            function, numparams, usage, helptext = cmds[cmd]
-            print("| %s | %s" % (usage.ljust(30), helptext))
-        print("="*78)
-
-    def lastError( self, params ):
-        """Show the reason or log that was produced by the last BitBake event exception"""
-        if last_exception is None:
-            print("SHELL: No Errors yet (Phew)...")
-        else:
-            reason, event = last_exception.args
-            print("SHELL: Reason for the last error: '%s'" % reason)
-            if ':' in reason:
-                msg, filename = reason.split( ':' )
-                filename = filename.strip()
-                print("SHELL: Dumping log file for last error:")
-                try:
-                    print(open( filename ).read())
-                except IOError:
-                    print("ERROR: Couldn't open '%s'" % filename)
-
-    def match( self, params ):
-        """Dump all files or providers matching a glob expression"""
-        what, globexpr = params
-        if what == "files":
-            self._checkParsed()
-            for key in globfilter( cooker.status.pkg_fn, globexpr ): print(key)
-        elif what == "providers":
-            self._checkParsed()
-            for key in globfilter( cooker.status.pkg_pn, globexpr ): print(key)
-        else:
-            print("Usage: match %s" % self.print_.usage)
-    match.usage = "<files|providers> <glob>"
-
-    def new( self, params ):
-        """Create a new .bb file and open the editor"""
-        dirname, filename = params
-        packages = '/'.join( data.getVar( "BBFILES", cooker.configuration.data, 1 ).split('/')[:-2] )
-        fulldirname = "%s/%s" % ( packages, dirname )
-
-        if not os.path.exists( fulldirname ):
-            print("SHELL: Creating '%s'" % fulldirname)
-            os.mkdir( fulldirname )
-        if os.path.exists( fulldirname ) and os.path.isdir( fulldirname ):
-            if os.path.exists( "%s/%s" % ( fulldirname, filename ) ):
-                print("SHELL: ERROR: %s/%s already exists" % ( fulldirname, filename ))
-                return False
-            print("SHELL: Creating '%s/%s'" % ( fulldirname, filename ))
-            newpackage = open( "%s/%s" % ( fulldirname, filename ), "w" )
-            print("""DESCRIPTION = ""
-SECTION = ""
-AUTHOR = ""
-HOMEPAGE = ""
-MAINTAINER = ""
-LICENSE = "GPL"
-PR = "r0"
-
-SRC_URI = ""
-
-#inherit base
-
-#do_configure() {
-#
-#}
-
-#do_compile() {
-#
-#}
-
-#do_stage() {
-#
-#}
-
-#do_install() {
-#
-#}
-""", file=newpackage)
-            newpackage.close()
-            os.system( "%s %s/%s" % ( os.environ.get( "EDITOR" ), fulldirname, filename ) )
-    new.usage = "<directory> <filename>"
-
-    def package( self, params ):
-        """Execute 'package' on a providee"""
-        self.build( params, "package" )
-    package.usage = "<providee>"
-
-    def pasteBin( self, params ):
-        """Send a command + output buffer to the pastebin at http://rafb.net/paste"""
-        index = params[0]
-        contents = self._shell.myout.buffer( int( index ) )
-        sendToPastebin( "output of " + params[0], contents )
-    pasteBin.usage = "<index>"
-
-    def pasteLog( self, params ):
-        """Send the last event exception error log (if there is one) to http://rafb.net/paste"""
-        if last_exception is None:
-            print("SHELL: No Errors yet (Phew)...")
-        else:
-            reason, event = last_exception.args
-            print("SHELL: Reason for the last error: '%s'" % reason)
-            if ':' in reason:
-                msg, filename = reason.split( ':' )
-                filename = filename.strip()
-                print("SHELL: Pasting log file to pastebin...")
-
-                file = open( filename ).read()
-                sendToPastebin( "contents of " + filename, file )
-
-    def patch( self, params ):
-        """Execute 'patch' command on a providee"""
-        self.build( params, "patch" )
-    patch.usage = "<providee>"
-
-    def parse( self, params ):
-        """(Re-)parse .bb files and calculate the dependency graph"""
-        cooker.status = cache.CacheData(cooker.caches_array)
-        ignore = data.getVar("ASSUME_PROVIDED", cooker.configuration.data, 1) or ""
-        cooker.status.ignored_dependencies = set( ignore.split() )
-        cooker.handleCollections( data.getVar("BBFILE_COLLECTIONS", cooker.configuration.data, 1) )
-
-        (filelist, masked) = cooker.collect_bbfiles()
-        cooker.parse_bbfiles(filelist, masked, cooker.myProgressCallback)
-        cooker.buildDepgraph()
-        global parsed
-        parsed = True
-        print()
-
-    def reparse( self, params ):
-        """(re)Parse a providee's bb file"""
-        bbfile = self._findProvider( params[0] )
-        if bbfile is not None:
-            print("SHELL: Found bbfile '%s' for '%s'" % ( bbfile, params[0] ))
-            self.fileReparse( [ bbfile ] )
-        else:
-            print("ERROR: Nothing provides '%s'" % params[0])
-    reparse.usage = "<providee>"
-
-    def getvar( self, params ):
-        """Dump the contents of an outer BitBake environment variable"""
-        var = params[0]
-        value = data.getVar( var, cooker.configuration.data, 1 )
-        print(value)
-    getvar.usage = "<variable>"
-
-    def peek( self, params ):
-        """Dump contents of variable defined in providee's metadata"""
-        name, var = params
-        bbfile = self._findProvider( name )
-        if bbfile is not None:
-            the_data = cache.Cache.loadDataFull(bbfile, cooker.configuration.data)
-            value = the_data.getVar( var, 1 )
-            print(value)
-        else:
-            print("ERROR: Nothing provides '%s'" % name)
-    peek.usage = "<providee> <variable>"
-
-    def poke( self, params ):
-        """Set contents of variable defined in providee's metadata"""
-        name, var, value = params
-        bbfile = self._findProvider( name )
-        if bbfile is not None:
-            print("ERROR: Sorry, this functionality is currently broken")
-            #d = cooker.pkgdata[bbfile]
-            #data.setVar( var, value, d )
-
-            # mark the change semi persistant
-            #cooker.pkgdata.setDirty(bbfile, d)
-            #print "OK"
-        else:
-            print("ERROR: Nothing provides '%s'" % name)
-    poke.usage = "<providee> <variable> <value>"
-
-    def print_( self, params ):
-        """Dump all files or providers"""
-        what = params[0]
-        if what == "files":
-            self._checkParsed()
-            for key in cooker.status.pkg_fn: print(key)
-        elif what == "providers":
-            self._checkParsed()
-            for key in cooker.status.providers: print(key)
-        else:
-            print("Usage: print %s" % self.print_.usage)
-    print_.usage = "<files|providers>"
-
-    def python( self, params ):
-        """Enter the expert mode - an interactive BitBake Python Interpreter"""
-        sys.ps1 = "EXPERT BB>>> "
-        sys.ps2 = "EXPERT BB... "
-        import code
-        interpreter = code.InteractiveConsole( dict( globals() ) )
-        interpreter.interact( "SHELL: Expert Mode - BitBake Python %s\nType 'help' for more information, press CTRL-D to switch back to BBSHELL." % sys.version )
-
-    def showdata( self, params ):
-        """Execute 'showdata' on a providee"""
-        cooker.showEnvironment(None, params)
-    showdata.usage = "<providee>"
-
-    def setVar( self, params ):
-        """Set an outer BitBake environment variable"""
-        var, value = params
-        data.setVar( var, value, cooker.configuration.data )
-        print("OK")
-    setVar.usage = "<variable> <value>"
-
-    def rebuild( self, params ):
-        """Clean and rebuild a .bb file or a providee"""
-        self.build( params, "clean" )
-        self.build( params, "build" )
-    rebuild.usage = "<providee>"
-
-    def shell( self, params ):
-        """Execute a shell command and dump the output"""
-        if params != "":
-            print(commands.getoutput( " ".join( params ) ))
-    shell.usage = "<...>"
-
-    def stage( self, params ):
-        """Execute 'stage' on a providee"""
-        self.build( params, "populate_staging" )
-    stage.usage = "<providee>"
-
-    def status( self, params ):
-        """<just for testing>"""
-        print("-" * 78)
-        print("building list = '%s'" % cooker.building_list)
-        print("build path = '%s'" % cooker.build_path)
-        print("consider_msgs_cache = '%s'" % cooker.consider_msgs_cache)
-        print("build stats = '%s'" % cooker.stats)
-        if last_exception is not None: print("last_exception = '%s'" % repr( last_exception.args ))
-        print("memory output contents = '%s'" % self._shell.myout._buffer)
-
-    def test( self, params ):
-        """<just for testing>"""
-        print("testCommand called with '%s'" % params)
-
-    def unpack( self, params ):
-        """Execute 'unpack' on a providee"""
-        self.build( params, "unpack" )
-    unpack.usage = "<providee>"
-
-    def which( self, params ):
-        """Computes the providers for a given providee"""
-        # Need to use taskData for this information
-        item = params[0]
-
-        self._checkParsed()
-
-        preferred = data.getVar( "PREFERRED_PROVIDER_%s" % item, cooker.configuration.data, 1 )
-        if not preferred: preferred = item
-
-        try:
-            lv, lf, pv, pf = Providers.findBestProvider(preferred, cooker.configuration.data, cooker.status)
-        except KeyError:
-            lv, lf, pv, pf = (None,)*4
-
-        try:
-            providers = cooker.status.providers[item]
-        except KeyError:
-            print("SHELL: ERROR: Nothing provides", preferred)
-        else:
-            for provider in providers:
-                if provider == pf: provider = " (***) %s" % provider
-                else:              provider = "       %s" % provider
-                print(provider)
-    which.usage = "<providee>"
-
-##########################################################################
-# Common helper functions
-##########################################################################
-
-def completeFilePath( bbfile ):
-    """Get the complete bbfile path"""
-    if not cooker.status: return bbfile
-    if not cooker.status.pkg_fn: return bbfile
-    for key in cooker.status.pkg_fn:
-        if key.endswith( bbfile ):
-            return key
-    return bbfile
-
-def sendToPastebin( desc, content ):
-    """Send content to http://oe.pastebin.com"""
-    mydata = {}
-    mydata["lang"] = "Plain Text"
-    mydata["desc"] = desc
-    mydata["cvt_tabs"] = "No"
-    mydata["nick"] = "%s@%s" % ( os.environ.get( "USER", "unknown" ), socket.gethostname() or "unknown" )
-    mydata["text"] = content
-    params = urllib.urlencode( mydata )
-    headers = {"Content-type": "application/x-www-form-urlencoded", "Accept": "text/plain"}
-
-    host = "rafb.net"
-    conn = httplib.HTTPConnection( "%s:80" % host )
-    conn.request("POST", "/paste/paste.php", params, headers )
-
-    response = conn.getresponse()
-    conn.close()
-
-    if response.status == 302:
-        location = response.getheader( "location" ) or "unknown"
-        print("SHELL: Pasted to http://%s%s" % ( host, location ))
-    else:
-        print("ERROR: %s %s" % ( response.status, response.reason ))
-
-def completer( text, state ):
-    """Return a possible readline completion"""
-    debugOut( "completer called with text='%s', state='%d'" % ( text, state ) )
-
-    if state == 0:
-        line = readline.get_line_buffer()
-        if " " in line:
-            line = line.split()
-            # we are in second (or more) argument
-            if line[0] in cmds and hasattr( cmds[line[0]][0], "usage" ): # known command and usage
-                u = getattr( cmds[line[0]][0], "usage" ).split()[0]
-                if u == "<variable>":
-                    allmatches = cooker.configuration.data.keys()
-                elif u == "<bbfile>":
-                    if cooker.status.pkg_fn is None: allmatches = [ "(No Matches Available. Parsed yet?)" ]
-                    else: allmatches = [ x.split("/")[-1] for x in cooker.status.pkg_fn ]
-                elif u == "<providee>":
-                    if cooker.status.pkg_fn is None: allmatches = [ "(No Matches Available. Parsed yet?)" ]
-                    else: allmatches = cooker.status.providers.iterkeys()
-                else: allmatches = [ "(No tab completion available for this command)" ]
-            else: allmatches = [ "(No tab completion available for this command)" ]
-        else:
-            # we are in first argument
-            allmatches = cmds.iterkeys()
-
-        completer.matches = [ x for x in allmatches if x[:len(text)] == text ]
-        #print "completer.matches = '%s'" % completer.matches
-    if len( completer.matches ) > state:
-        return completer.matches[state]
-    else:
-        return None
-
-def debugOut( text ):
-    if debug:
-        sys.stderr.write( "( %s )\n" % text )
-
-def columnize( alist, width = 80 ):
-    """
-    A word-wrap function that preserves existing line breaks
-    and most spaces in the text. Expects that existing line
-    breaks are posix newlines (\n).
-    """
-    return reduce(lambda line, word, width=width: '%s%s%s' %
-                  (line,
-                   ' \n'[(len(line[line.rfind('\n')+1:])
-                         + len(word.split('\n', 1)[0]
-                              ) >= width)],
-                   word),
-                  alist
-                 )
-
-def globfilter( names, pattern ):
-    return fnmatch.filter( names, pattern )
-
-##########################################################################
-# Class MemoryOutput
-##########################################################################
-
-class MemoryOutput:
-    """File-like output class buffering the output of the last 10 commands"""
-    def __init__( self, delegate ):
-        self.delegate = delegate
-        self._buffer = []
-        self.text = []
-        self._command = None
-
-    def startCommand( self, command ):
-        self._command = command
-        self.text = []
-    def endCommand( self ):
-        if self._command is not None:
-            if len( self._buffer ) == 10: del self._buffer[0]
-            self._buffer.append( ( self._command, self.text ) )
-    def removeLast( self ):
-        if self._buffer:
-            del self._buffer[ len( self._buffer ) - 1 ]
-        self.text = []
-        self._command = None
-    def lastBuffer( self ):
-        if self._buffer:
-            return self._buffer[ len( self._buffer ) -1 ][1]
-    def bufferedCommands( self ):
-        return [ cmd for cmd, output in self._buffer ]
-    def buffer( self, i ):
-        if i < len( self._buffer ):
-            return "BB>> %s\n%s" % ( self._buffer[i][0], "".join( self._buffer[i][1] ) )
-        else: return "ERROR: Invalid buffer number. Buffer needs to be in (0, %d)" % ( len( self._buffer ) - 1 )
-    def write( self, text ):
-        if self._command is not None and text != "BB>> ": self.text.append( text )
-        if self.delegate is not None: self.delegate.write( text )
-    def flush( self ):
-        return self.delegate.flush()
-    def fileno( self ):
-        return self.delegate.fileno()
-    def isatty( self ):
-        return self.delegate.isatty()
-
-##########################################################################
-# Class BitBakeShell
-##########################################################################
-
-class BitBakeShell:
-
-    def __init__( self ):
-        """Register commands and set up readline"""
-        self.commandQ = Queue.Queue()
-        self.commands = BitBakeShellCommands( self )
-        self.myout = MemoryOutput( sys.stdout )
-        self.historyfilename = os.path.expanduser( "~/.bbsh_history" )
-        self.startupfilename = os.path.expanduser( "~/.bbsh_startup" )
-
-        readline.set_completer( completer )
-        readline.set_completer_delims( " " )
-        readline.parse_and_bind("tab: complete")
-
-        try:
-            readline.read_history_file( self.historyfilename )
-        except IOError:
-            pass  # It doesn't exist yet.
-
-        print(__credits__)
-
-    def cleanup( self ):
-        """Write readline history and clean up resources"""
-        debugOut( "writing command history" )
-        try:
-            readline.write_history_file( self.historyfilename )
-        except:
-            print("SHELL: Unable to save command history")
-
-    def registerCommand( self, command, function, numparams = 0, usage = "", helptext = "" ):
-        """Register a command"""
-        if usage == "": usage = command
-        if helptext == "": helptext = function.__doc__ or "<not yet documented>"
-        cmds[command] = ( function, numparams, usage, helptext )
-
-    def processCommand( self, command, params ):
-        """Process a command. Check number of params and print a usage string, if appropriate"""
-        debugOut( "processing command '%s'..." % command )
-        try:
-            function, numparams, usage, helptext = cmds[command]
-        except KeyError:
-            print("SHELL: ERROR: '%s' command is not a valid command." % command)
-            self.myout.removeLast()
-        else:
-            if (numparams != -1) and (not len( params ) == numparams):
-                print("Usage: '%s'" % usage)
-                return
-
-            result = function( self.commands, params )
-            debugOut( "result was '%s'" % result )
-
-    def processStartupFile( self ):
-        """Read and execute all commands found in $HOME/.bbsh_startup"""
-        if os.path.exists( self.startupfilename ):
-            startupfile = open( self.startupfilename, "r" )
-            for cmdline in startupfile:
-                debugOut( "processing startup line '%s'" % cmdline )
-                if not cmdline:
-                    continue
-                if "|" in cmdline:
-                    print("ERROR: '|' in startup file is not allowed. Ignoring line")
-                    continue
-                self.commandQ.put( cmdline.strip() )
-
-    def main( self ):
-        """The main command loop"""
-        while not leave_mainloop:
-            try:
-                if self.commandQ.empty():
-                    sys.stdout = self.myout.delegate
-                    cmdline = raw_input( "BB>> " )
-                    sys.stdout = self.myout
-                else:
-                    cmdline = self.commandQ.get()
-                if cmdline:
-                    allCommands = cmdline.split( ';' )
-                    for command in allCommands:
-                        pipecmd = None
-                        #
-                        # special case for expert mode
-                        if command == 'python':
-                            sys.stdout = self.myout.delegate
-                            self.processCommand( command, "" )
-                            sys.stdout = self.myout
-                        else:
-                            self.myout.startCommand( command )
-                            if '|' in command: # disable output
-                                command, pipecmd = command.split( '|' )
-                                delegate = self.myout.delegate
-                                self.myout.delegate = None
-                            tokens = shlex.split( command, True )
-                            self.processCommand( tokens[0], tokens[1:] or "" )
-                            self.myout.endCommand()
-                            if pipecmd is not None: # restore output
-                                self.myout.delegate = delegate
-
-                                pipe = popen2.Popen4( pipecmd )
-                                pipe.tochild.write( "\n".join( self.myout.lastBuffer() ) )
-                                pipe.tochild.close()
-                                sys.stdout.write( pipe.fromchild.read() )
-                        #
-            except EOFError:
-                print()
-                return
-            except KeyboardInterrupt:
-                print()
-
-##########################################################################
-# Start function - called from the BitBake command line utility
-##########################################################################
-
-def start( aCooker ):
-    global cooker
-    cooker = aCooker
-    bbshell = BitBakeShell()
-    bbshell.processStartupFile()
-    bbshell.main()
-    bbshell.cleanup()
-
-if __name__ == "__main__":
-    print("SHELL: Sorry, this program should only be called by BitBake.")

bitbake/lib/bb/siggen.py

@@ -347,7 +347,7 @@ class SignatureGeneratorBasicHash(SignatureGeneratorBasic):
 
     def stampcleanmask(self, stampbase, fn, taskname, extrainfo):
         return self.stampfile(stampbase, fn, taskname, extrainfo, clean=True)
-        
+
     def invalidate_task(self, task, d, fn):
         bb.note("Tainting hash to force rebuild of task %s, %s" % (fn, task))
         bb.build.write_taint(task, d, fn)
@@ -636,7 +636,7 @@ def compare_sigfiles(a, b, recursecb=None, color=False, collapsed=False):
                         if collapsed:
                             output.extend(recout)
                         else:
-                            # If a dependent hash changed, might as well print the line above and then defer to the changes in 
+                            # If a dependent hash changed, might as well print the line above and then defer to the changes in
                             # that hash since in all likelyhood, they're the same changes this task also saw.
                             output = [output[-1]] + recout
 

bitbake/lib/bb/tests/event.py

@@ -30,28 +30,45 @@ import time
 import pickle
 from unittest.mock import Mock
 from unittest.mock import call
+from bb.msg import BBLogFormatter
 
 
-class EventQueueStub():
+class EventQueueStubBase(object):
+    """ Base class for EventQueueStub classes """
+    def __init__(self):
+        self.event_calls = []
+        return
+
+    def _store_event_data_string(self, event):
+        if isinstance(event, logging.LogRecord):
+            formatter = BBLogFormatter("%(levelname)s: %(message)s")
+            self.event_calls.append(formatter.format(event))
+        else:
+            self.event_calls.append(bb.event.getName(event))
+        return
+
+
+class EventQueueStub(EventQueueStubBase):
     """ Class used as specification for UI event handler queue stub objects """
     def __init__(self):
-        return
+        super(EventQueueStub, self).__init__()
 
     def send(self, event):
-        return
+        super(EventQueueStub, self)._store_event_data_string(event)
 
 
-class PickleEventQueueStub():
+class PickleEventQueueStub(EventQueueStubBase):
     """ Class used as specification for UI event handler queue stub objects
         with sendpickle method """
     def __init__(self):
-        return
+        super(PickleEventQueueStub, self).__init__()
 
     def sendpickle(self, pickled_event):
-        return
+        event = pickle.loads(pickled_event)
+        super(PickleEventQueueStub, self)._store_event_data_string(event)
 
 
-class UIClientStub():
+class UIClientStub(object):
     """ Class used as specification for UI event handler stub objects """
     def __init__(self):
         self.event = None
@@ -59,7 +76,7 @@ class UIClientStub():
 
 class EventHandlingTest(unittest.TestCase):
     """ Event handling test class """
-    _threadlock_test_calls = []
+
 
     def setUp(self):
         self._test_process = Mock()
@@ -179,6 +196,33 @@ class EventHandlingTest(unittest.TestCase):
         self.assertEqual(self._test_process.event_handler2.call_args_list,
                          expected_event_handler2)
 
+    def test_class_handler_filters(self):
+        """ Test filters for class handlers """
+        mask = ["bb.event.OperationStarted"]
+        result = bb.event.register("event_handler1",
+                                   self._test_process.event_handler1,
+                                   mask)
+        self.assertEqual(result, bb.event.Registered)
+        result = bb.event.register("event_handler2",
+                                   self._test_process.event_handler2,
+                                   "*")
+        self.assertEqual(result, bb.event.Registered)
+        bb.event.set_eventfilter(
+            lambda name, handler, event, d :
+            name == 'event_handler2' and
+            bb.event.getName(event) == "OperationStarted")
+        event1 = bb.event.OperationStarted()
+        event2 = bb.event.OperationCompleted(total=123)
+        bb.event.fire_class_handlers(event1, None)
+        bb.event.fire_class_handlers(event2, None)
+        bb.event.fire_class_handlers(event2, None)
+        expected_event_handler1 = []
+        expected_event_handler2 = [call(event1)]
+        self.assertEqual(self._test_process.event_handler1.call_args_list,
+                         expected_event_handler1)
+        self.assertEqual(self._test_process.event_handler2.call_args_list,
+                         expected_event_handler2)
+
     def test_change_handler_event_mapping(self):
         """ Test changing the event mapping for class handlers """
         event1 = bb.event.OperationStarted()
@@ -196,8 +240,8 @@ class EventHandlingTest(unittest.TestCase):
                          expected)
 
         # unregister handler and register it only for OperationStarted
-        result = bb.event.remove("event_handler1",
-                                 self._test_process.event_handler1)
+        bb.event.remove("event_handler1",
+                        self._test_process.event_handler1)
         mask = ["bb.event.OperationStarted"]
         result = bb.event.register("event_handler1",
                                    self._test_process.event_handler1,
@@ -210,8 +254,8 @@ class EventHandlingTest(unittest.TestCase):
                          expected)
 
         # unregister handler and register it only for OperationCompleted
-        result = bb.event.remove("event_handler1",
-                                 self._test_process.event_handler1)
+        bb.event.remove("event_handler1",
+                        self._test_process.event_handler1)
         mask = ["bb.event.OperationCompleted"]
         result = bb.event.register("event_handler1",
                                    self._test_process.event_handler1,
@@ -259,6 +303,61 @@ class EventHandlingTest(unittest.TestCase):
         self.assertEqual(self._test_ui2.event.sendpickle.call_args_list,
                          expected)
 
+    def test_ui_handler_mask_filter(self):
+        """ Test filters for UI handlers """
+        mask = ["bb.event.OperationStarted"]
+        debug_domains = {}
+        self._test_ui1.event = Mock(spec_set=EventQueueStub)
+        result = bb.event.register_UIHhandler(self._test_ui1, mainui=True)
+        bb.event.set_UIHmask(result, logging.INFO, debug_domains, mask)
+        self._test_ui2.event = Mock(spec_set=PickleEventQueueStub)
+        result = bb.event.register_UIHhandler(self._test_ui2, mainui=True)
+        bb.event.set_UIHmask(result, logging.INFO, debug_domains, mask)
+
+        event1 = bb.event.OperationStarted()
+        event2 = bb.event.OperationCompleted(total=1)
+
+        bb.event.fire_ui_handlers(event1, None)
+        bb.event.fire_ui_handlers(event2, None)
+        expected = [call(event1)]
+        self.assertEqual(self._test_ui1.event.send.call_args_list,
+                         expected)
+        expected = [call(pickle.dumps(event1))]
+        self.assertEqual(self._test_ui2.event.sendpickle.call_args_list,
+                         expected)
+
+    def test_ui_handler_log_filter(self):
+        """ Test log filters for UI handlers """
+        mask = ["*"]
+        debug_domains = {'BitBake.Foo': logging.WARNING}
+
+        self._test_ui1.event = EventQueueStub()
+        result = bb.event.register_UIHhandler(self._test_ui1, mainui=True)
+        bb.event.set_UIHmask(result, logging.ERROR, debug_domains, mask)
+        self._test_ui2.event = PickleEventQueueStub()
+        result = bb.event.register_UIHhandler(self._test_ui2, mainui=True)
+        bb.event.set_UIHmask(result, logging.ERROR, debug_domains, mask)
+
+        event1 = bb.event.OperationStarted()
+        bb.event.fire_ui_handlers(event1, None)   # All events match
+
+        event_log_handler = bb.event.LogHandler()
+        logger = logging.getLogger("BitBake")
+        logger.addHandler(event_log_handler)
+        logger1 = logging.getLogger("BitBake.Foo")
+        logger1.warning("Test warning LogRecord1") # Matches debug_domains level
+        logger1.info("Test info LogRecord")        # Filtered out
+        logger2 = logging.getLogger("BitBake.Bar")
+        logger2.error("Test error LogRecord")      # Matches filter base level
+        logger2.warning("Test warning LogRecord2") # Filtered out
+        logger.removeHandler(event_log_handler)
+
+        expected = ['OperationStarted',
+                    'WARNING: Test warning LogRecord1',
+                    'ERROR: Test error LogRecord']
+        self.assertEqual(self._test_ui1.event.event_calls, expected)
+        self.assertEqual(self._test_ui2.event.event_calls, expected)
+
     def test_fire(self):
         """ Test fire method used to trigger class and ui event handlers """
         mask = ["bb.event.ConfigParsed"]
@@ -289,18 +388,28 @@ class EventHandlingTest(unittest.TestCase):
         self.assertEqual(self._test_ui1.event.send.call_args_list,
                          expected)
 
+    def test_worker_fire(self):
+        """ Test the triggering of bb.event.worker_fire callback """
+        bb.event.worker_fire = Mock()
+        event = bb.event.Event()
+        bb.event.fire(event, None)
+        expected = [call(event, None)]
+        self.assertEqual(bb.event.worker_fire.call_args_list, expected)
+
     def test_print_ui_queue(self):
         """ Test print_ui_queue method """
         event1 = bb.event.OperationStarted()
         event2 = bb.event.OperationCompleted(total=123)
         bb.event.fire(event1, None)
         bb.event.fire(event2, None)
+        event_log_handler = bb.event.LogHandler()
         logger = logging.getLogger("BitBake")
-        logger.addHandler(bb.event.LogHandler())
+        logger.addHandler(event_log_handler)
         logger.info("Test info LogRecord")
         logger.warning("Test warning LogRecord")
         with self.assertLogs("BitBake", level="INFO") as cm:
             bb.event.print_ui_queue()
+        logger.removeHandler(event_log_handler)
         self.assertEqual(cm.output,
                          ["INFO:BitBake:Test info LogRecord",
                           "WARNING:BitBake:Test warning LogRecord"])
@@ -364,6 +473,7 @@ class EventHandlingTest(unittest.TestCase):
         self.assertEqual(self._threadlock_test_calls,
                          ["w1_ui1", "w1_ui2", "w2_ui1", "w2_ui2"])
 
+
     def test_disable_threadlock(self):
         """ Test disable_threadlock method """
         self._set_threadlock_test_mockups()
@@ -375,3 +485,502 @@ class EventHandlingTest(unittest.TestCase):
         # processed before finishing handling the first worker event.
         self.assertEqual(self._threadlock_test_calls,
                          ["w1_ui1", "w2_ui1", "w1_ui2", "w2_ui2"])
+
+
+class EventClassesTest(unittest.TestCase):
+    """ Event classes test class """
+
+    _worker_pid = 54321
+
+    def setUp(self):
+        bb.event.worker_pid = EventClassesTest._worker_pid
+
+    def test_Event(self):
+        """ Test the Event base class """
+        event = bb.event.Event()
+        self.assertEqual(event.pid, EventClassesTest._worker_pid)
+
+    def test_HeartbeatEvent(self):
+        """ Test the HeartbeatEvent class """
+        time = 10
+        event = bb.event.HeartbeatEvent(time)
+        self.assertEqual(event.time, time)
+        self.assertEqual(event.pid, EventClassesTest._worker_pid)
+
+    def test_OperationStarted(self):
+        """ Test OperationStarted event class """
+        msg = "Foo Bar"
+        event = bb.event.OperationStarted(msg)
+        self.assertEqual(event.msg, msg)
+        self.assertEqual(event.pid, EventClassesTest._worker_pid)
+
+    def test_OperationCompleted(self):
+        """ Test OperationCompleted event class """
+        msg = "Foo Bar"
+        total = 123
+        event = bb.event.OperationCompleted(total, msg)
+        self.assertEqual(event.msg, msg)
+        self.assertEqual(event.total, total)
+        self.assertEqual(event.pid, EventClassesTest._worker_pid)
+
+    def test_OperationProgress(self):
+        """ Test OperationProgress event class """
+        msg = "Foo Bar"
+        total = 123
+        current = 111
+        event = bb.event.OperationProgress(current, total, msg)
+        self.assertEqual(event.msg, msg + ": %s/%s" % (current, total))
+        self.assertEqual(event.pid, EventClassesTest._worker_pid)
+
+    def test_ConfigParsed(self):
+        """ Test the ConfigParsed class """
+        event = bb.event.ConfigParsed()
+        self.assertEqual(event.pid, EventClassesTest._worker_pid)
+
+    def test_MultiConfigParsed(self):
+        """ Test MultiConfigParsed event class """
+        mcdata = {"foobar": "Foo Bar"}
+        event = bb.event.MultiConfigParsed(mcdata)
+        self.assertEqual(event.mcdata, mcdata)
+        self.assertEqual(event.pid, EventClassesTest._worker_pid)
+
+    def test_RecipeEvent(self):
+        """ Test RecipeEvent event base class """
+        callback = lambda a: 2 * a
+        event = bb.event.RecipeEvent(callback)
+        self.assertEqual(event.fn(1), callback(1))
+        self.assertEqual(event.pid, EventClassesTest._worker_pid)
+
+    def test_RecipePreFinalise(self):
+        """ Test RecipePreFinalise event class """
+        callback = lambda a: 2 * a
+        event = bb.event.RecipePreFinalise(callback)
+        self.assertEqual(event.fn(1), callback(1))
+        self.assertEqual(event.pid, EventClassesTest._worker_pid)
+
+    def test_RecipeTaskPreProcess(self):
+        """ Test RecipeTaskPreProcess event class """
+        callback = lambda a: 2 * a
+        tasklist = [("foobar", callback)]
+        event = bb.event.RecipeTaskPreProcess(callback, tasklist)
+        self.assertEqual(event.fn(1), callback(1))
+        self.assertEqual(event.tasklist, tasklist)
+        self.assertEqual(event.pid, EventClassesTest._worker_pid)
+
+    def test_RecipeParsed(self):
+        """ Test RecipeParsed event base class """
+        callback = lambda a: 2 * a
+        event = bb.event.RecipeParsed(callback)
+        self.assertEqual(event.fn(1), callback(1))
+        self.assertEqual(event.pid, EventClassesTest._worker_pid)
+
+    def test_StampUpdate(self):
+        targets = ["foo", "bar"]
+        stampfns = [lambda:"foobar"]
+        event = bb.event.StampUpdate(targets, stampfns)
+        self.assertEqual(event.targets, targets)
+        self.assertEqual(event.stampPrefix, stampfns)
+        self.assertEqual(event.pid, EventClassesTest._worker_pid)
+
+    def test_BuildBase(self):
+        """ Test base class for bitbake build events """
+        name = "foo"
+        pkgs = ["bar"]
+        failures = 123
+        event = bb.event.BuildBase(name, pkgs, failures)
+        self.assertEqual(event.name, name)
+        self.assertEqual(event.pkgs, pkgs)
+        self.assertEqual(event.getFailures(), failures)
+        name = event.name = "bar"
+        pkgs = event.pkgs = ["foo"]
+        self.assertEqual(event.name, name)
+        self.assertEqual(event.pkgs, pkgs)
+        self.assertEqual(event.getFailures(), failures)
+        self.assertEqual(event.pid, EventClassesTest._worker_pid)
+
+    def test_BuildInit(self):
+        """ Test class for bitbake build invocation events """
+        event = bb.event.BuildInit()
+        self.assertEqual(event.name, None)
+        self.assertEqual(event.pkgs, [])
+        self.assertEqual(event.getFailures(), 0)
+        name = event.name = "bar"
+        pkgs = event.pkgs = ["foo"]
+        self.assertEqual(event.name, name)
+        self.assertEqual(event.pkgs, pkgs)
+        self.assertEqual(event.getFailures(), 0)
+        self.assertEqual(event.pid, EventClassesTest._worker_pid)
+
+    def test_BuildStarted(self):
+        """ Test class for build started events """
+        name = "foo"
+        pkgs = ["bar"]
+        failures = 123
+        event = bb.event.BuildStarted(name, pkgs, failures)
+        self.assertEqual(event.name, name)
+        self.assertEqual(event.pkgs, pkgs)
+        self.assertEqual(event.getFailures(), failures)
+        self.assertEqual(event.msg, "Building Started")
+        name = event.name = "bar"
+        pkgs = event.pkgs = ["foo"]
+        msg = event.msg = "foobar"
+        self.assertEqual(event.name, name)
+        self.assertEqual(event.pkgs, pkgs)
+        self.assertEqual(event.getFailures(), failures)
+        self.assertEqual(event.msg, msg)
+        self.assertEqual(event.pid, EventClassesTest._worker_pid)
+
+    def test_BuildCompleted(self):
+        """ Test class for build completed events """
+        total = 1000
+        name = "foo"
+        pkgs = ["bar"]
+        failures = 123
+        interrupted = 1
+        event = bb.event.BuildCompleted(total, name, pkgs, failures,
+                                        interrupted)
+        self.assertEqual(event.name, name)
+        self.assertEqual(event.pkgs, pkgs)
+        self.assertEqual(event.getFailures(), failures)
+        self.assertEqual(event.msg, "Building Failed")
+        event2 = bb.event.BuildCompleted(total, name, pkgs)
+        self.assertEqual(event2.name, name)
+        self.assertEqual(event2.pkgs, pkgs)
+        self.assertEqual(event2.getFailures(), 0)
+        self.assertEqual(event2.msg, "Building Succeeded")
+        self.assertEqual(event2.pid, EventClassesTest._worker_pid)
+
+    def test_DiskFull(self):
+        """ Test DiskFull event class """
+        dev = "/dev/foo"
+        type = "ext4"
+        freespace = "104M"
+        mountpoint = "/"
+        event = bb.event.DiskFull(dev, type, freespace, mountpoint)
+        self.assertEqual(event.pid, EventClassesTest._worker_pid)
+
+    def test_MonitorDiskEvent(self):
+        """ Test MonitorDiskEvent class """
+        available_bytes = 10000000
+        free_bytes = 90000000
+        total_bytes = 1000000000
+        du = bb.event.DiskUsageSample(available_bytes, free_bytes,
+                                      total_bytes)
+        event = bb.event.MonitorDiskEvent(du)
+        self.assertEqual(event.disk_usage.available_bytes, available_bytes)
+        self.assertEqual(event.disk_usage.free_bytes, free_bytes)
+        self.assertEqual(event.disk_usage.total_bytes, total_bytes)
+        self.assertEqual(event.pid, EventClassesTest._worker_pid)
+
+    def test_NoProvider(self):
+        """ Test NoProvider event class """
+        item = "foobar"
+        event1 = bb.event.NoProvider(item)
+        self.assertEqual(event1.getItem(), item)
+        self.assertEqual(event1.isRuntime(), False)
+        self.assertEqual(str(event1), "Nothing PROVIDES 'foobar'")
+        runtime = True
+        dependees = ["foo", "bar"]
+        reasons = None
+        close_matches = ["foibar", "footbar"]
+        event2 = bb.event.NoProvider(item, runtime, dependees, reasons,
+                                     close_matches)
+        self.assertEqual(event2.isRuntime(), True)
+        expected = ("Nothing RPROVIDES 'foobar' (but foo, bar RDEPENDS"
+                    " on or otherwise requires it). Close matches:\n"
+                    "  foibar\n"
+                    "  footbar")
+        self.assertEqual(str(event2), expected)
+        reasons = ["Item does not exist on database"]
+        close_matches = ["foibar", "footbar"]
+        event3 = bb.event.NoProvider(item, runtime, dependees, reasons,
+                                     close_matches)
+        expected = ("Nothing RPROVIDES 'foobar' (but foo, bar RDEPENDS"
+                    " on or otherwise requires it)\n"
+                    "Item does not exist on database")
+        self.assertEqual(str(event3), expected)
+        self.assertEqual(event3.pid, EventClassesTest._worker_pid)
+
+    def test_MultipleProviders(self):
+        """ Test MultipleProviders event class """
+        item = "foobar"
+        candidates = ["foobarv1", "foobars"]
+        event1 = bb.event.MultipleProviders(item, candidates)
+        self.assertEqual(event1.isRuntime(), False)
+        self.assertEqual(event1.getItem(), item)
+        self.assertEqual(event1.getCandidates(), candidates)
+        expected = ("Multiple providers are available for foobar (foobarv1,"
+                    " foobars)\n"
+                    "Consider defining a PREFERRED_PROVIDER entry to match "
+                    "foobar")
+        self.assertEqual(str(event1), expected)
+        runtime = True
+        event2 = bb.event.MultipleProviders(item, candidates, runtime)
+        self.assertEqual(event2.isRuntime(), runtime)
+        expected = ("Multiple providers are available for runtime foobar "
+                    "(foobarv1, foobars)\n"
+                    "Consider defining a PREFERRED_RPROVIDER entry to match "
+                    "foobar")
+        self.assertEqual(str(event2), expected)
+        self.assertEqual(event2.pid, EventClassesTest._worker_pid)
+
+    def test_ParseStarted(self):
+        """ Test ParseStarted event class """
+        total = 123
+        event = bb.event.ParseStarted(total)
+        self.assertEqual(event.msg, "Recipe parsing Started")
+        self.assertEqual(event.total, total)
+        self.assertEqual(event.pid, EventClassesTest._worker_pid)
+
+    def test_ParseCompleted(self):
+        """ Test ParseCompleted event class """
+        cached = 10
+        parsed = 13
+        skipped = 7
+        virtuals = 2
+        masked = 1
+        errors = 0
+        total = 23
+        event = bb.event.ParseCompleted(cached, parsed, skipped, masked,
+                                        virtuals, errors, total)
+        self.assertEqual(event.msg, "Recipe parsing Completed")
+        expected = [cached, parsed, skipped, virtuals, masked, errors,
+                    cached + parsed, total]
+        actual = [event.cached, event.parsed, event.skipped, event.virtuals,
+                  event.masked, event.errors, event.sofar, event.total]
+        self.assertEqual(str(actual), str(expected))
+        self.assertEqual(event.pid, EventClassesTest._worker_pid)
+
+    def test_ParseProgress(self):
+        """ Test ParseProgress event class """
+        current = 10
+        total = 100
+        event = bb.event.ParseProgress(current, total)
+        self.assertEqual(event.msg,
+                         "Recipe parsing" + ": %s/%s" % (current, total))
+        self.assertEqual(event.pid, EventClassesTest._worker_pid)
+
+    def test_CacheLoadStarted(self):
+        """ Test CacheLoadStarted event class """
+        total = 123
+        event = bb.event.CacheLoadStarted(total)
+        self.assertEqual(event.msg, "Loading cache Started")
+        self.assertEqual(event.total, total)
+        self.assertEqual(event.pid, EventClassesTest._worker_pid)
+
+    def test_CacheLoadProgress(self):
+        """ Test CacheLoadProgress event class """
+        current = 10
+        total = 100
+        event = bb.event.CacheLoadProgress(current, total)
+        self.assertEqual(event.msg,
+                         "Loading cache" + ": %s/%s" % (current, total))
+        self.assertEqual(event.pid, EventClassesTest._worker_pid)
+
+    def test_CacheLoadCompleted(self):
+        """ Test CacheLoadCompleted event class """
+        total = 23
+        num_entries = 12
+        event = bb.event.CacheLoadCompleted(total, num_entries)
+        self.assertEqual(event.msg, "Loading cache Completed")
+        expected = [total, num_entries]
+        actual = [event.total, event.num_entries]
+        self.assertEqual(str(actual), str(expected))
+        self.assertEqual(event.pid, EventClassesTest._worker_pid)
+
+    def test_TreeDataPreparationStarted(self):
+        """ Test TreeDataPreparationStarted event class """
+        event = bb.event.TreeDataPreparationStarted()
+        self.assertEqual(event.msg, "Preparing tree data Started")
+        self.assertEqual(event.pid, EventClassesTest._worker_pid)
+
+    def test_TreeDataPreparationProgress(self):
+        """ Test TreeDataPreparationProgress event class """
+        current = 10
+        total = 100
+        event = bb.event.TreeDataPreparationProgress(current, total)
+        self.assertEqual(event.msg,
+                         "Preparing tree data" + ": %s/%s" % (current, total))
+        self.assertEqual(event.pid, EventClassesTest._worker_pid)
+
+    def test_TreeDataPreparationCompleted(self):
+        """ Test TreeDataPreparationCompleted event class """
+        total = 23
+        event = bb.event.TreeDataPreparationCompleted(total)
+        self.assertEqual(event.msg, "Preparing tree data Completed")
+        self.assertEqual(event.total, total)
+        self.assertEqual(event.pid, EventClassesTest._worker_pid)
+
+    def test_DepTreeGenerated(self):
+        """ Test DepTreeGenerated event class """
+        depgraph = Mock()
+        event = bb.event.DepTreeGenerated(depgraph)
+        self.assertEqual(event.pid, EventClassesTest._worker_pid)
+
+    def test_TargetsTreeGenerated(self):
+        """ Test TargetsTreeGenerated event class """
+        model = Mock()
+        event = bb.event.TargetsTreeGenerated(model)
+        self.assertEqual(event.pid, EventClassesTest._worker_pid)
+
+    def test_ReachableStamps(self):
+        """ Test ReachableStamps event class """
+        stamps = [Mock(), Mock()]
+        event = bb.event.ReachableStamps(stamps)
+        self.assertEqual(event.stamps, stamps)
+        self.assertEqual(event.pid, EventClassesTest._worker_pid)
+
+    def test_FilesMatchingFound(self):
+        """ Test FilesMatchingFound event class """
+        pattern = "foo.*bar"
+        matches = ["foobar"]
+        event = bb.event.FilesMatchingFound(pattern, matches)
+        self.assertEqual(event.pid, EventClassesTest._worker_pid)
+
+    def test_ConfigFilesFound(self):
+        """ Test ConfigFilesFound event class """
+        variable = "FOO_BAR"
+        values = ["foo", "bar"]
+        event = bb.event.ConfigFilesFound(variable, values)
+        self.assertEqual(event.pid, EventClassesTest._worker_pid)
+
+    def test_ConfigFilePathFound(self):
+        """ Test ConfigFilePathFound event class """
+        path = "/foo/bar"
+        event = bb.event.ConfigFilePathFound(path)
+        self.assertEqual(event.pid, EventClassesTest._worker_pid)
+
+    def test_message_classes(self):
+        """ Test message event classes """
+        msg = "foobar foo bar"
+        event = bb.event.MsgBase(msg)
+        self.assertEqual(event.pid, EventClassesTest._worker_pid)
+        event = bb.event.MsgDebug(msg)
+        self.assertEqual(event.pid, EventClassesTest._worker_pid)
+        event = bb.event.MsgNote(msg)
+        self.assertEqual(event.pid, EventClassesTest._worker_pid)
+        event = bb.event.MsgWarn(msg)
+        self.assertEqual(event.pid, EventClassesTest._worker_pid)
+        event = bb.event.MsgError(msg)
+        self.assertEqual(event.pid, EventClassesTest._worker_pid)
+        event = bb.event.MsgFatal(msg)
+        self.assertEqual(event.pid, EventClassesTest._worker_pid)
+        event = bb.event.MsgPlain(msg)
+        self.assertEqual(event.pid, EventClassesTest._worker_pid)
+
+    def test_LogExecTTY(self):
+        """ Test LogExecTTY event class """
+        msg = "foo bar"
+        prog = "foo.sh"
+        sleep_delay = 10
+        retries = 3
+        event = bb.event.LogExecTTY(msg, prog, sleep_delay, retries)
+        self.assertEqual(event.msg, msg)
+        self.assertEqual(event.prog, prog)
+        self.assertEqual(event.sleep_delay, sleep_delay)
+        self.assertEqual(event.retries, retries)
+        self.assertEqual(event.pid, EventClassesTest._worker_pid)
+
+    def _throw_zero_division_exception(self):
+        a = 1 / 0
+        return
+
+    def _worker_handler(self, event, d):
+        self._returned_event = event
+        return
+
+    def test_LogHandler(self):
+        """ Test LogHandler class """
+        logger = logging.getLogger("TestEventClasses")
+        logger.propagate = False
+        handler = bb.event.LogHandler(logging.INFO)
+        logger.addHandler(handler)
+        bb.event.worker_fire = self._worker_handler
+        try:
+            self._throw_zero_division_exception()
+        except ZeroDivisionError as ex:
+            logger.exception(ex)
+        event = self._returned_event
+        try:
+            pe = pickle.dumps(event)
+            newevent = pickle.loads(pe)
+        except:
+            self.fail('Logged event is not serializable')
+        self.assertEqual(event.taskpid, EventClassesTest._worker_pid)
+
+    def test_MetadataEvent(self):
+        """ Test MetadataEvent class """
+        eventtype = "footype"
+        eventdata = {"foo": "bar"}
+        event = bb.event.MetadataEvent(eventtype, eventdata)
+        self.assertEqual(event.type, eventtype)
+        self.assertEqual(event.pid, EventClassesTest._worker_pid)
+
+    def test_ProcessStarted(self):
+        """ Test ProcessStarted class """
+        processname = "foo"
+        total = 9783128974
+        event = bb.event.ProcessStarted(processname, total)
+        self.assertEqual(event.processname, processname)
+        self.assertEqual(event.total, total)
+        self.assertEqual(event.pid, EventClassesTest._worker_pid)
+
+    def test_ProcessProgress(self):
+        """ Test ProcessProgress class """
+        processname = "foo"
+        progress = 243224
+        event = bb.event.ProcessProgress(processname, progress)
+        self.assertEqual(event.processname, processname)
+        self.assertEqual(event.progress, progress)
+        self.assertEqual(event.pid, EventClassesTest._worker_pid)
+
+    def test_ProcessFinished(self):
+        """ Test ProcessFinished class """
+        processname = "foo"
+        total = 1242342344
+        event = bb.event.ProcessFinished(processname)
+        self.assertEqual(event.processname, processname)
+        self.assertEqual(event.pid, EventClassesTest._worker_pid)
+
+    def test_SanityCheck(self):
+        """ Test SanityCheck class """
+        event1 = bb.event.SanityCheck()
+        self.assertEqual(event1.generateevents, True)
+        self.assertEqual(event1.pid, EventClassesTest._worker_pid)
+        generateevents = False
+        event2 = bb.event.SanityCheck(generateevents)
+        self.assertEqual(event2.generateevents, generateevents)
+        self.assertEqual(event2.pid, EventClassesTest._worker_pid)
+
+    def test_SanityCheckPassed(self):
+        """ Test SanityCheckPassed class """
+        event = bb.event.SanityCheckPassed()
+        self.assertEqual(event.pid, EventClassesTest._worker_pid)
+
+    def test_SanityCheckFailed(self):
+        """ Test SanityCheckFailed class """
+        msg = "The sanity test failed."
+        event1 = bb.event.SanityCheckFailed(msg)
+        self.assertEqual(event1.pid, EventClassesTest._worker_pid)
+        network_error = True
+        event2 = bb.event.SanityCheckFailed(msg, network_error)
+        self.assertEqual(event2.pid, EventClassesTest._worker_pid)
+
+    def test_network_event_classes(self):
+        """ Test network event classes """
+        event1 = bb.event.NetworkTest()
+        generateevents = False
+        self.assertEqual(event1.pid, EventClassesTest._worker_pid)
+        event2 = bb.event.NetworkTest(generateevents)
+        self.assertEqual(event2.pid, EventClassesTest._worker_pid)
+        event3 = bb.event.NetworkTestPassed()
+        self.assertEqual(event3.pid, EventClassesTest._worker_pid)
+        event4 = bb.event.NetworkTestFailed()
+        self.assertEqual(event4.pid, EventClassesTest._worker_pid)
+
+    def test_FindSigInfoResult(self):
+        """ Test FindSigInfoResult event class """
+        result = [Mock()]
+        event = bb.event.FindSigInfoResult(result)
+        self.assertEqual(event.result, result)
+        self.assertEqual(event.pid, EventClassesTest._worker_pid)

bitbake/lib/bb/tests/fetch.py

@@ -20,6 +20,7 @@
 #
 
 import unittest
+import hashlib
 import tempfile
 import subprocess
 import collections
@@ -401,6 +402,12 @@ class MirrorUriTest(FetcherTest):
             : "git://somewhere.org/somedir/mtd-utils.git;tag=1234567890123456789012345678901234567890;protocol=http", 
         ("git://git.invalid.infradead.org/foo/mtd-utils.git;tag=1234567890123456789012345678901234567890", "git://.*/.*", "git://somewhere.org/somedir/MIRRORNAME;protocol=http") 
             : "git://somewhere.org/somedir/git.invalid.infradead.org.foo.mtd-utils.git;tag=1234567890123456789012345678901234567890;protocol=http", 
+        ("http://somewhere.org/somedir1/somedir2/somefile_1.2.3.tar.gz", "http://.*/.*", "http://somewhere2.org")
+            : "http://somewhere2.org/somefile_1.2.3.tar.gz",
+        ("http://somewhere.org/somedir1/somedir2/somefile_1.2.3.tar.gz", "http://.*/.*", "http://somewhere2.org/")
+            : "http://somewhere2.org/somefile_1.2.3.tar.gz",
+        ("git://someserver.org/bitbake;tag=1234567890123456789012345678901234567890;branch=master", "git://someserver.org/bitbake;branch=master", "git://git.openembedded.org/bitbake;protocol=http")
+            : "git://git.openembedded.org/bitbake;tag=1234567890123456789012345678901234567890;branch=master;protocol=http",
 
         #Renaming files doesn't work
         #("http://somewhere.org/somedir1/somefile_1.2.3.tar.gz", "http://somewhere.org/somedir1/somefile_1.2.3.tar.gz", "http://somewhere2.org/somedir3/somefile_2.3.4.tar.gz") : "http://somewhere2.org/somedir3/somefile_2.3.4.tar.gz"
@@ -522,6 +529,109 @@ class FetcherLocalTest(FetcherTest):
         with self.assertRaises(bb.fetch2.UnpackError):
             self.fetchUnpack(['file://a;subdir=/bin/sh'])
 
+class FetcherNoNetworkTest(FetcherTest):
+    def setUp(self):
+        super().setUp()
+        # all test cases are based on not having network
+        self.d.setVar("BB_NO_NETWORK", "1")
+
+    def test_missing(self):
+        string = "this is a test file\n".encode("utf-8")
+        self.d.setVarFlag("SRC_URI", "md5sum", hashlib.md5(string).hexdigest())
+        self.d.setVarFlag("SRC_URI", "sha256sum", hashlib.sha256(string).hexdigest())
+
+        self.assertFalse(os.path.exists(os.path.join(self.dldir, "test-file.tar.gz")))
+        self.assertFalse(os.path.exists(os.path.join(self.dldir, "test-file.tar.gz.done")))
+        fetcher = bb.fetch.Fetch(["http://invalid.yoctoproject.org/test-file.tar.gz"], self.d)
+        with self.assertRaises(bb.fetch2.NetworkAccess):
+            fetcher.download()
+
+    def test_valid_missing_donestamp(self):
+        # create the file in the download directory with correct hash
+        string = "this is a test file\n".encode("utf-8")
+        with open(os.path.join(self.dldir, "test-file.tar.gz"), "wb") as f:
+            f.write(string)
+
+        self.d.setVarFlag("SRC_URI", "md5sum", hashlib.md5(string).hexdigest())
+        self.d.setVarFlag("SRC_URI", "sha256sum", hashlib.sha256(string).hexdigest())
+
+        self.assertTrue(os.path.exists(os.path.join(self.dldir, "test-file.tar.gz")))
+        self.assertFalse(os.path.exists(os.path.join(self.dldir, "test-file.tar.gz.done")))
+        fetcher = bb.fetch.Fetch(["http://invalid.yoctoproject.org/test-file.tar.gz"], self.d)
+        fetcher.download()
+        self.assertTrue(os.path.exists(os.path.join(self.dldir, "test-file.tar.gz.done")))
+
+    def test_invalid_missing_donestamp(self):
+        # create an invalid file in the download directory with incorrect hash
+        string = "this is a test file\n".encode("utf-8")
+        with open(os.path.join(self.dldir, "test-file.tar.gz"), "wb"):
+            pass
+
+        self.d.setVarFlag("SRC_URI", "md5sum", hashlib.md5(string).hexdigest())
+        self.d.setVarFlag("SRC_URI", "sha256sum", hashlib.sha256(string).hexdigest())
+
+        self.assertTrue(os.path.exists(os.path.join(self.dldir, "test-file.tar.gz")))
+        self.assertFalse(os.path.exists(os.path.join(self.dldir, "test-file.tar.gz.done")))
+        fetcher = bb.fetch.Fetch(["http://invalid.yoctoproject.org/test-file.tar.gz"], self.d)
+        with self.assertRaises(bb.fetch2.NetworkAccess):
+            fetcher.download()
+        # the existing file should not exist or should have be moved to "bad-checksum"
+        self.assertFalse(os.path.exists(os.path.join(self.dldir, "test-file.tar.gz")))
+
+    def test_nochecksums_missing(self):
+        self.assertFalse(os.path.exists(os.path.join(self.dldir, "test-file.tar.gz")))
+        self.assertFalse(os.path.exists(os.path.join(self.dldir, "test-file.tar.gz.done")))
+        # ssh fetch does not support checksums
+        fetcher = bb.fetch.Fetch(["ssh://invalid@invalid.yoctoproject.org/test-file.tar.gz"], self.d)
+        # attempts to download with missing donestamp
+        with self.assertRaises(bb.fetch2.NetworkAccess):
+            fetcher.download()
+
+    def test_nochecksums_missing_donestamp(self):
+        # create a file in the download directory
+        with open(os.path.join(self.dldir, "test-file.tar.gz"), "wb"):
+            pass
+
+        self.assertTrue(os.path.exists(os.path.join(self.dldir, "test-file.tar.gz")))
+        self.assertFalse(os.path.exists(os.path.join(self.dldir, "test-file.tar.gz.done")))
+        # ssh fetch does not support checksums
+        fetcher = bb.fetch.Fetch(["ssh://invalid@invalid.yoctoproject.org/test-file.tar.gz"], self.d)
+        # attempts to download with missing donestamp
+        with self.assertRaises(bb.fetch2.NetworkAccess):
+            fetcher.download()
+
+    def test_nochecksums_has_donestamp(self):
+        # create a file in the download directory with the donestamp
+        with open(os.path.join(self.dldir, "test-file.tar.gz"), "wb"):
+            pass
+        with open(os.path.join(self.dldir, "test-file.tar.gz.done"), "wb"):
+            pass
+
+        self.assertTrue(os.path.exists(os.path.join(self.dldir, "test-file.tar.gz")))
+        self.assertTrue(os.path.exists(os.path.join(self.dldir, "test-file.tar.gz.done")))
+        # ssh fetch does not support checksums
+        fetcher = bb.fetch.Fetch(["ssh://invalid@invalid.yoctoproject.org/test-file.tar.gz"], self.d)
+        # should not fetch
+        fetcher.download()
+        # both files should still exist
+        self.assertTrue(os.path.exists(os.path.join(self.dldir, "test-file.tar.gz")))
+        self.assertTrue(os.path.exists(os.path.join(self.dldir, "test-file.tar.gz.done")))
+
+    def test_nochecksums_missing_has_donestamp(self):
+        # create a file in the download directory with the donestamp
+        with open(os.path.join(self.dldir, "test-file.tar.gz.done"), "wb"):
+            pass
+
+        self.assertFalse(os.path.exists(os.path.join(self.dldir, "test-file.tar.gz")))
+        self.assertTrue(os.path.exists(os.path.join(self.dldir, "test-file.tar.gz.done")))
+        # ssh fetch does not support checksums
+        fetcher = bb.fetch.Fetch(["ssh://invalid@invalid.yoctoproject.org/test-file.tar.gz"], self.d)
+        with self.assertRaises(bb.fetch2.NetworkAccess):
+            fetcher.download()
+        # both files should still exist
+        self.assertFalse(os.path.exists(os.path.join(self.dldir, "test-file.tar.gz")))
+        self.assertFalse(os.path.exists(os.path.join(self.dldir, "test-file.tar.gz.done")))
+
 class FetcherNetworkTest(FetcherTest):
     @skipIfNoNetwork()
     def test_fetch(self):
@@ -728,7 +838,7 @@ class URLHandle(unittest.TestCase):
     # decodeurl and we need to handle them
     decodedata = datatable.copy()
     decodedata.update({
-       "http://somesite.net;someparam=1": ('http', 'somesite.net', '', '', '', {'someparam': '1'}),
+       "http://somesite.net;someparam=1": ('http', 'somesite.net', '/', '', '', {'someparam': '1'}),
     })
 
     def test_decodeurl(self):
@@ -809,7 +919,7 @@ class FetchLatestVersionTest(FetcherTest):
             ud = bb.fetch2.FetchData(k[1], self.d)
             pupver= ud.method.latest_versionstring(ud, self.d)
             verstring = pupver[0]
-            self.assertTrue(verstring, msg="Could not find upstream version")
+            self.assertTrue(verstring, msg="Could not find upstream version for %s" % k[0])
             r = bb.utils.vercmp_string(v, verstring)
             self.assertTrue(r == -1 or r == 0, msg="Package %s, version: %s <= %s" % (k[0], v, verstring))
 
@@ -822,7 +932,7 @@ class FetchLatestVersionTest(FetcherTest):
             ud = bb.fetch2.FetchData(k[1], self.d)
             pupver = ud.method.latest_versionstring(ud, self.d)
             verstring = pupver[0]
-            self.assertTrue(verstring, msg="Could not find upstream version")
+            self.assertTrue(verstring, msg="Could not find upstream version for %s" % k[0])
             r = bb.utils.vercmp_string(v, verstring)
             self.assertTrue(r == -1 or r == 0, msg="Package %s, version: %s <= %s" % (k[0], v, verstring))
 
@@ -874,9 +984,6 @@ class FetchCheckStatusTest(FetcherTest):
 
 
 class GitMakeShallowTest(FetcherTest):
-    bitbake_dir = os.path.join(os.path.dirname(os.path.join(__file__)), '..', '..', '..')
-    make_shallow_path = os.path.join(bitbake_dir, 'bin', 'git-make-shallow')
-
     def setUp(self):
         FetcherTest.setUp(self)
         self.gitdir = os.path.join(self.tempdir, 'gitshallow')
@@ -905,7 +1012,7 @@ class GitMakeShallowTest(FetcherTest):
     def make_shallow(self, args=None):
         if args is None:
             args = ['HEAD']
-        return bb.process.run([self.make_shallow_path] + args, cwd=self.gitdir)
+        return bb.process.run([bb.fetch2.git.Git.make_shallow_path] + args, cwd=self.gitdir)
 
     def add_empty_file(self, path, msg=None):
         if msg is None:

bitbake/lib/bb/utils.py

@@ -187,7 +187,7 @@ def explode_deps(s):
             #r[-1] += ' ' + ' '.join(j)
     return r
 
-def explode_dep_versions2(s):
+def explode_dep_versions2(s, *, sort=True):
     """
     Take an RDEPENDS style string of format:
     "DEPEND1 (optional version) DEPEND2 (optional version) ..."
@@ -250,7 +250,8 @@ def explode_dep_versions2(s):
         if not (i in r and r[i]):
             r[lastdep] = []
 
-    r = collections.OrderedDict(sorted(r.items(), key=lambda x: x[0]))
+    if sort:
+        r = collections.OrderedDict(sorted(r.items(), key=lambda x: x[0]))
     return r
 
 def explode_dep_versions(s):
@@ -806,8 +807,8 @@ def movefile(src, dest, newmtime = None, sstat = None):
                 return None # failure
         try:
             if didcopy:
-                os.lchown(dest, sstat[stat.ST_UID], sstat[stat.ST_GID])
-                os.chmod(dest, stat.S_IMODE(sstat[stat.ST_MODE])) # Sticky is reset on chown
+                os.lchown(destpath, sstat[stat.ST_UID], sstat[stat.ST_GID])
+                os.chmod(destpath, stat.S_IMODE(sstat[stat.ST_MODE])) # Sticky is reset on chown
                 os.unlink(src)
         except Exception as e:
             print("movefile: Failed to chown/chmod/unlink", dest, e)

bitbake/lib/toaster/bldcontrol/localhostbecontroller.py

@@ -192,7 +192,7 @@ class LocalhostBEController(BuildEnvironmentController):
             if commit != "HEAD":
                 logger.debug("localhostbecontroller: checking out commit %s to %s " % (commit, localdirname))
                 ref = commit if re.match('^[a-fA-F0-9]+$', commit) else 'origin/%s' % commit
-                self._shellcmd('git fetch --all && git reset --hard "%s"' % ref, localdirname,env=git_env)
+                self._shellcmd('git fetch && git reset --hard "%s"' % ref, localdirname,env=git_env)
 
             # take the localdirname as poky dir if we can find the oe-init-build-env
             if self.pokydirname is None and os.path.exists(os.path.join(localdirname, "oe-init-build-env")):

bitbake/lib/toaster/bldcontrol/management/commands/checksettings.py

@@ -107,7 +107,10 @@ class Command(BaseCommand):
                                 action="ignore",
                                 message="^.*No fixture named.*$")
                             print("Importing custom settings if present")
-                            call_command("loaddata", "custom")
+                            try:
+                                call_command("loaddata", "custom")
+                            except:
+                                print("NOTE: optional fixture 'custom' not found")
 
                         # we run lsupdates after config update
                         print("\nFetching information from the layer index, "

bitbake/lib/toaster/orm/fixtures/oe-core.xml

@@ -8,9 +8,9 @@
 
   <!-- Bitbake versions which correspond to the metadata release -->
   <object model="orm.bitbakeversion" pk="1">
-    <field type="CharField" name="name">rocko</field>
+    <field type="CharField" name="name">sumo</field>
     <field type="CharField" name="giturl">git://git.openembedded.org/bitbake</field>
-    <field type="CharField" name="branch">1.36</field>
+    <field type="CharField" name="branch">1.38</field>
   </object>
   <object model="orm.bitbakeversion" pk="2">
     <field type="CharField" name="name">HEAD</field>
@@ -22,14 +22,19 @@
     <field type="CharField" name="giturl">git://git.openembedded.org/bitbake</field>
     <field type="CharField" name="branch">master</field>
   </object>
+  <object model="orm.bitbakeversion" pk="4">
+    <field type="CharField" name="name">rocko</field>
+    <field type="CharField" name="giturl">git://git.openembedded.org/bitbake</field>
+    <field type="CharField" name="branch">1.36</field>
+  </object>
 
   <!-- Releases available -->
   <object model="orm.release" pk="1">
     <field type="CharField" name="name">rocko</field>
-    <field type="CharField" name="description">Openembedded Rocko</field>
+    <field type="CharField" name="description">Openembedded Sumo</field>
     <field rel="ManyToOneRel" to="orm.bitbakeversion" name="bitbake_version">1</field>
-    <field type="CharField" name="branch_name">rocko</field>
-    <field type="TextField" name="helptext">Toaster will run your builds using the tip of the &lt;a href=\"http://cgit.openembedded.org/openembedded-core/log/?h=rocko\"&gt;OpenEmbedded Rocko&lt;/a&gt; branch.</field>
+    <field type="CharField" name="branch_name">sumo</field>
+    <field type="TextField" name="helptext">Toaster will run your builds using the tip of the &lt;a href=\"http://cgit.openembedded.org/openembedded-core/log/?h=sumo\"&gt;OpenEmbedded Sumo&lt;/a&gt; branch.</field>
   </object>
   <object model="orm.release" pk="2">
     <field type="CharField" name="name">local</field>
@@ -45,6 +50,13 @@
     <field type="CharField" name="branch_name">master</field>
     <field type="TextField" name="helptext">Toaster will run your builds using the tip of the &lt;a href=\"http://cgit.openembedded.org/openembedded-core/log/\"&gt;OpenEmbedded master&lt;/a&gt; branch.</field>
   </object>
+  <object model="orm.release" pk="4">
+    <field type="CharField" name="name">rocko</field>
+    <field type="CharField" name="description">Openembedded Rocko</field>
+    <field rel="ManyToOneRel" to="orm.bitbakeversion" name="bitbake_version">1</field>
+    <field type="CharField" name="branch_name">rocko</field>
+    <field type="TextField" name="helptext">Toaster will run your builds using the tip of the &lt;a href=\"http://cgit.openembedded.org/openembedded-core/log/?h=rocko\"&gt;OpenEmbedded Rocko&lt;/a&gt; branch.</field>
+  </object>
 
   <!-- Default layers for each release -->
   <object model="orm.releasedefaultlayer" pk="1">
@@ -59,6 +71,10 @@
     <field rel="ManyToOneRel" to="orm.release" name="release">3</field>
     <field type="CharField" name="layer_name">openembedded-core</field>
   </object>
+  <object model="orm.releasedefaultlayer" pk="4">
+    <field rel="ManyToOneRel" to="orm.release" name="release">4</field>
+    <field type="CharField" name="layer_name">openembedded-core</field>
+  </object>
 
 
   <!-- Layer for the Local release -->

bitbake/lib/toaster/orm/fixtures/poky.xml

@@ -8,9 +8,9 @@
 
   <!-- Bitbake versions which correspond to the metadata release -->
   <object model="orm.bitbakeversion" pk="1">
-    <field type="CharField" name="name">rocko</field>
+    <field type="CharField" name="name">sumo</field>
     <field type="CharField" name="giturl">git://git.yoctoproject.org/poky</field>
-    <field type="CharField" name="branch">rocko</field>
+    <field type="CharField" name="branch">sumo</field>
     <field type="CharField" name="dirpath">bitbake</field>
   </object>
   <object model="orm.bitbakeversion" pk="2">
@@ -25,15 +25,21 @@
     <field type="CharField" name="branch">master</field>
     <field type="CharField" name="dirpath">bitbake</field>
   </object>
+  <object model="orm.bitbakeversion" pk="4">
+    <field type="CharField" name="name">rocko</field>
+    <field type="CharField" name="giturl">git://git.yoctoproject.org/poky</field>
+    <field type="CharField" name="branch">rocko</field>
+    <field type="CharField" name="dirpath">bitbake</field>
+  </object>
 
 
   <!-- Releases available -->
   <object model="orm.release" pk="1">
-    <field type="CharField" name="name">rocko</field>
-    <field type="CharField" name="description">Yocto Project 2.4 "Rocko"</field>
+    <field type="CharField" name="name">sumo</field>
+    <field type="CharField" name="description">Yocto Project 2.5 "Sumo"</field>
     <field rel="ManyToOneRel" to="orm.bitbakeversion" name="bitbake_version">1</field>
-    <field type="CharField" name="branch_name">rocko</field>
-    <field type="TextField" name="helptext">Toaster will run your builds using the tip of the &lt;a href="http://git.yoctoproject.org/cgit/cgit.cgi/poky/log/?h=rocko"&gt;Yocto Project Rocko branch&lt;/a&gt;.</field>
+    <field type="CharField" name="branch_name">sumo</field>
+    <field type="TextField" name="helptext">Toaster will run your builds using the tip of the &lt;a href="http://git.yoctoproject.org/cgit/cgit.cgi/poky/log/?h=sumo"&gt;Yocto Project Sumo branch&lt;/a&gt;.</field>
   </object>
   <object model="orm.release" pk="2">
     <field type="CharField" name="name">local</field>
@@ -49,6 +55,13 @@
     <field type="CharField" name="branch_name">master</field>
     <field type="TextField" name="helptext">Toaster will run your builds using the tip of the &lt;a href="http://git.yoctoproject.org/cgit/cgit.cgi/poky/log/"&gt;Yocto Project Master branch&lt;/a&gt;.</field>
   </object>
+  <object model="orm.release" pk="4">
+    <field type="CharField" name="name">rocko</field>
+    <field type="CharField" name="description">Yocto Project 2.4 "Rocko"</field>
+    <field rel="ManyToOneRel" to="orm.bitbakeversion" name="bitbake_version">1</field>
+    <field type="CharField" name="branch_name">rocko</field>
+    <field type="TextField" name="helptext">Toaster will run your builds using the tip of the &lt;a href="http://git.yoctoproject.org/cgit/cgit.cgi/poky/log/?h=rocko"&gt;Yocto Project Rocko branch&lt;/a&gt;.</field>
+  </object>
 
   <!-- Default project layers for each release -->
   <object model="orm.releasedefaultlayer" pk="1">
@@ -87,6 +100,18 @@
     <field rel="ManyToOneRel" to="orm.release" name="release">3</field>
     <field type="CharField" name="layer_name">meta-yocto-bsp</field>
   </object>
+  <object model="orm.releasedefaultlayer" pk="10">
+    <field rel="ManyToOneRel" to="orm.release" name="release">4</field>
+    <field type="CharField" name="layer_name">openembedded-core</field>
+  </object>
+  <object model="orm.releasedefaultlayer" pk="11">
+    <field rel="ManyToOneRel" to="orm.release" name="release">4</field>
+    <field type="CharField" name="layer_name">meta-poky</field>
+  </object>
+  <object model="orm.releasedefaultlayer" pk="12">
+    <field rel="ManyToOneRel" to="orm.release" name="release">4</field>
+    <field type="CharField" name="layer_name">meta-yocto-bsp</field>
+  </object>
 
   <!-- Default layers provided by poky
        openembedded-core
@@ -105,7 +130,7 @@
     <field rel="ManyToOneRel" to="orm.layer" name="layer">1</field>
     <field type="IntegerField" name="layer_source">0</field>
     <field rel="ManyToOneRel" to="orm.release" name="release">1</field>
-    <field type="CharField" name="branch">rocko</field>
+    <field type="CharField" name="branch">sumo</field>
     <field type="CharField" name="dirpath">meta</field>
   </object>
   <object model="orm.layer_version" pk="2">
@@ -123,6 +148,13 @@
     <field type="CharField" name="branch">master</field>
     <field type="CharField" name="dirpath">meta</field>
   </object>
+  <object model="orm.layer_version" pk="4">
+    <field rel="ManyToOneRel" to="orm.layer" name="layer">1</field>
+    <field type="IntegerField" name="layer_source">0</field>
+    <field rel="ManyToOneRel" to="orm.release" name="release">4</field>
+    <field type="CharField" name="branch">rocko</field>
+    <field type="CharField" name="dirpath">meta</field>
+  </object>
 
   <object model="orm.layer" pk="2">
     <field type="CharField" name="name">meta-poky</field>
@@ -132,14 +164,14 @@
     <field type="CharField" name="vcs_web_tree_base_url">http://git.yoctoproject.org/cgit/cgit.cgi/poky/tree/%path%?h=%branch%</field>
     <field type="CharField" name="vcs_web_file_base_url">http://git.yoctoproject.org/cgit/cgit.cgi/poky/tree/%path%?h=%branch%</field>
   </object>
-  <object model="orm.layer_version" pk="4">
+  <object model="orm.layer_version" pk="5">
     <field rel="ManyToOneRel" to="orm.layer" name="layer">2</field>
     <field type="IntegerField" name="layer_source">0</field>
     <field rel="ManyToOneRel" to="orm.release" name="release">1</field>
-    <field type="CharField" name="branch">rocko</field>
+    <field type="CharField" name="branch">sumo</field>
     <field type="CharField" name="dirpath">meta-poky</field>
   </object>
-  <object model="orm.layer_version" pk="5">
+  <object model="orm.layer_version" pk="6">
     <field rel="ManyToOneRel" to="orm.layer" name="layer">2</field>
     <field type="IntegerField" name="layer_source">0</field>
     <field rel="ManyToOneRel" to="orm.release" name="release">2</field>
@@ -147,13 +179,20 @@
     <field type="CharField" name="commit">HEAD</field>
     <field type="CharField" name="dirpath">meta-poky</field>
   </object>
-  <object model="orm.layer_version" pk="6">
+  <object model="orm.layer_version" pk="7">
     <field rel="ManyToOneRel" to="orm.layer" name="layer">2</field>
     <field type="IntegerField" name="layer_source">0</field>
     <field rel="ManyToOneRel" to="orm.release" name="release">3</field>
     <field type="CharField" name="branch">master</field>
     <field type="CharField" name="dirpath">meta-poky</field>
   </object>
+  <object model="orm.layer_version" pk="8">
+    <field rel="ManyToOneRel" to="orm.layer" name="layer">2</field>
+    <field type="IntegerField" name="layer_source">0</field>
+    <field rel="ManyToOneRel" to="orm.release" name="release">4</field>
+    <field type="CharField" name="branch">rocko</field>
+    <field type="CharField" name="dirpath">meta-poky</field>
+  </object>
 
   <object model="orm.layer" pk="3">
     <field type="CharField" name="name">meta-yocto-bsp</field>
@@ -163,14 +202,14 @@
     <field type="CharField" name="vcs_web_tree_base_url">http://git.yoctoproject.org/cgit/cgit.cgi/poky/tree/%path%?h=%branch%</field>
     <field type="CharField" name="vcs_web_file_base_url">http://git.yoctoproject.org/cgit/cgit.cgi/poky/tree/%path%?h=%branch%</field>
   </object>
-  <object model="orm.layer_version" pk="7">
+  <object model="orm.layer_version" pk="9">
     <field rel="ManyToOneRel" to="orm.layer" name="layer">3</field>
     <field type="IntegerField" name="layer_source">0</field>
     <field rel="ManyToOneRel" to="orm.release" name="release">1</field>
-    <field type="CharField" name="branch">rocko</field>
+    <field type="CharField" name="branch">sumo</field>
     <field type="CharField" name="dirpath">meta-yocto-bsp</field>
   </object>
-  <object model="orm.layer_version" pk="8">
+  <object model="orm.layer_version" pk="10">
     <field rel="ManyToOneRel" to="orm.layer" name="layer">3</field>
     <field type="IntegerField" name="layer_source">0</field>
     <field rel="ManyToOneRel" to="orm.release" name="release">2</field>
@@ -178,11 +217,18 @@
     <field type="CharField" name="commit">HEAD</field>
     <field type="CharField" name="dirpath">meta-yocto-bsp</field>
   </object>
-  <object model="orm.layer_version" pk="9">
+  <object model="orm.layer_version" pk="11">
     <field rel="ManyToOneRel" to="orm.layer" name="layer">3</field>
     <field type="IntegerField" name="layer_source">0</field>
     <field rel="ManyToOneRel" to="orm.release" name="release">3</field>
     <field type="CharField" name="branch">master</field>
     <field type="CharField" name="dirpath">meta-yocto-bsp</field>
   </object>
+  <object model="orm.layer_version" pk="12">
+    <field rel="ManyToOneRel" to="orm.layer" name="layer">3</field>
+    <field type="IntegerField" name="layer_source">0</field>
+    <field rel="ManyToOneRel" to="orm.release" name="release">4</field>
+    <field type="CharField" name="branch">rocko</field>
+    <field type="CharField" name="dirpath">meta-yocto-bsp</field>
+  </object>
 </django-objects>

documentation/Makefile

@@ -48,7 +48,7 @@
 # Examples:
 #
 #     make DOC=bsp-guide
-#     make html DOC=yocto-project-qs
+#     make html DOC=brief-yoctoprojectqs
 #     make pdf DOC=ref-manual
 #     make DOC=dev-manual BRANCH=edison
 #     make DOC=mega-manual BRANCH=denzil
@@ -56,7 +56,7 @@
 # The first example generates the HTML and Eclipse help versions of the BSP Guide.
 # The second example generates the HTML version only of the Quick Start.  Note
 # that the Quick Start only has an HTML version available.  So, the
-# 'make DOC=yocto-project-qs' command would be equivalent. The third example
+# 'make DOC=brief-yoctoprojectqs' command would be equivalent. The third example
 # generates just the PDF version of the Yocto Project Reference Manual.
 # The fourth example generates the HTML 'edison' version and (if available)
 # the Eclipse help version of the YP Development Tasks Manual.  The last example
@@ -84,6 +84,40 @@
 # for the 'denzil' branch.
 #
 
+ifeq ($(DOC),brief-yoctoprojectqs)
+XSLTOPTS = --stringparam html.stylesheet brief-yoctoprojectqs-style.css \
+           --stringparam  chapter.autolabel 0 \
+           --stringparam  section.autolabel 0 \
+           --stringparam  section.label.includes.component.label 0 \
+           --xinclude
+ALLPREQ = html eclipse tarball
+TARFILES = brief-yoctoprojectqs-style.css brief-yoctoprojectqs.html figures/bypqs-title.png \
+           figures/yocto-project-transp.png
+MANUALS = $(DOC)/$(DOC).html $(DOC)/eclipse
+FIGURES = figures
+STYLESHEET = $(DOC)/*.css
+
+endif
+
+ifeq ($(DOC),overview-manual)
+XSLTOPTS = --xinclude
+ALLPREQ = html eclipse tarball
+TARFILES = overview-manual-style.css overview-manual.html figures/overview-manual-title.png \
+           figures/git-workflow.png figures/source-repos.png figures/index-downloads.png \
+           figures/yp-download.png figures/YP-flow-diagram.png figures/key-dev-elements.png \
+           figures/poky-reference-distribution.png figures/cross-development-toolchains.png \
+           figures/user-configuration.png figures/layer-input.png figures/source-input.png \
+           figures/package-feeds.png figures/patching.png figures/source-fetching.png \
+           figures/configuration-compile-autoreconf.png figures/analysis-for-package-splitting.png \
+           figures/image-generation.png figures/sdk-generation.png figures/images.png \
+           figures/sdk.png \
+           eclipse
+MANUALS = $(DOC)/$(DOC).html $(DOC)/eclipse
+FIGURES = figures
+STYLESHEET = $(DOC)/*.css
+
+endif
+
 ifeq ($(DOC),bsp-guide)
 XSLTOPTS = --xinclude
 ALLPREQ = html eclipse tarball
@@ -128,8 +162,8 @@ TARFILES = dev-style.css dev-manual.html \
            figures/source-repos.png figures/yp-download.png \
            figures/wip.png
         else
-TARFILES = dev-style.css dev-manual.html \
-           figures/dev-title.png \
+TARFILES = dev-style.css dev-manual.html figures/buildhistory-web.png \
+           figures/dev-title.png figures/buildhistory.png \
            figures/recipe-workflow.png figures/bitbake-build-flow.png \
            eclipse
 	endif
@@ -140,22 +174,6 @@ STYLESHEET = $(DOC)/*.css
 
 endif
 
-ifeq ($(DOC),yocto-project-qs)
-XSLTOPTS = --stringparam html.stylesheet qs-style.css \
-           --stringparam  chapter.autolabel 1 \
-           --stringparam  section.autolabel 1 \
-           --stringparam  section.label.includes.component.label 1 \
-           --xinclude
-ALLPREQ = html eclipse tarball
-
-TARFILES = yocto-project-qs.html qs-style.css \
-           figures/yocto-project-transp.png figures/ypqs-title.png \
-           eclipse
-MANUALS = $(DOC)/$(DOC).html $(DOC)/eclipse
-FIGURES = figures
-STYLESHEET = $(DOC)/*.css
-endif
-
 ifeq ($(DOC),mega-manual)
 XSLTOPTS = --stringparam html.stylesheet mega-style.css \
            --stringparam  chapter.autolabel 1 \
@@ -196,7 +214,7 @@ TARFILES = mega-manual.html mega-style.css figures/yocto-environment.png \
 	figures/wip.png
         else
 TARFILES = mega-manual.html mega-style.css \
-        figures/building-an-image.png figures/YP-flow-diagram.png \
+        figures/YP-flow-diagram.png \
 	figures/using-a-pre-built-image.png \
 	figures/poky-title.png figures/buildhistory.png \
         figures/buildhistory-web.png \
@@ -234,22 +252,23 @@ TARFILES = mega-manual.html mega-style.css \
         figures/sched-wakeup-profile.png figures/sysprof-callers.png \
         figures/sysprof-copy-from-user.png figures/sysprof-copy-to-user.png \
         figures/cross-development-toolchains.png \
-	figures/yocto-environment-ref.png figures/user-configuration.png \
+	figures/user-configuration.png \
         figures/source-input.png figures/package-feeds.png \
         figures/layer-input.png figures/images.png figures/sdk.png \
 	figures/source-fetching.png figures/patching.png \
         figures/configuration-compile-autoreconf.png \
 	figures/analysis-for-package-splitting.png \
-        figures/image-generation.png \
+        figures/image-generation.png figures/key-dev-elements.png\
 	figures/sdk-generation.png figures/recipe-workflow.png \
 	figures/build-workspace-directory.png figures/mega-title.png \
 	figures/toaster-title.png figures/hosted-service.png \
-	figures/simple-configuration.png \
+	figures/simple-configuration.png figures/poky-reference-distribution.png \
 	figures/compatible-layers.png figures/import-layer.png figures/new-project.png \
 	figures/sdk-environment.png figures/sdk-installed-standard-sdk-directory.png \
 	figures/sdk-devtool-add-flow.png figures/sdk-installed-extensible-sdk-directory.png \
 	figures/sdk-devtool-modify-flow.png figures/sdk-eclipse-dev-flow.png \
-	figures/sdk-devtool-upgrade-flow.png figures/bitbake-build-flow.png figures/ypqs-title.png
+	figures/sdk-devtool-upgrade-flow.png figures/bitbake-build-flow.png figures/bypqs-title.png \
+	figures/overview-manual-title.png figures/sdk-autotools-flow.png figures/sdk-makefile-flow.png
 	endif
 
 MANUALS = $(DOC)/$(DOC).html
@@ -261,17 +280,9 @@ endif
 ifeq ($(DOC),ref-manual)
 XSLTOPTS = --xinclude
 ALLPREQ = html eclipse tarball
-TARFILES = ref-manual.html ref-style.css figures/poky-title.png figures/YP-flow-diagram.png \
-	figures/buildhistory.png figures/buildhistory-web.png eclipse \
-        figures/cross-development-toolchains.png figures/layer-input.png \
-	figures/package-feeds.png figures/source-input.png \
-	figures/user-configuration.png figures/yocto-environment-ref.png \
-	figures/images.png figures/sdk.png figures/source-fetching.png \
-	figures/patching.png figures/configuration-compile-autoreconf.png \
-	figures/analysis-for-package-splitting.png figures/image-generation.png \
-	figures/sdk-generation.png figures/building-an-image.png \
-	figures/build-workspace-directory.png figures/source-repos.png \
-	figures/index-downloads.png figures/yp-download.png figures/git-workflow.png
+TARFILES = ref-manual.html ref-style.css figures/poky-title.png \
+	figures/build-workspace-directory.png \
+	eclipse
 MANUALS = $(DOC)/$(DOC).html $(DOC)/eclipse
 FIGURES = figures
 STYLESHEET = $(DOC)/*.css
@@ -284,7 +295,7 @@ TARFILES = sdk-manual.html sdk-style.css figures/sdk-title.png \
            figures/sdk-environment.png figures/sdk-installed-standard-sdk-directory.png \
 	   figures/sdk-installed-extensible-sdk-directory.png figures/sdk-devtool-add-flow.png \
 	   figures/sdk-devtool-modify-flow.png figures/sdk-eclipse-dev-flow.png \
-	   figures/sdk-devtool-upgrade-flow.png \
+	   figures/sdk-devtool-upgrade-flow.png figures/sdk-autotools-flow.png figures/sdk-makefile-flow.png \
            eclipse
 MANUALS = $(DOC)/$(DOC).html $(DOC)/eclipse
 FIGURES = figures
@@ -360,9 +371,9 @@ XSL_XHTML_URI = $(XSL_BASE_URI)/xhtml/docbook.xsl
 all: $(ALLPREQ)
 
 pdf:
-ifeq ($(DOC),yocto-project-qs)
+ifeq ($(DOC),brief-yoctoprojectqs)
 	@echo " "
-	@echo "ERROR: You cannot generate a yocto-project-qs PDF file."
+	@echo "ERROR: You cannot generate a PDF file for brief-yoctoprojectqs."
 	@echo " "
 
 else ifeq ($(DOC),mega-manual)
@@ -406,17 +417,18 @@ eclipse: eclipse-generate eclipse-resolve-links
 .PHONY : eclipse-generate eclipse-resolve-links
 
 eclipse-generate:
-ifeq ($(filter $(DOC), sdk-manual bsp-guide dev-manual kernel-dev profile-manual ref-manual yocto-project-qs),)
+ifeq ($(filter $(DOC), overview-manual sdk-manual bsp-guide dev-manual kernel-dev profile-manual ref-manual brief-yoctoprojectqs),)
 	@echo " "
 	@echo "ERROR: You can only create eclipse documentation"
 	@echo "       of the following documentation parts:"
+	@echo "       - overview-manual"
 	@echo "       - sdk-manual"
 	@echo "       - bsp-guide"
 	@echo "       - dev-manual"
 	@echo "       - kernel-dev"
 	@echo "       - profile-manual"
 	@echo "       - ref-manual"
-	@echo "       - yocto-project-qs"
+	@echo "       - brief-yoctoprojectqs"
 	@echo " "
 else
 	@echo " "

documentation/brief-yoctoprojectqs/brief-yoctoprojectqs-customization.xsl

@@ -0,0 +1,24 @@
+<?xml version='1.0'?>
+<xsl:stylesheet xmlns:xsl="http://www.w3.org/1999/XSL/Transform" xmlns="http://www.w3.org/1999/xhtml" xmlns:fo="http://www.w3.org/1999/XSL/Format" version="1.0">
+
+  <xsl:import href="http://downloads.yoctoproject.org/mirror/docbook-mirror/docbook-xsl-1.76.1/xhtml/docbook.xsl" />
+
+<!--
+
+  <xsl:import href="../template/1.76.1/docbook-xsl-1.76.1/xhtml/docbook.xsl" />
+
+  <xsl:import href="http://docbook.sourceforge.net/release/xsl/1.76.1/xhtml/docbook.xsl" />
+
+-->
+
+  <xsl:import href="brief-yoctoprojectqs-titlepage.xsl"/>
+
+  <xsl:include href="../template/permalinks.xsl"/>
+  <xsl:include href="../template/section.title.xsl"/>
+  <xsl:include href="../template/component.title.xsl"/>
+  <xsl:include href="../template/division.title.xsl"/>
+  <xsl:include href="../template/formal.object.heading.xsl"/>
+
+  <xsl:param name="generate.toc" select="'article nop'"></xsl:param>
+  <xsl:param name="html.stylesheet" select="'brief-yoctoprojectqs-style.css'" />
+</xsl:stylesheet>

documentation/brief-yoctoprojectqs/brief-yoctoprojectqs-eclipse-customization.xsl

@@ -16,13 +16,13 @@
 
 -->
 
-  <xsl:import href="yocto-project-qs-titlepage.xsl"/>
+  <xsl:import href="brief-yoctoprojectqs-titlepage.xsl"/>
 
   <xsl:param name="chunker.output.indent" select="'yes'"/>
   <xsl:param name="chunk.quietly" select="1"/>
   <xsl:param name="use.id.as.filename" select="1"/>
   <xsl:param name="ulink.target" select="'_self'" />
-  <xsl:param name="base.dir" select="'html/yocto-project-qs/'"/>
+  <xsl:param name="base.dir" select="'html/brief-yoctoprojectqs/'"/>
   <xsl:param name="chunk.section.depth" select="0"/>
   <xsl:param name="html.stylesheet" select="'../book.css'"/>
   <xsl:param name="eclipse.manifest" select="0"/>
@@ -32,3 +32,4 @@
   <xsl:param name="generate.toc" select="'article nop'"></xsl:param>
   <xsl:param name="html.stylesheet" select="'style.css'" />
 </xsl:stylesheet>
+

documentation/brief-yoctoprojectqs/brief-yoctoprojectqs-style.css

@@ -118,7 +118,7 @@ h6 {
   background-color: transparent;
   background-repeat: no-repeat;
   padding-top: 256px;
-  background-image: url("figures/ypqs-title.png");
+  background-image: url("figures/bypqs-title.png");
   background-position: left top;
   margin-top: -256px;
   padding-right: 50px;

documentation/brief-yoctoprojectqs/brief-yoctoprojectqs.xml

@@ -0,0 +1,491 @@
+<!DOCTYPE article PUBLIC "-//OASIS//DTD DocBook XML V4.2//EN"
+"http://www.oasis-open.org/docbook/xml/4.2/docbookx.dtd"
+[<!ENTITY % poky SYSTEM "../poky.ent"> %poky; ] >
+
+<article id='brief-yocto-project-qs-intro'>
+    <articleinfo>
+        <title>Yocto Project Quick Build</title>
+
+        <copyright>
+            <year>&COPYRIGHT_YEAR;</year>
+            <holder>Linux Foundation</holder>
+        </copyright>
+
+        <legalnotice>
+            <para>
+                Permission is granted to copy, distribute and/or modify this document under
+                the terms of the <ulink type="http" url="http://creativecommons.org/licenses/by-sa/2.0/uk/">Creative Commons Attribution-Share Alike 2.0 UK: England &amp; Wales</ulink> as published by Creative Commons.
+            </para>
+        </legalnotice>
+
+
+        <abstract>
+            <imagedata fileref="figures/yocto-project-transp.png"
+                        width="6in" depth="1in"
+                        align="right" scale="25" />
+        </abstract>
+    </articleinfo>
+
+    <section id='brief-welcome'>
+        <title>Welcome!</title>
+
+        <para>
+            Welcome!
+            This short document steps you through the process for a typical
+            image build using the Yocto Project.
+            The document also introduces how to configure a build for specific
+            hardware.
+            You will use Yocto Project to build a reference embedded OS
+            called Poky.
+            <note>
+                The examples in this paper assume you are using a native Linux
+                system running a recent Ubuntu Linux distribution.
+                If the machine you want to use
+                Yocto Project on to build an image is not a native Linux
+                system, you can still perform these steps by using CROss
+                PlatformS (CROPS) and setting up a Poky container.
+                See the
+                <ulink url='&YOCTO_DOCS_DEV_URL;#setting-up-to-use-crops'>Setting Up to Use CROss PlatformS (CROPS)</ulink>"
+                section in the Yocto Project Development Tasks Manual for more
+                information.
+            </note>
+        </para>
+
+        <para>
+            If you want more conceptual or background information on the
+            Yocto Project, see the
+            <ulink url='&YOCTO_DOCS_OM_URL;'>Yocto Project Overview and Concepts Manual</ulink>.
+        </para>
+    </section>
+
+    <section id='brief-compatible-distro'>
+        <title>Compatible Linux Distribution</title>
+
+        <para>
+            Make sure your
+            <ulink url='&YOCTO_DOCS_REF_URL;#hardware-build-system-term'>build host</ulink>
+            meets the following requirements:
+            <itemizedlist>
+                <listitem><para>
+                    50 Gbytes of free disk space
+                    </para></listitem>
+                <listitem><para>
+                    Runs a supported Linux distribution (i.e. recent releases of
+                    Fedora, openSUSE, CentOS, Debian, or Ubuntu). For a list of
+                    Linux distributions that support the Yocto Project, see the
+                    "<ulink url='&YOCTO_DOCS_REF_URL;#detailed-supported-distros'>Supported Linux Distributions</ulink>"
+                    section in the Yocto Project Reference Manual.
+                    </para></listitem>
+                <listitem><para>
+                    <itemizedlist>
+                        <listitem><para>
+                            Git 1.8.3.1 or greater
+                            </para></listitem>
+                        <listitem><para>
+                            tar 1.27 or greater
+                            </para></listitem>
+                        <listitem><para>
+                            Python 3.4.0 or greater.
+                            </para></listitem>
+                    </itemizedlist>
+                    If your build host does not meet any of these three listed
+                    version requirements, you can take steps to prepare the
+                    system so that you can still use the Yocto Project.
+                    See the
+                    "<ulink url='&YOCTO_DOCS_REF_URL;#required-git-tar-and-python-versions'>Required Git, tar, and Python Versions</ulink>"
+                    section in the Yocto Project Reference Manual for information.
+                    </para></listitem>
+            </itemizedlist>
+        </para>
+    </section>
+
+    <section id='brief-build-system-packages'>
+        <title>Build Host Packages</title>
+
+        <para>
+            You must install essential host packages on your
+            build host.
+            The following command installs the host packages based on an
+            Ubuntu distribution:
+            <note>
+                For host package requirements on all supported Linux
+                distributions, see the
+                "<ulink url='&YOCTO_DOCS_REF_URL;#required-packages-for-the-host-development-system'>Required Packages for the Host Development System</ulink>"
+                section in the Yocto Project Reference Manual.
+            </note>
+            <literallayout class='monospaced'>
+     $ sudo apt-get install &UBUNTU_HOST_PACKAGES_ESSENTIAL; libsdl1.2-dev xterm
+            </literallayout>
+        </para>
+    </section>
+
+    <section id='brief-use-git-to-clone-poky'>
+        <title>Use Git to Clone Poky</title>
+
+        <para>
+            Once you complete the setup instructions for your machine,
+            you need to get a copy of the Poky repository on your build
+            host.
+            Use the following commands to clone the Poky
+            repository and then checkout the &DISTRO_REL_TAG; release:
+            <literallayout class='monospaced'>
+     $ git clone git://git.yoctoproject.org/poky
+     Cloning into 'poky'...
+     remote: Counting objects: 361782, done.
+     remote: Compressing objects: 100% (87100/87100), done.
+     remote: Total 361782 (delta 268619), reused 361439 (delta 268277)
+     Receiving objects: 100% (361782/361782), 131.94 MiB | 6.88 MiB/s, done.
+     Resolving deltas: 100% (268619/268619), done.
+     Checking connectivity... done.
+     $ git checkout tags/yocto-2.5 -b my-yocto-2.5
+            </literallayout>
+            The previous Git checkout command creates a local branch
+            named my-&DISTRO_REL_TAG;. The files available to you in that
+            branch exactly match the repository's files in the
+            "&DISTRO_NAME_NO_CAP;" development branch at the time of the
+            Yocto Project &DISTRO; release.
+        </para>
+
+        <para>
+            For more options and information about accessing Yocto
+            Project related repositories, see the
+            "<ulink url='&YOCTO_DOCS_DEV_URL;#locating-yocto-project-source-files'>Locating Yocto Project Source Files</ulink>"
+            section in the Yocto Project Development Tasks Manual.
+        </para>
+    </section>
+
+    <section id='brief-building-your-image'>
+        <title>Building Your Image</title>
+
+        <para>
+            Use the following steps to build your image.
+            The build process creates an entire Linux distribution, including
+            the toolchain, from source.
+            <note>
+                <itemizedlist>
+                    <listitem><para>
+                        If you are working behind a firewall and your build
+                        host is not set up for proxies, you could encounter
+                        problems with the build process when fetching source
+                        code (e.g. fetcher failures or Git failures).
+                        </para></listitem>
+                    <listitem><para>
+                        If you do not know your proxy settings, consult your
+                        local network infrastructure resources and get that
+                        information.
+                        A good starting point could also be to check your
+                        web browser settings.
+                        Finally, you can find more information on the
+                        "<ulink url='https://wiki.yoctoproject.org/wiki/Working_Behind_a_Network_Proxy'>Working Behind a Network Proxy</ulink>"
+                        page of the Yocto Project Wiki.
+                        </para></listitem>
+                </itemizedlist>
+            </note>
+        </para>
+
+        <para>
+            <orderedlist>
+                <listitem><para>
+                    <emphasis>Initialize the Build Environment:</emphasis>
+                    Run the
+                    <ulink url='&YOCTO_DOCS_REF_URL;#structure-core-script'><filename>&OE_INIT_FILE;</filename></ulink>
+                    environment setup script to define Yocto Project's
+                    build environment on your build host.
+                    <literallayout class='monospaced'>
+     $ source &OE_INIT_FILE;
+                    </literallayout>
+                    Among other things, the script creates the
+                    <ulink url='&YOCTO_DOCS_REF_URL;#build-directory'>Build Directory</ulink>,
+                    which is <filename>build</filename> in this case
+                    and is located in the
+                    <ulink url='&YOCTO_DOCS_REF_URL;#source-directory'>Source Directory</ulink>.
+                    After the script runs, your current working directory
+                    is set to the Build Directory.
+                    Later, when the build completes, the Build Directory
+                    contains all the files created during the build.
+                    </para></listitem>
+                <listitem><para id='conf-file-step'>
+                    <emphasis>Examine Your Local Configuration File:</emphasis>
+                    When you set up the build environment, a local
+                    configuration file named
+                    <filename>local.conf</filename> becomes available in
+                    a <filename>conf</filename> subdirectory of the
+                    Build Directory.
+                    For this example, the defaults are set to build
+                    for a <filename>qemux86</filename> target, which is
+                    suitable for emulation.
+                    The package manager used is set to the RPM package
+                    manager.
+                    <tip>
+                        You can significantly speed up your build and guard
+                        against fetcher failures by using mirrors.
+                        To use mirrors, add these lines to your
+                        <filename>local.conf</filename> file in the Build
+                        directory:
+                        <literallayout class='monospaced'>
+     SSTATE_MIRRORS = "\
+     file://.* http://sstate.yoctoproject.org/dev/PATH;downloadfilename=PATH \n \
+     file://.* http://sstate.yoctoproject.org/&YOCTO_DOC_VERSION_MINUS_ONE;/PATH;downloadfilename=PATH \n \
+     file://.* http://sstate.yoctoproject.org/&YOCTO_DOC_VERSION;/PATH;downloadfilename=PATH \n \
+     "
+                        </literallayout>
+                        The previous examples showed how to add sstate
+                        paths for Yocto Project &YOCTO_DOC_VERSION_MINUS_ONE;,
+                        &YOCTO_DOC_VERSION;, and a development area.
+                        For a complete index of sstate locations, see
+                        <ulink url='http://sstate.yoctoproject.org/'></ulink>.
+                    </tip>
+                    </para></listitem>
+                <listitem><para>
+                    <emphasis>Start the Build:</emphasis>
+                    Continue with the following command to build an OS image
+                    for the target, which is
+                    <filename>core-image-sato</filename> in this example:
+                    <literallayout class='monospaced'>
+     $ bitbake core-image-sato
+                    </literallayout>
+                    For information on using the
+                    <filename>bitbake</filename> command, see the
+                    "<ulink url='&YOCTO_DOCS_OM_URL;#usingpoky-components-bitbake'>BitBake</ulink>"
+                    section in the Yocto Project Overview and Concepts Manual,
+                    or see the
+                    "<ulink url='&YOCTO_DOCS_BB_URL;#bitbake-user-manual-command'>BitBake Command</ulink>"
+                    section in the BitBake User Manual.
+                    </para></listitem>
+                <listitem><para>
+                    <emphasis>Simulate Your Image Using QEMU:</emphasis>
+                    Once this particular image is built, you can start
+                    QEMU, which is a Quick EMUlator that ships with
+                    the Yocto Project:
+                    <literallayout class='monospaced'>
+     $ runqemu qemux86
+                    </literallayout>
+                    If you want to learn more about running QEMU, see the
+                    "<ulink url="&YOCTO_DOCS_DEV_URL;#dev-manual-qemu">Using the Quick EMUlator (QEMU)</ulink>"
+                    chapter in the Yocto Project Development Tasks Manual.
+                    </para></listitem>
+                <listitem><para>
+                    <emphasis>Exit QEMU:</emphasis>
+                    Exit QEMU by either clicking on the shutdown icon or by
+                    typing <filename>Ctrl-C</filename> in the QEMU
+                    transcript window from which you evoked QEMU.
+                    </para></listitem>
+            </orderedlist>
+        </para>
+    </section>
+
+    <section id='customizing-your-build-for-specific-hardware'>
+        <title>Customizing Your Build for Specific Hardware</title>
+
+        <para>
+            So far, all you have done is quickly built an image suitable
+            for emulation only.
+            This section shows you how to customize your build for specific
+            hardware by adding a hardware layer into the Yocto Project
+            development environment.
+        </para>
+
+        <para>
+            In general, layers are repositories that contain related sets of
+            instructions and configurations that tell the Yocto Project what
+            to do.
+            Isolating related metadata into functionally specific layers
+            facilitates modular development and makes it easier to reuse the
+            layer metadata.
+            <note>
+                By convention, layer names start with the string "meta-".
+            </note>
+        </para>
+
+        <para>
+            Follow these steps to add a hardware layer:
+            <orderedlist>
+                <listitem><para>
+                    <emphasis>Find a Layer:</emphasis>
+                    Lots of hardware layers exist.
+                    The Yocto Project
+                    <ulink url='&YOCTO_GIT_URL;'>Source Repositories</ulink>
+                    has many hardware layers.
+                    This example adds the
+                    <ulink url='https://github.com/kraj/meta-altera'>meta-altera</ulink>
+                    hardware layer.
+                    </para></listitem>
+                <listitem><para>
+                    <emphasis>Clone the Layer</emphasis>
+                    Use Git to make a local copy of the layer on your machine.
+                    You can put the copy in the top level of the copy of the
+                    Poky repository created earlier:
+                    <literallayout class='monospaced'>
+     $ cd ~/poky
+     $ git clone https://github.com/kraj/meta-altera.git
+     Cloning into 'meta-altera'...
+     remote: Counting objects: 25170, done.
+     remote: Compressing objects: 100% (350/350), done.
+     remote: Total 25170 (delta 645), reused 719 (delta 538), pack-reused 24219
+     Receiving objects: 100% (25170/25170), 41.02 MiB | 1.64 MiB/s, done.
+     Resolving deltas: 100% (13385/13385), done.
+     Checking connectivity... done.
+                    </literallayout>
+                    The hardware layer now exists with other layers inside
+                    the Poky reference repository on your build host as
+                    <filename>meta-altera</filename> and contains all the
+                    metadata needed to support hardware from Altera, which
+                    is owned by Intel.
+                    </para></listitem>
+                <listitem><para>
+                    <emphasis>Change the Configuration to Build for a Specific Machine:</emphasis>
+                    The
+                    <ulink url='&YOCTO_DOCS_REF_URL;#var-MACHINE'><filename>MACHINE</filename></ulink>
+                    variable in the <filename>local.conf</filename> file
+                    specifies the machine for the build.
+                    For this example, set the <filename>MACHINE</filename>
+                    variable to "cyclone5".
+                    These configurations are used:
+                    <ulink url='https://github.com/kraj/meta-altera/blob/master/conf/machine/cyclone5.conf'></ulink>.
+                    <note>
+                        See the
+                        "<link linkend='conf-file-step'>Examine Your Local Configuration File</link>"
+                        step earlier for more information on configuring the
+                        build.
+                    </note>
+                    </para></listitem>
+                <listitem><para>
+                    <emphasis>Add Your Layer to the Layer Configuration File:</emphasis>
+                    Before you can use a layer during a build, you must add it
+                    to your <filename>bblayers.conf</filename> file, which
+                    is found in the
+                    <ulink url='&YOCTO_DOCS_REF_URL;#build-directory'>Build Directory's</ulink>
+                    <filename>conf</filename> directory.</para>
+
+                    <para>Use the <filename>bitbake-layers add-layer</filename>
+                    command to add the layer to the configuration file:
+                    <literallayout class='monospaced'>
+     $ cd ~/poky/build
+     $ bitbake-layers add-layer ../meta-altera
+     NOTE: Starting bitbake server...
+     Parsing recipes: 100% |##################################################################| Time: 0:00:32
+     Parsing of 918 .bb files complete (0 cached, 918 parsed). 1401 targets, 123 skipped, 0 masked, 0 errors.
+                    </literallayout>
+                    You can find more information on adding layers in the
+                    "<ulink url='&YOCTO_DOCS_DEV_URL;#adding-a-layer-using-the-bitbake-layers-script'>Adding a Layer Using the <filename>bitbake-layers</filename> Script</ulink>"
+                    section.
+                    </para></listitem>
+            </orderedlist>
+            Completing these steps has added the
+            <filename>meta-altera</filename> layer to your Yocto Project
+            development environment and configured it to build for the
+            "cyclone5" machine.
+            <note>
+                The previous steps are for demonstration purposes only.
+                If you were to attempt to build an image for the
+                "cyclone5" build, you should read the Altera
+                <filename>README</filename>.
+            </note>
+        </para>
+    </section>
+
+    <section id='creating-your-own-general-layer'>
+        <title>Creating Your Own General Layer</title>
+
+        <para>
+            Maybe you have an application or specific set of behaviors you
+            need to isolate.
+            You can create your own general layer using the
+            <filename>bitbake-layers create-layer</filename> command.
+            The tool automates layer creation by setting up a
+            subdirectory with a <filename>layer.conf</filename>
+            configuration file, a <filename>recipes-example</filename>
+            subdirectory that contains an <filename>example.bb</filename>
+            recipe, a licensing file, and a <filename>README</filename>.
+        </para>
+
+        <para>
+            The following commands run the tool to create a layer named
+            <filename>meta-mylayer</filename> in the
+            <filename>poky</filename> directory:
+            <literallayout class='monospaced'>
+     $ cd ~/poky
+     $ bitbake-layers create-layer meta-mylayer
+     NOTE: Starting bitbake server...
+     Add your new layer with 'bitbake-layers add-layer meta-mylayer'
+            </literallayout>
+            For more information on layers and how to create them, see the
+            "<ulink url='&YOCTO_DOCS_DEV_URL;#creating-a-general-layer-using-the-bitbake-layers-script'>Creating a General Layer Using the <filename>bitbake-layers</filename> Script</ulink>"
+            section in the Yocto Project Development Tasks Manual.
+        </para>
+    </section>
+
+    <section id='brief-where-to-go-next'>
+        <title>Where To Go Next</title>
+
+        <para>
+            Now that you have experienced using the Yocto Project, you might
+            be asking yourself "What now?"
+            The Yocto Project has many sources of information including
+            the website, wiki pages, and user manuals:
+            <itemizedlist>
+                <listitem><para>
+                    <emphasis>Website:</emphasis>
+                    The
+                    <ulink url='&YOCTO_HOME_URL;'>Yocto Project Website</ulink>
+                    provides background information, the latest builds,
+                    breaking news, full development documentation, and
+                    access to a rich Yocto Project Development Community
+                    into which you can tap.
+                    </para></listitem>
+                <listitem><para>
+                    <emphasis>Developer Screencast:</emphasis>
+                    The
+                    <ulink url='http://vimeo.com/36450321'>Getting Started with the Yocto Project - New Developer Screencast Tutorial</ulink>
+                    provides a 30-minute video created for users unfamiliar
+                    with the Yocto Project but familiar with Linux build
+                    hosts.
+                    While this screencast is somewhat dated, the
+                    introductory and fundamental concepts are useful for
+                    the beginner.
+                    </para></listitem>
+                <listitem><para>
+                    <emphasis>Yocto Project Overview and Concepts Manual:</emphasis>
+                    The
+                    <ulink url='&YOCTO_DOCS_OM_URL;'>Yocto Project Overview and Concepts Manual</ulink>
+                    is a great place to start to learn about the
+                    Yocto Project.
+                    This manual introduces you to the Yocto Project and its
+                    development environment.
+                    The manual also provides conceptual information for
+                    various aspects of the Yocto Project.
+                    </para></listitem>
+                <listitem><para>
+                    <emphasis>Yocto Project Wiki:</emphasis>
+                    The
+                    <ulink url='&YOCTO_WIKI_URL;'>Yocto Project Wiki</ulink>
+                    provides additional information on where to go next
+                    when ramping up with the Yocto Project, release
+                    information, project planning, and QA information.
+                    </para></listitem>
+                <listitem><para>
+                    <emphasis>Yocto Project Mailing Lists:</emphasis>
+                    Related mailing lists provide a forum for discussion,
+                    patch submission and announcements.
+                    Several mailing lists exist and are grouped according
+                    to areas of concern.
+                    See the
+                    "<ulink url='&YOCTO_DOCS_REF_URL;#resources-mailinglist'>Mailing lists</ulink>"
+                    section in the Yocto Project Reference Manual for a
+                    complete list of Yocto Project mailing lists.
+                    </para></listitem>
+                <listitem><para>
+                    <emphasis>Comprehensive List of Links and Other Documentation:</emphasis>
+                    The
+                    "<ulink url='&YOCTO_DOCS_REF_URL;#resources-links-and-related-documentation'>Links and Related Documentation</ulink>"
+                    section in the Yocto Project Reference Manual provides a
+                    comprehensive list of all related links and other
+                    user documentation.
+                    </para></listitem>
+            </itemizedlist>
+        </para>
+    </section>
+</article>
+<!--
+vim: expandtab tw=80 ts=4
+-->

documentation/bsp-guide/bsp-guide.xml

@@ -118,9 +118,14 @@
             </revision>
             <revision>
                 <revnumber>2.5</revnumber>
-                <date>April 2018</date>
+                <date>May 2018</date>
                 <revremark>Released with the Yocto Project 2.5 Release.</revremark>
             </revision>
+            <revision>
+                <revnumber>2.6</revnumber>
+                <date>&REL_MONTH_YEAR;</date>
+                <revremark>Released with the Yocto Project 2.6 Release.</revremark>
+            </revision>
         </revhistory>
 
     <copyright>
@@ -137,28 +142,40 @@
                <itemizedlist>
                    <listitem><para>
                        This version of the
-                       <emphasis>Yocto Project Board Support Package Developer's Guide</emphasis>
+                       <emphasis>Yocto Project Board Support Package (BSP) Developer's Guide</emphasis>
                        is for the &YOCTO_DOC_VERSION; release of the
                        Yocto Project.
                        To be sure you have the latest version of the manual
-                       for this release, use the manual from the
-                       <ulink url='&YOCTO_HOME_URL;/documentation'>Yocto Project documentation page</ulink>.
-                       </para></listitem>
-                   <listitem><para>
-                       For manuals associated with other releases of the Yocto
-                       Project, go to the
+                       for this release, go to the
                        <ulink url='&YOCTO_HOME_URL;/documentation'>Yocto Project documentation page</ulink>
-                       and use the drop-down "Active Releases" button
-                       and choose the manual associated with the desired
-                       Yocto Project.
+                       and select the manual from that site.
+                       Manuals from the site are more up-to-date than manuals
+                       derived from the Yocto Project released TAR files.
                        </para></listitem>
                    <listitem><para>
-                        To report any inaccuracies or problems with this
-                        manual, send an email to the Yocto Project
-                        discussion group at
-                        <filename>yocto@yoctoproject.com</filename> or log into
-                        the freenode <filename>#yocto</filename> channel.
-                        </para></listitem>
+                       If you located this manual through a web search, the
+                       version of the manual might not be the one you want
+                       (e.g. the search might have returned a manual much
+                       older than the Yocto Project version with which you
+                       are working).
+                       You can see all Yocto Project major releases by
+                       visiting the
+                       <ulink url='&YOCTO_WIKI_URL;/wiki/Releases'>Releases</ulink>
+                       page.
+                       If you need a version of this manual for a different
+                       Yocto Project release, visit the
+                       <ulink url='&YOCTO_HOME_URL;/documentation'>Yocto Project documentation page</ulink>
+                       and select the manual set by using the
+                       "ACTIVE RELEASES DOCUMENTATION" or "DOCUMENTS ARCHIVE"
+                       pull-down menus.
+                       </para></listitem>
+                   <listitem><para>
+                       To report any inaccuracies or problems with this
+                       manual, send an email to the Yocto Project
+                       discussion group at
+                       <filename>yocto@yoctoproject.com</filename> or log into
+                       the freenode <filename>#yocto</filename> channel.
+                       </para></listitem>
                </itemizedlist>
            </note>
     </legalnotice>

documentation/dev-manual/dev-manual-intro.xml

@@ -16,33 +16,29 @@
             The manual groups related procedures into higher-level sections.
             Procedures can consist of high-level steps or low-level steps
             depending on the topic.
-            You can find conceptual information related to a procedure by
-            following appropriate links to the Yocto Project Reference
-            Manual.
         </para>
 
         <para>
             The following list describes what you can get from this manual:
             <itemizedlist>
                 <listitem><para>
-                    <emphasis>Setup Procedures:</emphasis>
-                    Procedures that show you how to set
-                    up a Yocto Project Development environment and how
-                    to accomplish the change workflow through logging
-                    defects and submitting changes.
+                    Procedures that help you get going with the Yocto Project.
+                    For example, procedures that show you how to set up
+                    a build host and work with the Yocto Project
+                    source repositories.
                     </para></listitem>
                 <listitem><para>
-                    <emphasis>Emulation Procedures:</emphasis>
-                    Procedures that show you how to use the
-                    Yocto Project integrated QuickEMUlator (QEMU), which lets
-                    you simulate running on hardware an image you have built
-                    using the OpenEmbedded build system.
+                    Procedures that show you how to submit changes to the
+                    Yocto Project.
+                    Changes can be improvements, new features, or bug
+                    fixes.
                     </para></listitem>
                 <listitem><para>
-                    <emphasis>Common Procedures:</emphasis>
                     Procedures related to "everyday" tasks you perform while
                     developing images and applications using the Yocto
                     Project.
+                    For example, procedures to create a layer, customize an
+                    image, write a new recipe, and so forth.
                     </para></listitem>
             </itemizedlist>
         </para>
@@ -51,7 +47,7 @@
             This manual will not give you the following:
             <itemizedlist>
                 <listitem><para>
-                    <emphasis>Redundant Step-by-step Instructions:</emphasis>
+                    Redundant Step-by-step Instructions:
                     For example, the
                     <ulink url='&YOCTO_DOCS_SDK_URL;'>Yocto Project Application Development and the Extensible Software Development Kit (eSDK)</ulink>
                     manual contains detailed instructions on how to install an
@@ -59,14 +55,15 @@
                     hardware.
                     </para></listitem>
                 <listitem><para>
-                    <emphasis>Reference or Conceptual Material:</emphasis>
-                    This type of material resides in an appropriate reference manual.
+                    Reference or Conceptual Material:
+                    This type of material resides in an appropriate reference
+                    manual.
                     For example, system variables are documented in the
                     <ulink url='&YOCTO_DOCS_REF_URL;'>Yocto Project Reference Manual</ulink>.
                     </para></listitem>
                 <listitem><para>
-                    <emphasis>Detailed Public Information Not Specific to the
-                    Yocto Project:</emphasis>
+                    Detailed Public Information Not Specific to the
+                    Yocto Project:
                     For example, exhaustive information on how to use the
                     Source Control Manager Git is better covered with Internet
                     searches and official Git Documentation than through the
@@ -85,9 +82,10 @@
             comprehension.
             For introductory information on the Yocto Project, see the
             <ulink url='&YOCTO_HOME_URL;'>Yocto Project Website</ulink>.
-            You can find an introductory to using the Yocto Project by working
-            through the
-            <ulink url='&YOCTO_DOCS_QS_URL;'>Yocto Project Quick Start</ulink>.
+            If you want to build an image with no knowledge of Yocto Project
+            as a way of quickly testing it out, see the
+            <ulink url='&YOCTO_DOCS_BRIEF_URL;'>Yocto Project Quick Build</ulink>
+            document.
         </para>
 
         <para>

documentation/dev-manual/dev-manual-newbie.xml

@@ -1,989 +0,0 @@
-<!DOCTYPE chapter PUBLIC "-//OASIS//DTD DocBook XML V4.2//EN"
-"http://www.oasis-open.org/docbook/xml/4.2/docbookx.dtd"
-[<!ENTITY % poky SYSTEM "../poky.ent"> %poky; ] >
-
-<chapter id='dev-manual-newbie'>
-
-<title>The Yocto Project Open Source Development Environment</title>
-
-<section id="usingpoky-changes-collaborate">
-    <title>Setting Up a Team Yocto Project Development Environment</title>
-
-    <para>
-        It might not be immediately clear how you can use the Yocto
-        Project in a team development environment, or scale it for a large
-        team of developers.
-        One of the strengths of the Yocto Project is that it is extremely
-        flexible.
-        Thus, you can adapt it to many different use cases and scenarios.
-        However, these characteristics can cause a struggle if you are trying
-        to create a working setup that scales across a large team.
-    </para>
-
-    <para>
-        To help you understand how to set up this type of environment,
-        this section presents a procedure that gives you the information
-        to learn how to get the results you want.
-        The procedure is high-level and presents some of the project's most
-        successful experiences, practices, solutions, and available
-        technologies that work well.
-        Keep in mind, the procedure here is a starting point.
-        You can build off it and customize it to fit any
-        particular working environment and set of practices.
-        <orderedlist>
-            <listitem><para>
-                <emphasis>Determine Who is Going to be Developing:</emphasis>
-                You need to understand who is going to be doing anything
-                related to the Yocto Project and what their roles would be.
-                Making this determination is essential to completing the
-                steps two and three, which are to get your equipment together
-                and set up your development environment's hardware topology.
-                </para>
-
-                <para>The following roles exist:
-                <itemizedlist>
-                    <listitem><para>
-                        <emphasis>Application Development:</emphasis>
-                        These types of developers do application level work
-                        on top of an existing software stack.
-                        </para></listitem>
-                    <listitem><para>
-                        <emphasis>Core System Development:</emphasis>
-                        These types of developers work on the contents of the
-                        operating system image itself.
-                        </para></listitem>
-                    <listitem><para>
-                        <emphasis>Build Engineer:</emphasis>
-                        This type of developer manages Autobuilders and
-                        releases.
-                        Not all environments need a Build Engineer.
-                        </para></listitem>
-                    <listitem><para>
-                        <emphasis>Test Engineer:</emphasis>
-                        This type of developer creates and manages automated
-                        tests needed to ensure all application and core
-                        system development meets desired quality standards.
-                        </para></listitem>
-                </itemizedlist>
-                </para></listitem>
-            <listitem><para>
-                <emphasis>Gather the Hardware:</emphasis>
-                Based on the size and make-up of the team, get the hardware
-                together.
-                Any development, build, or test engineer should be using
-                a system that is running a supported Linux distribution.
-                Systems, in general, should be high performance (e.g. dual,
-                six-core Xeons with 24 Gbytes of RAM and plenty of disk space).
-                You can help ensure efficiency by having any machines used
-                for testing or that run Autobuilders be as high performance
-                as possible.
-                </para></listitem>
-            <listitem><para>
-                <emphasis>Understand the Hardware Topology of the Environment:</emphasis>
-                Now that you know how many developers and support engineers
-                are required, you can understand the topology of the
-                hardware environment.
-                The following figure shows a moderately sized Yocto Project
-                development environment.
-
-                <para role="writernotes">
-                Need figure.</para>
-
-                </para></listitem>
-            <listitem><para>
-                <emphasis>Use Git as Your Source Control Manager (SCM):</emphasis>
-                Keeping your
-                <ulink url='&YOCTO_DOCS_REF_URL;#metadata'>Metadata</ulink>
-                and any software you are developing under the
-                control of an SCM system that is compatible
-                with the OpenEmbedded build system is advisable.
-                Of the SCMs BitBake supports, the
-                Yocto Project team strongly recommends using
-                <ulink url='&YOCTO_DOCS_REF_URL;#git'>Git</ulink>.
-                Git is a distributed system that is easy to backup,
-                allows you to work remotely, and then connects back to the
-                infrastructure.
-                <note>
-                    For information about BitBake, see the
-                    <ulink url='&YOCTO_DOCS_BB_URL;'>BitBake User Manual</ulink>.
-                </note></para>
-
-                <para>It is relatively easy to set up Git services and create
-                infrastructure like
-                <ulink url='&YOCTO_GIT_URL;'>http://git.yoctoproject.org</ulink>,
-                which is based on server software called
-                <filename>gitolite</filename> with <filename>cgit</filename>
-                being used to generate the web interface that lets you view the
-                repositories.
-                The <filename>gitolite</filename> software identifies users
-                using SSH keys and allows branch-based
-                access controls to repositories that you can control as little
-                or as much as necessary.
-
-                <note>
-                   The setup of these services is beyond the scope of this
-                   manual.
-                   However, sites such as these exist that describe how to
-                   perform setup:
-                   <itemizedlist>
-                       <listitem><para>
-                           <ulink url='http://git-scm.com/book/ch4-8.html'>Git documentation</ulink>:
-                           Describes how to install <filename>gitolite</filename>
-                           on the server.
-                           </para></listitem>
-                       <listitem><para>
-                           <ulink url='http://sitaramc.github.com/gitolite/master-toc.html'>The <filename>gitolite</filename> master index</ulink>:
-                            All topics for <filename>gitolite</filename>.
-                            </para></listitem>
-                        <listitem><para>
-                            <ulink url='https://git.wiki.kernel.org/index.php/Interfaces,_frontends,_and_tools'>Interfaces, frontends, and tools</ulink>:
-                            Documentation on how to create interfaces and frontends
-                            for Git.
-                            </para></listitem>
-                    </itemizedlist>
-                </note>
-                </para></listitem>
-            <listitem><para>
-                <emphasis>Set up the Application Development Machines:</emphasis>
-                As mentioned earlier, application developers are creating
-                applications on top of existing software stacks.
-                Following are some best practices for setting up machines
-                that do application development:
-                <itemizedlist>
-                    <listitem><para>
-                        Use a pre-built toolchain that
-                        contains the software stack itself.
-                        Then, develop the application code on top of the
-                        stack.
-                        This method works well for small numbers of relatively
-                        isolated applications.
-                        </para></listitem>
-                    <listitem><para>
-                        When possible, use the Yocto Project
-                        plug-in for the
-                        <trademark class='trade'>Eclipse</trademark> IDE
-                        and SDK development practices.
-                        For more information, see the
-                        "<ulink url='&YOCTO_DOCS_SDK_URL;'>Yocto Project Application Development and the Extensible Software Development Kit (eSDK)</ulink>"
-                        manual.
-                        </para></listitem>
-                    <listitem><para>
-                        Keep your cross-development toolchains updated.
-                        You can do this through provisioning either as new
-                        toolchain downloads or as updates through a package
-                        update mechanism using <filename>opkg</filename>
-                        to provide updates to an existing toolchain.
-                        The exact mechanics of how and when to do this are a
-                        question for local policy.
-                        </para></listitem>
-                    <listitem><para>
-                        Use multiple toolchains installed locally
-                        into different locations to allow development across
-                        versions.
-                        </para></listitem>
-                </itemizedlist>
-                </para></listitem>
-            <listitem><para>
-                <emphasis>Set up the Core Development Machines:</emphasis>
-                As mentioned earlier, these types of developers work on the
-                contents of the operating system itself.
-                Following are some best practices for setting up machines
-                used for developing images:
-                <itemizedlist>
-                    <listitem><para>
-                        Have the Yocto Project build system itself available on
-                        the developer workstations so developers can run their own
-                        builds and directly rebuild the software stack.
-                        </para></listitem>
-                    <listitem><para>
-                        Keep the core system unchanged as much as
-                        possible and do your work in layers on top of the
-                        core system.
-                        Doing so gives you a greater level of portability when
-                        upgrading to new versions of the core system or Board
-                        Support Packages (BSPs).
-                        </para></listitem>
-                    <listitem><para>
-                        Share layers amongst the developers of a
-                        particular project and contain the policy configuration
-                        that defines the project.
-                        </para></listitem>
-                </itemizedlist>
-                </para></listitem>
-            <listitem><para>
-                <emphasis>Set up an Autobuilder:</emphasis>
-                Autobuilders are often the core of the development
-                environment.
-                It is here that changes from individual developers are brought
-                together and centrally tested and subsequent decisions about
-                releases can be made.
-                Autobuilders also allow for "continuous integration" style
-                testing of software components and regression identification
-                and tracking.</para>
-
-                <para>See "<ulink url='http://autobuilder.yoctoproject.org'>Yocto Project Autobuilder</ulink>"
-                for more information and links to buildbot.
-                The Yocto Project team has found this implementation
-                works well in this role.
-                A public example of this is the Yocto Project
-                Autobuilders, which we use to test the overall health of the
-                project.</para>
-
-                <para>The features of this system are:
-                <itemizedlist>
-                    <listitem><para>
-                         Highlights when commits break the build.
-                         </para></listitem>
-                    <listitem><para>
-                        Populates an sstate cache from which
-                        developers can pull rather than requiring local
-                        builds.
-                        </para></listitem>
-                    <listitem><para>
-                        Allows commit hook triggers,
-                        which trigger builds when commits are made.
-                        </para></listitem>
-                    <listitem><para>
-                        Allows triggering of automated image booting
-                        and testing under the QuickEMUlator (QEMU).
-                        </para></listitem>
-                    <listitem><para>
-                        Supports incremental build testing and
-                        from-scratch builds.
-                        </para></listitem>
-                    <listitem><para>
-                        Shares output that allows developer
-                        testing and historical regression investigation.
-                        </para></listitem>
-                    <listitem><para>
-                        Creates output that can be used for releases.
-                        </para></listitem>
-                    <listitem><para>
-                        Allows scheduling of builds so that resources
-                        can be used efficiently.
-                        </para></listitem>
-               </itemizedlist>
-               </para></listitem>
-           <listitem><para>
-               <emphasis>Set up Test Machines:</emphasis>
-               Use a small number of shared, high performance systems
-               for testing purposes.
-               Developers can use these systems for wider, more
-               extensive testing while they continue to develop
-               locally using their primary development system.
-               </para></listitem>
-           <listitem><para>
-               <emphasis>Document Policies and Change Flow:</emphasis>
-               The Yocto Project itself uses a hierarchical structure and a
-                pull model.
-                Scripts exist to create and send pull requests
-                (i.e. <filename>create-pull-request</filename> and
-                <filename>send-pull-request</filename>).
-                This model is in line with other open source projects where
-                maintainers are responsible for specific areas of the project
-                and a single maintainer handles the final "top-of-tree" merges.
-                <note>
-                    You can also use a more collective push model.
-                    The <filename>gitolite</filename> software supports both the
-                    push and pull models quite easily.
-                </note></para>
-
-                <para>As with any development environment, it is important
-                to document the policy used as well as any main project
-                guidelines so they are understood by everyone.
-                It is also a good idea to have well structured
-                commit messages, which are usually a part of a project's
-                guidelines.
-                Good commit messages are essential when looking back in time and
-                trying to understand why changes were made.</para>
-
-                <para>If you discover that changes are needed to the core
-                layer of the project, it is worth sharing those with the
-                community as soon as possible.
-                Chances are if you have discovered the need for changes,
-                someone else in the community needs them also.
-                </para></listitem>
-            <listitem><para>
-                <emphasis>Development Environment Summary:</emphasis>
-                Aside from the previous steps, some best practices exist
-                within the Yocto Project development environment.
-                Consider the following:
-                <itemizedlist>
-                    <listitem><para>
-                        Use <ulink url='&YOCTO_DOCS_REF_URL;#git'>Git</ulink>
-                        as the source control system.
-                        </para></listitem>
-                    <listitem><para>
-                        Maintain your Metadata in layers that make sense
-                        for your situation.
-                        See the "<link linkend='understanding-and-creating-layers'>Understanding
-                        and Creating Layers</link>" section for more information on
-                        layers.
-                        </para></listitem>
-                    <listitem><para>
-                        Separate the project's Metadata and code by using
-                        separate Git repositories.
-                        See the
-                        "<ulink url='&YOCTO_DOCS_REF_URL;#yocto-project-repositories'>Yocto Project Source Repositories</ulink>"
-                        section for information on these repositories.
-                        See the
-                        "<link linkend='working-with-yocto-project-source-files'>Working With Yocto Project Source Files</link>"
-                        section for information on how to set up local Git
-                        repositories for related upstream Yocto Project
-                        Git repositories.
-                        </para></listitem>
-                    <listitem><para>
-                        Set up the directory for the shared state cache
-                        (<ulink url='&YOCTO_DOCS_REF_URL;#var-SSTATE_DIR'><filename>SSTATE_DIR</filename></ulink>)
-                        where it makes sense.
-                        For example, set up the sstate cache on a system used
-                        by developers in the same organization and share the
-                        same source directories on their machines.
-                        </para></listitem>
-                    <listitem><para>
-                        Set up an Autobuilder and have it populate the
-                        sstate cache and source directories.
-                        </para></listitem>
-                    <listitem><para>
-                        The Yocto Project community encourages you
-                        to send patches to the project to fix bugs or add features.
-                        If you do submit patches, follow the project commit
-                        guidelines for writing good commit messages.
-                        See the "<link linkend='how-to-submit-a-change'>Submitting a Change to the Yocto Project</link>"
-                        section.
-                        </para></listitem>
-                    <listitem><para>
-                        Send changes to the core sooner than later
-                        as others are likely to run into the same issues.
-                        For some guidance on mailing lists to use, see the list in the
-                        "<link linkend='how-to-submit-a-change'>Submitting a Change to the Yocto Project</link>"
-                        section.
-                        For a description of the available mailing lists, see the
-                        "<ulink url='&YOCTO_DOCS_REF_URL;#resources-mailinglist'>Mailing Lists</ulink>"
-                        section in the Yocto Project Reference Manual.
-                        </para></listitem>
-                </itemizedlist>
-                </para></listitem>
-        </orderedlist>
-    </para>
-</section>
-
-<section id='submitting-a-defect-against-the-yocto-project'>
-    <title>Submitting a Defect Against the Yocto Project</title>
-
-    <para>
-        Use the Yocto Project implementation of
-        <ulink url='http://www.bugzilla.org/about/'>Bugzilla</ulink>
-        to submit a defect (bug) against the Yocto Project.
-        For additional information on this implementation of Bugzilla see the
-        "<ulink url='&YOCTO_DOCS_REF_URL;#resources-bugtracker'>Yocto Project Bugzilla</ulink>"
-        section in the Yocto Project Reference Manual.
-        For more detail on any of the following steps, see the Yocto Project
-        <ulink url='&YOCTO_WIKI_URL;/wiki/Bugzilla_Configuration_and_Bug_Tracking'>Bugzilla wiki page</ulink>.
-    </para>
-
-    <para>
-        Use the following general steps to submit a bug"
-
-        <orderedlist>
-            <listitem><para>
-                Open the Yocto Project implementation of
-                <ulink url='&YOCTO_BUGZILLA_URL;'>Bugzilla</ulink>.
-                </para></listitem>
-            <listitem><para>
-                Click "File a Bug" to enter a new bug.
-                </para></listitem>
-            <listitem><para>
-                Choose the appropriate "Classification", "Product", and
-                "Component" for which the bug was found.
-                Bugs for the Yocto Project fall into one of several
-                classifications, which in turn break down into several
-                products and components.
-                For example, for a bug against the
-                <filename>meta-intel</filename> layer, you would choose
-                "Build System, Metadata &amp; Runtime", "BSPs", and
-                "bsps-meta-intel", respectively.
-                </para></listitem>
-            <listitem><para>
-                Choose the "Version" of the Yocto Project for which you found
-                the bug (e.g. &DISTRO;).
-                </para></listitem>
-            <listitem><para>
-                Determine and select the "Severity" of the bug.
-                The severity indicates how the bug impacted your work.
-                </para></listitem>
-            <listitem><para>
-                Choose the "Hardware" that the bug impacts.
-                </para></listitem>
-            <listitem><para>
-                Choose the "Architecture" that the bug impacts.
-                </para></listitem>
-            <listitem><para>
-                Choose a "Documentation change" item for the bug.
-                Fixing a bug might or might not affect the Yocto Project
-                documentation.
-                If you are unsure of the impact to the documentation, select
-                "Don't Know".
-                </para></listitem>
-            <listitem><para>
-                Provide a brief "Summary" of the bug.
-                Try to limit your summary to just a line or two and be sure
-                to capture the essence of the bug.
-                </para></listitem>
-            <listitem><para>
-                Provide a detailed "Description" of the bug.
-                You should provide as much detail as you can about the context,
-                behavior, output, and so forth that surrounds the bug.
-                You can even attach supporting files for output from logs by
-                using the "Add an attachment" button.
-                </para></listitem>
-            <listitem><para>
-                Click the "Submit Bug" button submit the bug.
-                A new Bugzilla number is assigned to the bug and the defect
-                is logged in the bug tracking system.
-                </para></listitem>
-        </orderedlist>
-        Once you file a bug, the bug is processed by the Yocto Project Bug
-        Triage Team and further details concerning the bug are assigned
-        (e.g. priority and owner).
-        You are the "Submitter" of the bug and any further categorization,
-        progress, or comments on the bug result in Bugzilla sending you an
-        automated email concerning the particular change or progress to the
-        bug.
-    </para>
-</section>
-
-<section id='how-to-submit-a-change'>
-    <title>Submitting a Change to the Yocto Project</title>
-
-    <para>
-        Contributions to the Yocto Project and OpenEmbedded are very welcome.
-        Because the system is extremely configurable and flexible, we recognize
-        that developers will want to extend, configure or optimize it for
-        their specific uses.
-    </para>
-
-    <para>
-        The Yocto Project uses a mailing list and a patch-based workflow
-        that is similar to the Linux kernel but contains important
-        differences.
-        In general, a mailing list exists through which you can submit
-        patches.
-        You should send patches to the appropriate mailing list so that they
-        can be reviewed and merged by the appropriate maintainer.
-        The specific mailing list you need to use depends on the
-        location of the code you are changing.
-        Each component (e.g. layer) should have a
-        <filename>README</filename> file that indicates where to send
-        the changes and which process to follow.
-    </para>
-
-    <para>
-        You can send the patch to the mailing list using whichever approach
-        you feel comfortable with to generate the patch.
-        Once sent, the patch is usually reviewed by the community at large.
-        If somebody has concerns with the patch, they will usually voice
-        their concern over the mailing list.
-        If a patch does not receive any negative reviews, the maintainer of
-        the affected layer typically takes the patch, tests it, and then
-        based on successful testing, merges the patch.
-    </para>
-
-    <para id='figuring-out-the-mailing-list-to-use'>
-        The "poky" repository, which is the Yocto Project's reference build
-        environment, is a hybrid repository that contains several
-        individual pieces (e.g. BitBake, Metadata, documentation,
-        and so forth) built using the combo-layer tool.
-        The upstream location used for submitting changes varies by
-        component:
-        <itemizedlist>
-            <listitem><para>
-                <emphasis>Core Metadata:</emphasis>
-                Send your patch to the
-                <ulink url='http://lists.openembedded.org/mailman/listinfo/openembedded-core'>openembedded-core</ulink>
-                mailing list.  For example, a change to anything under
-                the <filename>meta</filename> or
-                <filename>scripts</filename> directories should be sent
-                to this mailing list.
-                </para></listitem>
-            <listitem><para>
-                <emphasis>BitBake:</emphasis>
-                For changes to BitBake (i.e. anything under the
-                <filename>bitbake</filename> directory), send your patch
-                to the
-                <ulink url='http://lists.openembedded.org/mailman/listinfo/bitbake-devel'>bitbake-devel</ulink>
-                mailing list.
-                </para></listitem>
-            <listitem><para>
-                <emphasis>"meta-*" trees:</emphasis>
-                These trees contain Metadata.
-                Use the
-                <ulink url='https://lists.yoctoproject.org/listinfo/poky'>poky</ulink>
-                mailing list.
-                </para></listitem>
-        </itemizedlist>
-    </para>
-
-    <para>
-        For changes to other layers hosted in the Yocto Project source
-        repositories (i.e. <filename>yoctoproject.org</filename>), tools,
-        and the Yocto Project documentation, use the
-        <ulink url='https://lists.yoctoproject.org/listinfo/yocto'>Yocto Project</ulink>
-        general mailing list.
-        <note>
-            Sometimes a layer's documentation specifies to use a
-            particular mailing list.
-            If so, use that list.
-        </note>
-        For additional recipes that do not fit into the core Metadata, you
-        should determine which layer the recipe should go into and submit
-        the change in the manner recommended by the documentation (e.g.
-        the <filename>README</filename> file) supplied with the layer.
-        If in doubt, please ask on the Yocto general mailing list or on
-        the openembedded-devel mailing list.
-    </para>
-
-    <para>
-        You can also push a change upstream and request a maintainer to
-        pull the change into the component's upstream repository.
-        You do this by pushing to a contribution repository that is upstream.
-        See the
-        "<ulink url='&YOCTO_DOCS_REF_URL;#workflows'>Workflows</ulink>"
-        section in the Yocto Project Reference Manual for additional
-        concepts on working in the Yocto Project development environment.
-    </para>
-
-    <para>
-        Two commonly used testing repositories exist for
-        OpenEmbedded-Core:
-        <itemizedlist>
-            <listitem><para>
-                <emphasis>"ross/mut" branch:</emphasis>
-                The "mut" (master-under-test) tree
-                exists in the <filename>poky-contrib</filename> repository
-                in the
-                <ulink url='&YOCTO_GIT_URL;'>Yocto Project source repositories</ulink>.
-                </para></listitem>
-            <listitem><para>
-                <emphasis>"master-next" branch:</emphasis>
-                This branch is part of the main
-                "poky" repository in the Yocto Project source repositories.
-                </para></listitem>
-        </itemizedlist>
-        Maintainers use these branches to test submissions prior to merging
-        patches.
-        Thus, you can get an idea of the status of a patch based on
-        whether the patch has been merged into one of these branches.
-        <note>
-            This system is imperfect and changes can sometimes get lost in the
-            flow.
-            Asking about the status of a patch or change is reasonable if the
-            change has been idle for a while with no feedback.
-            The Yocto Project does have plans to use
-            <ulink url='https://en.wikipedia.org/wiki/Patchwork_(software)'>Patchwork</ulink>
-            to track the status of patches and also to automatically preview
-            patches.
-        </note>
-    </para>
-
-    <para>
-        The following sections provide procedures for submitting a change.
-    </para>
-
-    <section id='pushing-a-change-upstream'>
-        <title>Using Scripts to Push a Change Upstream and Request a Pull</title>
-
-        <para>
-            Follow this procedure to push a change to an upstream "contrib"
-            Git repository:
-            <note>
-                You can find general Git information on how to push a change
-                upstream in the
-                <ulink url='http://git-scm.com/book/en/v2/Distributed-Git-Distributed-Workflows'>Git Community Book</ulink>.
-            </note>
-            <orderedlist>
-                <listitem><para>
-                    <emphasis>Make Your Changes Locally:</emphasis>
-                    Make your changes in your local Git repository.
-                    You should make small, controlled, isolated changes.
-                    Keeping changes small and isolated aids review,
-                    makes merging/rebasing easier and keeps the change
-                    history clean should anyone need to refer to it in
-                    future.
-                    </para></listitem>
-                <listitem><para>
-                    <emphasis>Stage Your Changes:</emphasis>
-                    Stage your changes by using the <filename>git add</filename>
-                    command on each file you changed.
-                    </para></listitem>
-                <listitem><para id='making-sure-you-have-correct-commit-information'>
-                    <emphasis>Commit Your Changes:</emphasis>
-                    Commit the change by using the
-                    <filename>git commit</filename> command.
-                    Make sure your commit information follows standards by
-                    following these accepted conventions:
-                    <itemizedlist>
-                        <listitem><para>
-                            Be sure to include a "Signed-off-by:" line in the
-                            same style as required by the Linux kernel.
-                            Adding this line signifies that you, the submitter,
-                            have agreed to the Developer's Certificate of
-                            Origin 1.1 as follows:
-                            <literallayout class='monospaced'>
-     Developer's Certificate of Origin 1.1
-
-     By making a contribution to this project, I certify that:
-
-     (a) The contribution was created in whole or in part by me and I
-         have the right to submit it under the open source license
-         indicated in the file; or
-
-     (b) The contribution is based upon previous work that, to the best
-         of my knowledge, is covered under an appropriate open source
-         license and I have the right under that license to submit that
-         work with modifications, whether created in whole or in part
-         by me, under the same open source license (unless I am
-         permitted to submit under a different license), as indicated
-         in the file; or
-
-     (c) The contribution was provided directly to me by some other
-         person who certified (a), (b) or (c) and I have not modified
-         it.
-
-     (d) I understand and agree that this project and the contribution
-         are public and that a record of the contribution (including all
-         personal information I submit with it, including my sign-off) is
-         maintained indefinitely and may be redistributed consistent with
-         this project or the open source license(s) involved.
-                            </literallayout>
-                            </para></listitem>
-                        <listitem><para>
-                            Provide a single-line summary of the change.
-                            and,
-                            if more explanation is needed, provide more
-                            detail in the body of the commit.
-                            This summary is typically viewable in the
-                            "shortlist" of changes.
-                            Thus, providing something short and descriptive
-                            that gives the reader a summary of the change is
-                            useful when viewing a list of many commits.
-                            You should prefix this short description with the
-                            recipe name (if changing a recipe), or else with
-                            the short form path to the file being changed.
-                            </para></listitem>
-                        <listitem><para>
-                            For the body of the commit message, provide
-                            detailed information that describes what you
-                            changed, why you made the change, and the approach
-                            you used.
-                            It might also be helpful if you mention how you
-                            tested the change.
-                            Provide as much detail as you can in the body of
-                            the commit message.
-                            <note>
-                                You do not need to provide a more detailed
-                                explanation of a change if the change is
-                                minor to the point of the single line
-                                summary providing all the information.
-                            </note>
-                            </para></listitem>
-                        <listitem><para>
-                            If the change addresses a specific bug or issue
-                            that is associated with a bug-tracking ID,
-                            include a reference to that ID in your detailed
-                            description.
-                            For example, the Yocto Project uses a specific
-                            convention for bug references - any commit that
-                            addresses a specific bug should use the following
-                            form for the detailed description.
-                            Be sure to use the actual bug-tracking ID from
-                            Bugzilla for
-                            <replaceable>bug-id</replaceable>:
-                            <literallayout class='monospaced'>
-     Fixes [YOCTO #<replaceable>bug-id</replaceable>]
-
-     <replaceable>detailed description of change</replaceable>
-                            </literallayout>
-                            </para></listitem>
-                    </itemizedlist>
-                    </para></listitem>
-                <listitem><para>
-                    <emphasis>Push Your Commits to a "Contrib" Upstream:</emphasis>
-                    If you have arranged for permissions to push to an
-                    upstream contrib repository, push the change to that
-                    repository:
-                    <literallayout class='monospaced'>
-     $ git push <replaceable>upstream_remote_repo</replaceable> <replaceable>local_branch_name</replaceable>
-                    </literallayout>
-                    For example, suppose you have permissions to push into the
-                    upstream <filename>meta-intel-contrib</filename>
-                    repository and you are working in a local branch named
-                    <replaceable>your_name</replaceable><filename>/README</filename>.
-                    The following command pushes your local commits to the
-                    <filename>meta-intel-contrib</filename> upstream
-                    repository and puts the commit in a branch named
-                    <replaceable>your_name</replaceable><filename>/README</filename>:
-                    <literallayout class='monospaced'>
-     $ git push meta-intel-contrib <replaceable>your_name</replaceable>/README
-                    </literallayout>
-                    </para></listitem>
-                <listitem><para id='push-determine-who-to-notify'>
-                    <emphasis>Determine Who to Notify:</emphasis>
-                    Determine the maintainer or the mailing list
-                    that you need to notify for the change.</para>
-
-                    <para>Before submitting any change, you need to be sure
-                    who the maintainer is or what mailing list that you need
-                    to notify.
-                    Use either these methods to find out:
-                    <itemizedlist>
-                        <listitem><para>
-                            <emphasis>Maintenance File:</emphasis>
-                            Examine the <filename>maintainers.inc</filename>
-                            file, which is located in the
-                            <ulink url='&YOCTO_DOCS_REF_URL;#source-directory'>Source Directory</ulink>
-                            at
-                            <filename>meta/conf/distro/include</filename>,
-                            to see who is responsible for code.
-                            </para></listitem>
-                        <listitem><para>
-                            <emphasis>Search by File:</emphasis>
-                            Using <ulink url='&YOCTO_DOCS_REF_URL;#git'>Git</ulink>,
-                            you can enter the following command to bring up a
-                            short list of all commits against a specific file:
-                            <literallayout class='monospaced'>
-     git shortlog -- <replaceable>filename</replaceable>
-                            </literallayout>
-                            Just provide the name of the file for which you
-                            are interested.
-                            The information returned is not ordered by history
-                            but does include a list of everyone who has
-                            committed grouped by name.
-                            From the list, you can see who is responsible for
-                            the bulk of the changes against the file.
-                            </para></listitem>
-                        <listitem><para>
-                            <emphasis>Examine the List of Mailing Lists:</emphasis>
-                            For a list of the Yocto Project and related mailing
-                            lists, see the
-                            "<ulink url='&YOCTO_DOCS_REF_URL;#resources-mailinglist'>Mailing lists</ulink>"
-                            section in the Yocto Project Reference Manual.
-                            </para></listitem>
-                    </itemizedlist>
-                    </para></listitem>
-                <listitem><para>
-                    <emphasis>Make a Pull Request:</emphasis>
-                    Notify the maintainer or the mailing list that you have
-                    pushed a change by making a pull request.</para>
-
-                    <para>The Yocto Project provides two scripts that
-                    conveniently let you generate and send pull requests to the
-                    Yocto Project.
-                    These scripts are <filename>create-pull-request</filename>
-                    and <filename>send-pull-request</filename>.
-                    You can find these scripts in the
-                    <filename>scripts</filename> directory within the
-                    <ulink url='&YOCTO_DOCS_REF_URL;#source-directory'>Source Directory</ulink>
-                    (e.g. <filename>~/poky/scripts</filename>).
-                    </para>
-
-                    <para>Using these scripts correctly formats the requests
-                    without introducing any whitespace or HTML formatting.
-                    The maintainer that receives your patches either directly
-                    or through the mailing list needs to be able to save and
-                    apply them directly from your emails.
-                    Using these scripts is the preferred method for sending
-                    patches.</para>
-
-                    <para>First, create the pull request.
-                    For example, the following command runs the script,
-                    specifies the upstream repository in the contrib directory
-                    into which you pushed the change, and provides a subject
-                    line in the created patch files:
-                    <literallayout class='monospaced'>
-     $ ~/poky/scripts/create-pull-request -u meta-intel-contrib -s "Updated Manual Section Reference in README"
-                    </literallayout>
-                    Running this script forms
-                    <filename>*.patch</filename> files in a folder named
-                    <filename>pull-</filename><replaceable>PID</replaceable>
-                    in the current directory.
-                    One of the patch files is a cover letter.</para>
-
-                    <para>Before running the
-                    <filename>send-pull-request</filename> script, you must
-                    edit the cover letter patch to insert information about
-                    your change.
-                    After editing the cover letter, send the pull request.
-                    For example, the following command runs the script and
-                    specifies the patch directory and email address.
-                    In this example, the email address is a mailing list:
-                    <literallayout class='monospaced'>
-     $ ~/poky/scripts/send-pull-request -p ~/meta-intel/pull-10565 -t meta-intel@yoctoproject.org
-                    </literallayout>
-                    You need to follow the prompts as the script is
-                    interactive.
-                    <note>
-                        For help on using these scripts, simply provide the
-                        <filename>-h</filename> argument as follows:
-                        <literallayout class='monospaced'>
-     $ poky/scripts/create-pull-request -h
-     $ poky/scripts/send-pull-request -h
-                        </literallayout>
-                    </note>
-                    </para></listitem>
-            </orderedlist>
-        </para>
-    </section>
-
-    <section id='submitting-a-patch'>
-        <title>Using Email to Submit a Patch</title>
-
-        <para>
-            You can submit patches without using the
-            <filename>create-pull-request</filename> and
-            <filename>send-pull-request</filename> scripts described in the
-            previous section.
-            However, keep in mind, the preferred method is to use the scripts.
-        </para>
-
-        <para>
-            Depending on the components changed, you need to submit the email
-            to a specific mailing list.
-            For some guidance on which mailing list to use, see the
-            <link linkend='figuring-out-the-mailing-list-to-use'>beginning</link>
-            of this section.
-            For a description of all the available mailing lists, see the
-            "<ulink url='&YOCTO_DOCS_REF_URL;#resources-mailinglist'>Mailing Lists</ulink>"
-            section in the Yocto Project Reference Manual.
-        </para>
-
-        <para>
-            Here is the general procedure on how to submit a patch through
-            email without using the scripts:
-            <orderedlist>
-                <listitem><para>
-                    <emphasis>Make Your Changes Locally:</emphasis>
-                    Make your changes in your local Git repository.
-                    You should make small, controlled, isolated changes.
-                    Keeping changes small and isolated aids review,
-                    makes merging/rebasing easier and keeps the change
-                    history clean should anyone need to refer to it in
-                    future.
-                    </para></listitem>
-                <listitem><para>
-                    <emphasis>Stage Your Changes:</emphasis>
-                    Stage your changes by using the <filename>git add</filename>
-                    command on each file you changed.
-                    </para></listitem>
-                <listitem><para>
-                    <emphasis>Commit Your Changes:</emphasis>
-                    Commit the change by using the
-                    <filename>git commit --signoff</filename> command.
-                    Using the <filename>--signoff</filename> option identifies
-                    you as the person making the change and also satisfies
-                    the Developer's Certificate of Origin (DCO) shown earlier.
-                    </para>
-
-                    <para>When you form a commit, you must follow certain
-                    standards established by the Yocto Project development
-                    team.
-                    See
-                    <link linkend='making-sure-you-have-correct-commit-information'>Step 3</link>
-                    in the previous section for information on how to
-                    provide commit information that meets Yocto Project
-                    commit message standards.
-                    </para></listitem>
-                <listitem><para>
-                    <emphasis>Format the Commit:</emphasis>
-                    Format the commit into an email message.
-                    To format commits, use the
-                    <filename>git format-patch</filename> command.
-                    When you provide the command, you must include a revision
-                    list or a number of patches as part of the command.
-                    For example, either of these two commands takes your most
-                    recent single commit and formats it as an email message in
-                    the current directory:
-                    <literallayout class='monospaced'>
-     $ git format-patch -1
-                    </literallayout>
-                    or
-                    <literallayout class='monospaced'>
-     $ git format-patch HEAD~
-                    </literallayout></para>
-
-                    <para>After the command is run, the current directory
-                    contains a numbered <filename>.patch</filename> file for
-                    the commit.</para>
-
-                    <para>If you provide several commits as part of the
-                    command, the <filename>git format-patch</filename> command
-                    produces a series of numbered files in the current
-                    directory – one for each commit.
-                    If you have more than one patch, you should also use the
-                    <filename>--cover</filename> option with the command,
-                    which generates a cover letter as the first "patch" in
-                    the series.
-                    You can then edit the cover letter to provide a
-                    description for the series of patches.
-                    For information on the
-                    <filename>git format-patch</filename> command,
-                    see <filename>GIT_FORMAT_PATCH(1)</filename> displayed
-                    using the <filename>man git-format-patch</filename>
-                    command.
-                    <note>
-                        If you are or will be a frequent contributor to the
-                        Yocto Project or to OpenEmbedded, you might consider
-                        requesting a contrib area and the necessary associated
-                        rights.
-                    </note>
-                    </para></listitem>
-                <listitem><para>
-                    <emphasis>Import the Files Into Your Mail Client:</emphasis>
-                    Import the files into your mail client by using the
-                    <filename>git send-email</filename> command.
-                    <note>
-                        In order to use <filename>git send-email</filename>,
-                        you must have the proper Git packages installed on
-                        your host.
-                        For Ubuntu, Debian, and Fedora the package is
-                        <filename>git-email</filename>.
-                    </note></para>
-
-                    <para>The <filename>git send-email</filename> command
-                    sends email by using a local or remote Mail Transport Agent
-                    (MTA) such as <filename>msmtp</filename>,
-                    <filename>sendmail</filename>, or through a direct
-                    <filename>smtp</filename> configuration in your Git
-                    <filename>~/.gitconfig</filename> file.
-                    If you are submitting patches through email only, it is
-                    very important that you submit them without any whitespace
-                    or HTML formatting that either you or your mailer
-                    introduces.
-                    The maintainer that receives your patches needs to be able
-                    to save and apply them directly from your emails.
-                    A good way to verify that what you are sending will be
-                    applicable by the maintainer is to do a dry run and send
-                    them to yourself and then save and apply them as the
-                    maintainer would.</para>
-
-                    <para>The <filename>git send-email</filename> command is
-                    the preferred method for sending your patches using
-                    email since there is no risk of compromising whitespace
-                    in the body of the message, which can occur when you use
-                    your own mail client.
-                    The command also has several options that let you
-                    specify recipients and perform further editing of the
-                    email message.
-                    For information on how to use the
-                    <filename>git send-email</filename> command,
-                    see <filename>GIT-SEND-EMAIL(1)</filename> displayed using
-                    the <filename>man git-send-email</filename> command.
-                    </para></listitem>
-            </orderedlist>
-        </para>
-    </section>
-</section>
-</chapter>
-<!--
-vim: expandtab tw=80 ts=4
--->

documentation/dev-manual/dev-manual-qemu.xml

@@ -7,15 +7,51 @@
 <title>Using the Quick EMUlator (QEMU)</title>
 
     <para>
-        This chapter provides procedures that show you how to use the
-        Quick EMUlator (QEMU), which is an Open Source project the Yocto
-        Project uses as part of its development "tool set".
-        For reference information on the Yocto Project implementation of QEMU,
-        see the
-        "<ulink url='&YOCTO_DOCS_REF_URL;#ref-quick-emulator-qemu'>Quick EMUlator (QEMU)</ulink>"
-        section in the Yocto Project Reference Manual.
+        The Yocto Project uses an implementation of the Quick EMUlator (QEMU)
+        Open Source project as part of the Yocto Project development "tool
+        set".
+        This chapter provides both procedures that show you how to use the
+        Quick EMUlator (QEMU) and other QEMU information helpful for
+        development purposes.
     </para>
 
+    <section id='qemu-dev-overview'>
+        <title>Overview</title>
+
+        <para>
+            Within the context of the Yocto Project, QEMU is an
+            emulator and virtualization machine that allows you to run a
+            complete image you have built using the Yocto Project as just
+            another task on your build system.
+            QEMU is useful for running and testing images and applications on
+            supported Yocto Project architectures without having actual
+            hardware.
+            Among other things, the Yocto Project uses QEMU to run automated
+            Quality Assurance (QA) tests on final images shipped with each
+            release.
+            <note>
+                This implementation is not the same as QEMU in general.
+            </note>
+            This section provides a brief reference for the Yocto Project
+            implementation of QEMU.
+        </para>
+
+        <para>
+            For official information and documentation on QEMU in general, see
+            the following references:
+            <itemizedlist>
+                <listitem><para>
+                    <emphasis><ulink url='http://wiki.qemu.org/Main_Page'>QEMU Website</ulink>:</emphasis>
+                    The official website for the QEMU Open Source project.
+                    </para></listitem>
+                <listitem><para>
+                    <emphasis><ulink url='http://wiki.qemu.org/Manual'>Documentation</ulink>:</emphasis>
+                    The QEMU user manual.
+                    </para></listitem>
+            </itemizedlist>
+        </para>
+    </section>
+
     <section id='qemu-running-qemu'>
         <title>Running QEMU</title>
 
@@ -27,6 +63,9 @@
             <orderedlist>
                 <listitem><para>
                     <emphasis>Install QEMU:</emphasis>
+                    QEMU is made available with the Yocto Project a number of
+                    ways.
+                    One method is to install a Software Development Kit (SDK).
                     See
                     "<ulink url='&YOCTO_DOCS_SDK_URL;#the-qemu-emulator'>The QEMU Emulator</ulink>"
                     section in the Yocto Project Application Development and
@@ -303,6 +342,345 @@
             </note>
         </para>
     </section>
+
+    <section id='qemu-kvm-cpu-compatibility'>
+        <title>QEMU CPU Compatibility Under KVM</title>
+
+        <para>
+            By default, the QEMU build compiles for and targets 64-bit and x86
+            <trademark class='registered'>Intel</trademark> <trademark class='trademark'>Core</trademark>2
+            Duo processors and 32-bit x86
+            <trademark class='registered'>Intel</trademark> <trademark class='registered'>Pentium</trademark>
+            II processors.
+            QEMU builds for and targets these CPU types because they display
+            a broad range of CPU feature compatibility with many commonly
+            used CPUs.
+        </para>
+
+        <para>
+            Despite this broad range of compatibility, the CPUs could support
+            a feature that your host CPU does not support.
+            Although this situation is not a problem when QEMU uses software
+            emulation of the feature, it can be a problem when QEMU is
+            running with KVM enabled.
+            Specifically, software compiled with a certain CPU feature crashes
+            when run on a CPU under KVM that does not support that feature.
+            To work around this problem, you can override QEMU's runtime CPU
+            setting by changing the <filename>QB_CPU_KVM</filename>
+            variable in <filename>qemuboot.conf</filename> in the
+            <ulink url='&YOCTO_DOCS_REF_URL;#build-directory'>Build Directory's</ulink>
+            <filename>deploy/image</filename> directory.
+            This setting specifies a <filename>-cpu</filename> option
+            passed into QEMU in the <filename>runqemu</filename> script.
+            Running <filename>qemu -cpu help</filename> returns a list of
+            available supported CPU types.
+        </para>
+    </section>
+
+    <section id='qemu-dev-performance'>
+        <title>QEMU Performance</title>
+
+        <para>
+            Using QEMU to emulate your hardware can result in speed issues
+            depending on the target and host architecture mix.
+            For example, using the <filename>qemux86</filename> image in the
+            emulator on an Intel-based 32-bit (x86) host machine is fast
+            because the target and host architectures match.
+            On the other hand, using the <filename>qemuarm</filename> image
+            on the same Intel-based host can be slower.
+            But, you still achieve faithful emulation of ARM-specific issues.
+        </para>
+
+        <para>
+            To speed things up, the QEMU images support using
+            <filename>distcc</filename> to call a cross-compiler outside the
+            emulated system.
+            If you used <filename>runqemu</filename> to start QEMU, and the
+            <filename>distccd</filename> application is present on the host
+            system, any BitBake cross-compiling toolchain available from the
+            build system is automatically used from within QEMU simply by
+            calling <filename>distcc</filename>.
+            You can accomplish this by defining the cross-compiler variable
+            (e.g. <filename>export CC="distcc"</filename>).
+            Alternatively, if you are using a suitable SDK image or the
+            appropriate stand-alone toolchain is present, the toolchain is
+            also automatically used.
+            <note>
+                Several mechanisms exist that let you connect to the system
+                running on the QEMU emulator:
+                <itemizedlist>
+                    <listitem><para>
+                        QEMU provides a framebuffer interface that makes
+                        standard consoles available.
+                        </para></listitem>
+                    <listitem><para>
+                        Generally, headless embedded devices have a serial port.
+                        If so, you can configure the operating system of the
+                        running image to use that port to run a console.
+                        The connection uses standard IP networking.
+                        </para></listitem>
+                    <listitem><para>
+                        SSH servers exist in some QEMU images.
+                        The <filename>core-image-sato</filename> QEMU image
+                        has a Dropbear secure shell (SSH) server that runs
+                        with the root password disabled.
+                        The <filename>core-image-full-cmdline</filename> and
+                        <filename>core-image-lsb</filename> QEMU images
+                        have OpenSSH instead of Dropbear.
+                        Including these SSH servers allow you to use standard
+                        <filename>ssh</filename> and <filename>scp</filename>
+                        commands.
+                        The <filename>core-image-minimal</filename> QEMU image,
+                        however, contains no SSH server.
+                        </para></listitem>
+                    <listitem><para>
+                        You can use a provided, user-space NFS server to boot
+                        the QEMU session using a local copy of the root
+                        filesystem on the host.
+                        In order to make this connection, you must extract a
+                        root filesystem tarball by using the
+                        <filename>runqemu-extract-sdk</filename> command.
+                        After running the command, you must then point the
+                        <filename>runqemu</filename>
+                        script to the extracted directory instead of a root
+                        filesystem image file.
+                        See the
+                        "<link linkend='qemu-running-under-a-network-file-system-nfs-server'>Running Under a Network File System (NFS) Server</link>"
+                        section for more information.
+                        </para></listitem>
+                </itemizedlist>
+            </note>
+        </para>
+    </section>
+
+    <section id='qemu-dev-command-line-syntax'>
+        <title>QEMU Command-Line Syntax</title>
+
+        <para>
+            The basic <filename>runqemu</filename> command syntax is as
+            follows:
+            <literallayout class='monospaced'>
+     $ runqemu [<replaceable>option</replaceable> ]  [...]
+            </literallayout>
+            Based on what you provide on the command line,
+            <filename>runqemu</filename> does a good job of figuring out what
+            you are trying to do.
+            For example, by default, QEMU looks for the most recently built
+            image according to the timestamp when it needs to look for an
+            image.
+            Minimally, through the use of options, you must provide either
+            a machine name, a virtual machine image
+            (<filename>*wic.vmdk</filename>), or a kernel image
+            (<filename>*.bin</filename>).
+        </para>
+
+        <para>
+            Following is the command-line help output for the
+            <filename>runqemu</filename> command:
+            <literallayout class='monospaced'>
+     $ runqemu --help
+
+     Usage: you can run this script with any valid combination
+     of the following environment variables (in any order):
+       KERNEL - the kernel image file to use
+       ROOTFS - the rootfs image file or nfsroot directory to use
+       MACHINE - the machine name (optional, autodetected from KERNEL filename if unspecified)
+       Simplified QEMU command-line options can be passed with:
+         nographic - disable video console
+         serial - enable a serial console on /dev/ttyS0
+         slirp - enable user networking, no root privileges is required
+         kvm - enable KVM when running x86/x86_64 (VT-capable CPU required)
+         kvm-vhost - enable KVM with vhost when running x86/x86_64 (VT-capable CPU required)
+         publicvnc - enable a VNC server open to all hosts
+         audio - enable audio
+         [*/]ovmf* - OVMF firmware file or base name for booting with UEFI
+       tcpserial=&lt;port&gt; - specify tcp serial port number
+       biosdir=&lt;dir&gt; - specify custom bios dir
+       biosfilename=&lt;filename&gt; - specify bios filename
+       qemuparams=&lt;xyz&gt; - specify custom parameters to QEMU
+       bootparams=&lt;xyz&gt; - specify custom kernel parameters during boot
+       help, -h, --help: print this text
+
+     Examples:
+       runqemu
+       runqemu qemuarm
+       runqemu tmp/deploy/images/qemuarm
+       runqemu tmp/deploy/images/qemux86/&lt;qemuboot.conf&gt;
+       runqemu qemux86-64 core-image-sato ext4
+       runqemu qemux86-64 wic-image-minimal wic
+       runqemu path/to/bzImage-qemux86.bin path/to/nfsrootdir/ serial
+       runqemu qemux86 iso/hddimg/wic.vmdk/wic.qcow2/wic.vdi/ramfs/cpio.gz...
+       runqemu qemux86 qemuparams="-m 256"
+       runqemu qemux86 bootparams="psplash=false"
+       runqemu path/to/&lt;image&gt;-&lt;machine&gt;.wic
+       runqemu path/to/&lt;image&gt;-&lt;machine&gt;.wic.vmdk
+            </literallayout>
+        </para>
+    </section>
+
+    <section id='qemu-dev-runqemu-command-line-options'>
+        <title><filename>runqemu</filename> Command-Line Options</title>
+
+        <para>
+            Following is a description of <filename>runqemu</filename>
+            options you can provide on the command line:
+            <note><title>Tip</title>
+                If you do provide some "illegal" option combination or perhaps
+                you do not provide enough in the way of options,
+                <filename>runqemu</filename> provides appropriate error
+                messaging to help you correct the problem.
+            </note>
+            <itemizedlist>
+                <listitem><para>
+                    <replaceable>QEMUARCH</replaceable>:
+                    The QEMU machine architecture, which must be "qemuarm",
+                    "qemuarm64", "qemumips", "qemumips64", "qemuppc",
+                    "qemux86", or "qemux86-64".
+                    </para></listitem>
+                <listitem><para>
+                    <filename><replaceable>VM</replaceable></filename>:
+                    The virtual machine image, which must be a
+                    <filename>.wic.vmdk</filename> file.
+                    Use this option when you want to boot a
+                    <filename>.wic.vmdk</filename> image.
+                    The image filename you provide must contain one of the
+                    following strings: "qemux86-64", "qemux86", "qemuarm",
+                    "qemumips64", "qemumips", "qemuppc", or "qemush4".
+                    </para></listitem>
+                <listitem><para>
+                    <replaceable>ROOTFS</replaceable>:
+                    A root filesystem that has one of the following
+                    filetype extensions: "ext2", "ext3", "ext4", "jffs2",
+                    "nfs", or "btrfs".
+                    If the filename you provide for this option uses “nfs”, it
+                    must provide an explicit root filesystem path.
+                    </para></listitem>
+                <listitem><para>
+                    <replaceable>KERNEL</replaceable>:
+                    A kernel image, which is a <filename>.bin</filename> file.
+                    When you provide a <filename>.bin</filename> file,
+                    <filename>runqemu</filename> detects it and assumes the
+                    file is a kernel image.
+                    </para></listitem>
+                <listitem><para>
+                    <replaceable>MACHINE</replaceable>:
+                    The architecture of the QEMU machine, which must be one
+                    of the following: "qemux86", "qemux86-64", "qemuarm",
+                    "qemuarm64", "qemumips", “qemumips64", or "qemuppc".
+                    The <replaceable>MACHINE</replaceable> and
+                    <replaceable>QEMUARCH</replaceable> options are basically
+                    identical.
+                    If you do not provide a <replaceable>MACHINE</replaceable>
+                    option, <filename>runqemu</filename> tries to determine
+                    it based on other options.
+                    </para></listitem>
+                <listitem><para>
+                    <filename>ramfs</filename>:
+                    Indicates you are booting an initial RAM disk (initramfs)
+                    image, which means the <filename>FSTYPE</filename> is
+                    <filename>cpio.gz</filename>.
+                    </para></listitem>
+                <listitem><para>
+                    <filename>iso</filename>:
+                    Indicates you are booting an ISO image, which means the
+                    <filename>FSTYPE</filename> is
+                    <filename>.iso</filename>.
+                    </para></listitem>
+                <listitem><para>
+                    <filename>nographic</filename>:
+                    Disables the video console, which sets the console to
+                    "ttys0".
+                    </para></listitem>
+                <listitem><para>
+                    <filename>serial</filename>:
+                    Enables a serial console on
+                    <filename>/dev/ttyS0</filename>.
+                    </para></listitem>
+                <listitem><para>
+                    <filename>biosdir</filename>:
+                    Establishes a custom directory for BIOS, VGA BIOS and
+                    keymaps.
+                    </para></listitem>
+                <listitem><para>
+                    <filename>biosfilename</filename>:
+                    Establishes a custom BIOS name.
+                    </para></listitem>
+                <listitem><para>
+                    <filename>qemuparams=\"<replaceable>xyz</replaceable>\"</filename>:
+                    Specifies custom QEMU parameters.
+                    Use this option to pass options other than the simple
+                    "kvm" and "serial" options.
+                    </para></listitem>
+                <listitem><para><filename>bootparams=\"<replaceable>xyz</replaceable>\"</filename>:
+                    Specifies custom boot parameters for the kernel.
+                    </para></listitem>
+                <listitem><para>
+                    <filename>audio</filename>:
+                    Enables audio in QEMU.
+                    The <replaceable>MACHINE</replaceable> option must be
+                    either "qemux86" or "qemux86-64" in order for audio to be
+                    enabled.
+                    Additionally, the <filename>snd_intel8x0</filename>
+                    or <filename>snd_ens1370</filename> driver must be
+                    installed in linux guest.
+                    </para></listitem>
+                <listitem><para>
+                    <filename>slirp</filename>:
+                    Enables "slirp" networking, which is a different way
+                    of networking that does not need root access
+                    but also is not as easy to use or comprehensive
+                    as the default.
+                    </para></listitem>
+                <listitem><para id='kvm-cond'>
+                    <filename>kvm</filename>:
+                    Enables KVM when running "qemux86" or "qemux86-64"
+                    QEMU architectures.
+                    For KVM to work, all the following conditions must be met:
+                    <itemizedlist>
+                        <listitem><para>
+                            Your <replaceable>MACHINE</replaceable> must be either
+qemux86" or "qemux86-64".
+                            </para></listitem>
+                        <listitem><para>
+                            Your build host has to have the KVM modules
+                            installed, which are
+                            <filename>/dev/kvm</filename>.
+                            </para></listitem>
+                        <listitem><para>
+                            The  build host <filename>/dev/kvm</filename>
+                            directory has to be both writable and readable.
+                            </para></listitem>
+                    </itemizedlist>
+                    </para></listitem>
+                <listitem><para>
+                    <filename>kvm-vhost</filename>:
+                    Enables KVM with VHOST support when running "qemux86"
+                    or "qemux86-64" QEMU architectures.
+                    For KVM with VHOST to work, the following conditions must
+                    be met:
+                    <itemizedlist>
+                        <listitem><para>
+                            <link linkend='kvm-cond'>kvm</link> option
+                            conditions must be met.
+                            </para></listitem>
+                        <listitem><para>
+                            Your build host has to have virtio net device, which
+                            are <filename>/dev/vhost-net</filename>.
+                            </para></listitem>
+                        <listitem><para>
+                            The build host <filename>/dev/vhost-net</filename>
+                            directory has to be either readable or writable
+                            and “slirp-enabled”.
+                            </para></listitem>
+                    </itemizedlist>
+                    </para></listitem>
+                <listitem><para>
+                    <filename>publicvnc</filename>:
+                    Enables a VNC server open to all hosts.
+                    </para></listitem>
+            </itemizedlist>
+        </para>
+    </section>
 </chapter>
 <!--
 vim: expandtab tw=80 ts=4

documentation/dev-manual/dev-manual-start.xml

@@ -4,18 +4,386 @@
 
 <chapter id='dev-manual-start'>
 
-<title>Getting Started with the Yocto Project</title>
+<title>Setting Up to Use the Yocto Project</title>
 
 <para>
     This chapter provides procedures related to getting set up to use the
     Yocto Project.
-    For a more front-to-end process that takes you from minimally preparing
-    a build host through building an image, see the
-    <ulink url='&YOCTO_DOCS_QS_URL;'>Yocto Project Quick Start</ulink>.
+    You can learn about creating a team environment that develops using the
+    Yocto Project, how to set up a build host, how to locate Yocto Project
+    source repositories, and how to create local Git repositories.
 </para>
 
+<section id="usingpoky-changes-collaborate">
+    <title>Creating a Team Development Environment</title>
+
+    <para>
+        It might not be immediately clear how you can use the Yocto
+        Project in a team development environment, or scale it for a large
+        team of developers.
+        One of the strengths of the Yocto Project is that it is extremely
+        flexible.
+        Thus, you can adapt it to many different use cases and scenarios.
+        However, these characteristics can cause a struggle if you are trying
+        to create a working setup that scales across a large team.
+    </para>
+
+    <para>
+        To help you understand how to set up this type of environment,
+        this section presents a procedure that gives you the information
+        to learn how to get the results you want.
+        The procedure is high-level and presents some of the project's most
+        successful experiences, practices, solutions, and available
+        technologies that work well.
+        Keep in mind, the procedure here is a starting point.
+        You can build off it and customize it to fit any
+        particular working environment and set of practices.
+        <orderedlist>
+            <listitem><para>
+                <emphasis>Determine Who is Going to be Developing:</emphasis>
+                You need to understand who is going to be doing anything
+                related to the Yocto Project and what their roles would be.
+                Making this determination is essential to completing the
+                steps two and three, which are to get your equipment together
+                and set up your development environment's hardware topology.
+                </para>
+
+                <para>The following roles exist:
+                <itemizedlist>
+                    <listitem><para>
+                        <emphasis>Application Development:</emphasis>
+                        These types of developers do application level work
+                        on top of an existing software stack.
+                        </para></listitem>
+                    <listitem><para>
+                        <emphasis>Core System Development:</emphasis>
+                        These types of developers work on the contents of the
+                        operating system image itself.
+                        </para></listitem>
+                    <listitem><para>
+                        <emphasis>Build Engineer:</emphasis>
+                        This type of developer manages Autobuilders and
+                        releases.
+                        Not all environments need a Build Engineer.
+                        </para></listitem>
+                    <listitem><para>
+                        <emphasis>Test Engineer:</emphasis>
+                        This type of developer creates and manages automated
+                        tests needed to ensure all application and core
+                        system development meets desired quality standards.
+                        </para></listitem>
+                </itemizedlist>
+                </para></listitem>
+            <listitem><para>
+                <emphasis>Gather the Hardware:</emphasis>
+                Based on the size and make-up of the team, get the hardware
+                together.
+                Any development, build, or test engineer should be using
+                a system that is running a supported Linux distribution.
+                Systems, in general, should be high performance (e.g. dual,
+                six-core Xeons with 24 Gbytes of RAM and plenty of disk space).
+                You can help ensure efficiency by having any machines used
+                for testing or that run Autobuilders be as high performance
+                as possible.
+                </para></listitem>
+            <listitem><para>
+                <emphasis>Understand the Hardware Topology of the Environment:</emphasis>
+                Once you understand the hardware involved and the make-up
+                of the team, you can understand the hardware topology of the
+                development environment.
+                You can get a visual idea of the machines and their roles
+                across the development environment.
+
+<!--
+                The following figure shows a moderately sized Yocto Project
+                development environment.
+
+                <para role="writernotes">
+                Need figure.</para>
+-->
+
+                </para></listitem>
+            <listitem><para>
+                <emphasis>Use Git as Your Source Control Manager (SCM):</emphasis>
+                Keeping your
+                <ulink url='&YOCTO_DOCS_REF_URL;#metadata'>Metadata</ulink>
+                and any software you are developing under the
+                control of an SCM system that is compatible
+                with the OpenEmbedded build system is advisable.
+                Of the SCMs BitBake supports, the
+                Yocto Project team strongly recommends using
+                <ulink url='&YOCTO_DOCS_OM_URL;#git'>Git</ulink>.
+                Git is a distributed system that is easy to backup,
+                allows you to work remotely, and then connects back to the
+                infrastructure.
+                <note>
+                    For information about BitBake, see the
+                    <ulink url='&YOCTO_DOCS_BB_URL;'>BitBake User Manual</ulink>.
+                </note></para>
+
+                <para>It is relatively easy to set up Git services and create
+                infrastructure like
+                <ulink url='&YOCTO_GIT_URL;'>http://git.yoctoproject.org</ulink>,
+                which is based on server software called
+                <filename>gitolite</filename> with <filename>cgit</filename>
+                being used to generate the web interface that lets you view the
+                repositories.
+                The <filename>gitolite</filename> software identifies users
+                using SSH keys and allows branch-based
+                access controls to repositories that you can control as little
+                or as much as necessary.
+
+                <note>
+                   The setup of these services is beyond the scope of this
+                   manual.
+                   However, sites such as these exist that describe how to
+                   perform setup:
+                   <itemizedlist>
+                       <listitem><para>
+                           <ulink url='http://git-scm.com/book/ch4-8.html'>Git documentation</ulink>:
+                           Describes how to install <filename>gitolite</filename>
+                           on the server.
+                           </para></listitem>
+                       <listitem><para>
+                           <ulink url='http://gitolite.com'>Gitolite</ulink>:
+                            Information for <filename>gitolite</filename>.
+                            </para></listitem>
+                        <listitem><para>
+                            <ulink url='https://git.wiki.kernel.org/index.php/Interfaces,_frontends,_and_tools'>Interfaces, frontends, and tools</ulink>:
+                            Documentation on how to create interfaces and frontends
+                            for Git.
+                            </para></listitem>
+                    </itemizedlist>
+                </note>
+                </para></listitem>
+            <listitem><para>
+                <emphasis>Set up the Application Development Machines:</emphasis>
+                As mentioned earlier, application developers are creating
+                applications on top of existing software stacks.
+                Following are some best practices for setting up machines
+                that do application development:
+                <itemizedlist>
+                    <listitem><para>
+                        Use a pre-built toolchain that
+                        contains the software stack itself.
+                        Then, develop the application code on top of the
+                        stack.
+                        This method works well for small numbers of relatively
+                        isolated applications.
+                        </para></listitem>
+                    <listitem><para>
+                        When possible, use the Yocto Project
+                        plug-in for the
+                        <trademark class='trade'>Eclipse</trademark> IDE
+                        and SDK development practices.
+                        For more information, see the
+                        "<ulink url='&YOCTO_DOCS_SDK_URL;'>Yocto Project Application Development and the Extensible Software Development Kit (eSDK)</ulink>"
+                        manual.
+                        </para></listitem>
+                    <listitem><para>
+                        Keep your cross-development toolchains updated.
+                        You can do this through provisioning either as new
+                        toolchain downloads or as updates through a package
+                        update mechanism using <filename>opkg</filename>
+                        to provide updates to an existing toolchain.
+                        The exact mechanics of how and when to do this are a
+                        question for local policy.
+                        </para></listitem>
+                    <listitem><para>
+                        Use multiple toolchains installed locally
+                        into different locations to allow development across
+                        versions.
+                        </para></listitem>
+                </itemizedlist>
+                </para></listitem>
+            <listitem><para>
+                <emphasis>Set up the Core Development Machines:</emphasis>
+                As mentioned earlier, these types of developers work on the
+                contents of the operating system itself.
+                Following are some best practices for setting up machines
+                used for developing images:
+                <itemizedlist>
+                    <listitem><para>
+                        Have the Yocto Project build system itself available on
+                        the developer workstations so developers can run their own
+                        builds and directly rebuild the software stack.
+                        </para></listitem>
+                    <listitem><para>
+                        Keep the core system unchanged as much as
+                        possible and do your work in layers on top of the
+                        core system.
+                        Doing so gives you a greater level of portability when
+                        upgrading to new versions of the core system or Board
+                        Support Packages (BSPs).
+                        </para></listitem>
+                    <listitem><para>
+                        Share layers amongst the developers of a
+                        particular project and contain the policy configuration
+                        that defines the project.
+                        </para></listitem>
+                </itemizedlist>
+                </para></listitem>
+            <listitem><para>
+                <emphasis>Set up an Autobuilder:</emphasis>
+                Autobuilders are often the core of the development
+                environment.
+                It is here that changes from individual developers are brought
+                together and centrally tested and subsequent decisions about
+                releases can be made.
+                Autobuilders also allow for "continuous integration" style
+                testing of software components and regression identification
+                and tracking.</para>
+
+                <para>See "<ulink url='http://autobuilder.yoctoproject.org'>Yocto Project Autobuilder</ulink>"
+                for more information and links to buildbot.
+                The Yocto Project team has found this implementation
+                works well in this role.
+                A public example of this is the Yocto Project
+                Autobuilders, which we use to test the overall health of the
+                project.</para>
+
+                <para>The features of this system are:
+                <itemizedlist>
+                    <listitem><para>
+                         Highlights when commits break the build.
+                         </para></listitem>
+                    <listitem><para>
+                        Populates an sstate cache from which
+                        developers can pull rather than requiring local
+                        builds.
+                        </para></listitem>
+                    <listitem><para>
+                        Allows commit hook triggers,
+                        which trigger builds when commits are made.
+                        </para></listitem>
+                    <listitem><para>
+                        Allows triggering of automated image booting
+                        and testing under the QuickEMUlator (QEMU).
+                        </para></listitem>
+                    <listitem><para>
+                        Supports incremental build testing and
+                        from-scratch builds.
+                        </para></listitem>
+                    <listitem><para>
+                        Shares output that allows developer
+                        testing and historical regression investigation.
+                        </para></listitem>
+                    <listitem><para>
+                        Creates output that can be used for releases.
+                        </para></listitem>
+                    <listitem><para>
+                        Allows scheduling of builds so that resources
+                        can be used efficiently.
+                        </para></listitem>
+               </itemizedlist>
+               </para></listitem>
+           <listitem><para>
+               <emphasis>Set up Test Machines:</emphasis>
+               Use a small number of shared, high performance systems
+               for testing purposes.
+               Developers can use these systems for wider, more
+               extensive testing while they continue to develop
+               locally using their primary development system.
+               </para></listitem>
+           <listitem><para>
+               <emphasis>Document Policies and Change Flow:</emphasis>
+               The Yocto Project itself uses a hierarchical structure and a
+                pull model.
+                Scripts exist to create and send pull requests
+                (i.e. <filename>create-pull-request</filename> and
+                <filename>send-pull-request</filename>).
+                This model is in line with other open source projects where
+                maintainers are responsible for specific areas of the project
+                and a single maintainer handles the final "top-of-tree" merges.
+                <note>
+                    You can also use a more collective push model.
+                    The <filename>gitolite</filename> software supports both the
+                    push and pull models quite easily.
+                </note></para>
+
+                <para>As with any development environment, it is important
+                to document the policy used as well as any main project
+                guidelines so they are understood by everyone.
+                It is also a good idea to have well structured
+                commit messages, which are usually a part of a project's
+                guidelines.
+                Good commit messages are essential when looking back in time and
+                trying to understand why changes were made.</para>
+
+                <para>If you discover that changes are needed to the core
+                layer of the project, it is worth sharing those with the
+                community as soon as possible.
+                Chances are if you have discovered the need for changes,
+                someone else in the community needs them also.
+                </para></listitem>
+            <listitem><para>
+                <emphasis>Development Environment Summary:</emphasis>
+                Aside from the previous steps, some best practices exist
+                within the Yocto Project development environment.
+                Consider the following:
+                <itemizedlist>
+                    <listitem><para>
+                        Use
+                        <ulink url='&YOCTO_DOCS_OM_URL;#git'>Git</ulink>
+                        as the source control system.
+                        </para></listitem>
+                    <listitem><para>
+                        Maintain your Metadata in layers that make sense
+                        for your situation.
+                        See the "<link linkend='understanding-and-creating-layers'>Understanding
+                        and Creating Layers</link>" section for more information on
+                        layers.
+                        </para></listitem>
+                    <listitem><para>
+                        Separate the project's Metadata and code by using
+                        separate Git repositories.
+                        See the
+                        "<ulink url='&YOCTO_DOCS_OM_URL;#yocto-project-repositories'>Yocto Project Source Repositories</ulink>"
+                        section for information on these repositories.
+                        See the
+                        "<link linkend='locating-yocto-project-source-files'>Locating Yocto Project Source Files</link>"
+                        section for information on how to set up local Git
+                        repositories for related upstream Yocto Project
+                        Git repositories.
+                        </para></listitem>
+                    <listitem><para>
+                        Set up the directory for the shared state cache
+                        (<ulink url='&YOCTO_DOCS_REF_URL;#var-SSTATE_DIR'><filename>SSTATE_DIR</filename></ulink>)
+                        where it makes sense.
+                        For example, set up the sstate cache on a system used
+                        by developers in the same organization and share the
+                        same source directories on their machines.
+                        </para></listitem>
+                    <listitem><para>
+                        Set up an Autobuilder and have it populate the
+                        sstate cache and source directories.
+                        </para></listitem>
+                    <listitem><para>
+                        The Yocto Project community encourages you
+                        to send patches to the project to fix bugs or add features.
+                        If you do submit patches, follow the project commit
+                        guidelines for writing good commit messages.
+                        See the "<link linkend='how-to-submit-a-change'>Submitting a Change to the Yocto Project</link>"
+                        section.
+                        </para></listitem>
+                    <listitem><para>
+                        Send changes to the core sooner than later
+                        as others are likely to run into the same issues.
+                        For some guidance on mailing lists to use, see the list in the
+                        "<link linkend='how-to-submit-a-change'>Submitting a Change to the Yocto Project</link>"
+                        section.
+                        For a description of the available mailing lists, see the
+                        "<ulink url='&YOCTO_DOCS_REF_URL;#resources-mailinglist'>Mailing Lists</ulink>"
+                        section in the Yocto Project Reference Manual.
+                        </para></listitem>
+                </itemizedlist>
+                </para></listitem>
+        </orderedlist>
+    </para>
+</section>
+
 <section id='setting-up-the-development-host-to-use-the-yocto-project'>
-    <title>Setting Up the Development Host to Use the Yocto Project</title>
+    <title>Preparing the Build Host</title>
 
     <para>
         This section provides procedures to set up your development host to
@@ -177,7 +545,7 @@
                     site.
                     </para></listitem>
                 <listitem><para>
-                    <emphasis>Go the Install Site for Your Platform:</emphasis>
+                    <emphasis>Go to the Install Site for Your Platform:</emphasis>
                     Click the link for the Docker edition associated with
                     your development host machine's native software.
                     For example, if your machine is running Microsoft
@@ -213,8 +581,8 @@
                     the type of the software you need to install.
                     </para></listitem>
                 <listitem><para>
-                    <emphasis>Optionally Orient Yourself With Dockers:</emphasis>
-                    If you are unfamiliar with Dockers and the container
+                    <emphasis>Optionally Orient Yourself With Docker:</emphasis>
+                    If you are unfamiliar with Docker and the container
                     concept, you can learn more here -
                     <ulink url='https://docs.docker.com/get-started/'></ulink>.
                     You should be able to launch Docker or the Docker Toolbox
@@ -250,25 +618,25 @@
     </section>
 </section>
 
-<section id='working-with-yocto-project-source-files'>
-    <title>Working With Yocto Project Source Files</title>
+<section id='locating-yocto-project-source-files'>
+    <title>Locating Yocto Project Source Files</title>
 
     <para>
-        This section contains procedures related to locating and securing
-        Yocto Project files.
+        This section contains procedures related to locating Yocto Project
+        files.
         You establish and use these local files to work on projects.
         <note><title>Notes</title>
             <itemizedlist>
                 <listitem><para>
                     For concepts and introductory information about Git as it
                     is used in the Yocto Project, see the
-                    "<ulink url='&YOCTO_DOCS_REF_URL;#git'>Git</ulink>"
-                    section in the Yocto Project Reference Manual.
+                    "<ulink url='&YOCTO_DOCS_OM_URL;#git'>Git</ulink>"
+                    section in the Yocto Project Overview and Concepts Manual.
                     </para></listitem>
                 <listitem><para>
                     For concepts on Yocto Project source repositories, see the
-                    "<ulink url='&YOCTO_DOCS_REF_URL;#yocto-project-repositories'>Yocto Project Source Repositories</ulink>"
-                    section in the Yocto Project Reference Manual."
+                    "<ulink url='&YOCTO_DOCS_OM_URL;#yocto-project-repositories'>Yocto Project Source Repositories</ulink>"
+                    section in the Yocto Project Overview and Concepts Manual."
                     </para></listitem>
             </itemizedlist>
         </note>
@@ -278,9 +646,20 @@
         <title>Accessing Source Repositories</title>
 
         <para>
-            Yocto Project maintains upstream Git
-            <ulink url='&YOCTO_DOCS_REF_URL;#source-repositories'>Source Repositories</ulink>
-            that you can examine and access using a browser-based UI:
+            Working from a copy of the upstream Yocto Project
+            <ulink url='&YOCTO_DOCS_OM_URL;#source-repositories'>Source Repositories</ulink>
+            is the preferred method for obtaining and using a Yocto Project
+            release.
+            You can view the Yocto Project Source Repositories at
+            <ulink url='&YOCTO_GIT_URL;'></ulink>.
+            In particular, you can find the
+            <filename>poky</filename> repository at
+            <ulink url='http://git.yoctoproject.org/cgit/cgit.cgi/poky/'></ulink>.
+        </para>
+
+        <para>
+            Use the following procedure to locate the latest upstream copy of
+            the <filename>poky</filename> Git repository:
             <orderedlist>
                 <listitem><para>
                     <emphasis>Access Repositories:</emphasis>
@@ -290,24 +669,21 @@
                     repositories.
                     </para></listitem>
                 <listitem><para>
-                    <emphasis>Select a Repository:</emphasis>
-                    Click on any repository in which you are interested (e.g.
+                    <emphasis>Select the Repository:</emphasis>
+                    Click on the repository in which you are interested (i.e.
                     <filename>poky</filename>).
                     </para></listitem>
                 <listitem><para>
                     <emphasis>Find the URL Used to Clone the Repository:</emphasis>
                     At the bottom of the page, note the URL used to
-                    <ulink url='&YOCTO_DOCS_REF_URL;#git-commands-clone'>clone</ulink>
+                    <ulink url='&YOCTO_DOCS_OM_URL;#git-commands-clone'>clone</ulink>
                     that repository (e.g.
                     <filename>&YOCTO_GIT_URL;/poky</filename>).
-                    </para></listitem>
-                <listitem><para>
-                    <emphasis>Examine Change History of the Repository:</emphasis>
-                    At the top of the page, click on any branch in which you
-                    might be interested (e.g.
-                    <filename>&DISTRO_NAME_NO_CAP;</filename>).
-                    You can then view the commit log or tree view for that
-                    development branch.
+                    <note>
+                        For information on cloning a repository, see the
+                        "<link linkend='cloning-the-poky-repository'>Cloning the <filename>poky</filename> Repository</link>"
+                        section.
+                    </note>
                     </para></listitem>
             </orderedlist>
         </para>
@@ -319,12 +695,12 @@
         <para>
             Yocto Project maintains an Index of Releases area that contains
             related files that contribute to the Yocto Project.
-            Rather than Git repositories, these files represent snapshot
-            tarballs.
+            Rather than Git repositories, these files are tarballs that
+            represent snapshots in time of a given component.
             <note><title>Tip</title>
                 The recommended method for accessing Yocto Project
-                components is to use Git to clone a repository and work from
-                within that local repository.
+                components is to use Git to clone the upstream repository and
+                work from within that locally cloned repository.
                 The procedure in this section exists should you desire a
                 tarball snapshot of any given component.
             </note>
@@ -342,8 +718,8 @@
                         full array of released Poky tarballs.
                         The <filename>poky</filename> directory in the
                         Index of Releases was historically used for very
-                        early releases and exists for retroactive
-                        completeness only.
+                        early releases and exists now only for retroactive
+                        completeness.
                     </note>
                     </para></listitem>
                 <listitem><para>
@@ -361,7 +737,7 @@
                     </para></listitem>
                 <listitem><para>
                     <emphasis>Download the Tarball:</emphasis>
-                    Click a tarball to download and save a snapshot of a
+                    Click the tarball to download and save a snapshot of the
                     given component.
                     </para></listitem>
             </orderedlist>
@@ -374,7 +750,7 @@
         <para>
             The
             <ulink url='&YOCTO_HOME_URL;'>Yocto Project Website</ulink>
-            uses a "Downloads" area from which you can locate and download
+            uses a "DOWNLOADS" page from which you can locate and download
             tarballs of any Yocto Project release.
             Rather than Git repositories, these files represent snapshot
             tarballs.
@@ -394,51 +770,99 @@
                     </para></listitem>
                 <listitem><para>
                     <emphasis>Get to the Downloads Area:</emphasis>
-                    Click the "Downloads" tab.
+                    Select the "DOWNLOADS" item from the pull-down
+                    "SOFTWARE" tab menu.
                     </para></listitem>
                 <listitem><para>
-                    <emphasis>Select the Type of Files:</emphasis>
-                    Click the type of files you want (i.e "Build System",
-                    "Tools", or "Board Support Packages (BSPs)".
+                    <emphasis>Select a Yocto Project Release:</emphasis>
+                    Use the menu next to "RELEASE" to display and choose
+                    a Yocto Project release (e.g. sumo, rocko, pyro, and
+                    so forth.
+                    For a "map" of Yocto Project releases to version numbers,
+                    see the
+                    <ulink url='https://wiki.yoctoproject.org/wiki/Releases'>Releases</ulink>
+                    wiki page.
                     </para></listitem>
                 <listitem><para>
-                    <emphasis>Locate and Download the Tarball:</emphasis>
-                    From the list of releases, locate the appropriate
-                    download link and download the files.
+                    <emphasis>Download Tools or Board Support Packages (BSPs):</emphasis>
+                    From the "DOWNLOADS" page, you can download tools or
+                    BSPs as well.
+                    Just scroll down the page and look for what you need.
                     </para></listitem>
             </orderedlist>
         </para>
     </section>
 
+    <section id='accessing-nightly-builds'>
+        <title>Accessing Nightly Builds</title>
+
+        <para>
+            Yocto Project maintains an area for nightly builds that contains
+            tarball releases at <ulink url='&YOCTO_AB_NIGHTLY_URL;'/>.
+            These builds include Yocto Project releases, SDK installation
+            scripts, and experimental builds.
+        </para>
+
+        <para>
+            Should you ever want to access a nightly build of a particular
+            Yocto Project component, use the following procedure:
+            <orderedlist>
+                <listitem><para>
+                    <emphasis>Access the Nightly Builds:</emphasis>
+                    Open a browser and go to
+                    <ulink url='&YOCTO_AB_NIGHTLY_URL;'/> to access the
+                    Nightly Builds.
+                    </para></listitem>
+                <listitem><para>
+                    <emphasis>Select a Build:</emphasis>
+                    Click on any build by date in which you are interested.
+                    </para></listitem>
+                <listitem><para>
+                    <emphasis>Find the Tarball:</emphasis>
+                    Drill down to find the associated tarball.
+                    </para></listitem>
+                <listitem><para>
+                    <emphasis>Download the Tarball:</emphasis>
+                    Click the tarball to download and save a snapshot of the
+                    given component.
+                    </para></listitem>
+            </orderedlist>
+        </para>
+    </section>
+</section>
+
+<section id='cloning-and-checking-out-branchs'>
+    <title>Cloning and Checking Out Branches</title>
+
+    <para>
+        To use the Yocto Project, you need a release of the Yocto Project
+        locally installed on your development system.
+        The locally installed set of files is referred to as the
+        <ulink url='&YOCTO_DOCS_REF_URL;#source-directory'>Source Directory</ulink>
+        in the Yocto Project documentation.
+    </para>
+
+    <para>
+        You create your Source Directory by using
+        <ulink url='&YOCTO_DOCS_OM_URL;#git'>Git</ulink> to clone a local
+        copy of the upstream <filename>poky</filename> repository.
+        <note><title>Tip</title>
+            The preferred method of getting the Yocto Project Source
+            Directory set up is to clone the repository.
+        </note>
+        Working from a copy of the upstream repository allows you
+        to contribute back into the Yocto Project or simply work with
+        the latest software on a development branch.
+        Because Git maintains and creates an upstream repository with
+        a complete history of changes and you are working with a local
+        clone of that repository, you have access to all the Yocto
+        Project development branches and tag names used in the upstream
+        repository.
+    </para>
+
     <section id='cloning-the-poky-repository'>
         <title>Cloning the <filename>poky</filename> Repository</title>
 
-        <para>
-            To use the Yocto Project, you need a release of the Yocto Project
-            locally installed on your development system.
-            The locally installed set of files is referred to as the
-            <ulink url='&YOCTO_DOCS_REF_URL;#source-directory'>Source Directory</ulink>
-            in the Yocto Project documentation.
-        </para>
-
-        <para>
-            You create your Source Directory by using
-            <ulink url='&YOCTO_DOCS_REF_URL;#git'>Git</ulink> to clone a local
-            copy of the upstream <filename>poky</filename> repository.
-            <note><title>Tip</title>
-                The preferred method of getting the Yocto Project Source
-                Directory set up is to clone the repository.
-            </note>
-            Working from a copy of the upstream repository allows you
-            to contribute back into the Yocto Project or simply work with
-            the latest software on a development branch.
-            Because Git maintains and creates an upstream repository with
-            a complete history of changes and you are working with a local
-            clone of that repository, you have access to all the Yocto
-            Project development branches and tag names used in the upstream
-            repository.
-        </para>
-
         <para>
             Follow these steps to create a local version of the
             upstream
@@ -473,8 +897,8 @@
                     branch based on a tag name, see the
                     "<link linkend='checking-out-by-branch-in-poky'>Checking Out By Branch in Poky</link>"
                     and
-                    <link linkend='checkout-out-by-tag-in-poky'>Checking Out By Tag in Poky</link>",
-                    respectively.</para>
+                    <link linkend='checkout-out-by-tag-in-poky'>Checking Out By Tag in Poky</link>"
+                    sections, respectively.</para>
 
                     <para>Once the repository is created, you can change to
                     that directory and check its status.
@@ -635,7 +1059,7 @@
                 <listitem><para>
                     <emphasis>Checkout the Branch:</emphasis>
                     <literallayout class='monospaced'>
-     $ git checkout tags/&DISTRO; -b my_yocto_&DISTRO;
+     $ git checkout tags/&DISTRO_REL_TAG; -b my_yocto_&DISTRO;
      Switched to a new branch 'my_yocto_&DISTRO;'
      $ git branch
        master
@@ -656,95 +1080,6 @@
     </section>
 </section>
 
-<section id='performing-a-simple-build'>
-    <title>Performing a Simple Build</title>
-
-    <para>
-        Several methods exist that allow you to build an image within the
-        Yocto Project.
-        This procedure shows how to build an image using BitBake from a
-        Linux host.
-        <note><title>Notes</title>
-            <itemizedlist>
-                <listitem><para>
-                    For information on how to build an image using
-                    <ulink url='&YOCTO_DOCS_REF_URL;#toaster-term'>Toaster</ulink>,
-                    see the
-                    <ulink url='&YOCTO_DOCS_TOAST_URL;'>Yocto Project Toaster Manual</ulink>.
-                    </para></listitem>
-                <listitem><para>
-                    For information on how to use
-                    <filename>devtool</filename> to build images, see the
-                    "<ulink url='&YOCTO_DOCS_SDK_URL;#using-devtool-in-your-sdk-workflow'>Using <filename>devtool</filename> in Your SDK Workflow</ulink>"
-                    section in the Yocto Project Application Development and
-                    the Extensible Software Development Kit (eSDK) manual.
-                    </para></listitem>
-            </itemizedlist>
-        </note>
-    </para>
-
-    <para>
-        The build process creates an entire Linux distribution from source
-        and places it in your
-        <ulink url='&YOCTO_DOCS_REF_URL;#build-directory'>Build Directory</ulink>
-        under <filename>tmp/deploy/images</filename>.
-        For detailed information on the build process using BitBake, see the
-        "<ulink url='&YOCTO_DOCS_REF_URL;#images-dev-environment'>Images</ulink>"
-        section in the Yocto Project Reference Manual.
-        You can also reference the
-        "<ulink url='&YOCTO_DOCS_QS_URL;#qs-building-images'>Building Images</ulink>"
-        section in the Yocto Project Quick Start.
-    </para>
-
-    <para>
-        The following figure and list overviews the build process:
-        <imagedata fileref="figures/bitbake-build-flow.png" width="7in" depth="4in" align="center" scalefit="1" />
-        <orderedlist>
-            <listitem><para>
-                <emphasis>Set up Your Host Development System to Support
-                Development Using the Yocto Project</emphasis>:
-                See the
-                "<ulink url='&YOCTO_DOCS_QS_URL;#yp-resources'>Setting Up to Use the Yocto Project</ulink>"
-                section in the Yocto Project Quick Start for options on how
-                to get a build host ready to use the Yocto Project.
-                </para></listitem>
-            <listitem><para>
-                <emphasis>Initialize the Build Environment:</emphasis>
-                Initialize the build environment by sourcing the build
-                environment script (i.e.
-                <ulink url='&YOCTO_DOCS_REF_URL;#structure-core-script'><filename>&OE_INIT_FILE;</filename></ulink>).
-                </para></listitem>
-            <listitem><para>
-                <emphasis>Make Sure Your <filename>local.conf</filename>
-                File is Correct:</emphasis>
-                Ensure the <filename>conf/local.conf</filename> configuration
-                file, which is found in the
-                <ulink url='&YOCTO_DOCS_REF_URL;#build-directory'>Build Directory</ulink>,
-                is set up how you want it.
-                This file defines many aspects of the build environment
-                including the target machine architecture through the
-                <filename><ulink url='&YOCTO_DOCS_REF_URL;#var-MACHINE'>MACHINE</ulink></filename> variable,
-                the packaging format used during the build
-                (<ulink url='&YOCTO_DOCS_REF_URL;#var-PACKAGE_CLASSES'><filename>PACKAGE_CLASSES</filename></ulink>),
-                and a centralized tarball download directory through the
-                <filename><ulink url='&YOCTO_DOCS_REF_URL;#var-DL_DIR'>DL_DIR</ulink></filename> variable.
-                </para></listitem>
-            <listitem><para>
-                <emphasis>Build the Image:</emphasis>
-                Build the image using the <filename>bitbake</filename> command.
-                For example, the following command builds the
-                <filename>core-image-minimal</filename> image:
-                <literallayout class='monospaced'>
-     $ bitbake core-image-minimal
-                </literallayout>
-                For information on BitBake, see the
-                <ulink url='&YOCTO_DOCS_BB_URL;'>BitBake User Manual</ulink>.
-                </para></listitem>
-        </orderedlist>
-    </para>
-</section>
-
--->
 </chapter>
 <!--
 vim: expandtab tw=80 ts=4

documentation/dev-manual/dev-manual.xml

@@ -103,9 +103,14 @@
             </revision>
             <revision>
                 <revnumber>2.5</revnumber>
-                <date>April 2018</date>
+                <date>May 2018</date>
                 <revremark>Released with the Yocto Project 2.5 Release.</revremark>
             </revision>
+            <revision>
+                <revnumber>2.6</revnumber>
+                <date>&REL_MONTH_YEAR;</date>
+                <revremark>Released with the Yocto Project 2.6 Release.</revremark>
+            </revision>
         </revhistory>
 
     <copyright>
@@ -128,24 +133,36 @@
                        is for the &YOCTO_DOC_VERSION; release of the
                        Yocto Project.
                        To be sure you have the latest version of the manual
-                       for this release, use the manual from the
-                       <ulink url='&YOCTO_HOME_URL;/documentation'>Yocto Project documentation page</ulink>.
-                       </para></listitem>
-                   <listitem><para>
-                       For manuals associated with other releases of the Yocto
-                       Project, go to the
+                       for this release, go to the
                        <ulink url='&YOCTO_HOME_URL;/documentation'>Yocto Project documentation page</ulink>
-                       and use the drop-down "Active Releases" button
-                       and choose the manual associated with the desired
-                       Yocto Project.
+                       and select the manual from that site.
+                       Manuals from the site are more up-to-date than manuals
+                       derived from the Yocto Project released TAR files.
                        </para></listitem>
                    <listitem><para>
-                        To report any inaccuracies or problems with this
-                        manual, send an email to the Yocto Project
-                        discussion group at
-                        <filename>yocto@yoctoproject.com</filename> or log into
-                        the freenode <filename>#yocto</filename> channel.
-                        </para></listitem>
+                       If you located this manual through a web search, the
+                       version of the manual might not be the one you want
+                       (e.g. the search might have returned a manual much
+                       older than the Yocto Project version with which you
+                       are working).
+                       You can see all Yocto Project major releases by
+                       visiting the
+                       <ulink url='&YOCTO_WIKI_URL;/wiki/Releases'>Releases</ulink>
+                       page.
+                       If you need a version of this manual for a different
+                       Yocto Project release, visit the
+                       <ulink url='&YOCTO_HOME_URL;/documentation'>Yocto Project documentation page</ulink>
+                       and select the manual set by using the
+                       "ACTIVE RELEASES DOCUMENTATION" or "DOCUMENTS ARCHIVE"
+                       pull-down menus.
+                       </para></listitem>
+                   <listitem><para>
+                       To report any inaccuracies or problems with this
+                       manual, send an email to the Yocto Project
+                       discussion group at
+                       <filename>yocto@yoctoproject.com</filename> or log into
+                       the freenode <filename>#yocto</filename> channel.
+                       </para></listitem>
                </itemizedlist>
            </note>
     </legalnotice>
@@ -156,8 +173,6 @@
 
     <xi:include href="dev-manual-start.xml"/>
 
-    <xi:include href="dev-manual-newbie.xml"/>
-
     <xi:include href="dev-manual-common-tasks.xml"/>
 
     <xi:include href="dev-manual-qemu.xml"/>

documentation/kernel-dev/kernel-dev-advanced.xml

@@ -21,7 +21,7 @@
     <para>
         Kernel Metadata exists in many places.
         One area in the Yocto Project
-        <ulink url='&YOCTO_DOCS_REF_URL;#source-repositories'>Source Repositories</ulink>
+        <ulink url='&YOCTO_DOCS_OM_URL;#source-repositories'>Source Repositories</ulink>
         is the <filename>yocto-kernel-cache</filename> Git repository.
         You can find this repository grouped under the "Yocto Linux Kernel"
         heading in the
@@ -64,8 +64,7 @@
         <ulink url='&YOCTO_DOCS_REF_URL;#var-KMACHINE'><filename>KMACHINE</filename></ulink>
         variable.
         This variable is typically set to the same value as the
-        <ulink url='&YOCTO_DOCS_REF_URL;#var-MACHINE'><filename>MACHINE</filename></ulink>
-        variable, which is used by
+        <filename>MACHINE</filename> variable, which is used by
         <ulink url='&YOCTO_DOCS_REF_URL;#bitbake-term'>BitBake</ulink>.
         However, in some cases, the variable might instead refer to the
         underlying platform of the <filename>MACHINE</filename>.
@@ -77,8 +76,7 @@
         Multiple Corei7-based BSPs could share the same "intel-corei7-64"
         value for <filename>KMACHINE</filename>.
         It is important to realize that <filename>KMACHINE</filename> is
-        just for kernel mapping, while
-        <ulink url='&YOCTO_DOCS_REF_URL;#var-MACHINE'><filename>MACHINE</filename></ulink>
+        just for kernel mapping, while <filename>MACHINE</filename>
         is the machine type within a BSP Layer.
         Even with this distinction, however, these two variables can hold
         the same value.
@@ -116,8 +114,7 @@
         used in assembling the configuration.
         If you do not specify a <filename>LINUX_KERNEL_TYPE</filename>,
         it defaults to "standard".
-        Together with
-        <ulink url='&YOCTO_DOCS_REF_URL;#var-KMACHINE'><filename>KMACHINE</filename></ulink>,
+        Together with <filename>KMACHINE</filename>,
         <filename>LINUX_KERNEL_TYPE</filename> defines the search
         arguments used by the kernel tools to find the
         appropriate description within the kernel Metadata with which to
@@ -631,8 +628,10 @@
             <note>
                 For BSPs supported by the Yocto Project, the BSP description
                 files are located in the <filename>bsp</filename> directory
-                of the <filename>yocto-kernel-cache</filename> repository
-                organized under the "Yocto Linux Kernel" heading in the
+                of the
+                <ulink url='&YOCTO_GIT_URL;/cgit/cgit.cgi/yocto-kernel-cache/tree/bsp'><filename>yocto-kernel-cache</filename></ulink>
+                repository organized under the "Yocto Linux Kernel" heading
+                in the
                 <ulink url='http://git.yoctoproject.org/cgit/cgit.cgi'>Yocto Project Source Repositories</ulink>.
             </note>
         </para>
@@ -641,27 +640,30 @@
             This section overviews the BSP description structure, the
             aggregation concepts, and presents a detailed example using
             a BSP supported by the Yocto Project (i.e. BeagleBone Board).
+            For complete information on BSP layer file hierarchy, see the
+            <ulink url='&YOCTO_DOCS_BSP_URL;'>Yocto Project Board Support Package (BSP) Developer's Guide</ulink>.
         </para>
 
         <section id='bsp-description-file-overview'>
             <title>Overview</title>
 
             <para>
-                For simplicity, consider the following top-level BSP
+                For simplicity, consider the following root BSP layer
                 description files for the BeagleBone board.
-                Top-level BSP descriptions files employ both a structure
-                and naming convention for consistency.
+                These files employ both a structure and naming convention
+                for consistency.
                 The naming convention for the file is as follows:
                 <literallayout class='monospaced'>
-     <replaceable>bsp_name</replaceable>-<replaceable>kernel_type</replaceable>.scc
+     <replaceable>bsp_root_name</replaceable>-<replaceable>kernel_type</replaceable>.scc
                 </literallayout>
-                Here are some example top-level BSP filenames for the
+                Here are some example root layer BSP filenames for the
                 BeagleBone Board BSP, which is supported by the Yocto Project:
                 <literallayout class='monospaced'>
      beaglebone-standard.scc
      beaglebone-preempt-rt.scc
                 </literallayout>
-                Each file uses the BSP name followed by the kernel type.
+                Each file uses the root name (i.e "beaglebone") BSP name
+                followed by the kernel type.
             </para>
 
             <para>
@@ -850,7 +852,7 @@
 
             <para>
                 Now consider the "minnow" description for the "tiny" kernel
-                type (i.e. <filename>minnow-tiny.scc</filename>:
+                type (i.e. <filename>minnow-tiny.scc</filename>):
                 <literallayout class='monospaced'>
         define KMACHINE minnow
         define KTYPE tiny
@@ -1012,8 +1014,7 @@
 
         <para>
             If you modify the Metadata, you must not forget to update the
-            <ulink url='&YOCTO_DOCS_REF_URL;#var-SRCREV'><filename>SRCREV</filename></ulink>
-            statements in the kernel's recipe.
+            <filename>SRCREV</filename> statements in the kernel's recipe.
             In particular, you need to update the
             <filename>SRCREV_meta</filename> variable to match the commit in
             the <filename>KMETA</filename> branch you wish to use.
@@ -1221,9 +1222,13 @@
                 </para></listitem>
             <listitem><para>
                 <filename>define</filename>:
-                Defines variables, such as <filename>KMACHINE</filename>,
-                <filename>KTYPE</filename>, <filename>KARCH</filename>,
-                and <filename>KFEATURE_DESCRIPTION</filename>.</para></listitem>
+                Defines variables, such as
+                <ulink url='&YOCTO_DOCS_REF_URL;#var-KMACHINE'><filename>KMACHINE</filename></ulink>,
+                <ulink url='&YOCTO_DOCS_REF_URL;#var-KTYPE'><filename>KTYPE</filename></ulink>,
+                <ulink url='&YOCTO_DOCS_REF_URL;#var-KARCH'><filename>KARCH</filename></ulink>,
+                and
+                <ulink url='&YOCTO_DOCS_REF_URL;#var-KFEATURE_DESCRIPTION'><filename>KFEATURE_DESCRIPTION</filename></ulink>.
+                </para></listitem>
             <listitem><para>
                 <filename>include SCC_FILE</filename>:
                 Includes an SCC file in the current file.

documentation/kernel-dev/kernel-dev-common.xml

@@ -25,7 +25,7 @@
             Before you can do any kernel development, you need to be
             sure your build host is set up to use the Yocto Project.
             For information on how to get set up, see the
-            "<ulink url='&YOCTO_DOCS_DEV_URL;#setting-up-the-development-host-to-use-the-yocto-project'>Setting Up to Use the Yocto Project</ulink>"
+            "<ulink url='&YOCTO_DOCS_DEV_URL;#setting-up-the-development-host-to-use-the-yocto-project'>Preparing the Build Host</ulink>"
             section in the Yocto Project Development Tasks Manual.
             Part of preparing the system is creating a local Git
             repository of the
@@ -79,7 +79,7 @@
                         </literallayout>
                         <note>
                             The previous commands assume the
-                            <ulink url='&YOCTO_DOCS_REF_URL;#source-repositories'>Source Repositories</ulink>
+                            <ulink url='&YOCTO_DOCS_OM_URL;#source-repositories'>Source Repositories</ulink>
                             (i.e. <filename>poky</filename>) have been cloned
                             using Git and the local repository is named
                             "poky".
@@ -136,7 +136,7 @@
                             Developer's Guide, respectively.
                             For information on how to use the
                             <filename>bitbake-layers create-layer</filename>
-                            command, see the
+                            command to quickly set up a layer, see the
                             "<ulink url='&YOCTO_DOCS_DEV_URL;#creating-a-general-layer-using-the-bitbake-layers-script'>Creating a General Layer Using the <filename>bitbake-layers</filename> Script</ulink>"
                             section in the Yocto Project Development Tasks
                             Manual.
@@ -303,7 +303,7 @@
                         </literallayout>
                         <note>
                             The previous commands assume the
-                            <ulink url='&YOCTO_DOCS_REF_URL;#source-repositories'>Source Repositories</ulink>
+                            <ulink url='&YOCTO_DOCS_OM_URL;#source-repositories'>Source Repositories</ulink>
                             (i.e. <filename>poky</filename>) have been cloned
                             using Git and the local repository is named
                             "poky".
@@ -360,7 +360,7 @@
                             Developer's Guide, respectively.
                             For information on how to use the
                             <filename>bitbake-layers create-layer</filename>
-                            command, see the
+                            command to quickly set up a layer, see the
                             "<ulink url='&YOCTO_DOCS_DEV_URL;#creating-a-general-layer-using-the-bitbake-layers-script'>Creating a General Layer Using the <filename>bitbake-layers</filename> Script</ulink>"
                             section in the Yocto Project Development Tasks
                             Manual.
@@ -387,7 +387,7 @@
                         You can find Git repositories of supported Yocto Project
                         kernels organized under "Yocto Linux Kernel" in the
                         Yocto Project Source Repositories at
-                        <ulink url='&YOCTO_GIT_URL;/cgit.cgi'></ulink>.
+                        <ulink url='&YOCTO_GIT_URL;'></ulink>.
                         </para>
 
                         <para>
@@ -489,7 +489,8 @@
                 See the
                 "<ulink url='&YOCTO_DOCS_DEV_URL;#creating-a-general-layer-using-the-bitbake-layers-script'>Creating a General Layer Using the <filename>bitbake-layers</filename> Script</ulink>"
                 section in the Yocto Project Development Tasks Manual for
-                information on how to use this script.
+                information on how to use this script to quick set up a
+                new layer.
             </note>
         </para>
 
@@ -1224,18 +1225,6 @@
             the
             "<link linkend='getting-ready-for-traditional-kernel-development'>Getting Ready for Traditional Kernel Development</link>"
             Section.
-        </para>
-
-        <para>
-            Although this example uses Git and shell commands to generate the
-            patch, you could use the <filename>yocto-kernel</filename> script
-            found in the <ulink url='&YOCTO_DOCS_REF_URL;#source-directory'>Source Directory</ulink>
-            under <filename>scripts</filename> to add and manage kernel
-            patches and configuration.
-            See the "<ulink url='&YOCTO_DOCS_BSP_URL;#managing-kernel-patches-and-config-items-with-yocto-kernel'>Managing kernel Patches and Config Items with yocto-kernel</ulink>"
-            section in the Yocto Project Board Support Packages (BSP)
-            Developer's Guide for more information on the
-            <filename>yocto-kernel</filename> script.
             <orderedlist>
                 <listitem><para>
                     <emphasis>Edit the Source Files</emphasis>
@@ -1413,9 +1402,9 @@
      SRC_URI_append = " file://0001-calibrate.c-Added-some-printk-statements.patch"
                     </literallayout>
                     The
-                    <ulink url='&YOCTO_DOCS_REF_URL;var-FILESEXTRAPATHS'><filename>FILESEXTRAPATHS</filename></ulink>
+                    <ulink url='&YOCTO_DOCS_REF_URL;#var-FILESEXTRAPATHS'><filename>FILESEXTRAPATHS</filename></ulink>
                     and
-                    <ulink url='&YOCTO_DOCS_REF_URL;var-SRC_URI'><filename>SRC_URI</filename></ulink>
+                    <ulink url='&YOCTO_DOCS_REF_URL;#var-SRC_URI'><filename>SRC_URI</filename></ulink>
                     statements enable the OpenEmbedded build system to find
                     the patch file.</para>
 
@@ -1667,7 +1656,7 @@
                     after applying the existing defconfig file configurations.
                 </note>
                 For more information on configuring the kernel, see the
-                "<link link='changing-the-configuration'>Changing the Configuration</link>"
+                "<link linkend='changing-the-configuration'>Changing the Configuration</link>"
                 section.
             </para>
         </section>
@@ -2429,7 +2418,7 @@
                 modules.
                 If your module <filename>Makefile</filename> uses a different
                 variable, you might want to override the
-                <ulink url='&YOCTO_DOCS_REF_URL;#ref-tasks-compile'><filename>do_compile()</filename></ulink>
+                <ulink url='&YOCTO_DOCS_REF_URL;#ref-tasks-compile'><filename>do_compile</filename></ulink>
                 step, or create a patch to
                 the <filename>Makefile</filename> to work with the more typical
                 <filename>KERNEL_SRC</filename> or
@@ -2505,7 +2494,7 @@
                     the Git commands.
                     You can see the branch names through the web interface
                     to the Yocto Project source repositories at
-                    <ulink url='http://git.yoctoproject.org/cgit.cgi'></ulink>.
+                    <ulink url='&YOCTO_GIT_URL;'></ulink>.
                 </note>
                 To see a full range of the changes, use the
                 <filename>git whatchanged</filename> command and specify a

documentation/kernel-dev/kernel-dev-concepts-appx.xml

@@ -49,7 +49,7 @@
 
         <para>
             You can find a web interface to the Yocto Linux kernels in the
-            <ulink url='&YOCTO_DOCS_REF_URL;#source-repositories'>Source Repositories</ulink>
+            <ulink url='&YOCTO_DOCS_OM_URL;#source-repositories'>Source Repositories</ulink>
             at
             <ulink url='&YOCTO_GIT_URL;'></ulink>.
             If you look at the interface, you will see to the left a
@@ -239,8 +239,9 @@
                         <ulink url='http://git-scm.com/documentation'></ulink>.
                         You can also get an introduction to Git as it
                         applies to the Yocto Project in the
-                        "<ulink url='&YOCTO_DOCS_REF_URL;#git'>Git</ulink>"
-                        section in the Yocto Project Reference Manual.
+                        "<ulink url='&YOCTO_DOCS_OM_URL;#git'>Git</ulink>"
+                        section in the Yocto Project Overview and Concepts
+                        Manual.
                         The latter reference provides an overview of
                         Git and presents a minimal set of Git commands
                         that allows you to be functional using Git.
@@ -381,7 +382,7 @@
                 generic kernel just for conceptual purposes.
                 Also keep in mind that this structure represents the Yocto
                 Project
-                <ulink url='&YOCTO_DOCS_REF_URL;#source-repositories'>Source Repositories</ulink>
+                <ulink url='&YOCTO_DOCS_OM_URL;#source-repositories'>Source Repositories</ulink>
                 that are either pulled from during the build or established
                 on the host development system prior to the build by either
                 cloning a particular kernel's Git repository or by

documentation/kernel-dev/kernel-dev-intro.xml

@@ -108,7 +108,11 @@
         review and understand the following documentation:
         <itemizedlist>
             <listitem><para>
-                <ulink url='&YOCTO_DOCS_QS_URL;'>Yocto Project Quick Start</ulink>
+                <ulink url='&YOCTO_DOCS_BRIEF_URL;'>Yocto Project Quick Build</ulink>
+                document.
+                </para></listitem>
+            <listitem><para>
+                <ulink url='&YOCTO_DOCS_OM_URL;'>Yocto Project Overview and Concepts Manual</ulink>.
                 </para></listitem>
             <listitem><para>
                 <ulink url='&YOCTO_DOCS_SDK_URL;#using-devtool-in-your-sdk-workflow'><filename>devtool</filename> workflow</ulink>
@@ -127,18 +131,6 @@
                 </para></listitem>
         </itemizedlist>
     </para>
-
-    <para>
-        Finally, while this document focuses on the manual creation of
-        recipes, patches, and configuration files, the Yocto Project
-        Board Support Package (BSP) tools are available to automate
-        this process with existing content and work well to create the
-        initial framework and boilerplate code.
-        For details on these tools, see the
-        "<ulink url='&YOCTO_DOCS_BSP_URL;#using-the-yocto-projects-bsp-tools'>Using the Yocto Project's BSP Tools</ulink>"
-        section in the Yocto Project Board Support Package (BSP) Developer's
-        Guide.
-    </para>
 </section>
 
 <section id='kernel-modification-workflow'>
@@ -165,12 +157,15 @@
     <para>
         <orderedlist>
             <listitem><para>
-                <emphasis>Set Up Your Host Development System to Support
-                Development Using the Yocto Project:</emphasis>
+
+
+                <emphasis>Set up Your Host Development System to Support
+                Development Using the Yocto Project</emphasis>:
                 See the
-                "<ulink url='&YOCTO_DOCS_QS_URL;#yp-resources'>Setting Up to Use the Yocto Project</ulink>"
-                section in the Yocto Project Quick Start for options on how
-                to get a build host ready to use the Yocto Project.
+                "<ulink url='&YOCTO_DOCS_DEV_URL;#dev-manual-start'>Setting Up the Development Host to Use the Yocto Project</ulink>"
+                section in the Yocto Project Development Tasks Manual for
+                options on how to get a build host ready to use the Yocto
+                Project.
                 </para></listitem>
             <listitem><para>
                 <emphasis>Set Up Your Host Development System for Kernel Development:</emphasis>
@@ -243,11 +238,7 @@
 
                 <para>Additionally, if you are working in a BSP layer
                 and need to modify the BSP's kernel's configuration,
-                you can use the
-                <ulink url='&YOCTO_DOCS_BSP_URL;#managing-kernel-patches-and-config-items-with-yocto-kernel'><filename>yocto-kernel</filename></ulink>
-                script as well as <filename>menuconfig</filename>.
-                The <filename>yocto-kernel</filename> script lets
-                you interactively set up kernel configurations.
+                you can use <filename>menuconfig</filename>.
                 </para></listitem>
             <listitem><para>
                 <emphasis>Rebuild the Kernel Image With Your Changes:</emphasis>

documentation/kernel-dev/kernel-dev-maint-appx.xml

@@ -14,7 +14,7 @@
             create Yocto Linux kernel repositories.
             These kernel repositories are found under the heading "Yocto Linux
             Kernel" at
-            <ulink url='&YOCTO_GIT_URL;/cgit.cgi'>&YOCTO_GIT_URL;/cgit.cgi</ulink>
+            <ulink url='&YOCTO_GIT_URL;'>&YOCTO_GIT_URL;</ulink>
             and are shipped as part of a Yocto Project release.
             The team creates these repositories by compiling and executing the
             set of feature descriptions for every BSP and feature in the
@@ -118,13 +118,13 @@
             The following steps describe what happens when the Yocto Project
             Team constructs the Yocto Project kernel source Git repository
             (or tree) found at
-            <ulink url='&YOCTO_GIT_URL;/cgit.cgi'></ulink> given the
+            <ulink url='&YOCTO_GIT_URL;'></ulink> given the
             introduction of a new top-level kernel feature or BSP.
-            These are the actions that effectively provide the Metadata
-            and create the tree that includes the new feature, patch or BSP:
+            The following actions effectively provide the Metadata
+            and create the tree that includes the new feature, patch, or BSP:
             <orderedlist>
                 <listitem><para>
-                    <emphasis>Pass Feature to Build Subsystem:</emphasis>
+                    <emphasis>Pass Feature to the OpenEmbedded Build System:</emphasis>
                     A top-level kernel feature is passed to the kernel build
                     subsystem.
                     Normally, this feature is a BSP for a particular kernel
@@ -138,8 +138,10 @@
                         <listitem><para>
                             The in-tree kernel-cache directories, which are
                             located in the
-                            <filename>yocto-kernel-cache</filename>
-                            repository
+                            <ulink url='&YOCTO_GIT_URL;/cgit/cgit.cgi/yocto-kernel-cache/tree/bsp'><filename>yocto-kernel-cache</filename></ulink>
+                            repository organized under the "Yocto Linux Kernel"
+                            heading in the
+                            <ulink url='http://git.yoctoproject.org/cgit/cgit.cgi'>Yocto Project Source Repositories</ulink>.
                             </para></listitem>
                         <listitem><para>
                             Areas pointed to by <filename>SRC_URI</filename>
@@ -148,9 +150,11 @@
                     </itemizedlist>
                     For a typical build, the target of the search is a
                     feature description in an <filename>.scc</filename> file
-                    whose name follows this format:
+                    whose name follows this format (e.g.
+                    <filename>beaglebone-standard.scc</filename> and
+                    <filename>beaglebone-preempt-rt.scc</filename>):
                     <literallayout class='monospaced'>
-     <replaceable>bsp_name</replaceable>-<replaceable>kernel_type</replaceable>.scc
+     <replaceable>bsp_root_name</replaceable>-<replaceable>kernel_type</replaceable>.scc
                     </literallayout>
                 </para></listitem>
                 <listitem><para>
@@ -213,7 +217,7 @@
                         end of an existing branch.
                         The full repository generation that is found in the
                         official Yocto Project kernel repositories at
-                        <ulink url='&YOCTO_GIT_URL;/cgit.cgi'>http://git.yoctoproject.org/cgit.cgi</ulink>
+                        <ulink url='&YOCTO_GIT_URL;'>http://git.yoctoproject.org</ulink>
                         is the combination of all supported boards and
                         configurations.
                         </para></listitem>
@@ -227,7 +231,7 @@
                         </para></listitem>
                     <listitem><para>
                         The full kernel tree that you see on
-                        <ulink url='&YOCTO_GIT_URL;/cgit.cgi'></ulink> is
+                        <ulink url='&YOCTO_GIT_URL;'></ulink> is
                         generated through repeating the above steps for all
                         valid BSPs.
                         The end result is a branched, clean history tree that

documentation/kernel-dev/kernel-dev.xml

@@ -88,9 +88,14 @@
             </revision>
             <revision>
                 <revnumber>2.5</revnumber>
-                <date>April 2018</date>
+                <date>May 2018</date>
                 <revremark>Released with the Yocto Project 2.5 Release.</revremark>
             </revision>
+            <revision>
+                <revnumber>2.6</revnumber>
+                <date>&REL_MONTH_YEAR;</date>
+                <revremark>Released with the Yocto Project 2.6 Release.</revremark>
+            </revision>
         </revhistory>
 
     <copyright>
@@ -111,24 +116,36 @@
                        is for the &YOCTO_DOC_VERSION; release of the
                        Yocto Project.
                        To be sure you have the latest version of the manual
-                       for this release, use the manual from the
-                       <ulink url='&YOCTO_HOME_URL;/documentation'>Yocto Project documentation page</ulink>.
-                       </para></listitem>
-                   <listitem><para>
-                       For manuals associated with other releases of the Yocto
-                       Project, go to the
+                       for this release, go to the
                        <ulink url='&YOCTO_HOME_URL;/documentation'>Yocto Project documentation page</ulink>
-                       and use the drop-down "Active Releases" button
-                       and choose the manual associated with the desired
-                       Yocto Project.
+                       and select the manual from that site.
+                       Manuals from the site are more up-to-date than manuals
+                       derived from the Yocto Project released TAR files.
                        </para></listitem>
                    <listitem><para>
-                        To report any inaccuracies or problems with this
-                        manual, send an email to the Yocto Project
-                        discussion group at
-                        <filename>yocto@yoctoproject.com</filename> or log into
-                        the freenode <filename>#yocto</filename> channel.
-                        </para></listitem>
+                       If you located this manual through a web search, the
+                       version of the manual might not be the one you want
+                       (e.g. the search might have returned a manual much
+                       older than the Yocto Project version with which you
+                       are working).
+                       You can see all Yocto Project major releases by
+                       visiting the
+                       <ulink url='&YOCTO_WIKI_URL;/wiki/Releases'>Releases</ulink>
+                       page.
+                       If you need a version of this manual for a different
+                       Yocto Project release, visit the
+                       <ulink url='&YOCTO_HOME_URL;/documentation'>Yocto Project documentation page</ulink>
+                       and select the manual set by using the
+                       "ACTIVE RELEASES DOCUMENTATION" or "DOCUMENTS ARCHIVE"
+                       pull-down menus.
+                       </para></listitem>
+                   <listitem><para>
+                       To report any inaccuracies or problems with this
+                       manual, send an email to the Yocto Project
+                       discussion group at
+                       <filename>yocto@yoctoproject.com</filename> or log into
+                       the freenode <filename>#yocto</filename> channel.
+                       </para></listitem>
                </itemizedlist>
            </note>
     </legalnotice>

documentation/mega-manual/mega-manual.xml

@@ -72,9 +72,14 @@
             </revision>
             <revision>
                 <revnumber>2.5</revnumber>
-                <date>April 2018</date>
+                <date>May 2018</date>
                 <revremark>Released with the Yocto Project 2.5 Release.</revremark>
             </revision>
+            <revision>
+                <revnumber>2.6</revnumber>
+                <date>&REL_MONTH_YEAR;</date>
+                <revremark>Released with the Yocto Project 2.6 Release.</revremark>
+            </revision>
        </revhistory>
 
     <copyright>
@@ -95,24 +100,36 @@
                        is for the &YOCTO_DOC_VERSION; release of the
                        Yocto Project.
                        To be sure you have the latest version of the manual
-                       for this release, use the manual from the
-                       <ulink url='&YOCTO_HOME_URL;/documentation'>Yocto Project documentation page</ulink>.
-                       </para></listitem>
-                   <listitem><para>
-                       For manuals associated with other releases of the Yocto
-                       Project, go to the
+                       for this release, go to the
                        <ulink url='&YOCTO_HOME_URL;/documentation'>Yocto Project documentation page</ulink>
-                       and use the drop-down "Active Releases" button
-                       and choose the manual associated with the desired
-                       Yocto Project.
+                       and select the manual from that site.
+                       Manuals from the site are more up-to-date than manuals
+                       derived from the Yocto Project released TAR files.
                        </para></listitem>
                    <listitem><para>
-                        To report any inaccuracies or problems with this
-                        manual, send an email to the Yocto Project
-                        discussion group at
-                        <filename>yocto@yoctoproject.com</filename> or log into
-                        the freenode <filename>#yocto</filename> channel.
-                        </para></listitem>
+                       If you located this manual through a web search, the
+                       version of the manual might not be the one you want
+                       (e.g. the search might have returned a manual much
+                       older than the Yocto Project version with which you
+                       are working).
+                       You can see all Yocto Project major releases by
+                       visiting the
+                       <ulink url='&YOCTO_WIKI_URL;/wiki/Releases'>Releases</ulink>
+                       page.
+                       If you need a version of this manual for a different
+                       Yocto Project release, visit the
+                       <ulink url='&YOCTO_HOME_URL;/documentation'>Yocto Project documentation page</ulink>
+                       and select the manual set by using the
+                       "ACTIVE RELEASES DOCUMENTATION" or "DOCUMENTS ARCHIVE"
+                       pull-down menus.
+                       </para></listitem>
+                   <listitem><para>
+                       To report any inaccuracies or problems with this
+                       manual, send an email to the Yocto Project
+                       discussion group at
+                       <filename>yocto@yoctoproject.com</filename> or log into
+                       the freenode <filename>#yocto</filename> channel.
+                       </para></listitem>
                </itemizedlist>
            </note>
 
@@ -120,14 +137,32 @@
 
     </bookinfo>
 
-<!-- Includes yocto-project-qs -->
+<!-- Includes brief-yoctoprojectqs -->
 
     <para>
-        <imagedata fileref="figures/ypqs-title.png" width="100%" align="left" scalefit="1" />
+        <imagedata fileref="figures/bypqs-title.png" width="100%" align="left" scalefit="1" />
     </para>
 
     <xi:include
-        xmlns:xi="http://www.w3.org/2003/XInclude" href="../yocto-project-qs/qs.xml"/>
+        xmlns:xi="http://www.w3.org/2003/XInclude" href="../brief-yoctoprojectqs/brief-yoctoprojectqs.xml"/>
+
+<!-- Includes overview-manual title image and then overview-manual chapters -->
+
+    <para>
+        <imagedata fileref="figures/overview-manual-title.png" width="100%" align="left" scalefit="1" />
+    </para>
+
+    <xi:include
+        xmlns:xi="http://www.w3.org/2003/XInclude" href="../overview-manual/overview-manual-intro.xml"/>
+
+    <xi:include
+        xmlns:xi="http://www.w3.org/2003/XInclude" href="../overview-manual/overview-manual-yp-intro.xml"/>
+
+    <xi:include
+        xmlns:xi="http://www.w3.org/2003/XInclude" href="../overview-manual/overview-manual-development-environment.xml"/>
+
+    <xi:include
+        xmlns:xi="http://www.w3.org/2003/XInclude" href="../overview-manual/overview-manual-concepts.xml"/>
 
 <!-- Includes dev-manual title image and then dev-manual chapters -->
 
@@ -139,8 +174,6 @@
         xmlns:xi="http://www.w3.org/2003/XInclude" href="../dev-manual/dev-manual-intro.xml"/>
     <xi:include
         xmlns:xi="http://www.w3.org/2003/XInclude" href="../dev-manual/dev-manual-start.xml"/>
-    <xi:include
-        xmlns:xi="http://www.w3.org/2003/XInclude" href="../dev-manual/dev-manual-newbie.xml"/>
     <xi:include
         xmlns:xi="http://www.w3.org/2003/XInclude" href="../dev-manual/dev-manual-common-tasks.xml"/>
     <xi:include
@@ -169,7 +202,7 @@
     <xi:include
         xmlns:xi="http://www.w3.org/2003/XInclude" href="../sdk-manual/sdk-appendix-customizing-standard.xml"/>
     <xi:include
-        xmlns:xi="http://www.w3.org/2003/XInclude" href="../sdk-manual/sdk-appendix-mars.xml"/>
+        xmlns:xi="http://www.w3.org/2003/XInclude" href="../sdk-manual/sdk-appendix-neon.xml"/>
 
 <!-- Includes bsp-guide title image and then bsp-guide chapters -->
 
@@ -219,16 +252,10 @@
     </para>
 
     <xi:include
-        xmlns:xi="http://www.w3.org/2003/XInclude" href="../ref-manual/introduction.xml"/>
+        xmlns:xi="http://www.w3.org/2003/XInclude" href="../ref-manual/ref-system-requirements.xml"/>
 
     <xi:include
-        xmlns:xi="http://www.w3.org/2003/XInclude" href="../ref-manual/usingpoky.xml"/>
-
-    <xi:include
-        xmlns:xi="http://www.w3.org/2003/XInclude" href="../ref-manual/ref-development-environment.xml"/>
-
-    <xi:include
-        xmlns:xi="http://www.w3.org/2003/XInclude" href="../ref-manual/technical-details.xml"/>
+        xmlns:xi="http://www.w3.org/2003/XInclude" href="../ref-manual/ref-terms.xml"/>
 
     <xi:include
         xmlns:xi="http://www.w3.org/2003/XInclude" href="../ref-manual/ref-release-process.xml"/>