bitbake: daemonize: Always print any remaning UI events at exit

If there are events in the UI queue we want to print them regardless of
whether we're handling an exception or something like SystemExit.

This improves error messages for some other failure cases where currently
no logging would get printed and leave the user confused as to what went
wrong.

(Bitbake rev: 76c4f2c20216719736766e8ae7d089ccd061b71b)

Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>

README.LSB

@@ -0,0 +1,25 @@
+OE-Core aims to be able to provide basic LSB compatible images. There
+are some challenges for OE as LSB isn't always 100% relevant to its
+target embedded and IoT audiences. 
+
+One challenge is that the LSB spec is no longer being actively
+developed [https://github.com/LinuxStandardBase/lsb] and has 
+components which are end of life or significantly dated. OE 
+therefore provides compatibility with the following caveats:
+
+* Qt4 is provided by the separate meta-qt4 layer. Its noted that Qt4 
+  is end of life and this isn't something the core project regularly 
+  tests any longer. Users are recommended to group together to support
+  maintenance of that layer. [http://git.yoctoproject.org/cgit/cgit.cgi/meta-qt4/]
+
+* mailx has been dropped since its no longer being developed upstream 
+  and there are better, more modern replacements such as s-nail 
+  (http://sdaoden.eu/code.html) or mailutils (http://mailutils.org/).
+
+* A few perl modules that were required by LSB 4.x aren't provided:
+  libclass-isa, libenv, libdumpvalue, libfile-checktree,
+  libi18n-collate, libpod-plainer.
+
+* libpng 1.2 isn't provided; oe-core includes the latest release of libpng
+  instead.
+

bitbake/bin/bitbake

@@ -38,7 +38,7 @@ from bb.main import bitbake_main, BitBakeConfigParameters, BBMainException
 if sys.getfilesystemencoding() != "utf-8":
     sys.exit("Please use a locale setting which supports utf-8.\nPython can't change the filesystem locale after loading so we need a utf-8 when python starts or things won't work.")
 
-__version__ = "1.34.0"
+__version__ = "1.35.0"
 
 if __name__ == "__main__":
     if __version__ != bb.__version__:

bitbake/bin/bitbake-diffsigs

@@ -34,18 +34,39 @@ import bb.msg
 
 logger = bb.msg.logger_create('bitbake-diffsigs')
 
+def find_siginfo(tinfoil, pn, taskname, sigs=None):
+    result = None
+    tinfoil.set_event_mask(['bb.event.FindSigInfoResult',
+                            'logging.LogRecord',
+                            'bb.command.CommandCompleted',
+                            'bb.command.CommandFailed'])
+    ret = tinfoil.run_command('findSigInfo', pn, taskname, sigs)
+    if ret:
+        while True:
+            event = tinfoil.wait_event(1)
+            if event:
+                if isinstance(event, bb.command.CommandCompleted):
+                    break
+                elif isinstance(event, bb.command.CommandFailed):
+                    logger.error(str(event))
+                    sys.exit(2)
+                elif isinstance(event, bb.event.FindSigInfoResult):
+                    result = event.result
+                elif isinstance(event, logging.LogRecord):
+                    logger.handle(event)
+    else:
+        logger.error('No result returned from findSigInfo command')
+        sys.exit(2)
+    return result
+
 def find_compare_task(bbhandler, pn, taskname, sig1=None, sig2=None, color=False):
     """ Find the most recent signature files for the specified PN/task and compare them """
 
-    if not hasattr(bb.siggen, 'find_siginfo'):
-        logger.error('Metadata does not support finding signature data files')
-        sys.exit(1)
-
     if not taskname.startswith('do_'):
         taskname = 'do_%s' % taskname
 
     if sig1 and sig2:
-        sigfiles = bb.siggen.find_siginfo(pn, taskname, [sig1, sig2], bbhandler.config_data)
+        sigfiles = find_siginfo(bbhandler, pn, taskname, [sig1, sig2])
         if len(sigfiles) == 0:
             logger.error('No sigdata files found matching %s %s matching either %s or %s' % (pn, taskname, sig1, sig2))
             sys.exit(1)
@@ -57,7 +78,7 @@ def find_compare_task(bbhandler, pn, taskname, sig1=None, sig2=None, color=False
             sys.exit(1)
         latestfiles = [sigfiles[sig1], sigfiles[sig2]]
     else:
-        filedates = bb.siggen.find_siginfo(pn, taskname, None, bbhandler.config_data)
+        filedates = find_siginfo(bbhandler, pn, taskname)
         latestfiles = sorted(filedates.keys(), key=lambda f: filedates[f])[-3:]
         if not latestfiles:
             logger.error('No sigdata files found matching %s %s' % (pn, taskname))
@@ -69,7 +90,7 @@ def find_compare_task(bbhandler, pn, taskname, sig1=None, sig2=None, color=False
     # Define recursion callback
     def recursecb(key, hash1, hash2):
         hashes = [hash1, hash2]
-        hashfiles = bb.siggen.find_siginfo(key, None, hashes, bbhandler.config_data)
+        hashfiles = find_siginfo(bbhandler, key, None, hashes)
 
         recout = []
         if len(hashfiles) == 0:

bitbake/bin/bitbake-layers

@@ -90,7 +90,7 @@ def main():
 
         if getattr(args, 'parserecipes', False):
             tinfoil.config_data.disableTracking()
-            tinfoil.parseRecipes()
+            tinfoil.parse_recipes()
             tinfoil.config_data.enableTracking()
 
         return args.func(args)

bitbake/bin/bitbake-selftest

@@ -28,6 +28,7 @@ except RuntimeError as exc:
 tests = ["bb.tests.codeparser",
          "bb.tests.cow",
          "bb.tests.data",
+         "bb.tests.event",
          "bb.tests.fetch",
          "bb.tests.parse",
          "bb.tests.utils"]

bitbake/bin/toaster

@@ -24,6 +24,29 @@ Usage: source toaster start|stop [webport=<address:port>] [noweb]
         [webport] Set the development server (default: localhost:8000)
 "
 
+databaseCheck()
+{
+    retval=0
+    # you can always add a superuser later via
+    # ../bitbake/lib/toaster/manage.py createsuperuser --username=<ME>
+    $MANAGE migrate --noinput || retval=1
+
+    if [ $retval -eq 1 ]; then
+        echo "Failed migrations, aborting system start" 1>&2
+        return $retval
+    fi
+    # Make sure that checksettings can pick up any value for TEMPLATECONF
+    export TEMPLATECONF
+    $MANAGE checksettings --traceback || retval=1
+
+    if [ $retval -eq 1 ]; then
+        printf "\nError while checking settings; aborting\n"
+        return $retval
+    fi
+
+    return $retval
+}
+
 webserverKillAll()
 {
     local pidfile
@@ -48,22 +71,9 @@ webserverStartAll()
     fi
 
     retval=0
-    # you can always add a superuser later via
-    # ../bitbake/lib/toaster/manage.py createsuperuser --username=<ME>
-    $MANAGE migrate --noinput || retval=1
 
-    if [ $retval -eq 1 ]; then
-        echo "Failed migrations, aborting system start" 1>&2
-        return $retval
-    fi
-    # Make sure that checksettings can pick up any value for TEMPLATECONF
-    export TEMPLATECONF
-    $MANAGE checksettings --traceback || retval=1
-
-    if [ $retval -eq 1 ]; then
-        printf "\nError while checking settings; aborting\n"
-        return $retval
-    fi
+    # check the database
+    databaseCheck || return 1
 
     echo "Starting webserver..."
 
@@ -106,8 +116,14 @@ verify_prereq() {
     # Verify Django version
     reqfile=$(python3 -c "import os; print(os.path.realpath('$BBBASEDIR/toaster-requirements.txt'))")
     exp='s/Django\([><=]\+\)\([^,]\+\),\([><=]\+\)\(.\+\)/'
-    exp=$exp'import sys,django;version=django.get_version().split(".");'
-    exp=$exp'sys.exit(not (version \1 "\2".split(".") and version \3 "\4".split(".")))/p'
+    # expand version parts to 2 digits to support 1.10.x > 1.8
+    # (note:helper functions hard to insert in-line)
+    exp=$exp'import sys,django;'
+    exp=$exp'version=["%02d" % int(n) for n in django.get_version().split(".")];'
+    exp=$exp'vmin=["%02d" % int(n) for n in "\2".split(".")];'
+    exp=$exp'vmax=["%02d" % int(n) for n in "\4".split(".")];'
+    exp=$exp'sys.exit(not (version \1 vmin and version \3 vmax))'
+    exp=$exp'/p'
     if ! sed -n "$exp" $reqfile | python3 - ; then
         req=`grep ^Django $reqfile`
         echo "This program needs $req"
@@ -240,6 +256,16 @@ case $CMD in
         line='INHERIT+="toaster buildhistory"'
         grep -q "$line" $conf || echo $line >> $conf
 
+        if [ $WEBSERVER -eq 0 ] ; then
+            # Do not update the database for "noweb" unless
+            # it does not yet exist
+            if [ ! -f "$TOASTER_DIR/toaster.sqlite" ] ; then
+                if ! databaseCheck; then
+                    echo "Failed ${CMD}."
+                  return 4
+                fi
+            fi
+        fi
         if [ $WEBSERVER -gt 0 ] && ! webserverStartAll; then
             echo "Failed ${CMD}."
             return 4

bitbake/doc/bitbake-user-manual/bitbake-user-manual-hello.xml

@@ -269,7 +269,7 @@
                 and define some key BitBake variables.
                 For more information on the <filename>bitbake.conf</filename>,
                 see
-                <ulink url='http://hambedded.org/blog/2012/11/24/from-bitbake-hello-world-to-an-image/#an-overview-of-bitbakeconf'></ulink>
+                <ulink url='http://git.openembedded.org/bitbake/tree/conf/bitbake.conf'></ulink>.
                 </para>
                 <para>Use the following commands to create the <filename>conf</filename>
                 directory in the project directory:
@@ -375,8 +375,8 @@ ERROR: Unable to parse base: ParseError in configuration INHERITs: Could not inh
                 code separate from the general metadata used by BitBake.
                 Thus, this example creates and uses a layer called "mylayer".
                 <note>
-                    You can find additional information on adding a layer at
-                    <ulink url='http://hambedded.org/blog/2012/11/24/from-bitbake-hello-world-to-an-image/#adding-an-example-layer'></ulink>.
+                    You can find additional information on layers at
+                    <ulink url='http://www.yoctoproject.org/docs/2.3/bitbake-user-manual/bitbake-user-manual.html#layers'></ulink>.
                 </note>
                 </para>
                 <para>Minimally, you need a recipe file and a layer configuration

bitbake/doc/bitbake-user-manual/bitbake-user-manual-ref-variables.xml

@@ -1143,8 +1143,6 @@
             <glossdef>
                 <para>
                     Sets the base location where layers are stored.
-                    By default, this location is set to
-                    <filename>${COREBASE}</filename>.
                     This setting is used in conjunction with
                     <filename>bitbake-layers layerindex-fetch</filename> and
                     tells <filename>bitbake-layers</filename> where to place

bitbake/lib/bb/__init__.py

@@ -21,7 +21,7 @@
 # with this program; if not, write to the Free Software Foundation, Inc.,
 # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
 
-__version__ = "1.34.0"
+__version__ = "1.35.0"
 
 import sys
 if sys.version_info < (3, 4, 0):

bitbake/lib/bb/cache.py

@@ -86,9 +86,9 @@ class RecipeInfoCommon(object):
 class CoreRecipeInfo(RecipeInfoCommon):
     __slots__ = ()
 
-    cachefile = "bb_cache.dat"   
+    cachefile = "bb_cache.dat"
 
-    def __init__(self, filename, metadata):      
+    def __init__(self, filename, metadata):
         self.file_depends = metadata.getVar('__depends', False)
         self.timestamp = bb.parse.cached_mtime(filename)
         self.variants = self.listvar('__VARIANTS', metadata) + ['']
@@ -107,7 +107,7 @@ class CoreRecipeInfo(RecipeInfoCommon):
 
         self.pn = self.getvar('PN', metadata)
         self.packages = self.listvar('PACKAGES', metadata)
-        if not self.pn in self.packages:
+        if not self.packages:
             self.packages.append(self.pn)
 
         self.basetaskhashes = self.taskvar('BB_BASEHASH', self.tasks, metadata)
@@ -122,7 +122,7 @@ class CoreRecipeInfo(RecipeInfoCommon):
         self.defaultpref = self.intvar('DEFAULT_PREFERENCE', metadata)
         self.not_world = self.getvar('EXCLUDE_FROM_WORLD', metadata)
         self.stamp = self.getvar('STAMP', metadata)
-        self.stampclean = self.getvar('STAMPCLEAN', metadata)        
+        self.stampclean = self.getvar('STAMPCLEAN', metadata)
         self.stamp_extrainfo = self.flaglist('stamp-extra-info', self.tasks, metadata)
         self.file_checksums = self.flaglist('file-checksums', self.tasks, metadata, True)
         self.packages_dynamic = self.listvar('PACKAGES_DYNAMIC', metadata)
@@ -217,7 +217,7 @@ class CoreRecipeInfo(RecipeInfoCommon):
             cachedata.packages_dynamic[package].append(fn)
 
         # Build hash of runtime depends and recommends
-        for package in self.packages + [self.pn]:
+        for package in self.packages:
             cachedata.rundeps[fn][package] = list(self.rdepends) + self.rdepends_pkg[package]
             cachedata.runrecs[fn][package] = list(self.rrecommends) + self.rrecommends_pkg[package]
 
@@ -375,8 +375,8 @@ class Cache(NoCache):
         data = databuilder.data
 
         # Pass caches_array information into Cache Constructor
-        # It will be used later for deciding whether we 
-        # need extra cache file dump/load support 
+        # It will be used later for deciding whether we
+        # need extra cache file dump/load support
         self.caches_array = caches_array
         self.cachedir = data.getVar("CACHE")
         self.clean = set()
@@ -421,7 +421,7 @@ class Cache(NoCache):
                 cachesize += os.fstat(cachefile.fileno()).st_size
 
         bb.event.fire(bb.event.CacheLoadStarted(cachesize), self.data)
-        
+
         for cache_class in self.caches_array:
             cachefile = getCacheFile(self.cachedir, cache_class.cachefile, self.data_hash)
             with open(cachefile, "rb") as cachefile:
@@ -438,8 +438,8 @@ class Cache(NoCache):
                     logger.info('Cache version mismatch, rebuilding...')
                     return
                 elif bitbake_ver != bb.__version__:
-                     logger.info('Bitbake version mismatch, rebuilding...')
-                     return
+                    logger.info('Bitbake version mismatch, rebuilding...')
+                    return
 
                 # Load the rest of the cache file
                 current_progress = 0
@@ -616,13 +616,13 @@ class Cache(NoCache):
                     a = fl.find(":True")
                     b = fl.find(":False")
                     if ((a < 0) and b) or ((b > 0) and (b < a)):
-                       f = fl[:b+6]
-                       fl = fl[b+7:]
+                        f = fl[:b+6]
+                        fl = fl[b+7:]
                     elif ((b < 0) and a) or ((a > 0) and (a < b)):
-                       f = fl[:a+5]
-                       fl = fl[a+6:]
+                        f = fl[:a+5]
+                        fl = fl[a+6:]
                     else:
-                       break
+                        break
                     fl = fl.strip()
                     if "*" in f:
                         continue
@@ -886,4 +886,3 @@ class MultiProcessCache(object):
             p.dump([data, self.__class__.CACHE_VERSION])
 
         bb.utils.unlockfile(glf)
-

bitbake/lib/bb/command.py

@@ -50,6 +50,8 @@ class CommandFailed(CommandExit):
     def __init__(self, message):
         self.error = message
         CommandExit.__init__(self, 1)
+    def __str__(self):
+        return "Command execution failed: %s" % self.error
 
 class CommandError(Exception):
     pass
@@ -238,6 +240,11 @@ class CommandsSync:
         return command.cooker.matchFile(fMatch)
     matchFile.needconfig = False
 
+    def getUIHandlerNum(self, command, params):
+        return bb.event.get_uihandler()
+    getUIHandlerNum.needconfig = False
+    getUIHandlerNum.readonly = True
+
     def setEventMask(self, command, params):
         handlerNum = params[0]
         llevel = params[1]
@@ -305,6 +312,38 @@ class CommandsSync:
         return command.cooker.recipecaches[mc].pkg_pepvpr
     getRecipeVersions.readonly = True
 
+    def getRecipeProvides(self, command, params):
+        try:
+            mc = params[0]
+        except IndexError:
+            mc = ''
+        return command.cooker.recipecaches[mc].fn_provides
+    getRecipeProvides.readonly = True
+
+    def getRecipePackages(self, command, params):
+        try:
+            mc = params[0]
+        except IndexError:
+            mc = ''
+        return command.cooker.recipecaches[mc].packages
+    getRecipePackages.readonly = True
+
+    def getRecipePackagesDynamic(self, command, params):
+        try:
+            mc = params[0]
+        except IndexError:
+            mc = ''
+        return command.cooker.recipecaches[mc].packages_dynamic
+    getRecipePackagesDynamic.readonly = True
+
+    def getRProviders(self, command, params):
+        try:
+            mc = params[0]
+        except IndexError:
+            mc = ''
+        return command.cooker.recipecaches[mc].rproviders
+    getRProviders.readonly = True
+
     def getRuntimeDepends(self, command, params):
         ret = []
         try:
@@ -543,11 +582,14 @@ class CommandsAsync:
         bfile = params[0]
         task = params[1]
         if len(params) > 2:
-            hidewarning = params[2]
+            internal = params[2]
         else:
-            hidewarning = False
+            internal = False
 
-        command.cooker.buildFile(bfile, task, hidewarning)
+        if internal:
+            command.cooker.buildFileInternal(bfile, task, fireevents=False, quietlog=True)
+        else:
+            command.cooker.buildFile(bfile, task)
     buildFile.needcache = False
 
     def buildTargets(self, command, params):
@@ -704,3 +746,14 @@ class CommandsAsync:
         command.finishAsyncCommand()
     clientComplete.needcache = False
 
+    def findSigInfo(self, command, params):
+        """
+        Find signature info files via the signature generator
+        """
+        pn = params[0]
+        taskname = params[1]
+        sigs = params[2]
+        res = bb.siggen.find_siginfo(pn, taskname, sigs, command.cooker.data)
+        bb.event.fire(bb.event.FindSigInfoResult(res), command.cooker.data)
+        command.finishAsyncCommand()
+    findSigInfo.needcache = False

bitbake/lib/bb/cooker.py

@@ -165,7 +165,7 @@ class BBCooker:
     Manages one bitbake build run
     """
 
-    def __init__(self, configuration, featureSet=None):
+    def __init__(self, configuration, featureSet=None, readypipe=None):
         self.recipecaches = None
         self.skiplist = {}
         self.featureset = CookerFeatures()
@@ -181,13 +181,13 @@ class BBCooker:
         self.confignotifier = pyinotify.Notifier(self.configwatcher, self.config_notifications)
         self.watchmask = pyinotify.IN_CLOSE_WRITE | pyinotify.IN_CREATE | pyinotify.IN_DELETE | \
                          pyinotify.IN_DELETE_SELF | pyinotify.IN_MODIFY | pyinotify.IN_MOVE_SELF | \
-                         pyinotify.IN_MOVED_FROM | pyinotify.IN_MOVED_TO 
+                         pyinotify.IN_MOVED_FROM | pyinotify.IN_MOVED_TO
         self.watcher = pyinotify.WatchManager()
         self.watcher.bbseen = []
         self.watcher.bbwatchedfiles = []
         self.notifier = pyinotify.Notifier(self.watcher, self.notifications)
 
-        # If being called by something like tinfoil, we need to clean cached data 
+        # If being called by something like tinfoil, we need to clean cached data
         # which may now be invalid
         bb.parse.__mtime_cache = {}
         bb.parse.BBHandler.cached_statements = {}
@@ -215,19 +215,6 @@ class BBCooker:
 
         self.configuration.server_register_idlecallback(_process_inotify_updates, [self.confignotifier, self.notifier])
 
-        # Take a lock so only one copy of bitbake can run against a given build
-        # directory at a time
-        if not self.lockBitbake():
-            bb.fatal("Only one copy of bitbake should be run against a build directory")
-        try:
-            self.lock.seek(0)
-            self.lock.truncate()
-            if len(configuration.interface) >= 2:
-                self.lock.write("%s:%s\n" % (configuration.interface[0], configuration.interface[1]));
-            self.lock.flush()
-        except:
-            pass
-
         # TOSTOP must not be set or our children will hang when they output
         try:
             fd = sys.stdout.fileno()
@@ -250,6 +237,10 @@ class BBCooker:
         # Let SIGHUP exit as SIGTERM
         signal.signal(signal.SIGHUP, self.sigterm_exception)
 
+        if readypipe:
+            os.write(readypipe, b"ready")
+            os.close(readypipe)
+
     def config_notifications(self, event):
         if event.maskname == "IN_Q_OVERFLOW":
             bb.warn("inotify event queue overflowed, invalidating caches.")
@@ -282,7 +273,7 @@ class BBCooker:
             watchtarget = None
             while True:
                 # We try and add watches for files that don't exist but if they did, would influence
-                # the parser. The parent directory of these files may not exist, in which case we need 
+                # the parser. The parent directory of these files may not exist, in which case we need
                 # to watch any parent that does exist for changes.
                 try:
                     watcher.add_watch(f, self.watchmask, quiet=False)
@@ -440,7 +431,7 @@ class BBCooker:
                 del self.configuration.env[k]
                 clean = False
             if k not in self.configuration.env and k not in environment:
-                 continue
+                continue
             if environment[k] != self.configuration.env[k]:
                 logger.debug(1, "Updating environment variable %s from %s to %s" % (k, self.configuration.env[k], environment[k]))
                 self.configuration.env[k] = environment[k]
@@ -687,12 +678,12 @@ class BBCooker:
                     depend_tree["pn"][pn][ei] = vars(self.recipecaches[mc])[ei][taskfn]
 
 
+            dotname = "%s.%s" % (pn, bb.runqueue.taskname_from_tid(tid))
+            if not dotname in depend_tree["tdepends"]:
+                depend_tree["tdepends"][dotname] = []
             for dep in rq.rqdata.runtaskentries[tid].depends:
                 (depmc, depfn, deptaskname, deptaskfn) = bb.runqueue.split_tid_mcfn(dep)
                 deppn = self.recipecaches[mc].pkg_fn[deptaskfn]
-                dotname = "%s.%s" % (pn, bb.runqueue.taskname_from_tid(tid))
-                if not dotname in depend_tree["tdepends"]:
-                    depend_tree["tdepends"][dotname] = []
                 depend_tree["tdepends"][dotname].append("%s.%s" % (deppn, bb.runqueue.taskname_from_tid(dep)))
             if taskfn not in seen_fns:
                 seen_fns.append(taskfn)
@@ -1172,12 +1163,14 @@ class BBCooker:
         """
         Setup any variables needed before starting a build
         """
-        t = time.gmtime() 
-        if not self.data.getVar("BUILDNAME", False):
-            self.data.setVar("BUILDNAME", "${DATE}${TIME}")
-        self.data.setVar("BUILDSTART", time.strftime('%m/%d/%Y %H:%M:%S', t))
-        self.data.setVar("DATE", time.strftime('%Y%m%d', t))
-        self.data.setVar("TIME", time.strftime('%H%M%S', t))
+        t = time.gmtime()
+        for mc in self.databuilder.mcdata:
+            ds = self.databuilder.mcdata[mc]
+            if not ds.getVar("BUILDNAME", False):
+                ds.setVar("BUILDNAME", "${DATE}${TIME}")
+            ds.setVar("BUILDSTART", time.strftime('%m/%d/%Y %H:%M:%S', t))
+            ds.setVar("DATE", time.strftime('%Y%m%d', t))
+            ds.setVar("TIME", time.strftime('%H%M%S', t))
 
     def reset_mtime_caches(self):
         """
@@ -1230,21 +1223,27 @@ class BBCooker:
             raise NoSpecificMatch
         return matches[0]
 
-    def buildFile(self, buildfile, task, hidewarning=False):
+    def buildFile(self, buildfile, task):
         """
         Build the file matching regexp buildfile
         """
         bb.event.fire(bb.event.BuildInit(), self.data)
 
-        if not hidewarning:
-            # Too many people use -b because they think it's how you normally
-            # specify a target to be built, so show a warning
-            bb.warn("Buildfile specified, dependencies will not be handled. If this is not what you want, do not use -b / --buildfile.")
+        # Too many people use -b because they think it's how you normally
+        # specify a target to be built, so show a warning
+        bb.warn("Buildfile specified, dependencies will not be handled. If this is not what you want, do not use -b / --buildfile.")
 
         # Parse the configuration here. We need to do it explicitly here since
         # buildFile() doesn't use the cache
         self.parseConfiguration()
 
+        self.buildFileInternal(buildfile, task)
+
+    def buildFileInternal(self, buildfile, task, fireevents=True, quietlog=False):
+        """
+        Build the file matching regexp buildfile
+        """
+
         # If we are told to do the None task then query the default task
         if (task == None):
             task = self.configuration.cmd
@@ -1281,8 +1280,8 @@ class BBCooker:
         # Remove external dependencies
         self.recipecaches[mc].task_deps[fn]['depends'] = {}
         self.recipecaches[mc].deps[fn] = []
-        self.recipecaches[mc].rundeps[fn] = []
-        self.recipecaches[mc].runrecs[fn] = []
+        self.recipecaches[mc].rundeps[fn] = defaultdict(list)
+        self.recipecaches[mc].runrecs[fn] = defaultdict(list)
 
         # Invalidate task for target if force mode active
         if self.configuration.force:
@@ -1292,10 +1291,15 @@ class BBCooker:
         # Setup taskdata structure
         taskdata = {}
         taskdata[mc] = bb.taskdata.TaskData(self.configuration.abort)
-        taskdata[mc].add_provider(self.data, self.recipecaches[mc], item)
+        taskdata[mc].add_provider(self.databuilder.mcdata[mc], self.recipecaches[mc], item)
 
-        buildname = self.data.getVar("BUILDNAME")
-        bb.event.fire(bb.event.BuildStarted(buildname, [item]), self.data)
+        if quietlog:
+            rqloglevel = bb.runqueue.logger.getEffectiveLevel()
+            bb.runqueue.logger.setLevel(logging.WARNING)
+
+        buildname = self.databuilder.mcdata[mc].getVar("BUILDNAME")
+        if fireevents:
+            bb.event.fire(bb.event.BuildStarted(buildname, [item]), self.databuilder.mcdata[mc])
 
         # Execute the runqueue
         runlist = [[mc, item, task, fn]]
@@ -1322,11 +1326,16 @@ class BBCooker:
                 retval = False
             except SystemExit as exc:
                 self.command.finishAsyncCommand(str(exc))
+                if quietlog:
+                    bb.runqueue.logger.setLevel(rqloglevel)
                 return False
 
             if not retval:
-                bb.event.fire(bb.event.BuildCompleted(len(rq.rqdata.runtaskentries), buildname, item, failures, interrupted), self.data)
+                if fireevents:
+                    bb.event.fire(bb.event.BuildCompleted(len(rq.rqdata.runtaskentries), buildname, item, failures, interrupted), self.databuilder.mcdata[mc])
                 self.command.finishAsyncCommand(msg)
+                if quietlog:
+                    bb.runqueue.logger.setLevel(rqloglevel)
                 return False
             if retval is True:
                 return True
@@ -1362,7 +1371,8 @@ class BBCooker:
 
             if not retval:
                 try:
-                    bb.event.fire(bb.event.BuildCompleted(len(rq.rqdata.runtaskentries), buildname, targets, failures, interrupted), self.data)
+                    for mc in self.multiconfigs:
+                        bb.event.fire(bb.event.BuildCompleted(len(rq.rqdata.runtaskentries), buildname, targets, failures, interrupted), self.databuilder.mcdata[mc])
                 finally:
                     self.command.finishAsyncCommand(msg)
                 return False
@@ -1395,7 +1405,8 @@ class BBCooker:
                 ntargets.append("multiconfig:%s:%s:%s" % (target[0], target[1], target[2]))
             ntargets.append("%s:%s" % (target[1], target[2]))
 
-        bb.event.fire(bb.event.BuildStarted(buildname, ntargets), self.data)
+        for mc in self.multiconfigs:
+            bb.event.fire(bb.event.BuildStarted(buildname, ntargets), self.databuilder.mcdata[mc])
 
         rq = bb.runqueue.RunQueue(self, self.data, self.recipecaches, taskdata, runlist)
         if 'universe' in targets:
@@ -1553,33 +1564,6 @@ class BBCooker:
     def post_serve(self):
         prserv.serv.auto_shutdown(self.data)
         bb.event.fire(CookerExit(), self.data)
-        lockfile = self.lock.name
-        self.lock.close()
-        self.lock = None
-
-        while not self.lock:
-            with bb.utils.timeout(3):
-                self.lock = bb.utils.lockfile(lockfile, shared=False, retry=False, block=True)
-                if not self.lock:
-                    # Some systems may not have lsof available
-                    procs = None
-                    try:
-                        procs = subprocess.check_output(["lsof", '-w', lockfile], stderr=subprocess.STDOUT)
-                    except OSError as e:
-                        if e.errno != errno.ENOENT:
-                            raise
-                    if procs is None:
-                        # Fall back to fuser if lsof is unavailable
-                        try:
-                            procs = subprocess.check_output(["fuser", '-v', lockfile], stderr=subprocess.STDOUT)
-                        except OSError as e:
-                            if e.errno != errno.ENOENT:
-                                raise
-
-                    msg = "Delaying shutdown due to active processes which appear to be holding bitbake.lock"
-                    if procs:
-                        msg += ":\n%s" % str(procs)
-                    print(msg)
 
 
     def shutdown(self, force = False):
@@ -1601,46 +1585,10 @@ class BBCooker:
 
     def clientComplete(self):
         """Called when the client is done using the server"""
-        if self.configuration.server_only:
-            self.finishcommand()
-        else:
-            self.shutdown(True)
+        self.finishcommand()
+        self.databuilder.reset()
+        self.data = self.databuilder.data
 
-    def lockBitbake(self):
-        if not hasattr(self, 'lock'):
-            self.lock = None
-            if self.data:
-                lockfile = self.data.expand("${TOPDIR}/bitbake.lock")
-                if lockfile:
-                    self.lock = bb.utils.lockfile(lockfile, False, False)
-        return self.lock
-
-    def unlockBitbake(self):
-        if hasattr(self, 'lock') and self.lock:
-            bb.utils.unlockfile(self.lock)
-
-def server_main(cooker, func, *args):
-    cooker.pre_serve()
-
-    if cooker.configuration.profile:
-        try:
-            import cProfile as profile
-        except:
-            import profile
-        prof = profile.Profile()
-
-        ret = profile.Profile.runcall(prof, func, *args)
-
-        prof.dump_stats("profile.log")
-        bb.utils.process_profilelog("profile.log")
-        print("Raw profiling information saved to profile.log and processed statistics to profile.log.processed")
-
-    else:
-        ret = func(*args)
-
-    cooker.post_serve()
-
-    return ret
 
 class CookerExit(bb.event.Event):
     """

bitbake/lib/bb/cookerdata.py

@@ -73,15 +73,15 @@ class ConfigParameters(object):
 
     def updateToServer(self, server, environment):
         options = {}
-        for o in ["abort", "tryaltconfigs", "force", "invalidate_stamp", 
-                  "verbose", "debug", "dry_run", "dump_signatures", 
+        for o in ["abort", "tryaltconfigs", "force", "invalidate_stamp",
+                  "verbose", "debug", "dry_run", "dump_signatures",
                   "debug_domains", "extra_assume_provided", "profile",
-                  "prefile", "postfile", "tracking"]:
+                  "prefile", "postfile", "tracking", "server_timeout"]:
             options[o] = getattr(self.options, o)
 
         ret, error = server.runCommand(["updateConfig", options, environment, sys.argv])
         if error:
-                raise Exception("Unable to update the server configuration with local parameters: %s" % error)
+            raise Exception("Unable to update the server configuration with local parameters: %s" % error)
 
     def parseActions(self):
         # Parse any commandline into actions
@@ -144,7 +144,8 @@ class CookerConfiguration(object):
         self.dump_signatures = []
         self.dry_run = False
         self.tracking = False
-        self.interface = []
+        self.xmlrpcinterface = []
+        self.server_timeout = None
         self.writeeventlog = False
         self.server_only = False
         self.limited_deps = False
@@ -173,7 +174,7 @@ class CookerConfiguration(object):
 
     def __setstate__(self,state):
         for k in state:
-            setattr(self, k, state[k]) 
+            setattr(self, k, state[k])
 
 
 def catch_parse_error(func):
@@ -230,6 +231,27 @@ def findConfigFile(configfile, data):
 
     return None
 
+#
+# We search for a conf/bblayers.conf under an entry in BBPATH or in cwd working 
+# up to /. If that fails, we search for a conf/bitbake.conf in BBPATH.
+#
+
+def findTopdir():
+    d = bb.data.init()
+    bbpath = None
+    if 'BBPATH' in os.environ:
+        bbpath = os.environ['BBPATH']
+        d.setVar('BBPATH', bbpath)
+
+    layerconf = findConfigFile("bblayers.conf", d)
+    if layerconf:
+        return os.path.dirname(os.path.dirname(layerconf))
+    if bbpath:
+        bitbakeconf = bb.utils.which(bbpath, "conf/bitbake.conf")
+        if bitbakeconf:
+            return os.path.dirname(os.path.dirname(bitbakeconf))
+    return None
+
 class CookerDataBuilder(object):
 
     def __init__(self, cookercfg, worker = False):
@@ -255,7 +277,7 @@ class CookerDataBuilder(object):
         filtered_keys = bb.utils.approved_variables()
         bb.data.inheritFromOS(self.basedata, self.savedenv, filtered_keys)
         self.basedata.setVar("BB_ORIGENV", self.savedenv)
-        
+
         if worker:
             self.basedata.setVar("BB_WORKERCONTEXT", "1")
 
@@ -294,6 +316,8 @@ class CookerDataBuilder(object):
                 mcdata = self.parseConfigurationFiles(self.prefiles, self.postfiles, config)
                 bb.event.fire(bb.event.ConfigParsed(), mcdata)
                 self.mcdata[config] = mcdata
+            if multiconfig:
+                bb.event.fire(bb.event.MultiConfigParsed(self.mcdata), self.data)
 
         except (SyntaxError, bb.BBHandledException):
             raise bb.BBHandledException
@@ -304,6 +328,15 @@ class CookerDataBuilder(object):
             logger.exception("Error parsing configuration files")
             raise bb.BBHandledException
 
+        # Create a copy so we can reset at a later date when UIs disconnect
+        self.origdata = self.data
+        self.data = bb.data.createCopy(self.origdata)
+        self.mcdata[''] = self.data
+
+    def reset(self):
+        self.data = bb.data.createCopy(self.origdata)
+        self.mcdata[''] = self.data
+
     def _findLayerConf(self, data):
         return findConfigFile("bblayers.conf", data)
 

bitbake/lib/bb/daemonize.py

@@ -1,48 +1,14 @@
 """
 Python Daemonizing helper
 
-Configurable daemon behaviors:
-
-    1.) The current working directory set to the "/" directory.
-    2.) The current file creation mode mask set to 0.
-    3.) Close all open files (1024). 
-    4.) Redirect standard I/O streams to "/dev/null".
-
-A failed call to fork() now raises an exception.
-
-References:
-    1) Advanced Programming in the Unix Environment: W. Richard Stevens
-	http://www.apuebook.com/apue3e.html
-    2) The Linux Programming Interface: Michael Kerrisk
-	http://man7.org/tlpi/index.html
-    3) Unix Programming Frequently Asked Questions:
-	http://www.faqs.org/faqs/unix-faq/programmer/faq/
-
-Modified to allow a function to be daemonized and return for 
-bitbake use by Richard Purdie
+Originally based on code Copyright (C) 2005 Chad J. Schroeder but now heavily modified
+to allow a function to be daemonized and return for bitbake use by Richard Purdie
 """
 
-__author__ = "Chad J. Schroeder"
-__copyright__ = "Copyright (C) 2005 Chad J. Schroeder"
-__version__ = "0.2"
-
-# Standard Python modules.
-import os                    # Miscellaneous OS interfaces.
-import sys                   # System-specific parameters and functions.
-
-# Default daemon parameters.
-# File mode creation mask of the daemon.
-# For BitBake's children, we do want to inherit the parent umask.
-UMASK = None
-
-# Default maximum for the number of available file descriptors.
-MAXFD = 1024
-
-# The standard I/O file descriptors are redirected to /dev/null by default.
-if (hasattr(os, "devnull")):
-    REDIRECT_TO = os.devnull
-else:
-    REDIRECT_TO = "/dev/null"
+import os
+import sys
+import io
+import traceback
 
 def createDaemon(function, logfile):
     """
@@ -65,36 +31,6 @@ def createDaemon(function, logfile):
         # leader of the new process group, we call os.setsid().  The process is
         # also guaranteed not to have a controlling terminal.
         os.setsid()
-
-        # Is ignoring SIGHUP necessary?
-        #
-        # It's often suggested that the SIGHUP signal should be ignored before
-        # the second fork to avoid premature termination of the process.  The
-        # reason is that when the first child terminates, all processes, e.g.
-        # the second child, in the orphaned group will be sent a SIGHUP.
-        #
-        # "However, as part of the session management system, there are exactly
-        # two cases where SIGHUP is sent on the death of a process:
-        #
-        #    1) When the process that dies is the session leader of a session that
-        #        is attached to a terminal device, SIGHUP is sent to all processes
-        #        in the foreground process group of that terminal device.
-        #    2) When the death of a process causes a process group to become
-        #        orphaned, and one or more processes in the orphaned group are
-        #        stopped, then SIGHUP and SIGCONT are sent to all members of the
-        #        orphaned group." [2]
-        #
-        # The first case can be ignored since the child is guaranteed not to have
-        # a controlling terminal.  The second case isn't so easy to dismiss.
-        # The process group is orphaned when the first child terminates and
-        # POSIX.1 requires that every STOPPED process in an orphaned process
-        # group be sent a SIGHUP signal followed by a SIGCONT signal.  Since the
-        # second child is not STOPPED though, we can safely forego ignoring the
-        # SIGHUP signal.  In any case, there are no ill-effects if it is ignored.
-        #
-        # import signal              # Set handlers for asynchronous events.
-        # signal.signal(signal.SIGHUP, signal.SIG_IGN)
-
         try:
             # Fork a second child and exit immediately to prevent zombies.  This
             # causes the second child process to be orphaned, making the init
@@ -108,86 +44,39 @@ def createDaemon(function, logfile):
         except OSError as e:
             raise Exception("%s [%d]" % (e.strerror, e.errno))
 
-        if (pid == 0):  # The second child.
-            # We probably don't want the file mode creation mask inherited from
-            # the parent, so we give the child complete control over permissions.
-            if UMASK is not None:
-                os.umask(UMASK)
-        else:
+        if (pid != 0):
             # Parent (the first child) of the second child.
+            # exit() or _exit()?
+            # _exit is like exit(), but it doesn't call any functions registered
+            # with atexit (and on_exit) or any registered signal handlers.  It also
+            # closes any open file descriptors.  Using exit() may cause all stdio
+            # streams to be flushed twice and any temporary files may be unexpectedly
+            # removed.  It's therefore recommended that child branches of a fork()
+            # and the parent branch(es) of a daemon use _exit().
             os._exit(0)
     else:
-        # exit() or _exit()?
-        # _exit is like exit(), but it doesn't call any functions registered
-        # with atexit (and on_exit) or any registered signal handlers.  It also
-        # closes any open file descriptors.  Using exit() may cause all stdio
-        # streams to be flushed twice and any temporary files may be unexpectedly
-        # removed.  It's therefore recommended that child branches of a fork()
-        # and the parent branch(es) of a daemon use _exit().
+        os.waitpid(pid, 0)
         return
 
-    # Close all open file descriptors.  This prevents the child from keeping
-    # open any file descriptors inherited from the parent.  There is a variety
-    # of methods to accomplish this task.  Three are listed below.
-    #
-    # Try the system configuration variable, SC_OPEN_MAX, to obtain the maximum
-    # number of open file descriptors to close.  If it doesn't exist, use
-    # the default value (configurable).
-    #
-    # try:
-    #     maxfd = os.sysconf("SC_OPEN_MAX")
-    # except (AttributeError, ValueError):
-    #     maxfd = MAXFD
-    #
-    # OR
-    #
-    # if (os.sysconf_names.has_key("SC_OPEN_MAX")):
-    #     maxfd = os.sysconf("SC_OPEN_MAX")
-    # else:
-    #     maxfd = MAXFD
-    #
-    # OR
-    #
-    # Use the getrlimit method to retrieve the maximum file descriptor number
-    # that can be opened by this process.  If there is no limit on the
-    # resource, use the default value.
-    #
-    import resource             # Resource usage information.
-    maxfd = resource.getrlimit(resource.RLIMIT_NOFILE)[1]
-    if (maxfd == resource.RLIM_INFINITY):
-        maxfd = MAXFD
-  
-    # Iterate through and close all file descriptors.
-#    for fd in range(0, maxfd):
-#        try:
-#            os.close(fd)
-#        except OSError:        # ERROR, fd wasn't open to begin with (ignored)
-#            pass
-
-    # Redirect the standard I/O file descriptors to the specified file.  Since
-    # the daemon has no controlling terminal, most daemons redirect stdin,
-    # stdout, and stderr to /dev/null.  This is done to prevent side-effects
-    # from reads and writes to the standard I/O file descriptors.
-
-    # This call to open is guaranteed to return the lowest file descriptor,
-    # which will be 0 (stdin), since it was closed above.
-#    os.open(REDIRECT_TO, os.O_RDWR)    # standard input (0)
-
-    # Duplicate standard input to standard output and standard error.
-#    os.dup2(0, 1)                      # standard output (1)
-#    os.dup2(0, 2)                      # standard error (2)
-
+    # The second child.
 
+    # Replace standard fds with our own
     si = open('/dev/null', 'r')
-    so = open(logfile, 'w')
-    se = so
-
-
-    # Replace those fds with our own
     os.dup2(si.fileno(), sys.stdin.fileno())
-    os.dup2(so.fileno(), sys.stdout.fileno())
-    os.dup2(se.fileno(), sys.stderr.fileno())
 
-    function()
+    try:
+        so = open(logfile, 'a+')
+        se = so
+        os.dup2(so.fileno(), sys.stdout.fileno())
+        os.dup2(se.fileno(), sys.stderr.fileno())
+    except io.UnsupportedOperation:
+        sys.stdout = open(logfile, 'a+')
+        sys.stderr = sys.stdout
 
-    os._exit(0)
+    try:
+        function()
+    except Exception as e:
+        traceback.print_exc()
+    finally:
+        bb.event.print_ui_queue()
+        os._exit(0)

bitbake/lib/bb/data.py

@@ -290,7 +290,7 @@ def build_dependencies(key, keys, shelldeps, varflagsexcl, d):
             return deps, value
         varflags = d.getVarFlags(key, ["vardeps", "vardepvalue", "vardepsexclude", "exports", "postfuncs", "prefuncs", "lineno", "filename"]) or {}
         vardeps = varflags.get("vardeps")
-        value = d.getVar(key, False)
+        value = d.getVarFlag(key, "_content", False)
 
         def handle_contains(value, contains, d):
             newvalue = ""

bitbake/lib/bb/event.py

@@ -149,23 +149,30 @@ def print_ui_queue():
 
         # First check to see if we have any proper messages
         msgprint = False
+        msgerrs = False
+
+        # Should we print to stderr?
+        for event in ui_queue[:]:
+            if isinstance(event, logging.LogRecord) and event.levelno >= logging.WARNING:
+                msgerrs = True
+                break
+
+        if msgerrs:
+            logger.addHandler(stderr)
+        else:
+            logger.addHandler(stdout)
+
         for event in ui_queue[:]:
             if isinstance(event, logging.LogRecord):
                 if event.levelno > logging.DEBUG:
-                    if event.levelno >= logging.WARNING:
-                        logger.addHandler(stderr)
-                    else:
-                        logger.addHandler(stdout)
                     logger.handle(event)
                     msgprint = True
-        if msgprint:
-            return
 
         # Nope, so just print all of the messages we have (including debug messages)
-        logger.addHandler(stdout)
-        for event in ui_queue[:]:
-            if isinstance(event, logging.LogRecord):
-                logger.handle(event)
+        if not msgprint:
+            for event in ui_queue[:]:
+                if isinstance(event, logging.LogRecord):
+                    logger.handle(event)
 
 def fire_ui_handlers(event, d):
     global _thread_lock
@@ -212,6 +219,12 @@ def fire(event, d):
     if worker_fire:
         worker_fire(event, d)
     else:
+        # If messages have been queued up, clear the queue
+        global _uiready, ui_queue
+        if _uiready and ui_queue:
+            for queue_event in ui_queue:
+                fire_ui_handlers(queue_event, d)
+            ui_queue = []
         fire_ui_handlers(event, d)
 
 def fire_from_worker(event, d):
@@ -264,6 +277,11 @@ def register(name, handler, mask=None, filename=None, lineno=None):
 def remove(name, handler):
     """Remove an Event handler"""
     _handlers.pop(name)
+    if name in _catchall_handlers:
+        _catchall_handlers.pop(name)
+    for event in _event_handler_map.keys():
+        if name in _event_handler_map[event]:
+            _event_handler_map[event].pop(name)
 
 def get_handlers():
     return _handlers
@@ -277,20 +295,28 @@ def set_eventfilter(func):
     _eventfilter = func
 
 def register_UIHhandler(handler, mainui=False):
-    if mainui:
-        global _uiready
-        _uiready = True
     bb.event._ui_handler_seq = bb.event._ui_handler_seq + 1
     _ui_handlers[_ui_handler_seq] = handler
     level, debug_domains = bb.msg.constructLogOptions()
     _ui_logfilters[_ui_handler_seq] = UIEventFilter(level, debug_domains)
+    if mainui:
+        global _uiready
+        _uiready = _ui_handler_seq
     return _ui_handler_seq
 
-def unregister_UIHhandler(handlerNum):
+def unregister_UIHhandler(handlerNum, mainui=False):
+    if mainui:
+        global _uiready
+        _uiready = False
     if handlerNum in _ui_handlers:
         del _ui_handlers[handlerNum]
     return
 
+def get_uihandler():
+    if _uiready is False:
+        return None
+    return _uiready
+
 # Class to allow filtering of events and specific filtering of LogRecords *before* we put them over the IPC
 class UIEventFilter(object):
     def __init__(self, level, debug_domains):
@@ -353,6 +379,12 @@ class OperationProgress(Event):
 class ConfigParsed(Event):
     """Configuration Parsing Complete"""
 
+class MultiConfigParsed(Event):
+    """Multi-Config Parsing Complete"""
+    def __init__(self, mcdata):
+        self.mcdata = mcdata
+        Event.__init__(self)
+
 class RecipeEvent(Event):
     def __init__(self, fn):
         self.fn = fn
@@ -496,6 +528,28 @@ class NoProvider(Event):
     def isRuntime(self):
         return self._runtime
 
+    def __str__(self):
+        msg = ''
+        if self._runtime:
+            r = "R"
+        else:
+            r = ""
+
+        extra = ''
+        if not self._reasons:
+            if self._close_matches:
+                extra = ". Close matches:\n  %s" % '\n  '.join(self._close_matches)
+
+        if self._dependees:
+            msg = "Nothing %sPROVIDES '%s' (but %s %sDEPENDS on or otherwise requires it)%s" % (r, self._item, ", ".join(self._dependees), r, extra)
+        else:
+            msg = "Nothing %sPROVIDES '%s'%s" % (r, self._item, extra)
+        if self._reasons:
+            for reason in self._reasons:
+                msg += '\n' + reason
+        return msg
+
+
 class MultipleProviders(Event):
     """Multiple Providers"""
 
@@ -523,6 +577,16 @@ class MultipleProviders(Event):
         """
         return self._candidates
 
+    def __str__(self):
+        msg = "Multiple providers are available for %s%s (%s)" % (self._is_runtime and "runtime " or "",
+                            self._item,
+                            ", ".join(self._candidates))
+        rtime = ""
+        if self._is_runtime:
+            rtime = "R"
+        msg += "\nConsider defining a PREFERRED_%sPROVIDER entry to match %s" % (rtime, self._item)
+        return msg
+
 class ParseStarted(OperationStarted):
     """Recipe parsing for the runqueue has begun"""
     def __init__(self, total):
@@ -616,14 +680,6 @@ class FilesMatchingFound(Event):
         self._pattern = pattern
         self._matches = matches
 
-class CoreBaseFilesFound(Event):
-    """
-    Event when a list of appropriate config files has been generated
-    """
-    def __init__(self, paths):
-        Event.__init__(self)
-        self._paths = paths
-
 class ConfigFilesFound(Event):
     """
     Event when a list of appropriate config files has been generated
@@ -694,19 +750,6 @@ class LogHandler(logging.Handler):
         record.taskpid = worker_pid
         return True
 
-class RequestPackageInfo(Event):
-    """
-    Event to request package information
-    """
-
-class PackageInfo(Event):
-    """
-    Package information for GUI
-    """
-    def __init__(self, pkginfolist):
-        Event.__init__(self)
-        self._pkginfolist = pkginfolist
-
 class MetadataEvent(Event):
     """
     Generic event that target for OE-Core classes
@@ -784,3 +827,10 @@ class NetworkTestFailed(Event):
     Event to indicate network test has failed
     """
 
+class FindSigInfoResult(Event):
+    """
+    Event to return results from findSigInfo command
+    """
+    def __init__(self, result):
+        Event.__init__(self)
+        self.result = result

bitbake/lib/bb/fetch2/__init__.py

@@ -39,6 +39,7 @@ import errno
 import bb.persist_data, bb.utils
 import bb.checksum
 import bb.process
+import bb.event
 
 __version__ = "2"
 _checksum_cache = bb.checksum.FileChecksumCache()
@@ -48,11 +49,11 @@ logger = logging.getLogger("BitBake.Fetcher")
 class BBFetchException(Exception):
     """Class all fetch exceptions inherit from"""
     def __init__(self, message):
-         self.msg = message
-         Exception.__init__(self, message)
+        self.msg = message
+        Exception.__init__(self, message)
 
     def __str__(self):
-         return self.msg
+        return self.msg
 
 class UntrustedUrl(BBFetchException):
     """Exception raised when encountering a host not listed in BB_ALLOWED_NETWORKS"""
@@ -68,24 +69,24 @@ class UntrustedUrl(BBFetchException):
 class MalformedUrl(BBFetchException):
     """Exception raised when encountering an invalid url"""
     def __init__(self, url, message=''):
-         if message:
-             msg = message
-         else:
-             msg = "The URL: '%s' is invalid and cannot be interpreted" % url
-         self.url = url
-         BBFetchException.__init__(self, msg)
-         self.args = (url,)
+        if message:
+            msg = message
+        else:
+            msg = "The URL: '%s' is invalid and cannot be interpreted" % url
+        self.url = url
+        BBFetchException.__init__(self, msg)
+        self.args = (url,)
 
 class FetchError(BBFetchException):
     """General fetcher exception when something happens incorrectly"""
     def __init__(self, message, url = None):
-         if url:
+        if url:
             msg = "Fetcher failure for URL: '%s'. %s" % (url, message)
-         else:
+        else:
             msg = "Fetcher failure: %s" % message
-         self.url = url
-         BBFetchException.__init__(self, msg)
-         self.args = (message, url)
+        self.url = url
+        BBFetchException.__init__(self, msg)
+        self.args = (message, url)
 
 class ChecksumError(FetchError):
     """Exception when mismatched checksum encountered"""
@@ -99,49 +100,56 @@ class NoChecksumError(FetchError):
 class UnpackError(BBFetchException):
     """General fetcher exception when something happens incorrectly when unpacking"""
     def __init__(self, message, url):
-         msg = "Unpack failure for URL: '%s'. %s" % (url, message)
-         self.url = url
-         BBFetchException.__init__(self, msg)
-         self.args = (message, url)
+        msg = "Unpack failure for URL: '%s'. %s" % (url, message)
+        self.url = url
+        BBFetchException.__init__(self, msg)
+        self.args = (message, url)
 
 class NoMethodError(BBFetchException):
     """Exception raised when there is no method to obtain a supplied url or set of urls"""
     def __init__(self, url):
-         msg = "Could not find a fetcher which supports the URL: '%s'" % url
-         self.url = url
-         BBFetchException.__init__(self, msg)
-         self.args = (url,)
+        msg = "Could not find a fetcher which supports the URL: '%s'" % url
+        self.url = url
+        BBFetchException.__init__(self, msg)
+        self.args = (url,)
 
 class MissingParameterError(BBFetchException):
     """Exception raised when a fetch method is missing a critical parameter in the url"""
     def __init__(self, missing, url):
-         msg = "URL: '%s' is missing the required parameter '%s'" % (url, missing)
-         self.url = url
-         self.missing = missing
-         BBFetchException.__init__(self, msg)
-         self.args = (missing, url)
+        msg = "URL: '%s' is missing the required parameter '%s'" % (url, missing)
+        self.url = url
+        self.missing = missing
+        BBFetchException.__init__(self, msg)
+        self.args = (missing, url)
 
 class ParameterError(BBFetchException):
     """Exception raised when a url cannot be proccessed due to invalid parameters."""
     def __init__(self, message, url):
-         msg = "URL: '%s' has invalid parameters. %s" % (url, message)
-         self.url = url
-         BBFetchException.__init__(self, msg)
-         self.args = (message, url)
+        msg = "URL: '%s' has invalid parameters. %s" % (url, message)
+        self.url = url
+        BBFetchException.__init__(self, msg)
+        self.args = (message, url)
 
 class NetworkAccess(BBFetchException):
     """Exception raised when network access is disabled but it is required."""
     def __init__(self, url, cmd):
-         msg = "Network access disabled through BB_NO_NETWORK (or set indirectly due to use of BB_FETCH_PREMIRRORONLY) but access requested with command %s (for url %s)" % (cmd, url)
-         self.url = url
-         self.cmd = cmd
-         BBFetchException.__init__(self, msg)
-         self.args = (url, cmd)
+        msg = "Network access disabled through BB_NO_NETWORK (or set indirectly due to use of BB_FETCH_PREMIRRORONLY) but access requested with command %s (for url %s)" % (cmd, url)
+        self.url = url
+        self.cmd = cmd
+        BBFetchException.__init__(self, msg)
+        self.args = (url, cmd)
 
 class NonLocalMethod(Exception):
     def __init__(self):
         Exception.__init__(self)
 
+class MissingChecksumEvent(bb.event.Event):
+    def __init__(self, url, md5sum, sha256sum):
+        self.url = url
+        self.checksums = {'md5sum': md5sum,
+                          'sha256sum': sha256sum}
+        bb.event.Event.__init__(self)
+
 
 class URI(object):
     """
@@ -455,7 +463,7 @@ def uri_replace(ud, uri_find, uri_replace, replacements, d, mirrortarball=None):
                 result_decoded[loc][k] = uri_replace_decoded[loc][k]
         elif (re.match(regexp, uri_decoded[loc])):
             if not uri_replace_decoded[loc]:
-                result_decoded[loc] = ""    
+                result_decoded[loc] = ""
             else:
                 for k in replacements:
                     uri_replace_decoded[loc] = uri_replace_decoded[loc].replace(k, replacements[k])
@@ -584,6 +592,14 @@ def verify_checksum(ud, d, precomputed={}):
                               ud.sha256_name, sha256data))
             raise NoChecksumError('Missing SRC_URI checksum', ud.url)
 
+        bb.event.fire(MissingChecksumEvent(ud.url, md5data, sha256data), d)
+
+        if strict == "ignore":
+            return {
+                _MD5_KEY: md5data,
+                _SHA256_KEY: sha256data
+            }
+
         # Log missing sums so user can more easily add them
         logger.warning('Missing md5 SRC_URI checksum for %s, consider adding to the recipe:\n'
                        'SRC_URI[%s] = "%s"',
@@ -733,7 +749,7 @@ def get_srcrev(d, method_name='sortable_revision'):
     In the multi SCM case, we build a value based on SRCREV_FORMAT which must
     have been set.
 
-    The idea here is that we put the string "AUTOINC+" into return value if the revisions are not 
+    The idea here is that we put the string "AUTOINC+" into return value if the revisions are not
     incremental, other code is then responsible for turning that into an increasing value (if needed)
 
     A method_name can be supplied to retrieve an alternatively formatted revision from a fetcher, if
@@ -785,7 +801,7 @@ def get_srcrev(d, method_name='sortable_revision'):
     format = re.sub(name_to_rev_re, lambda match: name_to_rev[match.group(0)], format)
 
     if seenautoinc:
-       format = "AUTOINC+" + format
+        format = "AUTOINC+" + format
 
     return format
 
@@ -921,7 +937,7 @@ def build_mirroruris(origud, mirrors, ld):
                     logger.debug(1, "Mirror fetch failure for url %s (original url: %s)" % (newuri, origud.url))
                     logger.debug(1, str(e))
                     try:
-                        # setup_localpath of file:// urls may fail, we should still see 
+                        # setup_localpath of file:// urls may fail, we should still see
                         # if mirrors of the url exist
                         adduri(newud, uris, uds, localmirrors, tarballs)
                     except UnboundLocalError:
@@ -1017,7 +1033,7 @@ def try_mirror_url(fetch, origud, ud, ld, check = False):
 
     except IOError as e:
         if e.errno in [os.errno.ESTALE]:
-            logger.warn("Stale Error Observed %s." % ud.url)
+            logger.warning("Stale Error Observed %s." % ud.url)
             return False
         raise
 
@@ -1117,7 +1133,7 @@ def srcrev_internal_helper(ud, d, name):
     attempts.append("SRCREV")
 
     for a in attempts:
-        srcrev = d.getVar(a)              
+        srcrev = d.getVar(a)
         if srcrev and srcrev != "INVALID":
             break
 
@@ -1230,7 +1246,7 @@ class FetchData(object):
         for m in methods:
             if m.supports(self, d):
                 self.method = m
-                break                
+                break
 
         if not self.method:
             raise NoMethodError(url)
@@ -1265,7 +1281,7 @@ class FetchData(object):
         elif self.basepath or self.basename:
             basepath = dldir + os.sep + (self.basepath or self.basename)
         else:
-             bb.fatal("Can't determine lock path for url %s" % url)
+            bb.fatal("Can't determine lock path for url %s" % url)
 
         self.donestamp = basepath + '.done'
         self.lockfile = basepath + '.lock'
@@ -1328,13 +1344,13 @@ class FetchMethod(object):
         if os.path.isdir(urldata.localpath) == True:
             return False
         if urldata.localpath.find("*") != -1:
-             return False
+            return False
 
         return True
 
     def recommends_checksum(self, urldata):
         """
-        Is the backend on where checksumming is recommended (should warnings 
+        Is the backend on where checksumming is recommended (should warnings
         be displayed if there is no checksum)?
         """
         return False
@@ -1614,13 +1630,22 @@ class Fetch(object):
 
             try:
                 self.d.setVar("BB_NO_NETWORK", network)
- 
+
                 if verify_donestamp(ud, self.d) and not m.need_update(ud, self.d):
                     localpath = ud.localpath
                 elif m.try_premirror(ud, self.d):
                     logger.debug(1, "Trying PREMIRRORS")
                     mirrors = mirror_from_string(self.d.getVar('PREMIRRORS'))
                     localpath = try_mirrors(self, self.d, ud, mirrors, False)
+                    if localpath:
+                        try:
+                            # early checksum verification so that if the checksum of the premirror
+                            # contents mismatch the fetcher can still try upstream and mirrors
+                            update_stamp(ud, self.d)
+                        except ChecksumError as e:
+                            logger.warning("Checksum failure encountered with premirror download of %s - will attempt other sources." % u)
+                            logger.debug(1, str(e))
+                            localpath = ""
 
                 if premirroronly:
                     self.d.setVar("BB_NO_NETWORK", "1")
@@ -1701,9 +1726,8 @@ class Fetch(object):
             ret = try_mirrors(self, self.d, ud, mirrors, True)
             if not ret:
                 # Next try checking from the original uri, u
-                try:
-                    ret = m.checkstatus(self, ud, self.d)
-                except:
+                ret = m.checkstatus(self, ud, self.d)
+                if not ret:
                     # Finally, try checking uri, u, from MIRRORS
                     mirrors = mirror_from_string(self.d.getVar('MIRRORS'))
                     ret = try_mirrors(self, self.d, ud, mirrors, True)
@@ -1713,7 +1737,7 @@ class Fetch(object):
 
     def unpack(self, root, urls=None):
         """
-        Check all urls exist upstream
+        Unpack urls to root
         """
 
         if not urls:

bitbake/lib/bb/fetch2/npm.py

@@ -92,7 +92,7 @@ class Npm(FetchMethod):
 
         ud.write_tarballs = ((d.getVar("BB_GENERATE_MIRROR_TARBALLS") or "0") != "0")
         mirrortarball = 'npm_%s-%s.tar.xz' % (ud.pkgname, ud.version)
-        mirrortarball = ud.mirrortarball.replace('/', '-')
+        mirrortarball = mirrortarball.replace('/', '-')
         ud.fullmirror = os.path.join(d.getVar("DL_DIR"), mirrortarball)
         ud.mirrortarballs = [mirrortarball]
 

bitbake/lib/bb/fetch2/wget.py

@@ -30,6 +30,7 @@ import tempfile
 import subprocess
 import os
 import logging
+import errno
 import bb
 import bb.progress
 import urllib.request, urllib.parse, urllib.error
@@ -89,13 +90,13 @@ class Wget(FetchMethod):
 
         self.basecmd = d.getVar("FETCHCMD_wget") or "/usr/bin/env wget -t 2 -T 30 --passive-ftp --no-check-certificate"
 
-    def _runwget(self, ud, d, command, quiet):
+    def _runwget(self, ud, d, command, quiet, workdir=None):
 
         progresshandler = WgetProgressHandler(d)
 
         logger.debug(2, "Fetching %s using command '%s'" % (ud.url, command))
         bb.fetch2.check_network_access(d, command, ud.url)
-        runfetchcmd(command + ' --progress=dot -v', d, quiet, log=progresshandler)
+        runfetchcmd(command + ' --progress=dot -v', d, quiet, log=progresshandler, workdir=workdir)
 
     def download(self, ud, d):
         """Fetch urls"""
@@ -206,8 +207,21 @@ class Wget(FetchMethod):
                     h.request(req.get_method(), req.selector, req.data, headers)
                 except socket.error as err: # XXX what error?
                     # Don't close connection when cache is enabled.
+                    # Instead, try to detect connections that are no longer
+                    # usable (for example, closed unexpectedly) and remove
+                    # them from the cache.
                     if fetch.connection_cache is None:
                         h.close()
+                    elif isinstance(err, OSError) and err.errno == errno.EBADF:
+                        # This happens when the server closes the connection despite the Keep-Alive.
+                        # Apparently urllib then uses the file descriptor, expecting it to be
+                        # connected, when in reality the connection is already gone.
+                        # We let the request fail and expect it to be
+                        # tried once more ("try_again" in check_status()),
+                        # with the dead connection removed from the cache.
+                        # If it still fails, we give up, which can happend for bad
+                        # HTTP proxy settings.
+                        fetch.connection_cache.remove_connection(h.host, h.port)
                     raise urllib.error.URLError(err)
                 else:
                     try:
@@ -269,11 +283,6 @@ class Wget(FetchMethod):
             """
             http_error_403 = http_error_405
 
-            """
-            Some servers (e.g. FusionForge) returns 406 Not Acceptable when they
-            actually mean 405 Method Not Allowed.
-            """
-            http_error_406 = http_error_405
 
         class FixedHTTPRedirectHandler(urllib.request.HTTPRedirectHandler):
             """
@@ -302,7 +311,9 @@ class Wget(FetchMethod):
             uri = ud.url.split(";")[0]
             r = urllib.request.Request(uri)
             r.get_method = lambda: "HEAD"
-
+            # Some servers (FusionForge, as used on Alioth) require that the
+            # optional Accept header is set.
+            r.add_header("Accept", "*/*")
             def add_basic_auth(login_str, request):
                 '''Adds Basic auth to http request, pass in login:password as string'''
                 import base64
@@ -408,17 +419,16 @@ class Wget(FetchMethod):
         Run fetch checkstatus to get directory information
         """
         f = tempfile.NamedTemporaryFile()
+        with tempfile.TemporaryDirectory(prefix="wget-index-") as workdir, tempfile.NamedTemporaryFile(dir=workdir, prefix="wget-listing-") as f:
+            agent = "Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.9.2.12) Gecko/20101027 Ubuntu/9.10 (karmic) Firefox/3.6.12"
+            fetchcmd = self.basecmd
+            fetchcmd += " -O " + f.name + " --user-agent='" + agent + "' '" + uri + "'"
+            try:
+                self._runwget(ud, d, fetchcmd, True, workdir=workdir)
+                fetchresult = f.read()
+            except bb.fetch2.BBFetchException:
+                fetchresult = ""
 
-        agent = "Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.9.2.12) Gecko/20101027 Ubuntu/9.10 (karmic) Firefox/3.6.12"
-        fetchcmd = self.basecmd
-        fetchcmd += " -O " + f.name + " --user-agent='" + agent + "' '" + uri + "'"
-        try:
-            self._runwget(ud, d, fetchcmd, True)
-            fetchresult = f.read()
-        except bb.fetch2.BBFetchException:
-            fetchresult = ""
-
-        f.close()
         return fetchresult
 
     def _check_latest_version(self, url, package, package_regex, current_version, ud, d):

bitbake/lib/bb/main.py

@@ -28,6 +28,8 @@ import logging
 import optparse
 import warnings
 import fcntl
+import time
+import traceback
 
 import bb
 from bb import event
@@ -37,6 +39,9 @@ from bb import ui
 from bb import server
 from bb import cookerdata
 
+import bb.server.process
+import bb.server.xmlrpcclient
+
 logger = logging.getLogger("BitBake")
 
 class BBMainException(Exception):
@@ -58,9 +63,6 @@ class BitbakeHelpFormatter(optparse.IndentedHelpFormatter):
         if option.dest == 'ui':
             valid_uis = list_extension_modules(bb.ui, 'main')
             option.help = option.help.replace('@CHOICES@', present_options(valid_uis))
-        elif option.dest == 'servertype':
-            valid_server_types = list_extension_modules(bb.server, 'BitBakeServer')
-            option.help = option.help.replace('@CHOICES@', present_options(valid_server_types))
 
         return optparse.IndentedHelpFormatter.format_option(self, option)
 
@@ -238,11 +240,6 @@ class BitBakeConfigParameters(cookerdata.ConfigParameters):
                           default=os.environ.get('BITBAKE_UI', 'knotty'),
                           help="The user interface to use (@CHOICES@ - default %default).")
 
-        # @CHOICES@ is substituted out by BitbakeHelpFormatter above
-        parser.add_option("-t", "--servertype", action="store", dest="servertype",
-                          default=["process", "xmlrpc"]["BBSERVER" in os.environ],
-                          help="Choose which server type to use (@CHOICES@ - default %default).")
-
         parser.add_option("", "--token", action="store", dest="xmlrpctoken",
                           default=os.environ.get("BBTOKEN"),
                           help="Specify the connection token to be used when connecting "
@@ -258,14 +255,11 @@ class BitBakeConfigParameters(cookerdata.ConfigParameters):
                           help="Run bitbake without a UI, only starting a server "
                                "(cooker) process.")
 
-        parser.add_option("", "--foreground", action="store_true",
-                          help="Run bitbake server in foreground.")
-
         parser.add_option("-B", "--bind", action="store", dest="bind", default=False,
-                          help="The name/address for the bitbake server to bind to.")
+                          help="The name/address for the bitbake xmlrpc server to bind to.")
 
-        parser.add_option("-T", "--idle-timeout", type=int,
-                          default=int(os.environ.get("BBTIMEOUT", "0")),
+        parser.add_option("-T", "--idle-timeout", type=float, dest="server_timeout",
+                          default=float(os.environ.get("BB_SERVER_TIMEOUT", 0)) or None,
                           help="Set timeout to unload bitbake server due to inactivity")
 
         parser.add_option("", "--no-setscene", action="store_true",
@@ -283,7 +277,7 @@ class BitBakeConfigParameters(cookerdata.ConfigParameters):
 
         parser.add_option("-m", "--kill-server", action="store_true",
                           dest="kill_server", default=False,
-                          help="Terminate the remote server.")
+                          help="Terminate any running bitbake server.")
 
         parser.add_option("", "--observe-only", action="store_true",
                           dest="observe_only", default=False,
@@ -322,70 +316,20 @@ class BitBakeConfigParameters(cookerdata.ConfigParameters):
             eventlog = "bitbake_eventlog_%s.json" % datetime.now().strftime("%Y%m%d%H%M%S")
             options.writeeventlog = eventlog
 
-        # if BBSERVER says to autodetect, let's do that
-        if options.remote_server:
-            port = -1
-            if options.remote_server != 'autostart':
-                host, port = options.remote_server.split(":", 2)
+        if options.bind:
+            try:
+                #Checking that the port is a number and is a ':' delimited value
+                (host, port) = options.bind.split(':')
                 port = int(port)
-            # use automatic port if port set to -1, means read it from
-            # the bitbake.lock file; this is a bit tricky, but we always expect
-            # to be in the base of the build directory if we need to have a
-            # chance to start the server later, anyway
-            if port == -1:
-                lock_location = "./bitbake.lock"
-                # we try to read the address at all times; if the server is not started,
-                # we'll try to start it after the first connect fails, below
-                try:
-                    lf = open(lock_location, 'r')
-                    remotedef = lf.readline()
-                    [host, port] = remotedef.split(":")
-                    port = int(port)
-                    lf.close()
-                    options.remote_server = remotedef
-                except Exception as e:
-                    if options.remote_server != 'autostart':
-                        raise BBMainException("Failed to read bitbake.lock (%s), invalid port" % str(e))
+            except (ValueError,IndexError):
+                raise BBMainException("FATAL: Malformed host:port bind parameter")
+            options.xmlrpcinterface = (host, port)
+        else:
+            options.xmlrpcinterface = (None, 0)
 
         return options, targets[1:]
 
 
-def start_server(servermodule, configParams, configuration, features):
-    server = servermodule.BitBakeServer()
-    single_use = not configParams.server_only and os.getenv('BBSERVER') != 'autostart'
-    if configParams.bind:
-        (host, port) = configParams.bind.split(':')
-        server.initServer((host, int(port)), single_use=single_use,
-                          idle_timeout=configParams.idle_timeout)
-        configuration.interface = [server.serverImpl.host, server.serverImpl.port]
-    else:
-        server.initServer(single_use=single_use)
-        configuration.interface = []
-
-    try:
-        configuration.setServerRegIdleCallback(server.getServerIdleCB())
-
-        cooker = bb.cooker.BBCooker(configuration, features)
-
-        server.addcooker(cooker)
-        server.saveConnectionDetails()
-    except Exception as e:
-        while hasattr(server, "event_queue"):
-            import queue
-            try:
-                event = server.event_queue.get(block=False)
-            except (queue.Empty, IOError):
-                break
-            if isinstance(event, logging.LogRecord):
-                logger.handle(event)
-        raise
-    if not configParams.foreground:
-        server.detach()
-        cooker.shutdown()
-    cooker.lock.close()
-    return server
-
-
 def bitbake_main(configParams, configuration):
 
     # Python multiprocessing requires /dev/shm on Linux
@@ -406,45 +350,15 @@ def bitbake_main(configParams, configuration):
 
     configuration.setConfigParameters(configParams)
 
-    if configParams.server_only:
-        if configParams.servertype != "xmlrpc":
-            raise BBMainException("FATAL: If '--server-only' is defined, we must set the "
-                                  "servertype as 'xmlrpc'.\n")
-        if not configParams.bind:
-            raise BBMainException("FATAL: The '--server-only' option requires a name/address "
-                                  "to bind to with the -B option.\n")
-        else:
-            try:
-                #Checking that the port is a number
-                int(configParams.bind.split(":")[1])
-            except (ValueError,IndexError):
-                raise BBMainException(
-                        "FATAL: Malformed host:port bind parameter")
-        if configParams.remote_server:
+    if configParams.server_only and configParams.remote_server:
             raise BBMainException("FATAL: The '--server-only' option conflicts with %s.\n" %
                                   ("the BBSERVER environment variable" if "BBSERVER" in os.environ \
                                    else "the '--remote-server' option"))
 
-    elif configParams.foreground:
-        raise BBMainException("FATAL: The '--foreground' option can only be used "
-                              "with --server-only.\n")
-
-    if configParams.bind and configParams.servertype != "xmlrpc":
-        raise BBMainException("FATAL: If '-B' or '--bind' is defined, we must "
-                              "set the servertype as 'xmlrpc'.\n")
-
-    if configParams.remote_server and configParams.servertype != "xmlrpc":
-        raise BBMainException("FATAL: If '--remote-server' is defined, we must "
-                              "set the servertype as 'xmlrpc'.\n")
-
-    if configParams.observe_only and (not configParams.remote_server or configParams.bind):
+    if configParams.observe_only and not (configParams.remote_server or configParams.bind):
         raise BBMainException("FATAL: '--observe-only' can only be used by UI clients "
                               "connecting to a server.\n")
 
-    if configParams.kill_server and not configParams.remote_server:
-        raise BBMainException("FATAL: '--kill-server' can only be used to "
-                              "terminate a remote server")
-
     if "BBDEBUG" in os.environ:
         level = int(os.environ["BBDEBUG"])
         if level > configuration.debug:
@@ -453,9 +367,13 @@ def bitbake_main(configParams, configuration):
     bb.msg.init_msgconfig(configParams.verbose, configuration.debug,
                           configuration.debug_domains)
 
-    server, server_connection, ui_module = setup_bitbake(configParams, configuration)
-    if server_connection is None and configParams.kill_server:
-        return 0
+    server_connection, ui_module = setup_bitbake(configParams, configuration)
+    # No server connection
+    if server_connection is None:
+        if configParams.status_only:
+            return 1
+        if configParams.kill_server:
+            return 0
 
     if not configParams.server_only:
         if configParams.status_only:
@@ -463,16 +381,15 @@ def bitbake_main(configParams, configuration):
             return 0
 
         try:
+            for event in bb.event.ui_queue:
+                server_connection.events.queue_event(event)
+            bb.event.ui_queue = []
+
             return ui_module.main(server_connection.connection, server_connection.events,
                                   configParams)
         finally:
-            bb.event.ui_queue = []
             server_connection.terminate()
     else:
-        print("Bitbake server address: %s, server port: %s" % (server.serverImpl.host,
-                                                               server.serverImpl.port))
-        if configParams.foreground:
-            server.serverImpl.serve_forever()
         return 0
 
     return 1
@@ -495,58 +412,73 @@ def setup_bitbake(configParams, configuration, extrafeatures=None, setup_logging
         # Collect the feature set for the UI
         featureset = getattr(ui_module, "featureSet", [])
 
-    if configParams.server_only:
-        for param in ('prefile', 'postfile'):
-            value = getattr(configParams, param)
-            if value:
-                setattr(configuration, "%s_server" % param, value)
-                param = "%s_server" % param
-
     if extrafeatures:
         for feature in extrafeatures:
             if not feature in featureset:
                 featureset.append(feature)
 
-    servermodule = import_extension_module(bb.server,
-                                            configParams.servertype,
-                                            'BitBakeServer')
+    server_connection = None
+
     if configParams.remote_server:
-        if os.getenv('BBSERVER') == 'autostart':
-            if configParams.remote_server == 'autostart' or \
-               not servermodule.check_connection(configParams.remote_server, timeout=2):
-                configParams.bind = 'localhost:0'
-                srv = start_server(servermodule, configParams, configuration, featureset)
-                configParams.remote_server = '%s:%d' % tuple(configuration.interface)
-                bb.event.ui_queue = []
-        # we start a stub server that is actually a XMLRPClient that connects to a real server
-        from bb.server.xmlrpc import BitBakeXMLRPCClient
-        server = servermodule.BitBakeXMLRPCClient(configParams.observe_only,
-                                                  configParams.xmlrpctoken)
-        server.saveConnectionDetails(configParams.remote_server)
+        # Connect to a remote XMLRPC server
+        server_connection = bb.server.xmlrpcclient.connectXMLRPC(configParams.remote_server, featureset,
+                                                                 configParams.observe_only, configParams.xmlrpctoken)
     else:
-        # we start a server with a given configuration
-        server = start_server(servermodule, configParams, configuration, featureset)
+        retries = 8
+        while retries:
+            try:
+                topdir, lock = lockBitbake()
+                sockname = topdir + "/bitbake.sock"
+                if lock:
+                    if configParams.status_only or configParams.kill_server:
+                        logger.info("bitbake server is not running.")
+                        lock.close()
+                        return None, None
+                    # we start a server with a given configuration
+                    logger.info("Starting bitbake server...")
+                    server = bb.server.process.BitBakeServer(lock, sockname, configuration, featureset)
+                    # The server will handle any events already in the queue
+                    bb.event.ui_queue = []
+                else:
+                    logger.info("Reconnecting to bitbake server...")
+                    if not os.path.exists(sockname):
+                        print("Previous bitbake instance shutting down?, waiting to retry...")
+                        time.sleep(5)
+                        raise bb.server.process.ProcessTimeout("Bitbake still shutting down as socket exists but no lock?")
+                if not configParams.server_only:
+                    server_connection = bb.server.process.connectProcessServer(sockname, featureset)
+                if server_connection or configParams.server_only:
+                    break
+            except (Exception, bb.server.process.ProcessTimeout) as e:
+                if not retries:
+                    raise
+                retries -= 1
+                if isinstance(e, (bb.server.process.ProcessTimeout, BrokenPipeError)):
+                    logger.info("Retrying server connection...")
+                else:
+                    logger.info("Retrying server connection... (%s)" % traceback.format_exc())
+            if not retries:
+                bb.fatal("Unable to connect to bitbake server, or start one")
+            if retries < 5:
+                time.sleep(5)
+
+    if configParams.kill_server:
+        server_connection.connection.terminateServer()
+        server_connection.terminate()
         bb.event.ui_queue = []
+        logger.info("Terminated bitbake server.")
+        return None, None
 
-    if configParams.server_only:
-        server_connection = None
-    else:
-        try:
-            server_connection = server.establishConnection(featureset)
-        except Exception as e:
-            bb.fatal("Could not connect to server %s: %s" % (configParams.remote_server, str(e)))
+    # Restore the environment in case the UI needs it
+    for k in cleanedvars:
+        os.environ[k] = cleanedvars[k]
 
-        if configParams.kill_server:
-            server_connection.connection.terminateServer()
-            bb.event.ui_queue = []
-            return None, None, None
+    logger.removeHandler(handler)
 
-        server_connection.setupEventQueue()
+    return server_connection, ui_module
 
-        # Restore the environment in case the UI needs it
-        for k in cleanedvars:
-            os.environ[k] = cleanedvars[k]
+def lockBitbake():
+    topdir = bb.cookerdata.findTopdir()
+    lockfile = topdir + "/bitbake.lock"
+    return topdir, bb.utils.lockfile(lockfile, False, False)
 
-        logger.removeHandler(handler)
-
-    return server, server_connection, ui_module

bitbake/lib/bb/parse/parse_py/BBHandler.py

@@ -144,7 +144,7 @@ def handle(fn, d, include):
     try:
         statements.eval(d)
     except bb.parse.SkipRecipe:
-        bb.data.setVar("__SKIPPED", True, d)
+        d.setVar("__SKIPPED", True)
         if include == 0:
             return { "" : d }
 

bitbake/lib/bb/runqueue.py

@@ -2372,7 +2372,7 @@ class RunQueueExecuteScenequeue(RunQueueExecute):
         self.rq.scenequeue_covered = self.scenequeue_covered
         self.rq.scenequeue_notcovered = self.scenequeue_notcovered
 
-        logger.debug(1, 'We can skip tasks %s', sorted(self.rq.scenequeue_covered))
+        logger.debug(1, 'We can skip tasks %s', "\n".join(sorted(self.rq.scenequeue_covered)))
 
         self.rq.state = runQueueRunInit
 
@@ -2488,6 +2488,9 @@ class runQueueTaskFailed(runQueueEvent):
         runQueueEvent.__init__(self, task, stats, rq)
         self.exitcode = exitcode
 
+    def __str__(self):
+        return "Task (%s) failed with exit code '%s'" % (self.taskstring, self.exitcode)
+
 class sceneQueueTaskFailed(sceneQueueEvent):
     """
     Event notifying a setscene task failed
@@ -2496,6 +2499,9 @@ class sceneQueueTaskFailed(sceneQueueEvent):
         sceneQueueEvent.__init__(self, task, stats, rq)
         self.exitcode = exitcode
 
+    def __str__(self):
+        return "Setscene task (%s) failed with exit code '%s' - real task will be run instead" % (self.taskstring, self.exitcode)
+
 class sceneQueueComplete(sceneQueueEvent):
     """
     Event when all the sceneQueue tasks are complete

bitbake/lib/bb/server/__init__.py

@@ -18,82 +18,4 @@
 # with this program; if not, write to the Free Software Foundation, Inc.,
 # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
 
-""" Base code for Bitbake server process
 
-Have a common base for that all Bitbake server classes ensures a consistent
-approach to the interface, and minimize risks associated with code duplication.
-
-"""
-
-"""  BaseImplServer() the base class for all XXServer() implementations.
-
-    These classes contain the actual code that runs the server side, i.e.
-    listens for the commands and executes them. Although these implementations
-    contain all the data of the original bitbake command, i.e the cooker instance,
-    they may well run on a different process or even machine.
-
-"""
-
-class BaseImplServer():
-    def __init__(self):
-        self._idlefuns = {}
-
-    def addcooker(self, cooker):
-        self.cooker = cooker
-
-    def register_idle_function(self, function, data):
-        """Register a function to be called while the server is idle"""
-        assert hasattr(function, '__call__')
-        self._idlefuns[function] = data
-
-
-
-""" BitBakeBaseServerConnection class is the common ancestor to all
-    BitBakeServerConnection classes.
-
-    These classes control the remote server. The only command currently
-    implemented is the terminate() command.
-
-"""
-
-class BitBakeBaseServerConnection():
-    def __init__(self, serverImpl):
-        pass
-
-    def terminate(self):
-        pass
-
-    def setupEventQueue(self):
-        pass
-
-
-""" BitBakeBaseServer class is the common ancestor to all Bitbake servers
-
-    Derive this class in order to implement a BitBakeServer which is the
-    controlling stub for the actual server implementation
-
-"""
-class BitBakeBaseServer(object):
-    def initServer(self):
-        self.serverImpl = None  # we ensure a runtime crash if not overloaded
-        self.connection = None
-        return
-
-    def addcooker(self, cooker):
-        self.cooker = cooker
-        self.serverImpl.addcooker(cooker)
-
-    def getServerIdleCB(self):
-        return self.serverImpl.register_idle_function
-
-    def saveConnectionDetails(self):
-        return
-
-    def detach(self):
-        return
-
-    def establishConnection(self, featureset):
-        raise   "Must redefine the %s.establishConnection()" % self.__class__.__name__
-
-    def endSession(self):
-        self.connection.terminate()

bitbake/lib/bb/server/process.py

@@ -22,125 +22,232 @@
 
 import bb
 import bb.event
-import itertools
 import logging
 import multiprocessing
+import threading
+import array
 import os
-import signal
 import sys
 import time
 import select
-from queue import Empty
-from multiprocessing import Event, Process, util, Queue, Pipe, queues, Manager
-
-from . import BitBakeBaseServer, BitBakeBaseServerConnection, BaseImplServer
+import socket
+import subprocess
+import errno
+import bb.server.xmlrpcserver
+from bb import daemonize
+from multiprocessing import queues
 
 logger = logging.getLogger('BitBake')
 
-class ServerCommunicator():
-    def __init__(self, connection, event_handle, server):
-        self.connection = connection
-        self.event_handle = event_handle
-        self.server = server
+class ProcessTimeout(SystemExit):
+    pass
 
-    def runCommand(self, command):
-        # @todo try/except
-        self.connection.send(command)
-
-        if not self.server.is_alive():
-            raise SystemExit
-
-        while True:
-            # don't let the user ctrl-c while we're waiting for a response
-            try:
-                for idx in range(0,4): # 0, 1, 2, 3
-                    if self.connection.poll(5):
-                        return self.connection.recv()
-                    else:
-                        bb.warn("Timeout while attempting to communicate with bitbake server")
-                bb.fatal("Gave up; Too many tries: timeout while attempting to communicate with bitbake server")
-            except KeyboardInterrupt:
-                pass
-
-    def getEventHandle(self):
-        return self.event_handle.value
-
-class EventAdapter():
-    """
-    Adapter to wrap our event queue since the caller (bb.event) expects to
-    call a send() method, but our actual queue only has put()
-    """
-    def __init__(self, queue):
-        self.queue = queue
-
-    def send(self, event):
-        try:
-            self.queue.put(event)
-        except Exception as err:
-            print("EventAdapter puked: %s" % str(err))
-
-
-class ProcessServer(Process, BaseImplServer):
+class ProcessServer(multiprocessing.Process):
     profile_filename = "profile.log"
     profile_processed_filename = "profile.log.processed"
 
-    def __init__(self, command_channel, event_queue, featurelist):
-        BaseImplServer.__init__(self)
-        Process.__init__(self)
-        self.command_channel = command_channel
-        self.event_queue = event_queue
-        self.event = EventAdapter(event_queue)
-        self.featurelist = featurelist
+    def __init__(self, lock, sock, sockname):
+        multiprocessing.Process.__init__(self)
+        self.command_channel = False
+        self.command_channel_reply = False
         self.quit = False
         self.heartbeat_seconds = 1 # default, BB_HEARTBEAT_EVENT will be checked once we have a datastore.
         self.next_heartbeat = time.time()
 
-        self.quitin, self.quitout = Pipe()
-        self.event_handle = multiprocessing.Value("i")
+        self.event_handle = None
+        self.haveui = False
+        self.lastui = False
+        self.xmlrpc = False
+
+        self._idlefuns = {}
+
+        self.bitbake_lock = lock
+        self.sock = sock
+        self.sockname = sockname
+
+    def register_idle_function(self, function, data):
+        """Register a function to be called while the server is idle"""
+        assert hasattr(function, '__call__')
+        self._idlefuns[function] = data
 
     def run(self):
-        for event in bb.event.ui_queue:
-            self.event_queue.put(event)
-        self.event_handle.value = bb.event.register_UIHhandler(self, True)
+
+        if self.xmlrpcinterface[0]:
+            self.xmlrpc = bb.server.xmlrpcserver.BitBakeXMLRPCServer(self.xmlrpcinterface, self.cooker, self)
+
+            print("Bitbake XMLRPC server address: %s, server port: %s" % (self.xmlrpc.host, self.xmlrpc.port))
 
         heartbeat_event = self.cooker.data.getVar('BB_HEARTBEAT_EVENT')
         if heartbeat_event:
             try:
                 self.heartbeat_seconds = float(heartbeat_event)
             except:
-                # Throwing an exception here causes bitbake to hang.
-                # Just warn about the invalid setting and continue
                 bb.warn('Ignoring invalid BB_HEARTBEAT_EVENT=%s, must be a float specifying seconds.' % heartbeat_event)
-        bb.cooker.server_main(self.cooker, self.main)
+
+        self.timeout = self.server_timeout or self.cooker.data.getVar('BB_SERVER_TIMEOUT')
+        try:
+            if self.timeout:
+                self.timeout = float(self.timeout)
+        except:
+            bb.warn('Ignoring invalid BB_SERVER_TIMEOUT=%s, must be a float specifying seconds.' % self.timeout)
+
+
+        try:
+            self.bitbake_lock.seek(0)
+            self.bitbake_lock.truncate()
+            if self.xmlrpc:
+                self.bitbake_lock.write("%s %s:%s\n" % (os.getpid(), self.xmlrpc.host, self.xmlrpc.port))
+            else:
+                self.bitbake_lock.write("%s\n" % (os.getpid()))
+            self.bitbake_lock.flush()
+        except Exception as e:
+            print("Error writing to lock file: %s" % str(e))
+            pass
+
+        if self.cooker.configuration.profile:
+            try:
+                import cProfile as profile
+            except:
+                import profile
+            prof = profile.Profile()
+
+            ret = profile.Profile.runcall(prof, self.main)
+
+            prof.dump_stats("profile.log")
+            bb.utils.process_profilelog("profile.log")
+            print("Raw profiling information saved to profile.log and processed statistics to profile.log.processed")
+
+        else:
+            ret = self.main()
+
+        return ret
 
     def main(self):
-        # Ignore SIGINT within the server, as all SIGINT handling is done by
-        # the UI and communicated to us
-        self.quitin.close()
-        signal.signal(signal.SIGINT, signal.SIG_IGN)
+        self.cooker.pre_serve()
+
         bb.utils.set_process_name("Cooker")
+
+        ready = []
+
+        self.controllersock = False
+        fds = [self.sock]
+        if self.xmlrpc:
+            fds.append(self.xmlrpc)
+        print("Entering server connection loop")
         while not self.quit:
-            try:
-                if self.command_channel.poll():
-                    command = self.command_channel.recv()
-                    self.runCommand(command)
-                if self.quitout.poll():
-                    self.quitout.recv()
+            if self.sock in ready:
+                self.controllersock, address = self.sock.accept()
+                if self.haveui:
+                    print("Dropping connection attempt as we have a UI %s" % (str(ready)))
+                    self.controllersock.close()
+                else:
+                    print("Accepting %s" % (str(ready)))
+                    fds.append(self.controllersock)
+            if self.controllersock in ready:
+                try:
+                    print("Connecting Client")
+                    ui_fds = recvfds(self.controllersock, 3)
+
+                    # Where to write events to
+                    writer = ConnectionWriter(ui_fds[0])
+                    self.event_handle = bb.event.register_UIHhandler(writer, True)
+                    self.event_writer = writer
+
+                    # Where to read commands from
+                    reader = ConnectionReader(ui_fds[1])
+                    fds.append(reader)
+                    self.command_channel = reader
+
+                    # Where to send command return values to
+                    writer = ConnectionWriter(ui_fds[2])
+                    self.command_channel_reply = writer
+
+                    self.haveui = True
+
+                except EOFError:
+                    print("Disconnecting Client")
+                    fds.remove(self.controllersock)
+                    fds.remove(self.command_channel)
+                    bb.event.unregister_UIHhandler(self.event_handle, True)
+                    self.command_channel_reply.writer.close()
+                    self.event_writer.writer.close()
+                    del self.event_writer
+                    self.controllersock.close()
+                    self.haveui = False
+                    self.lastui = time.time()
+                    self.cooker.clientComplete()
+                    if self.timeout is None:
+                        print("No timeout, exiting.")
+                        self.quit = True
+            if not self.haveui and self.lastui and self.timeout and (self.lastui + self.timeout) < time.time():
+                print("Server timeout, exiting.")
+                self.quit = True
+
+            if self.command_channel in ready:
+                try:
+                    command = self.command_channel.get()
+                except EOFError:
+                    # Client connection shutting down
+                    continue
+                if command[0] == "terminateServer":
                     self.quit = True
+                    continue
+                try:
+                    print("Running command %s" % command)
+                    self.command_channel_reply.send(self.cooker.command.runCommand(command))
+                except Exception as e:
+                   logger.exception('Exception in server main event loop running command %s (%s)' % (command, str(e)))
+
+            if self.xmlrpc in ready:
+                self.xmlrpc.handle_requests()
+
+            ready = self.idle_commands(.1, fds)
+
+        print("Exiting")
+        try: 
+            self.cooker.shutdown(True)
+        except:
+            pass
+
+        self.cooker.post_serve()
+
+        # Remove the socket file so we don't get any more connections to avoid races
+        os.unlink(self.sockname)
+        self.sock.close()
+
+        # Finally release the lockfile but warn about other processes holding it open
+        lock = self.bitbake_lock
+        lockfile = lock.name
+        lock.close()
+        lock = None
+
+        while not lock:
+            with bb.utils.timeout(3):
+                lock = bb.utils.lockfile(lockfile, shared=False, retry=False, block=True)
+                if not lock:
+                    # Some systems may not have lsof available
+                    procs = None
                     try:
-                        self.runCommand(["stateForceShutdown"])
-                    except:
-                        pass
+                        procs = subprocess.check_output(["lsof", '-w', lockfile], stderr=subprocess.STDOUT)
+                    except OSError as e:
+                        if e.errno != errno.ENOENT:
+                            raise
+                    if procs is None:
+                        # Fall back to fuser if lsof is unavailable
+                        try:
+                            procs = subprocess.check_output(["fuser", '-v', lockfile], stderr=subprocess.STDOUT)
+                        except OSError as e:
+                            if e.errno != errno.ENOENT:
+                                raise
 
-                self.idle_commands(.1, [self.command_channel, self.quitout])
-            except Exception:
-                logger.exception('Running command %s', command)
-
-        self.event_queue.close()
-        bb.event.unregister_UIHhandler(self.event_handle.value)
-        self.command_channel.close()
-        self.cooker.shutdown(True)
-        self.quitout.close()
+                    msg = "Delaying shutdown due to active processes which appear to be holding bitbake.lock"
+                    if procs:
+                        msg += ":\n%s" % str(procs)
+                    print(msg)
+                    return
+        # We hold the lock so we can remove the file (hide stale pid data)
+        bb.utils.remove(lockfile)
+        bb.utils.unlockfile(lock)
 
     def idle_commands(self, delay, fds=None):
         nextsleep = delay
@@ -186,109 +293,294 @@ class ProcessServer(Process, BaseImplServer):
             nextsleep = self.next_heartbeat - now
 
         if nextsleep is not None:
-            select.select(fds,[],[],nextsleep)
+            if self.xmlrpc:
+                nextsleep = self.xmlrpc.get_timeout(nextsleep)
+            try:
+                return select.select(fds,[],[],nextsleep)[0]
+            except InterruptedError:
+                # Ignore EINTR
+                return []
+        else:
+            return []
+
+
+class ServerCommunicator():
+    def __init__(self, connection, recv):
+        self.connection = connection
+        self.recv = recv
 
     def runCommand(self, command):
-        """
-        Run a cooker command on the server
-        """
-        self.command_channel.send(self.cooker.command.runCommand(command))
+        self.connection.send(command)
+        if not self.recv.poll(5):
+            raise ProcessTimeout("Timeout while waiting for a reply from the bitbake server")
+        return self.recv.get()
 
-    def stop(self):
-        self.quitin.send("quit")
-        self.quitin.close()
-
-class BitBakeProcessServerConnection(BitBakeBaseServerConnection):
-    def __init__(self, serverImpl, ui_channel, event_queue):
-        self.procserver = serverImpl
-        self.ui_channel = ui_channel
-        self.event_queue = event_queue
-        self.connection = ServerCommunicator(self.ui_channel, self.procserver.event_handle, self.procserver)
-        self.events = self.event_queue
-        self.terminated = False
-
-    def sigterm_terminate(self):
-        bb.error("UI received SIGTERM")
-        self.terminate()
-
-    def terminate(self):
-        if self.terminated:
-            return
-        self.terminated = True
-        def flushevents():
-            while True:
-                try:
-                    event = self.event_queue.get(block=False)
-                except (Empty, IOError):
-                    break
-                if isinstance(event, logging.LogRecord):
-                    logger.handle(event)
-
-        self.procserver.stop()
-
-        while self.procserver.is_alive():
-            flushevents()
-            self.procserver.join(0.1)
-
-        self.ui_channel.close()
-        self.event_queue.close()
-        self.event_queue.setexit()
-        # XXX: Call explicity close in _writer to avoid
-        # fd leakage because isn't called on Queue.close()
-        self.event_queue._writer.close()
-
-# Wrap Queue to provide API which isn't server implementation specific
-class ProcessEventQueue(multiprocessing.queues.Queue):
-    def __init__(self, maxsize):
-        multiprocessing.queues.Queue.__init__(self, maxsize, ctx=multiprocessing.get_context())
-        self.exit = False
-        bb.utils.set_process_name("ProcessEQueue")
-
-    def setexit(self):
-        self.exit = True
-
-    def waitEvent(self, timeout):
-        if self.exit:
-            return self.getEvent()
-        try:
-            if not self.server.is_alive():
-                return self.getEvent()
-            return self.get(True, timeout)
-        except Empty:
-            return None
-
-    def getEvent(self):
-        try:
-            if not self.server.is_alive():
-                self.setexit()
-            return self.get(False)
-        except Empty:
-            if self.exit:
-                sys.exit(1)
-            return None
-
-class BitBakeServer(BitBakeBaseServer):
-    def initServer(self, single_use=True):
-        # establish communication channels.  We use bidirectional pipes for
-        # ui <--> server command/response pairs
-        # and a queue for server -> ui event notifications
-        #
-        self.ui_channel, self.server_channel = Pipe()
-        self.event_queue = ProcessEventQueue(0)
-        self.serverImpl = ProcessServer(self.server_channel, self.event_queue, None)
-        self.event_queue.server = self.serverImpl
-
-    def detach(self):
-        self.serverImpl.start()
-        return
-
-    def establishConnection(self, featureset):
-
-        self.connection = BitBakeProcessServerConnection(self.serverImpl, self.ui_channel, self.event_queue)
-
-        _, error = self.connection.connection.runCommand(["setFeatures", featureset])
+    def updateFeatureSet(self, featureset):
+        _, error = self.runCommand(["setFeatures", featureset])
         if error:
             logger.error("Unable to set the cooker to the correct featureset: %s" % error)
             raise BaseException(error)
-        signal.signal(signal.SIGTERM, lambda i, s: self.connection.sigterm_terminate())
-        return self.connection
+
+    def getEventHandle(self):
+        handle, error = self.runCommand(["getUIHandlerNum"])
+        if error:
+            logger.error("Unable to get UI Handler Number: %s" % error)
+            raise BaseException(error)
+
+        return handle
+
+    def terminateServer(self):
+        self.connection.send(['terminateServer'])
+        return
+
+class BitBakeProcessServerConnection(object):
+    def __init__(self, ui_channel, recv, eq, sock):
+        self.connection = ServerCommunicator(ui_channel, recv)
+        self.events = eq
+        # Save sock so it doesn't get gc'd for the life of our connection
+        self.socket_connection = sock
+
+    def terminate(self):
+        self.socket_connection.close()
+        self.connection.connection.close()
+        self.connection.recv.close()
+        return
+
+class BitBakeServer(object):
+    def __init__(self, lock, sockname, configuration, featureset):
+
+        self.configuration = configuration
+        self.featureset = featureset
+        self.sockname = sockname
+        self.bitbake_lock = lock
+        self.readypipe, self.readypipein = os.pipe()
+
+        # Create server control socket
+        if os.path.exists(sockname):
+            os.unlink(sockname)
+
+        self.sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
+        # AF_UNIX has path length issues so chdir here to workaround
+        cwd = os.getcwd()
+        logfile = os.path.join(cwd, "bitbake-cookerdaemon.log")
+
+        try:
+            os.chdir(os.path.dirname(sockname))
+            self.sock.bind(os.path.basename(sockname))
+        finally:
+            os.chdir(cwd)
+        self.sock.listen(1)
+
+        os.set_inheritable(self.sock.fileno(), True)
+        bb.daemonize.createDaemon(self._startServer, logfile)
+        self.sock.close()
+        self.bitbake_lock.close()
+
+        ready = ConnectionReader(self.readypipe)
+        r = ready.wait(8)
+        if not r:
+            ready.close()
+            bb.error("Unable to start bitbake server")
+            if os.path.exists(logfile):
+                with open(logfile, "r") as f:
+                    logs=f.readlines()
+                bb.error("Last 10 lines of server log %s:\n%s" % (logfile, "".join(logs[-10:])))
+            raise SystemExit(1)
+        ready.close()
+        os.close(self.readypipein)
+
+    def _startServer(self):
+        print("Starting bitbake server pid %d" % os.getpid())
+        server = ProcessServer(self.bitbake_lock, self.sock, self.sockname)
+        self.configuration.setServerRegIdleCallback(server.register_idle_function)
+
+        # Copy prefile and postfile to _server variants
+        for param in ('prefile', 'postfile'):
+            value = getattr(self.configuration, param)
+            if value:
+                setattr(self.configuration, "%s_server" % param, value)
+
+        self.cooker = bb.cooker.BBCooker(self.configuration, self.featureset, self.readypipein)
+        server.cooker = self.cooker
+        server.server_timeout = self.configuration.server_timeout
+        server.xmlrpcinterface = self.configuration.xmlrpcinterface
+        print("Started bitbake server pid %d" % os.getpid())
+        server.start()
+
+def connectProcessServer(sockname, featureset):
+    # Connect to socket
+    sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
+    # AF_UNIX has path length issues so chdir here to workaround
+    cwd = os.getcwd()
+
+    try:
+        os.chdir(os.path.dirname(sockname))
+        sock.connect(os.path.basename(sockname))
+    finally:
+        os.chdir(cwd)
+
+    readfd = writefd = readfd1 = writefd1 = readfd2 = writefd2 = None
+    eq = command_chan_recv = command_chan = None
+
+    try:
+
+        # Send an fd for the remote to write events to
+        readfd, writefd = os.pipe()
+        eq = BBUIEventQueue(readfd)
+        # Send an fd for the remote to recieve commands from
+        readfd1, writefd1 = os.pipe()
+        command_chan = ConnectionWriter(writefd1)
+        # Send an fd for the remote to write commands results to
+        readfd2, writefd2 = os.pipe()
+        command_chan_recv = ConnectionReader(readfd2)
+
+        sendfds(sock, [writefd, readfd1, writefd2])
+
+        server_connection = BitBakeProcessServerConnection(command_chan, command_chan_recv, eq, sock)
+
+        # Close the ends of the pipes we won't use
+        for i in [writefd, readfd1, writefd2]:
+            os.close(i)
+
+        server_connection.connection.updateFeatureSet(featureset)
+
+    except (Exception, SystemExit) as e:
+        if command_chan_recv:
+            command_chan_recv.close()
+        if command_chan:
+            command_chan.close()
+        for i in [writefd, readfd1, writefd2]:
+            try:
+                os.close(i)
+            except OSError:
+                pass
+        sock.close()
+        raise
+
+    return server_connection
+
+def sendfds(sock, fds):
+        '''Send an array of fds over an AF_UNIX socket.'''
+        fds = array.array('i', fds)
+        msg = bytes([len(fds) % 256])
+        sock.sendmsg([msg], [(socket.SOL_SOCKET, socket.SCM_RIGHTS, fds)])
+
+def recvfds(sock, size):
+        '''Receive an array of fds over an AF_UNIX socket.'''
+        a = array.array('i')
+        bytes_size = a.itemsize * size
+        msg, ancdata, flags, addr = sock.recvmsg(1, socket.CMSG_LEN(bytes_size))
+        if not msg and not ancdata:
+            raise EOFError
+        try:
+            if len(ancdata) != 1:
+                raise RuntimeError('received %d items of ancdata' %
+                                   len(ancdata))
+            cmsg_level, cmsg_type, cmsg_data = ancdata[0]
+            if (cmsg_level == socket.SOL_SOCKET and
+                cmsg_type == socket.SCM_RIGHTS):
+                if len(cmsg_data) % a.itemsize != 0:
+                    raise ValueError
+                a.frombytes(cmsg_data)
+                assert len(a) % 256 == msg[0]
+                return list(a)
+        except (ValueError, IndexError):
+            pass
+        raise RuntimeError('Invalid data received')
+
+class BBUIEventQueue:
+    def __init__(self, readfd):
+
+        self.eventQueue = []
+        self.eventQueueLock = threading.Lock()
+        self.eventQueueNotify = threading.Event()
+
+        self.reader = ConnectionReader(readfd)
+
+        self.t = threading.Thread()
+        self.t.setDaemon(True)
+        self.t.run = self.startCallbackHandler
+        self.t.start()
+
+    def getEvent(self):
+        self.eventQueueLock.acquire()
+
+        if len(self.eventQueue) == 0:
+            self.eventQueueLock.release()
+            return None
+
+        item = self.eventQueue.pop(0)
+
+        if len(self.eventQueue) == 0:
+            self.eventQueueNotify.clear()
+
+        self.eventQueueLock.release()
+        return item
+
+    def waitEvent(self, delay):
+        self.eventQueueNotify.wait(delay)
+        return self.getEvent()
+
+    def queue_event(self, event):
+        self.eventQueueLock.acquire()
+        self.eventQueue.append(event)
+        self.eventQueueNotify.set()
+        self.eventQueueLock.release()
+
+    def send_event(self, event):
+        self.queue_event(pickle.loads(event))
+
+    def startCallbackHandler(self):
+        bb.utils.set_process_name("UIEventQueue")
+        while True:
+            try:
+                self.reader.wait()
+                event = self.reader.get()
+                self.queue_event(event)
+            except EOFError:
+                # Easiest way to exit is to close the file descriptor to cause an exit
+                break
+        self.reader.close()
+
+class ConnectionReader(object):
+
+    def __init__(self, fd):
+        self.reader = multiprocessing.connection.Connection(fd, writable=False)
+        self.rlock = multiprocessing.Lock()
+
+    def wait(self, timeout=None):
+        return multiprocessing.connection.wait([self.reader], timeout)
+
+    def poll(self, timeout=None):
+        return self.reader.poll(timeout)
+
+    def get(self):
+        with self.rlock:
+            res = self.reader.recv_bytes()
+        return multiprocessing.reduction.ForkingPickler.loads(res)
+
+    def fileno(self):
+        return self.reader.fileno()
+
+    def close(self):
+        return self.reader.close()
+
+
+class ConnectionWriter(object):
+
+    def __init__(self, fd):
+        self.writer = multiprocessing.connection.Connection(fd, readable=False)
+        self.wlock = multiprocessing.Lock()
+        # Why bb.event needs this I have no idea
+        self.event = self
+
+    def send(self, obj):
+        obj = multiprocessing.reduction.ForkingPickler.dumps(obj)
+        with self.wlock:
+            self.writer.send_bytes(obj)
+
+    def fileno(self):
+        return self.writer.fileno()
+
+    def close(self):
+        return self.writer.close()

bitbake/lib/bb/server/xmlrpc.py

@@ -1,422 +0,0 @@
-#
-# BitBake XMLRPC Server
-#
-# Copyright (C) 2006 - 2007  Michael 'Mickey' Lauer
-# Copyright (C) 2006 - 2008  Richard Purdie
-#
-# This program is free software; you can redistribute it and/or modify
-# it under the terms of the GNU General Public License version 2 as
-# published by the Free Software Foundation.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License along
-# with this program; if not, write to the Free Software Foundation, Inc.,
-# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
-
-"""
-    This module implements an xmlrpc server for BitBake.
-
-    Use this by deriving a class from BitBakeXMLRPCServer and then adding
-    methods which you want to "export" via XMLRPC. If the methods have the
-    prefix xmlrpc_, then registering those function will happen automatically,
-    if not, you need to call register_function.
-
-    Use register_idle_function() to add a function which the xmlrpc server
-    calls from within server_forever when no requests are pending. Make sure
-    that those functions are non-blocking or else you will introduce latency
-    in the server's main loop.
-"""
-
-import os
-import sys
-
-import hashlib
-import time
-import socket
-import signal
-import threading
-import pickle
-import inspect
-import select
-import http.client
-import xmlrpc.client
-from xmlrpc.server import SimpleXMLRPCServer, SimpleXMLRPCRequestHandler
-
-import bb
-from bb import daemonize
-from bb.ui import uievent
-from . import BitBakeBaseServer, BitBakeBaseServerConnection, BaseImplServer
-
-DEBUG = False
-
-class BBTransport(xmlrpc.client.Transport):
-    def __init__(self, timeout):
-        self.timeout = timeout
-        self.connection_token = None
-        xmlrpc.client.Transport.__init__(self)
-
-    # Modified from default to pass timeout to HTTPConnection
-    def make_connection(self, host):
-        #return an existing connection if possible.  This allows
-        #HTTP/1.1 keep-alive.
-        if self._connection and host == self._connection[0]:
-            return self._connection[1]
-
-        # create a HTTP connection object from a host descriptor
-        chost, self._extra_headers, x509 = self.get_host_info(host)
-        #store the host argument along with the connection object
-        self._connection = host, http.client.HTTPConnection(chost, timeout=self.timeout)
-        return self._connection[1]
-
-    def set_connection_token(self, token):
-        self.connection_token = token
-
-    def send_content(self, h, body):
-        if self.connection_token:
-            h.putheader("Bitbake-token", self.connection_token)
-        xmlrpc.client.Transport.send_content(self, h, body)
-
-def _create_server(host, port, timeout = 60):
-    t = BBTransport(timeout)
-    s = xmlrpc.client.ServerProxy("http://%s:%d/" % (host, port), transport=t, allow_none=True, use_builtin_types=True)
-    return s, t
-
-def check_connection(remote, timeout):
-    try:
-        host, port = remote.split(":")
-        port = int(port)
-    except Exception as e:
-        bb.warn("Failed to read remote definition (%s)" % str(e))
-        raise e
-
-    server, _transport = _create_server(host, port, timeout)
-    try:
-        ret, err =  server.runCommand(['getVariable', 'TOPDIR'])
-        if err or not ret:
-            return False
-    except ConnectionError:
-        return False
-    return True
-
-class BitBakeServerCommands():
-
-    def __init__(self, server):
-        self.server = server
-        self.has_client = False
-
-    def registerEventHandler(self, host, port):
-        """
-        Register a remote UI Event Handler
-        """
-        s, t = _create_server(host, port)
-
-        # we don't allow connections if the cooker is running
-        if (self.cooker.state in [bb.cooker.state.parsing, bb.cooker.state.running]):
-            return None, "Cooker is busy: %s" % bb.cooker.state.get_name(self.cooker.state)
-
-        self.event_handle = bb.event.register_UIHhandler(s, True)
-        return self.event_handle, 'OK'
-
-    def unregisterEventHandler(self, handlerNum):
-        """
-        Unregister a remote UI Event Handler
-        """
-        return bb.event.unregister_UIHhandler(handlerNum)
-
-    def runCommand(self, command):
-        """
-        Run a cooker command on the server
-        """
-        return self.cooker.command.runCommand(command, self.server.readonly)
-
-    def getEventHandle(self):
-        return self.event_handle
-
-    def terminateServer(self):
-        """
-        Trigger the server to quit
-        """
-        self.server.quit = True
-        print("Server (cooker) exiting")
-        return
-
-    def addClient(self):
-        if self.has_client:
-            return None
-        token = hashlib.md5(str(time.time()).encode("utf-8")).hexdigest()
-        self.server.set_connection_token(token)
-        self.has_client = True
-        return token
-
-    def removeClient(self):
-        if self.has_client:
-            self.server.set_connection_token(None)
-            self.has_client = False
-            if self.server.single_use:
-                self.server.quit = True
-
-# This request handler checks if the request has a "Bitbake-token" header
-# field (this comes from the client side) and compares it with its internal
-# "Bitbake-token" field (this comes from the server). If the two are not
-# equal, it is assumed that a client is trying to connect to the server
-# while another client is connected to the server. In this case, a 503 error
-# ("service unavailable") is returned to the client.
-class BitBakeXMLRPCRequestHandler(SimpleXMLRPCRequestHandler):
-    def __init__(self, request, client_address, server):
-        self.server = server
-        SimpleXMLRPCRequestHandler.__init__(self, request, client_address, server)
-
-    def do_POST(self):
-        try:
-            remote_token = self.headers["Bitbake-token"]
-        except:
-            remote_token = None
-        if remote_token != self.server.connection_token and remote_token != "observer":
-            self.report_503()
-        else:
-            if remote_token == "observer":
-                self.server.readonly = True
-            else:
-                self.server.readonly = False
-            SimpleXMLRPCRequestHandler.do_POST(self)
-
-    def report_503(self):
-        self.send_response(503)
-        response = 'No more client allowed'
-        self.send_header("Content-type", "text/plain")
-        self.send_header("Content-length", str(len(response)))
-        self.end_headers()
-        self.wfile.write(bytes(response, 'utf-8'))
-
-
-class XMLRPCProxyServer(BaseImplServer):
-    """ not a real working server, but a stub for a proxy server connection
-
-    """
-    def __init__(self, host, port, use_builtin_types=True):
-        self.host = host
-        self.port = port
-
-class XMLRPCServer(SimpleXMLRPCServer, BaseImplServer):
-    # remove this when you're done with debugging
-    # allow_reuse_address = True
-
-    def __init__(self, interface, single_use=False, idle_timeout=0):
-        """
-        Constructor
-        """
-        BaseImplServer.__init__(self)
-        self.single_use = single_use
-        # Use auto port configuration
-        if (interface[1] == -1):
-            interface = (interface[0], 0)
-        SimpleXMLRPCServer.__init__(self, interface,
-                                    requestHandler=BitBakeXMLRPCRequestHandler,
-                                    logRequests=False, allow_none=True)
-        self.host, self.port = self.socket.getsockname()
-        self.connection_token = None
-        #self.register_introspection_functions()
-        self.commands = BitBakeServerCommands(self)
-        self.autoregister_all_functions(self.commands, "")
-        self.interface = interface
-        self.time = time.time()
-        self.idle_timeout = idle_timeout
-        if idle_timeout:
-            self.register_idle_function(self.handle_idle_timeout, self)
-
-    def addcooker(self, cooker):
-        BaseImplServer.addcooker(self, cooker)
-        self.commands.cooker = cooker
-
-    def autoregister_all_functions(self, context, prefix):
-        """
-        Convenience method for registering all functions in the scope
-        of this class that start with a common prefix
-        """
-        methodlist = inspect.getmembers(context, inspect.ismethod)
-        for name, method in methodlist:
-            if name.startswith(prefix):
-                self.register_function(method, name[len(prefix):])
-
-    def handle_idle_timeout(self, server, data, abort):
-        if not abort:
-            if time.time() - server.time > server.idle_timeout:
-                server.quit = True
-                print("Server idle timeout expired")
-        return []
-
-    def serve_forever(self):
-        # Start the actual XMLRPC server
-        bb.cooker.server_main(self.cooker, self._serve_forever)
-
-    def _serve_forever(self):
-        """
-        Serve Requests. Overloaded to honor a quit command
-        """
-        self.quit = False
-        while not self.quit:
-            fds = [self]
-            nextsleep = 0.1
-            for function, data in list(self._idlefuns.items()):
-                retval = None
-                try:
-                    retval = function(self, data, False)
-                    if retval is False:
-                        del self._idlefuns[function]
-                    elif retval is True:
-                        nextsleep = 0
-                    elif isinstance(retval, float):
-                        if (retval < nextsleep):
-                            nextsleep = retval
-                    else:
-                        fds = fds + retval
-                except SystemExit:
-                    raise
-                except:
-                    import traceback
-                    traceback.print_exc()
-                    if retval == None:
-                        # the function execute failed; delete it
-                        del self._idlefuns[function]
-                    pass
-
-            socktimeout = self.socket.gettimeout() or nextsleep
-            socktimeout = min(socktimeout, nextsleep)
-            # Mirror what BaseServer handle_request would do
-            try:
-                fd_sets = select.select(fds, [], [], socktimeout)
-                if fd_sets[0] and self in fd_sets[0]:
-                    if self.idle_timeout:
-                        self.time = time.time()
-                    self._handle_request_noblock()
-            except IOError:
-                # we ignore interrupted calls
-                pass
-
-        # Tell idle functions we're exiting
-        for function, data in list(self._idlefuns.items()):
-            try:
-                retval = function(self, data, True)
-            except:
-                pass
-        self.server_close()
-        return
-
-    def set_connection_token(self, token):
-        self.connection_token = token
-
-class BitBakeXMLRPCServerConnection(BitBakeBaseServerConnection):
-    def __init__(self, serverImpl, clientinfo=("localhost", 0), observer_only = False, featureset = None):
-        self.connection, self.transport = _create_server(serverImpl.host, serverImpl.port)
-        self.clientinfo = clientinfo
-        self.serverImpl = serverImpl
-        self.observer_only = observer_only
-        if featureset:
-            self.featureset = featureset
-        else:
-            self.featureset = []
-
-    def connect(self, token = None):
-        if token is None:
-            if self.observer_only:
-                token = "observer"
-            else:
-                token = self.connection.addClient()
-
-        if token is None:
-            return None
-
-        self.transport.set_connection_token(token)
-        return self
-
-    def setupEventQueue(self):
-        self.events = uievent.BBUIEventQueue(self.connection, self.clientinfo)
-        for event in bb.event.ui_queue:
-            self.events.queue_event(event)
-
-        _, error = self.connection.runCommand(["setFeatures", self.featureset])
-        if error:
-            # disconnect the client, we can't make the setFeature work
-            self.connection.removeClient()
-            # no need to log it here, the error shall be sent to the client
-            raise BaseException(error)
-
-    def removeClient(self):
-        if not self.observer_only:
-            self.connection.removeClient()
-
-    def terminate(self):
-        # Don't wait for server indefinitely
-        import socket
-        socket.setdefaulttimeout(2)
-        try:
-            self.events.system_quit()
-        except:
-            pass
-        try:
-            self.connection.removeClient()
-        except:
-            pass
-
-class BitBakeServer(BitBakeBaseServer):
-    def initServer(self, interface = ("localhost", 0),
-                   single_use = False, idle_timeout=0):
-        self.interface = interface
-        self.serverImpl = XMLRPCServer(interface, single_use, idle_timeout)
-
-    def detach(self):
-        daemonize.createDaemon(self.serverImpl.serve_forever, "bitbake-cookerdaemon.log")
-        del self.cooker
-
-    def establishConnection(self, featureset):
-        self.connection = BitBakeXMLRPCServerConnection(self.serverImpl, self.interface, False, featureset)
-        return self.connection.connect()
-
-    def set_connection_token(self, token):
-        self.connection.transport.set_connection_token(token)
-
-class BitBakeXMLRPCClient(BitBakeBaseServer):
-
-    def __init__(self, observer_only = False, token = None):
-        self.token = token
-
-        self.observer_only = observer_only
-        # if we need extra caches, just tell the server to load them all
-        pass
-
-    def saveConnectionDetails(self, remote):
-        self.remote = remote
-
-    def establishConnection(self, featureset):
-        # The format of "remote" must be "server:port"
-        try:
-            [host, port] = self.remote.split(":")
-            port = int(port)
-        except Exception as e:
-            bb.warn("Failed to read remote definition (%s)" % str(e))
-            raise e
-
-        # We need our IP for the server connection. We get the IP
-        # by trying to connect with the server
-        try:
-            s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
-            s.connect((host, port))
-            ip = s.getsockname()[0]
-            s.close()
-        except Exception as e:
-            bb.warn("Could not create socket for %s:%s (%s)" % (host, port, str(e)))
-            raise e
-        try:
-            self.serverImpl = XMLRPCProxyServer(host, port, use_builtin_types=True)
-            self.connection = BitBakeXMLRPCServerConnection(self.serverImpl, (ip, 0), self.observer_only, featureset)
-            return self.connection.connect(self.token)
-        except Exception as e:
-            bb.warn("Could not connect to server at %s:%s (%s)" % (host, port, str(e)))
-            raise e
-
-    def endSession(self):
-        self.connection.removeClient()

bitbake/lib/bb/server/xmlrpcclient.py

@@ -0,0 +1,154 @@
+#
+# BitBake XMLRPC Client Interface
+#
+# Copyright (C) 2006 - 2007  Michael 'Mickey' Lauer
+# Copyright (C) 2006 - 2008  Richard Purdie
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License version 2 as
+# published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License along
+# with this program; if not, write to the Free Software Foundation, Inc.,
+# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+import os
+import sys
+
+import socket
+import http.client
+import xmlrpc.client
+
+import bb
+from bb.ui import uievent
+
+class BBTransport(xmlrpc.client.Transport):
+    def __init__(self, timeout):
+        self.timeout = timeout
+        self.connection_token = None
+        xmlrpc.client.Transport.__init__(self)
+
+    # Modified from default to pass timeout to HTTPConnection
+    def make_connection(self, host):
+        #return an existing connection if possible.  This allows
+        #HTTP/1.1 keep-alive.
+        if self._connection and host == self._connection[0]:
+            return self._connection[1]
+
+        # create a HTTP connection object from a host descriptor
+        chost, self._extra_headers, x509 = self.get_host_info(host)
+        #store the host argument along with the connection object
+        self._connection = host, http.client.HTTPConnection(chost, timeout=self.timeout)
+        return self._connection[1]
+
+    def set_connection_token(self, token):
+        self.connection_token = token
+
+    def send_content(self, h, body):
+        if self.connection_token:
+            h.putheader("Bitbake-token", self.connection_token)
+        xmlrpc.client.Transport.send_content(self, h, body)
+
+def _create_server(host, port, timeout = 60):
+    t = BBTransport(timeout)
+    s = xmlrpc.client.ServerProxy("http://%s:%d/" % (host, port), transport=t, allow_none=True, use_builtin_types=True)
+    return s, t
+
+def check_connection(remote, timeout):
+    try:
+        host, port = remote.split(":")
+        port = int(port)
+    except Exception as e:
+        bb.warn("Failed to read remote definition (%s)" % str(e))
+        raise e
+
+    server, _transport = _create_server(host, port, timeout)
+    try:
+        ret, err =  server.runCommand(['getVariable', 'TOPDIR'])
+        if err or not ret:
+            return False
+    except ConnectionError:
+        return False
+    return True
+
+class BitBakeXMLRPCServerConnection(object):
+    def __init__(self, host, port, clientinfo=("localhost", 0), observer_only = False, featureset = None):
+        self.connection, self.transport = _create_server(host, port)
+        self.clientinfo = clientinfo
+        self.observer_only = observer_only
+        if featureset:
+            self.featureset = featureset
+        else:
+            self.featureset = []
+
+        self.events = uievent.BBUIEventQueue(self.connection, self.clientinfo)
+
+        _, error = self.connection.runCommand(["setFeatures", self.featureset])
+        if error:
+            # disconnect the client, we can't make the setFeature work
+            self.connection.removeClient()
+            # no need to log it here, the error shall be sent to the client
+            raise BaseException(error)
+
+    def connect(self, token = None):
+        if token is None:
+            if self.observer_only:
+                token = "observer"
+            else:
+                token = self.connection.addClient()
+
+        if token is None:
+            return None
+
+        self.transport.set_connection_token(token)
+        return self
+
+    def removeClient(self):
+        if not self.observer_only:
+            self.connection.removeClient()
+
+    def terminate(self):
+        # Don't wait for server indefinitely
+        socket.setdefaulttimeout(2)
+        try:
+            self.events.system_quit()
+        except:
+            pass
+        try:
+            self.connection.removeClient()
+        except:
+            pass
+
+def connectXMLRPC(remote, featureset, observer_only = False, token = None):
+    # The format of "remote" must be "server:port"
+    try:
+        [host, port] = remote.split(":")
+        port = int(port)
+    except Exception as e:
+        bb.warn("Failed to parse remote definition %s (%s)" % (remote, str(e)))
+        raise e
+
+    # We need our IP for the server connection. We get the IP
+    # by trying to connect with the server
+    try:
+        s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
+        s.connect((host, port))
+        ip = s.getsockname()[0]
+        s.close()
+    except Exception as e:
+        bb.warn("Could not create socket for %s:%s (%s)" % (host, port, str(e)))
+        raise e
+    try:
+        connection = BitBakeXMLRPCServerConnection(host, port, (ip, 0), observer_only, featureset)
+        return connection.connect(token)
+    except Exception as e:
+        bb.warn("Could not connect to server at %s:%s (%s)" % (host, port, str(e)))
+        raise e
+
+
+

bitbake/lib/bb/server/xmlrpcserver.py

@@ -0,0 +1,158 @@
+#
+# BitBake XMLRPC Server Interface
+#
+# Copyright (C) 2006 - 2007  Michael 'Mickey' Lauer
+# Copyright (C) 2006 - 2008  Richard Purdie
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License version 2 as
+# published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License along
+# with this program; if not, write to the Free Software Foundation, Inc.,
+# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+import os
+import sys
+
+import hashlib
+import time
+import inspect
+from xmlrpc.server import SimpleXMLRPCServer, SimpleXMLRPCRequestHandler
+
+import bb
+
+# This request handler checks if the request has a "Bitbake-token" header
+# field (this comes from the client side) and compares it with its internal
+# "Bitbake-token" field (this comes from the server). If the two are not
+# equal, it is assumed that a client is trying to connect to the server
+# while another client is connected to the server. In this case, a 503 error
+# ("service unavailable") is returned to the client.
+class BitBakeXMLRPCRequestHandler(SimpleXMLRPCRequestHandler):
+    def __init__(self, request, client_address, server):
+        self.server = server
+        SimpleXMLRPCRequestHandler.__init__(self, request, client_address, server)
+
+    def do_POST(self):
+        try:
+            remote_token = self.headers["Bitbake-token"]
+        except:
+            remote_token = None
+        if 0 and remote_token != self.server.connection_token and remote_token != "observer":
+            self.report_503()
+        else:
+            if remote_token == "observer":
+                self.server.readonly = True
+            else:
+                self.server.readonly = False
+            SimpleXMLRPCRequestHandler.do_POST(self)
+
+    def report_503(self):
+        self.send_response(503)
+        response = 'No more client allowed'
+        self.send_header("Content-type", "text/plain")
+        self.send_header("Content-length", str(len(response)))
+        self.end_headers()
+        self.wfile.write(bytes(response, 'utf-8'))
+
+class BitBakeXMLRPCServer(SimpleXMLRPCServer):
+    # remove this when you're done with debugging
+    # allow_reuse_address = True
+
+    def __init__(self, interface, cooker, parent):
+        # Use auto port configuration
+        if (interface[1] == -1):
+            interface = (interface[0], 0)
+        SimpleXMLRPCServer.__init__(self, interface,
+                                    requestHandler=BitBakeXMLRPCRequestHandler,
+                                    logRequests=False, allow_none=True)
+        self.host, self.port = self.socket.getsockname()
+        self.interface = interface
+
+        self.connection_token = None
+        self.commands = BitBakeXMLRPCServerCommands(self)
+        self.register_functions(self.commands, "")
+
+        self.cooker = cooker
+        self.parent = parent
+
+
+    def register_functions(self, context, prefix):
+        """
+        Convenience method for registering all functions in the scope
+        of this class that start with a common prefix
+        """
+        methodlist = inspect.getmembers(context, inspect.ismethod)
+        for name, method in methodlist:
+            if name.startswith(prefix):
+                self.register_function(method, name[len(prefix):])
+
+    def get_timeout(self, delay):
+        socktimeout = self.socket.gettimeout() or delay
+        return min(socktimeout, delay)
+
+    def handle_requests(self):
+        self._handle_request_noblock()
+
+class BitBakeXMLRPCServerCommands():
+
+    def __init__(self, server):
+        self.server = server
+        self.has_client = False
+
+    def registerEventHandler(self, host, port):
+        """
+        Register a remote UI Event Handler
+        """
+        s, t = bb.server.xmlrpcclient._create_server(host, port)
+
+        # we don't allow connections if the cooker is running
+        if (self.server.cooker.state in [bb.cooker.state.parsing, bb.cooker.state.running]):
+            return None, "Cooker is busy: %s" % bb.cooker.state.get_name(self.server.cooker.state)
+
+        self.event_handle = bb.event.register_UIHhandler(s, True)
+        return self.event_handle, 'OK'
+
+    def unregisterEventHandler(self, handlerNum):
+        """
+        Unregister a remote UI Event Handler
+        """
+        ret = bb.event.unregister_UIHhandler(handlerNum, True)
+        self.event_handle = None
+        return ret
+
+    def runCommand(self, command):
+        """
+        Run a cooker command on the server
+        """
+        return self.server.cooker.command.runCommand(command, self.server.readonly)
+
+    def getEventHandle(self):
+        return self.event_handle
+
+    def terminateServer(self):
+        """
+        Trigger the server to quit
+        """
+        self.server.parent.quit = True
+        print("XMLRPC Server triggering exit")
+        return
+
+    def addClient(self):
+        if self.server.parent.haveui:
+            return None
+        token = hashlib.md5(str(time.time()).encode("utf-8")).hexdigest()
+        self.server.connection_token = token
+        self.server.parent.haveui = True
+        return token
+
+    def removeClient(self):
+        if self.server.parent.haveui:
+            self.server.connection_token = None
+            self.server.parent.haveui = False
+

bitbake/lib/bb/tests/event.py

@@ -0,0 +1,377 @@
+# ex:ts=4:sw=4:sts=4:et
+# -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*-
+#
+# BitBake Tests for the Event implementation (event.py)
+#
+# Copyright (C) 2017 Intel Corporation
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License version 2 as
+# published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License along
+# with this program; if not, write to the Free Software Foundation, Inc.,
+# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+#
+
+import unittest
+import bb
+import logging
+import bb.compat
+import bb.event
+import importlib
+import threading
+import time
+import pickle
+from unittest.mock import Mock
+from unittest.mock import call
+
+
+class EventQueueStub():
+    """ Class used as specification for UI event handler queue stub objects """
+    def __init__(self):
+        return
+
+    def send(self, event):
+        return
+
+
+class PickleEventQueueStub():
+    """ Class used as specification for UI event handler queue stub objects
+        with sendpickle method """
+    def __init__(self):
+        return
+
+    def sendpickle(self, pickled_event):
+        return
+
+
+class UIClientStub():
+    """ Class used as specification for UI event handler stub objects """
+    def __init__(self):
+        self.event = None
+
+
+class EventHandlingTest(unittest.TestCase):
+    """ Event handling test class """
+    _threadlock_test_calls = []
+
+    def setUp(self):
+        self._test_process = Mock()
+        ui_client1 = UIClientStub()
+        ui_client2 = UIClientStub()
+        self._test_ui1 = Mock(wraps=ui_client1)
+        self._test_ui2 = Mock(wraps=ui_client2)
+        importlib.reload(bb.event)
+
+    def _create_test_handlers(self):
+        """ Method used to create a test handler ordered dictionary """
+        test_handlers = bb.compat.OrderedDict()
+        test_handlers["handler1"] = self._test_process.handler1
+        test_handlers["handler2"] = self._test_process.handler2
+        return test_handlers
+
+    def test_class_handlers(self):
+        """ Test set_class_handlers and get_class_handlers methods """
+        test_handlers = self._create_test_handlers()
+        bb.event.set_class_handlers(test_handlers)
+        self.assertEqual(test_handlers,
+                         bb.event.get_class_handlers())
+
+    def test_handlers(self):
+        """ Test set_handlers and get_handlers """
+        test_handlers = self._create_test_handlers()
+        bb.event.set_handlers(test_handlers)
+        self.assertEqual(test_handlers,
+                         bb.event.get_handlers())
+
+    def test_clean_class_handlers(self):
+        """ Test clean_class_handlers method """
+        cleanDict = bb.compat.OrderedDict()
+        self.assertEqual(cleanDict,
+                         bb.event.clean_class_handlers())
+
+    def test_register(self):
+        """ Test register method for class handlers """
+        result = bb.event.register("handler", self._test_process.handler)
+        self.assertEqual(result, bb.event.Registered)
+        handlers_dict = bb.event.get_class_handlers()
+        self.assertIn("handler", handlers_dict)
+
+    def test_already_registered(self):
+        """ Test detection of an already registed class handler """
+        bb.event.register("handler", self._test_process.handler)
+        handlers_dict = bb.event.get_class_handlers()
+        self.assertIn("handler", handlers_dict)
+        result = bb.event.register("handler", self._test_process.handler)
+        self.assertEqual(result, bb.event.AlreadyRegistered)
+
+    def test_register_from_string(self):
+        """ Test register method receiving code in string """
+        result = bb.event.register("string_handler", "    return True")
+        self.assertEqual(result, bb.event.Registered)
+        handlers_dict = bb.event.get_class_handlers()
+        self.assertIn("string_handler", handlers_dict)
+
+    def test_register_with_mask(self):
+        """ Test register method with event masking """
+        mask = ["bb.event.OperationStarted",
+                "bb.event.OperationCompleted"]
+        result = bb.event.register("event_handler",
+                                   self._test_process.event_handler,
+                                   mask)
+        self.assertEqual(result, bb.event.Registered)
+        handlers_dict = bb.event.get_class_handlers()
+        self.assertIn("event_handler", handlers_dict)
+
+    def test_remove(self):
+        """ Test remove method for class handlers """
+        test_handlers = self._create_test_handlers()
+        bb.event.set_class_handlers(test_handlers)
+        count = len(test_handlers)
+        bb.event.remove("handler1", None)
+        test_handlers = bb.event.get_class_handlers()
+        self.assertEqual(len(test_handlers), count - 1)
+        with self.assertRaises(KeyError):
+            bb.event.remove("handler1", None)
+
+    def test_execute_handler(self):
+        """ Test execute_handler method for class handlers """
+        mask = ["bb.event.OperationProgress"]
+        result = bb.event.register("event_handler",
+                                   self._test_process.event_handler,
+                                   mask)
+        self.assertEqual(result, bb.event.Registered)
+        event = bb.event.OperationProgress(current=10, total=100)
+        bb.event.execute_handler("event_handler",
+                                 self._test_process.event_handler,
+                                 event,
+                                 None)
+        self._test_process.event_handler.assert_called_once_with(event)
+
+    def test_fire_class_handlers(self):
+        """ Test fire_class_handlers method """
+        mask = ["bb.event.OperationStarted"]
+        result = bb.event.register("event_handler1",
+                                   self._test_process.event_handler1,
+                                   mask)
+        self.assertEqual(result, bb.event.Registered)
+        result = bb.event.register("event_handler2",
+                                   self._test_process.event_handler2,
+                                   "*")
+        self.assertEqual(result, bb.event.Registered)
+        event1 = bb.event.OperationStarted()
+        event2 = bb.event.OperationCompleted(total=123)
+        bb.event.fire_class_handlers(event1, None)
+        bb.event.fire_class_handlers(event2, None)
+        bb.event.fire_class_handlers(event2, None)
+        expected_event_handler1 = [call(event1)]
+        expected_event_handler2 = [call(event1),
+                                   call(event2),
+                                   call(event2)]
+        self.assertEqual(self._test_process.event_handler1.call_args_list,
+                         expected_event_handler1)
+        self.assertEqual(self._test_process.event_handler2.call_args_list,
+                         expected_event_handler2)
+
+    def test_change_handler_event_mapping(self):
+        """ Test changing the event mapping for class handlers """
+        event1 = bb.event.OperationStarted()
+        event2 = bb.event.OperationCompleted(total=123)
+
+        # register handler for all events
+        result = bb.event.register("event_handler1",
+                                   self._test_process.event_handler1,
+                                   "*")
+        self.assertEqual(result, bb.event.Registered)
+        bb.event.fire_class_handlers(event1, None)
+        bb.event.fire_class_handlers(event2, None)
+        expected = [call(event1), call(event2)]
+        self.assertEqual(self._test_process.event_handler1.call_args_list,
+                         expected)
+
+        # unregister handler and register it only for OperationStarted
+        result = bb.event.remove("event_handler1",
+                                 self._test_process.event_handler1)
+        mask = ["bb.event.OperationStarted"]
+        result = bb.event.register("event_handler1",
+                                   self._test_process.event_handler1,
+                                   mask)
+        self.assertEqual(result, bb.event.Registered)
+        bb.event.fire_class_handlers(event1, None)
+        bb.event.fire_class_handlers(event2, None)
+        expected = [call(event1), call(event2), call(event1)]
+        self.assertEqual(self._test_process.event_handler1.call_args_list,
+                         expected)
+
+        # unregister handler and register it only for OperationCompleted
+        result = bb.event.remove("event_handler1",
+                                 self._test_process.event_handler1)
+        mask = ["bb.event.OperationCompleted"]
+        result = bb.event.register("event_handler1",
+                                   self._test_process.event_handler1,
+                                   mask)
+        self.assertEqual(result, bb.event.Registered)
+        bb.event.fire_class_handlers(event1, None)
+        bb.event.fire_class_handlers(event2, None)
+        expected = [call(event1), call(event2), call(event1), call(event2)]
+        self.assertEqual(self._test_process.event_handler1.call_args_list,
+                         expected)
+
+    def test_register_UIHhandler(self):
+        """ Test register_UIHhandler method """
+        result = bb.event.register_UIHhandler(self._test_ui1, mainui=True)
+        self.assertEqual(result, 1)
+
+    def test_UIHhandler_already_registered(self):
+        """ Test registering an UIHhandler already existing """
+        result = bb.event.register_UIHhandler(self._test_ui1, mainui=True)
+        self.assertEqual(result, 1)
+        result = bb.event.register_UIHhandler(self._test_ui1, mainui=True)
+        self.assertEqual(result, 2)
+
+    def test_unregister_UIHhandler(self):
+        """ Test unregister_UIHhandler method """
+        result = bb.event.register_UIHhandler(self._test_ui1, mainui=True)
+        self.assertEqual(result, 1)
+        result = bb.event.unregister_UIHhandler(1)
+        self.assertIs(result, None)
+
+    def test_fire_ui_handlers(self):
+        """ Test fire_ui_handlers method """
+        self._test_ui1.event = Mock(spec_set=EventQueueStub)
+        result = bb.event.register_UIHhandler(self._test_ui1, mainui=True)
+        self.assertEqual(result, 1)
+        self._test_ui2.event = Mock(spec_set=PickleEventQueueStub)
+        result = bb.event.register_UIHhandler(self._test_ui2, mainui=True)
+        self.assertEqual(result, 2)
+        event1 = bb.event.OperationStarted()
+        bb.event.fire_ui_handlers(event1, None)
+        expected = [call(event1)]
+        self.assertEqual(self._test_ui1.event.send.call_args_list,
+                         expected)
+        expected = [call(pickle.dumps(event1))]
+        self.assertEqual(self._test_ui2.event.sendpickle.call_args_list,
+                         expected)
+
+    def test_fire(self):
+        """ Test fire method used to trigger class and ui event handlers """
+        mask = ["bb.event.ConfigParsed"]
+        result = bb.event.register("event_handler1",
+                                   self._test_process.event_handler1,
+                                   mask)
+
+        self._test_ui1.event = Mock(spec_set=EventQueueStub)
+        result = bb.event.register_UIHhandler(self._test_ui1, mainui=True)
+        self.assertEqual(result, 1)
+
+        event1 = bb.event.ConfigParsed()
+        bb.event.fire(event1, None)
+        expected = [call(event1)]
+        self.assertEqual(self._test_process.event_handler1.call_args_list,
+                         expected)
+        self.assertEqual(self._test_ui1.event.send.call_args_list,
+                         expected)
+
+    def test_fire_from_worker(self):
+        """ Test fire_from_worker method """
+        self._test_ui1.event = Mock(spec_set=EventQueueStub)
+        result = bb.event.register_UIHhandler(self._test_ui1, mainui=True)
+        self.assertEqual(result, 1)
+        event1 = bb.event.ConfigParsed()
+        bb.event.fire_from_worker(event1, None)
+        expected = [call(event1)]
+        self.assertEqual(self._test_ui1.event.send.call_args_list,
+                         expected)
+
+    def test_print_ui_queue(self):
+        """ Test print_ui_queue method """
+        event1 = bb.event.OperationStarted()
+        event2 = bb.event.OperationCompleted(total=123)
+        bb.event.fire(event1, None)
+        bb.event.fire(event2, None)
+        logger = logging.getLogger("BitBake")
+        logger.addHandler(bb.event.LogHandler())
+        logger.info("Test info LogRecord")
+        logger.warning("Test warning LogRecord")
+        with self.assertLogs("BitBake", level="INFO") as cm:
+            bb.event.print_ui_queue()
+        self.assertEqual(cm.output,
+                         ["INFO:BitBake:Test info LogRecord",
+                          "WARNING:BitBake:Test warning LogRecord"])
+
+    def _set_threadlock_test_mockups(self):
+        """ Create UI event handler mockups used in enable and disable
+            threadlock tests """
+        def ui1_event_send(event):
+            if type(event) is bb.event.ConfigParsed:
+                self._threadlock_test_calls.append("w1_ui1")
+            if type(event) is bb.event.OperationStarted:
+                self._threadlock_test_calls.append("w2_ui1")
+            time.sleep(2)
+
+        def ui2_event_send(event):
+            if type(event) is bb.event.ConfigParsed:
+                self._threadlock_test_calls.append("w1_ui2")
+            if type(event) is bb.event.OperationStarted:
+                self._threadlock_test_calls.append("w2_ui2")
+            time.sleep(2)
+
+        self._threadlock_test_calls = []
+        self._test_ui1.event = EventQueueStub()
+        self._test_ui1.event.send = ui1_event_send
+        result = bb.event.register_UIHhandler(self._test_ui1, mainui=True)
+        self.assertEqual(result, 1)
+        self._test_ui2.event = EventQueueStub()
+        self._test_ui2.event.send = ui2_event_send
+        result = bb.event.register_UIHhandler(self._test_ui2, mainui=True)
+        self.assertEqual(result, 2)
+
+    def _set_and_run_threadlock_test_workers(self):
+        """ Create and run the workers used to trigger events in enable and
+            disable threadlock tests """
+        worker1 = threading.Thread(target=self._thread_lock_test_worker1)
+        worker2 = threading.Thread(target=self._thread_lock_test_worker2)
+        worker1.start()
+        time.sleep(1)
+        worker2.start()
+        worker1.join()
+        worker2.join()
+
+    def _thread_lock_test_worker1(self):
+        """ First worker used to fire the ConfigParsed event for enable and
+            disable threadlocks tests """
+        bb.event.fire(bb.event.ConfigParsed(), None)
+
+    def _thread_lock_test_worker2(self):
+        """ Second worker used to fire the OperationStarted event for enable
+            and disable threadlocks tests """
+        bb.event.fire(bb.event.OperationStarted(), None)
+
+    def test_enable_threadlock(self):
+        """ Test enable_threadlock method """
+        self._set_threadlock_test_mockups()
+        bb.event.enable_threadlock()
+        self._set_and_run_threadlock_test_workers()
+        # Calls to UI handlers should be in order as all the registered
+        # handlers for the event coming from the first worker should be
+        # called before processing the event from the second worker.
+        self.assertEqual(self._threadlock_test_calls,
+                         ["w1_ui1", "w1_ui2", "w2_ui1", "w2_ui2"])
+
+    def test_disable_threadlock(self):
+        """ Test disable_threadlock method """
+        self._set_threadlock_test_mockups()
+        bb.event.disable_threadlock()
+        self._set_and_run_threadlock_test_workers()
+        # Calls to UI handlers should be intertwined together. Thanks to the
+        # delay in the registered handlers for the event coming from the first
+        # worker, the event coming from the second worker starts being
+        # processed before finishing handling the first worker event.
+        self.assertEqual(self._threadlock_test_calls,
+                         ["w1_ui1", "w2_ui1", "w1_ui2", "w2_ui2"])

bitbake/lib/bb/tests/fetch.py

@@ -1440,7 +1440,7 @@ class GitShallowTest(FetcherTest):
         def test_bitbake(self):
             self.git('remote add --mirror=fetch origin git://github.com/openembedded/bitbake', cwd=self.srcdir)
             self.git('config core.bare true', cwd=self.srcdir)
-            self.git('fetch --tags', cwd=self.srcdir)
+            self.git('fetch', cwd=self.srcdir)
 
             self.d.setVar('BB_GIT_SHALLOW_DEPTH', '0')
             # Note that the 1.10.0 tag is annotated, so this also tests

bitbake/lib/bb/tinfoil.py

@@ -2,6 +2,7 @@
 #
 # Copyright (C) 2012-2017 Intel Corporation
 # Copyright (C) 2011 Mentor Graphics Corporation
+# Copyright (C) 2006-2012 Richard Purdie
 #
 # This program is free software; you can redistribute it and/or modify
 # it under the terms of the GNU General Public License version 2 as
@@ -54,6 +55,7 @@ class TinfoilCommandFailed(Exception):
     """Exception raised when run_command fails"""
 
 class TinfoilDataStoreConnector:
+    """Connector object used to enable access to datastore objects via tinfoil"""
 
     def __init__(self, tinfoil, dsindex):
         self.tinfoil = tinfoil
@@ -172,6 +174,14 @@ class TinfoilCookerAdapter:
                 attrvalue = self.tinfoil.run_command('getBbFilePriority') or {}
             elif name == 'pkg_dp':
                 attrvalue = self.tinfoil.run_command('getDefaultPreference') or {}
+            elif name == 'fn_provides':
+                attrvalue = self.tinfoil.run_command('getRecipeProvides') or {}
+            elif name == 'packages':
+                attrvalue = self.tinfoil.run_command('getRecipePackages') or {}
+            elif name == 'packages_dynamic':
+                attrvalue = self.tinfoil.run_command('getRecipePackagesDynamic') or {}
+            elif name == 'rproviders':
+                attrvalue = self.tinfoil.run_command('getRProviders') or {}
             else:
                 raise AttributeError("%s instance has no attribute '%s'" % (self.__class__.__name__, name))
 
@@ -208,15 +218,110 @@ class TinfoilCookerAdapter:
         return self.tinfoil.find_best_provider(pn)
 
 
+class TinfoilRecipeInfo:
+    """
+    Provides a convenient representation of the cached information for a single recipe.
+    Some attributes are set on construction, others are read on-demand (which internally
+    may result in a remote procedure call to the bitbake server the first time).
+    Note that only information which is cached is available through this object - if
+    you need other variable values you will need to parse the recipe using
+    Tinfoil.parse_recipe().
+    """
+    def __init__(self, recipecache, d, pn, fn, fns):
+        self._recipecache = recipecache
+        self._d = d
+        self.pn = pn
+        self.fn = fn
+        self.fns = fns
+        self.inherit_files = recipecache.inherits[fn]
+        self.depends = recipecache.deps[fn]
+        (self.pe, self.pv, self.pr) = recipecache.pkg_pepvpr[fn]
+        self._cached_packages = None
+        self._cached_rprovides = None
+        self._cached_packages_dynamic = None
+
+    def __getattr__(self, name):
+        if name == 'alternates':
+            return [x for x in self.fns if x != self.fn]
+        elif name == 'rdepends':
+            return self._recipecache.rundeps[self.fn]
+        elif name == 'rrecommends':
+            return self._recipecache.runrecs[self.fn]
+        elif name == 'provides':
+            return self._recipecache.fn_provides[self.fn]
+        elif name == 'packages':
+            if self._cached_packages is None:
+                self._cached_packages = []
+                for pkg, fns in self._recipecache.packages.items():
+                    if self.fn in fns:
+                        self._cached_packages.append(pkg)
+            return self._cached_packages
+        elif name == 'packages_dynamic':
+            if self._cached_packages_dynamic is None:
+                self._cached_packages_dynamic = []
+                for pkg, fns in self._recipecache.packages_dynamic.items():
+                    if self.fn in fns:
+                        self._cached_packages_dynamic.append(pkg)
+            return self._cached_packages_dynamic
+        elif name == 'rprovides':
+            if self._cached_rprovides is None:
+                self._cached_rprovides = []
+                for pkg, fns in self._recipecache.rproviders.items():
+                    if self.fn in fns:
+                        self._cached_rprovides.append(pkg)
+            return self._cached_rprovides
+        else:
+            raise AttributeError("%s instance has no attribute '%s'" % (self.__class__.__name__, name))
+    def inherits(self, only_recipe=False):
+        """
+        Get the inherited classes for a recipe. Returns the class names only.
+        Parameters:
+            only_recipe: True to return only the classes inherited by the recipe
+                         itself, False to return all classes inherited within
+                         the context for the recipe (which includes globally
+                         inherited classes).
+        """
+        if only_recipe:
+            global_inherit = [x for x in (self._d.getVar('BBINCLUDED') or '').split() if x.endswith('.bbclass')]
+        else:
+            global_inherit = []
+        for clsfile in self.inherit_files:
+            if only_recipe and clsfile in global_inherit:
+                continue
+            clsname = os.path.splitext(os.path.basename(clsfile))[0]
+            yield clsname
+    def __str__(self):
+        return '%s' % self.pn
+
+
 class Tinfoil:
+    """
+    Tinfoil - an API for scripts and utilities to query
+    BitBake internals and perform build operations.
+    """
 
     def __init__(self, output=sys.stdout, tracking=False, setup_logging=True):
+        """
+        Create a new tinfoil object.
+        Parameters:
+            output: specifies where console output should be sent. Defaults
+                    to sys.stdout.
+            tracking: True to enable variable history tracking, False to
+                    disable it (default). Enabling this has a minor
+                    performance impact so typically it isn't enabled
+                    unless you need to query variable history.
+            setup_logging: True to setup a logger so that things like
+                    bb.warn() will work immediately and timeout warnings
+                    are visible; False to let BitBake do this itself.
+        """
         self.logger = logging.getLogger('BitBake')
         self.config_data = None
         self.cooker = None
         self.tracking = tracking
         self.ui_module = None
         self.server_connection = None
+        self.recipes_parsed = False
+        self.quiet = 0
         if setup_logging:
             # This is the *client-side* logger, nothing to do with
             # logging messages from the server
@@ -229,6 +334,39 @@ class Tinfoil:
         self.shutdown()
 
     def prepare(self, config_only=False, config_params=None, quiet=0, extra_features=None):
+        """
+        Prepares the underlying BitBake system to be used via tinfoil.
+        This function must be called prior to calling any of the other
+        functions in the API.
+        NOTE: if you call prepare() you must absolutely call shutdown()
+        before your code terminates. You can use a "with" block to ensure
+        this happens e.g.
+
+            with bb.tinfoil.Tinfoil() as tinfoil:
+                tinfoil.prepare()
+                ...
+
+        Parameters:
+            config_only: True to read only the configuration and not load
+                        the cache / parse recipes. This is useful if you just
+                        want to query the value of a variable at the global
+                        level or you want to do anything else that doesn't
+                        involve knowing anything about the recipes in the
+                        current configuration. False loads the cache / parses
+                        recipes.
+            config_params: optionally specify your own configuration
+                        parameters. If not specified an instance of
+                        TinfoilConfigParameters will be created internally.
+            quiet:      quiet level controlling console output - equivalent
+                        to bitbake's -q/--quiet option. Default of 0 gives
+                        the same output level as normal bitbake execution.
+            extra_features: extra features to be added to the feature
+                        set requested from the server. See
+                        CookerFeatures._feature_list for possible
+                        features.
+        """
+        self.quiet = quiet
+
         if self.tracking:
             extrafeatures = [bb.cooker.CookerFeatures.BASEDATASTORE_TRACKING]
         else:
@@ -243,7 +381,7 @@ class Tinfoil:
         cookerconfig = CookerConfiguration()
         cookerconfig.setConfigParameters(config_params)
 
-        server, self.server_connection, ui_module = setup_bitbake(config_params,
+        self.server_connection, ui_module = setup_bitbake(config_params,
                             cookerconfig,
                             extrafeatures,
                             setup_logging=False)
@@ -270,6 +408,7 @@ class Tinfoil:
                 self.run_command('parseConfiguration')
             else:
                 self.run_actions(config_params)
+                self.recipes_parsed = True
 
             self.config_data = bb.data.init()
             connector = TinfoilDataStoreConnector(self, None)
@@ -289,7 +428,13 @@ class Tinfoil:
 
     def parseRecipes(self):
         """
-        Force a parse of all recipes. Normally you should specify
+        Legacy function - use parse_recipes() instead.
+        """
+        self.parse_recipes()
+
+    def parse_recipes(self):
+        """
+        Load information on all recipes. Normally you should specify
         config_only=False when calling prepare() instead of using this
         function; this function is designed for situations where you need
         to initialise Tinfoil and use it with config_only=True first and
@@ -297,6 +442,7 @@ class Tinfoil:
         """
         config_params = TinfoilConfigParameters(config_only=False)
         self.run_actions(config_params)
+        self.recipes_parsed = True
 
     def run_command(self, command, *params):
         """
@@ -343,9 +489,16 @@ class Tinfoil:
         return self.server_connection.events.waitEvent(timeout)
 
     def get_overlayed_recipes(self):
+        """
+        Find recipes which are overlayed (i.e. where recipes exist in multiple layers)
+        """
         return defaultdict(list, self.run_command('getOverlayedRecipes'))
 
     def get_skipped_recipes(self):
+        """
+        Find recipes which were skipped (i.e. SkipRecipe was raised
+        during parsing).
+        """
         return OrderedDict(self.run_command('getSkippedRecipes'))
 
     def get_all_providers(self):
@@ -378,8 +531,77 @@ class Tinfoil:
         return best[3]
 
     def get_file_appends(self, fn):
+        """
+        Find the bbappends for a recipe file
+        """
         return self.run_command('getFileAppends', fn)
 
+    def all_recipes(self, mc='', sort=True):
+        """
+        Enable iterating over all recipes in the current configuration.
+        Returns an iterator over TinfoilRecipeInfo objects created on demand.
+        Parameters:
+            mc: The multiconfig, default of '' uses the main configuration.
+            sort: True to sort recipes alphabetically (default), False otherwise
+        """
+        recipecache = self.cooker.recipecaches[mc]
+        if sort:
+            recipes = sorted(recipecache.pkg_pn.items())
+        else:
+            recipes = recipecache.pkg_pn.items()
+        for pn, fns in recipes:
+            prov = self.find_best_provider(pn)
+            recipe = TinfoilRecipeInfo(recipecache,
+                                       self.config_data,
+                                       pn=pn,
+                                       fn=prov[3],
+                                       fns=fns)
+            yield recipe
+
+    def all_recipe_files(self, mc='', variants=True, preferred_only=False):
+        """
+        Enable iterating over all recipe files in the current configuration.
+        Returns an iterator over file paths.
+        Parameters:
+            mc: The multiconfig, default of '' uses the main configuration.
+            variants: True to include variants of recipes created through
+                      BBCLASSEXTEND (default) or False to exclude them
+            preferred_only: True to include only the preferred recipe where
+                      multiple exist providing the same PN, False to list
+                      all recipes
+        """
+        recipecache = self.cooker.recipecaches[mc]
+        if preferred_only:
+            files = []
+            for pn in recipecache.pkg_pn.keys():
+                prov = self.find_best_provider(pn)
+                files.append(prov[3])
+        else:
+            files = recipecache.pkg_fn.keys()
+        for fn in sorted(files):
+            if not variants and fn.startswith('virtual:'):
+                continue
+            yield fn
+
+
+    def get_recipe_info(self, pn, mc=''):
+        """
+        Get information on a specific recipe in the current configuration by name (PN).
+        Returns a TinfoilRecipeInfo object created on demand.
+        Parameters:
+            mc: The multiconfig, default of '' uses the main configuration.
+        """
+        recipecache = self.cooker.recipecaches[mc]
+        prov = self.find_best_provider(pn)
+        fn = prov[3]
+        actual_pn = recipecache.pkg_fn[fn]
+        recipe = TinfoilRecipeInfo(recipecache,
+                                    self.config_data,
+                                    pn=actual_pn,
+                                    fn=fn,
+                                    fns=recipecache.pkg_pn[actual_pn])
+        return recipe
+
     def parse_recipe(self, pn):
         """
         Parse the specified recipe and return a datastore object
@@ -415,14 +637,173 @@ class Tinfoil:
         else:
             return None
 
-    def build_file(self, buildfile, task):
+    def build_file(self, buildfile, task, internal=True):
         """
         Runs the specified task for just a single recipe (i.e. no dependencies).
-        This is equivalent to bitbake -b, except no warning will be printed.
+        This is equivalent to bitbake -b, except with the default internal=True
+        no warning about dependencies will be produced, normal info messages
+        from the runqueue will be silenced and BuildInit, BuildStarted and
+        BuildCompleted events will not be fired.
         """
-        return self.run_command('buildFile', buildfile, task, True)
+        return self.run_command('buildFile', buildfile, task, internal)
+
+    def build_targets(self, targets, task=None, handle_events=True, extra_events=None, event_callback=None):
+        """
+        Builds the specified targets. This is equivalent to a normal invocation
+        of bitbake. Has built-in event handling which is enabled by default and
+        can be extended if needed.
+        Parameters:
+            targets:
+                One or more targets to build. Can be a list or a
+                space-separated string.
+            task:
+                The task to run; if None then the value of BB_DEFAULT_TASK
+                will be used. Default None.
+            handle_events:
+                True to handle events in a similar way to normal bitbake
+                invocation with knotty; False to return immediately (on the
+                assumption that the caller will handle the events instead).
+                Default True.
+            extra_events:
+                An optional list of events to add to the event mask (if
+                handle_events=True). If you add events here you also need
+                to specify a callback function in event_callback that will
+                handle the additional events. Default None.
+            event_callback:
+                An optional function taking a single parameter which
+                will be called first upon receiving any event (if
+                handle_events=True) so that the caller can override or
+                extend the event handling. Default None.
+        """
+        if isinstance(targets, str):
+            targets = targets.split()
+        if not task:
+            task = self.config_data.getVar('BB_DEFAULT_TASK')
+
+        if handle_events:
+            # A reasonable set of default events matching up with those we handle below
+            eventmask = [
+                        'bb.event.BuildStarted',
+                        'bb.event.BuildCompleted',
+                        'logging.LogRecord',
+                        'bb.event.NoProvider',
+                        'bb.command.CommandCompleted',
+                        'bb.command.CommandFailed',
+                        'bb.build.TaskStarted',
+                        'bb.build.TaskFailed',
+                        'bb.build.TaskSucceeded',
+                        'bb.build.TaskFailedSilent',
+                        'bb.build.TaskProgress',
+                        'bb.runqueue.runQueueTaskStarted',
+                        'bb.runqueue.sceneQueueTaskStarted',
+                        'bb.event.ProcessStarted',
+                        'bb.event.ProcessProgress',
+                        'bb.event.ProcessFinished',
+                        ]
+            if extra_events:
+                eventmask.extend(extra_events)
+            ret = self.set_event_mask(eventmask)
+
+        ret = self.run_command('buildTargets', targets, task)
+        if handle_events:
+            result = False
+            # Borrowed from knotty, instead somewhat hackily we use the helper
+            # as the object to store "shutdown" on
+            helper = bb.ui.uihelper.BBUIHelper()
+            # We set up logging optionally in the constructor so now we need to
+            # grab the handlers to pass to TerminalFilter
+            console = None
+            errconsole = None
+            for handler in self.logger.handlers:
+                if isinstance(handler, logging.StreamHandler):
+                    if handler.stream == sys.stdout:
+                        console = handler
+                    elif handler.stream == sys.stderr:
+                        errconsole = handler
+            format_str = "%(levelname)s: %(message)s"
+            format = bb.msg.BBLogFormatter(format_str)
+            helper.shutdown = 0
+            parseprogress = None
+            termfilter = bb.ui.knotty.TerminalFilter(helper, helper, console, errconsole, format, quiet=self.quiet)
+            try:
+                while True:
+                    try:
+                        event = self.wait_event(0.25)
+                        if event:
+                            if event_callback and event_callback(event):
+                                continue
+                            if helper.eventHandler(event):
+                                continue
+                            if isinstance(event, bb.event.ProcessStarted):
+                                if self.quiet > 1:
+                                    continue
+                                parseprogress = bb.ui.knotty.new_progress(event.processname, event.total)
+                                parseprogress.start(False)
+                                continue
+                            if isinstance(event, bb.event.ProcessProgress):
+                                if self.quiet > 1:
+                                    continue
+                                if parseprogress:
+                                    parseprogress.update(event.progress)
+                                else:
+                                    bb.warn("Got ProcessProgress event for someting that never started?")
+                                continue
+                            if isinstance(event, bb.event.ProcessFinished):
+                                if self.quiet > 1:
+                                    continue
+                                if parseprogress:
+                                    parseprogress.finish()
+                                parseprogress = None
+                                continue
+                            if isinstance(event, bb.command.CommandCompleted):
+                                result = True
+                                break
+                            if isinstance(event, bb.command.CommandFailed):
+                                self.logger.error(str(event))
+                                result = False
+                                break
+                            if isinstance(event, logging.LogRecord):
+                                if event.taskpid == 0 or event.levelno > logging.INFO:
+                                    self.logger.handle(event)
+                                continue
+                            if isinstance(event, bb.event.NoProvider):
+                                self.logger.error(str(event))
+                                result = False
+                                break
+
+                        elif helper.shutdown > 1:
+                            break
+                        termfilter.updateFooter()
+                    except KeyboardInterrupt:
+                        termfilter.clearFooter()
+                        if helper.shutdown == 1:
+                            print("\nSecond Keyboard Interrupt, stopping...\n")
+                            ret = self.run_command("stateForceShutdown")
+                            if ret and ret[2]:
+                                self.logger.error("Unable to cleanly stop: %s" % ret[2])
+                        elif helper.shutdown == 0:
+                            print("\nKeyboard Interrupt, closing down...\n")
+                            interrupted = True
+                            ret = self.run_command("stateShutdown")
+                            if ret and ret[2]:
+                                self.logger.error("Unable to cleanly shutdown: %s" % ret[2])
+                        helper.shutdown = helper.shutdown + 1
+                termfilter.clearFooter()
+            finally:
+                termfilter.finish()
+            if helper.failed_tasks:
+                result = False
+            return result
+        else:
+            return ret
 
     def shutdown(self):
+        """
+        Shut down tinfoil. Disconnects from the server and gracefully
+        releases any associated resources. You must call this function if
+        prepare() has been called, or use a with... block when you create
+        the tinfoil object which will ensure that it gets called.
+        """
         if self.server_connection:
             self.run_command('clientComplete')
             _server_connections.remove(self.server_connection)

bitbake/lib/bb/ui/buildinfohelper.py

@@ -858,6 +858,12 @@ class MockEvent(object):
         self.pathname = None
         self.lineno = None
 
+    def getMessage(self):
+        """
+        Simulate LogRecord message return
+        """
+        return self.msg
+
 
 class BuildInfoHelper(object):
     """ This class gathers the build information from the server and sends it
@@ -981,6 +987,7 @@ class BuildInfoHelper(object):
             pathRE = pathRE[:-1]
 
         p = re.compile(pathRE)
+        path=re.sub(r'[$^]',r'',pathRE)
         # Heuristics: we always match recipe to the deepest layer path in
         # the discovered layers
         for lvo in sorted(self.orm_wrapper.layer_version_objects,
@@ -990,12 +997,16 @@ class BuildInfoHelper(object):
             if lvo.layer.local_source_dir:
                 if p.fullmatch(os.path.abspath(lvo.layer.local_source_dir)):
                     return lvo
+            if 0 == path.find(lvo.local_path):
+                # sub-layer path inside existing layer
+                return lvo
 
         # if we get here, we didn't read layers correctly;
         # dump whatever information we have on the error log
         logger.warning("Could not match layer dependency for path %s : %s",
                        pathRE,
                        self.orm_wrapper.layer_version_objects)
+        return None
 
     def _get_layer_version_for_path(self, path):
         self._ensure_build()
@@ -1396,9 +1407,9 @@ class BuildInfoHelper(object):
             for lv in event._depgraph['layer-priorities']:
                 (_, path, _, priority) = lv
                 layer_version_obj = self._get_layer_version_for_dependency(path)
-                assert layer_version_obj is not None
-                layer_version_obj.priority = priority
-                layer_version_obj.save()
+                if layer_version_obj:
+                    layer_version_obj.priority = priority
+                    layer_version_obj.save()
 
         # save recipe information
         self.internal_state['recipes'] = {}

bitbake/lib/bb/ui/knotty.py

@@ -207,8 +207,10 @@ class TerminalFilter(object):
             self.interactive = False
             bb.note("Unable to use interactive mode for this terminal, using fallback")
             return
-        console.addFilter(InteractConsoleLogFilter(self, format))
-        errconsole.addFilter(InteractConsoleLogFilter(self, format))
+        if console:
+            console.addFilter(InteractConsoleLogFilter(self, format))
+        if errconsole:
+            errconsole.addFilter(InteractConsoleLogFilter(self, format))
 
         self.main_progress = None
 
@@ -472,11 +474,11 @@ def main(server, eventHandler, params, tf = TerminalFilter):
                         continue
 
                     # Prefix task messages with recipe/task
-                    if event.taskpid in helper.running_tasks:
+                    if event.taskpid in helper.running_tasks and event.levelno != format.PLAIN:
                         taskinfo = helper.running_tasks[event.taskpid]
                         event.msg = taskinfo['title'] + ': ' + event.msg
                 if hasattr(event, 'fn'):
-                        event.msg = event.fn + ': ' + event.msg
+                    event.msg = event.fn + ': ' + event.msg
                 logger.handle(event)
                 continue
 
@@ -560,7 +562,7 @@ def main(server, eventHandler, params, tf = TerminalFilter):
                 return_value = event.exitcode
                 if event.error:
                     errors = errors + 1
-                    logger.error("Command execution failed: %s", event.error)
+                    logger.error(str(event))
                 main.shutdown = 2
                 continue
             if isinstance(event, bb.command.CommandExit):
@@ -571,39 +573,16 @@ def main(server, eventHandler, params, tf = TerminalFilter):
                 main.shutdown = 2
                 continue
             if isinstance(event, bb.event.MultipleProviders):
-                logger.info("multiple providers are available for %s%s (%s)", event._is_runtime and "runtime " or "",
-                            event._item,
-                            ", ".join(event._candidates))
-                rtime = ""
-                if event._is_runtime:
-                    rtime = "R"
-                logger.info("consider defining a PREFERRED_%sPROVIDER entry to match %s" % (rtime, event._item))
+                logger.info(str(event))
                 continue
             if isinstance(event, bb.event.NoProvider):
-                if event._runtime:
-                    r = "R"
-                else:
-                    r = ""
-
-                extra = ''
-                if not event._reasons:
-                    if event._close_matches:
-                        extra = ". Close matches:\n  %s" % '\n  '.join(event._close_matches)
-
                 # For universe builds, only show these as warnings, not errors
-                h = logger.warning
                 if not universe:
                     return_value = 1
                     errors = errors + 1
-                    h = logger.error
-
-                if event._dependees:
-                    h("Nothing %sPROVIDES '%s' (but %s %sDEPENDS on or otherwise requires it)%s", r, event._item, ", ".join(event._dependees), r, extra)
+                    logger.error(str(event))
                 else:
-                    h("Nothing %sPROVIDES '%s'%s", r, event._item, extra)
-                if event._reasons:
-                    for reason in event._reasons:
-                        h("%s", reason)
+                    logger.warning(str(event))
                 continue
 
             if isinstance(event, bb.runqueue.sceneQueueTaskStarted):
@@ -625,13 +604,11 @@ def main(server, eventHandler, params, tf = TerminalFilter):
             if isinstance(event, bb.runqueue.runQueueTaskFailed):
                 return_value = 1
                 taskfailures.append(event.taskstring)
-                logger.error("Task (%s) failed with exit code '%s'",
-                             event.taskstring, event.exitcode)
+                logger.error(str(event))
                 continue
 
             if isinstance(event, bb.runqueue.sceneQueueTaskFailed):
-                logger.warning("Setscene task (%s) failed with exit code '%s' - real task will be run instead",
-                               event.taskstring, event.exitcode)
+                logger.warning(str(event))
                 continue
 
             if isinstance(event, bb.event.DepTreeGenerated):
@@ -664,6 +641,7 @@ def main(server, eventHandler, params, tf = TerminalFilter):
                                   bb.event.MetadataEvent,
                                   bb.event.StampUpdate,
                                   bb.event.ConfigParsed,
+                                  bb.event.MultiConfigParsed,
                                   bb.event.RecipeParsed,
                                   bb.event.RecipePreFinalise,
                                   bb.runqueue.runQueueEvent,

bitbake/lib/bb/ui/ncurses.py

@@ -315,7 +315,7 @@ class NCursesUI:
                     # also allow them to now exit with a single ^C
                     shutdown = 2
                 if isinstance(event, bb.command.CommandFailed):
-                    mw.appendText("Command execution failed: %s" % event.error)
+                    mw.appendText(str(event))
                     time.sleep(2)
                     exitflag = True
                 if isinstance(event, bb.command.CommandExit):

bitbake/lib/bb/ui/taskexp.py

@@ -286,23 +286,7 @@ def main(server, eventHandler, params):
                 continue
 
             if isinstance(event, bb.event.NoProvider):
-                if event._runtime:
-                    r = "R"
-                else:
-                    r = ""
-
-                extra = ''
-                if not event._reasons:
-                    if event._close_matches:
-                        extra = ". Close matches:\n  %s" % '\n  '.join(event._close_matches)
-
-                if event._dependees:
-                    print("Nothing %sPROVIDES '%s' (but %s %sDEPENDS on or otherwise requires it)%s" % (r, event._item, ", ".join(event._dependees), r, extra))
-                else:
-                    print("Nothing %sPROVIDES '%s'%s" % (r, event._item, extra))
-                if event._reasons:
-                    for reason in event._reasons:
-                        print(reason)
+                print(str(event))
 
                 _, error = server.runCommand(["stateShutdown"])
                 if error:
@@ -310,7 +294,7 @@ def main(server, eventHandler, params):
                 break
 
             if isinstance(event, bb.command.CommandFailed):
-                print("Command execution failed: %s" % event.error)
+                print(str(event))
                 return event.exitcode
 
             if isinstance(event, bb.command.CommandExit):

bitbake/lib/bb/ui/toasterui.py

@@ -320,29 +320,13 @@ def main(server, eventHandler, params):
             if isinstance(event, bb.event.CacheLoadCompleted):
                 continue
             if isinstance(event, bb.event.MultipleProviders):
-                logger.info("multiple providers are available for %s%s (%s)", event._is_runtime and "runtime " or "",
-                            event._item,
-                            ", ".join(event._candidates))
-                logger.info("consider defining a PREFERRED_PROVIDER entry to match %s", event._item)
+                logger.info(str(event))
                 continue
 
             if isinstance(event, bb.event.NoProvider):
                 errors = errors + 1
-                if event._runtime:
-                    r = "R"
-                else:
-                    r = ""
-
-                if event._dependees:
-                    text = "Nothing %sPROVIDES '%s' (but %s %sDEPENDS on or otherwise requires it)" % (r, event._item, ", ".join(event._dependees), r)
-                else:
-                    text = "Nothing %sPROVIDES '%s'" % (r, event._item)
-
+                text = str(event)
                 logger.error(text)
-                if event._reasons:
-                    for reason in event._reasons:
-                        logger.error("%s", reason)
-                        text += reason
                 buildinfohelper.store_log_error(text)
                 continue
 
@@ -364,8 +348,7 @@ def main(server, eventHandler, params):
             if isinstance(event, bb.runqueue.runQueueTaskFailed):
                 buildinfohelper.update_and_store_task(event)
                 taskfailures.append(event.taskstring)
-                logger.error("Task (%s) failed with exit code '%s'",
-                             event.taskstring, event.exitcode)
+                logger.error(str(event))
                 continue
 
             if isinstance(event, (bb.runqueue.sceneQueueTaskCompleted, bb.runqueue.sceneQueueTaskFailed)):
@@ -382,7 +365,7 @@ def main(server, eventHandler, params):
                 if isinstance(event, bb.command.CommandFailed):
                     errors += 1
                     errorcode = 1
-                    logger.error("Command execution failed: %s", event.error)
+                    logger.error(str(event))
                 elif isinstance(event, bb.event.BuildCompleted):
                     buildinfohelper.scan_image_artifacts()
                     buildinfohelper.clone_required_sdk_artifacts()

bitbake/lib/bb/ui/uihelper.py

@@ -61,6 +61,9 @@ class BBUIHelper:
                 self.running_tasks[event.pid]['progress'] = event.progress
                 self.running_tasks[event.pid]['rate'] = event.rate
                 self.needUpdate = True
+        else:
+            return False
+        return True
 
     def getTasks(self):
         self.needUpdate = False

bitbake/lib/bb/utils.py

@@ -1502,7 +1502,7 @@ def export_proxies(d):
 
 def load_plugins(logger, plugins, pluginpath):
     def load_plugin(name):
-        logger.debug('Loading plugin %s' % name)
+        logger.debug(1, 'Loading plugin %s' % name)
         fp, pathname, description = imp.find_module(name, [pluginpath])
         try:
             return imp.load_module(name, fp, pathname, description)
@@ -1510,7 +1510,7 @@ def load_plugins(logger, plugins, pluginpath):
             if fp:
                 fp.close()
 
-    logger.debug('Loading plugins from %s...' % pluginpath)
+    logger.debug(1, 'Loading plugins from %s...' % pluginpath)
 
     expanded = (glob.glob(os.path.join(pluginpath, '*' + ext))
                 for ext in python_extensions)

bitbake/lib/prserv/serv.py

@@ -6,10 +6,11 @@ import queue
 import socket
 import io
 import sqlite3
-import bb.server.xmlrpc
+import bb.server.xmlrpcclient
 import prserv
 import prserv.db
 import errno
+import select
 
 logger = logging.getLogger("BitBake.PRserv")
 
@@ -59,6 +60,8 @@ class PRServer(SimpleXMLRPCServer):
         self.register_function(self.importone, "importone")
         self.register_introspection_functions()
 
+        self.quitpipein, self.quitpipeout = os.pipe()
+
         self.requestqueue = queue.Queue()
         self.handlerthread = threading.Thread(target = self.process_request_thread)
         self.handlerthread.daemon = False
@@ -153,11 +156,16 @@ class PRServer(SimpleXMLRPCServer):
 
     def quit(self):
         self.quit=True
+        os.write(self.quitpipeout, b"q")
+        os.close(self.quitpipeout)
         return
 
     def work_forever(self,):
         self.quit = False
-        self.timeout = 0.5
+        # This timeout applies to the poll in TCPServer, we need the select 
+        # below to wake on our quit pipe closing. We only ever call into handle_request
+        # if there is data there.
+        self.timeout = 0.01
 
         bb.utils.set_process_name("PRServ")
 
@@ -170,11 +178,16 @@ class PRServer(SimpleXMLRPCServer):
 
         self.handlerthread.start()
         while not self.quit:
-            self.handle_request()
+            ready = select.select([self.fileno(), self.quitpipein], [], [], 30)
+            if self.quit:
+                break
+            if self.fileno() in ready[0]:
+                self.handle_request()
         self.handlerthread.join()
         self.db.disconnect()
         logger.info("PRServer: stopping...")
         self.server_close()
+        os.close(self.quitpipein)
         return
 
     def start(self):
@@ -182,6 +195,7 @@ class PRServer(SimpleXMLRPCServer):
             pid = self.daemonize()
         else:
             pid = self.fork()
+            self.pid = pid
 
         # Ensure both the parent sees this and the child from the work_forever log entry above
         logger.info("Started PRServer with DBfile: %s, IP: %s, PORT: %s, PID: %s" %
@@ -300,7 +314,7 @@ class PRServerConnection(object):
             host, port = singleton.getinfo()
         self.host = host
         self.port = port
-        self.connection, self.transport = bb.server.xmlrpc._create_server(self.host, self.port)
+        self.connection, self.transport = bb.server.xmlrpcclient._create_server(self.host, self.port)
 
     def terminate(self):
         try:
@@ -472,6 +486,7 @@ def auto_shutdown(d=None):
             PRServerConnection(host, port).terminate()
         except:
             logger.critical("Stop PRService %s:%d failed" % (host,port))
+        os.waitpid(singleton.prserv.pid, 0)
         singleton = None
 
 def ping(host, port):

bitbake/lib/toaster/bldcollector/urls.py

@@ -1,7 +1,7 @@
 #
 # BitBake Toaster Implementation
 #
-# Copyright (C) 2014        Intel Corporation
+# Copyright (C) 2014-2017   Intel Corporation
 #
 # This program is free software; you can redistribute it and/or modify
 # it under the terms of the GNU General Public License version 2 as
@@ -19,7 +19,9 @@
 
 from django.conf.urls import patterns, include, url
 
-urlpatterns = patterns('bldcollector.views',
+import bldcollector.views
+
+urlpatterns = [
         # landing point for pushing a bitbake_eventlog.json file to this toaster instace
-        url(r'^eventfile$', 'eventfile', name='eventfile'),
-        )
+        url(r'^eventfile$', bldcollector.views.eventfile, name='eventfile'),
+]

bitbake/lib/toaster/bldcontrol/bbcontroller.py

@@ -37,8 +37,8 @@ class BitbakeController(object):
     """
 
     def __init__(self, be):
-        import bb.server.xmlrpc
-        self.connection = bb.server.xmlrpc._create_server(be.bbaddress,
+        import bb.server.xmlrpcclient
+        self.connection = bb.server.xmlrpcclient._create_server(be.bbaddress,
                                                           int(be.bbport))[0]
 
     def _runCommand(self, command):

bitbake/lib/toaster/bldcontrol/localhostbecontroller.py

@@ -24,6 +24,7 @@ import os
 import sys
 import re
 import shutil
+import time
 from django.db import transaction
 from django.db.models import Q
 from bldcontrol.models import BuildEnvironment, BRLayer, BRVariable, BRTarget, BRBitbake
@@ -85,6 +86,11 @@ class LocalhostBEController(BuildEnvironmentController):
         return local_checkout_path
 
 
+    def setCloneStatus(self,bitbake,status,total,current):
+        bitbake.req.build.repos_cloned=current
+        bitbake.req.build.repos_to_clone=total
+        bitbake.req.build.save()
+
     def setLayers(self, bitbake, layers, targets):
         """ a word of attention: by convention, the first layer for any build will be poky! """
 
@@ -147,7 +153,13 @@ class LocalhostBEController(BuildEnvironmentController):
         logger.info("Using pre-checked out source for layer %s", cached_layers)
 
         # 3. checkout the repositories
+        clone_count=0
+        clone_total=len(gitrepos.keys())
+        self.setCloneStatus(bitbake,'Started',clone_total,clone_count)
         for giturl, commit in gitrepos.keys():
+            self.setCloneStatus(bitbake,'progress',clone_total,clone_count)
+            clone_count += 1
+
             localdirname = os.path.join(self.be.sourcedir, self.getGitCloneDirectory(giturl, commit))
             logger.debug("localhostbecontroller: giturl %s:%s checking out in current directory %s" % (giturl, commit, localdirname))
 
@@ -198,6 +210,7 @@ class LocalhostBEController(BuildEnvironmentController):
                 if name != "bitbake":
                     layerlist.append(localdirpath.rstrip("/"))
 
+        self.setCloneStatus(bitbake,'complete',clone_total,clone_count)
         logger.debug("localhostbecontroller: current layer list %s " % pformat(layerlist))
 
         # 5. create custom layer and add custom recipes to it
@@ -319,12 +332,22 @@ class LocalhostBEController(BuildEnvironmentController):
         bitbake = os.path.join(self.pokydirname, 'bitbake', 'bin', 'bitbake')
         toasterlayers = os.path.join(builddir,"conf/toaster-bblayers.conf")
         self._shellcmd('bash -c \"source %s %s; BITBAKE_UI="knotty" %s --read %s --read %s '
-                       '--server-only -t xmlrpc -B 0.0.0.0:0\"' % (oe_init,
+                       '--server-only -B 0.0.0.0:0\"' % (oe_init,
                        builddir, bitbake, confpath, toasterlayers), self.be.sourcedir)
 
         # read port number from bitbake.lock
         self.be.bbport = ""
         bblock = os.path.join(builddir, 'bitbake.lock')
+        # allow 10 seconds for bb lock file to appear but also be populated
+        for lock_check in range(10):
+            if not os.path.exists(bblock):
+                logger.debug("localhostbecontroller: waiting for bblock file to appear")
+                time.sleep(1)
+                continue
+            if 10 < os.stat(bblock).st_size:
+                break
+            logger.debug("localhostbecontroller: waiting for bblock content to appear")
+            time.sleep(1)
         with open(bblock) as fplock:
             for line in fplock:
                 if ":" in line:
@@ -353,10 +376,10 @@ class LocalhostBEController(BuildEnvironmentController):
         log = os.path.join(builddir, 'toaster_ui.log')
         local_bitbake = os.path.join(os.path.dirname(os.getenv('BBBASEDIR')),
                                      'bitbake')
-        self._shellcmd(['bash -c \"(TOASTER_BRBE="%s" BBSERVER="0.0.0.0:-1" '
+        self._shellcmd(['bash -c \"(TOASTER_BRBE="%s" BBSERVER="0.0.0.0:%s" '
                         '%s %s -u toasterui --token="" >>%s 2>&1;'
-                        'BITBAKE_UI="knotty" BBSERVER=0.0.0.0:-1 %s -m)&\"' \
-                        % (brbe, local_bitbake, bbtargets, log, bitbake)],
+                        'BITBAKE_UI="knotty" BBSERVER=0.0.0.0:%s %s -m)&\"' \
+                        % (brbe, self.be.bbport, local_bitbake, bbtargets, log, self.be.bbport, bitbake)],
                         builddir, nowait=True)
 
         logger.debug('localhostbecontroller: Build launched, exiting. '

bitbake/lib/toaster/bldcontrol/management/commands/checksettings.py

@@ -1,4 +1,4 @@
-from django.core.management.base import NoArgsCommand, CommandError
+from django.core.management.base import BaseCommand, CommandError
 from django.db import transaction
 
 from django.core.management import call_command
@@ -18,7 +18,7 @@ def DN(path):
         return os.path.dirname(path)
 
 
-class Command(NoArgsCommand):
+class Command(BaseCommand):
     args = ""
     help = "Verifies that the configured settings are valid and usable, or prompts the user to fix the settings."
 
@@ -152,7 +152,7 @@ class Command(NoArgsCommand):
 
 
 
-    def handle_noargs(self, **options):
+    def handle(self, **options):
         retval = 0
         retval += self._verify_build_environment()
         retval += self._verify_default_settings()

bitbake/lib/toaster/bldcontrol/management/commands/runbuilds.py

@@ -1,4 +1,4 @@
-from django.core.management.base import NoArgsCommand
+from django.core.management.base import BaseCommand
 from django.db import transaction
 from django.db.models import Q
 
@@ -16,7 +16,7 @@ import os
 logger = logging.getLogger("toaster")
 
 
-class Command(NoArgsCommand):
+class Command(BaseCommand):
     args = ""
     help = "Schedules and executes build requests as possible. "\
            "Does not return (interrupt with Ctrl-C)"
@@ -168,7 +168,7 @@ class Command(NoArgsCommand):
         except Exception as e:
             logger.warn("runbuilds: schedule exception %s" % str(e))
 
-    def handle_noargs(self, **options):
+    def handle(self, **options):
         pidfile_path = os.path.join(os.environ.get("BUILDDIR", "."),
                                     ".runbuilds.pid")
 

bitbake/lib/toaster/orm/management/commands/lsupdates.py

@@ -4,7 +4,7 @@
 #
 # BitBake Toaster Implementation
 #
-# Copyright (C) 2016        Intel Corporation
+# Copyright (C) 2016-2017   Intel Corporation
 #
 # This program is free software; you can redistribute it and/or modify
 # it under the terms of the GNU General Public License version 2 as
@@ -19,10 +19,11 @@
 # with this program; if not, write to the Free Software Foundation, Inc.,
 # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
 
-from django.core.management.base import NoArgsCommand
+from django.core.management.base import BaseCommand
 
 from orm.models import LayerSource, Layer, Release, Layer_Version
 from orm.models import LayerVersionDependency, Machine, Recipe
+from orm.models import Distro
 
 import os
 import sys
@@ -56,7 +57,7 @@ class Spinner(threading.Thread):
         self.signal = False
 
 
-class Command(NoArgsCommand):
+class Command(BaseCommand):
     args = ""
     help = "Updates locally cached information from a layerindex server"
 
@@ -249,6 +250,24 @@ class Command(NoArgsCommand):
                                                              depends_on=lvd)
             self.mini_progress("Layer version dependencies", i, total)
 
+        # update Distros
+        logger.info("Fetching distro information")
+        distros_info = _get_json_response(
+            apilinks['distros'] + "?filter=layerbranch__branch__name:%s" %
+            "OR".join(whitelist_branch_names))
+
+        total = len(distros_info)
+        for i, di in enumerate(distros_info):
+            distro, created = Distro.objects.get_or_create(
+                name=di['name'],
+                layer_version=Layer_Version.objects.get(
+                    pk=li_layer_branch_id_to_toaster_lv_id[di['layerbranch']]))
+            distro.up_date = di['updated']
+            distro.name = di['name']
+            distro.description = di['description']
+            distro.save()
+            self.mini_progress("distros", i, total)
+
         # update machines
         logger.info("Fetching machine information")
         machines_info = _get_json_response(
@@ -307,5 +326,5 @@ class Command(NoArgsCommand):
 
         os.system('setterm -cursor on')
 
-    def handle_noargs(self, **options):
+    def handle(self, **options):
         self.update()

bitbake/lib/toaster/orm/migrations/0016_clone_progress.py

@@ -0,0 +1,24 @@
+# -*- coding: utf-8 -*-
+from __future__ import unicode_literals
+
+from django.db import migrations, models
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('orm', '0015_layer_local_source_dir'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='build',
+            name='repos_cloned',
+            field=models.IntegerField(default=1),
+        ),
+        migrations.AddField(
+            model_name='build',
+            name='repos_to_clone',
+            field=models.IntegerField(default=1), # (default off)
+        ),
+    ]
+

bitbake/lib/toaster/orm/migrations/0017_distro_clone.py

@@ -0,0 +1,25 @@
+# -*- coding: utf-8 -*-
+from __future__ import unicode_literals
+
+from django.db import migrations, models
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('orm', '0016_clone_progress'),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name='Distro',
+            fields=[
+                ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)),
+                ('up_id', models.IntegerField(default=None, null=True)),
+                ('up_date', models.DateTimeField(default=None, null=True)),
+                ('name', models.CharField(max_length=255)),
+                ('description', models.CharField(max_length=255)),
+                ('layer_version', models.ForeignKey(to='orm.Layer_Version')),
+            ],
+        ),
+    ]
+

bitbake/lib/toaster/orm/models.py

@@ -225,16 +225,16 @@ class Project(models.Model):
             return( -1 )
 
     def get_last_outcome(self):
-        build_id = self.get_last_build_id
+        build_id = self.get_last_build_id()
         if (-1 == build_id):
             return( "" )
         try:
-            return Build.objects.filter( id = self.get_last_build_id )[ 0 ].outcome
+            return Build.objects.filter( id = build_id )[ 0 ].outcome
         except (Build.DoesNotExist,IndexError):
             return( "not_found" )
 
     def get_last_target(self):
-        build_id = self.get_last_build_id
+        build_id = self.get_last_build_id()
         if (-1 == build_id):
             return( "" )
         try:
@@ -243,7 +243,7 @@ class Project(models.Model):
             return( "not_found" )
 
     def get_last_errors(self):
-        build_id = self.get_last_build_id
+        build_id = self.get_last_build_id()
         if (-1 == build_id):
             return( 0 )
         try:
@@ -252,7 +252,7 @@ class Project(models.Model):
             return( "not_found" )
 
     def get_last_warnings(self):
-        build_id = self.get_last_build_id
+        build_id = self.get_last_build_id()
         if (-1 == build_id):
             return( 0 )
         try:
@@ -269,7 +269,7 @@ class Project(models.Model):
         return last_build.get_image_file_extensions()
 
     def get_last_imgfiles(self):
-        build_id = self.get_last_build_id
+        build_id = self.get_last_build_id()
         if (-1 == build_id):
             return( "" )
         try:
@@ -321,6 +321,22 @@ class Project(models.Model):
 
         return queryset
 
+    def get_available_distros(self):
+        """ Returns QuerySet of all Distros which are provided by the
+        Layers currently added to the Project """
+        queryset = Distro.objects.filter(
+            layer_version__in=self.get_project_layer_versions())
+
+        return queryset
+
+    def get_all_compatible_distros(self):
+        """ Returns QuerySet of all the compatible Wind River distros available to the
+        project including ones from Layers not currently added """
+        queryset = Distro.objects.filter(
+            layer_version__in=self.get_all_compatible_layer_versions())
+
+        return queryset
+
     def get_available_recipes(self):
         """ Returns QuerySet of all the recipes that are provided by layers
         added to this project """
@@ -435,7 +451,13 @@ class Build(models.Model):
     recipes_to_parse = models.IntegerField(default=1)
 
     # number of recipes parsed so far for this build
-    recipes_parsed = models.IntegerField(default=0)
+    recipes_parsed = models.IntegerField(default=1)
+
+    # number of repos to clone for this build
+    repos_to_clone = models.IntegerField(default=1)
+
+    # number of repos cloned so far for this build (default off)
+    repos_cloned = models.IntegerField(default=1)
 
     @staticmethod
     def get_recent(project=None):
@@ -486,7 +508,7 @@ class Build(models.Model):
         tf = Task.objects.filter(build = self)
         tfc = tf.count()
         if tfc > 0:
-            completeper = tf.exclude(order__isnull=True).count()*100 // tfc
+            completeper = tf.exclude(outcome=Task.OUTCOME_NA).count()*100 // tfc
         else:
             completeper = 0
         return completeper
@@ -667,6 +689,13 @@ class Build(models.Model):
         else:
             return False
 
+    def is_cloning(self):
+        """
+        True if the build is still cloning repos
+        """
+        return self.outcome == Build.IN_PROGRESS and \
+            self.repos_cloned < self.repos_to_clone
+
     def is_parsing(self):
         """
         True if the build is still parsing recipes
@@ -680,10 +709,11 @@ class Build(models.Model):
         tasks.
 
         Note that the mechanism for testing whether a Task is "done" is whether
-        its order field is set, as per the completeper() method.
+        its outcome field is set, as per the completeper() method.
         """
         return self.outcome == Build.IN_PROGRESS and \
-            self.task_build.filter(order__isnull=False).count() == 0
+            self.task_build.exclude(outcome=Task.OUTCOME_NA).count() == 0
+
 
     def get_state(self):
         """
@@ -698,6 +728,8 @@ class Build(models.Model):
             return 'Cancelling';
         elif self.is_queued():
             return 'Queued'
+        elif self.is_cloning():
+            return 'Cloning'
         elif self.is_parsing():
             return 'Parsing'
         elif self.is_starting():
@@ -1485,12 +1517,12 @@ class Layer_Version(models.Model):
         return self._handle_url_path(self.layer.vcs_web_tree_base_url, '')
 
     def get_vcs_reference(self):
+        if self.commit is not None and len(self.commit) > 0:
+            return self.commit
         if self.branch is not None and len(self.branch) > 0:
             return self.branch
         if self.release is not None:
             return self.release.name
-        if self.commit is not None and len(self.commit) > 0:
-            return self.commit
         return 'N/A'
 
     def get_detailspage_url(self, project_id=None):
@@ -1780,6 +1812,21 @@ def signal_runbuilds():
     except FileNotFoundError:
         logger.info("Stopping existing runbuilds: no current process found")
 
+class Distro(models.Model):
+    search_allowed_fields = ["name", "description", "layer_version__layer__name"]
+    up_date = models.DateTimeField(null = True, default = None)
+
+    layer_version = models.ForeignKey('Layer_Version')
+    name = models.CharField(max_length=255)
+    description = models.CharField(max_length=255)
+
+    def get_vcs_distro_file_link_url(self):
+        path = self.name+'.conf'
+        return self.layer_version.get_vcs_file_link_url(path)
+
+    def __unicode__(self):
+        return "Distro " + self.name + "(" + self.description + ")"
+
 django.db.models.signals.post_save.connect(invalidate_cache)
 django.db.models.signals.post_delete.connect(invalidate_cache)
 django.db.models.signals.m2m_changed.connect(invalidate_cache)

bitbake/lib/toaster/toastergui/api.py

@@ -874,6 +874,12 @@ class XhrProject(View):
             machinevar.value = request.POST['machineName']
             machinevar.save()
 
+        # Distro name change
+        if 'distroName' in request.POST:
+            distrovar = prj.projectvariable_set.get(name="DISTRO")
+            distrovar.value = request.POST['distroName']
+            distrovar.save()
+
         return JsonResponse({"error": "ok"})
 
     def get(self, request, *args, **kwargs):
@@ -960,10 +966,11 @@ class XhrProject(View):
         except ProjectVariable.DoesNotExist:
             data["machine"] = None
         try:
-            data["distro"] = project.projectvariable_set.get(
-                name="DISTRO").value
+            data["distro"] = {"name":
+                               project.projectvariable_set.get(
+                                   name="DISTRO").value}
         except ProjectVariable.DoesNotExist:
-            data["distro"] = "-- not set yet"
+            data["distro"] = None
 
         data['error'] = "ok"
 

bitbake/lib/toaster/toastergui/static/js/mrbsection.js

@@ -61,6 +61,12 @@ function mrbSectionInit(ctx){
     return (cached.recipes_parsed_percentage !== build.recipes_parsed_percentage);
   }
 
+  // returns true if the number of repos cloned/to clone changed
+  function cloneProgressChanged(build) {
+    var cached = getCached(build);
+    return (cached.repos_cloned_percentage !== build.repos_cloned_percentage);
+  }
+
   function refreshMostRecentBuilds(){
     libtoaster.getMostRecentBuilds(
       libtoaster.ctx.mostRecentBuildsUrl,
@@ -100,6 +106,15 @@ function mrbSectionInit(ctx){
 
             container.html(html);
           }
+          else if (cloneProgressChanged(build)) {
+            // update the clone progress text
+            selector = '#repos-cloned-percentage-' + build.id;
+            $(selector).html(build.repos_cloned_percentage);
+
+            // update the recipe progress bar
+            selector = '#repos-cloned-percentage-bar-' + build.id;
+            $(selector).width(build.repos_cloned_percentage + '%');
+          }
           else if (tasksProgressChanged(build)) {
             // update the task progress text
             selector = '#build-pc-done-' + build.id;

bitbake/lib/toaster/toastergui/static/js/projectpage.js

@@ -15,6 +15,13 @@ function projectPageInit(ctx) {
   var machineInputForm = $("#machine-input-form");
   var invalidMachineNameHelp = $("#invalid-machine-name-help");
 
+  var distroChangeInput = $("#distro-change-input");
+  var distroChangeBtn = $("#distro-change-btn");
+  var distroForm = $("#select-distro-form");
+  var distroChangeFormToggle = $("#change-distro-toggle");
+  var distroNameTitle = $("#project-distro-name");
+  var distroChangeCancel = $("#cancel-distro-change");
+
   var freqBuildBtn =  $("#freq-build-btn");
   var freqBuildList = $("#freq-build-list");
 
@@ -26,6 +33,7 @@ function projectPageInit(ctx) {
 
   var currentLayerAddSelection;
   var currentMachineAddSelection = "";
+  var currentDistroAddSelection = "";
 
   var urlParams = libtoaster.parseUrlParams();
 
@@ -45,6 +53,17 @@ function projectPageInit(ctx) {
       updateMachineName(prjInfo.machine.name);
     }
 
+    /* If we're receiving a distro set from the url and it's different from
+     * our current distro then activate set machine sequence.
+     */
+    if (urlParams.hasOwnProperty('setDistro') &&
+        urlParams.setDistro !== prjInfo.distro.name){
+        distroChangeInput.val(urlParams.setDistro);
+        distroChangeBtn.click();
+    } else {
+      updateDistroName(prjInfo.distro.name);
+    }
+
    /* Now we're really ready show the page */
     $("#project-page").show();
 
@@ -278,6 +297,60 @@ function projectPageInit(ctx) {
   });
 
 
+  /* Change distro functionality */
+
+  distroChangeFormToggle.click(function(){
+    distroForm.slideDown();
+    distroNameTitle.hide();
+    $(this).hide();
+  });
+
+  distroChangeCancel.click(function(){
+    distroForm.slideUp(function(){
+      distroNameTitle.show();
+      distroChangeFormToggle.show();
+    });
+  });
+
+  function updateDistroName(distroName){
+    distroChangeInput.val(distroName);
+    distroNameTitle.text(distroName);
+  }
+
+  libtoaster.makeTypeahead(distroChangeInput,
+                           libtoaster.ctx.distrosTypeAheadUrl,
+                           { }, function(item){
+    currentDistroAddSelection = item.name;
+    distroChangeBtn.removeAttr("disabled");
+  });
+
+  distroChangeBtn.click(function(e){
+    e.preventDefault();
+    /* We accept any value regardless of typeahead selection or not */
+    if (distroChangeInput.val().length === 0)
+      return;
+
+    currentDistroAddSelection = distroChangeInput.val();
+
+    libtoaster.editCurrentProject(
+      { distroName : currentDistroAddSelection },
+      function(){
+        /* Success machine changed */
+        updateDistroName(currentDistroAddSelection);
+        distroChangeCancel.click();
+
+        /* Show the alert message */
+        var message = $('<span>You have changed the distro to: <strong><span id="notify-machine-name"></span></strong></span>');
+        message.find("#notify-machine-name").text(currentDistroAddSelection);
+        libtoaster.showChangeNotification(message);
+    },
+      function(){
+        /* Failed machine changed */
+        console.warn("Failed to change distro");
+    });
+  });
+
+
   /* Change release functionality */
   function updateProjectRelease(release){
     releaseTitle.text(release.description);

bitbake/lib/toaster/toastergui/static/js/projecttopbar.js

@@ -73,14 +73,14 @@ function projectTopBarInit(ctx) {
 
   newBuildTargetBuildBtn.click(function (e) {
     e.preventDefault();
-    if (!newBuildTargetInput.val()) {
+    if (!newBuildTargetInput.val().trim()) {
       return;
     }
     /* We use the value of the input field so as to maintain any command also
      * added e.g. core-image-minimal:clean and because we can build targets
      * that toaster doesn't yet know about
      */
-    selectedTarget = { name: newBuildTargetInput.val() };
+    selectedTarget = { name: newBuildTargetInput.val().trim() };
 
     /* Fire off the build */
     libtoaster.startABuild(null, selectedTarget.name,

bitbake/lib/toaster/toastergui/tables.py

@@ -23,6 +23,7 @@ from toastergui.widgets import ToasterTable
 from orm.models import Recipe, ProjectLayer, Layer_Version, Machine, Project
 from orm.models import CustomImageRecipe, Package, Target, Build, LogMessage, Task
 from orm.models import CustomImagePackage, Package_DependencyManager
+from orm.models import Distro
 from django.db.models import Q, Max, Sum, Count, When, Case, Value, IntegerField
 from django.conf.urls import url
 from django.core.urlresolvers import reverse, resolve
@@ -1536,3 +1537,93 @@ class ProjectBuildsTable(BuildsTable):
             context['build_in_progress_none_completed'] = False
 
         return context
+
+
+class DistrosTable(ToasterTable):
+    """Table of Distros in Toaster"""
+
+    def __init__(self, *args, **kwargs):
+        super(DistrosTable, self).__init__(*args, **kwargs)
+        self.empty_state = "Toaster has no distro information for this project. Sadly, 			   distro information cannot be obtained from builds, so this 				  page will remain empty."
+        self.title = "Compatible Distros"
+        self.default_orderby = "name"
+
+    def get_context_data(self, **kwargs):
+        context = super(DistrosTable, self).get_context_data(**kwargs)
+        context['project'] = Project.objects.get(pk=kwargs['pid'])
+        return context
+
+    def setup_filters(self, *args, **kwargs):
+        project = Project.objects.get(pk=kwargs['pid'])
+
+        in_current_project_filter = TableFilter(
+            "in_current_project",
+            "Filter by project Distros"
+        )
+
+        in_project_action = TableFilterActionToggle(
+            "in_project",
+            "Distro provided by layers added to this project",
+            ProjectFilters.in_project(self.project_layers)
+        )
+
+        not_in_project_action = TableFilterActionToggle(
+            "not_in_project",
+            "Distros provided by layers not added to this project",
+            ProjectFilters.not_in_project(self.project_layers)
+        )
+
+        in_current_project_filter.add_action(in_project_action)
+        in_current_project_filter.add_action(not_in_project_action)
+        self.add_filter(in_current_project_filter)
+
+    def setup_queryset(self, *args, **kwargs):
+        prj = Project.objects.get(pk = kwargs['pid'])
+        self.queryset = prj.get_all_compatible_distros()
+        self.queryset = self.queryset.order_by(self.default_orderby)
+
+        self.static_context_extra['current_layers'] = \
+                self.project_layers = \
+                prj.get_project_layer_versions(pk=True)
+
+    def setup_columns(self, *args, **kwargs):
+
+        self.add_column(title="Distro",
+                        hideable=False,
+                        orderable=True,
+                        field_name="name")
+
+        self.add_column(title="Description",
+                        field_name="description")
+
+        layer_link_template = '''
+        <a href="{% url 'layerdetails' extra.pid data.layer_version.id %}">
+        {{data.layer_version.layer.name}}</a>
+        '''
+
+        self.add_column(title="Layer",
+                        static_data_name="layer_version__layer__name",
+                        static_data_template=layer_link_template,
+                        orderable=True)
+
+        self.add_column(title="Git revision",
+                        help_text="The Git branch, tag or commit. For the layers from the OpenEmbedded layer source, the revision is always the branch compatible with the Yocto Project version you selected for this project",
+                        hidden=True,
+                        field_name="layer_version__get_vcs_reference")
+
+        wrtemplate_file_template = '''<code>conf/machine/{{data.name}}.conf</code>
+        <a href="{{data.get_vcs_machine_file_link_url}}" target="_blank"><span class="glyphicon glyphicon-new-window"></i></a>'''
+
+        self.add_column(title="Distro file",
+                        hidden=True,
+                        static_data_name="templatefile",
+                        static_data_template=wrtemplate_file_template)
+
+
+        self.add_column(title="Select",
+                        help_text="Sets the selected distro to the project",
+                        hideable=False,
+                        filter_name="in_current_project",
+                        static_data_name="add-del-layers",
+                        static_data_template='{% include "distro_btn.html" %}')
+

bitbake/lib/toaster/toastergui/templates/base.html

@@ -49,6 +49,7 @@
         recipesTypeAheadUrl: {% url 'xhr_recipestypeahead' project.id as paturl%}{{paturl|json}},
         layersTypeAheadUrl: {% url 'xhr_layerstypeahead' project.id as paturl%}{{paturl|json}},
         machinesTypeAheadUrl: {% url 'xhr_machinestypeahead' project.id as paturl%}{{paturl|json}},
+        distrosTypeAheadUrl: {% url 'xhr_distrostypeahead' project.id as paturl%}{{paturl|json}},
         projectBuildsUrl: {% url 'projectbuilds' project.id as pburl %}{{pburl|json}},
         xhrCustomRecipeUrl : "{% url 'xhr_customrecipe' %}",
         projectId : {{project.id}},
@@ -100,7 +101,8 @@
         <div class="collapse navbar-collapse" id="global-nav">
             <ul class="nav navbar-nav">
             {% if request.resolver_match.url_name != 'landing' and request.resolver_match.url_name != 'newproject' %}
-              <li {% if request.resolver_match.url_name == 'all-builds' %}
+              <li id="navbar-all-builds"
+              {% if request.resolver_match.url_name == 'all-builds' %}
               class="active"
               {% endif %}>
               <a href="{% url 'all-builds' %}">
@@ -108,7 +110,8 @@
                 All builds
               </a>
               </li>
-              <li {% if request.resolver_match.url_name == 'all-projects'  %}
+              <li id="navbar-all-projects"
+              {% if request.resolver_match.url_name == 'all-projects'  %}
               class="active"
               {% endif %}>
               <a href="{% url 'all-projects' %}">
@@ -117,7 +120,7 @@
               </a>
               </li>
             {% endif %}
-              <li>
+              <li id="navbar-docs">
               <a target="_blank" href="http://www.yoctoproject.org/docs/latest/toaster-manual/toaster-manual.html">
                 <i class="glyphicon glyphicon-book"></i>
                 Documentation

bitbake/lib/toaster/toastergui/templates/basebuildpage.html

@@ -132,16 +132,16 @@
           {% endif %}
           <li class="nav-header">Build</li>
             <li id="menu-configuration"><a href="{% url 'configuration' build.pk %}">Configuration</a></li>
-            <li><a href="{% url 'tasks' build.pk %}">Tasks</a></li>
-            <li><a href="{% url 'recipes' build.pk %}">Recipes</a></li>
-            <li><a href="{% url 'packages' build.pk %}">Packages</a></li>
+            <li id="menu-tasks"><a href="{% url 'tasks' build.pk %}">Tasks</a></li>
+            <li id="menu-recipes"><a href="{% url 'recipes' build.pk %}">Recipes</a></li>
+            <li id="menu-packages"><a href="{% url 'packages' build.pk %}">Packages</a></li>
             <li class="nav-header">Performance</li>
-            <li><a href="{% url 'buildtime' build.pk %}">Time</a></li>
-            <li><a href="{% url 'cputime' build.pk %}">CPU usage</a></li>
-            <li><a href="{% url 'diskio' build.pk %}">Disk I/O</a></li>
+            <li id="menu-time"><a href="{% url 'buildtime' build.pk %}">Time</a></li>
+            <li id="menu-cpu-time"><a href="{% url 'cputime' build.pk %}">CPU usage</a></li>
+            <li id="menu-disk-io"><a href="{% url 'diskio' build.pk %}">Disk I/O</a></li>
 
           <li class="nav-header">Actions</li>
-          <li>
+          <li id="menu-download-build-log">
             <a href="{% url 'build_artifact' build.id 'cookerlog' build.id %}">
               <span class="glyphicon glyphicon-download-alt"></span>
             Download build log
@@ -151,7 +151,7 @@
           {% with build.get_custom_image_recipes as custom_image_recipes %}
             {% if custom_image_recipes.count > 0 %}
               <!-- edit custom image built during this build -->
-              <li>
+              <li id="menu-edit-custom-image">
                 <a href="#" data-role="edit-custom-image-trigger">
                   <span class="glyphicon glyphicon-edit"></span>
                   Edit custom image
@@ -186,7 +186,7 @@
 
             <!-- new custom image from image recipe in this build -->
             {% if build.has_image_recipes %}
-            <li>
+            <li id="menu-new-custom-image">
               <a href="#"  data-role="new-custom-image-trigger">
                 <span class="glyphicon glyphicon-plus"></span>
                 New custom image
@@ -216,7 +216,7 @@
               </script>
             {% endif %}
 
-            <li>
+            <li  id="menu-delete-build">
             <a href="#delete-build-modal" id="delete-build" data-toggle="modal" data-target="#delete-build-modal" class="text-danger">
               <span class="glyphicon glyphicon-trash"></span>
               Delete build

bitbake/lib/toaster/toastergui/templates/baseprojectpage.html

@@ -32,6 +32,7 @@ $(document).ready(function(){
       <li><a href="{% url 'projectsoftwarerecipes' project.id %}">Software recipes</a></li>
       <li><a href="{% url 'projectmachines' project.id %}">Machines</a></li>
       <li><a href="{% url 'projectlayers' project.id %}">Layers</a></li>
+      <li><a href="{% url 'projectdistros' project.id %}">Distros</a></li>
       <li class="nav-header">Extra configuration</li>
       <li><a href="{% url 'projectconf' project.id %}">BitBake variables</a></li>
 

bitbake/lib/toaster/toastergui/templates/distro_btn.html

@@ -0,0 +1,20 @@
+<a href="{% url 'project' extra.pid %}?setDistro={{data.name}}" class="btn btn-default btn-block layer-exists-{{data.layer_version.id}}"
+    {% if data.layer_version.pk not in extra.current_layers %}
+    style="display:none;"
+    {% endif %}>
+  Set distro</a>
+<a class="btn btn-default btn-block layerbtn layer-add-{{data.layer_version.id}}" data-layer='{
+    "id": {{data.layer_version.id}},
+    "name":  "{{data.layer_version.layer.name}}",
+    "xhrLayerUrl": "{% url "xhr_layer" extra.pid data.pk %}",
+    "layerdetailurl": "{%url 'layerdetails' extra.pid data.layer_version.id %}"
+    }' data-directive="add"
+    {% if data.layer_version.pk in extra.current_layers %}
+    style="display:none;"
+    {% endif %}
+>
+  <span class="glyphicon glyphicon-plus"></span>
+  Add layer
+  <span class="glyphicon glyphicon-question-sign get-help" title="To select this distro, you must first add the {{data.layer_version.layer.name}} layer to your project"></i>
+</a>
+

bitbake/lib/toaster/toastergui/templates/health.html

@@ -0,0 +1,6 @@
+<!DOCTYPE html>
+<html lang="en">
+  <head><title>Toaster Health</title></head>
+  <body>Ok</body>
+</html>
+

bitbake/lib/toaster/toastergui/templates/mrb_section.html

@@ -64,7 +64,9 @@
   </div>
 
   <div data-build-state="<%:state%>">
-    <%if state == 'Parsing'%>
+    <%if state == 'Cloning'%>
+      <%include tmpl='#cloning-repos-build-template'/%>
+    <%else state == 'Parsing'%>
       <%include tmpl='#parsing-recipes-build-template'/%>
     <%else state == 'Queued'%>
       <%include tmpl='#queued-build-template'/%>
@@ -98,6 +100,31 @@
   </div>
 </script>
 
+<!-- cloning repos build -->
+<script id="cloning-repos-build-template" type="text/x-jsrender">
+  <!-- progress bar and parse completion percentage -->
+  <div data-role="build-status" class="col-md-4 col-md-offset-1 progress-info">
+    <!-- progress bar -->
+    <div class="progress">
+      <div id="repos-cloned-percentage-bar-<%:id%>"
+           style="width: <%:repos_cloned_percentage%>%;"
+           class="progress-bar">
+      </div>
+    </div>
+  </div>
+
+  <div class="col-md-4 progress-info">
+    <!-- parse completion percentage -->
+    <span class="glyphicon glyphicon-question-sign get-help get-help-blue"
+          title="Toaster is cloning the repos required for your build">
+    </span>
+
+    Cloning <span id="repos-cloned-percentage-<%:id%>"><%:repos_cloned_percentage%></span>% complete
+
+    <%include tmpl='#cancel-template'/%>
+  </div>
+</script>
+
 <!-- parsing recipes build -->
 <script id="parsing-recipes-build-template" type="text/x-jsrender">
   <!-- progress bar and parse completion percentage -->

bitbake/lib/toaster/toastergui/templates/project.html

@@ -77,6 +77,22 @@
       </form>
     </div>
 
+    <div class="well well-transparent" id="distro-section">
+      <h3>Distro</h3>
+
+      <p class="lead"><span id="project-distro-name"></span> <span class="glyphicon glyphicon-edit" id="change-distro-toggle"></span></p>
+
+      <form id="select-distro-form" style="display:none;" class="form-inline">
+        <span class="help-block">Distro suggestions come from the Layer Index</a></span>
+        <div class="form-group">
+          <input class="form-control" id="distro-change-input" autocomplete="off" value="" data-provide="typeahead" data-minlength="1" data-autocomplete="off" type="text">
+        </div>
+        <button id="distro-change-btn" class="btn btn-default" type="button">Save</button>
+        <a href="#" id="cancel-distro-change" class="btn btn-link">Cancel</a>
+        <p class="form-link"><a href="{% url 'projectdistros' project.id %}">View compatible distros</a></p>
+      </form>
+    </div>
+
     <div class="well well-transparent">
       <h3>Most built recipes</h3>
 

bitbake/lib/toaster/toastergui/typeaheads.py

@@ -100,6 +100,36 @@ class MachinesTypeAhead(ToasterTypeAhead):
         return results
 
 
+class DistrosTypeAhead(ToasterTypeAhead):
+    """ Typeahead for all the distros available in the current project's
+    configuration """
+    def __init__(self):
+        super(DistrosTypeAhead, self).__init__()
+
+    def apply_search(self, search_term, prj, request):
+        distros = prj.get_available_distros()
+        distros = distros.order_by("name")
+
+        primary_results = distros.filter(name__istartswith=search_term)
+        secondary_results = distros.filter(name__icontains=search_term).exclude(pk__in=primary_results)
+        tertiary_results = distros.filter(layer_version__layer__name__icontains=search_term).exclude(pk__in=primary_results).exclude(pk__in=secondary_results)
+
+        results = []
+
+        for distro in list(primary_results) + list(secondary_results) + list(tertiary_results):
+
+            detail = "[ %s ]" % (distro.layer_version.layer.name)
+            needed_fields = {
+                'id' : distro.pk,
+                'name' : distro.name,
+                'detail' : detail,
+            }
+
+            results.append(needed_fields)
+
+        return results
+
+
 class RecipesTypeAhead(ToasterTypeAhead):
     """ Typeahead for all the recipes available in the current project's
     configuration """

bitbake/lib/toaster/toastergui/urls.py

@@ -1,7 +1,7 @@
 #
 # BitBake Toaster Implementation
 #
-# Copyright (C) 2013        Intel Corporation
+# Copyright (C) 2013-2017    Intel Corporation
 #
 # This program is free software; you can redistribute it and/or modify
 # it under the terms of the GNU General Public License version 2 as
@@ -25,49 +25,50 @@ from toastergui import buildtables
 from toastergui import typeaheads
 from toastergui import api
 from toastergui import widgets
+from toastergui import views
 
-urlpatterns = patterns('toastergui.views',
+urlpatterns = [
         # landing page
-        url(r'^landing/$', 'landing', name='landing'),
+        url(r'^landing/$', views.landing, name='landing'),
 
         url(r'^builds/$',
             tables.AllBuildsTable.as_view(template_name="builds-toastertable.html"),
             name='all-builds'),
 
         # build info navigation
-        url(r'^build/(?P<build_id>\d+)$', 'builddashboard', name="builddashboard"),
+        url(r'^build/(?P<build_id>\d+)$', views.builddashboard, name="builddashboard"),
         url(r'^build/(?P<build_id>\d+)/tasks/$',
             buildtables.BuildTasksTable.as_view(
                 template_name="buildinfo-toastertable.html"),
             name='tasks'),
 
-        url(r'^build/(?P<build_id>\d+)/task/(?P<task_id>\d+)$', 'task', name='task'),
+        url(r'^build/(?P<build_id>\d+)/task/(?P<task_id>\d+)$', views.task, name='task'),
 
         url(r'^build/(?P<build_id>\d+)/recipes/$',
             buildtables.BuiltRecipesTable.as_view(
                 template_name="buildinfo-toastertable.html"),
             name='recipes'),
 
-        url(r'^build/(?P<build_id>\d+)/recipe/(?P<recipe_id>\d+)/active_tab/(?P<active_tab>\d{1})$', 'recipe', name='recipe'),
+        url(r'^build/(?P<build_id>\d+)/recipe/(?P<recipe_id>\d+)/active_tab/(?P<active_tab>\d{1})$', views.recipe, name='recipe'),
 
-        url(r'^build/(?P<build_id>\d+)/recipe/(?P<recipe_id>\d+)$', 'recipe', name='recipe'),
-        url(r'^build/(?P<build_id>\d+)/recipe_packages/(?P<recipe_id>\d+)$', 'recipe_packages', name='recipe_packages'),
+        url(r'^build/(?P<build_id>\d+)/recipe/(?P<recipe_id>\d+)$', views.recipe, name='recipe'),
+        url(r'^build/(?P<build_id>\d+)/recipe_packages/(?P<recipe_id>\d+)$', views.recipe_packages, name='recipe_packages'),
 
         url(r'^build/(?P<build_id>\d+)/packages/$',
             buildtables.BuiltPackagesTable.as_view(
                 template_name="buildinfo-toastertable.html"),
             name='packages'),
 
-        url(r'^build/(?P<build_id>\d+)/package/(?P<package_id>\d+)$', 'package_built_detail',
+        url(r'^build/(?P<build_id>\d+)/package/(?P<package_id>\d+)$', views.package_built_detail,
                 name='package_built_detail'),
         url(r'^build/(?P<build_id>\d+)/package_built_dependencies/(?P<package_id>\d+)$',
-            'package_built_dependencies', name='package_built_dependencies'),
+            views.package_built_dependencies, name='package_built_dependencies'),
         url(r'^build/(?P<build_id>\d+)/package_included_detail/(?P<target_id>\d+)/(?P<package_id>\d+)$',
-            'package_included_detail', name='package_included_detail'),
+            views.package_included_detail, name='package_included_detail'),
         url(r'^build/(?P<build_id>\d+)/package_included_dependencies/(?P<target_id>\d+)/(?P<package_id>\d+)$',
-            'package_included_dependencies', name='package_included_dependencies'),
+            views.package_included_dependencies, name='package_included_dependencies'),
         url(r'^build/(?P<build_id>\d+)/package_included_reverse_dependencies/(?P<target_id>\d+)/(?P<package_id>\d+)$',
-            'package_included_reverse_dependencies', name='package_included_reverse_dependencies'),
+            views.package_included_reverse_dependencies, name='package_included_reverse_dependencies'),
 
         url(r'^build/(?P<build_id>\d+)/target/(?P<target_id>\d+)$',
             buildtables.InstalledPackagesTable.as_view(
@@ -75,11 +76,11 @@ urlpatterns = patterns('toastergui.views',
             name='target'),
 
 
-        url(r'^dentries/build/(?P<build_id>\d+)/target/(?P<target_id>\d+)$', 'xhr_dirinfo', name='dirinfo_ajax'),
-        url(r'^build/(?P<build_id>\d+)/target/(?P<target_id>\d+)/dirinfo$', 'dirinfo', name='dirinfo'),
-        url(r'^build/(?P<build_id>\d+)/target/(?P<target_id>\d+)/dirinfo_filepath/_(?P<file_path>(?:/[^/\n]+)*)$', 'dirinfo', name='dirinfo_filepath'),
-        url(r'^build/(?P<build_id>\d+)/configuration$', 'configuration', name='configuration'),
-        url(r'^build/(?P<build_id>\d+)/configvars$', 'configvars', name='configvars'),
+        url(r'^dentries/build/(?P<build_id>\d+)/target/(?P<target_id>\d+)$', views.xhr_dirinfo, name='dirinfo_ajax'),
+        url(r'^build/(?P<build_id>\d+)/target/(?P<target_id>\d+)/dirinfo$', views.dirinfo, name='dirinfo'),
+        url(r'^build/(?P<build_id>\d+)/target/(?P<target_id>\d+)/dirinfo_filepath/_(?P<file_path>(?:/[^/\n]+)*)$', views.dirinfo, name='dirinfo_filepath'),
+        url(r'^build/(?P<build_id>\d+)/configuration$', views.configuration, name='configuration'),
+        url(r'^build/(?P<build_id>\d+)/configvars$', views.configvars, name='configvars'),
         url(r'^build/(?P<build_id>\d+)/buildtime$',
             buildtables.BuildTimeTable.as_view(
                 template_name="buildinfo-toastertable.html"),
@@ -97,26 +98,26 @@ urlpatterns = patterns('toastergui.views',
 
         # image information dir
         url(r'^build/(?P<build_id>\d+)/target/(?P<target_id>\d+)/packagefile/(?P<packagefile_id>\d+)$',
-             'image_information_dir', name='image_information_dir'),
+             views.image_information_dir, name='image_information_dir'),
 
         # build download artifact
-        url(r'^build/(?P<build_id>\d+)/artifact/(?P<artifact_type>\w+)/id/(?P<artifact_id>\w+)', 'build_artifact', name="build_artifact"),
+        url(r'^build/(?P<build_id>\d+)/artifact/(?P<artifact_type>\w+)/id/(?P<artifact_id>\w+)', views.build_artifact, name="build_artifact"),
 
         # project URLs
-        url(r'^newproject/$', 'newproject', name='newproject'),
+        url(r'^newproject/$', views.newproject, name='newproject'),
 
         url(r'^projects/$',
             tables.ProjectsTable.as_view(template_name="projects-toastertable.html"),
             name='all-projects'),
 
-        url(r'^project/(?P<pid>\d+)/$', 'project', name='project'),
-        url(r'^project/(?P<pid>\d+)/configuration$', 'projectconf', name='projectconf'),
+        url(r'^project/(?P<pid>\d+)/$', views.project, name='project'),
+        url(r'^project/(?P<pid>\d+)/configuration$', views.projectconf, name='projectconf'),
         url(r'^project/(?P<pid>\d+)/builds/$',
             tables.ProjectBuildsTable.as_view(template_name="projectbuilds-toastertable.html"),
             name='projectbuilds'),
 
         # the import layer is a project-specific functionality;
-        url(r'^project/(?P<pid>\d+)/importlayer$', 'importlayer', name='importlayer'),
+        url(r'^project/(?P<pid>\d+)/importlayer$', views.importlayer, name='importlayer'),
 
         # the table pages that have been converted to ToasterTable widget
         url(r'^project/(?P<pid>\d+)/machines/$',
@@ -142,7 +143,7 @@ urlpatterns = patterns('toastergui.views',
             name="projectlayers"),
 
         url(r'^project/(?P<pid>\d+)/layer/(?P<layerid>\d+)$',
-            'layerdetails', name='layerdetails'),
+            views.layerdetails, name='layerdetails'),
 
         url(r'^project/(?P<pid>\d+)/layer/(?P<layerid>\d+)/recipes/$',
             tables.LayerRecipesTable.as_view(template_name="generic-toastertable-page.html"),
@@ -157,6 +158,11 @@ urlpatterns = patterns('toastergui.views',
             name=tables.LayerMachinesTable.__name__.lower()),
 
 
+        url(r'^project/(?P<pid>\d+)/distros/$',
+            tables.DistrosTable.as_view(template_name="generic-toastertable-page.html"),
+            name="projectdistros"),
+
+
         url(r'^project/(?P<pid>\d+)/customrecipe/(?P<custrecipeid>\d+)/selectpackages/$',
             tables.SelectPackagesTable.as_view(), name="recipeselectpackages"),
 
@@ -166,7 +172,7 @@ urlpatterns = patterns('toastergui.views',
             name="customrecipe"),
 
         url(r'^project/(?P<pid>\d+)/customrecipe/(?P<recipe_id>\d+)/download$',
-            'customrecipe_download',
+            views.customrecipe_download,
             name="customrecipedownload"),
 
         url(r'^project/(?P<pid>\d+)/recipe/(?P<recipe_id>\d+)$',
@@ -186,9 +192,12 @@ urlpatterns = patterns('toastergui.views',
             typeaheads.GitRevisionTypeAhead.as_view(),
             name='xhr_gitrevtypeahead'),
 
-        url(r'^xhr_testreleasechange/(?P<pid>\d+)$', 'xhr_testreleasechange',
+        url(r'^xhr_typeahead/(?P<pid>\d+)/distros$',
+            typeaheads.DistrosTypeAhead.as_view(), name='xhr_distrostypeahead'),
+
+        url(r'^xhr_testreleasechange/(?P<pid>\d+)$', views.xhr_testreleasechange,
             name='xhr_testreleasechange'),
-        url(r'^xhr_configvaredit/(?P<pid>\d+)$', 'xhr_configvaredit',
+        url(r'^xhr_configvaredit/(?P<pid>\d+)$', views.xhr_configvaredit,
             name='xhr_configvaredit'),
 
         url(r'^xhr_layer/(?P<pid>\d+)/(?P<layerversion_id>\d+)$',
@@ -200,7 +209,7 @@ urlpatterns = patterns('toastergui.views',
             name='xhr_layer'),
 
         # JS Unit tests
-        url(r'^js-unit-tests/$', 'jsunittests', name='js-unit-tests'),
+        url(r'^js-unit-tests/$', views.jsunittests, name='js-unit-tests'),
 
         # image customisation functionality
         url(r'^xhr_customrecipe/(?P<recipe_id>\d+)'
@@ -235,6 +244,11 @@ urlpatterns = patterns('toastergui.views',
         url(r'^mostrecentbuilds$', widgets.MostRecentBuildsView.as_view(),
             name='most_recent_builds'),
 
-          # default redirection
+        # JSON data for aggregators
+        url(r'^api/builds$', views.json_builds, name='json_builds'),
+        url(r'^api/building$', views.json_building, name='json_building'),
+        url(r'^api/build/(?P<build_id>\d+)$', views.json_build, name='json_build'),
+
+        # default redirection
         url(r'^$', RedirectView.as_view(url='landing', permanent=True)),
-)
+]

bitbake/lib/toaster/toastergui/views.py

@@ -35,7 +35,7 @@ from orm.models import BitbakeVersion, CustomImageRecipe
 from django.core.urlresolvers import reverse, resolve
 from django.core.exceptions import MultipleObjectsReturned, ObjectDoesNotExist
 from django.core.paginator import Paginator, EmptyPage, PageNotAnInteger
-from django.http import HttpResponseNotFound
+from django.http import HttpResponseNotFound, JsonResponse
 from django.utils import timezone
 from datetime import timedelta, datetime
 from toastergui.templatetags.projecttags import json as jsonfilter
@@ -457,10 +457,15 @@ def builddashboard( request, build_id ):
         npkg = 0
         pkgsz = 0
         package = None
-        for package in Package.objects.filter(id__in = [x.package_id for x in t.target_installed_package_set.all()]):
-            pkgsz = pkgsz + package.size
-            if package.installed_name:
-                npkg = npkg + 1
+        # Chunk the query to avoid "too many SQL variables" error
+        package_set = t.target_installed_package_set.all()
+        package_set_len = len(package_set)
+        for ps_start in range(0,package_set_len,500):
+            ps_stop = min(ps_start+500,package_set_len)
+            for package in Package.objects.filter(id__in = [x.package_id for x in package_set[ps_start:ps_stop]]):
+                pkgsz = pkgsz + package.size
+                if package.installed_name:
+                    npkg = npkg + 1
         elem['npkg'] = npkg
         elem['pkgsz'] = pkgsz
         ti = Target_Image_File.objects.filter(target_id = t.id)
@@ -1251,6 +1256,89 @@ def managedcontextprocessor(request):
     }
     return ret
 
+# REST-based API calls to return build/building status to external Toaster
+# managers and aggregators via JSON
+
+def _json_build_status(build_id,extend):
+    build_stat = None
+    try:
+        build = Build.objects.get( pk = build_id )
+        build_stat = {}
+        build_stat['id'] = build.id
+        build_stat['name'] = build.build_name
+        build_stat['machine'] = build.machine
+        build_stat['distro'] = build.distro
+        build_stat['start'] = build.started_on
+        # look up target name
+        target= Target.objects.get( build = build )
+        if target:
+            if target.task:
+                build_stat['target'] = '%s:%s' % (target.target,target.task)
+            else:
+                build_stat['target'] = '%s' % (target.target)
+        else:
+            build_stat['target'] = ''
+        # look up project name
+        project = Project.objects.get( build = build )
+        if project:
+            build_stat['project'] = project.name
+        else:
+            build_stat['project'] = ''
+        if Build.IN_PROGRESS == build.outcome:
+            now = timezone.now()
+            timediff = now - build.started_on
+            build_stat['seconds']='%.3f' % timediff.total_seconds()
+            build_stat['clone']='%d:%d' % (build.repos_cloned,build.repos_to_clone)
+            build_stat['parse']='%d:%d' % (build.recipes_parsed,build.recipes_to_parse)
+            tf = Task.objects.filter(build = build)
+            tfc = tf.count()
+            if tfc > 0:
+                tfd = tf.exclude(order__isnull=True).count()
+            else:
+                tfd = 0
+            build_stat['task']='%d:%d' % (tfd,tfc)
+        else:
+            build_stat['outcome'] = build.get_outcome_text()
+            timediff = build.completed_on - build.started_on
+            build_stat['seconds']='%.3f' % timediff.total_seconds()
+            build_stat['stop'] = build.completed_on
+            messages = LogMessage.objects.all().filter(build = build)
+            errors = len(messages.filter(level=LogMessage.ERROR) |
+                 messages.filter(level=LogMessage.EXCEPTION) |
+                 messages.filter(level=LogMessage.CRITICAL))
+            build_stat['errors'] = errors
+            warnings = len(messages.filter(level=LogMessage.WARNING))
+            build_stat['warnings'] = warnings
+        if extend:
+            build_stat['cooker_log'] = build.cooker_log_path
+    except Exception as e:
+        build_state = str(e)
+    return build_stat
+
+def json_builds(request):
+    build_table = []
+    builds = []
+    try:
+        builds = Build.objects.exclude(outcome=Build.IN_PROGRESS).order_by("-started_on")
+        for build in builds:
+            build_table.append(_json_build_status(build.id,False))
+    except Exception as e:
+        build_table = str(e)
+    return JsonResponse({'builds' : build_table, 'count' : len(builds)})
+
+def json_building(request):
+    build_table = []
+    builds = []
+    try:
+        builds = Build.objects.filter(outcome=Build.IN_PROGRESS).order_by("-started_on")
+        for build in builds:
+            build_table.append(_json_build_status(build.id,False))
+    except Exception as e:
+        build_table = str(e)
+    return JsonResponse({'building' : build_table, 'count' : len(builds)})
+
+def json_build(request,build_id):
+    return JsonResponse({'build' : _json_build_status(build_id,True)})
 
 
 import toastermain.settings
@@ -1689,3 +1777,4 @@ if True:
                 return render(request, "unavailable_artifact.html")
         except (ObjectDoesNotExist, IOError):
             return render(request, "unavailable_artifact.html")
+

bitbake/lib/toaster/toastergui/widgets.py

@@ -511,6 +511,10 @@ class MostRecentBuildsView(View):
                 int((build_obj.recipes_parsed /
                      build_obj.recipes_to_parse) * 100)
 
+            build['repos_cloned_percentage'] = \
+                int((build_obj.repos_cloned /
+                     build_obj.repos_to_clone) * 100)
+
             tasks_complete_percentage = 0
             if build_obj.outcome in (Build.SUCCEEDED, Build.FAILED):
                 tasks_complete_percentage = 100

bitbake/lib/toaster/toastermain/management/commands/buildslist.py

@@ -1,13 +1,13 @@
-from django.core.management.base import NoArgsCommand, CommandError
+from django.core.management.base import BaseCommand, CommandError
 from orm.models import Build
 import os
 
 
 
-class Command(NoArgsCommand):
+class Command(BaseCommand):
     args    = ""
     help    = "Lists current builds"
 
-    def handle_noargs(self,**options):
+    def handle(self,**options):
         for b in Build.objects.all():
             print("%d: %s %s %s" % (b.pk, b.machine, b.distro, ",".join([x.target for x in b.target_set.all()])))

bitbake/lib/toaster/toastermain/settings.py

@@ -24,7 +24,6 @@
 import os
 
 DEBUG = True
-TEMPLATE_DEBUG = DEBUG
 
 # Set to True to see the SQL queries in console
 SQL_DEBUG = False
@@ -161,12 +160,47 @@ STATICFILES_FINDERS = (
 # Make this unique, and don't share it with anybody.
 SECRET_KEY = 'NOT_SUITABLE_FOR_HOSTED_DEPLOYMENT'
 
-# List of callables that know how to import templates from various sources.
-TEMPLATE_LOADERS = (
-    'django.template.loaders.filesystem.Loader',
-    'django.template.loaders.app_directories.Loader',
-#     'django.template.loaders.eggs.Loader',
-)
+class InvalidString(str):
+    def __mod__(self, other):
+        from django.template.base import TemplateSyntaxError
+        raise TemplateSyntaxError(
+            "Undefined variable or unknown value for: \"%s\"" % other)
+
+TEMPLATES = [
+    {
+        'BACKEND': 'django.template.backends.django.DjangoTemplates',
+        'DIRS': [
+            # Put strings here, like "/home/html/django_templates" or "C:/www/django/templates".
+            # Always use forward slashes, even on Windows.
+            # Don't forget to use absolute paths, not relative paths.
+        ],
+        'OPTIONS': {
+            'context_processors': [
+                # Insert your TEMPLATE_CONTEXT_PROCESSORS here or use this
+                # list if you haven't customized them:
+                'django.contrib.auth.context_processors.auth',
+                'django.template.context_processors.debug',
+                'django.template.context_processors.i18n',
+                'django.template.context_processors.media',
+                'django.template.context_processors.static',
+                'django.template.context_processors.tz',
+                'django.contrib.messages.context_processors.messages',
+                # Custom
+                'django.core.context_processors.request',
+                'toastergui.views.managedcontextprocessor',
+
+            ],
+            'loaders': [
+                # List of callables that know how to import templates from various sources.
+                'django.template.loaders.filesystem.Loader',
+                'django.template.loaders.app_directories.Loader',
+                #'django.template.loaders.eggs.Loader',
+            ],
+            'string_if_invalid': InvalidString("%s"),
+            'debug': DEBUG,
+        },
+    },
+]
 
 MIDDLEWARE_CLASSES = (
     'django.middleware.common.CommonMiddleware',
@@ -203,22 +237,6 @@ ROOT_URLCONF = 'toastermain.urls'
 # Python dotted path to the WSGI application used by Django's runserver.
 WSGI_APPLICATION = 'toastermain.wsgi.application'
 
-TEMPLATE_DIRS = (
-    # Put strings here, like "/home/html/django_templates" or "C:/www/django/templates".
-    # Always use forward slashes, even on Windows.
-    # Don't forget to use absolute paths, not relative paths.
-)
-
-TEMPLATE_CONTEXT_PROCESSORS = ('django.contrib.auth.context_processors.auth',
- 'django.core.context_processors.debug',
- 'django.core.context_processors.i18n',
- 'django.core.context_processors.media',
- 'django.core.context_processors.static',
- 'django.core.context_processors.tz',
- 'django.contrib.messages.context_processors.messages',
- "django.core.context_processors.request",
- 'toastergui.views.managedcontextprocessor',
- )
 
 INSTALLED_APPS = (
     'django.contrib.auth',
@@ -348,10 +366,4 @@ connection_created.connect(activate_synchronous_off)
 #
 
 
-class InvalidString(str):
-    def __mod__(self, other):
-        from django.template.base import TemplateSyntaxError
-        raise TemplateSyntaxError(
-            "Undefined variable or unknown value for: \"%s\"" % other)
 
-TEMPLATE_STRING_IF_INVALID = InvalidString("%s")

bitbake/lib/toaster/toastermain/urls.py

@@ -20,8 +20,9 @@
 # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
 
 from django.conf.urls import patterns, include, url
-from django.views.generic import RedirectView
+from django.views.generic import RedirectView, TemplateView
 from django.views.decorators.cache import never_cache
+import bldcollector.views
 
 import logging
 
@@ -31,7 +32,7 @@ logger = logging.getLogger("toaster")
 from django.contrib import admin
 admin.autodiscover()
 
-urlpatterns = patterns('',
+urlpatterns = [
 
     # Examples:
     # url(r'^toaster/', include('toaster.foo.urls')),
@@ -42,11 +43,13 @@ urlpatterns = patterns('',
 
     # This is here to maintain backward compatibility and will be deprecated
     # in the future.
-    url(r'^orm/eventfile$', 'bldcollector.views.eventfile'),
+    url(r'^orm/eventfile$', bldcollector.views.eventfile),
+
+    url(r'^health$', TemplateView.as_view(template_name="health.html"), name='Toaster Health'),
 
     # if no application is selected, we have the magic toastergui app here
     url(r'^$', never_cache(RedirectView.as_view(url='/toastergui/', permanent=True))),
-)
+]
 
 import toastermain.settings
 

bitbake/toaster-requirements.txt

@@ -1,3 +1,3 @@
-Django>1.8,<1.9
+Django>1.8,<1.9.9
 beautifulsoup4>=4.4.0
 pytz

documentation/Makefile

@@ -88,6 +88,7 @@ ifeq ($(DOC),bsp-guide)
 XSLTOPTS = --xinclude
 ALLPREQ = html eclipse tarball
 TARFILES = bsp-style.css bsp-guide.html figures/bsp-title.png \
+           figures/bsp-dev-flow.png \
            eclipse
 MANUALS = $(DOC)/$(DOC).html $(DOC)/eclipse
 FIGURES = figures
@@ -128,14 +129,8 @@ TARFILES = dev-style.css dev-manual.html \
            figures/wip.png
         else
 TARFILES = dev-style.css dev-manual.html \
-           figures/bsp-dev-flow.png \
-           figures/dev-title.png figures/git-workflow.png \
-           figures/index-downloads.png figures/kernel-dev-flow.png \
-           figures/kernel-overview-1.png figures/kernel-overview-2-generic.png \
-           figures/source-repos.png figures/yp-download.png \
-           figures/recipe-workflow.png \
-           figures/devtool-add-flow.png figures/devtool-modify-flow.png \
-           figures/devtool-upgrade-flow.png \
+           figures/dev-title.png \
+           figures/recipe-workflow.png figures/bitbake-build-flow.png \
            eclipse
 	endif
 
@@ -148,7 +143,7 @@ endif
 ifeq ($(DOC),yocto-project-qs)
 XSLTOPTS = --xinclude
 ALLPREQ = html eclipse tarball
-TARFILES = yocto-project-qs.html qs-style.css figures/yocto-environment.png \
+TARFILES = yocto-project-qs.html qs-style.css \
            figures/yocto-project-transp.png \
            eclipse
 MANUALS = $(DOC)/$(DOC).html $(DOC)/eclipse
@@ -196,7 +191,7 @@ TARFILES = mega-manual.html mega-style.css figures/yocto-environment.png \
 	figures/wip.png
         else
 TARFILES = mega-manual.html mega-style.css figures/yocto-environment.png \
-        figures/building-an-image.png  \
+        figures/building-an-image.png figures/YP-flow-diagram.png \
 	figures/using-a-pre-built-image.png \
 	figures/poky-title.png figures/buildhistory.png \
         figures/buildhistory-web.png \
@@ -206,7 +201,7 @@ TARFILES = mega-manual.html mega-style.css figures/yocto-environment.png \
         figures/dev-title.png \
 	figures/git-workflow.png figures/index-downloads.png \
         figures/kernel-dev-flow.png \
-	figures/kernel-overview-1.png figures/kernel-overview-2-generic.png \
+	figures/kernel-overview-2-generic.png \
 	figures/source-repos.png figures/yp-download.png \
         figures/profile-title.png figures/kernelshark-all.png \
         figures/kernelshark-choose-events.png \
@@ -244,13 +239,12 @@ TARFILES = mega-manual.html mega-style.css figures/yocto-environment.png \
 	figures/sdk-generation.png figures/recipe-workflow.png \
 	figures/build-workspace-directory.png figures/mega-title.png \
 	figures/toaster-title.png figures/hosted-service.png \
-	figures/simple-configuration.png figures/devtool-add-flow.png \
-	figures/devtool-modify-flow.png figures/devtool-upgrade-flow.png \
+	figures/simple-configuration.png \
 	figures/compatible-layers.png figures/import-layer.png figures/new-project.png \
 	figures/sdk-environment.png figures/sdk-installed-standard-sdk-directory.png \
 	figures/sdk-devtool-add-flow.png figures/sdk-installed-extensible-sdk-directory.png \
 	figures/sdk-devtool-modify-flow.png figures/sdk-eclipse-dev-flow.png \
-	figures/sdk-devtool-upgrade-flow.png
+	figures/sdk-devtool-upgrade-flow.png figures/bitbake-build-flow.png
 	endif
 
 MANUALS = $(DOC)/$(DOC).html
@@ -262,7 +256,7 @@ endif
 ifeq ($(DOC),ref-manual)
 XSLTOPTS = --xinclude
 ALLPREQ = html eclipse tarball
-TARFILES = ref-manual.html ref-style.css figures/poky-title.png \
+TARFILES = ref-manual.html ref-style.css figures/poky-title.png figures/YP-flow-diagram.png \
 	figures/buildhistory.png figures/buildhistory-web.png eclipse \
         figures/cross-development-toolchains.png figures/layer-input.png \
 	figures/package-feeds.png figures/source-input.png \
@@ -271,7 +265,8 @@ TARFILES = ref-manual.html ref-style.css figures/poky-title.png \
 	figures/patching.png figures/configuration-compile-autoreconf.png \
 	figures/analysis-for-package-splitting.png figures/image-generation.png \
 	figures/sdk-generation.png figures/building-an-image.png \
-	figures/build-workspace-directory.png
+	figures/build-workspace-directory.png figures/source-repos.png \
+	figures/index-downloads.png figures/yp-download.png figures/git-workflow.png
 MANUALS = $(DOC)/$(DOC).html $(DOC)/eclipse
 FIGURES = figures
 STYLESHEET = $(DOC)/*.css
@@ -330,8 +325,8 @@ ifeq ($(DOC),kernel-dev)
 XSLTOPTS = --xinclude
 ALLPREQ = html eclipse tarball
 TARFILES = kernel-dev.html kernel-dev-style.css \
-           figures/kernel-dev-title.png \
-           figures/kernel-architecture-overview.png \
+           figures/kernel-dev-title.png figures/kernel-overview-2-generic \
+           figures/kernel-architecture-overview.png figures/kernel-dev-flow.png \
            eclipse
 MANUALS = $(DOC)/$(DOC).html $(DOC)/eclipse
 FIGURES = figures

documentation/bsp-guide/bsp.xml

@@ -55,7 +55,7 @@
                 To help understand the BSP layer concept, consider the BSPs that the
                 Yocto Project supports and provides with each release.
                 You can see the layers in the
-                <ulink url='&YOCTO_DOCS_DEV_URL;#yocto-project-repositories'>Yocto Project Source Repositories</ulink>
+                <ulink url='&YOCTO_DOCS_REF_URL;#yocto-project-repositories'>Yocto Project Source Repositories</ulink>
                 through a web interface at
                 <ulink url='&YOCTO_GIT_URL;/cgit/cgit.cgi'></ulink>.
                 If you go to that interface, you will find near the bottom of the list
@@ -83,11 +83,11 @@
 
             <para>
                 For information on the BSP development workflow, see the
-                "<ulink url='&YOCTO_DOCS_DEV_URL;#developing-a-board-support-package-bsp'>Developing a Board Support Package (BSP)</ulink>"
-                section in the Yocto Project Development Manual.
+                "<link linkend='developing-a-board-support-package-bsp'>Developing a Board Support Package (BSP)</link>"
+                section.
                 For more information on how to set up a local copy of source files
                 from a Git repository, see the
-                "<ulink url='&YOCTO_DOCS_DEV_URL;#getting-setup'>Getting Set Up</ulink>"
+                "<ulink url='&YOCTO_DOCS_DEV_URL;#working-with-yocto-project-source-files'>Working With Yocto Project Source Files</ulink>"
                 section also in the Yocto Project Development Manual.
             </para>
 
@@ -98,7 +98,7 @@
                 This root is what you add to the
                 <ulink url='&YOCTO_DOCS_REF_URL;#var-BBLAYERS'><filename>BBLAYERS</filename></ulink>
                 variable in the <filename>conf/bblayers.conf</filename> file found in the
-                <ulink url='&YOCTO_DOCS_DEV_URL;#build-directory'>Build Directory</ulink>,
+                <ulink url='&YOCTO_DOCS_REF_URL;#build-directory'>Build Directory</ulink>,
                 which is established after you run one of the OpenEmbedded build environment
                 setup scripts (i.e.
                 <ulink url='&YOCTO_DOCS_REF_URL;#structure-core-script'><filename>&OE_INIT_FILE;</filename></ulink>
@@ -417,7 +417,7 @@
                     released with the BSP.
                     The information in the <filename>README.sources</filename>
                     file also helps you find the
-                    <ulink url='&YOCTO_DOCS_DEV_URL;#metadata'>Metadata</ulink>
+                    <ulink url='&YOCTO_DOCS_REF_URL;#metadata'>Metadata</ulink>
                     used to generate the images that ship with the BSP.
                     <note>
                         If the BSP's <filename>binary</filename> directory is
@@ -522,7 +522,7 @@
 
                 <para>
                     This file simply makes
-                    <ulink url='&YOCTO_DOCS_DEV_URL;#bitbake-term'>BitBake</ulink>
+                    <ulink url='&YOCTO_DOCS_REF_URL;#bitbake-term'>BitBake</ulink>
                     aware of the recipes and configuration directories.
                     The file must exist so that the OpenEmbedded build system can recognize the BSP.
                 </para>
@@ -710,6 +710,220 @@
             </section>
         </section>
 
+        <section id='developing-a-board-support-package-bsp'>
+            <title>Developing a Board Support Package (BSP)</title>
+
+            <para>
+                This section contains the high-level procedure you can follow
+                to create a BSP using the Yocto Project's
+                <link linkend='using-the-yocto-projects-bsp-tools'>BSP Tools</link>.
+                Although not required for BSP creation, the
+                <filename>meta-intel</filename> repository, which contains
+                many BSPs supported by the Yocto Project, is part of the
+                example.
+            </para>
+
+            <para>
+                For an example that shows how to create a new layer using
+                the tools, see the
+                "<link linkend='creating-a-new-bsp-layer-using-the-yocto-bsp-script'>Creating a New BSP Layer Using the yocto-bsp Script</link>"
+                 section.
+            </para>
+
+            <para>
+                The following illustration and list summarize the BSP
+                creation general workflow.
+            </para>
+
+            <para>
+                <imagedata fileref="figures/bsp-dev-flow.png" width="6in" depth="7in" align="center" scalefit="1" />
+            </para>
+
+            <para>
+                <orderedlist>
+                    <listitem><para>
+                        <emphasis>Set Up Your Host Development System to Support
+                        Development Using the Yocto Project:</emphasis>
+                        See the
+                        "<ulink url='&YOCTO_DOCS_QS_URL;#the-linux-distro'>The Linux Distribution</ulink>"
+                        and the
+                        "<ulink url='&YOCTO_DOCS_QS_URL;#packages'>The Build Host Packages</ulink>"
+                        sections both in the Yocto Project Quick Start for
+                        build host requirements.
+                        </para></listitem>
+                    <listitem><para>
+                        <emphasis>Establish a Local Copy of the Project Files on
+                        Your System:</emphasis>
+                        You need this
+                        <ulink url='&YOCTO_DOCS_REF_URL;#source-directory'>Source Directory</ulink>
+                        available on your host system.
+                        Having these files on your system gives you access to the
+                        build process and to the tools you need.
+                        For information on how to set up the Source Directory,
+                        see the
+                        "<ulink url='&YOCTO_DOCS_DEV_URL;#working-with-yocto-project-source-files'>Working With Yocto Project Source Files</ulink>"
+                        section in the Yocto Project Development Manual.
+                        </para></listitem>
+                    <listitem><para>
+                        <emphasis>Establish the <filename>meta-intel</filename>
+                        Repository on Your System:</emphasis>
+                        Having local copies of these supported BSP layers on
+                        your system gives you access to layers you might be able
+                        to build on or modify to create your BSP.
+                        For information on how to get these files, see the
+                        "<ulink url='&YOCTO_DOCS_DEV_URL;#setting-up-bsp-layers'>Setting Up BSP Layers</ulink>"
+                        section in the Yocto Project Development Manual.
+                        </para></listitem>
+                    <listitem><para>
+                        <emphasis>Create Your Own BSP Layer Using the
+                        <link linkend='creating-a-new-bsp-layer-using-the-yocto-bsp-script'><filename>yocto-bsp</filename></link>
+                        script:</emphasis>
+                        Layers are ideal for isolating and storing work for a
+                        given piece of hardware.
+                        A layer is really just a location or area in which you
+                        place the recipes and configurations for your BSP.
+                        In fact, a BSP is, in itself, a special type of layer.
+                        The simplest way to create a new BSP layer that is
+                        compliant with the Yocto Project is to use the
+                        <filename>yocto-bsp</filename> script.
+                        For information about that script, see the
+                        "<link linkend='creating-a-new-bsp-layer-using-the-yocto-bsp-script'>Creating a New BSP Layer Using the yocto-bsp Script</link>"
+                        section.</para>
+
+                        <para>Another example that illustrates a layer
+                        is an application.
+                        Suppose you are creating an application that has
+                        library or other dependencies in order for it to
+                        compile and run.
+                        The layer, in this case, would be where all the
+                        recipes that define those dependencies are kept.
+                        The key point for a layer is that it is an isolated
+                        area that contains all the relevant information for
+                        the project that the OpenEmbedded build system knows
+                        about.
+                        For more information on layers, see the
+                        "<ulink url='&YOCTO_DOCS_DEV_URL;#understanding-and-creating-layers'>Understanding and Creating Layers</ulink>"
+                        section in the Yocto Project Development Manual.
+                        For more information on BSP layers, see the
+                        "<link linkend='bsp-layers'>BSP Layers</link>"
+                        section.
+                        <note><title>Notes</title>
+                            <para>Five BSPs exist that are part of the Yocto
+                            Project release:
+                            <filename>beaglebone</filename> (ARM),
+                            <filename>mpc8315e</filename> (PowerPC),
+                            and <filename>edgerouter</filename> (MIPS).
+                            The recipes and configurations for these five BSPs
+                            are located and dispersed within the
+                            <ulink url='&YOCTO_DOCS_REF_URL;#source-directory'>Source Directory</ulink>.
+                            </para>
+
+                            <para>Three core Intel BSPs exist as part of the Yocto
+                            Project release in the
+                            <filename>meta-intel</filename> layer:
+                            <itemizedlist>
+                                <listitem><para>
+                                    <filename>intel-core2-32</filename>,
+                                    which is a BSP optimized for the Core2 family of CPUs
+                                    as well as all CPUs prior to the Silvermont core.
+                                    </para></listitem>
+                                <listitem><para>
+                                    <filename>intel-corei7-64</filename>,
+                                    which is a BSP optimized for Nehalem and later
+                                    Core and Xeon CPUs as well as Silvermont and later
+                                    Atom CPUs, such as the Baytrail SoCs.
+                                    </para></listitem>
+                                <listitem><para>
+                                    <filename>intel-quark</filename>,
+                                    which is a BSP optimized for the Intel Galileo
+                                    gen1 &amp; gen2 development boards.
+                                    </para></listitem>
+                            </itemizedlist></para>
+                        </note></para>
+
+                        <para>When you set up a layer for a new BSP, you should
+                        follow a standard layout.
+                        This layout is described in the
+                        "<link linkend='bsp-filelayout'>Example Filesystem Layout</link>"
+                        section.
+                        In the standard layout, you will notice a suggested
+                        structure for recipes and configuration information.
+                        You can see the standard layout for a BSP by examining
+                        any supported BSP found in the
+                        <filename>meta-intel</filename> layer inside the Source
+                        Directory.
+                        </para></listitem>
+                    <listitem><para>
+                        <emphasis>Make Configuration Changes to Your New BSP
+                        Layer:</emphasis>
+                        The standard BSP layer structure organizes the files
+                        you need to edit in <filename>conf</filename> and
+                        several <filename>recipes-*</filename>
+                        directories within the BSP layer.
+                        Configuration changes identify where your new layer
+                        is on the local system and identify which kernel you
+                        are going to use.
+                        When you run the <filename>yocto-bsp</filename> script,
+                        you are able to interactively configure many things for
+                        the BSP (e.g. keyboard, touchscreen, and so forth).
+                        </para></listitem>
+                    <listitem><para>
+                        <emphasis>Make Recipe Changes to Your New BSP
+                        Layer:</emphasis>
+                        Recipe changes include altering recipes
+                        (<filename>.bb</filename> files), removing recipes you
+                        do not use, and adding new recipes or append files
+                        (<filename>.bbappend</filename>) that you need to
+                        support your hardware.
+                        </para></listitem>
+                    <listitem><para>
+                        <emphasis>Prepare for the Build:</emphasis>
+                        Once you have made all the changes to your BSP layer,
+                        there remains a few things you need to do for the
+                        OpenEmbedded build system in order for it to create
+                        your image.
+                        You need to get the build environment ready by
+                        sourcing an environment setup script
+                        (i.e. <filename>oe-init-build-env</filename> or
+                        <filename>oe-init-build-env-memres</filename>)
+                        and you need to be sure two key configuration
+                        files are configured appropriately: the
+                        <filename>conf/local.conf</filename> and the
+                        <filename>conf/bblayers.conf</filename> file.
+                        You must make the OpenEmbedded build system aware
+                        of your new layer.
+                        See the
+                        "<ulink url='&YOCTO_DOCS_DEV_URL;#enabling-your-layer'>Enabling Your Layer</ulink>"
+                        section in the Yocto Project Development Manual
+                        for information on how to let the build system
+                        know about your new layer.</para>
+
+                        <para>The entire process for building an image is
+                        overviewed in the section
+                        "<ulink url='&YOCTO_DOCS_QS_URL;#qs-building-images'>Building Images</ulink>" section
+                        of the Yocto Project Quick Start.
+                        You might want to reference this information.
+                        </para></listitem>
+                    <listitem><para>
+                        <emphasis>Build the Image:</emphasis>
+                        The OpenEmbedded build system uses the BitBake tool
+                        to build images based on the type of image you want to
+                        create.
+                        You can find more information about BitBake in the
+                        <ulink url='&YOCTO_DOCS_BB_URL;'>BitBake User Manual</ulink>.
+                        </para>
+
+                        <para>The build process supports several types of
+                        images to satisfy different needs.
+                        See the
+                        "<ulink url='&YOCTO_DOCS_REF_URL;#ref-images'>Images</ulink>"
+                        chapter in the Yocto Project Reference Manual for
+                        information on supported images.
+                        </para></listitem>
+                </orderedlist>
+            </para>
+        </section>
+
         <section id='requirements-and-recommendations-for-released-bsps'>
             <title>Requirements and Recommendations for Released BSPs</title>
 
@@ -832,7 +1046,7 @@
                                     This is the person to whom patches and questions should
                                     be sent.
                                     For information on how to find the right person, see the
-                                    "<ulink url='&YOCTO_DOCS_DEV_URL;#how-to-submit-a-change'>How to Submit a Change</ulink>"
+                                    "<ulink url='&YOCTO_DOCS_DEV_URL;#how-to-submit-a-change'>Submitting a Change to the Yocto Project</ulink>"
                                     section in the Yocto Project Development Manual.
                                     </para></listitem>
                                 <listitem><para>Instructions on how to build the BSP using the BSP
@@ -1144,7 +1358,7 @@
 
                 <para>
                     Designed to have a  command interface somewhat like
-                    <ulink url='&YOCTO_DOCS_DEV_URL;#git'>Git</ulink>, each
+                    <ulink url='&YOCTO_DOCS_REF_URL;#git'>Git</ulink>, each
                     tool is structured as a set of sub-commands under a
                     top-level command.
                     The top-level command (<filename>yocto-bsp</filename>

documentation/dev-manual/dev-manual-intro.xml

@@ -5,101 +5,72 @@
 <chapter id='dev-manual-intro'>
 
 <title>The Yocto Project Development Manual</title>
-    <section id='dev-intro'>
-        <title>Introduction</title>
+    <section id='dev-welcome'>
+        <title>Welcome</title>
 
         <para>
             Welcome to the Yocto Project Development Manual!
-            This manual provides information on how to use the Yocto Project to
-            develop embedded Linux images and user-space applications that
-            run on targeted devices.
-            The manual provides an overview of image, kernel, and
-            user-space application development using the Yocto Project.
-            Because much of the information in this manual is general, it
-            contains many references to other sources where you can find more
-            detail.
-            For example, you can find detailed information on Git, repositories,
-            and open source in general in many places on the Internet.
-            Another example specific to the Yocto Project is how to quickly
-            set up your host development system and build an image, which you
-            find in the
-            <ulink url='&YOCTO_DOCS_QS_URL;'>Yocto Project Quick Start</ulink>.
+            This manual provides relevant procedures necessary for developing
+            in the Yocto Project environment (i.e. developing embedded Linux
+            images and user-space applications that run on targeted devices).
+            The manual groups related procedures into higher-level sections.
+            Procedures can consist of high-level steps or low-level steps
+            depending on the topic.
+            You can find conceptual information related to a procedure by
+            following appropriate links to the Yocto Project Reference
+            Manual.
         </para>
 
-        <para>
-            The Yocto Project Development Manual does, however, provide
-            guidance and examples on how to change the kernel source code,
-            reconfigure the kernel, and develop an application using
-            <filename>devtool</filename>.
-        </para>
-
-        <note>
-            By default, using the Yocto Project creates a Poky distribution.
-            However, you can create your own distribution by providing key
-            <link linkend='metadata'>Metadata</link>.
-            A good example is Angstrom, which has had a distribution
-            based on the Yocto Project since its inception.
-            Other examples include commercial distributions like
-            <ulink url='https://www.yoctoproject.org/organization/wind-river-systems'>Wind River Linux</ulink>,
-            <ulink url='https://www.yoctoproject.org/organization/mentor-graphics'>Mentor Embedded Linux</ulink>,
-            <ulink url='https://www.yoctoproject.org/organization/enea-ab'>ENEA Linux</ulink>
-            and <ulink url='https://www.yoctoproject.org/ecosystem/member-organizations'>others</ulink>.
-            See the "<link linkend='creating-your-own-distribution'>Creating Your Own Distribution</link>"
-            section for more information.
-        </note>
-    </section>
-
-    <section id='what-this-manual-provides'>
-        <title>What This Manual Provides</title>
-
         <para>
             The following list describes what you can get from this manual:
             <itemizedlist>
-                <listitem><para>Information that lets you get set
-                    up to develop using the Yocto Project.</para></listitem>
-                <listitem><para>Information to help developers who are new to
-                    the open source environment and to the distributed revision
-                    control system Git, which the Yocto Project uses.
+                <listitem><para>
+                    <emphasis>Setup Procedures:</emphasis>
+                    Procedures that show you how to set
+                    up a Yocto Project Development environment and how
+                    to accomplish the change workflow through logging
+                    defects and submitting changes.
                     </para></listitem>
-                <listitem><para>An understanding of common end-to-end
-                    development models and tasks.</para></listitem>
-                <listitem><para>Information about common development tasks
-                    generally used during image development for
-                    embedded devices.
+                <listitem><para>
+                    <emphasis>Emulation Procedures:</emphasis>
+                    Procedures that show you how to use the
+                    Yocto Project integrated QuickEMUlator (QEMU), which lets
+                    you simulate running on hardware an image you have built
+                    using the OpenEmbedded build system.
                     </para></listitem>
-                <listitem><para>Information on using the Yocto Project
-                    integration of the QuickEMUlator (QEMU), which lets you
-                    simulate running on hardware an image you have built using
-                    the OpenEmbedded build system.
+                <listitem><para>
+                    <emphasis>Common Procedures:</emphasis>
+                    Procedures related to "everyday" tasks you perform while
+                    developing images and applications using the Yocto
+                    Project.
                     </para></listitem>
-                <listitem><para>Many references to other sources of related
-                    information.</para></listitem>
             </itemizedlist>
         </para>
-    </section>
-
-    <section id='what-this-manual-does-not-provide'>
-        <title>What this Manual Does Not Provide</title>
 
         <para>
             This manual will not give you the following:
             <itemizedlist>
-                <listitem><para><emphasis>Step-by-step instructions when those instructions exist in other Yocto
-                    Project documentation:</emphasis>
+                <listitem><para>
+                    <emphasis>Redundant Step-by-step Instructions:</emphasis>
                     For example, the
                     <ulink url='&YOCTO_DOCS_SDK_URL;'>Yocto Project Software Development Kit (SDK) Developer's Guide</ulink>
-                    manual contains detailed instructions on how to install an
+                    contains detailed instructions on how to install an
                     SDK, which is used to develop applications for target
                     hardware.
                     </para></listitem>
-                <listitem><para><emphasis>Reference material:</emphasis>
+                <listitem><para>
+                    <emphasis>Reference or Conceptual Material:</emphasis>
                     This type of material resides in an appropriate reference manual.
                     For example, system variables are documented in the
                     <ulink url='&YOCTO_DOCS_REF_URL;'>Yocto Project Reference Manual</ulink>.
                     </para></listitem>
-                <listitem><para><emphasis>Detailed public information that is not specific to the Yocto Project:</emphasis>
-                    For example, exhaustive information on how to use Git is covered better through the
-                    Internet than in this manual.
+                <listitem><para>
+                    <emphasis>Detailed Public Information Not Specific to the
+                    Yocto Project:</emphasis>
+                    For example, exhaustive information on how to use the
+                    Source Control Manager Git is better covered with Internet
+                    searches and official Git Documentation than through the
+                    Yocto Project documentation.
                     </para></listitem>
             </itemizedlist>
         </para>
@@ -112,10 +83,22 @@
             Because this manual presents information for many different
             topics, supplemental information is recommended for full
             comprehension.
-            See the
+            For introductory information on the Yocto Project, see the
+            <ulink url='&YOCTO_HOME_URL;/ecosystem/yocto-project-backgrounders'>Yocto Project Backgrounders</ulink>
+            on the
+            <ulink url='&YOCTO_HOME_URL;'>Yocto Project Website</ulink>.
+            You can find an introductory to using the Yocto Project by working
+            through the
+            <ulink url='&YOCTO_DOCS_QS_URL;'>Yocto Project Quick Start</ulink>.
+        </para>
+
+        <para>
+            For a comprehensive list of links and other documentation, see the
             "<ulink url='&YOCTO_DOCS_REF_URL;#resources-links-and-related-documentation'>Links and Related Documentation</ulink>"
-            section in the Yocto Project Reference Manual for helpful
-            sources.
+            section in the Yocto Project Reference Manual.
+        </para>
+
+        <para>
         </para>
     </section>
 </chapter>

documentation/dev-manual/dev-manual-qemu.xml

@@ -6,427 +6,305 @@
 
 <title>Using the Quick EMUlator (QEMU)</title>
 
-<para>
-    Quick EMUlator (QEMU) is an Open Source project the Yocto Project uses
-    as part of its development "tool set".
-    As such, the information in this chapter is limited to the
-    Yocto Project integration of QEMU and not QEMU in general.
-    For official information and documentation on QEMU, see the
-    following references:
-    <itemizedlist>
-        <listitem><para><emphasis><ulink url='http://wiki.qemu.org/Main_Page'>QEMU Website</ulink>:</emphasis>
-            The official website for the QEMU Open Source project.
-            </para></listitem>
-        <listitem><para><emphasis><ulink url='http://wiki.qemu.org/Manual'>Documentation</ulink>:</emphasis>
-            The QEMU user manual.
-            </para></listitem>
-    </itemizedlist>
-</para>
-
-<para>
-    This chapter provides an overview of the Yocto Project's integration of
-    QEMU, a description of how you use QEMU and its various options, running
-    under a Network File System (NFS) server, and a few tips and tricks you
-    might find helpful when using QEMU.
-</para>
-
-<section id='qemu-overview'>
-    <title>Overview</title>
-
     <para>
-        Within the context of the Yocto Project, QEMU is an
-        emulator and virtualization machine that allows you to run a complete
-        image you have built using the Yocto Project as just another task
-        on your build system.
-        QEMU is useful for running and testing images and applications on
-        supported Yocto Project architectures without having actual hardware.
-        Among other things, the Yocto Project uses QEMU to run automated
-        Quality Assurance (QA) tests on final images shipped with each
-        release.
+        This chapter provides procedures that show you how to use the
+        Quick EMUlator (QEMU), which is an Open Source project the Yocto
+        Project uses as part of its development "tool set".
+        For reference information on the Yocto Project implementation of QEMU,
+        see the
+        "<ulink url='&YOCTO_DOCS_REF_URL;#ref-quick-emulator-qemu'>Quick EMUlator (QEMU)</ulink>"
+        section in the Yocto Project Reference Manual.
     </para>
 
-    <para>
-        QEMU is made available with the Yocto Project a number of ways.
-        One method is to install a Software Development Kit (SDK).
-        For more information on how to make sure you have
-        QEMU available, see the
-        <ulink url='&YOCTO_DOCS_SDK_URL;#sdk-intro'>Yocto Project Software Development Kit (SDK) Developer's Guide</ulink>.
-    </para>
-</section>
-
-<section id='qemu-running-qemu'>
-    <title>Running QEMU</title>
-
-    <para>
-        Running QEMU involves having your build environment set up, having the
-        right artifacts available, and understanding how to use the many
-        options that are available to you when you start QEMU using the
-        <filename>runqemu</filename> command.
-    </para>
-
-    <section id='qemu-setting-up-the-environment'>
-        <title>Setting Up the Environment</title>
+    <section id='qemu-running-qemu'>
+        <title>Running QEMU</title>
 
         <para>
-            You run QEMU in the same environment from which you run BitBake.
-            This means you need to source a build environment script (i.e.
-            <ulink url='&YOCTO_DOCS_REF_URL;#structure-core-script'><filename>&OE_INIT_FILE;</filename></ulink>
-            or
-            <ulink url='&YOCTO_DOCS_REF_URL;#structure-memres-core-script'><filename>oe-init-build-env-memres</filename></ulink>).
-        </para>
-    </section>
+            To use QEMU, you need to have QEMU installed and initialized as
+            well as have the proper artifacts (i.e. image files and root
+            filesystems) available.
+            Follow these general steps to run QEMU:
+            <orderedlist>
+                <listitem><para>
+                    <emphasis>Install QEMU:</emphasis>
+                    See
+                    "<ulink url='&YOCTO_DOCS_SDK_URL;#the-qemu-emulator'>The QEMU Emulator</ulink>"
+                    section in the Yocto Project Software Development Kit (SDK)
+                    Developer's Guide for information on how to install
+                    QEMU.
+                    </para></listitem>
+                <listitem><para>
+                    <emphasis>Setting Up the Environment:</emphasis>
+                    How you set up the QEMU environment depends on how you
+                    installed QEMU:
+                    <itemizedlist>
+                        <listitem><para>
+                            If you cloned the <filename>poky</filename>
+                            repository or you downloaded and unpacked a
+                            Yocto Project release tarball, you can source
+                            the build environment script (i.e.
+                            <ulink url='&YOCTO_DOCS_REF_URL;#structure-core-script'><filename>&OE_INIT_FILE;</filename></ulink>
+                            or
+                            <ulink url='&YOCTO_DOCS_REF_URL;#structure-memres-core-script'><filename>oe-init-build-env-memres</filename></ulink>):
+                            <literallayout class='monospaced'>
+     $ cd ~/poky
+     $ source oe-init-build-env
+                            </literallayout>
+                            </para></listitem>
+                        <listitem><para>
+                            If you installed a cross-toolchain, you can
+                            run the script that initializes the toolchain.
+                            For example, the following commands run the
+                            initialization script from the default
+                            <filename>poky_sdk</filename> directory:
+                            <literallayout class='monospaced'>
+     . ~/poky_sdk/environment-setup-core2-64-poky-linux
+                            </literallayout>
+                            </para></listitem>
+                    </itemizedlist>
+                    </para></listitem>
+                <listitem><para>
+                    <emphasis>Ensure the Artifacts are in Place:</emphasis>
+                    You need to be sure you have a pre-built kernel that
+                    will boot in QEMU.
+                    You also need the target root filesystem for your target
+                    machine’s architecture:
+                    <itemizedlist>
+                        <listitem><para>
+                            If you have previously built an image for QEMU
+                            (e.g. <filename>qemux86</filename>,
+                            <filename>qemuarm</filename>, and so forth),
+                            then the artifacts are in place in your
+                            <ulink url='&YOCTO_DOCS_REF_URL;#build-directory'>Build Directory</ulink>.
+                            </para></listitem>
+                        <listitem><para>
+                            If you have not built an image, you can go to the
+                            <ulink url='&YOCTO_MACHINES_DL_URL;'>machines/qemu</ulink>
+                            area and download a pre-built image that matches
+                            your architecture and can be run on QEMU.
+                            </para></listitem>
+                    </itemizedlist></para>
 
-    <section id='qemu-using-the-runqemu-command'>
-        <title>Using the <filename>runqemu</filename> Command</title>
-
-        <para>
-            The basic <filename>runqemu</filename> command syntax is as
-            follows:
-            <literallayout class='monospaced'>
+                    <para>See the
+                    "<ulink url='&YOCTO_DOCS_SDK_URL;#sdk-extracting-the-root-filesystem'>Extracting the Root Filesystem</ulink>"
+                    section in the Yocto Project Software Development Kit (SDK)
+                    Developer's Guide for information on how to extract a
+                    root filesystem.
+                    </para></listitem>
+                <listitem><para>
+                    <emphasis>Run QEMU:</emphasis>
+                    The basic <filename>runqemu</filename> command syntax is as
+                    follows:
+                    <literallayout class='monospaced'>
      $ runqemu [<replaceable>option</replaceable> ]  [...]
-            </literallayout>
-            Based on what you provide on the command line,
-            <filename>runqemu</filename> does a good job of figuring out what
-            you are trying to do.
-            For example, by default, QEMU looks for the most recently built
-            image according to the timestamp when it needs to look for an
-            image.
-            Minimally, through the use of options, you must provide either
-            a machine name, a virtual machine image
-            (<filename>*.vmdk</filename>), or a kernel image
-            (<filename>*.bin</filename>).
-        </para>
+                    </literallayout>
+                    Based on what you provide on the command line,
+                    <filename>runqemu</filename> does a good job of figuring
+                    out what you are trying to do.
+                    For example, by default, QEMU looks for the most recently
+                    built image according to the timestamp when it needs to
+                    look for an image.
+                    Minimally, through the use of options, you must provide
+                    either a machine name, a virtual machine image
+                    (<filename>*.vmdk</filename>), or a kernel image
+                    (<filename>*.bin</filename>).</para>
 
-        <para>
-            Following is a description of <filename>runqemu</filename>
-            options you can provide on the command line:
-            <note><title>Tip</title>
-                If you do provide some "illegal" option combination or perhaps
-                you do not provide enough in the way of options,
-                <filename>runqemu</filename> provides appropriate error
-                messaging to help you correct the problem.
-            </note>
-            <itemizedlist>
-                <listitem><para><replaceable>QEMUARCH</replaceable>:
-                    The QEMU machine architecture, which must be "qemux86",
-                    "qemux86_64", "qemuarm", "qemumips", "qemumipsel",
-                    “qemumips64", "qemush4", "qemuppc", "qemumicroblaze",
-                    or "qemuzynq".
-                    </para></listitem>
-                <listitem><para><filename><replaceable>VM</replaceable></filename>:
-                    The virtual machine image, which must be a
-                    <filename>.vmdk</filename> file.
-                    Use this option when you want to boot a
-                    <filename>.vmdk</filename> image.
-                    The image filename you provide must contain one of the
-                    following strings: "qemux86-64", "qemux86", "qemuarm",
-                    "qemumips64", "qemumips", "qemuppc", or "qemush4".
-                    </para></listitem>
-                <listitem><para><replaceable>ROOTFS</replaceable>:
-                    A root filesystem that has one of the following
-                    filetype extensions: "ext2", "ext3", "ext4", "jffs2",
-                    "nfs", or "btrfs".
-                    If the filename you provide for this option uses “nfs”, it
-                    must provide an explicit root filesystem path.
-                    </para></listitem>
-                <listitem><para><replaceable>KERNEL</replaceable>:
-                    A kernel image, which is a <filename>.bin</filename> file.
-                    When you provide a <filename>.bin</filename> file,
-                    <filename>runqemu</filename> detects it and assumes the
-                    file is a kernel image.
-                    </para></listitem>
-                <listitem><para><replaceable>MACHINE</replaceable>:
-                    The architecture of the QEMU machine, which must be one
-                    of the following: "qemux86",
-                    "qemux86-64", "qemuarm", "qemumips", "qemumipsel",
-                    “qemumips64", "qemush4", "qemuppc", "qemumicroblaze",
-                    or "qemuzynq".
-                    The <replaceable>MACHINE</replaceable> and
-                    <replaceable>QEMUARCH</replaceable> options are basically
-                    identical.
-                    If you do not provide a <replaceable>MACHINE</replaceable>
-                    option, <filename>runqemu</filename> tries to determine
-                    it based on other options.
-                    </para></listitem>
-                <listitem><para><filename>ramfs</filename>:
-                    Indicates you are booting an initial RAM disk (initramfs)
-                    image, which means the <filename>FSTYPE</filename> is
-                    <filename>cpio.gz</filename>.
-                    </para></listitem>
-                <listitem><para><filename>iso</filename>:
-                    Indicates you are booting an ISO image, which means the
-                    <filename>FSTYPE</filename> is
-                    <filename>.iso</filename>.
-                    </para></listitem>
-                <listitem><para><filename>nographic</filename>:
-                    Disables the video console, which sets the console to
-                    "ttys0".
-                    </para></listitem>
-                <listitem><para><filename>serial</filename>:
-                    Enables a serial console on
-                    <filename>/dev/ttyS0</filename>.
-                    </para></listitem>
-                <listitem><para><filename>biosdir</filename>:
-                    Establishes a custom directory for BIOS, VGA BIOS and
-                    keymaps.
-                    </para></listitem>
-                <listitem><para><filename>biosfilename</filename>:
-                    Establishes a custom BIOS name.
-                    </para></listitem>
-                <listitem><para><filename>qemuparams=\"<replaceable>xyz</replaceable>\"</filename>:
-                    Specifies custom QEMU parameters.
-                    Use this option to pass options other than the simple
-                    "kvm" and "serial" options.
-                    </para></listitem>
-                <listitem><para><filename>bootparams=\"<replaceable>xyz</replaceable>\"</filename>:
-                    Specifies custom boot parameters for the kernel.
-                    </para></listitem>
-                <listitem><para><filename>audio</filename>:
-                    Enables audio in QEMU.
-                    The <replaceable>MACHINE</replaceable> option must be
-                    either "qemux86" or "qemux86-64" in order for audio to be
-                    enabled.
-                    Additionally, the <filename>snd_intel8x0</filename>
-                    or <filename>snd_ens1370</filename> driver must be
-                    installed in linux guest.
-                    </para></listitem>
-                <listitem><para><filename>slirp</filename>:
-                    Enables "slirp" networking, which is a different way
-                    of networking that does not need root access
-                    but also is not as easy to use or comprehensive
-                    as the default.
-                    </para></listitem>
-                <listitem><para id='kvm-cond'><filename>kvm</filename>:
-                    Enables KVM when running "qemux86" or "qemux86-64"
-                    QEMU architectures.
-                    For KVM to work, all the following conditions must be met:
+                    <para>Here are some additional examples to help illustrate
+                    further QEMU:
                     <itemizedlist>
                         <listitem><para>
-                            Your <replaceable>MACHINE</replaceable> must be either
-qemux86" or "qemux86-64".
+                            This example starts QEMU with
+                            <replaceable>MACHINE</replaceable> set to "qemux86".
+                            Assuming a standard
+                            <ulink url='&YOCTO_DOCS_REF_URL;#build-directory'>Build Directory</ulink>,
+                            <filename>runqemu</filename> automatically finds the
+                            <filename>bzImage-qemux86.bin</filename> image file and
+                            the
+                            <filename>core-image-minimal-qemux86-20140707074611.rootfs.ext3</filename>
+                            (assuming the current build created a
+                            <filename>core-image-minimal</filename> image).
+                            <note>
+                            When more than one image with the same name exists, QEMU finds
+                            and uses the most recently built image according to the
+                            timestamp.
+                            </note>
+                            <literallayout class='monospaced'>
+     $ runqemu qemux86
+                            </literallayout>
                             </para></listitem>
                         <listitem><para>
-                            Your build host has to have the KVM modules
-                            installed, which are
-                            <filename>/dev/kvm</filename>.
-                            </para></listitem>
-                        <listitem><para>
-                            The  build host <filename>/dev/kvm</filename>
-                            directory has to be both writable and readable.
-                            </para></listitem>
-                    </itemizedlist>
-                    </para></listitem>
-                <listitem><para><filename>kvm-vhost</filename>:
-                    Enables KVM with VHOST support when running "qemux86" or "qemux86-64"
-                    QEMU architectures.
-                    For KVM with VHOST to work, the following conditions must
-                    be met:
-                    <itemizedlist>
-                        <listitem><para>
-                            <link linkend='kvm-cond'>kvm</link> option
-                            conditions must be met.
-                            </para></listitem>
-                        <listitem><para>
-                            Your build host has to have virtio net device, which
-                            are <filename>/dev/vhost-net</filename>.
-                            </para></listitem>
-                        <listitem><para>
-                            The build host <filename>/dev/vhost-net</filename>
-                            directory has to be either readable or writable
-                            and “slirp-enabled”.
-                            </para></listitem>
-                    </itemizedlist>
-                    </para></listitem>
-                <listitem><para><filename>publicvnc</filename>:
-                    Enables a VNC server open to all hosts.
-                    </para></listitem>
-            </itemizedlist>
-        </para>
-
-        <para>
-            For further understanding regarding option use with
-            <filename>runqemu</filename>, consider some examples.
-        </para>
-
-        <para>
-            This example starts QEMU with
-            <replaceable>MACHINE</replaceable> set to "qemux86".
-            Assuming a standard
-            <link linkend='build-directory'>Build Directory</link>,
-            <filename>runqemu</filename> automatically finds the
-            <filename>bzImage-qemux86.bin</filename> image file and
-            the
-            <filename>core-image-minimal-qemux86-20140707074611.rootfs.ext3</filename>
-            (assuming the current build created a
-            <filename>core-image-minimal</filename> image).
-            <note>
-                When more than one image with the same name exists, QEMU finds
-                and uses the most recently built image according to the
-                timestamp.
-            </note>
-            <literallayout class='monospaced'>
-    $ runqemu qemux86
-            </literallayout>
-            This example produces the exact same results as the
-            previous example.
-            This command, however, specifically provides the image
-            and root filesystem type.
-            <literallayout class='monospaced'>
+                            This example produces the exact same results as the
+                            previous example.
+                            This command, however, specifically provides the image
+                            and root filesystem type.
+                            <literallayout class='monospaced'>
      $ runqemu qemux86 core-image-minimal ext3
-            </literallayout>
-            This example specifies to boot an initial RAM disk image
-            and to enable audio in QEMU.
-            For this case, <filename>runqemu</filename> set the
-            internal variable <filename>FSTYPE</filename> to
-            "cpio.gz".
-            Also, for audio to be enabled, an appropriate driver must
-            be installed (see the previous description for the
-            <filename>audio</filename> option for more information).
-            <literallayout class='monospaced'>
+                            </literallayout>
+                            </para></listitem>
+                        <listitem><para>
+                            This example specifies to boot an initial RAM disk image
+                            and to enable audio in QEMU.
+                            For this case, <filename>runqemu</filename> set the
+                            internal variable <filename>FSTYPE</filename> to
+                            "cpio.gz".
+                            Also, for audio to be enabled, an appropriate driver must
+                            be installed (see the previous description for the
+                            <filename>audio</filename> option for more information).
+                            <literallayout class='monospaced'>
      $ runqemu qemux86 ramfs audio
-            </literallayout>
-            This example does not provide enough information for
-            QEMU to launch.
-            While the command does provide a root filesystem type, it
-            must also minimally provide a
-            <replaceable>MACHINE</replaceable>,
-            <replaceable>KERNEL</replaceable>, or
-            <replaceable>VM</replaceable> option.
-            <literallayout class='monospaced'>
+                            </literallayout>
+                            </para></listitem>
+                        <listitem><para>
+                            This example does not provide enough information for
+                            QEMU to launch.
+                            While the command does provide a root filesystem type, it
+                            must also minimally provide a
+                            <replaceable>MACHINE</replaceable>,
+                            <replaceable>KERNEL</replaceable>, or
+                            <replaceable>VM</replaceable> option.
+                            <literallayout class='monospaced'>
      $ runqemu ext3
-            </literallayout>
-            This example specifies to boot a virtual machine image
-            (<filename>.vmdk</filename> file).
-            From the <filename>.vmdk</filename>,
-            <filename>runqemu</filename> determines the QEMU
-            architecture (<replaceable>MACHINE</replaceable>) to be
-            "qemux86" and the root filesystem type to be "vmdk".
-            <literallayout class='monospaced'>
+                            </literallayout>
+                            </para></listitem>
+                        <listitem><para>
+                            This example specifies to boot a virtual machine image
+                            (<filename>.vmdk</filename> file).
+                            From the <filename>.vmdk</filename>,
+                            <filename>runqemu</filename> determines the QEMU
+                            architecture (<replaceable>MACHINE</replaceable>) to be
+                            "qemux86" and the root filesystem type to be "vmdk".
+                            <literallayout class='monospaced'>
      $ runqemu /home/scott-lenovo/vm/core-image-minimal-qemux86.vmdk
-            </literallayout>
+                            </literallayout>
+                            </para></listitem>
+                    </itemizedlist>
+                    </para></listitem>
+            </orderedlist>
         </para>
     </section>
-</section>
 
-<section id='qemu-running-under-a-network-file-system-nfs-server'>
-    <title>Running Under a Network File System (NFS) Server</title>
-
-    <para>
-        One method for running QEMU is to run it on an NFS server.
-        This is useful when you need to access the same file system from both
-        the build and the emulated system at the same time.
-        It is also worth noting that the system does not need root privileges
-        to run.
-        It uses a user space NFS server to avoid that.
-        This section describes how to set up for running QEMU using an NFS
-        server and then how you can start and stop the server.
-    </para>
-
-    <section id='qemu-setting-up-to-use-nfs'>
-        <title>Setting Up to Use NFS</title>
+    <section id='switching-between-consoles'>
+        <title>Switching Between Consoles</title>
 
         <para>
-            Once you are able to run QEMU in your environment, you can use the
-            <filename>runqemu-extract-sdk</filename> script, which is located
-            in the <filename>scripts</filename> directory along with
-            <filename>runqemu</filename> script.
-            The <filename>runqemu-extract-sdk</filename> takes a root
-            file system tarball and extracts it into a location that you
-            specify.
-            Then, when you run <filename>runqemu</filename>, you can specify
-            the location that has the file system to pass it to QEMU.
-            Here is an example that takes a file system and extracts it to
-            a directory named <filename>test-nfs</filename>:
-            <literallayout class='monospaced'>
+            When booting or running QEMU, you can switch between
+            supported consoles by using
+            Ctrl+Alt+<replaceable>number</replaceable>.
+            For example, Ctrl+Alt+3 switches you to the serial console
+            as long as that console is enabled.
+            Being able to switch consoles is helpful, for example, if
+            the main QEMU console breaks for some reason.
+            <note>
+                Usually, "2" gets you to the main console and "3"
+                gets you to the serial console.
+            </note>
+        </para>
+    </section>
+
+    <section id='removing-the-splash-screen'>
+        <title>Removing the Splash Screen</title>
+
+        <para>
+            You can remove the splash screen when QEMU is booting by
+            using Alt+left.
+            Removing the splash screen allows you to see what is
+            happening in the background.
+        </para>
+    </section>
+
+    <section id='disabling-the-cursor-grab'>
+        <title>Disabling the Cursor Grab</title>
+
+        <para>
+            The default QEMU integration captures the cursor within the
+            main window.
+            It does this since standard mouse devices only provide
+            relative input and not absolute coordinates.
+            You then have to break out of the grab using the "Ctrl+Alt"
+            key combination.
+            However, the Yocto Project's integration of QEMU enables
+            the wacom USB touch pad driver by default to allow input
+            of absolute coordinates.
+            This default means that the mouse can enter and leave the
+            main window without the grab taking effect leading to a
+            better user experience.
+        </para>
+    </section>
+
+    <section id='qemu-running-under-a-network-file-system-nfs-server'>
+        <title>Running Under a Network File System (NFS) Server</title>
+
+        <para>
+            One method for running QEMU is to run it on an NFS server.
+            This is useful when you need to access the same file system
+            from both the build and the emulated system at the same time.
+            It is also worth noting that the system does not need root
+            privileges to run.
+            It uses a user space NFS server to avoid that.
+            Follow these steps to set up for running QEMU using an NFS
+            server.
+            <orderedlist>
+                <listitem><para>
+                    <emphasis>Extract a Root Filesystem:</emphasis>
+                    Once you are able to run QEMU in your environment, you can
+                    use the <filename>runqemu-extract-sdk</filename> script,
+                    which is located in the <filename>scripts</filename>
+                    directory along with the <filename>runqemu</filename>
+                    script.</para>
+
+                    <para>The <filename>runqemu-extract-sdk</filename> takes a
+                    root filesystem tarball and extracts it into a location
+                    that you specify.
+                    Here is an example that takes a file system and
+                    extracts it to a directory named
+                    <filename>test-nfs</filename>:
+                    <literallayout class='monospaced'>
      runqemu-extract-sdk ./tmp/deploy/images/qemux86/core-image-sato-qemux86.tar.bz2 test-nfs
-            </literallayout>
-            Once you have extracted the file system, you can run
-            <filename>runqemu</filename> normally with the additional
-            location of the file system.
-            You can then also make changes to the files within
-            <filename>./test-nfs</filename> and see those changes appear in the
-            image in real time.
-            Here is an example using the <filename>qemux86</filename> image:
-            <literallayout class='monospaced'>
+                    </literallayout>
+                    </para></listitem>
+                <listitem><para>
+                    <emphasis>Start QEMU:</emphasis>
+                    Once you have extracted the file system, you can run
+                    <filename>runqemu</filename> normally with the additional
+                    location of the file system.
+                    You can then also make changes to the files within
+                    <filename>./test-nfs</filename> and see those changes
+                    appear in the image in real time.
+                    Here is an example using the <filename>qemux86</filename>
+                    image:
+                    <literallayout class='monospaced'>
      runqemu qemux86 ./test-nfs
-            </literallayout>
-        </para>
-    </section>
-
-    <section id='qemu-starting-and-stopping-nfs'>
-        <title>Starting and Stopping NFS</title>
-
-        <para>
-            You can manually start and stop the NFS share using these
-            commands:
-            <itemizedlist>
-                <listitem><para><emphasis><filename>start</filename>:</emphasis>
-                    Starts the NFS share:
-                    <literallayout class='monospaced'>
+                    </literallayout>
+                    </para></listitem>
+            </orderedlist>
+            <note>
+                <para>
+                    Should you need to start, stop, or restart the NFS share,
+                    you can use the following commands:
+                    <itemizedlist>
+                        <listitem><para>
+                            The following command starts the NFS share:
+                            <literallayout class='monospaced'>
      runqemu-export-rootfs start <replaceable>file-system-location</replaceable>
-                    </literallayout>
-                    </para></listitem>
-                <listitem><para><emphasis><filename>stop</filename>:</emphasis>
-                    Stops the NFS share:
-                    <literallayout class='monospaced'>
-     runqemu-export-rootfs stop <replaceable>file-system-location</replaceable>
-                    </literallayout>
-                    </para></listitem>
-                <listitem><para><emphasis><filename>restart</filename>:</emphasis>
-                    Restarts the NFS share:
-                    <literallayout class='monospaced'>
+                            </literallayout>
+                            </para></listitem>
+                        <listitem><para>
+                            The following command stops the NFS share:
+                            <literallayout class='monospaced'>
+         runqemu-export-rootfs stop <replaceable>file-system-location</replaceable>
+                            </literallayout>
+                            </para></listitem>
+                        <listitem><para>
+                            The following command restarts the NFS share:
+                            <literallayout class='monospaced'>
      runqemu-export-rootfs restart <replaceable>file-system-location</replaceable>
-                    </literallayout>
-                    </para></listitem>
-            </itemizedlist>
+                            </literallayout>
+                            </para></listitem>
+                    </itemizedlist>
+                </para>
+            </note>
         </para>
     </section>
-</section>
-
-<section id='qemu-tips-and-tricks'>
-    <title>Tips and Tricks</title>
-
-    <para>
-        The following list describes things you can do to make running QEMU
-        in the context of the Yocto Project a better experience:
-        <itemizedlist>
-            <listitem><para><emphasis>Switching Between Consoles:</emphasis>
-                When booting or running QEMU, you can switch between
-                supported consoles by using
-                Ctrl+Alt+<replaceable>number</replaceable>.
-                For example, Ctrl+Alt+3 switches you to the serial console as
-                long as that console is enabled.
-                Being able to switch consoles is helpful, for example, if the
-                main QEMU console breaks for some reason.
-                <note>
-                    Usually, "2" gets you to the main console and "3" gets you
-                    to the serial console.
-                </note>
-                </para></listitem>
-            <listitem><para><emphasis>Removing the Splash Screen:</emphasis>
-                You can remove the splash screen when QEMU is booting by
-                using Alt+left.
-                Removing the splash screen allows you to see what is happening
-                in the background.
-                </para></listitem>
-            <listitem><para><emphasis>Disabling the Cursor Grab:</emphasis>
-                The default QEMU integration captures the cursor within the
-                main window.
-                It does this since standard mouse devices only provide relative
-                input and not absolute coordinates.
-                You then have to break out of the grab using the "Ctrl+Alt" key
-                combination.
-                However, the Yocto Project's integration of QEMU enables the
-                wacom USB touch pad driver by default to allow input of absolute
-                coordinates.
-                This default means that the mouse can enter and leave the
-                main window without the grab taking effect leading to a better
-                user experience.
-                </para></listitem>
-        </itemizedlist>
-    </para>
-</section>
-
 </chapter>
 <!--
 vim: expandtab tw=80 ts=4

documentation/dev-manual/dev-manual.xml

@@ -24,7 +24,7 @@
             <author>
                 <firstname>Scott</firstname> <surname>Rifenbark</surname>
                 <affiliation>
-                    <orgname>Intel Corporation</orgname>
+                    <orgname>Scotty's Documentation Services, Inc.</orgname>
                 </affiliation>
                 <email>srifenbark@gmail.com</email>
             </author>
@@ -152,8 +152,6 @@
 
     <xi:include href="dev-manual-newbie.xml"/>
 
-    <xi:include href="dev-manual-model.xml"/>
-
     <xi:include href="dev-manual-common-tasks.xml"/>
 
     <xi:include href="dev-manual-qemu.xml"/>

documentation/kernel-dev/kernel-dev-advanced.xml

@@ -11,7 +11,7 @@
     <para>
         In addition to supporting configuration fragments and patches, the
         Yocto Project kernel tools also support rich
-        <ulink url='&YOCTO_DOCS_DEV_URL;#metadata'>Metadata</ulink> that you can
+        <ulink url='&YOCTO_DOCS_REF_URL;#metadata'>Metadata</ulink> that you can
         use to define complex policies and Board Support Package (BSP) support.
         The purpose of the Metadata and the tools that manage it, known as
         the kern-tools (<filename>kern-tools-native_git.bb</filename>), is
@@ -48,7 +48,7 @@
         This variable is typically set to the same value as the
         <ulink url='&YOCTO_DOCS_REF_URL;#var-MACHINE'><filename>MACHINE</filename></ulink>
         variable, which is used by
-        <ulink url='&YOCTO_DOCS_DEV_URL;#bitbake-term'>BitBake</ulink>.
+        <ulink url='&YOCTO_DOCS_REF_URL;#bitbake-term'>BitBake</ulink>.
         However, in some cases, the variable might instead refer to the
         underlying platform of the <filename>MACHINE</filename>.
     </para>
@@ -658,7 +658,7 @@
                 Consider the Minnow BSP definition from the
                 <filename>linux-yocto-4.4</filename> in the
                 Yocto Project
-                <ulink url='&YOCTO_DOCS_DEV_URL;#source-repositories'>Source Repositories</ulink>
+                <ulink url='&YOCTO_DOCS_REF_URL;#source-repositories'>Source Repositories</ulink>
                 (i.e.
                 <filename>yocto-kernel-cache/bsp/minnow</filename>):
                 <literallayout class='monospaced'>

documentation/kernel-dev/kernel-dev-common.xml

@@ -26,7 +26,7 @@
             that you create and prepare your own layer in which to do your
             work.
             Your layer contains its own
-            <ulink url='&YOCTO_DOCS_DEV_URL;#bitbake-term'>BitBake</ulink>
+            <ulink url='&YOCTO_DOCS_REF_URL;#bitbake-term'>BitBake</ulink>
             append files
             (<filename>.bbappend</filename>) and provides a convenient
             mechanism to create your own recipe files
@@ -759,7 +759,7 @@
             working with your own sources.
             When you use your own sources, you will not be able to
             leverage the existing kernel
-            <ulink url='&YOCTO_DOCS_DEV_URL;#metadata'>Metadata</ulink> and
+            <ulink url='&YOCTO_DOCS_REF_URL;#metadata'>Metadata</ulink> and
             stabilization work of the linux-yocto sources.
             However, you will be able to manage your own Metadata in the same
             format as the linux-yocto sources.

documentation/kernel-dev/kernel-dev-concepts-appx.xml

@@ -7,245 +7,464 @@
 
     <section id='kernel-big-picture'>
         <title>Yocto Project Kernel Development and Maintenance</title>
+
         <para>
-            Kernels available through the Yocto Project, like other kernels, are based off the Linux
-            kernel releases from <ulink url='http://www.kernel.org'></ulink>.
-            At the beginning of a major development cycle, the Yocto Project team
-            chooses its kernel based on factors such as release timing, the anticipated release
-            timing of final upstream <filename>kernel.org</filename> versions, and Yocto Project
+            Kernels available through the Yocto Project (Linux Yocto kernels),
+            like other kernels, are based off the Linux kernel releases from
+            <ulink url='http://www.kernel.org'></ulink>.
+            At the beginning of a major Linux kernel development cycle, the
+            Yocto Project team chooses a Linux kernel based on factors such as
+            release timing, the anticipated release timing of final upstream
+            <filename>kernel.org</filename> versions, and Yocto Project
             feature requirements.
-            Typically, the kernel chosen is in the
-            final stages of development by the community.
-            In other words, the kernel is in the release
-            candidate or "rc" phase and not yet a final release.
-            But, by being in the final stages of external development, the team knows that the
-            <filename>kernel.org</filename> final release will clearly be within the early stages of
-            the Yocto Project development window.
+            Typically, the Linux kernel chosen is in the final stages of
+            development by the Linux community.
+            In other words, the Linux kernel is in the release candidate
+            or "rc" phase and has yet to reach final release.
+            But, by being in the final stages of external development, the
+            team knows that the <filename>kernel.org</filename> final release
+            will clearly be within the early stages of the Yocto Project
+            development window.
         </para>
+
         <para>
-            This balance allows the team to deliver the most up-to-date kernel
-            possible, while still ensuring that the team has a stable official release for
-            the baseline Linux kernel version.
+            This balance allows the Yocto Project team to deliver the most
+            up-to-date Linux Yocto kernel possible, while still ensuring that
+            the team has a stable official release for the baseline Linux
+            kernel version.
         </para>
+
         <para>
-            The ultimate source for kernels available through the Yocto Project are released kernels
-            from <filename>kernel.org</filename>.
-            In addition to a foundational kernel from <filename>kernel.org</filename>, the
-            kernels available contain a mix of important new mainline
-            developments, non-mainline developments (when there is no alternative),
-            Board Support Package (BSP) developments,
-            and custom features.
-            These additions result in a commercially released Yocto Project Linux kernel that caters
-            to specific embedded designer needs for targeted hardware.
+            As implied earlier, the ultimate source for Linux Yocto kernels
+            are released kernels from <filename>kernel.org</filename>.
+            In addition to a foundational kernel from
+            <filename>kernel.org</filename>, the available Yocto Linux kernels
+            contain a mix of important new mainline developments, non-mainline
+            developments (when there is no alternative), Board Support Package
+            (BSP) developments, and custom features.
+            These additions result in a commercially released Yocto
+            Project Linux kernel that caters to specific embedded designer
+            needs for targeted hardware.
         </para>
+
         <para>
-            Once a kernel is officially released, the Yocto Project team goes into
-            their next development cycle, or upward revision (uprev) cycle, while still
-            continuing maintenance on the released kernel.
+            You can find a web interface to the Yocto Linux kernels in the
+            <ulink url='&YOCTO_DOCS_REF_URL;#source-repositories'>Source Repositories</ulink>
+            at
+            <ulink url='&YOCTO_GIT_URL;'></ulink>.
+            If you look at the interface, you will see to the left a
+            grouping of Git repositories titled "Yocto Linux Kernel".
+            Within this group, you will find several Linux Yocto kernels
+            developed and included with Yocto Project releases:
+            <itemizedlist>
+                <listitem><para>
+                    <emphasis><filename>linux-yocto-4.1</filename>:</emphasis>
+                    The stable Yocto Project kernel to use with the Yocto
+                    Project Release 2.0.
+                    This kernel is based on the Linux 4.1 released kernel.
+                    </para></listitem>
+                <listitem><para>
+                    <emphasis><filename>linux-yocto-4.4</filename>:</emphasis>
+                    The stable Yocto Project kernel to use with the Yocto
+                    Project Release 2.1.
+                    This kernel is based on the Linux 4.4 released kernel.
+                    </para></listitem>
+                <listitem><para>
+                    <emphasis><filename>linux-yocto-4.9</filename>:</emphasis>
+                    The stable Yocto Project kernel to use with the Yocto
+                    Project Release 2.3.
+                    This kernel is based on the Linux 4.9 released kernel.
+                    </para></listitem>
+                <listitem><para>
+                    <emphasis><filename>linux-yocto-4.10</filename>:</emphasis>
+                    The default stable Yocto Project kernel to use with the
+                    Yocto Project Release 2.3.
+                    This kernel is based on the Linux 4.10 released kernel.
+                    </para></listitem>
+                <listitem><para>
+                    <emphasis><filename>linux-yocto-dev</filename>:</emphasis>
+                    A development kernel based on the latest upstream release
+                    candidate available.
+                    </para></listitem>
+            </itemizedlist>
+            <note><title>Notes</title>
+                Long Term Support Initiative (LTSI) for Yocto Linux
+                kernels is as follows:
+                <itemizedlist>
+                    <listitem><para>
+                        For Yocto Project releases 1.7, 1.8, and 2.0,
+                        the LTSI kernel is
+                        <filename>linux-yocto-3.14</filename>.
+                        </para></listitem>
+                    <listitem><para>
+                        For Yocto Project releases 2.1, 2.2, and 2.3,
+                        the LTSI kernel is <filename>linux-yocto-4.1</filename>.
+                        </para></listitem>
+                    <listitem><para>
+                        <filename>linux-yocto-4.1</filename>,
+                        <filename>linux-yocto-4.4</filename>, and
+                        <filename>linux-yocto-4.9</filename> are all LTS
+                        kernels.
+                        </para></listitem>
+                </itemizedlist>
+            </note>
+        </para>
+
+        <para>
+            Once a Yocto Linux kernel is officially released, the Yocto
+            Project team goes into their next development cycle, or upward
+            revision (uprev) cycle, while still continuing maintenance on the
+            released kernel.
             It is important to note that the most sustainable and stable way
-            to include feature development upstream is through a kernel uprev process.
-            Back-porting hundreds of individual fixes and minor features from various
-            kernel versions is not sustainable and can easily compromise quality.
+            to include feature development upstream is through a kernel uprev
+            process.
+            Back-porting hundreds of individual fixes and minor features from
+            various kernel versions is not sustainable and can easily
+            compromise quality.
         </para>
+
         <para>
-            During the uprev cycle, the Yocto Project team uses an ongoing analysis of
-            kernel development, BSP support, and release timing to select the best
-            possible <filename>kernel.org</filename> version.
-            The team continually monitors community kernel
-            development to look for significant features of interest.
-            The team does consider back-porting large features if they have a significant advantage.
-            User or community demand can also trigger a back-port or creation of new
-            functionality in the Yocto Project baseline kernel during the uprev cycle.
+            During the uprev cycle, the Yocto Project team uses an ongoing
+            analysis of Linux kernel development, BSP support, and release
+            timing to select the best possible <filename>kernel.org</filename>
+            Linux kernel version on which to base subsequent Yocto Linux
+            kernel development.
+            The team continually monitors Linux community kernel development
+            to look for significant features of interest.
+            The team does consider back-porting large features if they have a
+            significant advantage.
+            User or community demand can also trigger a back-port or creation
+            of new functionality in the Yocto Project baseline kernel during
+            the uprev cycle.
         </para>
+
         <para>
-            Generally speaking, every new kernel both adds features and introduces new bugs.
-            These consequences are the basic properties of upstream kernel development and are
-            managed by the Yocto Project team's kernel strategy.
-            It is the Yocto Project team's policy to not back-port minor features to the released kernel.
-            They only consider back-porting significant technological jumps - and, that is done
-            after a complete gap analysis.
-            The reason for this policy is that back-porting any small to medium sized change
-            from an evolving kernel can easily create mismatches, incompatibilities and very
-            subtle errors.
+            Generally speaking, every new Linux kernel both adds features and
+            introduces new bugs.
+            These consequences are the basic properties of upstream
+            Linux kernel development and are managed by the Yocto Project
+            team's Linux Yocto kernel development strategy.
+            It is the Yocto Project team's policy to not back-port minor
+            features to the released Linux Yocto kernel.
+            They only consider back-porting significant technological
+            jumps &dash; and, that is done after a complete gap analysis.
+            The reason for this policy is that back-porting any small to
+            medium sized change from an evolving Linux kernel can easily
+            create mismatches, incompatibilities and very subtle errors.
         </para>
+
         <para>
-            These policies result in both a stable and a cutting
-            edge kernel that mixes forward ports of existing features and significant and critical
-            new functionality.
-            Forward porting functionality in the kernels available through the Yocto Project kernel
-            can be thought of as a "micro uprev."
-            The many “micro uprevs” produce a kernel version with a mix of
-            important new mainline, non-mainline, BSP developments and feature integrations.
-            This kernel gives insight into new features and allows focused
-            amounts of testing to be done on the kernel, which prevents
-            surprises when selecting the next major uprev.
-            The quality of these cutting edge kernels is evolving and the kernels are used in leading edge
-            feature and BSP development.
+            The policies described in this section result in both a stable
+            and a cutting edge Linux Yocto kernel that mixes forward ports of
+            existing Linux kernel features and significant and critical new
+            functionality.
+            Forward porting Linux kernel functionality into the Linux Yocto
+            kernels available through the Yocto Project can be thought of as
+            a "micro uprev."
+            The many “micro uprevs” produce a Linux Yocto kernel version with
+            a mix of important new mainline, non-mainline, BSP developments
+            and feature integrations.
+            This Yocto Linux kernel gives insight into new features and
+            allows focused amounts of testing to be done on the kernel,
+            which prevents surprises when selecting the next major uprev.
+            The quality of these cutting edge Linux Yocto kernels is evolving
+            and the kernels are used in leading edge feature and BSP
+            development.
         </para>
     </section>
 
-    <section id='kernel-architecture'>
-        <title>Kernel Architecture</title>
+    <section id='yocto-linux-kernel-architecture-and-branching-strategies'>
+        <title>Yocto Linux Kernel Architecture and Branching Strategies</title>
+
         <para>
-            This section describes the architecture of the kernels available through the
-            Yocto Project and provides information
-            on the mechanisms used to achieve that architecture.
+            As mentioned earlier, a key goal of the Yocto Project is
+            to present the developer with a kernel that has a clear and
+            continuous history that is visible to the user.
+            The architecture and mechanisms, in particular the branching
+            strategies, used achieve that goal in a manner similar to
+            upstream Linux kernel development in
+            <filename>kernel.org</filename>.
         </para>
 
-        <section id='architecture-overview'>
-            <title>Overview</title>
-            <para>
-                As mentioned earlier, a key goal of the Yocto Project is to present the
-                developer with
-                a kernel that has a clear and continuous history that is visible to the user.
-                The architecture and mechanisms used achieve that goal in a manner similar to the
-                upstream <filename>kernel.org</filename>.
-            </para>
-            <para>
-                You can think of a Yocto Project kernel as consisting of a baseline Linux kernel with
-                added features logically structured on top of the baseline.
-                The features are tagged and organized by way of a branching strategy implemented by the
-                source code manager (SCM) Git.
-                For information on Git as applied to the Yocto Project, see the
-                "<ulink url='&YOCTO_DOCS_DEV_URL;#git'>Git</ulink>" section in the
-                Yocto Project Development Manual.
-            </para>
-            <para>
-                The result is that the user has the ability to see the added features and
-                the commits that make up those features.
-                In addition to being able to see added features, the user can also view the history of what
-                made up the baseline kernel.
-            </para>
-            <para>
-                The following illustration shows the conceptual Yocto Project kernel.
-            </para>
-            <para>
-                <imagedata fileref="figures/kernel-architecture-overview.png" width="6in" depth="7in" align="center" scale="100" />
-            </para>
-            <para>
-                In the illustration, the "Kernel.org Branch Point"
-                marks the specific spot (or release) from
-                which the Yocto Project kernel is created.
-                From this point "up" in the tree, features and differences are organized and tagged.
-            </para>
-            <para>
-                The "Yocto Project Baseline Kernel" contains functionality that is common to every kernel
-                type and BSP that is organized further up the tree.
-                Placing these common features in the
-                tree this way means features do not have to be duplicated along individual branches of the
-                structure.
-            </para>
-            <para>
-                From the Yocto Project Baseline Kernel, branch points represent specific functionality
-                for individual BSPs as well as real-time kernels.
-                The illustration represents this through three BSP-specific branches and a real-time
-                kernel branch.
-                Each branch represents some unique functionality for the BSP or a real-time kernel.
-            </para>
-            <para>
-                In this example structure, the real-time kernel branch has common features for all
-                real-time kernels and contains
-                more branches for individual BSP-specific real-time kernels.
-                The illustration shows three branches as an example.
-                Each branch points the way to specific, unique features for a respective real-time
-                kernel as they apply to a given BSP.
-            </para>
-            <para>
-                The resulting tree structure presents a clear path of markers (or branches) to the
-                developer that, for all practical purposes, is the kernel needed for any given set
-                of requirements.
-            </para>
-        </section>
+        <para>
+            You can think of a Yocto Linux kernel as consisting of a
+            baseline Linux kernel with added features logically structured
+            on top of the baseline.
+            The features are tagged and organized by way of a branching
+            strategy implemented by the Yocto Project team using the
+            Source Code Manager (SCM) Git.
+            <note><title>Notes</title>
+                <itemizedlist>
+                    <listitem><para>
+                        Git is the obvious SCM for meeting the Yocto Linux
+                        kernel organizational and structural goals
+                        described in this section.
+                        Not only is Git the SCM for Linux kernel
+                        development in <filename>kernel.org</filename> but,
+                        Git continues to grow in popularity and supports
+                        many different work flows, front-ends and
+                        management techniques.
+                        </para></listitem>
+                    <listitem><para>
+                        You can find documentation on Git at
+                        <ulink url='http://git-scm.com/documentation'></ulink>.
+                        You can also get an introduction to Git as it
+                        applies to the Yocto Project in the
+                        "<ulink url='&YOCTO_DOCS_REF_URL;#git'>Git</ulink>"
+                        section in the Yocto Project Reference Manual.
+                        The latter reference provides an overview of
+                        Git and presents a minimal set of Git commands
+                        that allows you to be functional using Git.
+                        You can use as much, or as little, of what Git
+                        has to offer to accomplish what you need for your
+                        project.
+                        You do not have to be a "Git Expert" in order to
+                        use it with the Yocto Project.
+                        </para></listitem>
+                </itemizedlist>
+            </note>
+        </para>
 
-        <section id='branching-and-workflow'>
-            <title>Branching Strategy and Workflow</title>
-            <para>
-                The Yocto Project team creates kernel branches at points where functionality is
-                no longer shared and thus, needs to be isolated.
-                For example, board-specific incompatibilities would require different functionality
-                and would require a branch to separate the features.
-                Likewise, for specific kernel features, the same branching strategy is used.
-            </para>
-            <para>
-                This branching strategy results in a tree that has features organized to be specific
-                for particular functionality, single kernel types, or a subset of kernel types.
-                This strategy also results in not having to store the same feature twice
-                internally in the tree.
-                Rather, the kernel team stores the unique differences required to apply the
-                feature onto the kernel type in question.
-                <note>
-                    The Yocto Project team strives to place features in the tree such that they can be
-                    shared by all boards and kernel types where possible.
-                    However, during development cycles or when large features are merged,
-                    the team cannot always follow this practice.
-                    In those cases, the team uses isolated branches to merge features.
-                </note>
-            </para>
-            <para>
-                BSP-specific code additions are handled in a similar manner to kernel-specific additions.
-                Some BSPs only make sense given certain kernel types.
-                So, for these types, the team creates branches off the end of that kernel type for all
-                of the BSPs that are supported on that kernel type.
-                From the perspective of the tools that create the BSP branch, the BSP is really no
-                different than a feature.
-                Consequently, the same branching strategy applies to BSPs as it does to features.
-                So again, rather than store the BSP twice, the team only stores the unique
-                differences for the BSP across the supported multiple kernels.
-            </para>
-            <para>
-                While this strategy can result in a tree with a significant number of branches, it is
-                important to realize that from the developer's point of view, there is a linear
-                path that travels from the baseline <filename>kernel.org</filename>, through a select
-                group of features and ends with their BSP-specific commits.
-                In other words, the divisions of the kernel are transparent and are not relevant
-                to the developer on a day-to-day basis.
-                From the developer's perspective, this path is the "master" branch.
-                The developer does not need to be aware of the existence of any other branches at all.
-                Of course, there is value in the existence of these branches
-                in the tree, should a person decide to explore them.
-                For example, a comparison between two BSPs at either the commit level or at the line-by-line
-                code <filename>diff</filename> level is now a trivial operation.
-            </para>
-            <para>
-                Working with the kernel as a structured tree follows recognized community best practices.
-                In particular, the kernel as shipped with the product, should be
-                considered an "upstream source" and viewed as a series of
-                historical and documented modifications (commits).
-                These modifications represent the development and stabilization done
-                by the Yocto Project kernel development team.
-            </para>
-            <para>
-                Because commits only change at significant release points in the product life cycle,
-                developers can work on a branch created
-                from the last relevant commit in the shipped Yocto Project kernel.
-                As mentioned previously, the structure is transparent to the developer
-                because the kernel tree is left in this state after cloning and building the kernel.
-            </para>
-        </section>
+        <para>
+            Using Git's tagging and branching features, the Yocto Project
+            team creates kernel branches at points where functionality is
+            no longer shared and thus, needs to be isolated.
+            For example, board-specific incompatibilities would require
+            different functionality and would require a branch to
+            separate the features.
+            Likewise, for specific kernel features, the same branching
+            strategy is used.
+        </para>
 
-        <section id='source-code-manager-git'>
-            <title>Source Code Manager - Git</title>
-            <para>
-                The Source Code Manager (SCM) is Git.
-                This SCM is the obvious mechanism for meeting the previously mentioned goals.
-                Not only is it the SCM for <filename>kernel.org</filename> but,
-                Git continues to grow in popularity and supports many different work flows,
-                front-ends and management techniques.
-            </para>
-            <para>
-                You can find documentation on Git at <ulink url='http://git-scm.com/documentation'></ulink>.
-                You can also get an introduction to Git as it applies to the Yocto Project in the
-                "<ulink url='&YOCTO_DOCS_DEV_URL;#git'>Git</ulink>"
-                section in the Yocto Project Development Manual.
-                These referenced sections overview Git and describe a minimal set of
-                commands that allows you to be functional using Git.
-                <note>
-                    You can use as much, or as little, of what Git has to offer to accomplish what
-                    you need for your project.
-                    You do not have to be a "Git Master" in order to use it with the Yocto Project.
-                </note>
-            </para>
-        </section>
+        <para>
+            This "tree-like" architecture results in a tree that has
+            features organized to be specific for particular
+            functionality, single kernel types, or a subset of kernel
+            types.
+            Thus, the user has the ability to see the added
+            features and the commits that make up those features.
+            In addition to being able to see added features, the user
+            can also view the history of what made up the baseline
+            Linux kernel.
+        </para>
+
+        <para>
+            Another consequence of this strategy results in not having to
+            store the same feature twice internally in the tree.
+            Rather, the kernel team stores the unique differences required
+            to apply the feature onto the kernel type in question.
+            <note>
+                The Yocto Project team strives to place features in the tree
+                such that features can be shared by all boards and kernel
+                types where possible.
+                However, during development cycles or when large features
+                are merged, the team cannot always follow this practice.
+                In those cases, the team uses isolated branches to merge
+                features.
+            </note>
+        </para>
+
+        <para>
+            BSP-specific code additions are handled in a similar manner to
+            kernel-specific additions.
+            Some BSPs only make sense given certain kernel types.
+            So, for these types, the team creates branches off the end
+            of that kernel type for all of the BSPs that are supported on
+            that kernel type.
+            From the perspective of the tools that create the BSP branch,
+            the BSP is really no different than a feature.
+            Consequently, the same branching strategy applies to BSPs as
+            it does to kernel features.
+            So again, rather than store the BSP twice, the team only
+            stores the unique differences for the BSP across the supported
+            multiple kernels.
+        </para>
+
+        <para>
+            While this strategy can result in a tree with a significant number
+            of branches, it is important to realize that from the developer's
+            point of view, there is a linear path that travels from the
+            baseline <filename>kernel.org</filename>, through a select
+            group of features and ends with their BSP-specific commits.
+            In other words, the divisions of the kernel are transparent and
+            are not relevant to the developer on a day-to-day basis.
+            From the developer's perspective, this path is the "master" branch
+            in Git terms.
+            The developer does not need to be aware of the existence of any
+            other branches at all.
+            Of course, value exists in the having these branches in the tree,
+            should a person decide to explore them.
+            For example, a comparison between two BSPs at either the commit
+            level or at the line-by-line code <filename>diff</filename> level
+            is now a trivial operation.
+        </para>
+
+        <para>
+            The following illustration shows the conceptual Yocto
+            Linux kernel.
+            <imagedata fileref="figures/kernel-architecture-overview.png" width="6in" depth="7in" align="center" scale="100" />
+        </para>
+
+        <para>
+            In the illustration, the "Kernel.org Branch Point" marks the
+            specific spot (or Linux kernel release) from which the
+            Yocto Linux kernel is created.
+            From this point forward in the tree, features and differences
+            are organized and tagged.
+        </para>
+
+        <para>
+            The "Yocto Project Baseline Kernel" contains functionality that
+            is common to every kernel type and BSP that is organized
+            further along in the tree.
+            Placing these common features in the tree this way means
+            features do not have to be duplicated along individual
+            branches of the tree structure.
+        </para>
+
+        <para>
+            From the "Yocto Project Baseline Kernel", branch points represent
+            specific functionality for individual Board Support Packages
+            (BSPs) as well as real-time kernels.
+            The illustration represents this through three BSP-specific
+            branches and a real-time kernel branch.
+            Each branch represents some unique functionality for the BSP
+            or for a real-time Yocto Linux kernel.
+        </para>
+
+        <para>
+            In this example structure, the "Real-time (rt) Kernel" branch has
+            common features for all real-time Yocto Linux kernels and
+            contains more branches for individual BSP-specific real-time
+            kernels.
+            The illustration shows three branches as an example.
+            Each branch points the way to specific, unique features for a
+            respective real-time kernel as they apply to a given BSP.
+        </para>
+
+        <para>
+            The resulting tree structure presents a clear path of markers
+            (or branches) to the developer that, for all practical
+            purposes, is the Yocto Linux kernel needed for any given set of
+            requirements.
+            <note>
+                Keep in mind the figure does not take into account all the
+                supported Yocto Linux kernels, but rather shows a single
+                generic kernel just for conceptual purposes.
+                Also keep in mind that this structure represents the Yocto
+                Project
+                <ulink url='&YOCTO_DOCS_REF_URL;#source-repositories'>Source Repositories</ulink>
+                that are either pulled from during the build or established
+                on the host development system prior to the build by either
+                cloning a particular kernel's Git repository or by
+                downloading and unpacking a tarball.
+            </note>
+        </para>
+
+        <para>
+            Working with the kernel as a structured tree follows recognized
+            community best practices.
+            In particular, the kernel as shipped with the product, should be
+            considered an "upstream source" and viewed as a series of
+            historical and documented modifications (commits).
+            These modifications represent the development and stabilization
+            done by the Yocto Project kernel development team.
+        </para>
+
+        <para>
+            Because commits only change at significant release points in the
+            product life cycle, developers can work on a branch created
+            from the last relevant commit in the shipped Yocto Project kernel.
+            As mentioned previously, the structure is transparent to the
+            developer because the kernel tree is left in this state after
+            cloning and building the kernel.
+        </para>
+    </section>
+
+    <section id='kernel-build-file-hierarchy'>
+        <title>Kernel Build File Hierarchy</title>
+
+        <para>
+            Upstream storage of all the available kernel source code is
+            one thing, while representing and using the code on your host
+            development system is another.
+            Conceptually, you can think of the kernel source repositories
+            as all the source files necessary for all the supported
+            Yocto Linux kernels.
+            As a developer, you are just interested in the source files
+            for the kernel on which you are working.
+            And, furthermore, you need them available on your host system.
+        </para>
+
+        <para>
+            Kernel source code is available on your host system several
+            different ways:
+            <itemizedlist>
+                <listitem><para>
+                    <emphasis>Files Accessed While using <filename>devtool</filename>:</emphasis>
+                    <filename>devtool</filename>, which is available with the
+                    Yocto Project, is the preferred method by which to
+                    modify the kernel.
+                    See the
+                    "<link linkend='kernel-modification-workflow'>Kernel Modification Workflow</link>"
+                    section.
+                    </para></listitem>
+                <listitem><para>
+                    <emphasis>Cloned Repository:</emphasis>
+                    If you are working in the kernel all the time, you probably
+                    would want to set up your own local Git repository of the
+                    Yocto Linux kernel tree.
+                    For information on how to clone a Yocto Linux kernel
+                    Git repository, see the
+                    "<ulink url='&YOCTO_DOCS_DEV_URL;#local-kernel-files'>Setting Up to Work On a Kernel</ulink>"
+                    section in the Yocto Project Development Manual.
+                    </para></listitem>
+                <listitem><para>
+                    <emphasis>Temporary Source Files from a Build:</emphasis>
+                    If you just need to make some patches to the kernel using
+                    a traditional BitBake workflow (i.e. not using the
+                    <filename>devtool</filename>), you can access temporary
+                    kernel source files that were extracted and used during
+                    a kernel build.
+                    </para></listitem>
+            </itemizedlist>
+        </para>
+
+        <para>
+            The temporary kernel source files resulting from a build using
+            BitBake have a particular hierarchy.
+            When you build the kernel on your development system, all files
+            needed for the build are taken from the source repositories
+            pointed to by the
+            <ulink url='&YOCTO_DOCS_REF_URL;#var-SRC_URI'><filename>SRC_URI</filename></ulink>
+            variable and gathered in a temporary work area where they are
+            subsequently used to create the unique kernel.
+            Thus, in a sense, the process constructs a local source tree
+            specific to your kernel from which to generate the new kernel
+            image.
+        </para>
+
+        <para>
+            The following figure shows the temporary file structure
+            created on your host system when you build the kernel using
+            Bitbake.
+            This
+            <ulink url='&YOCTO_DOCS_REF_URL;#build-directory'>Build Directory</ulink>
+            contains all the source files used during the build.
+            <imagedata fileref="figures/kernel-overview-2-generic.png"
+                width="6in" depth="5in" align="center" scale="100" />
+        </para>
+
+        <para>
+            Again, for additional information on the Yocto Project kernel's
+            architecture and its branching strategy, see the
+            "<link linkend='yocto-linux-kernel-architecture-and-branching-strategies'>Yocto Linux Kernel Architecture and Branching Strategies</link>"
+            section.
+            You can also reference the
+            "<ulink url='&YOCTO_DOCS_DEV_URL;#patching-the-kernel'>Patching the Kernel</ulink>"
+            section in the Yocto Project Development Manual for a detailed
+            example that modifies the kernel.
+        </para>
     </section>
 </appendix>
 <!--

documentation/kernel-dev/kernel-dev-examples.xml

@@ -241,8 +241,8 @@
             You can find Git documentation at
             <ulink url='http://git-scm.com/documentation'></ulink>.
             You can find a simple overview of using Git with the Yocto Project in the
-            "<ulink url='&YOCTO_DOCS_DEV_URL;#git'>Git</ulink>"
-            section of the Yocto Project Development Manual.
+            "<ulink url='&YOCTO_DOCS_REF_URL;#git'>Git</ulink>"
+            section of the Yocto Project Reference Manual.
         </para>
 
         <section id='change-inspection-kernel-changes-commits'>
@@ -693,8 +693,8 @@
                     The messages used to commit changes are a large part of these standards.
                     Consequently, be sure that the headers for each commit have the required information.
                     For information on how to follow the Yocto Project commit message standards, see the
-                    "<ulink url='&YOCTO_DOCS_DEV_URL;#how-to-submit-a-change'>How to Submit a
-                    Change</ulink>" section in the Yocto Project Development Manual.
+                    "<ulink url='&YOCTO_DOCS_DEV_URL;#how-to-submit-a-change'>Submitting a Change to the Yocto Project</ulink>"
+                    section in the Yocto Project Development Manual.
                 </para>
 
                 <para>

documentation/kernel-dev/kernel-dev-intro.xml

@@ -5,144 +5,282 @@
 <chapter id='kernel-dev-intro'>
 <title>Introduction</title>
 
-<!--
-<para>
-    <emphasis>AR - Darren Hart:</emphasis>  See if the concepts in these
-    three bullets are adequately covered in somewhere in this manual:
-    <itemizedlist>
-        <listitem><para>Do we convey that our kernel Git repositories
-            have a clear and continuous history, similar to the way the
-            kernel Git repositories for <filename>kernel.org</filename>
-            do.
-            </para></listitem>
-        <listitem><para>Does the manual note that Yocto Project delivers
-            a key set of supported kernel types, where
-            each type is tailored to meet a specific use (e.g. networking,
-            consumer, devices, and so forth).</para></listitem>
-        <listitem><para>Do we convey that the Yocto Project uses a
-            Git branching strategy that, from a
-            developer's point of view, results in a linear path from the
-            baseline kernel.org, through a select group of features and
-            ends with their BSP-specific commits.</para></listitem>
-    </itemizedlist>
-</para>
--->
+<section id='kernel-dev-overview'>
+    <title>Overview</title>
 
-    <section id='kernel-dev-overview'>
-        <title>Overview</title>
+    <para>
+        Regardless of how you intend to make use of the Yocto Project,
+        chances are you will work with the Linux kernel.
+        This manual introduces the kernel development process and
+        provides background information on the Yocto Linux kernel
+        <ulink url='&YOCTO_DOCS_REF_URL;#metadata'>Metadata</ulink>,
+        describes common tasks you can perform using the kernel tools,
+        and shows you how to use the kernel Metadata needed to work with
+        the kernel inside the Yocto Project.
+   </para>
 
-        <para>
-            Regardless of how you intend to make use of the Yocto Project,
-            chances are you will work with the Linux kernel.
-            This manual provides background information on the Yocto Linux kernel
-            <ulink url='&YOCTO_DOCS_DEV_URL;#metadata'>Metadata</ulink>,
-            describes common tasks you can perform using the kernel tools,
-            and shows you how to use the kernel Metadata needed to work with
-            the kernel inside the Yocto Project.
-       </para>
+   <para>
+        Each Yocto Project release has a set of Yocto Linux kernel recipes,
+        whose Git repositories you can view in the Yocto
+        <ulink url='&YOCTO_GIT_URL;'>Source Repositories</ulink> under
+        the "Yocto Linux Kernel" heading.
+        New recipes for the release track the latest Linux kernel
+        upstream developments from
+        <ulink url='http://www.kernel.org'></ulink> and introduce
+        newly-supported platforms.
+        Previous recipes in the release are refreshed and supported for at
+        least one additional Yocto Project release.
+        As they align, these previous releases are updated to include the
+        latest from the
+        <ulink url='&YOCTO_HOME_URL;/organization/long-term-support-initiative-ltsi'>Long Term Support Initiative</ulink>
+        (LTSI) project.
+        Also included is a Yocto Linux kernel development recipe
+        (<filename>linux-yocto-dev.bb</filename>) should you want to work
+        with the very latest in upstream Yocto Linux kernel development and
+        kernel Metadata development.
+        <note>
+            For more on Yocto Linux kernels, see the
+            "<link linkend='kernel-big-picture'>Yocto Project Kernel Development and Maintenance</link>
+            section.
+        </note>
+    </para>
 
-       <para>
-            Each Yocto Project release has a set of linux-yocto recipes, whose
-            Git repositories you can view in the Yocto
-            <ulink url='&YOCTO_GIT_URL;'>Source Repositories</ulink> under
-            the "Yocto Linux Kernel" heading.
-            New recipes for the release track the latest upstream developments
-            and introduce newly-supported platforms.
-            Previous recipes in the release are refreshed and supported for at
-            least one additional release.
-            As they align, these previous releases are updated to include the
-            latest from the
-            <ulink url='&YOCTO_HOME_URL;/organization/long-term-support-initiative-ltsi'>Long Term Support Initiative</ulink>
-            (LTSI) project.
-            Also included is a linux-yocto development recipe
-            (<filename>linux-yocto-dev.bb</filename>) should you want to work
-            with the very latest in upstream Linux kernel development and
-            kernel Metadata development.
-        </para>
+    <para>
+        The Yocto Project also provides a powerful set of kernel
+        tools for managing Yocto Linux kernel sources and configuration data.
+        You can use these tools to make a single configuration change,
+        apply multiple patches, or work with your own kernel sources.
+    </para>
 
-        <para>
-            The Yocto Project also provides a powerful set of kernel
-            tools for managing Linux kernel sources and configuration data.
-            You can use these tools to make a single configuration change,
-            apply multiple patches, or work with your own kernel sources.
-        </para>
+    <para>
+        In particular, the kernel tools allow you to generate configuration
+        fragments that specify only what you must, and nothing more.
+        Configuration fragments only need to contain the highest level
+        visible <filename>CONFIG</filename> options as presented by the
+        Yocto Linux kernel <filename>menuconfig</filename> system.
+        Contrast this against a complete Yocto Linux kernel
+        <filename>.config</filename> file, which includes all the automatically
+        selected <filename>CONFIG</filename> options.
+        This efficiency reduces your maintenance effort and allows you
+        to further separate your configuration in ways that make sense for
+        your project.
+        A common split separates policy and hardware.
+        For example, all your kernels might support the
+        <filename>proc</filename> and <filename>sys</filename> filesystems,
+        but only specific boards require sound, USB, or specific drivers.
+        Specifying these configurations individually allows you to aggregate
+        them together as needed, but maintains them in only one place.
+        Similar logic applies to separating source changes.
+    </para>
 
-        <para>
-            In particular, the kernel tools allow you to generate configuration
-            fragments that specify only what you must, and nothing more.
-            Configuration fragments only need to contain the highest level
-            visible <filename>CONFIG</filename> options as presented by the Linux
-            kernel <filename>menuconfig</filename> system.
-            Contrast this against a complete Linux kernel
-            <filename>.config</filename>, which includes all the automatically
-            selected <filename>CONFIG</filename> options.
-            This efficiency reduces your maintenance effort and allows you
-            to further separate your configuration in ways that make sense for
-            your project.
-            A common split separates policy and hardware.
-            For example, all your kernels might support
-            the <filename>proc</filename> and <filename>sys</filename> filesystems,
-            but only specific boards require sound, USB, or specific drivers.
-            Specifying these configurations individually allows you to aggregate
-            them together as needed, but maintains them in only one place.
-            Similar logic applies to separating source changes.
-        </para>
+    <para>
+        If you do not maintain your own kernel sources and need to make
+        only minimal changes to the sources, the released recipes provide a
+        vetted base upon which to layer your changes.
+        Doing so allows you to benefit from the continual kernel
+        integration and testing performed during development of the
+        Yocto Project.
+    </para>
 
-        <para>
-            If you do not maintain your own kernel sources and need to make
-            only minimal changes to the sources, the released recipes provide a
-            vetted base upon which to layer your changes.
-            Doing so allows you to benefit from the continual kernel
-            integration and testing performed during development of the
-            Yocto Project.
-        </para>
+    <para>
+        If, instead, you have a very specific Linux kernel source tree
+        and are unable to align with one of the official Yocto Linux kernel
+        recipes, an alternative exists by which you can use the Yocto
+        Project Linux kernel tools with your own kernel sources.
+    </para>
+</section>
 
-        <para>
-            If, instead, you have a very specific Linux kernel source tree
-            and are unable to align with one of the official linux-yocto
-            recipes, an alternative exists by which you can use the Yocto
-            Project Linux kernel tools with your own kernel sources.
-        </para>
-    </section>
+<section id='kernel-modification-workflow'>
+    <title>Kernel Modification Workflow</title>
 
-    <section id='kernel-dev-other-resources'>
-        <title>Other Resources</title>
+    <para>
+        Kernel modification involves changing the Yocto Project kernel,
+        which could involve changing configuration options as well as adding
+        new kernel recipes.
+        Configuration changes can be added in the form of configuration
+        fragments, while recipe modification comes through the kernel's
+        <filename>recipes-kernel</filename> area in a kernel layer you create.
+    </para>
 
-        <para>
-            The sections that follow provide instructions for completing
-            specific Linux kernel development tasks.
-            These instructions assume you are comfortable working with
-            <ulink url='http://openembedded.org/wiki/Bitbake'>BitBake</ulink>
-            recipes and basic open-source development tools.
-            Understanding these concepts will facilitate the process of working
-            with the kernel recipes.
-            If you find you need some additional background, please be sure to
-            review and understand the following documentation:
-            <itemizedlist>
-                <listitem><para><ulink url='&YOCTO_DOCS_QS_URL;'>Yocto Project Quick Start</ulink>
-                    </para></listitem>
-                <listitem><para>The "<ulink url='&YOCTO_DOCS_DEV_URL;#dev-modifying-source-code'>Modifying Source Code</ulink>"
-                    section in the Yocto Project Development Manual
-                    </para></listitem>
-                <listitem><para>The "<ulink url='&YOCTO_DOCS_DEV_URL;#understanding-and-creating-layers'>Understanding and Creating Layers</ulink>" section
-                    in the Yocto Project Development Manual</para></listitem>
-                <listitem><para>The "<ulink url='&YOCTO_DOCS_DEV_URL;#modifying-the-kernel'>Modifying the Kernel</ulink>" section
-                    in the Yocto Project Development Manual.</para></listitem>
-            </itemizedlist>
-        </para>
+    <para>
+        This section presents a high-level overview of the Yocto Project
+        kernel modification workflow.
+        You can find additional information here:
+        <itemizedlist>
+            <listitem><para>
+                "<ulink url='&YOCTO_DOCS_DEV_URL;#patching-the-kernel'>Patching the Kernel</ulink>"
+                in the Yocto Project Development Manual.
+                </para></listitem>
+            <listitem><para>
+                "<ulink url='&YOCTO_DOCS_DEV_URL;#configuring-the-kernel'>Configuring the Kernel</ulink>"
+                in the Yocto Project Development Manual.
+                </para></listitem>
+        </itemizedlist>
+        This illustration and the following list summarizes the kernel
+        modification general workflow.
+        <imagedata fileref="figures/kernel-dev-flow.png"
+            width="6in" depth="5in" align="center" scalefit="1" />
+    </para>
+
+    <para>
+        <orderedlist>
+            <listitem><para>
+                <emphasis>Set up Your Host Development System to Support
+                Development Using the Yocto Project</emphasis>:
+                See
+                "<ulink url='&YOCTO_DOCS_QS_URL;#the-linux-distro'>The Linux Distribution</ulink>"
+                and
+                "<ulink url='&YOCTO_DOCS_QS_URL;#packages'>The Build Host Packages</ulink>"
+                sections both in the Yocto Project Quick Start for
+                requirements.
+                </para></listitem>
+            <listitem><para>
+                <emphasis>Clone the <filename>poky</filename> Repository</emphasis>:
+                Having the
+                <ulink url='&YOCTO_DOCS_REF_URL;#source-directory'>Source Directory</ulink>
+                on your system gives you access to the build process and tools
+                you need.
+                For information on how to get these files, see the
+                "<ulink url='&YOCTO_DOCS_DEV_URL;#cloning-the-poky-repository'>Cloning the <filename>poky</filename> Repository</ulink>"
+                section in the Yocto Project Development Manual.
+                </para></listitem>
+            <listitem><para>
+                <emphasis>Establish the Temporary Kernel Source Files</emphasis>:
+                Temporary kernel source files are kept in the
+                <ulink url='&YOCTO_DOCS_REF_URL;#build-directory'>Build Directory</ulink>
+                created by the OpenEmbedded build system when you run BitBake.
+                If you have never built the kernel in which you are
+                interested, you need to run an initial build to
+                establish local kernel source files.</para>
+
+                <para>If you are building an image for the first time, you
+                need to get the build environment ready by sourcing an
+                environment setup script
+                (i.e. <filename>oe-init-build-env</filename> or
+                <filename>oe-init-build-env-memres</filename>).
+                You also need to be sure two key configuration files
+                (<filename>local.conf</filename> and
+                <filename>bblayers.conf</filename>) are configured
+                appropriately.</para>
+
+                <para>The entire process for building an image is overviewed
+                in the
+                "<ulink url='&YOCTO_DOCS_QS_URL;#qs-building-images'>Building Images</ulink>"
+                section of the Yocto Project Quick Start.
+                You might want to reference this information.
+                You can find more information on BitBake in the
+                <ulink url='&YOCTO_DOCS_BB_URL;'>BitBake User Manual</ulink>.
+                </para>
+
+                <para>The build process supports several types of images to
+                satisfy different needs.
+                See the "<ulink url='&YOCTO_DOCS_REF_URL;#ref-images'>Images</ulink>"
+                chapter in the Yocto Project Reference Manual for information
+                on supported images.
+                </para></listitem>
+            <listitem><para>
+                <emphasis>Make Changes to the Kernel Source Code if
+                applicable</emphasis>:
+                Modifying the kernel does not always mean directly
+                changing source files.
+                However, if you have to do this, you make the changes to the
+                files in the Build Directory.
+                </para></listitem>
+            <listitem><para>
+                <emphasis>Make Kernel Configuration Changes if
+                Applicable</emphasis>:
+                If your situation calls for changing the kernel's
+                configuration, you can use
+                <link linkend='generating-configuration-files'><filename>menuconfig</filename></link>,
+                which allows you to interactively develop and test the
+                configuration changes you are making to the kernel.
+                Saving changes you make with <filename>menuconfig</filename>
+                updates the kernel's <filename>.config</filename> file.
+                <note><title>Warning</title>
+                    Try to resist the temptation to directly edit an
+                    existing <filename>.config</filename> file, which is
+                    found in the Build Directory among the source code
+                    used for the build (e.g. see the workflow illustration
+                    in the
+                    "<link linkend='kernel-modification-workflow'>Kernel Modification Workflow</link>"
+                    section).
+                    Doing so, can produce unexpected results when the
+                    OpenEmbedded build system regenerates the configuration
+                    file.
+                </note>
+                Once you are satisfied with the configuration
+                changes made using <filename>menuconfig</filename>
+                and you have saved them, you can directly compare the
+                resulting <filename>.config</filename> file against an
+                existing original and gather those changes into a
+                <ulink url='&YOCTO_DOCS_DEV_URL;#creating-config-fragments'>configuration fragment file</ulink>
+                to be referenced from within the kernel's
+                <filename>.bbappend</filename> file.</para>
+
+                <para>Additionally, if you are working in a BSP layer
+                and need to modify the BSP's kernel's configuration,
+                you can use the
+                <ulink url='&YOCTO_DOCS_BSP_URL;#managing-kernel-patches-and-config-items-with-yocto-kernel'><filename>yocto-kernel</filename></ulink>
+                script as well as <filename>menuconfig</filename>.
+                The <filename>yocto-kernel</filename> script lets
+                you interactively set up kernel configurations.
+                </para></listitem>
+            <listitem><para>
+                <emphasis>Rebuild the Kernel Image With Your Changes</emphasis>:
+                Rebuilding the kernel image applies your changes.
+                </para></listitem>
+        </orderedlist>
+    </para>
+</section>
+
+<section id='kernel-dev-other-resources'>
+    <title>Other Resources</title>
+
+    <para>
+        The remainder of this manual provides instructions for completing
+        specific Linux kernel development tasks.
+        These instructions assume you are comfortable working with
+        <ulink url='http://openembedded.org/wiki/Bitbake'>BitBake</ulink>
+        recipes and basic open-source development tools.
+        Understanding these concepts will facilitate the process of working
+        with the kernel recipes.
+        If you find you need some additional background, please be sure to
+        review and understand the following documentation:
+        <itemizedlist>
+            <listitem><para>
+                <ulink url='&YOCTO_DOCS_QS_URL;'>Yocto Project Quick Start</ulink>
+                </para></listitem>
+            <listitem><para>
+                <ulink url='&YOCTO_DOCS_SDK_URL;#using-devtool-in-your-sdk-workflow'><filename>devtool</filename> workflow</ulink>
+                as described in the Yocto Project Software Development Kit
+                (SDK) Developer's Guide.
+                </para></listitem>
+            <listitem><para>
+                The
+                "<ulink url='&YOCTO_DOCS_DEV_URL;#understanding-and-creating-layers'>Understanding and Creating Layers</ulink>" section
+                in the Yocto Project Development Manual
+                </para></listitem>
+            <listitem><para>
+                The
+                "<link linkend='kernel-modification-workflow'>Kernel Modification Workflow</link>"
+                section.
+                </para></listitem>
+        </itemizedlist>
+    </para>
+
+    <para>
+        Finally, while this document focuses on the manual creation of
+        recipes, patches, and configuration files, the Yocto Project
+        Board Support Package (BSP) tools are available to automate
+        this process with existing content and work well to create the
+        initial framework and boilerplate code.
+        For details on these tools, see the
+        "<ulink url='&YOCTO_DOCS_BSP_URL;#using-the-yocto-projects-bsp-tools'>Using the Yocto Project's BSP Tools</ulink>"
+        section in the Yocto Project Board Support Package (BSP) Developer's
+        Guide.
+    </para>
+</section>
 
-        <para>
-            Finally, while this document focuses on the manual creation of
-            recipes, patches, and configuration files, the Yocto Project
-            Board Support Package (BSP) tools are available to automate
-            this process with existing content and work well to create the
-            initial framework and boilerplate code.
-            For details on these tools, see the
-            "<ulink url='&YOCTO_DOCS_BSP_URL;#using-the-yocto-projects-bsp-tools'>Using the Yocto Project's BSP Tools</ulink>"
-            section in the Yocto Project Board Support Package (BSP) Developer's
-            Guide.
-        </para>
-    </section>
 </chapter>
 <!--
 vim: expandtab tw=80 ts=4

documentation/kernel-dev/kernel-dev-maint-appx.xml

@@ -209,7 +209,7 @@
             The files include the final <filename>.config</filename> file, all the <filename>.o</filename>
             files, the <filename>.a</filename> files, and so forth.
             Since each machine or BSP has its own separate
-            <ulink url='&YOCTO_DOCS_DEV_URL;#build-directory'>Build Directory</ulink>
+            <ulink url='&YOCTO_DOCS_REF_URL;#build-directory'>Build Directory</ulink>
             in its own separate branch
             of the Git repository, you can easily switch between different builds.
         </para>

documentation/kernel-dev/kernel-dev-style.css

@@ -730,6 +730,10 @@ div.navfooter {
   border-color: black;
 }
 
+.writernotes {
+  color: red;
+}
+
 
   /*********** /
  /  graphics  /

documentation/mega-manual/mega-manual.xml

@@ -130,8 +130,6 @@
         xmlns:xi="http://www.w3.org/2003/XInclude" href="../dev-manual/dev-manual-start.xml"/>
     <xi:include
         xmlns:xi="http://www.w3.org/2003/XInclude" href="../dev-manual/dev-manual-newbie.xml"/>
-    <xi:include
-        xmlns:xi="http://www.w3.org/2003/XInclude" href="../dev-manual/dev-manual-model.xml"/>
     <xi:include
         xmlns:xi="http://www.w3.org/2003/XInclude" href="../dev-manual/dev-manual-common-tasks.xml"/>
     <xi:include
@@ -214,7 +212,7 @@
         xmlns:xi="http://www.w3.org/2003/XInclude" href="../ref-manual/usingpoky.xml"/>
 
     <xi:include
-        xmlns:xi="http://www.w3.org/2003/XInclude" href="../ref-manual/closer-look.xml"/>
+        xmlns:xi="http://www.w3.org/2003/XInclude" href="../ref-manual/ref-development-environment.xml"/>
 
     <xi:include
         xmlns:xi="http://www.w3.org/2003/XInclude" href="../ref-manual/technical-details.xml"/>

documentation/mega-manual/mega-style.css

@@ -731,6 +731,11 @@ div.navfooter {
 }
 
 
+.writernotes {
+  color: red;
+}
+
+
   /*********** /
  /  graphics  /
 / ***********/

documentation/poky.ent

@@ -3,6 +3,9 @@
 <!ENTITY DISTRO_NAME_NO_CAP "tbd">
 <!ENTITY DISTRO_NAME "Tbd">
 <!ENTITY YOCTO_DOC_VERSION "2.4">
+<!ENTITY DISTRO_REL_TAG "yocto-2.4">
+<!ENTITY METAINTELVERSION "8.0">
+<!ENTITY META_INTEL_REL_TAG "&METAINTELVERSION;-tbd-&YOCTO_DOC_VERSION;">
 <!ENTITY POKYVERSION "19.0.0">
 <!ENTITY POKYVERSION_COMPRESSED "1900">
 <!ENTITY YOCTO_POKY "poky-&DISTRO_NAME_NO_CAP;-&POKYVERSION;">
@@ -66,7 +69,7 @@
 <!ENTITY FEDORA_HOST_PACKAGES_ESSENTIAL "gawk make wget tar bzip2 gzip python3 unzip perl patch \
      diffutils diffstat git cpp gcc gcc-c++ glibc-devel texinfo chrpath \
      ccache perl-Data-Dumper perl-Text-ParseWords perl-Thread-Queue perl-bignum socat \
-     python3-pexpect findutils which file cpio python python3-pip xz which">
+     python3-pexpect findutils which file cpio python python3-pip xz">
 <!ENTITY OPENSUSE_HOST_PACKAGES_ESSENTIAL "python gcc gcc-c++ git chrpath make wget python-xml \
      diffstat makeinfo python-curses patch socat python3 python3-curses tar python3-pip \
      python3-pexpect xz which">
@@ -74,5 +77,5 @@
      $ sudo yum makecache
      $ sudo yum install gawk make wget tar bzip2 gzip python unzip perl patch \
      diffutils diffstat git cpp gcc gcc-c++ glibc-devel texinfo chrpath socat \
-     perl-Data-Dumper perl-Text-ParseWords perl-Thread-Queue python34-pip xz \
+     perl-Data-Dumper perl-Text-ParseWords perl-Thread-Queue python3-pip xz \
      which">

documentation/ref-manual/faq.xml

@@ -17,7 +17,7 @@
                 refers to the specific reference build system that
                 the Yocto Project provides.
                 Poky is based on <ulink url='&YOCTO_DOCS_DEV_URL;#oe-core'>OE-Core</ulink>
-                and <ulink url='&YOCTO_DOCS_DEV_URL;#bitbake-term'>BitBake</ulink>.
+                and <link linkend='bitbake-term'>BitBake</link>.
                 Thus, the generic term used here for the build system is
                 the "OpenEmbedded build system."
                 Development in the Yocto Project using Poky is closely tied to OpenEmbedded, with
@@ -416,7 +416,7 @@
 
             <para>
                 You can find more information on licensing in the
-                "<ulink url='&YOCTO_DOCS_DEV_URL;#licensing'>Licensing</ulink>"
+                "<ulink url='&YOCTO_DOCS_REF_URL;#licensing'>Licensing</ulink>"
                 and "<ulink url='&YOCTO_DOCS_DEV_URL;#maintaining-open-source-license-compliance-during-your-products-lifecycle'>Maintaining Open Source License Compliance During Your Product's Lifecycle</ulink>"
                 sections, both of which are in the Yocto Project Development
                 Manual.
@@ -702,7 +702,8 @@
                 <link linkend='structure-core-script'><filename>&OE_INIT_FILE;</filename></link>
                 or
                 <link linkend='structure-memres-core-script'><filename>oe-init-build-env-memres</filename></link>).
-                By default, this <ulink url='&YOCTO_DOCS_DEV_URL;#build-directory'>Build Directory</ulink>
+                By default, this
+                <link linkend='build-directory'>Build Directory</link>
                 is named <filename>build</filename> but can be named
                 anything you want.
             </para>
@@ -765,7 +766,7 @@
 
             <para>
                 Meanwhile, <filename>DESTDIR</filename> is a path within the
-                <ulink url='&YOCTO_DOCS_DEV_URL;#build-directory'>Build Directory</ulink>.
+                <link linkend='build-directory'>Build Directory</link>.
                 However, when the recipe builds a native program (i.e. one
                 that is intended to run on the build machine), that program
                 is never installed directly to the build machine's root
@@ -810,7 +811,7 @@
             <para>
                 This situation results when a build system does
                 not recognize the environment variables supplied to it by
-                <ulink url='&YOCTO_DOCS_DEV_URL;#bitbake-term'>BitBake</ulink>.
+                <link linkend='bitbake-term'>BitBake</link>.
                 The incident that prompted this FAQ entry involved a Makefile
                 that used an environment variable named
                 <filename>BINDIR</filename> instead of the more standard