build-appliance-image: Update to sumo head revision

(From OE-Core rev: 0a2db923fd17019d07d88204b355aa46590f0b97)

Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>

bitbake/bin/bitbake-diffsigs

@@ -1,7 +1,7 @@
 #!/usr/bin/env python3
 
-# bitbake-diffsigs
-# BitBake task signature data comparison utility
+# bitbake-diffsigs / bitbake-dumpsig
+# BitBake task signature data dump and comparison utility
 #
 # Copyright (C) 2012-2013, 2017 Intel Corporation
 #
@@ -21,7 +21,6 @@
 import os
 import sys
 import warnings
-import fnmatch
 import argparse
 import logging
 import pickle
@@ -32,7 +31,10 @@ import bb.tinfoil
 import bb.siggen
 import bb.msg
 
-logger = bb.msg.logger_create('bitbake-diffsigs')
+myname = os.path.basename(sys.argv[0])
+logger = bb.msg.logger_create(myname)
+
+is_dump = myname == 'bitbake-dumpsig'
 
 def find_siginfo(tinfoil, pn, taskname, sigs=None):
     result = None
@@ -59,8 +61,8 @@ def find_siginfo(tinfoil, pn, taskname, sigs=None):
         sys.exit(2)
     return result
 
-def find_compare_task(bbhandler, pn, taskname, sig1=None, sig2=None, color=False):
-    """ Find the most recent signature files for the specified PN/task and compare them """
+def find_siginfo_task(bbhandler, pn, taskname, sig1=None, sig2=None):
+    """ Find the most recent signature files for the specified PN/task """
 
     if not taskname.startswith('do_'):
         taskname = 'do_%s' % taskname
@@ -79,73 +81,81 @@ def find_compare_task(bbhandler, pn, taskname, sig1=None, sig2=None, color=False
         latestfiles = [sigfiles[sig1], sigfiles[sig2]]
     else:
         filedates = find_siginfo(bbhandler, pn, taskname)
-        latestfiles = sorted(filedates.keys(), key=lambda f: filedates[f])[-3:]
+        latestfiles = sorted(filedates.keys(), key=lambda f: filedates[f])[-2:]
         if not latestfiles:
             logger.error('No sigdata files found matching %s %s' % (pn, taskname))
             sys.exit(1)
-        elif len(latestfiles) < 2:
-            logger.error('Only one matching sigdata file found for the specified task (%s %s)' % (pn, taskname))
-            sys.exit(1)
 
-    # Define recursion callback
-    def recursecb(key, hash1, hash2):
-        hashes = [hash1, hash2]
-        hashfiles = find_siginfo(bbhandler, key, None, hashes)
+    return latestfiles
 
-        recout = []
-        if len(hashfiles) == 0:
-            recout.append("Unable to find matching sigdata for %s with hashes %s or %s" % (key, hash1, hash2))
-        elif not hash1 in hashfiles:
-            recout.append("Unable to find matching sigdata for %s with hash %s" % (key, hash1))
-        elif not hash2 in hashfiles:
-            recout.append("Unable to find matching sigdata for %s with hash %s" % (key, hash2))
-        else:
-            out2 = bb.siggen.compare_sigfiles(hashfiles[hash1], hashfiles[hash2], recursecb, color=color)
-            for change in out2:
-                for line in change.splitlines():
-                    recout.append('  ' + line)
 
-        return recout
+# Define recursion callback
+def recursecb(key, hash1, hash2):
+    hashes = [hash1, hash2]
+    hashfiles = find_siginfo(tinfoil, key, None, hashes)
 
-    # Recurse into signature comparison
-    logger.debug("Signature file (previous): %s" % latestfiles[-2])
-    logger.debug("Signature file (latest): %s" % latestfiles[-1])
-    output = bb.siggen.compare_sigfiles(latestfiles[-2], latestfiles[-1], recursecb, color=color)
-    if output:
-        print('\n'.join(output))
-    sys.exit(0)
+    recout = []
+    if len(hashfiles) == 0:
+        recout.append("Unable to find matching sigdata for %s with hashes %s or %s" % (key, hash1, hash2))
+    elif not hash1 in hashfiles:
+        recout.append("Unable to find matching sigdata for %s with hash %s" % (key, hash1))
+    elif not hash2 in hashfiles:
+        recout.append("Unable to find matching sigdata for %s with hash %s" % (key, hash2))
+    else:
+        out2 = bb.siggen.compare_sigfiles(hashfiles[hash1], hashfiles[hash2], recursecb, color=color)
+        for change in out2:
+            for line in change.splitlines():
+                recout.append('  ' + line)
 
+    return recout
 
 
 parser = argparse.ArgumentParser(
-    description="Compares siginfo/sigdata files written out by BitBake")
+    description=("Dumps" if is_dump else "Compares") + " siginfo/sigdata files written out by BitBake")
 
-parser.add_argument('-d', '--debug',
+parser.add_argument('-D', '--debug',
                     help='Enable debug output',
                     action='store_true')
 
-parser.add_argument('--color',
-        help='Colorize output (where %(metavar)s is %(choices)s)',
-        choices=['auto', 'always', 'never'], default='auto', metavar='color')
+if is_dump:
+    parser.add_argument("-t", "--task",
+            help="find the signature data file for the last run of the specified task",
+            action="store", dest="taskargs", nargs=2, metavar=('recipename', 'taskname'))
 
-parser.add_argument("-t", "--task",
-        help="find the signature data files for last two runs of the specified task and compare them",
-        action="store", dest="taskargs", nargs=2, metavar=('recipename', 'taskname'))
+    parser.add_argument("sigdatafile1",
+            help="Signature file to dump. Not used when using -t/--task.",
+            action="store", nargs='?', metavar="sigdatafile")
+else:
+    parser.add_argument('-c', '--color',
+            help='Colorize the output (where %(metavar)s is %(choices)s)',
+            choices=['auto', 'always', 'never'], default='auto', metavar='color')
 
-parser.add_argument("-s", "--signature",
-        help="With -t/--task, specify the signatures to look for instead of taking the last two",
-        action="store", dest="sigargs", nargs=2, metavar=('fromsig', 'tosig'))
+    parser.add_argument('-d', '--dump',
+            help='Dump the last signature data instead of comparing (equivalent to using bitbake-dumpsig)',
+            action='store_true')
 
-parser.add_argument("sigdatafile1",
-        help="First signature file to compare (or signature file to dump, if second not specified). Not used when using -t/--task.",
-        action="store", nargs='?')
+    parser.add_argument("-t", "--task",
+            help="find the signature data files for the last two runs of the specified task and compare them",
+            action="store", dest="taskargs", nargs=2, metavar=('recipename', 'taskname'))
 
-parser.add_argument("sigdatafile2",
-        help="Second signature file to compare",
-        action="store", nargs='?')
+    parser.add_argument("-s", "--signature",
+            help="With -t/--task, specify the signatures to look for instead of taking the last two",
+            action="store", dest="sigargs", nargs=2, metavar=('fromsig', 'tosig'))
 
+    parser.add_argument("sigdatafile1",
+            help="First signature file to compare (or signature file to dump, if second not specified). Not used when using -t/--task.",
+            action="store", nargs='?')
+
+    parser.add_argument("sigdatafile2",
+            help="Second signature file to compare",
+            action="store", nargs='?')
 
 options = parser.parse_args()
+if is_dump:
+    options.color = 'never'
+    options.dump = True
+    options.sigdatafile2 = None
+    options.sigargs = None
 
 if options.debug:
     logger.setLevel(logging.DEBUG)
@@ -155,17 +165,32 @@ color = (options.color == 'always' or (options.color == 'auto' and sys.stdout.is
 if options.taskargs:
     with bb.tinfoil.Tinfoil() as tinfoil:
         tinfoil.prepare(config_only=True)
-        if options.sigargs:
-            find_compare_task(tinfoil, options.taskargs[0], options.taskargs[1], options.sigargs[0], options.sigargs[1], color=color)
+        if not options.dump and options.sigargs:
+            files = find_siginfo_task(tinfoil, options.taskargs[0], options.taskargs[1], options.sigargs[0], options.sigargs[1])
         else:
-            find_compare_task(tinfoil, options.taskargs[0], options.taskargs[1], color=color)
+            files = find_siginfo_task(tinfoil, options.taskargs[0], options.taskargs[1])
+
+        if options.dump:
+            logger.debug("Signature file: %s" % files[-1])
+            output = bb.siggen.dump_sigfile(files[-1])
+        else:
+            if len(files) < 2:
+                logger.error('Only one matching sigdata file found for the specified task (%s %s)' % (options.taskargs[0], options.taskargs[1]))
+                sys.exit(1)
+
+            # Recurse into signature comparison
+            logger.debug("Signature file (previous): %s" % files[-2])
+            logger.debug("Signature file (latest): %s" % files[-1])
+            output = bb.siggen.compare_sigfiles(files[-2], files[-1], recursecb, color=color)
 else:
     if options.sigargs:
         logger.error('-s/--signature can only be used together with -t/--task')
         sys.exit(1)
     try:
-        if options.sigdatafile1 and options.sigdatafile2:
-            output = bb.siggen.compare_sigfiles(options.sigdatafile1, options.sigdatafile2, color=color)
+        if not options.dump and options.sigdatafile1 and options.sigdatafile2:
+            with bb.tinfoil.Tinfoil() as tinfoil:
+                tinfoil.prepare(config_only=True)
+                output = bb.siggen.compare_sigfiles(options.sigdatafile1, options.sigdatafile2, recursecb, color=color)
         elif options.sigdatafile1:
             output = bb.siggen.dump_sigfile(options.sigdatafile1)
         else:
@@ -179,5 +204,5 @@ else:
         logger.error('Invalid signature data - ensure you are specifying sigdata/siginfo files')
         sys.exit(1)
 
-    if output:
-        print('\n'.join(output))
+if output:
+    print('\n'.join(output))

bitbake/bin/bitbake-dumpsig

@@ -1,94 +0,0 @@
-#!/usr/bin/env python3
-
-# bitbake-dumpsig
-# BitBake task signature dump utility
-#
-# Copyright (C) 2013 Intel Corporation
-#
-# This program is free software; you can redistribute it and/or modify
-# it under the terms of the GNU General Public License version 2 as
-# published by the Free Software Foundation.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License along
-# with this program; if not, write to the Free Software Foundation, Inc.,
-# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
-
-import os
-import sys
-import warnings
-import optparse
-import logging
-import pickle
-
-sys.path.insert(0, os.path.join(os.path.dirname(os.path.dirname(sys.argv[0])), 'lib'))
-
-import bb.tinfoil
-import bb.siggen
-import bb.msg
-
-logger = bb.msg.logger_create('bitbake-dumpsig')
-
-def find_siginfo_task(bbhandler, pn, taskname):
-    """ Find the most recent signature file for the specified PN/task """
-
-    if not hasattr(bb.siggen, 'find_siginfo'):
-        logger.error('Metadata does not support finding signature data files')
-        sys.exit(1)
-
-    if not taskname.startswith('do_'):
-        taskname = 'do_%s' % taskname
-
-    filedates = bb.siggen.find_siginfo(pn, taskname, None, bbhandler.config_data)
-    latestfiles = sorted(filedates.keys(), key=lambda f: filedates[f])[-1:]
-    if not latestfiles:
-        logger.error('No sigdata files found matching %s %s' % (pn, taskname))
-        sys.exit(1)
-
-    return latestfiles[0]
-
-parser = optparse.OptionParser(
-    description = "Dumps siginfo/sigdata files written out by BitBake",
-    usage = """
-  %prog -t recipename taskname
-  %prog sigdatafile""")
-
-parser.add_option("-D", "--debug",
-        help = "enable debug",
-        action = "store_true", dest="debug", default = False)
-
-parser.add_option("-t", "--task",
-        help = "find the signature data file for the specified task",
-        action="store", dest="taskargs", nargs=2, metavar='recipename taskname')
-
-options, args = parser.parse_args(sys.argv)
-
-if options.debug:
-    logger.setLevel(logging.DEBUG)
-
-if options.taskargs:
-    tinfoil = bb.tinfoil.Tinfoil()
-    tinfoil.prepare(config_only = True)
-    file = find_siginfo_task(tinfoil, options.taskargs[0], options.taskargs[1])
-    logger.debug("Signature file: %s" % file)
-elif len(args) == 1:
-    parser.print_help()
-    sys.exit(0)
-else:
-    file = args[1]
-
-try:
-    output = bb.siggen.dump_sigfile(file)
-except IOError as e:
-    logger.error(str(e))
-    sys.exit(1)
-except (pickle.UnpicklingError, EOFError):
-    logger.error('Invalid signature data - ensure you are specifying a sigdata/siginfo file')
-    sys.exit(1)
-
-if output:
-    print('\n'.join(output))

bitbake/bin/bitbake-dumpsig

@@ -0,0 +1 @@
+bitbake-diffsigs

bitbake/bin/toaster

@@ -18,11 +18,12 @@
 # along with this program. If not, see http://www.gnu.org/licenses/.
 
 HELP="
-Usage: source toaster start|stop [webport=<address:port>] [noweb] [nobuild]
+Usage: source toaster start|stop [webport=<address:port>] [noweb] [nobuild] [toasterdir]
     Optional arguments:
         [nobuild] Setup the environment for capturing builds with toaster but disable managed builds
         [noweb] Setup the environment for capturing builds with toaster but don't start the web server
         [webport] Set the development server (default: localhost:8000)
+        [toasterdir] Set absolute path to be used as TOASTER_DIR (default: BUILDDIR/../)
 "
 
 custom_extention()
@@ -160,7 +161,9 @@ fi
 
 export BBBASEDIR=`dirname $TOASTER`/..
 MANAGE="python3 $BBBASEDIR/lib/toaster/manage.py"
-OE_ROOT=`dirname $TOASTER`/../..
+if [ -z "$OE_ROOT" ]; then
+    OE_ROOT=`dirname $TOASTER`/../..
+fi
 
 # this is the configuraton file we are using for toaster
 # we are using the same logic that oe-setup-builddir uses
@@ -186,6 +189,7 @@ unset OE_ROOT
 WEBSERVER=1
 export TOASTER_BUILDSERVER=1
 ADDR_PORT="localhost:8000"
+TOASTERDIR=`dirname $BUILDDIR`
 unset CMD
 for param in $*; do
     case $param in
@@ -211,6 +215,9 @@ for param in $*; do
                 ADDR_PORT="localhost:$PORT"
             fi
     ;;
+    toasterdir=*)
+            TOASTERDIR="${param#*=}"
+    ;;
     --help)
             echo "$HELP"
             return 0
@@ -241,7 +248,7 @@ fi
 # 2) the build dir (in build)
 # 3) the sqlite db if that is being used.
 # 4) pid's we need to clean up on exit/shutdown
-export TOASTER_DIR=`dirname $BUILDDIR`
+export TOASTER_DIR=$TOASTERDIR
 export BB_ENV_EXTRAWHITE="$BB_ENV_EXTRAWHITE TOASTER_DIR"
 
 # Determine the action. If specified by arguments, fine, if not, toggle it

bitbake/doc/bitbake-user-manual/bitbake-user-manual-fetching.xml

@@ -588,6 +588,14 @@
                         The name of the path in which to place the checkout.
                         By default, the path is <filename>git/</filename>.
                         </para></listitem>
+                    <listitem><para><emphasis>"usehead":</emphasis>
+                        Enables local <filename>git://</filename> URLs to use the
+                        current branch HEAD as the revision for use with
+                        <filename>AUTOREV</filename>.
+                        The "usehead" parameter implies no branch and only works
+                        when the transfer protocol is
+                        <filename>file://</filename>.
+                        </para></listitem>
                 </itemizedlist>
                 Here are some example URLs:
                 <literallayout class='monospaced'>

bitbake/doc/bitbake-user-manual/bitbake-user-manual-metadata.xml

@@ -502,7 +502,7 @@
         </section>
 
         <section id='unsetting-variables'>
-            <title>Unseting variables</title>
+            <title>Unsetting variables</title>
 
             <para>
                 It is possible to completely remove a variable or a variable flag

bitbake/lib/bb/COW.py

@@ -150,7 +150,7 @@ class COWDictMeta(COWMeta):
                 yield value
             if type == "items":
                 yield (key, value)
-        raise StopIteration()
+        return
 
     def iterkeys(cls):
         return cls.iter("keys")

bitbake/lib/bb/checksum.py

@@ -97,6 +97,8 @@ class FileChecksumCache(MultiProcessCache):
 
         def checksum_dir(pth):
             # Handle directories recursively
+            if pth == "/":
+                bb.fatal("Refusing to checksum /")
             dirchecksums = []
             for root, dirs, files in os.walk(pth):
                 for name in files:

bitbake/lib/bb/cooker.py

@@ -175,18 +175,31 @@ class BBCooker:
 
         self.configuration = configuration
 
+        bb.debug(1, "BBCooker starting %s" % time.time())
+        sys.stdout.flush()
+
         self.configwatcher = pyinotify.WatchManager()
+        bb.debug(1, "BBCooker pyinotify1 %s" % time.time())
+        sys.stdout.flush()
+
         self.configwatcher.bbseen = []
         self.configwatcher.bbwatchedfiles = []
         self.confignotifier = pyinotify.Notifier(self.configwatcher, self.config_notifications)
+        bb.debug(1, "BBCooker pyinotify2 %s" % time.time())
+        sys.stdout.flush()
         self.watchmask = pyinotify.IN_CLOSE_WRITE | pyinotify.IN_CREATE | pyinotify.IN_DELETE | \
                          pyinotify.IN_DELETE_SELF | pyinotify.IN_MODIFY | pyinotify.IN_MOVE_SELF | \
                          pyinotify.IN_MOVED_FROM | pyinotify.IN_MOVED_TO
         self.watcher = pyinotify.WatchManager()
+        bb.debug(1, "BBCooker pyinotify3 %s" % time.time())
+        sys.stdout.flush()
         self.watcher.bbseen = []
         self.watcher.bbwatchedfiles = []
         self.notifier = pyinotify.Notifier(self.watcher, self.notifications)
 
+        bb.debug(1, "BBCooker pyinotify complete %s" % time.time())
+        sys.stdout.flush()
+
         # If being called by something like tinfoil, we need to clean cached data
         # which may now be invalid
         bb.parse.clear_cache()
@@ -196,6 +209,9 @@ class BBCooker:
 
         self.initConfigurationData()
 
+        bb.debug(1, "BBCooker parsed base configuration %s" % time.time())
+        sys.stdout.flush()
+
         # we log all events to a file if so directed
         if self.configuration.writeeventlog:
             # register the log file writer as UI Handler
@@ -233,6 +249,9 @@ class BBCooker:
         # Let SIGHUP exit as SIGTERM
         signal.signal(signal.SIGHUP, self.sigterm_exception)
 
+        bb.debug(1, "BBCooker startup complete %s" % time.time())
+        sys.stdout.flush()
+
     def process_inotify_updates(self):
         for n in [self.confignotifier, self.notifier]:
             if n.check_events(timeout=0):

bitbake/lib/bb/fetch2/__init__.py

@@ -1457,7 +1457,7 @@ class FetchMethod(object):
                 else:
                     cmd = 'rpm2cpio.sh %s | cpio -id' % (file)
             elif file.endswith('.deb') or file.endswith('.ipk'):
-                output = subprocess.check_output('ar -t %s' % file, preexec_fn=subprocess_setup, shell=True)
+                output = subprocess.check_output(['ar', '-t', file], preexec_fn=subprocess_setup)
                 datafile = None
                 if output:
                     for line in output.decode().splitlines():

bitbake/lib/bb/fetch2/git.py

@@ -354,10 +354,9 @@ class Git(FetchMethod):
             if not self._contains_ref(ud, d, name, ud.clonedir):
                 needupdate = True
         if needupdate:
-            try: 
-                runfetchcmd("%s remote rm origin" % ud.basecmd, d, workdir=ud.clonedir)
-            except bb.fetch2.FetchError:
-                logger.debug(1, "No Origin")
+            output = runfetchcmd("%s remote" % ud.basecmd, d, quiet=True, workdir=ud.clonedir)
+            if "origin" in output:
+              runfetchcmd("%s remote rm origin" % ud.basecmd, d, workdir=ud.clonedir)
 
             runfetchcmd("%s remote add --mirror=fetch origin %s" % (ud.basecmd, repourl), d, workdir=ud.clonedir)
             fetch_cmd = "LANG=C %s fetch -f --prune --progress %s refs/*:refs/*" % (ud.basecmd, repourl)

bitbake/lib/bb/main.py

@@ -405,9 +405,6 @@ def setup_bitbake(configParams, configuration, extrafeatures=None):
         # In status only mode there are no logs and no UI
         logger.addHandler(handler)
 
-    # Clear away any spurious environment variables while we stoke up the cooker
-    cleanedvars = bb.utils.clean_environment()
-
     if configParams.server_only:
         featureset = []
         ui_module = None
@@ -423,6 +420,10 @@ def setup_bitbake(configParams, configuration, extrafeatures=None):
 
     server_connection = None
 
+    # Clear away any spurious environment variables while we stoke up the cooker
+    # (done after import_extension_module() above since for example import gi triggers env var usage)
+    cleanedvars = bb.utils.clean_environment()
+
     if configParams.remote_server:
         # Connect to a remote XMLRPC server
         server_connection = bb.server.xmlrpcclient.connectXMLRPC(configParams.remote_server, featureset,
@@ -447,7 +448,7 @@ def setup_bitbake(configParams, configuration, extrafeatures=None):
                 else:
                     logger.info("Reconnecting to bitbake server...")
                     if not os.path.exists(sockname):
-                        print("Previous bitbake instance shutting down?, waiting to retry...")
+                        logger.info("Previous bitbake instance shutting down?, waiting to retry...")
                         i = 0
                         lock = None
                         # Wait for 5s or until we can get the lock

bitbake/lib/bb/runqueue.py

@@ -1371,6 +1371,12 @@ class RunQueue:
                 bb.event.fire(bb.event.DepTreeGenerated(depgraph), self.cooker.data)
 
         if self.state is runQueueSceneInit:
+            if not self.dm_event_handler_registered:
+                 res = bb.event.register(self.dm_event_handler_name,
+                                         lambda x: self.dm.check(self) if self.state in [runQueueSceneRun, runQueueRunning, runQueueCleanUp] else False,
+                                         ('bb.event.HeartbeatEvent',))
+                 self.dm_event_handler_registered = True
+
             dump = self.cooker.configuration.dump_signatures
             if dump:
                 self.rqdata.init_progress_reporter.finish()
@@ -1387,11 +1393,6 @@ class RunQueue:
                 self.rqexe = RunQueueExecuteScenequeue(self)
 
         if self.state is runQueueSceneRun:
-            if not self.dm_event_handler_registered:
-                 res = bb.event.register(self.dm_event_handler_name,
-                                         lambda x: self.dm.check(self) if self.state in [runQueueSceneRun, runQueueRunning, runQueueCleanUp] else False,
-                                         ('bb.event.HeartbeatEvent',))
-                 self.dm_event_handler_registered = True
             retval = self.rqexe.execute()
 
         if self.state is runQueueRunInit:

bitbake/lib/bb/server/process.py

@@ -130,6 +130,7 @@ class ProcessServer(multiprocessing.Process):
         bb.utils.set_process_name("Cooker")
 
         ready = []
+        newconnections = []
 
         self.controllersock = False
         fds = [self.sock]
@@ -138,37 +139,48 @@ class ProcessServer(multiprocessing.Process):
         print("Entering server connection loop")
 
         def disconnect_client(self, fds):
-            if not self.haveui:
-                return
             print("Disconnecting Client")
-            fds.remove(self.controllersock)
-            fds.remove(self.command_channel)
-            bb.event.unregister_UIHhandler(self.event_handle, True)
-            self.command_channel_reply.writer.close()
-            self.event_writer.writer.close()
-            del self.event_writer
-            self.controllersock.close()
-            self.controllersock = False
-            self.haveui = False
-            self.lastui = time.time()
-            self.cooker.clientComplete()
-            if self.timeout is None:
+            if self.controllersock:
+                fds.remove(self.controllersock)
+                self.controllersock.close()
+                self.controllersock = False
+            if self.haveui:
+                fds.remove(self.command_channel)
+                bb.event.unregister_UIHhandler(self.event_handle, True)
+                self.command_channel_reply.writer.close()
+                self.event_writer.writer.close()
+                self.command_channel.close()
+                self.command_channel = False
+                del self.event_writer
+                self.lastui = time.time()
+                self.cooker.clientComplete()
+                self.haveui = False
+            ready = select.select(fds,[],[],0)[0]
+            if newconnections:
+                print("Starting new client")
+                conn = newconnections.pop(-1)
+                fds.append(conn)
+                self.controllersock = conn
+            elif self.timeout is None and not ready:
                 print("No timeout, exiting.")
                 self.quit = True
 
         while not self.quit:
             if self.sock in ready:
-                self.controllersock, address = self.sock.accept()
-                if self.haveui:
-                    print("Dropping connection attempt as we have a UI %s" % (str(ready)))
-                    self.controllersock.close()
-                else:
-                    print("Accepting %s" % (str(ready)))
-                    fds.append(self.controllersock)
+                while select.select([self.sock],[],[],0)[0]:
+                    controllersock, address = self.sock.accept()
+                    if self.controllersock:
+                        print("Queuing %s (%s)" % (str(ready), str(newconnections)))
+                        newconnections.append(controllersock)
+                    else:
+                        print("Accepting %s (%s)" % (str(ready), str(newconnections)))
+                        self.controllersock = controllersock
+                        fds.append(controllersock)
             if self.controllersock in ready:
                 try:
-                    print("Connecting Client")
+                    print("Processing Client")
                     ui_fds = recvfds(self.controllersock, 3)
+                    print("Connecting Client")
 
                     # Where to write events to
                     writer = ConnectionWriter(ui_fds[0])
@@ -239,6 +251,12 @@ class ProcessServer(multiprocessing.Process):
         while not lock:
             with bb.utils.timeout(3):
                 lock = bb.utils.lockfile(lockfile, shared=False, retry=False, block=True)
+                if lock:
+                    # We hold the lock so we can remove the file (hide stale pid data)
+                    bb.utils.remove(lockfile)
+                    bb.utils.unlockfile(lock)
+                    return
+
                 if not lock:
                     # Some systems may not have lsof available
                     procs = None
@@ -259,10 +277,6 @@ class ProcessServer(multiprocessing.Process):
                     if procs:
                         msg += ":\n%s" % str(procs)
                     print(msg)
-                    return
-        # We hold the lock so we can remove the file (hide stale pid data)
-        bb.utils.remove(lockfile)
-        bb.utils.unlockfile(lock)
 
     def idle_commands(self, delay, fds=None):
         nextsleep = delay
@@ -396,7 +410,10 @@ class BitBakeServer(object):
         self.bitbake_lock.close()
 
         ready = ConnectionReader(self.readypipe)
-        r = ready.poll(30)
+        r = ready.poll(5)
+        if not r:
+            bb.note("Bitbake server didn't start within 5 seconds, waiting for 90")
+            r = ready.poll(90)
         if r:
             r = ready.get()
         if not r or r != "ready":
@@ -406,28 +423,40 @@ class BitBakeServer(object):
                 logstart_re = re.compile(self.start_log_format % ('([0-9]+)', '([0-9-]+ [0-9:.]+)'))
                 started = False
                 lines = []
+                lastlines = []
                 with open(logfile, "r") as f:
                     for line in f:
                         if started:
                             lines.append(line)
                         else:
+                            lastlines.append(line)
                             res = logstart_re.match(line.rstrip())
                             if res:
                                 ldatetime = datetime.datetime.strptime(res.group(2), self.start_log_datetime_format)
                                 if ldatetime >= startdatetime:
                                     started = True
                                     lines.append(line)
+                        if len(lastlines) > 60:
+                            lastlines = lastlines[-60:]
                 if lines:
-                    if len(lines) > 10:
-                        bb.error("Last 10 lines of server log for this session (%s):\n%s" % (logfile, "".join(lines[-10:])))
+                    if len(lines) > 60:
+                        bb.error("Last 60 lines of server log for this session (%s):\n%s" % (logfile, "".join(lines[-60:])))
                     else:
                         bb.error("Server log for this session (%s):\n%s" % (logfile, "".join(lines)))
+                elif lastlines:
+                        bb.error("Server didn't start, last 60 loglines (%s):\n%s" % (logfile, "".join(lastlines)))
+            else:
+                bb.error("%s doesn't exist" % logfile)
+
             raise SystemExit(1)
+
         ready.close()
         os.close(self.readypipein)
 
     def _startServer(self):
         print(self.start_log_format % (os.getpid(), datetime.datetime.now().strftime(self.start_log_datetime_format)))
+        sys.stdout.flush()
+
         server = ProcessServer(self.bitbake_lock, self.sock, self.sockname)
         self.configuration.setServerRegIdleCallback(server.register_idle_function)
         writer = ConnectionWriter(self.readypipein)
@@ -443,6 +472,8 @@ class BitBakeServer(object):
         server.server_timeout = self.configuration.server_timeout
         server.xmlrpcinterface = self.configuration.xmlrpcinterface
         print("Started bitbake server pid %d" % os.getpid())
+        sys.stdout.flush()
+
         server.start()
 
 def connectProcessServer(sockname, featureset):
@@ -451,16 +482,24 @@ def connectProcessServer(sockname, featureset):
     # AF_UNIX has path length issues so chdir here to workaround
     cwd = os.getcwd()
 
-    try:
-        os.chdir(os.path.dirname(sockname))
-        sock.connect(os.path.basename(sockname))
-    finally:
-        os.chdir(cwd)
-
     readfd = writefd = readfd1 = writefd1 = readfd2 = writefd2 = None
     eq = command_chan_recv = command_chan = None
 
+    sock.settimeout(10)
+
     try:
+        try:
+            os.chdir(os.path.dirname(sockname))
+            finished = False
+            while not finished:
+                try:
+                    sock.connect(os.path.basename(sockname))
+                    finished = True
+                except IOError as e:
+                    if e.errno == errno.EWOULDBLOCK:
+                        pass
+        finally:
+            os.chdir(cwd)
 
         # Send an fd for the remote to write events to
         readfd, writefd = os.pipe()
@@ -489,7 +528,8 @@ def connectProcessServer(sockname, featureset):
             command_chan.close()
         for i in [writefd, readfd1, writefd2]:
             try:
-                os.close(i)
+                if i:
+                    os.close(i)
             except OSError:
                 pass
         sock.close()

bitbake/lib/bb/tests/fetch.py

@@ -861,12 +861,12 @@ class FetchLatestVersionTest(FetcherTest):
         ("dtc", "git://git.qemu.org/dtc.git", "65cc4d2748a2c2e6f27f1cf39e07a5dbabd80ebf", "")
             : "1.4.0",
         # combination version pattern
-        ("sysprof", "git://git.gnome.org/sysprof", "cd44ee6644c3641507fb53b8a2a69137f2971219", "")
+        ("sysprof", "git://gitlab.gnome.org/GNOME/sysprof;protocol=https", "cd44ee6644c3641507fb53b8a2a69137f2971219", "")
             : "1.2.0",
         ("u-boot-mkimage", "git://git.denx.de/u-boot.git;branch=master;protocol=git", "62c175fbb8a0f9a926c88294ea9f7e88eb898f6c", "")
             : "2014.01",
         # version pattern "yyyymmdd"
-        ("mobile-broadband-provider-info", "git://git.gnome.org/mobile-broadband-provider-info", "4ed19e11c2975105b71b956440acdb25d46a347d", "")
+        ("mobile-broadband-provider-info", "git://gitlab.gnome.org/GNOME/mobile-broadband-provider-info;protocol=https", "4ed19e11c2975105b71b956440acdb25d46a347d", "")
             : "20120614",
         # packages with a valid UPSTREAM_CHECK_GITTAGREGEX
         ("xf86-video-omap", "git://anongit.freedesktop.org/xorg/driver/xf86-video-omap", "ae0394e687f1a77e966cf72f895da91840dffb8f", "(?P<pver>(\d+\.(\d\.?)*))")

bitbake/lib/bb/utils.py

@@ -524,12 +524,17 @@ def md5_file(filename):
     """
     Return the hex string representation of the MD5 checksum of filename.
     """
-    import hashlib
-    m = hashlib.md5()
+    import hashlib, mmap
 
     with open(filename, "rb") as f:
-        for line in f:
-            m.update(line)
+        m = hashlib.md5()
+        try:
+            with mmap.mmap(f.fileno(), 0, access=mmap.ACCESS_READ) as mm:
+                for chunk in iter(lambda: mm.read(8192), b''):
+                    m.update(chunk)
+        except ValueError:
+            # You can't mmap() an empty file so silence this exception
+            pass
     return m.hexdigest()
 
 def sha256_file(filename):
@@ -1285,7 +1290,7 @@ def edit_metadata_file(meta_file, variables, varfunc):
     return updated
 
 
-def edit_bblayers_conf(bblayers_conf, add, remove):
+def edit_bblayers_conf(bblayers_conf, add, remove, edit_cb=None):
     """Edit bblayers.conf, adding and/or removing layers
     Parameters:
         bblayers_conf: path to bblayers.conf file to edit
@@ -1293,6 +1298,8 @@ def edit_bblayers_conf(bblayers_conf, add, remove):
             list to add nothing
         remove: layer path (or list of layer paths) to remove; None or
             empty list to remove nothing
+        edit_cb: optional callback function that will be called after
+            processing adds/removes once per existing entry.
     Returns a tuple:
         notadded: list of layers specified to be added but weren't
             (because they were already in the list)
@@ -1356,6 +1363,17 @@ def edit_bblayers_conf(bblayers_conf, add, remove):
                     bblayers.append(addlayer)
             del addlayers[:]
 
+        if edit_cb:
+            newlist = []
+            for layer in bblayers:
+                res = edit_cb(layer, canonicalise_path(layer))
+                if res != layer:
+                    newlist.append(res)
+                    updated = True
+                else:
+                    newlist.append(layer)
+            bblayers = newlist
+
         if updated:
             if op == '+=' and not bblayers:
                 bblayers = None

bitbake/lib/bblayers/layerindex.py

@@ -133,7 +133,7 @@ class LayerIndexPlugin(ActionPlugin):
         layerdir = os.path.join(repodir, subdir)
         if not os.path.exists(repodir):
             if fetch_layer:
-                result = subprocess.call('git clone %s %s' % (url, repodir), shell = True)
+                result = subprocess.call(['git', 'clone', url, repodir])
                 if result:
                     logger.error("Failed to download %s" % url)
                     return None, None

bitbake/lib/toaster/bldcontrol/localhostbecontroller.py

@@ -217,9 +217,21 @@ class LocalhostBEController(BuildEnvironmentController):
         self.setCloneStatus(bitbake,'complete',clone_total,clone_count)
         logger.debug("localhostbecontroller: current layer list %s " % pformat(layerlist))
 
-        if self.pokydirname is None and os.path.exists(os.path.join(self.be.sourcedir, "oe-init-build-env")):
-            logger.debug("localhostbecontroller: selected poky dir name %s" % self.be.sourcedir)
-            self.pokydirname = self.be.sourcedir
+        # Resolve self.pokydirname if not resolved yet, consider the scenario
+        # where all layers are local, that's the else clause
+        if self.pokydirname is None:
+            if os.path.exists(os.path.join(self.be.sourcedir, "oe-init-build-env")):
+                logger.debug("localhostbecontroller: selected poky dir name %s" % self.be.sourcedir)
+                self.pokydirname = self.be.sourcedir
+            else:
+                # Alternatively, scan local layers for relative "oe-init-build-env" location
+                for layer in layers:
+                    if os.path.exists(os.path.join(layer.layer_version.layer.local_source_dir,"..","oe-init-build-env")):
+                        logger.debug("localhostbecontroller, setting pokydirname to %s" % (layer.layer_version.layer.local_source_dir))
+                        self.pokydirname = os.path.join(layer.layer_version.layer.local_source_dir,"..")
+                        break
+                else:
+                    logger.error("pokydirname is not set, you will run into trouble!")
 
         # 5. create custom layer and add custom recipes to it
         for target in targets:
@@ -339,8 +351,21 @@ class LocalhostBEController(BuildEnvironmentController):
         # clean the Toaster to build environment
         env_clean = 'unset BBPATH;' # clean BBPATH for <= YP-2.4.0
 
-        # run bitbake server from the clone
+        # run bitbake server from the clone if available
+        # otherwise pick it from the PATH
         bitbake = os.path.join(self.pokydirname, 'bitbake', 'bin', 'bitbake')
+        if not os.path.exists(bitbake):
+            logger.info("Bitbake not available under %s, will try to use it from PATH" %
+                        self.pokydirname)
+            for path in os.environ["PATH"].split(os.pathsep):
+                if os.path.exists(os.path.join(path, 'bitbake')):
+                    bitbake = os.path.join(path, 'bitbake')
+                    logger.info("Found Bitbake at: %s" % path)
+                    break
+            else:
+                logger.error("Looks like Bitbake is not available, please fix your environment")
+
+        # run bitbake server from the clone
         toasterlayers = os.path.join(builddir,"conf/toaster-bblayers.conf")
         self._shellcmd('%s bash -c \"source %s %s; BITBAKE_UI="knotty" %s --read %s --read %s '
                        '--server-only -B 0.0.0.0:0\"' % (env_clean, oe_init,

bitbake/lib/toaster/bldcontrol/management/commands/checksettings.py

@@ -74,8 +74,9 @@ class Command(BaseCommand):
                         print("Loading default settings")
                         call_command("loaddata", "settings")
                         template_conf = os.environ.get("TEMPLATECONF", "")
+                        custom_xml_only = os.environ.get("CUSTOM_XML_ONLY")
 
-                        if ToasterSetting.objects.filter(name='CUSTOM_XML_ONLY').count() > 0:
+                        if ToasterSetting.objects.filter(name='CUSTOM_XML_ONLY').count() > 0 or (not custom_xml_only == None):
                             # only use the custom settings
                             pass
                         elif "poky" in template_conf:

bitbake/lib/toaster/orm/models.py

@@ -1663,6 +1663,9 @@ class CustomImageRecipe(Recipe):
 
         path_schema_two = self.base_recipe.file_path
 
+        path_schema_three = "%s/%s" % (self.base_recipe.layer_version.layer.local_source_dir,
+                                     self.base_recipe.file_path)
+
         if os.path.exists(path_schema_one):
             return path_schema_one
 
@@ -1670,6 +1673,10 @@ class CustomImageRecipe(Recipe):
         if os.path.exists(path_schema_two):
             return path_schema_two
 
+        # Or a local path if all layers are local
+        if os.path.exists(path_schema_three):
+            return path_schema_three
+
         return None
 
     def generate_recipe_file_contents(self):

bitbake/lib/toaster/toastergui/static/js/layerdetails.js

@@ -359,7 +359,8 @@ function layerDetailsPageInit (ctx) {
     if ($(this).is("dt")) {
       var dd = $(this).next("dd");
       if (!dd.children("form:visible")|| !dd.find(".current-value").html()){
-        if (ctx.layerVersion.layer_source == ctx.layerSourceTypes.TYPE_IMPORTED){
+        if (ctx.layerVersion.layer_source == ctx.layerSourceTypes.TYPE_IMPORTED ||
+            ctx.layerVersion.layer_source == ctx.layerSourceTypes.TYPE_LOCAL) {
         /* There's no current value and the layer is editable
          * so show the "Not set" and hide the delete icon
          */

bitbake/lib/toaster/toastergui/templates/newproject.html

@@ -54,12 +54,12 @@
                     <span class="help-block">{{release.helptext|safe}}</span>
                   </div>
                 {% endfor %}
+                </div>
+              </div>
             {% else %}
               <input type="hidden" name="projectversion" value="{{releases.0.id}}"/>
             {% endif %}
                 </div>
-              </div>
-            </fieldset>
         {% endif %}
             <div class="top-air">
               <input type="submit" id="create-project-button" class="btn btn-primary btn-lg" value="Create project"/>

bitbake/lib/toaster/toastergui/templates/recipe.html

@@ -176,7 +176,7 @@
                     <td>{{task.get_executed_display}}</td>
 
                     <td>{{task.get_outcome_display}} 
-                        {% if task.outcome = task.OUTCOME_FAILED %}
+                        {% if task.outcome == task.OUTCOME_FAILED %}
                             <a href="{% url 'build_artifact' build.pk "tasklogfile" task.pk %}">
                                 <span class="glyphicon glyphicon-download-alt
                                     get-help" title="Download task log

bitbake/lib/toaster/toastergui/widgets.py

@@ -511,13 +511,18 @@ class MostRecentBuildsView(View):
                 buildrequest_id = build_obj.buildrequest.pk
             build['buildrequest_id'] = buildrequest_id
 
-            build['recipes_parsed_percentage'] = \
-                int((build_obj.recipes_parsed /
-                     build_obj.recipes_to_parse) * 100)
-
-            build['repos_cloned_percentage'] = \
-                int((build_obj.repos_cloned /
-                     build_obj.repos_to_clone) * 100)
+            if build_obj.recipes_to_parse > 0:
+                build['recipes_parsed_percentage'] = \
+                    int((build_obj.recipes_parsed /
+                         build_obj.recipes_to_parse) * 100)
+            else:
+                build['recipes_parsed_percentage'] = 0
+            if build_obj.repos_to_clone > 0:
+                build['repos_cloned_percentage'] = \
+                    int((build_obj.repos_cloned /
+                         build_obj.repos_to_clone) * 100)
+            else:
+                build['repos_cloned_percentage'] = 0
 
             tasks_complete_percentage = 0
             if build_obj.outcome in (Build.SUCCEEDED, Build.FAILED):

documentation/brief-yoctoprojectqs/brief-yoctoprojectqs.xml

@@ -131,19 +131,43 @@
             <literallayout class='monospaced'>
      $ git clone git://git.yoctoproject.org/poky
      Cloning into 'poky'...
-     remote: Counting objects: 361782, done.
-     remote: Compressing objects: 100% (87100/87100), done.
-     remote: Total 361782 (delta 268619), reused 361439 (delta 268277)
-     Receiving objects: 100% (361782/361782), 131.94 MiB | 6.88 MiB/s, done.
-     Resolving deltas: 100% (268619/268619), done.
+     remote: Counting objects: 431956, done.
+     remote: Compressing objects: 100% (101918/101918), done.
+     remote: Total 431956 (delta 322982), reused 431910 (delta 322936)
+     Receiving objects: 100% (431956/431956), 153.76 MiB | 6.86 MiB/s, done.
+     Resolving deltas: 100% (322982/322982), done.
      Checking connectivity... done.
-     $ git checkout tags/yocto-2.5 -b my-yocto-2.5
             </literallayout>
-            The previous Git checkout command creates a local branch
-            named my-&DISTRO_REL_TAG;. The files available to you in that
-            branch exactly match the repository's files in the
-            "&DISTRO_NAME_NO_CAP;" development branch at the time of the
-            Yocto Project &DISTRO; release.
+            Move to the poky directory and take a look at the tags:
+            <literallayout class='monospaced'>
+     $ cd poky
+     $ git fetch --tags
+     $ git tag
+     1.1_M1.final
+     1.1_M1.rc1
+     1.1_M1.rc2
+     1.1_M2.final
+     1.1_M2.rc1
+        .
+        .
+        .
+     yocto-2.5.2
+     yocto-2.5.3
+     yocto-2.6
+     yocto-2.6.1
+     yocto_1.5_M5.rc8
+            </literallayout>
+            For this example, check out the branch based on the
+            yocto-&DISTRO; release:
+            <literallayout class='monospaced'>
+     $ git checkout tags/yocto-&DISTRO; -b my-yocto-&DISTRO;
+     Switched to a new branch 'my-yocto-&DISTRO;'
+            </literallayout>
+            The previous Git checkout command creates a local branch named
+            my-yocto-&DISTRO;.
+            The files available to you in that branch exactly match the
+            repository's files in the "&DISTRO_NAME_NO_CAP;" development
+            branch at the time of the Yocto Project yocto-&DISTRO; release.
         </para>
 
         <para>

documentation/bsp-guide/bsp-guide.xml

@@ -121,6 +121,21 @@
                 <date>May 2018</date>
                 <revremark>Released with the Yocto Project 2.5 Release.</revremark>
             </revision>
+            <revision>
+                <revnumber>2.5.1</revnumber>
+                <date>September 2018</date>
+                <revremark>The initial document released with the Yocto Project 2.5.1 Release.</revremark>
+            </revision>
+            <revision>
+                <revnumber>2.5.2</revnumber>
+                <date>January 2019</date>
+                <revremark>The initial document released with the Yocto Project 2.5.2 Release.</revremark>
+            </revision>
+            <revision>
+                <revnumber>2.5.3</revnumber>
+                <date>&REL_MONTH_YEAR;</date>
+                <revremark>The initial document released with the Yocto Project 2.5.3 Release.</revremark>
+            </revision>
         </revhistory>
 
     <copyright>

documentation/dev-manual/dev-manual-start.xml

@@ -881,11 +881,11 @@
                     <literallayout class='monospaced'>
      $ git clone git://git.yoctoproject.org/poky
      Cloning into 'poky'...
-     remote: Counting objects: 367178, done.
-     remote: Compressing objects: 100% (88161/88161), done.
-     remote: Total 367178 (delta 272761), reused 366942 (delta 272525)
-     Receiving objects: 100% (367178/367178), 133.26 MiB | 6.40 MiB/s, done.
-     Resolving deltas: 100% (272761/272761), done.
+     remote: Counting objects: 431956, done.
+     remote: Compressing objects: 100% (101918/101918), done.
+     remote: Total 431956 (delta 322982), reused 431910 (delta 322936)
+     Receiving objects: 100% (431956/431956), 153.76 MiB | 6.86 MiB/s, done.
+     Resolving deltas: 100% (322982/322982), done.
      Checking connectivity... done.
                     </literallayout>
                     Unless you specify a specific development branch or
@@ -963,13 +963,12 @@
            .
            .
            .
-       remotes/origin/master-next
-       remotes/origin/master-next2
-       remotes/origin/morty
-       remotes/origin/pinky
-       remotes/origin/purple
-       remotes/origin/pyro
        remotes/origin/rocko
+       remotes/origin/rocko-next
+       remotes/origin/sumo
+       remotes/origin/sumo-next
+       remotes/origin/thud
+       remotes/origin/thud-next
                     </literallayout>
                     </para></listitem>
                 <listitem><para>
@@ -1048,11 +1047,10 @@
         .
         .
         .
-     yocto-2.2
-     yocto-2.2.1
-     yocto-2.3
-     yocto-2.3.1
-     yocto-2.4
+     yocto-2.5.2
+     yocto-2.5.3
+     yocto-2.6
+     yocto-2.6.1
      yocto_1.5_M5.rc8
                     </literallayout>
                     </para></listitem>

documentation/dev-manual/dev-manual.xml

@@ -106,6 +106,21 @@
                 <date>May 2018</date>
                 <revremark>Released with the Yocto Project 2.5 Release.</revremark>
             </revision>
+            <revision>
+                <revnumber>2.5.1</revnumber>
+                <date>September 2018</date>
+                <revremark>The initial document released with the Yocto Project 2.5.1 Release.</revremark>
+            </revision>
+            <revision>
+                <revnumber>2.5.2</revnumber>
+                <date>January 2019</date>
+                <revremark>The initial document released with the Yocto Project 2.5.2 Release.</revremark>
+            </revision>
+            <revision>
+                <revnumber>2.5.3</revnumber>
+                <date>&REL_MONTH_YEAR;</date>
+                <revremark>The initial document released with the Yocto Project 2.5.3 Release.</revremark>
+            </revision>
         </revhistory>
 
     <copyright>

documentation/kernel-dev/kernel-dev-common.xml

@@ -1603,8 +1603,11 @@
             <title>Creating a&nbsp;&nbsp;<filename>defconfig</filename> File</title>
 
             <para>
-                A <filename>defconfig</filename> file is simply a
-                <filename>.config</filename> renamed to "defconfig".
+                A <filename>defconfig</filename> file in the context of
+                the Yocto Project is often a <filename>.config</filename>
+                file that is copied from a build or a
+                <filename>defconfig</filename> taken from the kernel tree
+                and moved into recipe space.
                 You can use a <filename>defconfig</filename> file
                 to retain a known set of kernel configurations from which the
                 OpenEmbedded build system can draw to create the final

documentation/kernel-dev/kernel-dev.xml

@@ -91,6 +91,21 @@
                 <date>May 2018</date>
                 <revremark>Released with the Yocto Project 2.5 Release.</revremark>
             </revision>
+            <revision>
+                <revnumber>2.5.1</revnumber>
+                <date>September 2018</date>
+                <revremark>The initial document released with the Yocto Project 2.5.1 Release.</revremark>
+            </revision>
+            <revision>
+                <revnumber>2.5.2</revnumber>
+                <date>January 2019</date>
+                <revremark>The initial document released with the Yocto Project 2.5.2 Release.</revremark>
+            </revision>
+            <revision>
+                <revnumber>2.5.3</revnumber>
+                <date>&REL_MONTH_YEAR;</date>
+                <revremark>The initial document released with the Yocto Project 2.5.3 Release.</revremark>
+            </revision>
         </revhistory>
 
     <copyright>

documentation/mega-manual/mega-manual.xml

@@ -75,6 +75,21 @@
                 <date>May 2018</date>
                 <revremark>Released with the Yocto Project 2.5 Release.</revremark>
             </revision>
+            <revision>
+                <revnumber>2.5.1</revnumber>
+                <date>September 2018</date>
+                <revremark>The initial document released with the Yocto Project 2.5.1 Release.</revremark>
+            </revision>
+            <revision>
+                <revnumber>2.5.2</revnumber>
+                <date>January 2019</date>
+                <revremark>The initial document released with the Yocto Project 2.5.2 Release.</revremark>
+            </revision>
+            <revision>
+                <revnumber>2.5.3</revnumber>
+                <date>&REL_MONTH_YEAR;</date>
+                <revremark>The initial document released with the Yocto Project 2.5.3 Release.</revremark>
+            </revision>
        </revhistory>
 
     <copyright>

documentation/overview-manual/overview-manual.xml

@@ -36,6 +36,21 @@
                 <date>May 2018</date>
                 <revremark>The initial document released with the Yocto Project 2.5 Release.</revremark>
             </revision>
+            <revision>
+                <revnumber>2.5.1</revnumber>
+                <date>September 2018</date>
+                <revremark>The initial document released with the Yocto Project 2.5.1 Release.</revremark>
+            </revision>
+            <revision>
+                <revnumber>2.5.2</revnumber>
+                <date>January 2019</date>
+                <revremark>The initial document released with the Yocto Project 2.5.2 Release.</revremark>
+            </revision>
+            <revision>
+                <revnumber>2.5.3</revnumber>
+                <date>&REL_MONTH_YEAR;</date>
+                <revremark>The initial document released with the Yocto Project 2.5.3 Release.</revremark>
+            </revision>
         </revhistory>
 
     <copyright>

documentation/poky.ent

@@ -1,19 +1,19 @@
-<!ENTITY DISTRO "2.5">
-<!ENTITY DISTRO_COMPRESSED "25">
+<!ENTITY DISTRO "2.5.3">
+<!ENTITY DISTRO_COMPRESSED "253">
 <!ENTITY DISTRO_NAME_NO_CAP "sumo">
 <!ENTITY DISTRO_NAME "Sumo">
 <!ENTITY DISTRO_NAME_NO_CAP_MINUS_ONE "rocko">
 <!ENTITY DISTRO_NAME_MINUS_ONE "Rocko">
-<!ENTITY YOCTO_DOC_VERSION "2.5">
-<!ENTITY YOCTO_DOC_VERSION_MINUS_ONE "2.4">
-<!ENTITY DISTRO_REL_TAG "yocto-2.5">
-<!ENTITY METAINTELVERSION "9.0">
-<!ENTITY REL_MONTH_YEAR "May 2018">
+<!ENTITY YOCTO_DOC_VERSION "2.5.3">
+<!ENTITY YOCTO_DOC_VERSION_MINUS_ONE "2.4.4">
+<!ENTITY DISTRO_REL_TAG "yocto-2.5.3">
+<!ENTITY METAINTELVERSION "9.2">
+<!ENTITY REL_MONTH_YEAR "March 2019">
 <!ENTITY META_INTEL_REL_TAG "&METAINTELVERSION;-&DISTRO_NAME_NO_CAP;-&YOCTO_DOC_VERSION;">
-<!ENTITY POKYVERSION "20.0.0">
-<!ENTITY POKYVERSION_COMPRESSED "2000">
+<!ENTITY POKYVERSION "20.0.3">
+<!ENTITY POKYVERSION_COMPRESSED "2003">
 <!ENTITY YOCTO_POKY "poky-&DISTRO_NAME_NO_CAP;-&POKYVERSION;">
-<!ENTITY COPYRIGHT_YEAR "2010-2018">
+<!ENTITY COPYRIGHT_YEAR "2010-2019">
 <!ENTITY YOCTO_DL_URL "http://downloads.yoctoproject.org">
 <!ENTITY YOCTO_HOME_URL "http://www.yoctoproject.org">
 <!ENTITY YOCTO_LISTS_URL "http://lists.yoctoproject.org">

documentation/profile-manual/profile-manual.xml

@@ -91,6 +91,21 @@
                 <date>May 2018</date>
                 <revremark>Released with the Yocto Project 2.5 Release.</revremark>
             </revision>
+            <revision>
+                <revnumber>2.5.1</revnumber>
+                <date>September 2018</date>
+                <revremark>The initial document released with the Yocto Project 2.5.1 Release.</revremark>
+            </revision>
+            <revision>
+                <revnumber>2.5.2</revnumber>
+                <date>January 2019</date>
+                <revremark>The initial document released with the Yocto Project 2.5.2 Release.</revremark>
+            </revision>
+            <revision>
+                <revnumber>2.5.3</revnumber>
+                <date>&REL_MONTH_YEAR;</date>
+                <revremark>The initial document released with the Yocto Project 2.5.3 Release.</revremark>
+            </revision>
         </revhistory>
 
     <copyright>

documentation/ref-manual/ref-manual.xml

@@ -122,6 +122,21 @@
                 <date>May 2018</date>
                 <revremark>Released with the Yocto Project 2.5 Release.</revremark>
             </revision>
+            <revision>
+                <revnumber>2.5.1</revnumber>
+                <date>September 2018</date>
+                <revremark>The initial document released with the Yocto Project 2.5.1 Release.</revremark>
+            </revision>
+            <revision>
+                <revnumber>2.5.2</revnumber>
+                <date>January 2019</date>
+                <revremark>The initial document released with the Yocto Project 2.5.2 Release.</revremark>
+            </revision>
+            <revision>
+                <revnumber>2.5.3</revnumber>
+                <date>&REL_MONTH_YEAR;</date>
+                <revremark>The initial document released with the Yocto Project 2.5.3 Release.</revremark>
+            </revision>
         </revhistory>
 
     <copyright>

documentation/ref-manual/ref-variables.xml

@@ -3636,10 +3636,17 @@
                 <para role="glossdeffirst">
 <!--                <para role="glossdeffirst"><imagedata fileref="figures/define-generic.png" /> -->
                     The short name of the distribution.
-                    This variable corresponds to a distribution
-                    configuration file whose root name is the same as the
-                    variable's argument and whose filename extension is
-                    <filename>.conf</filename>.
+                    For information on the long name of the distribution, see
+                    the
+                    <link linkend='var-DISTRO_NAME'><filename>DISTRO_NAME</filename></link>
+                    variable.
+                </para>
+
+                <para>
+                    The <filename>DISTRO</filename> variable corresponds to a
+                    distribution configuration file whose root name is the
+                    same as the variable's argument and whose filename
+                    extension is <filename>.conf</filename>.
                     For example, the distribution configuration file for the
                     Poky distribution is named <filename>poky.conf</filename>
                     and resides in the
@@ -3664,9 +3671,9 @@
                     The value for <filename>DISTRO</filename> must not contain
                     spaces, and is typically all lower-case.
                     <note>
-                        If the <filename>DISTRO</filename> variable is blank, a set
-                        of default configurations are used, which are specified
-                        within
+                        If the <filename>DISTRO</filename> variable is blank,
+                        a set of default configurations are used, which are
+                        specified within
                         <filename>meta/conf/distro/defaultsetup.conf</filename>
                         also in the Source Directory.
                     </note>
@@ -3931,6 +3938,46 @@
                 <para role="glossdeffirst">
 <!--                <para role="glossdeffirst"><imagedata fileref="figures/define-generic.png" /> -->
                     The long name of the distribution.
+                    For information on the short name of the distribution, see
+                    the
+                    <link linkend='var-DISTRO'><filename>DISTRO</filename></link>
+                    variable.
+                </para>
+
+                <para>
+                    The <filename>DISTRO_NAME</filename> variable corresponds
+                    to a distribution configuration file whose root name is the
+                    same as the variable's argument and whose filename
+                    extension is <filename>.conf</filename>.
+                    For example, the distribution configuration file for the
+                    Poky distribution is named <filename>poky.conf</filename>
+                    and resides in the
+                    <filename>meta-poky/conf/distro</filename> directory of
+                    the
+                    <link linkend='source-directory'>Source Directory</link>.
+                </para>
+
+                <para>
+                    Within that <filename>poky.conf</filename> file, the
+                    <filename>DISTRO_NAME</filename> variable is set as
+                    follows:
+                    <literallayout class='monospaced'>
+     DISTRO_NAME = "Poky (Yocto Project Reference Distro)"
+                    </literallayout>
+                </para>
+
+                <para>
+                    Distribution configuration files are located in a
+                    <filename>conf/distro</filename> directory within the
+                    <link linkend='metadata'>Metadata</link>
+                    that contains the distribution configuration.
+                    <note>
+                        If the <filename>DISTRO_NAME</filename> variable is
+                        blank, a set of default configurations are used, which
+                        are specified within
+                        <filename>meta/conf/distro/defaultsetup.conf</filename>
+                        also in the Source Directory.
+                    </note>
                 </para>
             </glossdef>
         </glossentry>
@@ -9917,7 +9964,7 @@ recipes-graphics/xorg-font/font-alias_1.0.3.bb:PR = "${INC_PR}.3"
                     execute at rootfs creation time rather than on the
                     target but depends on a native tool in order to execute,
                     you need to list the tools in
-                    <filename>PACKAGE_WRITE_DEPENDS</filename>.
+                    <filename>PACKAGE_WRITE_DEPS</filename>.
                 </para>
 
                 <para>
@@ -12175,13 +12222,26 @@ recipes-graphics/xorg-font/font-alias_1.0.3.bb:PR = "${INC_PR}.3"
                     A list of classes to remove from the
                     <link linkend='var-INHERIT'><filename>INHERIT</filename></link>
                     value globally within the extensible SDK configuration.
-                    The default value is "buildhistory icecc".
+                    The
+                    <link linkend='ref-classes-populate-sdk-*'><filename>populate-sdk-ext</filename></link>
+                    class sets the default value:
+                    <literallayout class='monospaced'>
+     SDK_INHERIT_BLACKLIST ?= "buildhistory icecc"
+                    </literallayout>
                 </para>
 
                 <para>
                     Some classes are not generally applicable within
-                    the extensible SDK context and you can use this variable
-                    to disable them.
+                    the extensible SDK context.
+                    You can use this variable to disable those classes.
+                </para>
+
+                <para>
+                    For additional information on how to customize the
+                    extensible SDK's configuration, see the
+                    "<ulink url='&YOCTO_DOCS_SDK_URL;#sdk-configuring-the-extensible-sdk'>Configuring the Extensible SDK</ulink>"
+                    section in the Yocto Project Application Development and
+                    the Extensible Software Development Kit (eSDK) manual.
                 </para>
             </glossdef>
         </glossentry>
@@ -12193,12 +12253,40 @@ recipes-graphics/xorg-font/font-alias_1.0.3.bb:PR = "${INC_PR}.3"
             <glossdef>
                 <para role="glossdeffirst">
 <!--                <para role="glossdeffirst"><imagedata fileref="figures/define-generic.png" /> -->
-                    A list of variables not allowed through from the build
-                    system configuration into the extensible SDK configuration.
+                    A list of variables not allowed through from the
+                    OpenEmbedded build system configuration into the extensible
+                    SDK configuration.
                     Usually, these are variables that are specific to the
                     machine on which the build system is running and thus
                     would be potentially problematic within the extensible SDK.
+                    </para>
+
+                    <para>By default,
+                    <filename>SDK_LOCAL_CONF_BLACKLIST</filename> is set in the
+                    <link linkend='ref-classes-populate-sdk-*'><filename>populate-sdk-ext</filename></link>
+                    class and excludes the following variables:
+                    <literallayout class='monospaced'>
+     <link linkend='var-CONF_VERSION'>CONF_VERSION</link>
+     <link linkend='var-BB_NUMBER_THREADS'>BB_NUMBER_THREADS</link>
+     <ulink url='&YOCTO_DOCS_BB_URL;#var-BB_NUMBER_PARSE_THREADS'>BB_NUMBER_PARSE_THREADS</ulink>
+     <link linkend='var-PARALLEL_MAKE'>PARALLEL_MAKE</link>
+     <link linkend='var-PRSERV_HOST'>PRSERV_HOST</link>
+     <link linkend='var-SSTATE_MIRRORS'>SSTATE_MIRRORS</link>
+     <link linkend='var-DL_DIR'>DL_DIR</link>
+     <link linkend='var-SSTATE_DIR'>SSTATE_DIR</link>
+     <link linkend='var-TMPDIR'>TMPDIR</link>
+     <link linkend='var-BB_SERVER_TIMEOUT'>BB_SERVER_TIMEOUT</link>
+                    </literallayout>
                 </para>
+
+                <para>
+                    For additional information on how to customize the
+                    extensible SDK's configuration, see the
+                    "<ulink url='&YOCTO_DOCS_SDK_URL;#sdk-configuring-the-extensible-sdk'>Configuring the Extensible SDK</ulink>"
+                    section in the Yocto Project Application Development and
+                    the Extensible Software Development Kit (eSDK) manual.
+                </para>
+
             </glossdef>
         </glossentry>
 
@@ -12209,8 +12297,16 @@ recipes-graphics/xorg-font/font-alias_1.0.3.bb:PR = "${INC_PR}.3"
             <glossdef>
                 <para role="glossdeffirst">
 <!--                <para role="glossdeffirst"><imagedata fileref="figures/define-generic.png" /> -->
-                    A list of variables allowed through from the build system
-                    configuration into the extensible SDK configuration.
+                    A list of variables allowed through from the OpenEmbedded
+                    build system configuration into the extensible SDK
+                    configuration.
+                    By default, the list of variables is empty and is set in
+                    the
+                    <link linkend='ref-classes-populate-sdk-*'><filename>populate-sdk-ext</filename></link>
+                    class.
+                </para>
+
+                <para>
                     This list overrides the variables specified using the
                     <link linkend='var-SDK_LOCAL_CONF_BLACKLIST'><filename>SDK_LOCAL_CONF_BLACKLIST</filename></link>
                     variable as well as any variables identified by automatic
@@ -12219,6 +12315,14 @@ recipes-graphics/xorg-font/font-alias_1.0.3.bb:PR = "${INC_PR}.3"
                     path and thus might not be valid on the system where the
                     SDK is installed.
                 </para>
+
+                <para>
+                    For additional information on how to customize the
+                    extensible SDK's configuration, see the
+                    "<ulink url='&YOCTO_DOCS_SDK_URL;#sdk-configuring-the-extensible-sdk'>Configuring the Extensible SDK</ulink>"
+                    section in the Yocto Project Application Development and
+                    the Extensible Software Development Kit (eSDK) manual.
+                </para>
             </glossdef>
         </glossentry>
 
@@ -13197,7 +13301,7 @@ recipes-graphics/xorg-font/font-alias_1.0.3.bb:PR = "${INC_PR}.3"
                             </para></listitem>
                         <listitem><para><emphasis><filename>maxdate</filename> -</emphasis>
                             Apply the patch only if <filename>SRCDATE</filename>
-                            is not later than <filename>mindate</filename>.
+                            is not later than <filename>maxdate</filename>.
                             </para></listitem>
                         <listitem><para><emphasis><filename>minrev</filename> -</emphasis>
                             Apply the patch only if <filename>SRCREV</filename>
@@ -16308,7 +16412,7 @@ recipes-graphics/xorg-font/font-alias_1.0.3.bb:PR = "${INC_PR}.3"
      USERADD_PACKAGES = "${PN}"
                     </literallayout>
                     <note>
-                        If follows that if you are going to use the
+                        It follows that if you are going to use the
                         <filename>USERADD_PACKAGES</filename> variable,
                         you need to set one or more of the
                         <link linkend='var-USERADD_PARAM'><filename>USERADD_PARAM</filename></link>,

documentation/sdk-manual/sdk-appendix-customizing.xml

@@ -17,26 +17,31 @@
         The extensible SDK primarily consists of a pre-configured copy of
         the OpenEmbedded build system from which it was produced.
         Thus, the SDK's configuration is derived using that build system and
-        the following filters, which the OpenEmbedded build system applies
-        against <filename>local.conf</filename> and
-        <filename>auto.conf</filename> if they are present:
+        the filters shown in the following list.
+        When these filters are present, the OpenEmbedded build system applies
+        them against <filename>local.conf</filename> and
+        <filename>auto.conf</filename>:
         <itemizedlist>
             <listitem><para>
                 Variables whose values start with "/" are excluded since the
                 assumption is that those values are paths that are likely to
-                be specific to the build host.
+                be specific to the
+                <ulink url='&YOCTO_DOCS_REF_URL;#hardware-build-system-term'>build host</ulink>.
                 </para></listitem>
             <listitem><para>
                 Variables listed in
                 <ulink url='&YOCTO_DOCS_REF_URL;#var-SDK_LOCAL_CONF_BLACKLIST'><filename>SDK_LOCAL_CONF_BLACKLIST</filename></ulink>
                 are excluded.
-                The default value blacklists
-                <ulink url='&YOCTO_DOCS_REF_URL;#var-CONF_VERSION'><filename>CONF_VERSION</filename></ulink>,
-                <ulink url='&YOCTO_DOCS_REF_URL;#var-BB_NUMBER_THREADS'><filename>BB_NUMBER_THREADS</filename></ulink>,
-                <ulink url='&YOCTO_DOCS_REF_URL;#var-PARALLEL_MAKE'><filename>PARALLEL_MAKE</filename></ulink>,
-                <ulink url='&YOCTO_DOCS_REF_URL;#var-PRSERV_HOST'><filename>PRSERV_HOST</filename></ulink>,
-                and
-                <ulink url='&YOCTO_DOCS_REF_URL;#var-SSTATE_MIRRORS'><filename>SSTATE_MIRRORS</filename></ulink>.
+                These variables are not allowed through from the OpenEmbedded
+                build system configuration into the extensible SDK
+                configuration.
+                Typically, these variables are specific to the machine on
+                which the build system is running and could be problematic
+                as part of the extensible SDK configuration.</para>
+
+                <para>For a list of the variables excluded by default, see the
+                <ulink url='&YOCTO_DOCS_REF_URL;#var-SDK_LOCAL_CONF_BLACKLIST'><filename>SDK_LOCAL_CONF_BLACKLIST</filename></ulink>
+                in the glossary of the Yocto Project Reference Manual.
                 </para></listitem>
             <listitem><para>
                 Variables listed in
@@ -44,7 +49,7 @@
                 are included.
                 Including a variable in the value of
                 <filename>SDK_LOCAL_CONF_WHITELIST</filename> overrides either
-                of the above two conditions.
+                of the previous two filters.
                 The default value is blank.
                 </para></listitem>
             <listitem><para>
@@ -54,7 +59,7 @@
                 <ulink url='&YOCTO_DOCS_REF_URL;#var-SDK_INHERIT_BLACKLIST'><filename>SDK_INHERIT_BLACKLIST</filename></ulink>
                 are disabled.
                 Using <filename>SDK_INHERIT_BLACKLIST</filename> to disable
-                these classes is is the typical method to disable classes that
+                these classes is the typical method to disable classes that
                 are problematic or unnecessary in the SDK context.
                 The default value blacklists the
                 <ulink url='&YOCTO_DOCS_REF_URL;#ref-classes-buildhistory'><filename>buildhistory</filename></ulink>
@@ -73,11 +78,13 @@
     </para>
 </section>
 
-<section id='adjusting-the-extensible-sdk-to-suit-your-build-system-setup'>
-    <title>Adjusting the Extensible SDK to Suit Your Build System Setup</title>
+<section id='adjusting-the-extensible-sdk-to-suit-your-build-hosts-setup'>
+    <title>Adjusting the Extensible SDK to Suit Your Build Host's Setup</title>
 
     <para>
-        In most cases, the extensible SDK defaults should work.
+        In most cases, the extensible SDK defaults should work with your
+        <ulink url='&YOCTO_DOCS_REF_URL;#hardware-build-system-term'>build host's</ulink>
+        setup.
         However, some cases exist for which you might consider making
         adjustments:
         <itemizedlist>
@@ -88,33 +95,43 @@
                 variable and you do not need or want those classes enabled in
                 the SDK, you can blacklist them by adding them to the
                 <ulink url='&YOCTO_DOCS_REF_URL;#var-SDK_INHERIT_BLACKLIST'><filename>SDK_INHERIT_BLACKLIST</filename></ulink>
-                variable.
-                The default value of <filename>SDK_INHERIT_BLACKLIST</filename>
-                is set using the "?=" operator.
-                Consequently, you will need to either set the complete value
-                using "=" or append the value using "_append".
+                variable as described in the fourth bullet of the previous
+                section.
+                <note>
+                    The default value of
+                    <filename>SDK_INHERIT_BLACKLIST</filename> is set using
+                    the "?=" operator.
+                    Consequently, you will need to either define the entire
+                    list by using the "=" operator, or you will need to append
+                    a value using either "_append" or the "+=" operator.
+                    You can learn more about these operators in the
+                    "<ulink url='&YOCTO_DOCS_BB_URL;#basic-syntax'>Basic Syntax</ulink>"
+                    section of the BitBake User Manual.
+                </note>.
                 </para></listitem>
             <listitem><para>
                 If you have classes or recipes that add additional tasks to
-                the standard build flow (i.e. that execute as part of building
-                the recipe as opposed to needing to be called explicitly), then
-                you need to do one of the following:
+                the standard build flow (i.e. the tasks execute as the recipe
+                builds as opposed to being called explicitly), then you need
+                to do one of the following:
                 <itemizedlist>
                     <listitem><para>
-                        Ensure the tasks are shared state tasks (i.e. their
-                        output is saved to and can be restored from the shared
-                        state cache), or that the tasks are able to be
-                        produced quickly from a task that is a shared state
-                        task and add the task name to the value of
+                        After ensuring the tasks are
+                        <ulink url='&YOCTO_DOCS_OM_URL;#shared-state-cache'>shared state</ulink>
+                        tasks (i.e. the output of the task is saved to and
+                        can be restored from the shared state cache) or
+                        ensuring the tasks are able to be produced quickly from
+                        a task that is a shared state task, add the task name
+                        to the value of
                         <ulink url='&YOCTO_DOCS_REF_URL;#var-SDK_RECRDEP_TASKS'><filename>SDK_RECRDEP_TASKS</filename></ulink>.
                         </para></listitem>
                     <listitem><para>
                         Disable the tasks if they are added by a class and
                         you do not need the functionality the class provides
                         in the extensible SDK.
-                        To disable the tasks, add the class to
-                        <filename>SDK_INHERIT_BLACKLIST</filename> as previously
-                        described.
+                        To disable the tasks, add the class to the
+                        <filename>SDK_INHERIT_BLACKLIST</filename> variable
+                        as described in the previous section.
                         </para></listitem>
                 </itemizedlist>
                 </para></listitem>
@@ -132,7 +149,7 @@
                 <ulink url='&YOCTO_DOCS_REF_URL;#var-SDK_UPDATE_URL'><filename>SDK_UPDATE_URL</filename></ulink>
                 variable.
                 For more information, see the
-                "<link linkend='sdk-providing-updates-after-installing-the-extensible-sdk'>Providing Updates After Installing the Extensible SDK</link>"
+                "<link linkend='sdk-providing-updates-to-the-extensible-sdk-after-installation'>Providing Updates to the Extensible SDK After Installation</link>"
                 section.
                 </para></listitem>
             <listitem><para>
@@ -162,11 +179,12 @@
     </para>
 </section>
 
-<section id='sdk-changing-the-appearance-of-the-extensible-sdk'>
-    <title>Changing the Appearance of the Extensible SDK</title>
+<section id='sdk-changing-the-sdk-installer-title'>
+    <title>Changing the Extensible SDK Installer Title</title>
 
     <para>
-        You can change the title shown by the SDK installer by setting the
+        You can change the displayed title for the SDK installer by setting
+        the
         <ulink url='&YOCTO_DOCS_REF_URL;#var-SDK_TITLE'><filename>SDK_TITLE</filename></ulink>
         variable.
         By default, this title is derived from
@@ -177,21 +195,37 @@
         <ulink url='&YOCTO_DOCS_REF_URL;#var-DISTRO'><filename>DISTRO</filename></ulink>
         variable.
     </para>
+
+    <para>
+        The
+        <ulink url='&YOCTO_DOCS_REF_URL;#ref-classes-populate-sdk-*'><filename>populate_sdk_ext</filename></ulink>
+        class defines the default value of the <filename>SDK_TITLE</filename>
+        variable as follows:
+        <literallayout class='monospaced'>
+     SDK_TITLE_task-populate-sdk-ext = "${@d.getVar('DISTRO_NAME') or d.getVar('DISTRO')} Extensible SDK"
+        </literallayout>
+    </para>
 </section>
 
-<section id='sdk-providing-updates-after-installing-the-extensible-sdk'>
-    <title>Providing Updates After Installing the Extensible SDK</title>
+<section id='sdk-providing-updates-to-the-extensible-sdk-after-installation'>
+    <title>Providing Updates to the Extensible SDK After Installation</title>
 
     <para>
         When you make changes to your configuration or to the metadata and
         if you want those changes to be reflected in installed SDKs, you need
-        to perform additional steps to make it possible for those that use
-        the SDK to update their installations with the
+        to perform additional steps.
+        These steps make it possible for anyone using the installed SDKs to
+        update the installed SDKs by using the
         <filename>devtool sdk-update</filename> command:
         <orderedlist>
             <listitem><para>
-                Arrange to be created a directory that can be shared over
-                HTTP or HTTPS.
+                Create a directory that can be shared over HTTP or HTTPS.
+                You can do this by setting up a web server such as an
+                <ulink url='https://en.wikipedia.org/wiki/Apache_HTTP_Server'>Apache HTTP Server</ulink>
+                or
+                <ulink url='https://en.wikipedia.org/wiki/Nginx'>Nginx</ulink>
+                server in the cloud to host the directory.
+                This directory must contain the published SDK.
                 </para></listitem>
             <listitem><para>
                 Set the
@@ -199,7 +233,10 @@
                 variable to point to the corresponding HTTP or HTTPS URL.
                 Setting this variable causes any SDK built to default to that
                 URL and thus, the user does not have to pass the URL to the
-                <filename>devtool sdk-update</filename> command.
+                <filename>devtool sdk-update</filename> command as described
+                in the
+                "<link linkend='sdk-applying-updates-to-an-installed-extensible-sdk'>Applying Updates to an Installed Extensible SDK</link>"
+                section.
                 </para></listitem>
             <listitem><para>
                 Build the extensible SDK normally (i.e., use the
@@ -209,7 +246,7 @@
             <listitem><para>
                 Publish the SDK using the following command:
                 <literallayout class='monospaced'>
-     $ oe-publish-sdk <replaceable>some_path</replaceable>/sdk-installer.sh <replaceable>path_to_shared/http_directory</replaceable>
+     $ oe-publish-sdk <replaceable>some_path</replaceable>/sdk-installer.sh <replaceable>path_to_shared_http_directory</replaceable>
                 </literallayout>
                 You must repeat this step each time you rebuild the SDK
                 with changes that you want to make available through the
@@ -219,11 +256,11 @@
     </para>
 
     <para>
-        Completing the above steps allows users of the existing SDKs to
-        simply run <filename>devtool sdk-update</filename> to retrieve the
-        latest updates.
+        Completing the above steps allows users of the existing installed
+        SDKs to simply run <filename>devtool sdk-update</filename> to
+        retrieve and apply the latest updates.
         See the
-        "<link linkend='sdk-updating-the-extensible-sdk'>Updating the Extensible SDK</link>"
+        "<link linkend='sdk-applying-updates-to-an-installed-extensible-sdk'>Applying Updates to an Installed Extensible SDK</link>"
         section for further information.
     </para>
 </section>
@@ -232,27 +269,38 @@
     <title>Providing Additional Installable Extensible SDK Content</title>
 
     <para>
-        If you want the users of the extensible SDK you are building to be
-        able to add items to the SDK without needing to build the
-        items from source, you need to do a number of things:
+        If you want the users of an extensible SDK you build to be
+        able to add items to the SDK without requiring the users to build
+        the items from source, you need to do a number of things:
         <orderedlist>
             <listitem><para>
                 Ensure the additional items you want the user to be able to
-                install are actually built.
-                You can ensure these items are built a number of different
-                ways: 1) Build them explicitly, perhaps using one or more
-                "meta" recipes that depend on lists of other recipes to keep
-                things tidy, or 2) Build the "world" target and set
-                <filename>EXCLUDE_FROM_WORLD_pn-</filename><replaceable>recipename</replaceable>
-                for the recipes you do not want built.
-                See the
-                <ulink url='&YOCTO_DOCS_REF_URL;#var-EXCLUDE_FROM_WORLD'><filename>EXCLUDE_FROM_WORLD</filename></ulink>
-                variable for additional information.
+                install are already built:
+                <itemizedlist>
+                    <listitem><para>
+                        Build the items explicitly.
+                        You could use one or more "meta" recipes that depend
+                        on lists of other recipes.
+                        </para></listitem>
+                    <listitem><para>
+                        Build the "world" target and set
+                        <filename>EXCLUDE_FROM_WORLD_pn-</filename><replaceable>recipename</replaceable>
+                        for the recipes you do not want built.
+                        See the
+                        <ulink url='&YOCTO_DOCS_REF_URL;#var-EXCLUDE_FROM_WORLD'><filename>EXCLUDE_FROM_WORLD</filename></ulink>
+                        variable for additional information.
+                        </para></listitem>
+                </itemizedlist>
                 </para></listitem>
             <listitem><para>
                 Expose the <filename>sstate-cache</filename> directory
                 produced by the build.
-                Typically, you expose this directory over HTTP or HTTPS.
+                Typically, you expose this directory by making it available
+                through an
+                <ulink url='https://en.wikipedia.org/wiki/Apache_HTTP_Server'>Apache HTTP Server</ulink>
+                or
+                <ulink url='https://en.wikipedia.org/wiki/Nginx'>Nginx</ulink>
+                server.
                 </para></listitem>
             <listitem><para>
                 Set the appropriate configuration so that the produced SDK
@@ -285,7 +333,7 @@
                         Alternatively, if you just want to set the
                         <filename>SSTATE_MIRRORS</filename> variable's value
                         for the SDK alone, create a
-                        <filename>conf/sdk-extra.conf</filename> either in
+                        <filename>conf/sdk-extra.conf</filename> file either in
                         your
                         <ulink url='&YOCTO_DOCS_REF_URL;#build-directory'>Build Directory</ulink>
                         or within any layer and put your
@@ -324,24 +372,24 @@
         size, which downloads and installs quickly.
         You need to realize, though, that the minimal installer does not
         install any libraries or tools out of the box.
-        These must be installed either "on the fly" or through actions you
-        perform using <filename>devtool</filename> or explicitly with the
-        <filename>devtool sdk-install</filename> command.
+        These libraries and tools must be installed either "on the fly" or
+        through actions you perform using <filename>devtool</filename> or
+        explicitly with the <filename>devtool sdk-install</filename> command.
     </para>
 
     <para>
-        In most cases, when building a minimal SDK you will need to also enable
+        In most cases, when building a minimal SDK you need to also enable
         bringing in the information on a wider range of packages produced by
         the system.
-        This is particularly true so that <filename>devtool add</filename>
-        is able to effectively map dependencies it discovers in a source tree
-        to the appropriate recipes.
-        Also so that the <filename>devtool search</filename> command
-        is able to return useful results.
+        Requiring this wider range of information is particularly true
+        so that <filename>devtool add</filename> is able to effectively map
+        dependencies it discovers in a source tree to the appropriate recipes.
+        Additionally, the information enables the
+        <filename>devtool search</filename> command to return useful results.
     </para>
 
     <para>
-        To facilitate this wider range of information, you would additionally
+        To facilitate this wider range of information, you would need to
         set the following:
         <literallayout class='monospaced'>
      SDK_INCLUDE_PKGDATA = "1"
@@ -384,8 +432,8 @@
         <ulink url='&YOCTO_DOCS_REF_URL;#var-SDK_INCLUDE_TOOLCHAIN'><filename>SDK_INCLUDE_TOOLCHAIN</filename></ulink>
         variable to "1".
         In particular, it is useful to include the toolchain when you
-        have set <filename>SDK_EXT_TYPE</filename> to
-        "minimal", which by default, excludes the toolchain.
+        have set <filename>SDK_EXT_TYPE</filename> to "minimal", which by
+        default, excludes the toolchain.
         Also, it is helpful if you are building a small SDK for use with
         an IDE, such as <trademark class='trade'>Eclipse</trademark>, or some
         other tool where you do not want to take extra steps to install a

documentation/sdk-manual/sdk-appendix-obtain.xml

@@ -159,6 +159,20 @@
                 <ulink url='&YOCTO_DOCS_REF_URL;#var-SDKMACHINE'><filename>SDKMACHINE</filename></ulink>
                 variable in the <filename>local.conf</filename> file in your
                 Build Directory is correctly set.
+                <note>
+                    If you are building an SDK installer for the Extensible
+                    SDK, the <filename>SDKMACHINE</filename> value must be
+                    set for the architecture of the machine you are using to
+                    build the installer.
+                    If <filename>SDKMACHINE</filename> is not set appropriately,
+                    the build fails and provides an error message similar to
+                    the following:
+                    <literallayout class='monospaced'>
+     The extensible SDK can currently only be built for the same architecture as the machine being built on - SDK_ARCH is
+     set to i686 (likely via setting SDKMACHINE) which is different from the architecture of the build machine (x86_64).
+     Unable to continue.
+                    </literallayout>
+                </note>
                 </para></listitem>
             <listitem><para>
                 <emphasis>Build the SDK Installer:</emphasis>
@@ -264,13 +278,14 @@
                 Root Filesystem Image File:</emphasis>
                 You need to find and download the root filesystem image
                 file that is appropriate for your target system.
-                These files are kept in the
+                These files are kept in machine-specific folders in the
                 <ulink url='&YOCTO_DL_URL;/releases/yocto/yocto-&DISTRO;/machines/'>Index of Releases</ulink>
                 in the "machines" directory.</para>
 
-                <para>The "machines" directory contains tarballs
-                (<filename>*.tar.bz2</filename>) for supported machines.
-                The directory also contains flattened root filesystem
+                <para>The machine-specific folders of the "machines" directory
+                contain tarballs (<filename>*.tar.bz2</filename>) for supported
+                machines.
+                These directories also contain flattened root filesystem
                 image files (<filename>*.ext4</filename>), which you can use
                 with QEMU directly.</para>
 
@@ -284,14 +299,15 @@
 
      Where:
          <replaceable>profile</replaceable> is the filesystem image's profile:
-                   lsb, lsb-dev, lsb-sdk, lsb-qt3, minimal, minimal-dev, sato,
-                   sato-dev, sato-sdk, minimal-initramfs, or sdk-ptest. For
-                   information on these types of image profiles, see the
-                   "Images" chapter in the Yocto Project Reference Manual.
+                   lsb, lsb-dev, lsb-sdk, minimal, minimal-dev, minimal-initramfs,
+                   sato, sato-dev, sato-sdk, sato-sdk-ptest. For information on
+                   these types of image profiles, see the "<ulink url='&YOCTO_DOCS_REF_URL;#ref-images'>Images</ulink>" chapter in
+                   the Yocto Project Reference Manual.
 
          <replaceable>arch</replaceable> is a string representing the target architecture:
-                   beaglebone, edgerouter, genericx86, genericx86-64, mpc8315e-rdb,
-                   and several versions for qemu*.
+                   beaglebone-yocto, beaglebone-yocto-lsb, edgerouter, edgerouter-lsb,
+                   genericx86, genericx86-64, genericx86-64-lsb, genericx86-lsb,
+                   mpc8315e-rdb, mpc8315e-rdb-lsb, and qemu*.
 
 <!-->
          <replaceable>date_time</replaceable> is a date and time stamp.
@@ -303,26 +319,28 @@
                 <filename>core-image-minimal</filename> images.
                 </para>
 
-                <para>For example, if your target hardware system is a
-                BeagleBone board and your image is a
-                <filename>core-image-minimal</filename> image, you need
-                to download the following root filesystem image file:
+                <para>For example, if you plan on using a BeagleBone device
+                as your target hardware and your image is a
+                <filename>core-image-sato-sdk</filename>
+                image, you can download the following file:
                 <literallayout class='monospaced'>
-     core-image-minimal-beaglebone-yocto.tar.bz2
+     core-image-sato-sdk-beaglebone-yocto.tar.bz2
                 </literallayout>
                 </para></listitem>
             <listitem><para>
                 <emphasis>Initialize the Cross-Development Environment:</emphasis>
-                You must <filename>source</filename>
-                the cross-development environment setup script to establish
-                necessary environment variables.</para>
+                You must <filename>source</filename> the cross-development
+                environment setup script to establish necessary environment
+                variables.</para>
 
                 <para>This script is located in the top-level directory in
                 which you installed the toolchain (e.g.
                 <filename>poky_sdk</filename>).</para>
 
-                <para>Following is an example for the BeagleBone Board
-                previously shown:
+                <para>Following is an example based on the toolchain installed
+                in the
+                "<link linkend='sdk-locating-pre-built-sdk-installers'>Locating Pre-Built SDK Installers</link>"
+                section:
                 <literallayout class='monospaced'>
      $ source ~/poky_sdk/environment-setup-core2-64-poky-linux
                 </literallayout>
@@ -339,10 +357,10 @@
                 This command extracts the root filesystem into the
                 <filename>core2-64-sato</filename> directory:
                 <literallayout class='monospaced'>
-     $ runqemu-extract-sdk ~/Downloads/core-image-sato-core2-64.tar.bz2 ~/core2-64-sato
+     $ runqemu-extract-sdk ~/Downloads/core-image-sato-sdk-beaglebone-yocto.tar.bz2 ~/beaglebone-sato
                 </literallayout>
                 You could now point to the target sysroot at
-                <filename>core2-64-sato</filename>.
+                <filename>beablebone-sato</filename>.
                 </para></listitem>
         </orderedlist>
     </para>
@@ -358,7 +376,7 @@
     </para>
 
     <para>
-        <imagedata fileref="figures/sdk-installed-standard-sdk-directory.png" scale="60" align="center" />
+        <imagedata fileref="figures/sdk-installed-standard-sdk-directory.png" scale="80" align="center" />
     </para>
 
     <para>
@@ -399,7 +417,7 @@
     </para>
 
     <para>
-        <imagedata fileref="figures/sdk-installed-extensible-sdk-directory.png" scale="60" align="center" />
+        <imagedata fileref="figures/sdk-installed-extensible-sdk-directory.png" scale="80" align="center" />
     </para>
 
     <para>
@@ -414,8 +432,8 @@
     <para>
         Of note in the directory structure are an environment setup script
         for the SDK, a configuration file for the target, a version file for
-        the target, and a log file for the OpenEmbedded build system
-        preparation script run by the installer.
+        the target, and log files for the OpenEmbedded build system
+        preparation script run by the installer and BitBake.
     </para>
 
     <para>
@@ -423,11 +441,9 @@
         portions of the file or directory name.
         For example,
         <replaceable>install_dir</replaceable> is the directory where the SDK
-        is installed, which is <filename>poky_sdk</filename> by default.
+        is installed, which is <filename>poky_sdk</filename> by default, and
         <replaceable>target</replaceable> represents the target
-        architecture (e.g. <filename>i586</filename>) and
-        <replaceable>host</replaceable> represents the development system's
-        architecture (e.g. <filename>x86_64</filename>).
+        architecture (e.g. <filename>i586</filename>).
     </para>
 </section>
 

documentation/sdk-manual/sdk-extensible.xml

@@ -1715,31 +1715,35 @@
         </para>
     </section>
 
-    <section id='sdk-updating-the-extensible-sdk'>
-        <title>Updating the Extensible SDK</title>
+    <section id='sdk-applying-updates-to-an-installed-extensible-sdk'>
+        <title>Applying Updates to an Installed Extensible SDK</title>
 
         <para>
-            If you are working with an extensible SDK that gets occasionally
-            updated (e.g. typically when that SDK has been provided to you by
-            another party), then you will need to manually pull down those
-            updates to your installed SDK.
+            If you are working with an installed extensible SDK that gets
+            occasionally updated (e.g. a third-party SDK), then you will need
+            to manually "pull down" the updates into the installed SDK.
         </para>
 
         <para>
-            To update your installed SDK, run the following:
+            To update your installed SDK, use <filename>devtool</filename> as
+            follows:
             <literallayout class='monospaced'>
      $ devtool sdk-update
             </literallayout>
             The previous command assumes your SDK provider has set the default
-            update URL for you.
-            If that URL has not been set, you need to specify it yourself as
-            follows:
+            update URL for you through the
+            <ulink url='&YOCTO_DOCS_REF_URL;#var-SDK_UPDATE_URL'><filename>SDK_UPDATE_URL</filename></ulink>
+            variable as described in the
+            "<link linkend='sdk-providing-updates-to-the-extensible-sdk-after-installation'>Providing Updates to the Extensible SDK After Installation</link>"
+            section.
+            If the SDK provider has not set that default URL, you need to
+            specify it yourself in the command as follows:
             <literallayout class='monospaced'>
      $ devtool sdk-update <replaceable>path_to_update_directory</replaceable>
             </literallayout>
             <note>
-                The URL needs to point specifically to a published SDK and not an
-                SDK installer that you would download and install.
+                The URL needs to point specifically to a published SDK and
+                not to an SDK installer that you would download and install.
             </note>
         </para>
     </section>

documentation/sdk-manual/sdk-manual.xml

@@ -56,6 +56,21 @@
                 <date>May 2018</date>
                 <revremark>Released with the Yocto Project 2.5 Release.</revremark>
             </revision>
+            <revision>
+                <revnumber>2.5.1</revnumber>
+                <date>September 2018</date>
+                <revremark>The initial document released with the Yocto Project 2.5.1 Release.</revremark>
+            </revision>
+            <revision>
+                <revnumber>2.5.2</revnumber>
+                <date>January 2019</date>
+                <revremark>The initial document released with the Yocto Project 2.5.2 Release.</revremark>
+            </revision>
+            <revision>
+                <revnumber>2.5.3</revnumber>
+                <date>&REL_MONTH_YEAR;</date>
+                <revremark>The initial document released with the Yocto Project 2.5.3 Release.</revremark>
+            </revision>
        </revhistory>
 
     <copyright>

documentation/toaster-manual/toaster-manual.xml

@@ -66,6 +66,21 @@
                 <date>May 2018</date>
                 <revremark>Released with the Yocto Project 2.5 Release.</revremark>
             </revision>
+            <revision>
+                <revnumber>2.5.1</revnumber>
+                <date>September 2018</date>
+                <revremark>The initial document released with the Yocto Project 2.5.1 Release.</revremark>
+            </revision>
+            <revision>
+                <revnumber>2.5.2</revnumber>
+                <date>January 2019</date>
+                <revremark>The initial document released with the Yocto Project 2.5.2 Release.</revremark>
+            </revision>
+            <revision>
+                <revnumber>2.5.3</revnumber>
+                <date>&REL_MONTH_YEAR;</date>
+                <revremark>The initial document released with the Yocto Project 2.5.3 Release.</revremark>
+            </revision>
        </revhistory>
 
     <copyright>

documentation/tools/mega-manual.sed

@@ -2,39 +2,39 @@
 # This style is for manual folders like "yocto-project-qs" and "poky-ref-manual".
 # This is the old way that did it.  Can't do that now that we have "bitbake-user-manual" strings
 # in the mega-manual.
-# s@"ulink" href="http://www.yoctoproject.org/docs/2.5/[a-z]*-[a-z]*-[a-z]*/[a-z]*-[a-z]*-[a-z]*.html#@"link" href="#@g
-s@"ulink" href="http://www.yoctoproject.org/docs/2.5/yocto-project-qs/yocto-project-qs.html#@"link" href="#@g
-s@"ulink" href="http://www.yoctoproject.org/docs/2.5/poky-ref-manual/poky-ref-manual.html#@"link" href="#@g
+# s@"ulink" href="http://www.yoctoproject.org/docs/2.5.2/[a-z]*-[a-z]*-[a-z]*/[a-z]*-[a-z]*-[a-z]*.html#@"link" href="#@g
+s@"ulink" href="http://www.yoctoproject.org/docs/2.5.2/yocto-project-qs/yocto-project-qs.html#@"link" href="#@g
+s@"ulink" href="http://www.yoctoproject.org/docs/2.5.2/poky-ref-manual/poky-ref-manual.html#@"link" href="#@g
 
 # Processes all other manuals (<word>-<word> style) except for the BitBake User Manual because
 # it is not included in the mega-manual.
 # This style is for manual folders that use two word, which is the standard now (e.g. "ref-manual").
 # This was the one-liner that worked before we introduced the BitBake User Manual, which is
 # not in the mega-manual.
-# s@"ulink" href="http://www.yoctoproject.org/docs/2.5/[a-z]*-[a-z]*/[a-z]*-[a-z]*.html#@"link" href="#@g
+# s@"ulink" href="http://www.yoctoproject.org/docs/2.5.2/[a-z]*-[a-z]*/[a-z]*-[a-z]*.html#@"link" href="#@g
 
-s@"ulink" href="http://www.yoctoproject.org/docs/2.5/sdk-manual/sdk-manual.html#@"link" href="#@g
-s@"ulink" href="http://www.yoctoproject.org/docs/2.5/bsp-guide/bsp-guide.html#@"link" href="#@g
-s@"ulink" href="http://www.yoctoproject.org/docs/2.5/dev-manual/dev-manual.html#@"link" href="#@g
-s@"ulink" href="http://www.yoctoproject.org/docs/2.5/overview-manual/overview-manual.html#@"link" href="#@g
-s@"ulink" href="http://www.yoctoproject.org/docs/2.5/brief-yoctoprojectqs/brief-yoctoprojectqs.html#@"link" href="#@g
-s@"ulink" href="http://www.yoctoproject.org/docs/2.5/kernel-dev/kernel-dev.html#@"link" href="#@g
-s@"ulink" href="http://www.yoctoproject.org/docs/2.5/profile-manual/profile-manual.html#@"link" href="#@g
-s@"ulink" href="http://www.yoctoproject.org/docs/2.5/ref-manual/ref-manual.html#@"link" href="#@g
-s@"ulink" href="http://www.yoctoproject.org/docs/2.5/toaster-manual/toaster-manual.html#@"link" href="#@g
+s@"ulink" href="http://www.yoctoproject.org/docs/2.5.2/sdk-manual/sdk-manual.html#@"link" href="#@g
+s@"ulink" href="http://www.yoctoproject.org/docs/2.5.2/bsp-guide/bsp-guide.html#@"link" href="#@g
+s@"ulink" href="http://www.yoctoproject.org/docs/2.5.2/dev-manual/dev-manual.html#@"link" href="#@g
+s@"ulink" href="http://www.yoctoproject.org/docs/2.5.2/overview-manual/overview-manual.html#@"link" href="#@g
+s@"ulink" href="http://www.yoctoproject.org/docs/2.5.2/brief-yoctoprojectqs/brief-yoctoprojectqs.html#@"link" href="#@g
+s@"ulink" href="http://www.yoctoproject.org/docs/2.5.2/kernel-dev/kernel-dev.html#@"link" href="#@g
+s@"ulink" href="http://www.yoctoproject.org/docs/2.5.2/profile-manual/profile-manual.html#@"link" href="#@g
+s@"ulink" href="http://www.yoctoproject.org/docs/2.5.2/ref-manual/ref-manual.html#@"link" href="#@g
+s@"ulink" href="http://www.yoctoproject.org/docs/2.5.2/toaster-manual/toaster-manual.html#@"link" href="#@g
 
 # Process cases where just an external manual is referenced without an id anchor
-s@<a class="ulink" href="http://www.yoctoproject.org/docs/2.5/brief-yoctoprojectqs/brief-yoctoprojectqs.html" target="_top">Yocto Project Quick Build</a>@Yocto Project Quick Build@g
-s@<a class="ulink" href="http://www.yoctoproject.org/docs/2.5/yocto-project-qs/yocto-project-qs.html" target="_top">Yocto Project Quick Start</a>@Yocto Project Quick Start@g
-s@<a class="ulink" href="http://www.yoctoproject.org/docs/2.5/dev-manual/dev-manual.html" target="_top">Yocto Project Development Tasks Manual</a>@Yocto Project Development Tasks Manual@g
-s@<a class="ulink" href="http://www.yoctoproject.org/docs/2.5/overview-manual/overview-manual.html" target="_top">Yocto Project Overview and Concepts Manual</a>@Yocto project Overview and Concepts Manual@g
-s@<a class="ulink" href="http://www.yoctoproject.org/docs/2.5/sdk-manual/sdk-manual.html" target="_top">Yocto Project Application Development and the Extensible Software Development Kit (eSDK)</a>@Yocto Project Application Development and the Extensible Software Development Kit (eSDK)@g
-s@<a class="ulink" href="http://www.yoctoproject.org/docs/2.5/bsp-guide/bsp-guide.html" target="_top">Yocto Project Board Support Package (BSP) Developer's Guide</a>@Yocto Project Board Support Package (BSP) Developer's Guide@g
-s@<a class="ulink" href="http://www.yoctoproject.org/docs/2.5/profile-manual/profile-manual.html" target="_top">Yocto Project Profiling and Tracing Manual</a>@Yocto Project Profiling and Tracing Manual@g
-s@<a class="ulink" href="http://www.yoctoproject.org/docs/2.5/kernel-dev/kernel-dev.html" target="_top">Yocto Project Linux Kernel Development Manual</a>@Yocto Project Linux Kernel Development Manual@g
-s@<a class="ulink" href="http://www.yoctoproject.org/docs/2.5/ref-manual/ref-manual.html" target="_top">Yocto Project Reference Manual</a>@Yocto Project Reference Manual@g
-s@<a class="ulink" href="http://www.yoctoproject.org/docs/2.5/toaster-manual/toaster-manual.html" target="_top">Toaster User Manual</a>@Toaster User Manual@g
+s@<a class="ulink" href="http://www.yoctoproject.org/docs/2.5.2/brief-yoctoprojectqs/brief-yoctoprojectqs.html" target="_top">Yocto Project Quick Build</a>@Yocto Project Quick Build@g
+s@<a class="ulink" href="http://www.yoctoproject.org/docs/2.5.2/yocto-project-qs/yocto-project-qs.html" target="_top">Yocto Project Quick Start</a>@Yocto Project Quick Start@g
+s@<a class="ulink" href="http://www.yoctoproject.org/docs/2.5.2/dev-manual/dev-manual.html" target="_top">Yocto Project Development Tasks Manual</a>@Yocto Project Development Tasks Manual@g
+s@<a class="ulink" href="http://www.yoctoproject.org/docs/2.5.2/overview-manual/overview-manual.html" target="_top">Yocto Project Overview and Concepts Manual</a>@Yocto project Overview and Concepts Manual@g
+s@<a class="ulink" href="http://www.yoctoproject.org/docs/2.5.2/sdk-manual/sdk-manual.html" target="_top">Yocto Project Application Development and the Extensible Software Development Kit (eSDK)</a>@Yocto Project Application Development and the Extensible Software Development Kit (eSDK)@g
+s@<a class="ulink" href="http://www.yoctoproject.org/docs/2.5.2/bsp-guide/bsp-guide.html" target="_top">Yocto Project Board Support Package (BSP) Developer's Guide</a>@Yocto Project Board Support Package (BSP) Developer's Guide@g
+s@<a class="ulink" href="http://www.yoctoproject.org/docs/2.5.2/profile-manual/profile-manual.html" target="_top">Yocto Project Profiling and Tracing Manual</a>@Yocto Project Profiling and Tracing Manual@g
+s@<a class="ulink" href="http://www.yoctoproject.org/docs/2.5.2/kernel-dev/kernel-dev.html" target="_top">Yocto Project Linux Kernel Development Manual</a>@Yocto Project Linux Kernel Development Manual@g
+s@<a class="ulink" href="http://www.yoctoproject.org/docs/2.5.2/ref-manual/ref-manual.html" target="_top">Yocto Project Reference Manual</a>@Yocto Project Reference Manual@g
+s@<a class="ulink" href="http://www.yoctoproject.org/docs/2.5.2/toaster-manual/toaster-manual.html" target="_top">Toaster User Manual</a>@Toaster User Manual@g
 
 # Process a single, rouge occurrence of a linked reference to the Mega-Manual.
-s@<a class="ulink" href="http://www.yoctoproject.org/docs/2.5/mega-manual/mega-manual.html" target="_top">Yocto Project Mega-Manual</a>@Yocto Project Mega-Manual@g
+s@<a class="ulink" href="http://www.yoctoproject.org/docs/2.5.2/mega-manual/mega-manual.html" target="_top">Yocto Project Mega-Manual</a>@Yocto Project Mega-Manual@g
 

meta-poky/conf/distro/poky.conf

@@ -1,6 +1,6 @@
 DISTRO = "poky"
 DISTRO_NAME = "Poky (Yocto Project Reference Distro)"
-DISTRO_VERSION = "2.5.1"
+DISTRO_VERSION = "2.5.3"
 DISTRO_CODENAME = "sumo"
 SDK_VENDOR = "-pokysdk"
 SDK_VERSION := "${@'${DISTRO_VERSION}'.replace('snapshot-${DATE}','snapshot')}"
@@ -68,14 +68,12 @@ SANITY_TESTED_DISTROS ?= " \
             poky-2.5 \n \
             ubuntu-15.04 \n \
             ubuntu-16.04 \n \
-            ubuntu-16.10 \n \
-            ubuntu-17.04 \n \
-            fedora-26 \n \
+            ubuntu-18.04 \n \
+            fedora-28 \n \
             centos-7 \n \
             debian-8 \n \
             debian-9 \n \
-            opensuse-42.1 \n \
-            opensuse-42.2 \n \
+            opensuse-42.3 \n \
             "
 #
 # OELAYOUT_ABI allows us to notify users when the format of TMPDIR changes in

meta-selftest/lib/oeqa/runtime/cases/dnf_runtime.py

@@ -27,7 +27,7 @@ class DnfSelftest(DnfTest):
         Expected: 1. Feeds were correctly set for dnf
                   2. Update recovers packages from host's repo
         Author: Humberto Ibarra <humberto.ibarra.lopez@intel.com>
-        Author: Alexander Kanavin <alexander.kanavin@intel.com>
+        Author: Alexander Kanavin <alex.kanavin@gmail.com>
         """
         # When we created an image, we had to supply fake ip and port
         # for the feeds. Now we can patch the real ones into the config file.

meta-selftest/recipes-test/delay/delay.bb

@@ -0,0 +1,12 @@
+SUMMARY = "Recipe with a fixed delay task"
+DESCRIPTION = "Contains a delay task to be used to for testing."
+LICENSE = "MIT"
+
+INHIBIT_DEFAULT_DEPS = "1"
+EXCLUDE_FROM_WORLD = "1"
+
+do_delay() {
+	sleep 5
+}
+do_delay[nostamp] = "1"
+addtask delay

meta-yocto-bsp/recipes-kernel/linux/linux-yocto_4.14.bbappend

@@ -8,8 +8,8 @@ KMACHINE_genericx86 ?= "common-pc"
 KMACHINE_genericx86-64 ?= "common-pc-64"
 KMACHINE_beaglebone-yocto ?= "beaglebone"
 
-SRCREV_machine_genericx86    ?= "2bf00ed1aee9c26b48c26b10c731c2f8970d50c2"
-SRCREV_machine_genericx86-64 ?= "2bf00ed1aee9c26b48c26b10c731c2f8970d50c2"
+SRCREV_machine_genericx86    ?= "084af9624d268ddf4fd65b2f9e8e50ca2f22e62b"
+SRCREV_machine_genericx86-64 ?= "084af9624d268ddf4fd65b2f9e8e50ca2f22e62b"
 SRCREV_machine_edgerouter ?= "65d1c849534179bbfa494f77947f8be615e9871a"
 SRCREV_machine_beaglebone-yocto ?= "d62ddfa26a59c83f6128790b7aa2baadc04d2960"
 SRCREV_machine_mpc8315e-rdb ?= "80a106ecf737b69864e07b955eccb62f809cf120"
@@ -20,8 +20,8 @@ COMPATIBLE_MACHINE_edgerouter = "edgerouter"
 COMPATIBLE_MACHINE_beaglebone-yocto = "beaglebone-yocto"
 COMPATIBLE_MACHINE_mpc8315e-rdb = "mpc8315e-rdb"
 
-LINUX_VERSION_genericx86 = "4.14.48"
-LINUX_VERSION_genericx86-64 = "4.14.48"
+LINUX_VERSION_genericx86 = "4.14.67"
+LINUX_VERSION_genericx86-64 = "4.14.67"
 LINUX_VERSION_edgerouter = "4.14.48"
 LINUX_VERSION_beaglebone-yocto = "4.14.48"
 LINUX_VERSION_mpc8315e-rdb = "4.14.48"

meta/classes/archiver.bbclass

@@ -40,8 +40,11 @@ ARCHIVER_MODE[recipe] ?= "0"
 DEPLOY_DIR_SRC ?= "${DEPLOY_DIR}/sources"
 ARCHIVER_TOPDIR ?= "${WORKDIR}/deploy-sources"
 ARCHIVER_OUTDIR = "${ARCHIVER_TOPDIR}/${TARGET_SYS}/${PF}/"
+ARCHIVER_RPMTOPDIR ?= "${WORKDIR}/deploy-sources-rpm"
+ARCHIVER_RPMOUTDIR = "${ARCHIVER_RPMTOPDIR}/${TARGET_SYS}/${PF}/"
 ARCHIVER_WORKDIR = "${WORKDIR}/archiver-work/"
 
+
 do_dumpdata[dirs] = "${ARCHIVER_OUTDIR}"
 do_ar_recipe[dirs] = "${ARCHIVER_OUTDIR}"
 do_ar_original[dirs] = "${ARCHIVER_OUTDIR} ${ARCHIVER_WORKDIR}"
@@ -117,6 +120,9 @@ python () {
     if d.getVarFlag('ARCHIVER_MODE', 'srpm') == "1" and d.getVar('PACKAGES'):
         if "package_rpm" in d.getVar('PACKAGE_CLASSES'):
             d.appendVarFlag('do_deploy_archives', 'depends', ' %s:do_package_write_rpm' % pn)
+            d.appendVarFlag('do_package_write_rpm', 'dirs', ' ${ARCHIVER_RPMTOPDIR}')
+            d.appendVarFlag('do_package_write_rpm', 'sstate-inputdirs', ' ${ARCHIVER_RPMTOPDIR}')
+            d.appendVarFlag('do_package_write_rpm', 'sstate-outputdirs', ' ${DEPLOY_DIR_SRC}')
             if ar_dumpdata == "1":
                 d.appendVarFlag('do_package_write_rpm', 'depends', ' %s:do_dumpdata' % pn)
             if ar_recipe == "1":
@@ -267,6 +273,14 @@ python do_ar_configured() {
         create_tarball(d, srcdir, 'configured', ar_outdir)
 }
 
+def exclude_useless_paths(tarinfo):
+    if tarinfo.isdir():
+        if tarinfo.name.endswith('/temp') or tarinfo.name.endswith('/patches') or tarinfo.name.endswith('/.pc'):
+            return None
+        elif tarinfo.name == 'temp' or tarinfo.name == 'patches' or tarinfo.name == '.pc':
+            return None
+    return tarinfo
+
 def create_tarball(d, srcdir, suffix, ar_outdir):
     """
     create the tarball from srcdir
@@ -291,7 +305,7 @@ def create_tarball(d, srcdir, suffix, ar_outdir):
 
     bb.note('Creating %s' % tarname)
     tar = tarfile.open(tarname, 'w:gz')
-    tar.add(srcdir, arcname=os.path.basename(srcdir))
+    tar.add(srcdir, arcname=os.path.basename(srcdir), filter=exclude_useless_paths)
     tar.close()
 
 # creating .diff.gz between source.orig and source

meta/classes/base.bbclass

@@ -100,8 +100,8 @@ def get_lic_checksum_file_list(d):
         # We only care about items that are absolute paths since
         # any others should be covered by SRC_URI.
         try:
-            path = bb.fetch.decodeurl(url)[2]
-            if not path:
+            (method, host, path, user, pswd, parm) = bb.fetch.decodeurl(url)
+            if method != "file" or not path:
                 raise bb.fetch.MalformedUrl(url)
 
             if path[0] == '/':
@@ -303,7 +303,9 @@ base_do_configure() {
 			if [ "${CLEANBROKEN}" != "1" -a \( -e Makefile -o -e makefile -o -e GNUmakefile \) ]; then
 				oe_runmake clean
 			fi
-			find ${B} -ignore_readdir_race -name \*.la -delete
+			# -ignore_readdir_race does not work correctly with -delete;
+			# use xargs to avoid spurious build failures
+			find ${B} -ignore_readdir_race -name \*.la -type f -print0 | xargs -0 rm -f
 		fi
 	fi
 	if [ -n "${CONFIGURESTAMPFILE}" ]; then

meta/classes/clutter.bbclass

@@ -14,7 +14,7 @@ REALNAME = "${@get_real_name("${BPN}")}"
 
 CLUTTER_SRC_FTP = "${GNOME_MIRROR}/${REALNAME}/${VERMINOR}/${REALNAME}-${PV}.tar.xz;name=archive"
 
-CLUTTER_SRC_GIT = "git://git.gnome.org/${REALNAME}"
+CLUTTER_SRC_GIT = "git://gitlab.gnome.org/GNOME/${REALNAME};protocol=https"
 
 SRC_URI = "${CLUTTER_SRC_FTP}"
 S = "${WORKDIR}/${REALNAME}-${PV}"

meta/classes/crosssdk.bbclass

@@ -21,10 +21,10 @@ TARGET_PREFIX = "${SDK_PREFIX}"
 TARGET_CC_ARCH = "${SDK_CC_ARCH}"
 TARGET_LD_ARCH = "${SDK_LD_ARCH}"
 TARGET_AS_ARCH = "${SDK_AS_ARCH}"
-TARGET_CPPFLAGS = "${BUILD_CPPFLAGS}"
-TARGET_CFLAGS = "${BUILD_CFLAGS}"
-TARGET_CXXFLAGS = "${BUILD_CXXFLAGS}"
-TARGET_LDFLAGS = "${BUILD_LDFLAGS}"
+TARGET_CPPFLAGS = ""
+TARGET_CFLAGS = ""
+TARGET_CXXFLAGS = ""
+TARGET_LDFLAGS = ""
 TARGET_FPU = ""
 
 

meta/classes/cve-check.bbclass

@@ -146,15 +146,17 @@ def get_patches_cves(d):
                 with open(patch_file, "r", encoding="iso8859-1") as f:
                     patch_text = f.read()
 
-        # Search for the "CVE: " line
-        match = cve_match.search(patch_text)
-        if match:
+        # Search for one or more "CVE: " lines
+        text_match = False
+        for match in cve_match.finditer(patch_text):
             # Get only the CVEs without the "CVE: " tag
             cves = patch_text[match.start()+5:match.end()]
             for cve in cves.split():
                 bb.debug(2, "Patch %s solves %s" % (patch_file, cve))
                 patched_cves.add(cve)
-        elif not fname_match:
+                text_match = True
+
+        if not fname_match and not text_match:
             bb.debug(2, "Patch %s doesn't solve CVEs" % patch_file)
 
     return patched_cves
@@ -177,7 +179,7 @@ def check_cves(d, patched_cves):
     cve_db_dir = d.getVar("CVE_CHECK_DB_DIR")
     cve_whitelist = ast.literal_eval(d.getVar("CVE_CHECK_CVE_WHITELIST"))
     cve_cmd = "cve-check-tool"
-    cmd = [cve_cmd, "--no-html", "--csv", "--not-affected", "-t", "faux", "-d", cve_db_dir]
+    cmd = [cve_cmd, "--no-html", "--skip-update", "--csv", "--not-affected", "-t", "faux", "-d", cve_db_dir]
 
     # If the recipe has been whitlisted we return empty lists
     if d.getVar("PN") in d.getVar("CVE_CHECK_PN_WHITELIST").split():
@@ -188,7 +190,8 @@ def check_cves(d, patched_cves):
         # Write the faux CSV file to be used with cve-check-tool
         fd, faux = tempfile.mkstemp(prefix="cve-faux-")
         with os.fdopen(fd, "w") as f:
-            f.write("%s,%s,%s," % (bpn, pv, cves))
+            for pn in bpn.split():
+                f.write("%s,%s,%s,\n" % (pn, pv, cves))
         cmd.append(faux)
 
         output = subprocess.check_output(cmd).decode("utf-8")

meta/classes/externalsrc.bbclass

@@ -41,8 +41,8 @@ python () {
     # re-parsed so that the file-checksums function for do_compile is run every
     # time.
     bpn = d.getVar('BPN')
-    if bpn == d.getVar('PN'):
-        classextend = (d.getVar('BBCLASSEXTEND') or '').split()
+    classextend = (d.getVar('BBCLASSEXTEND') or '').split()
+    if bpn == d.getVar('PN') or not classextend:
         if (externalsrc or
                 ('native' in classextend and
                  d.getVar('EXTERNALSRC_pn-%s-native' % bpn)) or

meta/classes/image-buildinfo.bbclass

@@ -64,6 +64,8 @@ def buildinfo_target(d):
 
 # Write build information to target filesystem
 python buildinfo () {
+    if not d.getVar('IMAGE_BUILDINFO_FILE'):
+        return
     with open(d.expand('${IMAGE_ROOTFS}${IMAGE_BUILDINFO_FILE}'), 'w') as build:
         build.writelines((
             '''-----------------------

meta/classes/image_types.bbclass

@@ -239,6 +239,7 @@ EXTRA_IMAGECMD_ext4 ?= "-i 4096"
 EXTRA_IMAGECMD_btrfs ?= "-n 4096"
 EXTRA_IMAGECMD_f2fs ?= ""
 
+do_image_cpio[depends] += "cpio-native:do_populate_sysroot"
 do_image_jffs2[depends] += "mtd-utils-native:do_populate_sysroot"
 do_image_cramfs[depends] += "util-linux-native:do_populate_sysroot"
 do_image_ext2[depends] += "e2fsprogs-native:do_populate_sysroot"

meta/classes/kernel-yocto.bbclass

@@ -275,6 +275,9 @@ addtask kernel_metadata after do_validate_branches do_unpack before do_patch
 do_kernel_metadata[depends] = "kern-tools-native:do_populate_sysroot"
 do_validate_branches[depends] = "kern-tools-native:do_populate_sysroot"
 
+do_kernel_configme[depends] += "virtual/${TARGET_PREFIX}binutils:do_populate_sysroot"
+do_kernel_configme[depends] += "virtual/${TARGET_PREFIX}gcc:do_populate_sysroot"
+do_kernel_configme[depends] += "bc-native:do_populate_sysroot bison-native:do_populate_sysroot"
 do_kernel_configme[dirs] += "${S} ${B}"
 do_kernel_configme() {
 	set +e
@@ -304,7 +307,7 @@ do_kernel_configme() {
 		bbfatal_log "Could not find configuration queue (${meta_dir}/config.queue)"
 	fi
 
-	CFLAGS="${CFLAGS} ${TOOLCHAIN_OPTIONS}"	ARCH=${ARCH} merge_config.sh -O ${B} ${config_flags} ${configs} > ${meta_dir}/cfg/merge_config_build.log 2>&1
+	CFLAGS="${CFLAGS} ${TOOLCHAIN_OPTIONS}"	HOSTCC="${BUILD_CC} ${BUILD_CFLAGS} ${BUILD_LDFLAGS}" HOSTCPP="${BUILD_CPP}" CC="${KERNEL_CC}" ARCH=${ARCH} merge_config.sh -O ${B} ${config_flags} ${configs} > ${meta_dir}/cfg/merge_config_build.log 2>&1
 	if [ $? -ne 0 ]; then
 		bbfatal_log "Could not configure ${KMACHINE}-${LINUX_KERNEL_TYPE}"
 	fi

meta/classes/kernel.bbclass

@@ -152,8 +152,8 @@ PACKAGES_DYNAMIC += "^${KERNEL_PACKAGE_NAME}-firmware-.*"
 export OS = "${TARGET_OS}"
 export CROSS_COMPILE = "${TARGET_PREFIX}"
 export KBUILD_BUILD_VERSION = "1"
-export KBUILD_BUILD_USER = "oe-user"
-export KBUILD_BUILD_HOST = "oe-host"
+export KBUILD_BUILD_USER ?= "oe-user"
+export KBUILD_BUILD_HOST ?= "oe-host"
 
 KERNEL_RELEASE ?= "${KERNEL_VERSION}"
 
@@ -489,7 +489,7 @@ sysroot_stage_all () {
 	:
 }
 
-KERNEL_CONFIG_COMMAND ?= "oe_runmake_call -C ${S} O=${B} oldnoconfig"
+KERNEL_CONFIG_COMMAND ?= "oe_runmake_call -C ${S} CC="${KERNEL_CC}" O=${B} oldnoconfig"
 
 python check_oldest_kernel() {
     oldest_kernel = d.getVar('OLDEST_KERNEL')

meta/classes/license.bbclass

@@ -480,7 +480,9 @@ def find_license_files(d):
 
     for url in lic_files.split():
         try:
-            (type, host, path, user, pswd, parm) = bb.fetch.decodeurl(url)
+            (method, host, path, user, pswd, parm) = bb.fetch.decodeurl(url)
+            if method != "file" or not path:
+                raise bb.fetch.MalformedUrl()
         except bb.fetch.MalformedUrl:
             bb.fatal("%s: LIC_FILES_CHKSUM contains an invalid URL:  %s" % (d.getVar('PF'), url))
         # We want the license filename and path

meta/classes/mirrors.bbclass

@@ -69,7 +69,7 @@ ${CPAN_MIRROR}  http://search.cpan.org/CPAN/ \n \
 
 MIRRORS += "\
 git://salsa.debian.org/.*     git://salsa.debian.org/PATH;protocol=https \n \
-git://git.gnome.org/.*        git://git.gnome.org/browse/PATH;protocol=https \n \
+git://git.gnome.org/.*        git://gitlab.gnome.org/GNOME/PATH;protocol=https \n \
 git://git.savannah.gnu.org/.* git://git.savannah.gnu.org/git/PATH;protocol=https \n \
 git://git.yoctoproject.org/.* git://git.yoctoproject.org/git/PATH;protocol=https \n \
 git://.*/.*                   git://HOST/PATH;protocol=https \n \

meta/classes/multilib_script.bbclass

@@ -0,0 +1,32 @@
+#
+# Recipe needs to set MULTILIB_SCRIPTS in the form <pkgname>:<scriptname>, e.g.
+# MULTILIB_SCRIPTS = "${PN}-dev:${bindir}/file1 ${PN}:${base_bindir}/file2"
+# to indicate which script files to process from which packages.
+#
+
+inherit update-alternatives
+
+MULTILIB_SUFFIX = "${@d.getVar('base_libdir',1).split('/')[-1]}"
+
+PACKAGE_PREPROCESS_FUNCS += "multilibscript_rename"
+
+multilibscript_rename() {
+	:
+}
+
+python () {
+    # Do nothing if multilib isn't being used
+    if not d.getVar("MULTILIB_VARIANTS"):
+       return
+    # Do nothing for native/cross
+    if bb.data.inherits_class('native', d) or bb.data.inherits_class('cross', d):
+       return
+
+    for entry in (d.getVar("MULTILIB_SCRIPTS", False) or "").split():
+       pkg, script = entry.split(":")
+       scriptname = os.path.basename(script)
+       d.setVar("ALTERNATIVE_" + pkg, scriptname)
+       d.setVarFlag("ALTERNATIVE_LINK_NAME", scriptname, script)
+       d.setVarFlag("ALTERNATIVE_TARGET", scriptname, script + "-${MULTILIB_SUFFIX}")
+       d.appendVar("multilibscript_rename",  "\n	mv ${PKGD}" + script + " ${PKGD}" + script + "-${MULTILIB_SUFFIX}")
+}

meta/classes/package.bbclass

@@ -359,7 +359,8 @@ def append_source_info(file, sourcefile, d, fatal=True):
     # of rpm's debugedit, which was writing them out that way, and the code elsewhere
     # is still assuming that.
     debuglistoutput = '\0'.join(debugsources) + '\0'
-    open(sourcefile, 'a').write(debuglistoutput)
+    with open(sourcefile, 'a') as sf:
+        sf.write(debuglistoutput)
 
 
 def splitdebuginfo(file, debugfile, debugsrcdir, sourcefile, d):
@@ -1411,9 +1412,6 @@ fi
     pkgdest = d.getVar('PKGDEST')
     pkgdatadir = d.getVar('PKGDESTWORK')
 
-    # Take shared lock since we're only reading, not writing
-    lf = bb.utils.lockfile(d.expand("${PACKAGELOCK}"), True)
-
     data_file = pkgdatadir + d.expand("/${PN}" )
     f = open(data_file, 'w')
     f.write("PACKAGES: %s\n" % packages)
@@ -1518,7 +1516,6 @@ fi
     if bb.data.inherits_class('allarch', d) and not bb.data.inherits_class('packagegroup', d):
         write_extra_runtime_pkgs(global_variants, packages, pkgdatadir)
 
-    bb.utils.unlockfile(lf)
 }
 emit_pkgdata[dirs] = "${PKGDESTWORK}/runtime ${PKGDESTWORK}/runtime-reverse ${PKGDESTWORK}/runtime-rprovides"
 
@@ -1617,9 +1614,6 @@ python package_do_shlibs() {
 
     shlibswork_dir = d.getVar('SHLIBSWORKDIR')
 
-    # Take shared lock since we're only reading, not writing
-    lf = bb.utils.lockfile(d.expand("${PACKAGELOCK}"), True)
-
     def linux_so(file, needed, sonames, renames, pkgver):
         needs_ldconfig = False
         ldir = os.path.dirname(file).replace(pkgdest + "/" + pkg, '')
@@ -1732,7 +1726,11 @@ python package_do_shlibs() {
     use_ldconfig = bb.utils.contains('DISTRO_FEATURES', 'ldconfig', True, False, d)
 
     needed = {}
+
+    # Take shared lock since we're only reading, not writing
+    lf = bb.utils.lockfile(d.expand("${PACKAGELOCK}"), True)
     shlib_provider = oe.package.read_shlib_providers(d)
+    bb.utils.unlockfile(lf)
 
     for pkg in packages.split():
         private_libs = d.getVar('PRIVATE_LIBS_' + pkg) or d.getVar('PRIVATE_LIBS') or ""
@@ -1788,8 +1786,6 @@ python package_do_shlibs() {
             d.setVar('pkg_postinst_%s' % pkg, postinst)
         bb.debug(1, 'LIBNAMES: pkg %s sonames %s' % (pkg, sonames))
 
-    bb.utils.unlockfile(lf)
-
     assumed_libs = d.getVar('ASSUME_SHLIBS')
     if assumed_libs:
         libdir = d.getVar("libdir")
@@ -1902,9 +1898,6 @@ python package_do_pkgconfig () {
                             if hdr == 'Requires':
                                 pkgconfig_needed[pkg] += exp.replace(',', ' ').split()
 
-    # Take shared lock since we're only reading, not writing
-    lf = bb.utils.lockfile(d.expand("${PACKAGELOCK}"), True)
-
     for pkg in packages.split():
         pkgs_file = os.path.join(shlibswork_dir, pkg + ".pclist")
         if pkgconfig_provided[pkg] != []:
@@ -1913,6 +1906,9 @@ python package_do_pkgconfig () {
                 f.write('%s\n' % p)
             f.close()
 
+    # Take shared lock since we're only reading, not writing
+    lf = bb.utils.lockfile(d.expand("${PACKAGELOCK}"), True)
+
     # Go from least to most specific since the last one found wins
     for dir in reversed(shlibs_dirs):
         if not os.path.exists(dir):
@@ -1928,6 +1924,8 @@ python package_do_pkgconfig () {
                 for l in lines:
                     pkgconfig_provided[pkg].append(l.rstrip())
 
+    bb.utils.unlockfile(lf)
+
     for pkg in packages.split():
         deps = []
         for n in pkgconfig_needed[pkg]:
@@ -1945,8 +1943,6 @@ python package_do_pkgconfig () {
             for dep in deps:
                 fd.write(dep + '\n')
             fd.close()
-
-    bb.utils.unlockfile(lf)
 }
 
 def read_libdep_files(d):

meta/classes/package_rpm.bbclass

@@ -113,6 +113,10 @@ python write_specfile () {
             source_list = os.listdir(ar_outdir)
             source_number = 0
             for source in source_list:
+                # do_deploy_archives may have already run (from sstate) meaning a .src.rpm may already 
+                # exist in ARCHIVER_OUTDIR so skip if present.
+                if source.endswith(".src.rpm"):
+                    continue
                 # The rpmbuild doesn't need the root permission, but it needs
                 # to know the file's user and group name, the only user and
                 # group in fakeroot is "root" when working in fakeroot.
@@ -690,7 +694,7 @@ python do_package_rpm () {
     cmd = cmd + " --define '_tmppath " + workdir + "'"
     if d.getVarFlag('ARCHIVER_MODE', 'srpm') == '1' and bb.data.inherits_class('archiver', d):
         cmd = cmd + " --define '_sourcedir " + d.getVar('ARCHIVER_OUTDIR') + "'"
-        cmdsrpm = cmd + " --define '_srcrpmdir " + d.getVar('ARCHIVER_OUTDIR') + "'"
+        cmdsrpm = cmd + " --define '_srcrpmdir " + d.getVar('ARCHIVER_RPMOUTDIR') + "'"
         cmdsrpm = cmdsrpm + " -bs " + outspecfile
         # Build the .src.rpm
         d.setVar('SBUILDSPEC', cmdsrpm + "\n")

meta/classes/patch.bbclass

@@ -153,6 +153,7 @@ python patch_do_patch() {
 patch_do_patch[vardepsexclude] = "PATCHRESOLVE"
 
 addtask patch after do_unpack
+do_patch[umask] = "022"
 do_patch[dirs] = "${WORKDIR}"
 do_patch[depends] = "${PATCHDEPENDENCY}"
 

meta/classes/populate_sdk_ext.bbclass

@@ -325,8 +325,9 @@ python copy_buildsystem () {
             f.write('TCLIBCAPPEND = ""\n')
             f.write('DL_DIR = "${TOPDIR}/downloads"\n')
 
-            f.write('INHERIT += "%s"\n' % 'uninative')
-            f.write('UNINATIVE_CHECKSUM[%s] = "%s"\n\n' % (d.getVar('BUILD_ARCH'), uninative_checksum))
+            if bb.data.inherits_class('uninative', d):
+               f.write('INHERIT += "%s"\n' % 'uninative')
+               f.write('UNINATIVE_CHECKSUM[%s] = "%s"\n\n' % (d.getVar('BUILD_ARCH'), uninative_checksum))
             f.write('CONF_VERSION = "%s"\n\n' % d.getVar('CONF_VERSION', False))
 
             # Some classes are not suitable for SDK, remove them from INHERIT

meta/classes/sanity.bbclass

@@ -511,6 +511,16 @@ def check_make_version(sanity_data):
     return None
 
 
+# Check if we're running on WSL (Windows Subsystem for Linux). Its known not to
+# work but we should tell the user that upfront.
+def check_wsl(d):
+    with open("/proc/version", "r") as f:
+        verdata = f.readlines()
+    for l in verdata:
+        if "Microsoft" in l:
+            return "OpenEmbedded doesn't work under WSL at this time, sorry"
+    return None
+
 # Tar version 1.24 and onwards handle overwriting symlinks correctly
 # but earlier versions do not; this needs to work properly for sstate
 def check_tar_version(sanity_data):
@@ -625,6 +635,7 @@ def check_sanity_version_change(status, d):
     status.addresult(check_tar_version(d))
     status.addresult(check_git_version(d))
     status.addresult(check_perl_modules(d))
+    status.addresult(check_wsl(d))
 
     missing = ""
 

meta/classes/testimage.bbclass

@@ -2,7 +2,7 @@
 #
 # Released under the MIT license (see COPYING.MIT)
 
-
+inherit metadata_scm
 # testimage.bbclass enables testing of qemu images using python unittests.
 # Most of the tests are commands run on target image over ssh.
 # To use it add testimage to global inherit and call your target image with -c testimage
@@ -26,6 +26,7 @@
 # TEST_LOG_DIR contains a command ssh log and may contain infromation about what command is running, output and return codes and for qemu a boot log till login.
 # Booting is handled by this class, and it's not a test in itself.
 # TEST_QEMUBOOT_TIMEOUT can be used to set the maximum time in seconds the launch code will wait for the login prompt.
+# TEST_QEMUPARAMS can be used to pass extra parameters to qemu, e.g. "-m 1024" for setting the amount of ram to 1 GB.
 
 TEST_LOG_DIR ?= "${WORKDIR}/testimage"
 
@@ -35,26 +36,13 @@ TEST_NEEDED_PACKAGES_DIR ?= "${WORKDIR}/testimage/packages"
 TEST_EXTRACTED_DIR ?= "${TEST_NEEDED_PACKAGES_DIR}/extracted"
 TEST_PACKAGED_DIR ?= "${TEST_NEEDED_PACKAGES_DIR}/packaged"
 
-RPMTESTSUITE = "${@bb.utils.contains('IMAGE_PKGTYPE', 'rpm', 'dnf rpm', '', d)}"
-SYSTEMDSUITE = "${@bb.utils.filter('DISTRO_FEATURES', 'systemd', d)}"
-MINTESTSUITE = "ping"
-NETTESTSUITE = "${MINTESTSUITE} ssh df date scp oe_syslog ${SYSTEMDSUITE}"
-DEVTESTSUITE = "gcc kernelmodule ldd"
+BASICTESTSUITE = "\
+    ping date df ssh scp python perl gi ptest parselogs \
+    logrotate connman systemd oe_syslog pam stap ldd xorg \
+    kernelmodule gcc buildcpio buildlzip buildgalculator \
+    dnf rpm opkg apt"
 
-DEFAULT_TEST_SUITES = "${MINTESTSUITE} auto"
-DEFAULT_TEST_SUITES_pn-core-image-minimal = "${MINTESTSUITE}"
-DEFAULT_TEST_SUITES_pn-core-image-minimal-dev = "${MINTESTSUITE}"
-DEFAULT_TEST_SUITES_pn-core-image-full-cmdline = "${NETTESTSUITE} perl python logrotate ptest"
-DEFAULT_TEST_SUITES_pn-core-image-x11 = "${MINTESTSUITE}"
-DEFAULT_TEST_SUITES_pn-core-image-lsb = "${NETTESTSUITE} pam parselogs ${RPMTESTSUITE} ptest"
-DEFAULT_TEST_SUITES_pn-core-image-sato = "${NETTESTSUITE} connman xorg parselogs ${RPMTESTSUITE} \
-    ${@bb.utils.contains('IMAGE_PKGTYPE', 'rpm', 'python', '', d)} ptest gi"
-DEFAULT_TEST_SUITES_pn-core-image-sato-sdk = "${NETTESTSUITE} buildcpio buildlzip buildgalculator \
-    connman ${DEVTESTSUITE} logrotate perl parselogs python ${RPMTESTSUITE} xorg ptest gi stap"
-DEFAULT_TEST_SUITES_pn-core-image-lsb-dev = "${NETTESTSUITE} pam perl python parselogs ${RPMTESTSUITE} ptest gi"
-DEFAULT_TEST_SUITES_pn-core-image-lsb-sdk = "${NETTESTSUITE} buildcpio buildlzip buildgalculator \
-    connman ${DEVTESTSUITE} logrotate pam parselogs perl python ${RPMTESTSUITE} ptest gi stap"
-DEFAULT_TEST_SUITES_pn-meta-toolchain = "auto"
+DEFAULT_TEST_SUITES = "${BASICTESTSUITE}"
 
 # aarch64 has no graphics
 DEFAULT_TEST_SUITES_remove_aarch64 = "xorg"
@@ -71,6 +59,7 @@ TEST_SUITES ?= "${DEFAULT_TEST_SUITES}"
 
 TEST_QEMUBOOT_TIMEOUT ?= "1000"
 TEST_TARGET ?= "qemu"
+TEST_QEMUPARAMS ?= ""
 
 TESTIMAGEDEPENDS = ""
 TESTIMAGEDEPENDS_qemuall = "qemu-native:do_populate_sysroot qemu-helper-native:do_populate_sysroot qemu-helper-native:do_addto_recipe_sysroot"
@@ -85,7 +74,7 @@ TESTIMAGEDEPENDS += "${@bb.utils.contains('IMAGE_PKGTYPE', 'rpm', 'createrepo-c-
 TESTIMAGELOCK = "${TMPDIR}/testimage.lock"
 TESTIMAGELOCK_qemuall = ""
 
-TESTIMAGE_DUMP_DIR ?= "/tmp/oe-saved-tests/"
+TESTIMAGE_DUMP_DIR ?= "${LOG_DIR}/runtime-hostdump/"
 
 TESTIMAGE_UPDATE_VARS ?= "DL_DIR WORKDIR DEPLOY_DIR"
 
@@ -132,6 +121,30 @@ def testimage_sanity(d):
         bb.fatal('When TEST_TARGET is set to "simpleremote" '
                  'TEST_TARGET_IP and TEST_SERVER_IP are needed too.')
 
+def get_testimage_configuration(d, test_type, machine):
+    import platform
+    from oeqa.utils.metadata import get_layers
+    configuration = {'TEST_TYPE': test_type,
+                    'MACHINE': machine,
+                    'DISTRO': d.getVar("DISTRO"),
+                    'IMAGE_BASENAME': d.getVar("IMAGE_BASENAME"),
+                    'IMAGE_PKGTYPE': d.getVar("IMAGE_PKGTYPE"),
+                    'STARTTIME': d.getVar("DATETIME"),
+                    'HOST_DISTRO': oe.lsb.distro_identifier().replace(' ', '-'),
+                    'LAYERS': get_layers(d.getVar("BBLAYERS"))}
+    return configuration
+get_testimage_configuration[vardepsexclude] = "DATETIME"
+
+def get_testimage_json_result_dir(d):
+    json_result_dir = os.path.join(d.getVar("LOG_DIR"), 'oeqa')
+    custom_json_result_dir = d.getVar("OEQA_JSON_RESULT_DIR")
+    if custom_json_result_dir:
+        json_result_dir = custom_json_result_dir
+    return json_result_dir
+
+def get_testimage_result_id(configuration):
+    return '%s_%s_%s_%s' % (configuration['TEST_TYPE'], configuration['IMAGE_BASENAME'], configuration['MACHINE'], configuration['STARTTIME'])
+
 def testimage_main(d):
     import os
     import json
@@ -185,12 +198,13 @@ def testimage_main(d):
     machine = d.getVar("MACHINE")
 
     # Get rootfs
-    fstypes = [fs for fs in d.getVar('IMAGE_FSTYPES').split(' ')
-                  if fs in supported_fstypes]
-    if not fstypes:
-        bb.fatal('Unsupported image type built. Add a comptible image to '
-                 'IMAGE_FSTYPES. Supported types: %s' %
-                 ', '.join(supported_fstypes))
+    fstypes = d.getVar('IMAGE_FSTYPES').split()
+    if d.getVar("TEST_TARGET") == "qemu":
+        fstypes = [fs for fs in fstypes if fs in supported_fstypes]
+        if not fstypes:
+            bb.fatal('Unsupported image type built. Add a comptible image to '
+                     'IMAGE_FSTYPES. Supported types: %s' %
+                     ', '.join(supported_fstypes))
     rootfs = '%s.%s' % (image_name, fstypes[0])
 
     # Get tmpdir (not really used, just for compatibility)
@@ -214,13 +228,11 @@ def testimage_main(d):
     boottime = int(d.getVar("TEST_QEMUBOOT_TIMEOUT"))
 
     # Get use_kvm
-    qemu_use_kvm = d.getVar("QEMU_USE_KVM")
-    if qemu_use_kvm and \
-       (oe.types.boolean(qemu_use_kvm) and 'x86' in machine or \
-        d.getVar('MACHINE') in qemu_use_kvm.split()):
-        kvm = True
-    else:
-        kvm = False
+    kvm = oe.types.qemu_use_kvm(d.getVar('QEMU_USE_KVM'), d.getVar('TARGET_ARCH'))
+
+    slirp = False
+    if d.getVar("QEMU_USE_SLIRP"):
+        slirp = True
 
     # TODO: We use the current implementatin of qemu runner because of
     # time constrains, qemu runner really needs a refactor too.
@@ -233,6 +245,8 @@ def testimage_main(d):
                       'boottime'    : boottime,
                       'bootlog'     : bootlog,
                       'kvm'         : kvm,
+                      'slirp'       : slirp,
+                      'dump_dir'    : d.getVar("TESTIMAGE_DUMP_DIR"),
                     }
 
     # TODO: Currently BBPATH is needed for custom loading of targets.
@@ -272,17 +286,12 @@ def testimage_main(d):
 
     package_extraction(d, tc.suites)
 
-    bootparams = None
-    if d.getVar('VIRTUAL-RUNTIME_init_manager', '') == 'systemd':
-        # Add systemd.log_level=debug to enable systemd debug logging
-        bootparams = 'systemd.log_target=console'
-
     results = None
     orig_sigterm_handler = signal.signal(signal.SIGTERM, sigterm_exception)
     try:
         # We need to check if runqemu ends unexpectedly
         # or if the worker send us a SIGTERM
-        tc.target.start(extra_bootparams=bootparams)
+        tc.target.start(params=d.getVar("TEST_QEMUPARAMS"))
         results = tc.runTests()
     except (RuntimeError, BlockingIOError) as err:
         if isinstance(err, RuntimeError):
@@ -299,7 +308,10 @@ def testimage_main(d):
     # Show results (if we have them)
     if not results:
         bb.fatal('%s - FAILED - tests were interrupted during execution' % pn, forcelog=True)
-    results.logDetails()
+    configuration = get_testimage_configuration(d, 'runtime', machine)
+    results.logDetails(get_testimage_json_result_dir(d),
+                       configuration,
+                       get_testimage_result_id(configuration))
     results.logSummary(pn)
     if not results.wasSuccessful():
         bb.fatal('%s - FAILED - check the task log and the ssh log' % pn, forcelog=True)
@@ -336,6 +348,7 @@ def create_index(arg):
     return None
 
 def create_rpm_index(d):
+    import glob
     # Index RPMs
     rpm_createrepo = bb.utils.which(os.getenv('PATH'), "createrepo_c")
     index_cmds = []
@@ -352,9 +365,13 @@ def create_rpm_index(d):
         lf = bb.utils.lockfile(lockfilename, False)
         oe.path.copyhardlinktree(rpm_dir, idx_path)
         # Full indexes overload a 256MB image so reduce the number of rpms
-        # in the feed. Filter to r* since we use the run-postinst packages and
-        # this leaves some allarch and machine arch packages too.
-        bb.utils.remove(idx_path + "*/[a-qs-z]*.rpm")
+        # in the feed by filtering to specific packages needed by the tests.
+        package_list = glob.glob(idx_path + "*/*.rpm")
+
+        for pkg in package_list:
+            if not os.path.basename(pkg).startswith(("rpm", "run-postinsts", "busybox", "bash", "update-alternatives", "libc6", "curl", "musl")):
+                bb.utils.remove(pkg)
+
         bb.utils.unlockfile(lf)
         cmd = '%s --update -q %s' % (rpm_createrepo, idx_path)
 

meta/classes/testsdk.bbclass

@@ -14,6 +14,30 @@
 #
 # where "<image-name>" is an image like core-image-sato.
 
+def get_sdk_configuration(d, test_type):
+    import platform
+    from oeqa.utils.metadata import get_layers
+    configuration = {'TEST_TYPE': test_type,
+                    'MACHINE': d.getVar("MACHINE"),
+                    'SDKMACHINE': d.getVar("SDKMACHINE"),
+                    'IMAGE_BASENAME': d.getVar("IMAGE_BASENAME"),
+                    'IMAGE_PKGTYPE': d.getVar("IMAGE_PKGTYPE"),
+                    'STARTTIME': d.getVar("DATETIME"),
+                    'HOST_DISTRO': oe.lsb.distro_identifier().replace(' ', '-'),
+                    'LAYERS': get_layers(d.getVar("BBLAYERS"))}
+    return configuration
+get_sdk_configuration[vardepsexclude] = "DATETIME"
+
+def get_sdk_json_result_dir(d):
+    json_result_dir = os.path.join(d.getVar("LOG_DIR"), 'oeqa')
+    custom_json_result_dir = d.getVar("OEQA_JSON_RESULT_DIR")
+    if custom_json_result_dir:
+        json_result_dir = custom_json_result_dir
+    return json_result_dir
+
+def get_sdk_result_id(configuration):
+    return '%s_%s_%s_%s_%s' % (configuration['TEST_TYPE'], configuration['IMAGE_BASENAME'], configuration['SDKMACHINE'], configuration['MACHINE'], configuration['STARTTIME'])
+
 def testsdk_main(d):
     import os
     import subprocess
@@ -24,8 +48,6 @@ def testsdk_main(d):
     from oeqa.sdk.context import OESDKTestContext, OESDKTestContextExecutor
     from oeqa.utils import make_logger_bitbake_compatible
 
-    bb.event.enable_threadlock()
-
     pn = d.getVar("PN")
     logger = make_logger_bitbake_compatible(logging.getLogger("BitBake"))
 
@@ -71,8 +93,10 @@ def testsdk_main(d):
 
         component = "%s %s" % (pn, OESDKTestContextExecutor.name)
         context_msg = "%s:%s" % (os.path.basename(tcname), os.path.basename(sdk_env))
-
-        result.logDetails()
+        configuration = get_sdk_configuration(d, 'sdk')
+        result.logDetails(get_sdk_json_result_dir(d),
+                          configuration,
+                          get_sdk_result_id(configuration))
         result.logSummary(component, context_msg)
 
         if not result.wasSuccessful():
@@ -99,8 +123,6 @@ def testsdkext_main(d):
     from oeqa.utils import avoid_paths_in_environ, make_logger_bitbake_compatible, subprocesstweak
     from oeqa.sdkext.context import OESDKExtTestContext, OESDKExtTestContextExecutor
 
-    bb.event.enable_threadlock()
-
     pn = d.getVar("PN")
     logger = make_logger_bitbake_compatible(logging.getLogger("BitBake"))
 
@@ -176,8 +198,10 @@ def testsdkext_main(d):
 
         component = "%s %s" % (pn, OESDKExtTestContextExecutor.name)
         context_msg = "%s:%s" % (os.path.basename(tcname), os.path.basename(sdk_env))
-
-        result.logDetails()
+        configuration = get_sdk_configuration(d, 'sdkext')
+        result.logDetails(get_sdk_json_result_dir(d),
+                          configuration,
+                          get_sdk_result_id(configuration))
         result.logSummary(component, context_msg)
 
         if not result.wasSuccessful():

meta/conf/bitbake.conf

@@ -624,7 +624,7 @@ BBLAYERS_FETCH_DIR ??= "${COREBASE}"
 APACHE_MIRROR = "http://archive.apache.org/dist"
 DEBIAN_MIRROR = "http://ftp.debian.org/debian/pool"
 GENTOO_MIRROR = "http://distfiles.gentoo.org/distfiles"
-GNOME_GIT = "git://git.gnome.org"
+GNOME_GIT = "git://gitlab.gnome.org/GNOME"
 GNOME_MIRROR = "http://ftp.gnome.org/pub/GNOME/sources"
 GNU_MIRROR = "http://ftp.gnu.org/gnu"
 GNUPG_MIRROR = "https://www.gnupg.org/ftp/gcrypt"

meta/conf/distro/include/default-versions.inc

@@ -2,6 +2,6 @@
 # Default preferred versions
 #
 
-PREFERRED_VERSION_openssl = "1.0.%"
-PREFERRED_VERSION_openssl-native = "1.0.%"
-PREFERRED_VERSION_nativesdk-openssl = "1.0.%"
+PREFERRED_VERSION_openssl ?= "1.0.%"
+PREFERRED_VERSION_openssl-native ?= "1.0.%"
+PREFERRED_VERSION_nativesdk-openssl ?= "1.0.%"

meta/conf/distro/include/security_flags.inc

@@ -6,7 +6,7 @@
 # in the DISTRO="poky-lsb" configuration.
 
 GCCPIE ?= "--enable-default-pie"
-GLIBCPIE ?= "--enable-static-pie"
+# If static PIE is known to work well, GLIBCPIE="--enable-static-pie" can be set
 
 # _FORTIFY_SOURCE requires -O1 or higher, so disable in debug builds as they use
 # -O0 which then results in a compiler warning.

meta/conf/distro/include/yocto-uninative.inc

@@ -6,9 +6,8 @@
 # to the distro running on the build machine.
 #
 
-UNINATIVE_MAXGLIBCVERSION = "2.27"
-
-UNINATIVE_URL ?= "http://downloads.yoctoproject.org/releases/uninative/2.2/"
-UNINATIVE_CHECKSUM[i686] ?= "036b60092fe4acfa1a321d110673030db20344a2d56f33a4d047f0279498bdad"
-UNINATIVE_CHECKSUM[x86_64] ?= "e3b77208169bf1ac4e89496f3cdbf27695f5b18a2694a908a793390f28b67f83"
+UNINATIVE_MAXGLIBCVERSION = "2.29"
 
+UNINATIVE_URL ?= "http://downloads.yoctoproject.org/releases/uninative/2.4/"
+UNINATIVE_CHECKSUM[i686] ?= "fafacfc537a6ce2bd122bd16c146881ab5ac69bd575abf6cb68a0dd33fa70ea2"
+UNINATIVE_CHECKSUM[x86_64] ?= "06f91685b782f2ccfedf3070b3ba0fe4a5ba2f0766dad5c9d1642dccf95accd0"

meta/files/common-licenses/FreeType

@@ -48,7 +48,7 @@ Introduction
   encourage you to use the following text:
 
    """  
-    Portions of this software are copyright <EFBFBD> <year> The FreeType
+    Portions of this software are copyright © <year> The FreeType
     Project (www.freetype.org).  All rights reserved.
    """
 
@@ -163,7 +163,7 @@ Legal Terms
 
   Our home page can be found at
 
-    http://www.freetype.org
+    https://www.freetype.org
 
 
 --- end of FTL.TXT ---

meta/lib/oe/rootfs.py

@@ -849,9 +849,8 @@ class OpkgRootfs(DpkgOpkgRootfs):
         opkg_pre_process_cmds = self.d.getVar('OPKG_PREPROCESS_COMMANDS')
         opkg_post_process_cmds = self.d.getVar('OPKG_POSTPROCESS_COMMANDS')
 
-        # update PM index files, unless users provide their own feeds
-        if (self.d.getVar('BUILD_IMAGES_FROM_FEEDS') or "") != "1":
-            self.pm.write_index()
+        # update PM index files
+        self.pm.write_index()
 
         execute_pre_post_process(self.d, opkg_pre_process_cmds)
 

meta/lib/oe/types.py

@@ -151,3 +151,27 @@ def path(value, relativeto='', normalize='true', mustexist='false'):
                 raise ValueError("{0}: {1}".format(value, os.strerror(errno.ENOENT)))
 
     return value
+
+def is_x86(arch):
+    """
+    Check whether arch is x86 or x86_64
+    """
+    if arch.startswith('x86_') or re.match('i.*86', arch):
+        return True
+    else:
+        return False
+
+def qemu_use_kvm(kvm, target_arch):
+    """
+    Enable kvm if target_arch == build_arch or both of them are x86 archs.
+    """
+
+    use_kvm = False
+    if kvm and boolean(kvm):
+        build_arch = os.uname()[4]
+        if is_x86(build_arch) and is_x86(target_arch):
+            use_kvm = True
+        elif build_arch == target_arch:
+            use_kvm = True
+    return use_kvm
+

meta/lib/oeqa/core/context.py

@@ -27,7 +27,6 @@ class OETestContext(object):
         self.logger = logger
         self._registry = {}
         self._registry['cases'] = collections.OrderedDict()
-        self._results = {}
 
     def _read_modules_from_manifest(self, manifest):
         if not os.path.exists(manifest):

meta/lib/oeqa/core/decorator/depends.py

@@ -3,7 +3,6 @@
 
 from unittest import SkipTest
 
-from oeqa.core.threaded import OETestRunnerThreaded
 from oeqa.core.exception import OEQADependency
 
 from . import OETestDiscover, registerDecorator
@@ -64,16 +63,10 @@ def _order_test_case_by_depends(cases, depends):
     return [cases[case_id] for case_id in cases_ordered]
 
 def _skipTestDependency(case, depends):
-    if isinstance(case.tc.runner, OETestRunnerThreaded):
-        import threading
-        results = case.tc._results[threading.get_ident()]
-    else:
-        results = case.tc._results
-
     skipReasons = ['errors', 'failures', 'skipped']
 
     for reason in skipReasons:
-        for test, _ in results[reason]:
+        for test, _ in getattr(case.tc.results, reason):
             if test.id() in depends:
                 raise SkipTest("Test case %s depends on %s and was in %s." \
                         % (case.id(), test.id(), reason))

meta/lib/oeqa/core/decorator/oetimeout.py

@@ -1,12 +1,8 @@
 # Copyright (C) 2016 Intel Corporation
 # Released under the MIT license (see COPYING.MIT)
 
-from . import OETestDecorator, registerDecorator
-
 import signal
-from threading import Timer
-
-from oeqa.core.threaded import OETestRunnerThreaded
+from . import OETestDecorator, registerDecorator
 from oeqa.core.exception import OEQATimeoutError
 
 @registerDecorator
@@ -14,32 +10,16 @@ class OETimeout(OETestDecorator):
     attrs = ('oetimeout',)
 
     def setUpDecorator(self):
-        self.logger.debug("Setting up a %d second(s) timeout" % self.oetimeout)
-
-        if isinstance(self.case.tc.runner, OETestRunnerThreaded):
-            self.timeouted = False
-            def _timeoutHandler():
-                self.timeouted = True
-
-            self.timer = Timer(self.oetimeout, _timeoutHandler)
-            self.timer.start()
-        else:
-            timeout = self.oetimeout
-            def _timeoutHandler(signum, frame):
-                raise OEQATimeoutError("Timed out after %s "
+        timeout = self.oetimeout
+        def _timeoutHandler(signum, frame):
+            raise OEQATimeoutError("Timed out after %s "
                     "seconds of execution" % timeout)
 
-            self.alarmSignal = signal.signal(signal.SIGALRM, _timeoutHandler)
-            signal.alarm(self.oetimeout)
+        self.logger.debug("Setting up a %d second(s) timeout" % self.oetimeout)
+        self.alarmSignal = signal.signal(signal.SIGALRM, _timeoutHandler)
+        signal.alarm(self.oetimeout)
 
     def tearDownDecorator(self):
-        if isinstance(self.case.tc.runner, OETestRunnerThreaded):
-            self.timer.cancel()
-            self.logger.debug("Removed Timer handler")
-            if self.timeouted:
-                raise OEQATimeoutError("Timed out after %s "
-                    "seconds of execution" % self.oetimeout)
-        else:
-            signal.alarm(0)
-            signal.signal(signal.SIGALRM, self.alarmSignal)
-            self.logger.debug("Removed SIGALRM handler")
+        signal.alarm(0)
+        signal.signal(signal.SIGALRM, self.alarmSignal)
+        self.logger.debug("Removed SIGALRM handler")

meta/lib/oeqa/core/loader.py

@@ -24,7 +24,7 @@ from oeqa.core.decorator import decoratorClasses, OETestDecorator, \
 # Generate the function definition because this differ across python versions
 # Python >= 3.4.4 uses tree parameters instead four but for example Python 3.5.3
 # ueses four parameters so isn't incremental.
-_failed_test_args = inspect.getargspec(unittest.loader._make_failed_test).args
+_failed_test_args = inspect.getfullargspec(unittest.loader._make_failed_test).args
 exec("""def _make_failed_test(%s): raise exception""" % ', '.join(_failed_test_args))
 unittest.loader._make_failed_test = _make_failed_test
 

meta/lib/oeqa/core/runner.py

@@ -6,17 +6,10 @@ import time
 import unittest
 import logging
 import re
+import json
 
-xmlEnabled = False
-try:
-    import xmlrunner
-    from xmlrunner.result import _XMLTestResult as _TestResult
-    from xmlrunner.runner import XMLTestRunner as _TestRunner
-    xmlEnabled = True
-except ImportError:
-    # use the base runner instead
-    from unittest import TextTestResult as _TestResult
-    from unittest import TextTestRunner as _TestRunner
+from unittest import TextTestResult as _TestResult
+from unittest import TextTestRunner as _TestRunner
 
 class OEStreamLogger(object):
     def __init__(self, logger):
@@ -42,8 +35,12 @@ class OETestResult(_TestResult):
     def __init__(self, tc, *args, **kwargs):
         super(OETestResult, self).__init__(*args, **kwargs)
 
+        self.successes = []
+
+        # Inject into tc so that TestDepends decorator can see results
+        tc.results = self
+
         self.tc = tc
-        self._tc_map_results()
 
     def startTest(self, test):
         # Allow us to trigger the testcase buffer mode on a per test basis
@@ -53,12 +50,6 @@ class OETestResult(_TestResult):
             self.buffer = test.buffer
         super(OETestResult, self).startTest(test)
 
-    def _tc_map_results(self):
-        self.tc._results['failures'] = self.failures
-        self.tc._results['errors'] = self.errors
-        self.tc._results['skipped'] = self.skipped
-        self.tc._results['expectedFailures'] = self.expectedFailures
-
     def logSummary(self, component, context_msg=''):
         elapsed_time = self.tc._run_end_time - self.tc._run_start_time
         self.tc.logger.info("SUMMARY:")
@@ -70,67 +61,60 @@ class OETestResult(_TestResult):
             msg = "%s - OK - All required tests passed" % component
         else:
             msg = "%s - FAIL - Required tests failed" % component
-        skipped = len(self.tc._results['skipped'])
-        if skipped: 
-            msg += " (skipped=%d)" % skipped
+        msg += " (successes=%d, skipped=%d, failures=%d, errors=%d)" % (len(self.successes), len(self.skipped), len(self.failures), len(self.errors))
         self.tc.logger.info(msg)
 
-    def _getDetailsNotPassed(self, case, type, desc):
-        found = False
+    def _getTestResultDetails(self, case):
+        result_types = {'failures': 'FAILED', 'errors': 'ERROR', 'skipped': 'SKIPPED',
+                        'expectedFailures': 'EXPECTEDFAIL', 'successes': 'PASSED'}
 
-        for (scase, msg) in self.tc._results[type]:
-            # XXX: When XML reporting is enabled scase is
-            # xmlrunner.result._TestInfo instance instead of
-            # string.
-            if xmlEnabled:
-                if case.id() == scase.test_id:
+        for rtype in result_types:
+            found = False
+            for (scase, msg) in getattr(self, rtype):
+                if case.id() == scase.id():
                     found = True
                     break
-                scase_str = scase.test_id
-            else:
-                if case == scase:
-                    found = True
-                    break
-                scase_str = str(scase)
+                scase_str = str(scase.id())
 
-            # When fails at module or class level the class name is passed as string
-            # so figure out to see if match
-            m = re.search("^setUpModule \((?P<module_name>.*)\)$", scase_str)
-            if m:
-                if case.__class__.__module__ == m.group('module_name'):
-                    found = True
-                    break
+                # When fails at module or class level the class name is passed as string
+                # so figure out to see if match
+                m = re.search("^setUpModule \((?P<module_name>.*)\)$", scase_str)
+                if m:
+                    if case.__class__.__module__ == m.group('module_name'):
+                        found = True
+                        break
 
-            m = re.search("^setUpClass \((?P<class_name>.*)\)$", scase_str)
-            if m:
-                class_name = "%s.%s" % (case.__class__.__module__,
-                        case.__class__.__name__)
+                m = re.search("^setUpClass \((?P<class_name>.*)\)$", scase_str)
+                if m:
+                    class_name = "%s.%s" % (case.__class__.__module__,
+                                            case.__class__.__name__)
 
-                if class_name == m.group('class_name'):
-                    found = True
-                    break
+                    if class_name == m.group('class_name'):
+                        found = True
+                        break
 
-        if found:
-            return (found, msg)
+            if found:
+                return result_types[rtype], msg
 
-        return (found, None)
+        return 'UNKNOWN', None
 
-    def logDetails(self):
+    def addSuccess(self, test):
+        #Added so we can keep track of successes too
+        self.successes.append((test, None))
+        super(OETestResult, self).addSuccess(test)
+
+    def logDetails(self, json_file_dir=None, configuration=None, result_id=None):
         self.tc.logger.info("RESULTS:")
+
+        result = {}
+        logs = {}
+        if hasattr(self.tc, "extraresults"):
+            result = self.tc.extraresults
+
         for case_name in self.tc._registry['cases']:
             case = self.tc._registry['cases'][case_name]
 
-            result_types = ['failures', 'errors', 'skipped', 'expectedFailures']
-            result_desc = ['FAILED', 'ERROR', 'SKIPPED', 'EXPECTEDFAIL']
-
-            fail = False
-            desc = None
-            for idx, name in enumerate(result_types):
-                (fail, msg) = self._getDetailsNotPassed(case, result_types[idx],
-                        result_desc[idx])
-                if fail:
-                    desc = result_desc[idx]
-                    break
+            (status, log) = self._getTestResultDetails(case)
 
             oeid = -1
             if hasattr(case, 'decorators'):
@@ -138,12 +122,27 @@ class OETestResult(_TestResult):
                     if hasattr(d, 'oeid'):
                         oeid = d.oeid
 
-            if fail:
-                self.tc.logger.info("RESULTS - %s - Testcase %s: %s" % (case.id(),
-                    oeid, desc))
+            if status not in logs:
+                logs[status] = []
+            logs[status].append("RESULTS - %s - Testcase %s: %s" % (case.id(), oeid, status))
+            if log:
+                result[case.id()] = {'status': status, 'log': log}
             else:
-                self.tc.logger.info("RESULTS - %s - Testcase %s: %s" % (case.id(),
-                    oeid, 'PASSED'))
+                result[case.id()] = {'status': status}
+
+        for i in ['PASSED', 'SKIPPED', 'EXPECTEDFAIL', 'ERROR', 'FAILED', 'UNKNOWN']:
+            if i not in logs:
+                continue
+            for l in logs[i]:
+                self.tc.logger.info(l)
+
+        if json_file_dir:
+            tresultjsonhelper = OETestResultJSONHelper()
+            tresultjsonhelper.dump_testresult_file(json_file_dir, configuration, result_id, result)
+
+    def wasSuccessful(self):
+        # Override as we unexpected successes aren't failures for us
+        return (len(self.failures) == len(self.errors) == 0)
 
 class OEListTestsResult(object):
     def wasSuccessful(self):
@@ -153,33 +152,14 @@ class OETestRunner(_TestRunner):
     streamLoggerClass = OEStreamLogger
 
     def __init__(self, tc, *args, **kwargs):
-        if xmlEnabled:
-            if not kwargs.get('output'):
-                kwargs['output'] = os.path.join(os.getcwd(),
-                        'TestResults_%s_%s' % (time.strftime("%Y%m%d%H%M%S"), os.getpid()))
-
         kwargs['stream'] = self.streamLoggerClass(tc.logger)
         super(OETestRunner, self).__init__(*args, **kwargs)
         self.tc = tc
         self.resultclass = OETestResult
 
-    # XXX: The unittest-xml-reporting package defines _make_result method instead
-    # of _makeResult standard on unittest.
-    if xmlEnabled:
-        def _make_result(self):
-            """
-            Creates a TestResult object which will be used to store
-            information about the executed tests.
-            """
-            # override in subclasses if necessary.
-            return self.resultclass(self.tc,
-                self.stream, self.descriptions, self.verbosity, self.elapsed_times
-            )
-    else:
-        def _makeResult(self):
-            return self.resultclass(self.tc, self.stream, self.descriptions,
-                    self.verbosity)
-
+    def _makeResult(self):
+        return self.resultclass(self.tc, self.stream, self.descriptions,
+                self.verbosity)
 
     def _walk_suite(self, suite, func):
         for obj in suite:
@@ -275,3 +255,29 @@ class OETestRunner(_TestRunner):
             self._list_tests_module(suite)
 
         return OEListTestsResult()
+
+class OETestResultJSONHelper(object):
+
+    testresult_filename = 'testresults.json'
+
+    def _get_existing_testresults_if_available(self, write_dir):
+        testresults = {}
+        file = os.path.join(write_dir, self.testresult_filename)
+        if os.path.exists(file):
+            with open(file, "r") as f:
+                testresults = json.load(f)
+        return testresults
+
+    def _write_file(self, write_dir, file_name, file_content):
+        file_path = os.path.join(write_dir, file_name)
+        with open(file_path, 'w') as the_file:
+            the_file.write(file_content)
+
+    def dump_testresult_file(self, write_dir, configuration, result_id, test_result):
+        bb.utils.mkdirhier(write_dir)
+        lf = bb.utils.lockfile(os.path.join(write_dir, 'jsontestresult.lock'))
+        test_results = self._get_existing_testresults_if_available(write_dir)
+        test_results[result_id] = {'configuration': configuration, 'result': test_result}
+        json_testresults = json.dumps(test_results, sort_keys=True, indent=4)
+        self._write_file(write_dir, self.testresult_filename, json_testresults)
+        bb.utils.unlockfile(lf)

meta/lib/oeqa/core/target/qemu.py

@@ -12,15 +12,14 @@ from oeqa.utils.qemurunner import QemuRunner
 supported_fstypes = ['ext3', 'ext4', 'cpio.gz', 'wic']
 
 class OEQemuTarget(OESSHTarget):
-    def __init__(self, logger, ip, server_ip, timeout=300, user='root',
+    def __init__(self, logger, server_ip, timeout=300, user='root',
             port=None, machine='', rootfs='', kernel='', kvm=False,
             dump_dir='', dump_host_cmds='', display='', bootlog='',
             tmpdir='', dir_image='', boottime=60, **kwargs):
 
-        super(OEQemuTarget, self).__init__(logger, ip, server_ip, timeout,
+        super(OEQemuTarget, self).__init__(logger, None, server_ip, timeout,
                 user, port)
 
-        self.ip = ip
         self.server_ip = server_ip
         self.machine = machine
         self.rootfs = rootfs

meta/lib/oeqa/core/tests/cases/loader/threaded/threaded.py

@@ -1,12 +0,0 @@
-# Copyright (C) 2017 Intel Corporation
-# Released under the MIT license (see COPYING.MIT)
-
-from oeqa.core.case import OETestCase
-
-class ThreadedTest(OETestCase):
-    def test_threaded_no_depends(self):
-        self.assertTrue(True, msg='How is this possible?')
-
-class ThreadedTest2(OETestCase):
-    def test_threaded_same_module(self):
-        self.assertTrue(True, msg='How is this possible?')

meta/lib/oeqa/core/tests/cases/loader/threaded/threaded_alone.py

@@ -1,8 +0,0 @@
-# Copyright (C) 2017 Intel Corporation
-# Released under the MIT license (see COPYING.MIT)
-
-from oeqa.core.case import OETestCase
-
-class ThreadedTestAlone(OETestCase):
-    def test_threaded_alone(self):
-        self.assertTrue(True, msg='How is this possible?')

meta/lib/oeqa/core/tests/cases/loader/threaded/threaded_depends.py

@@ -1,10 +0,0 @@
-# Copyright (C) 2017 Intel Corporation
-# Released under the MIT license (see COPYING.MIT)
-
-from oeqa.core.case import OETestCase
-from oeqa.core.decorator.depends import OETestDepends
-
-class ThreadedTest3(OETestCase):
-    @OETestDepends(['threaded.ThreadedTest.test_threaded_no_depends'])
-    def test_threaded_depends(self):
-        self.assertTrue(True, msg='How is this possible?')

meta/lib/oeqa/core/tests/cases/loader/threaded/threaded_module.py

@@ -1,12 +0,0 @@
-# Copyright (C) 2017 Intel Corporation
-# Released under the MIT license (see COPYING.MIT)
-
-from oeqa.core.case import OETestCase
-
-class ThreadedTestModule(OETestCase):
-    def test_threaded_module(self):
-        self.assertTrue(True, msg='How is this possible?')
-
-class ThreadedTestModule2(OETestCase):
-    def test_threaded_module2(self):
-        self.assertTrue(True, msg='How is this possible?')

meta/lib/oeqa/core/tests/common.py

@@ -33,13 +33,3 @@ class TestBase(unittest.TestCase):
         tc.loadTests(self.cases_path, modules=modules, tests=tests,
                      filters=filters)
         return tc
-
-    def _testLoaderThreaded(self, d={}, modules=[],
-            tests=[], filters={}):
-        from oeqa.core.threaded import OETestContextThreaded
-
-        tc = OETestContextThreaded(d, self.logger)
-        tc.loadTests(self.cases_path, modules=modules, tests=tests,
-                     filters=filters)
-
-        return tc

meta/lib/oeqa/core/tests/test_data.py

@@ -21,7 +21,7 @@ class TestData(TestBase):
 
         tc = self._testLoader(modules=self.modules)
         self.assertEqual(False, tc.runTests().wasSuccessful())
-        for test, data in tc._results['errors']:
+        for test, data in tc.errors:
             expect = False
             if expectedException in data:
                 expect = True
@@ -34,7 +34,7 @@ class TestData(TestBase):
 
         tc = self._testLoader(d=d, modules=self.modules)
         self.assertEqual(False, tc.runTests().wasSuccessful())
-        for test, data in tc._results['failures']:
+        for test, data in tc.failures:
             expect = False
             if expectedError in data:
                 expect = True

meta/lib/oeqa/core/tests/test_decorators.py

@@ -131,17 +131,5 @@ class TestTimeoutDecorator(TestBase):
         msg = "OETestTimeout didn't restore SIGALRM"
         self.assertIs(alarm_signal, signal.getsignal(signal.SIGALRM), msg=msg)
 
-    def test_timeout_thread(self):
-        tests = ['timeout.TimeoutTest.testTimeoutPass']
-        msg = 'Failed to run test using OETestTimeout'
-        tc = self._testLoaderThreaded(modules=self.modules, tests=tests)
-        self.assertTrue(tc.runTests().wasSuccessful(), msg=msg)
-
-    def test_timeout_threaded_fail(self):
-        tests = ['timeout.TimeoutTest.testTimeoutFail']
-        msg = "OETestTimeout test didn't timeout as expected"
-        tc = self._testLoaderThreaded(modules=self.modules, tests=tests)
-        self.assertFalse(tc.runTests().wasSuccessful(), msg=msg)
-
 if __name__ == '__main__':
     unittest.main()

meta/lib/oeqa/core/tests/test_loader.py

@@ -1,6 +1,6 @@
 #!/usr/bin/env python3
 
-# Copyright (C) 2016-2017 Intel Corporation
+# Copyright (C) 2016 Intel Corporation
 # Released under the MIT license (see COPYING.MIT)
 
 import os
@@ -82,33 +82,5 @@ class TestLoader(TestBase):
         msg = 'Expected modules from two different paths'
         self.assertEqual(modules, expected_modules, msg=msg)
 
-    def test_loader_threaded(self):
-        cases_path = self.cases_path
-
-        self.cases_path = [os.path.join(self.cases_path, 'loader', 'threaded')]
-
-        tc = self._testLoaderThreaded()
-        self.assertEqual(len(tc.suites), 3, "Expected to be 3 suites")
-
-        case_ids = ['threaded.ThreadedTest.test_threaded_no_depends',
-                'threaded.ThreadedTest2.test_threaded_same_module',
-                'threaded_depends.ThreadedTest3.test_threaded_depends']
-        for case in tc.suites[0]._tests:
-            self.assertEqual(case.id(),
-                    case_ids[tc.suites[0]._tests.index(case)])
-
-        case_ids = ['threaded_alone.ThreadedTestAlone.test_threaded_alone']
-        for case in tc.suites[1]._tests:
-            self.assertEqual(case.id(),
-                    case_ids[tc.suites[1]._tests.index(case)])
-
-        case_ids = ['threaded_module.ThreadedTestModule.test_threaded_module',
-                'threaded_module.ThreadedTestModule2.test_threaded_module2']
-        for case in tc.suites[2]._tests:
-            self.assertEqual(case.id(),
-                    case_ids[tc.suites[2]._tests.index(case)])
-
-        self.cases_path = cases_path
-
 if __name__ == '__main__':
     unittest.main()

meta/lib/oeqa/core/threaded.py

@@ -1,275 +0,0 @@
-# Copyright (C) 2017 Intel Corporation
-# Released under the MIT license (see COPYING.MIT)
-
-import threading
-import multiprocessing
-import queue
-import time
-
-from unittest.suite import TestSuite
-
-from oeqa.core.loader import OETestLoader
-from oeqa.core.runner import OEStreamLogger, OETestResult, OETestRunner
-from oeqa.core.context import OETestContext
-
-class OETestLoaderThreaded(OETestLoader):
-    def __init__(self, tc, module_paths, modules, tests, modules_required,
-            filters, process_num=0, *args, **kwargs):
-        super(OETestLoaderThreaded, self).__init__(tc, module_paths, modules,
-                tests, modules_required, filters, *args, **kwargs)
-
-        self.process_num = process_num
-
-    def discover(self):
-        suite = super(OETestLoaderThreaded, self).discover()
-
-        if self.process_num <= 0:
-            self.process_num = min(multiprocessing.cpu_count(),
-                    len(suite._tests))
-
-        suites = []
-        for _ in range(self.process_num):
-            suites.append(self.suiteClass())
-
-        def _search_for_module_idx(suites, case):
-            """
-                Cases in the same module needs to be run
-                in the same thread because PyUnit keeps track
-                of setUp{Module, Class,} and tearDown{Module, Class,}.
-            """
-
-            for idx in range(self.process_num):
-                suite = suites[idx]
-                for c in suite._tests:
-                    if case.__module__ == c.__module__:
-                        return idx
-
-            return -1
-
-        def _search_for_depend_idx(suites, depends):
-            """
-                Dependency cases needs to be run in the same
-                thread, because OEQA framework look at the state
-                of dependant test to figure out if skip or not.
-            """
-
-            for idx in range(self.process_num):
-                suite = suites[idx]
-
-                for case in suite._tests:
-                    if case.id() in depends:
-                        return idx
-            return -1
-
-        def _get_best_idx(suites):
-            sizes = [len(suite._tests) for suite in suites]
-            return sizes.index(min(sizes))
-
-        def _fill_suites(suite):
-            idx = -1
-            for case in suite:
-                if isinstance(case, TestSuite):
-                    _fill_suites(case)
-                else:
-                    idx = _search_for_module_idx(suites, case)
-
-                    depends = {}
-                    if 'depends' in self.tc._registry:
-                        depends = self.tc._registry['depends']
-
-                    if idx == -1 and case.id() in depends:
-                        case_depends = depends[case.id()] 
-                        idx = _search_for_depend_idx(suites, case_depends)
-
-                    if idx == -1:
-                        idx = _get_best_idx(suites)
-
-                    suites[idx].addTest(case)
-        _fill_suites(suite)
-
-        suites_tmp = suites
-        suites = []
-        for suite in suites_tmp:
-            if len(suite._tests) > 0:
-                suites.append(suite)
-
-        return suites
-
-class OEStreamLoggerThreaded(OEStreamLogger):
-    _lock = threading.Lock()
-    buffers = {}
-
-    def write(self, msg):
-        tid = threading.get_ident()
-
-        if not tid in self.buffers:
-            self.buffers[tid] = ""
-
-        if msg:
-            self.buffers[tid] += msg
-
-    def finish(self):
-        tid = threading.get_ident()
-        
-        self._lock.acquire()
-        self.logger.info('THREAD: %d' % tid)
-        self.logger.info('-' * 70)
-        for line in self.buffers[tid].split('\n'):
-            self.logger.info(line)
-        self._lock.release()
-
-class OETestResultThreadedInternal(OETestResult):
-    def _tc_map_results(self):
-        tid = threading.get_ident()
-        
-        # PyUnit generates a result for every test module run, test
-        # if the thread already has an entry to avoid lose the previous
-        # test module results.
-        if not tid in self.tc._results:
-            self.tc._results[tid] = {}
-            self.tc._results[tid]['failures'] = self.failures
-            self.tc._results[tid]['errors'] = self.errors
-            self.tc._results[tid]['skipped'] = self.skipped
-            self.tc._results[tid]['expectedFailures'] = self.expectedFailures
-
-class OETestResultThreaded(object):
-    _results = {}
-    _lock = threading.Lock()
-
-    def __init__(self, tc):
-        self.tc = tc
-
-    def _fill_tc_results(self):
-        tids = list(self.tc._results.keys())
-        fields = ['failures', 'errors', 'skipped', 'expectedFailures']
-
-        for tid in tids:
-            result = self.tc._results[tid]
-            for field in fields:
-                if not field in self.tc._results:
-                    self.tc._results[field] = []
-                self.tc._results[field].extend(result[field])
-
-    def addResult(self, result, run_start_time, run_end_time):
-        tid = threading.get_ident()
-
-        self._lock.acquire()
-        self._results[tid] = {}
-        self._results[tid]['result'] = result
-        self._results[tid]['run_start_time'] = run_start_time 
-        self._results[tid]['run_end_time'] = run_end_time 
-        self._results[tid]['result'] = result
-        self._lock.release()
-
-    def wasSuccessful(self):
-        wasSuccessful = True
-        for tid in self._results.keys():
-            wasSuccessful = wasSuccessful and \
-                    self._results[tid]['result'].wasSuccessful()
-        return wasSuccessful
-
-    def stop(self):
-        for tid in self._results.keys():
-            self._results[tid]['result'].stop()
-
-    def logSummary(self, component, context_msg=''):
-        elapsed_time = (self.tc._run_end_time - self.tc._run_start_time)
-
-        self.tc.logger.info("SUMMARY:")
-        self.tc.logger.info("%s (%s) - Ran %d tests in %.3fs" % (component,
-            context_msg, len(self.tc._registry['cases']), elapsed_time))
-        if self.wasSuccessful():
-            msg = "%s - OK - All required tests passed" % component
-        else:
-            msg = "%s - FAIL - Required tests failed" % component
-        self.tc.logger.info(msg)
-
-    def logDetails(self):
-        if list(self._results):
-            tid = list(self._results)[0]
-            result = self._results[tid]['result']
-            result.logDetails()
-
-class _Worker(threading.Thread):
-    """Thread executing tasks from a given tasks queue"""
-    def __init__(self, tasks, result, stream):
-        threading.Thread.__init__(self)
-        self.tasks = tasks
-
-        self.result = result
-        self.stream = stream
-
-    def run(self):
-        while True:
-            try:
-                func, args, kargs = self.tasks.get(block=False)
-            except queue.Empty:
-                break
-
-            try:
-                run_start_time = time.time()
-                rc = func(*args, **kargs)
-                run_end_time = time.time()
-                self.result.addResult(rc, run_start_time, run_end_time)
-                self.stream.finish()
-            except Exception as e:
-                print(e)
-            finally:
-                self.tasks.task_done()
-
-class _ThreadedPool:
-    """Pool of threads consuming tasks from a queue"""
-    def __init__(self, num_workers, num_tasks, stream=None, result=None):
-        self.tasks = queue.Queue(num_tasks)
-        self.workers = []
-
-        for _ in range(num_workers):
-            worker = _Worker(self.tasks, result, stream)
-            self.workers.append(worker)
-
-    def start(self):
-        for worker in self.workers:
-            worker.start()
-
-    def add_task(self, func, *args, **kargs):
-        """Add a task to the queue"""
-        self.tasks.put((func, args, kargs))
-
-    def wait_completion(self):
-        """Wait for completion of all the tasks in the queue"""
-        self.tasks.join()
-        for worker in self.workers:
-            worker.join()
-
-class OETestRunnerThreaded(OETestRunner):
-    streamLoggerClass = OEStreamLoggerThreaded
-
-    def __init__(self, tc, *args, **kwargs):
-        super(OETestRunnerThreaded, self).__init__(tc, *args, **kwargs)
-        self.resultclass = OETestResultThreadedInternal # XXX: XML reporting overrides at __init__
-
-    def run(self, suites):
-        result = OETestResultThreaded(self.tc)
-
-        pool = _ThreadedPool(len(suites), len(suites), stream=self.stream,
-                result=result)
-        for s in suites:
-            pool.add_task(super(OETestRunnerThreaded, self).run, s)
-        pool.start()
-        pool.wait_completion()
-        result._fill_tc_results()
-
-        return result
-
-class OETestContextThreaded(OETestContext):
-    loaderClass = OETestLoaderThreaded
-    runnerClass = OETestRunnerThreaded
-
-    def loadTests(self, module_paths, modules=[], tests=[],
-            modules_manifest="", modules_required=[], filters={}, process_num=0):
-        if modules_manifest:
-            modules = self._read_modules_from_manifest(modules_manifest)
-
-        self.loader = self.loaderClass(self, module_paths, modules, tests,
-                modules_required, filters, process_num)
-        self.suites = self.loader.discover()

meta/lib/oeqa/files/test.pl

@@ -1,2 +0,0 @@
-$a = 9.01e+21 - 9.01e+21 + 0.01;
-print ("the value of a is ", $a, "\n");

meta/lib/oeqa/files/testresults/testresults.json

@@ -0,0 +1,40 @@
+{
+    "runtime_core-image-minimal_qemuarm_20181225195701": {
+        "configuration": {
+            "DISTRO": "poky",
+            "HOST_DISTRO": "ubuntu-16.04",
+            "IMAGE_BASENAME": "core-image-minimal",
+            "IMAGE_PKGTYPE": "rpm",
+            "LAYERS": {
+                "meta": {
+                    "branch": "master",
+                    "commit": "801745d918e83f976c706f29669779f5b292ade3",
+                    "commit_count": 52782
+                },
+                "meta-poky": {
+                    "branch": "master",
+                    "commit": "801745d918e83f976c706f29669779f5b292ade3",
+                    "commit_count": 52782
+                },
+                "meta-yocto-bsp": {
+                    "branch": "master",
+                    "commit": "801745d918e83f976c706f29669779f5b292ade3",
+                    "commit_count": 52782
+                }
+            },
+            "MACHINE": "qemuarm",
+            "STARTTIME": "20181225195701",
+            "TEST_TYPE": "runtime"
+        },
+        "result": {
+            "apt.AptRepoTest.test_apt_install_from_repo": {
+                "log": "Test requires apt to be installed",
+                "status": "PASSED"
+            },
+            "buildcpio.BuildCpioTest.test_cpio": {
+                "log": "Test requires autoconf to be installed",
+                "status": "ERROR"
+            }            
+        }
+    }
+}

meta/lib/oeqa/runtime/cases/apt.py

@@ -18,7 +18,7 @@ class AptRepoTest(AptTest):
     @classmethod
     def setUpClass(cls):
         service_repo = os.path.join(cls.tc.td['DEPLOY_DIR_DEB'], 'all')
-        cls.repo_server = HTTPService(service_repo, cls.tc.target.server_ip)
+        cls.repo_server = HTTPService(service_repo, cls.tc.target.server_ip, logger=cls.tc.logger)
         cls.repo_server.start()
 
     @classmethod

meta/lib/oeqa/runtime/cases/buildcpio.py

@@ -1,7 +1,7 @@
 from oeqa.runtime.case import OERuntimeTestCase
 from oeqa.core.decorator.depends import OETestDepends
 from oeqa.core.decorator.oeid import OETestID
-from oeqa.core.decorator.data import skipIfNotFeature
+from oeqa.runtime.decorator.package import OEHasPackage
 
 from oeqa.runtime.utils.targetbuildproject import TargetBuildProject
 
@@ -13,17 +13,18 @@ class BuildCpioTest(OERuntimeTestCase):
         cls.project = TargetBuildProject(cls.tc.target,
                                          uri,
                                          dl_dir = cls.tc.td['DL_DIR'])
-        cls.project.download_archive()
 
     @classmethod
     def tearDownClass(cls):
         cls.project.clean()
 
     @OETestID(205)
-    @skipIfNotFeature('tools-sdk',
-                      'Test requires tools-sdk to be in IMAGE_FEATURES')
     @OETestDepends(['ssh.SSHTest.test_ssh'])
+    @OEHasPackage(['gcc'])
+    @OEHasPackage(['make'])
+    @OEHasPackage(['autoconf'])
     def test_cpio(self):
+        self.project.download_archive()
         self.project.run_configure()
         self.project.run_make()
         self.project.run_install()

meta/lib/oeqa/runtime/cases/buildgalculator.py

@@ -1,7 +1,7 @@
 from oeqa.runtime.case import OERuntimeTestCase
 from oeqa.core.decorator.depends import OETestDepends
 from oeqa.core.decorator.oeid import OETestID
-from oeqa.core.decorator.data import skipIfNotFeature
+from oeqa.runtime.decorator.package import OEHasPackage
 
 from oeqa.runtime.utils.targetbuildproject import TargetBuildProject
 
@@ -13,16 +13,17 @@ class GalculatorTest(OERuntimeTestCase):
         cls.project = TargetBuildProject(cls.tc.target,
                                          uri,
                                          dl_dir = cls.tc.td['DL_DIR'])
-        cls.project.download_archive()
 
     @classmethod
     def tearDownClass(cls):
         cls.project.clean()
 
     @OETestID(1526)
-    @skipIfNotFeature('tools-sdk',
-                      'Test requires tools-sdk to be in IMAGE_FEATURES')
     @OETestDepends(['ssh.SSHTest.test_ssh'])
+    @OEHasPackage(['gcc'])
+    @OEHasPackage(['make'])
+    @OEHasPackage(['autoconf'])
     def test_galculator(self):
+        self.project.download_archive()
         self.project.run_configure()
         self.project.run_make()

meta/lib/oeqa/runtime/cases/buildlzip.py

@@ -1,7 +1,7 @@
 from oeqa.runtime.case import OERuntimeTestCase
 from oeqa.core.decorator.depends import OETestDepends
 from oeqa.core.decorator.oeid import OETestID
-from oeqa.core.decorator.data import skipIfNotFeature
+from oeqa.runtime.decorator.package import OEHasPackage
 
 from oeqa.runtime.utils.targetbuildproject import TargetBuildProject
 
@@ -14,21 +14,19 @@ class BuildLzipTest(OERuntimeTestCase):
         cls.project = TargetBuildProject(cls.tc.target,
                                          uri,
                                          dl_dir = cls.tc.td['DL_DIR'])
-        cls.project.download_archive()
 
     @classmethod
     def tearDownClass(cls):
         cls.project.clean()
 
     @OETestID(206)
-    @skipIfNotFeature('tools-sdk',
-                      'Test requires tools-sdk to be in IMAGE_FEATURES')
     @OETestDepends(['ssh.SSHTest.test_ssh'])
+    @OEHasPackage(['gcc'])
+    @OEHasPackage(['make'])
+    @OEHasPackage(['autoconf'])
     def test_lzip(self):
+        self.project.download_archive()
         self.project.run_configure()
         self.project.run_make()
         self.project.run_install()
 
-    @classmethod
-    def tearDownClass(self):
-        self.project.clean()