build-appliance-image: Update to sumo head revision

(From OE-Core rev: 0a2db923fd17019d07d88204b355aa46590f0b97)

Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>

bitbake/bin/bitbake-diffsigs

@@ -1,7 +1,7 @@
 #!/usr/bin/env python3
 
-# bitbake-diffsigs
-# BitBake task signature data comparison utility
+# bitbake-diffsigs / bitbake-dumpsig
+# BitBake task signature data dump and comparison utility
 #
 # Copyright (C) 2012-2013, 2017 Intel Corporation
 #
@@ -21,7 +21,6 @@
 import os
 import sys
 import warnings
-import fnmatch
 import argparse
 import logging
 import pickle
@@ -32,7 +31,10 @@ import bb.tinfoil
 import bb.siggen
 import bb.msg
 
-logger = bb.msg.logger_create('bitbake-diffsigs')
+myname = os.path.basename(sys.argv[0])
+logger = bb.msg.logger_create(myname)
+
+is_dump = myname == 'bitbake-dumpsig'
 
 def find_siginfo(tinfoil, pn, taskname, sigs=None):
     result = None
@@ -59,8 +61,8 @@ def find_siginfo(tinfoil, pn, taskname, sigs=None):
         sys.exit(2)
     return result
 
-def find_compare_task(bbhandler, pn, taskname, sig1=None, sig2=None, color=False):
-    """ Find the most recent signature files for the specified PN/task and compare them """
+def find_siginfo_task(bbhandler, pn, taskname, sig1=None, sig2=None):
+    """ Find the most recent signature files for the specified PN/task """
 
     if not taskname.startswith('do_'):
         taskname = 'do_%s' % taskname
@@ -79,73 +81,81 @@ def find_compare_task(bbhandler, pn, taskname, sig1=None, sig2=None, color=False
         latestfiles = [sigfiles[sig1], sigfiles[sig2]]
     else:
         filedates = find_siginfo(bbhandler, pn, taskname)
-        latestfiles = sorted(filedates.keys(), key=lambda f: filedates[f])[-3:]
+        latestfiles = sorted(filedates.keys(), key=lambda f: filedates[f])[-2:]
         if not latestfiles:
             logger.error('No sigdata files found matching %s %s' % (pn, taskname))
             sys.exit(1)
-        elif len(latestfiles) < 2:
-            logger.error('Only one matching sigdata file found for the specified task (%s %s)' % (pn, taskname))
-            sys.exit(1)
 
-    # Define recursion callback
-    def recursecb(key, hash1, hash2):
-        hashes = [hash1, hash2]
-        hashfiles = find_siginfo(bbhandler, key, None, hashes)
+    return latestfiles
 
-        recout = []
-        if len(hashfiles) == 0:
-            recout.append("Unable to find matching sigdata for %s with hashes %s or %s" % (key, hash1, hash2))
-        elif not hash1 in hashfiles:
-            recout.append("Unable to find matching sigdata for %s with hash %s" % (key, hash1))
-        elif not hash2 in hashfiles:
-            recout.append("Unable to find matching sigdata for %s with hash %s" % (key, hash2))
-        else:
-            out2 = bb.siggen.compare_sigfiles(hashfiles[hash1], hashfiles[hash2], recursecb, color=color)
-            for change in out2:
-                for line in change.splitlines():
-                    recout.append('  ' + line)
 
-        return recout
+# Define recursion callback
+def recursecb(key, hash1, hash2):
+    hashes = [hash1, hash2]
+    hashfiles = find_siginfo(tinfoil, key, None, hashes)
 
-    # Recurse into signature comparison
-    logger.debug("Signature file (previous): %s" % latestfiles[-2])
-    logger.debug("Signature file (latest): %s" % latestfiles[-1])
-    output = bb.siggen.compare_sigfiles(latestfiles[-2], latestfiles[-1], recursecb, color=color)
-    if output:
-        print('\n'.join(output))
-    sys.exit(0)
+    recout = []
+    if len(hashfiles) == 0:
+        recout.append("Unable to find matching sigdata for %s with hashes %s or %s" % (key, hash1, hash2))
+    elif not hash1 in hashfiles:
+        recout.append("Unable to find matching sigdata for %s with hash %s" % (key, hash1))
+    elif not hash2 in hashfiles:
+        recout.append("Unable to find matching sigdata for %s with hash %s" % (key, hash2))
+    else:
+        out2 = bb.siggen.compare_sigfiles(hashfiles[hash1], hashfiles[hash2], recursecb, color=color)
+        for change in out2:
+            for line in change.splitlines():
+                recout.append('  ' + line)
 
+    return recout
 
 
 parser = argparse.ArgumentParser(
-    description="Compares siginfo/sigdata files written out by BitBake")
+    description=("Dumps" if is_dump else "Compares") + " siginfo/sigdata files written out by BitBake")
 
-parser.add_argument('-d', '--debug',
+parser.add_argument('-D', '--debug',
                     help='Enable debug output',
                     action='store_true')
 
-parser.add_argument('--color',
-        help='Colorize output (where %(metavar)s is %(choices)s)',
-        choices=['auto', 'always', 'never'], default='auto', metavar='color')
+if is_dump:
+    parser.add_argument("-t", "--task",
+            help="find the signature data file for the last run of the specified task",
+            action="store", dest="taskargs", nargs=2, metavar=('recipename', 'taskname'))
 
-parser.add_argument("-t", "--task",
-        help="find the signature data files for last two runs of the specified task and compare them",
-        action="store", dest="taskargs", nargs=2, metavar=('recipename', 'taskname'))
+    parser.add_argument("sigdatafile1",
+            help="Signature file to dump. Not used when using -t/--task.",
+            action="store", nargs='?', metavar="sigdatafile")
+else:
+    parser.add_argument('-c', '--color',
+            help='Colorize the output (where %(metavar)s is %(choices)s)',
+            choices=['auto', 'always', 'never'], default='auto', metavar='color')
 
-parser.add_argument("-s", "--signature",
-        help="With -t/--task, specify the signatures to look for instead of taking the last two",
-        action="store", dest="sigargs", nargs=2, metavar=('fromsig', 'tosig'))
+    parser.add_argument('-d', '--dump',
+            help='Dump the last signature data instead of comparing (equivalent to using bitbake-dumpsig)',
+            action='store_true')
 
-parser.add_argument("sigdatafile1",
-        help="First signature file to compare (or signature file to dump, if second not specified). Not used when using -t/--task.",
-        action="store", nargs='?')
+    parser.add_argument("-t", "--task",
+            help="find the signature data files for the last two runs of the specified task and compare them",
+            action="store", dest="taskargs", nargs=2, metavar=('recipename', 'taskname'))
 
-parser.add_argument("sigdatafile2",
-        help="Second signature file to compare",
-        action="store", nargs='?')
+    parser.add_argument("-s", "--signature",
+            help="With -t/--task, specify the signatures to look for instead of taking the last two",
+            action="store", dest="sigargs", nargs=2, metavar=('fromsig', 'tosig'))
 
+    parser.add_argument("sigdatafile1",
+            help="First signature file to compare (or signature file to dump, if second not specified). Not used when using -t/--task.",
+            action="store", nargs='?')
+
+    parser.add_argument("sigdatafile2",
+            help="Second signature file to compare",
+            action="store", nargs='?')
 
 options = parser.parse_args()
+if is_dump:
+    options.color = 'never'
+    options.dump = True
+    options.sigdatafile2 = None
+    options.sigargs = None
 
 if options.debug:
     logger.setLevel(logging.DEBUG)
@@ -155,17 +165,32 @@ color = (options.color == 'always' or (options.color == 'auto' and sys.stdout.is
 if options.taskargs:
     with bb.tinfoil.Tinfoil() as tinfoil:
         tinfoil.prepare(config_only=True)
-        if options.sigargs:
-            find_compare_task(tinfoil, options.taskargs[0], options.taskargs[1], options.sigargs[0], options.sigargs[1], color=color)
+        if not options.dump and options.sigargs:
+            files = find_siginfo_task(tinfoil, options.taskargs[0], options.taskargs[1], options.sigargs[0], options.sigargs[1])
         else:
-            find_compare_task(tinfoil, options.taskargs[0], options.taskargs[1], color=color)
+            files = find_siginfo_task(tinfoil, options.taskargs[0], options.taskargs[1])
+
+        if options.dump:
+            logger.debug("Signature file: %s" % files[-1])
+            output = bb.siggen.dump_sigfile(files[-1])
+        else:
+            if len(files) < 2:
+                logger.error('Only one matching sigdata file found for the specified task (%s %s)' % (options.taskargs[0], options.taskargs[1]))
+                sys.exit(1)
+
+            # Recurse into signature comparison
+            logger.debug("Signature file (previous): %s" % files[-2])
+            logger.debug("Signature file (latest): %s" % files[-1])
+            output = bb.siggen.compare_sigfiles(files[-2], files[-1], recursecb, color=color)
 else:
     if options.sigargs:
         logger.error('-s/--signature can only be used together with -t/--task')
         sys.exit(1)
     try:
-        if options.sigdatafile1 and options.sigdatafile2:
-            output = bb.siggen.compare_sigfiles(options.sigdatafile1, options.sigdatafile2, color=color)
+        if not options.dump and options.sigdatafile1 and options.sigdatafile2:
+            with bb.tinfoil.Tinfoil() as tinfoil:
+                tinfoil.prepare(config_only=True)
+                output = bb.siggen.compare_sigfiles(options.sigdatafile1, options.sigdatafile2, recursecb, color=color)
         elif options.sigdatafile1:
             output = bb.siggen.dump_sigfile(options.sigdatafile1)
         else:
@@ -179,5 +204,5 @@ else:
         logger.error('Invalid signature data - ensure you are specifying sigdata/siginfo files')
         sys.exit(1)
 
-    if output:
-        print('\n'.join(output))
+if output:
+    print('\n'.join(output))

bitbake/bin/bitbake-dumpsig

@@ -1,94 +0,0 @@
-#!/usr/bin/env python3
-
-# bitbake-dumpsig
-# BitBake task signature dump utility
-#
-# Copyright (C) 2013 Intel Corporation
-#
-# This program is free software; you can redistribute it and/or modify
-# it under the terms of the GNU General Public License version 2 as
-# published by the Free Software Foundation.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License along
-# with this program; if not, write to the Free Software Foundation, Inc.,
-# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
-
-import os
-import sys
-import warnings
-import optparse
-import logging
-import pickle
-
-sys.path.insert(0, os.path.join(os.path.dirname(os.path.dirname(sys.argv[0])), 'lib'))
-
-import bb.tinfoil
-import bb.siggen
-import bb.msg
-
-logger = bb.msg.logger_create('bitbake-dumpsig')
-
-def find_siginfo_task(bbhandler, pn, taskname):
-    """ Find the most recent signature file for the specified PN/task """
-
-    if not hasattr(bb.siggen, 'find_siginfo'):
-        logger.error('Metadata does not support finding signature data files')
-        sys.exit(1)
-
-    if not taskname.startswith('do_'):
-        taskname = 'do_%s' % taskname
-
-    filedates = bb.siggen.find_siginfo(pn, taskname, None, bbhandler.config_data)
-    latestfiles = sorted(filedates.keys(), key=lambda f: filedates[f])[-1:]
-    if not latestfiles:
-        logger.error('No sigdata files found matching %s %s' % (pn, taskname))
-        sys.exit(1)
-
-    return latestfiles[0]
-
-parser = optparse.OptionParser(
-    description = "Dumps siginfo/sigdata files written out by BitBake",
-    usage = """
-  %prog -t recipename taskname
-  %prog sigdatafile""")
-
-parser.add_option("-D", "--debug",
-        help = "enable debug",
-        action = "store_true", dest="debug", default = False)
-
-parser.add_option("-t", "--task",
-        help = "find the signature data file for the specified task",
-        action="store", dest="taskargs", nargs=2, metavar='recipename taskname')
-
-options, args = parser.parse_args(sys.argv)
-
-if options.debug:
-    logger.setLevel(logging.DEBUG)
-
-if options.taskargs:
-    tinfoil = bb.tinfoil.Tinfoil()
-    tinfoil.prepare(config_only = True)
-    file = find_siginfo_task(tinfoil, options.taskargs[0], options.taskargs[1])
-    logger.debug("Signature file: %s" % file)
-elif len(args) == 1:
-    parser.print_help()
-    sys.exit(0)
-else:
-    file = args[1]
-
-try:
-    output = bb.siggen.dump_sigfile(file)
-except IOError as e:
-    logger.error(str(e))
-    sys.exit(1)
-except (pickle.UnpicklingError, EOFError):
-    logger.error('Invalid signature data - ensure you are specifying a sigdata/siginfo file')
-    sys.exit(1)
-
-if output:
-    print('\n'.join(output))

bitbake/bin/bitbake-dumpsig

@@ -0,0 +1 @@
+bitbake-diffsigs

bitbake/doc/bitbake-user-manual/bitbake-user-manual-fetching.xml

@@ -588,6 +588,14 @@
                         The name of the path in which to place the checkout.
                         By default, the path is <filename>git/</filename>.
                         </para></listitem>
+                    <listitem><para><emphasis>"usehead":</emphasis>
+                        Enables local <filename>git://</filename> URLs to use the
+                        current branch HEAD as the revision for use with
+                        <filename>AUTOREV</filename>.
+                        The "usehead" parameter implies no branch and only works
+                        when the transfer protocol is
+                        <filename>file://</filename>.
+                        </para></listitem>
                 </itemizedlist>
                 Here are some example URLs:
                 <literallayout class='monospaced'>

bitbake/doc/bitbake-user-manual/bitbake-user-manual-metadata.xml

@@ -502,7 +502,7 @@
         </section>
 
         <section id='unsetting-variables'>
-            <title>Unseting variables</title>
+            <title>Unsetting variables</title>
 
             <para>
                 It is possible to completely remove a variable or a variable flag

bitbake/lib/bb/COW.py

@@ -150,7 +150,7 @@ class COWDictMeta(COWMeta):
                 yield value
             if type == "items":
                 yield (key, value)
-        raise StopIteration()
+        return
 
     def iterkeys(cls):
         return cls.iter("keys")

bitbake/lib/bb/cooker.py

@@ -175,18 +175,31 @@ class BBCooker:
 
         self.configuration = configuration
 
+        bb.debug(1, "BBCooker starting %s" % time.time())
+        sys.stdout.flush()
+
         self.configwatcher = pyinotify.WatchManager()
+        bb.debug(1, "BBCooker pyinotify1 %s" % time.time())
+        sys.stdout.flush()
+
         self.configwatcher.bbseen = []
         self.configwatcher.bbwatchedfiles = []
         self.confignotifier = pyinotify.Notifier(self.configwatcher, self.config_notifications)
+        bb.debug(1, "BBCooker pyinotify2 %s" % time.time())
+        sys.stdout.flush()
         self.watchmask = pyinotify.IN_CLOSE_WRITE | pyinotify.IN_CREATE | pyinotify.IN_DELETE | \
                          pyinotify.IN_DELETE_SELF | pyinotify.IN_MODIFY | pyinotify.IN_MOVE_SELF | \
                          pyinotify.IN_MOVED_FROM | pyinotify.IN_MOVED_TO
         self.watcher = pyinotify.WatchManager()
+        bb.debug(1, "BBCooker pyinotify3 %s" % time.time())
+        sys.stdout.flush()
         self.watcher.bbseen = []
         self.watcher.bbwatchedfiles = []
         self.notifier = pyinotify.Notifier(self.watcher, self.notifications)
 
+        bb.debug(1, "BBCooker pyinotify complete %s" % time.time())
+        sys.stdout.flush()
+
         # If being called by something like tinfoil, we need to clean cached data
         # which may now be invalid
         bb.parse.clear_cache()
@@ -196,6 +209,9 @@ class BBCooker:
 
         self.initConfigurationData()
 
+        bb.debug(1, "BBCooker parsed base configuration %s" % time.time())
+        sys.stdout.flush()
+
         # we log all events to a file if so directed
         if self.configuration.writeeventlog:
             # register the log file writer as UI Handler
@@ -233,6 +249,9 @@ class BBCooker:
         # Let SIGHUP exit as SIGTERM
         signal.signal(signal.SIGHUP, self.sigterm_exception)
 
+        bb.debug(1, "BBCooker startup complete %s" % time.time())
+        sys.stdout.flush()
+
     def process_inotify_updates(self):
         for n in [self.confignotifier, self.notifier]:
             if n.check_events(timeout=0):

bitbake/lib/bb/fetch2/__init__.py

@@ -1457,7 +1457,7 @@ class FetchMethod(object):
                 else:
                     cmd = 'rpm2cpio.sh %s | cpio -id' % (file)
             elif file.endswith('.deb') or file.endswith('.ipk'):
-                output = subprocess.check_output('ar -t %s' % file, preexec_fn=subprocess_setup, shell=True)
+                output = subprocess.check_output(['ar', '-t', file], preexec_fn=subprocess_setup)
                 datafile = None
                 if output:
                     for line in output.decode().splitlines():

bitbake/lib/bb/main.py

@@ -448,7 +448,7 @@ def setup_bitbake(configParams, configuration, extrafeatures=None):
                 else:
                     logger.info("Reconnecting to bitbake server...")
                     if not os.path.exists(sockname):
-                        print("Previous bitbake instance shutting down?, waiting to retry...")
+                        logger.info("Previous bitbake instance shutting down?, waiting to retry...")
                         i = 0
                         lock = None
                         # Wait for 5s or until we can get the lock

bitbake/lib/bb/runqueue.py

@@ -1371,6 +1371,12 @@ class RunQueue:
                 bb.event.fire(bb.event.DepTreeGenerated(depgraph), self.cooker.data)
 
         if self.state is runQueueSceneInit:
+            if not self.dm_event_handler_registered:
+                 res = bb.event.register(self.dm_event_handler_name,
+                                         lambda x: self.dm.check(self) if self.state in [runQueueSceneRun, runQueueRunning, runQueueCleanUp] else False,
+                                         ('bb.event.HeartbeatEvent',))
+                 self.dm_event_handler_registered = True
+
             dump = self.cooker.configuration.dump_signatures
             if dump:
                 self.rqdata.init_progress_reporter.finish()
@@ -1387,11 +1393,6 @@ class RunQueue:
                 self.rqexe = RunQueueExecuteScenequeue(self)
 
         if self.state is runQueueSceneRun:
-            if not self.dm_event_handler_registered:
-                 res = bb.event.register(self.dm_event_handler_name,
-                                         lambda x: self.dm.check(self) if self.state in [runQueueSceneRun, runQueueRunning, runQueueCleanUp] else False,
-                                         ('bb.event.HeartbeatEvent',))
-                 self.dm_event_handler_registered = True
             retval = self.rqexe.execute()
 
         if self.state is runQueueRunInit:

bitbake/lib/bb/server/process.py

@@ -130,6 +130,7 @@ class ProcessServer(multiprocessing.Process):
         bb.utils.set_process_name("Cooker")
 
         ready = []
+        newconnections = []
 
         self.controllersock = False
         fds = [self.sock]
@@ -138,37 +139,48 @@ class ProcessServer(multiprocessing.Process):
         print("Entering server connection loop")
 
         def disconnect_client(self, fds):
-            if not self.haveui:
-                return
             print("Disconnecting Client")
-            fds.remove(self.controllersock)
-            fds.remove(self.command_channel)
-            bb.event.unregister_UIHhandler(self.event_handle, True)
-            self.command_channel_reply.writer.close()
-            self.event_writer.writer.close()
-            del self.event_writer
-            self.controllersock.close()
-            self.controllersock = False
-            self.haveui = False
-            self.lastui = time.time()
-            self.cooker.clientComplete()
-            if self.timeout is None:
+            if self.controllersock:
+                fds.remove(self.controllersock)
+                self.controllersock.close()
+                self.controllersock = False
+            if self.haveui:
+                fds.remove(self.command_channel)
+                bb.event.unregister_UIHhandler(self.event_handle, True)
+                self.command_channel_reply.writer.close()
+                self.event_writer.writer.close()
+                self.command_channel.close()
+                self.command_channel = False
+                del self.event_writer
+                self.lastui = time.time()
+                self.cooker.clientComplete()
+                self.haveui = False
+            ready = select.select(fds,[],[],0)[0]
+            if newconnections:
+                print("Starting new client")
+                conn = newconnections.pop(-1)
+                fds.append(conn)
+                self.controllersock = conn
+            elif self.timeout is None and not ready:
                 print("No timeout, exiting.")
                 self.quit = True
 
         while not self.quit:
             if self.sock in ready:
-                self.controllersock, address = self.sock.accept()
-                if self.haveui:
-                    print("Dropping connection attempt as we have a UI %s" % (str(ready)))
-                    self.controllersock.close()
-                else:
-                    print("Accepting %s" % (str(ready)))
-                    fds.append(self.controllersock)
+                while select.select([self.sock],[],[],0)[0]:
+                    controllersock, address = self.sock.accept()
+                    if self.controllersock:
+                        print("Queuing %s (%s)" % (str(ready), str(newconnections)))
+                        newconnections.append(controllersock)
+                    else:
+                        print("Accepting %s (%s)" % (str(ready), str(newconnections)))
+                        self.controllersock = controllersock
+                        fds.append(controllersock)
             if self.controllersock in ready:
                 try:
-                    print("Connecting Client")
+                    print("Processing Client")
                     ui_fds = recvfds(self.controllersock, 3)
+                    print("Connecting Client")
 
                     # Where to write events to
                     writer = ConnectionWriter(ui_fds[0])
@@ -239,6 +251,12 @@ class ProcessServer(multiprocessing.Process):
         while not lock:
             with bb.utils.timeout(3):
                 lock = bb.utils.lockfile(lockfile, shared=False, retry=False, block=True)
+                if lock:
+                    # We hold the lock so we can remove the file (hide stale pid data)
+                    bb.utils.remove(lockfile)
+                    bb.utils.unlockfile(lock)
+                    return
+
                 if not lock:
                     # Some systems may not have lsof available
                     procs = None
@@ -259,10 +277,6 @@ class ProcessServer(multiprocessing.Process):
                     if procs:
                         msg += ":\n%s" % str(procs)
                     print(msg)
-                    return
-        # We hold the lock so we can remove the file (hide stale pid data)
-        bb.utils.remove(lockfile)
-        bb.utils.unlockfile(lock)
 
     def idle_commands(self, delay, fds=None):
         nextsleep = delay
@@ -396,7 +410,10 @@ class BitBakeServer(object):
         self.bitbake_lock.close()
 
         ready = ConnectionReader(self.readypipe)
-        r = ready.poll(30)
+        r = ready.poll(5)
+        if not r:
+            bb.note("Bitbake server didn't start within 5 seconds, waiting for 90")
+            r = ready.poll(90)
         if r:
             r = ready.get()
         if not r or r != "ready":
@@ -406,28 +423,40 @@ class BitBakeServer(object):
                 logstart_re = re.compile(self.start_log_format % ('([0-9]+)', '([0-9-]+ [0-9:.]+)'))
                 started = False
                 lines = []
+                lastlines = []
                 with open(logfile, "r") as f:
                     for line in f:
                         if started:
                             lines.append(line)
                         else:
+                            lastlines.append(line)
                             res = logstart_re.match(line.rstrip())
                             if res:
                                 ldatetime = datetime.datetime.strptime(res.group(2), self.start_log_datetime_format)
                                 if ldatetime >= startdatetime:
                                     started = True
                                     lines.append(line)
+                        if len(lastlines) > 60:
+                            lastlines = lastlines[-60:]
                 if lines:
-                    if len(lines) > 10:
-                        bb.error("Last 10 lines of server log for this session (%s):\n%s" % (logfile, "".join(lines[-10:])))
+                    if len(lines) > 60:
+                        bb.error("Last 60 lines of server log for this session (%s):\n%s" % (logfile, "".join(lines[-60:])))
                     else:
                         bb.error("Server log for this session (%s):\n%s" % (logfile, "".join(lines)))
+                elif lastlines:
+                        bb.error("Server didn't start, last 60 loglines (%s):\n%s" % (logfile, "".join(lastlines)))
+            else:
+                bb.error("%s doesn't exist" % logfile)
+
             raise SystemExit(1)
+
         ready.close()
         os.close(self.readypipein)
 
     def _startServer(self):
         print(self.start_log_format % (os.getpid(), datetime.datetime.now().strftime(self.start_log_datetime_format)))
+        sys.stdout.flush()
+
         server = ProcessServer(self.bitbake_lock, self.sock, self.sockname)
         self.configuration.setServerRegIdleCallback(server.register_idle_function)
         writer = ConnectionWriter(self.readypipein)
@@ -443,6 +472,8 @@ class BitBakeServer(object):
         server.server_timeout = self.configuration.server_timeout
         server.xmlrpcinterface = self.configuration.xmlrpcinterface
         print("Started bitbake server pid %d" % os.getpid())
+        sys.stdout.flush()
+
         server.start()
 
 def connectProcessServer(sockname, featureset):
@@ -451,16 +482,24 @@ def connectProcessServer(sockname, featureset):
     # AF_UNIX has path length issues so chdir here to workaround
     cwd = os.getcwd()
 
-    try:
-        os.chdir(os.path.dirname(sockname))
-        sock.connect(os.path.basename(sockname))
-    finally:
-        os.chdir(cwd)
-
     readfd = writefd = readfd1 = writefd1 = readfd2 = writefd2 = None
     eq = command_chan_recv = command_chan = None
 
+    sock.settimeout(10)
+
     try:
+        try:
+            os.chdir(os.path.dirname(sockname))
+            finished = False
+            while not finished:
+                try:
+                    sock.connect(os.path.basename(sockname))
+                    finished = True
+                except IOError as e:
+                    if e.errno == errno.EWOULDBLOCK:
+                        pass
+        finally:
+            os.chdir(cwd)
 
         # Send an fd for the remote to write events to
         readfd, writefd = os.pipe()
@@ -489,7 +528,8 @@ def connectProcessServer(sockname, featureset):
             command_chan.close()
         for i in [writefd, readfd1, writefd2]:
             try:
-                os.close(i)
+                if i:
+                    os.close(i)
             except OSError:
                 pass
         sock.close()

bitbake/lib/bb/utils.py

@@ -1290,7 +1290,7 @@ def edit_metadata_file(meta_file, variables, varfunc):
     return updated
 
 
-def edit_bblayers_conf(bblayers_conf, add, remove):
+def edit_bblayers_conf(bblayers_conf, add, remove, edit_cb=None):
     """Edit bblayers.conf, adding and/or removing layers
     Parameters:
         bblayers_conf: path to bblayers.conf file to edit
@@ -1298,6 +1298,8 @@ def edit_bblayers_conf(bblayers_conf, add, remove):
             list to add nothing
         remove: layer path (or list of layer paths) to remove; None or
             empty list to remove nothing
+        edit_cb: optional callback function that will be called after
+            processing adds/removes once per existing entry.
     Returns a tuple:
         notadded: list of layers specified to be added but weren't
             (because they were already in the list)
@@ -1361,6 +1363,17 @@ def edit_bblayers_conf(bblayers_conf, add, remove):
                     bblayers.append(addlayer)
             del addlayers[:]
 
+        if edit_cb:
+            newlist = []
+            for layer in bblayers:
+                res = edit_cb(layer, canonicalise_path(layer))
+                if res != layer:
+                    newlist.append(res)
+                    updated = True
+                else:
+                    newlist.append(layer)
+            bblayers = newlist
+
         if updated:
             if op == '+=' and not bblayers:
                 bblayers = None

bitbake/lib/bblayers/layerindex.py

@@ -133,7 +133,7 @@ class LayerIndexPlugin(ActionPlugin):
         layerdir = os.path.join(repodir, subdir)
         if not os.path.exists(repodir):
             if fetch_layer:
-                result = subprocess.call('git clone %s %s' % (url, repodir), shell = True)
+                result = subprocess.call(['git', 'clone', url, repodir])
                 if result:
                     logger.error("Failed to download %s" % url)
                     return None, None

documentation/brief-yoctoprojectqs/brief-yoctoprojectqs.xml

@@ -131,19 +131,43 @@
             <literallayout class='monospaced'>
      $ git clone git://git.yoctoproject.org/poky
      Cloning into 'poky'...
-     remote: Counting objects: 361782, done.
-     remote: Compressing objects: 100% (87100/87100), done.
-     remote: Total 361782 (delta 268619), reused 361439 (delta 268277)
-     Receiving objects: 100% (361782/361782), 131.94 MiB | 6.88 MiB/s, done.
-     Resolving deltas: 100% (268619/268619), done.
+     remote: Counting objects: 431956, done.
+     remote: Compressing objects: 100% (101918/101918), done.
+     remote: Total 431956 (delta 322982), reused 431910 (delta 322936)
+     Receiving objects: 100% (431956/431956), 153.76 MiB | 6.86 MiB/s, done.
+     Resolving deltas: 100% (322982/322982), done.
      Checking connectivity... done.
-     $ git checkout tags/yocto-2.5 -b my-yocto-2.5
             </literallayout>
-            The previous Git checkout command creates a local branch
-            named my-&DISTRO_REL_TAG;. The files available to you in that
-            branch exactly match the repository's files in the
-            "&DISTRO_NAME_NO_CAP;" development branch at the time of the
-            Yocto Project &DISTRO; release.
+            Move to the poky directory and take a look at the tags:
+            <literallayout class='monospaced'>
+     $ cd poky
+     $ git fetch --tags
+     $ git tag
+     1.1_M1.final
+     1.1_M1.rc1
+     1.1_M1.rc2
+     1.1_M2.final
+     1.1_M2.rc1
+        .
+        .
+        .
+     yocto-2.5.2
+     yocto-2.5.3
+     yocto-2.6
+     yocto-2.6.1
+     yocto_1.5_M5.rc8
+            </literallayout>
+            For this example, check out the branch based on the
+            yocto-&DISTRO; release:
+            <literallayout class='monospaced'>
+     $ git checkout tags/yocto-&DISTRO; -b my-yocto-&DISTRO;
+     Switched to a new branch 'my-yocto-&DISTRO;'
+            </literallayout>
+            The previous Git checkout command creates a local branch named
+            my-yocto-&DISTRO;.
+            The files available to you in that branch exactly match the
+            repository's files in the "&DISTRO_NAME_NO_CAP;" development
+            branch at the time of the Yocto Project yocto-&DISTRO; release.
         </para>
 
         <para>

documentation/bsp-guide/bsp-guide.xml

@@ -126,6 +126,16 @@
                 <date>September 2018</date>
                 <revremark>The initial document released with the Yocto Project 2.5.1 Release.</revremark>
             </revision>
+            <revision>
+                <revnumber>2.5.2</revnumber>
+                <date>January 2019</date>
+                <revremark>The initial document released with the Yocto Project 2.5.2 Release.</revremark>
+            </revision>
+            <revision>
+                <revnumber>2.5.3</revnumber>
+                <date>&REL_MONTH_YEAR;</date>
+                <revremark>The initial document released with the Yocto Project 2.5.3 Release.</revremark>
+            </revision>
         </revhistory>
 
     <copyright>

documentation/dev-manual/dev-manual-start.xml

@@ -881,11 +881,11 @@
                     <literallayout class='monospaced'>
      $ git clone git://git.yoctoproject.org/poky
      Cloning into 'poky'...
-     remote: Counting objects: 367178, done.
-     remote: Compressing objects: 100% (88161/88161), done.
-     remote: Total 367178 (delta 272761), reused 366942 (delta 272525)
-     Receiving objects: 100% (367178/367178), 133.26 MiB | 6.40 MiB/s, done.
-     Resolving deltas: 100% (272761/272761), done.
+     remote: Counting objects: 431956, done.
+     remote: Compressing objects: 100% (101918/101918), done.
+     remote: Total 431956 (delta 322982), reused 431910 (delta 322936)
+     Receiving objects: 100% (431956/431956), 153.76 MiB | 6.86 MiB/s, done.
+     Resolving deltas: 100% (322982/322982), done.
      Checking connectivity... done.
                     </literallayout>
                     Unless you specify a specific development branch or
@@ -963,13 +963,12 @@
            .
            .
            .
-       remotes/origin/master-next
-       remotes/origin/master-next2
-       remotes/origin/morty
-       remotes/origin/pinky
-       remotes/origin/purple
-       remotes/origin/pyro
        remotes/origin/rocko
+       remotes/origin/rocko-next
+       remotes/origin/sumo
+       remotes/origin/sumo-next
+       remotes/origin/thud
+       remotes/origin/thud-next
                     </literallayout>
                     </para></listitem>
                 <listitem><para>
@@ -1048,11 +1047,10 @@
         .
         .
         .
-     yocto-2.2
-     yocto-2.2.1
-     yocto-2.3
-     yocto-2.3.1
-     yocto-2.4
+     yocto-2.5.2
+     yocto-2.5.3
+     yocto-2.6
+     yocto-2.6.1
      yocto_1.5_M5.rc8
                     </literallayout>
                     </para></listitem>

documentation/dev-manual/dev-manual.xml

@@ -111,6 +111,16 @@
                 <date>September 2018</date>
                 <revremark>The initial document released with the Yocto Project 2.5.1 Release.</revremark>
             </revision>
+            <revision>
+                <revnumber>2.5.2</revnumber>
+                <date>January 2019</date>
+                <revremark>The initial document released with the Yocto Project 2.5.2 Release.</revremark>
+            </revision>
+            <revision>
+                <revnumber>2.5.3</revnumber>
+                <date>&REL_MONTH_YEAR;</date>
+                <revremark>The initial document released with the Yocto Project 2.5.3 Release.</revremark>
+            </revision>
         </revhistory>
 
     <copyright>

documentation/kernel-dev/kernel-dev.xml

@@ -96,6 +96,16 @@
                 <date>September 2018</date>
                 <revremark>The initial document released with the Yocto Project 2.5.1 Release.</revremark>
             </revision>
+            <revision>
+                <revnumber>2.5.2</revnumber>
+                <date>January 2019</date>
+                <revremark>The initial document released with the Yocto Project 2.5.2 Release.</revremark>
+            </revision>
+            <revision>
+                <revnumber>2.5.3</revnumber>
+                <date>&REL_MONTH_YEAR;</date>
+                <revremark>The initial document released with the Yocto Project 2.5.3 Release.</revremark>
+            </revision>
         </revhistory>
 
     <copyright>

documentation/mega-manual/mega-manual.xml

@@ -80,6 +80,16 @@
                 <date>September 2018</date>
                 <revremark>The initial document released with the Yocto Project 2.5.1 Release.</revremark>
             </revision>
+            <revision>
+                <revnumber>2.5.2</revnumber>
+                <date>January 2019</date>
+                <revremark>The initial document released with the Yocto Project 2.5.2 Release.</revremark>
+            </revision>
+            <revision>
+                <revnumber>2.5.3</revnumber>
+                <date>&REL_MONTH_YEAR;</date>
+                <revremark>The initial document released with the Yocto Project 2.5.3 Release.</revremark>
+            </revision>
        </revhistory>
 
     <copyright>

documentation/overview-manual/overview-manual.xml

@@ -41,6 +41,16 @@
                 <date>September 2018</date>
                 <revremark>The initial document released with the Yocto Project 2.5.1 Release.</revremark>
             </revision>
+            <revision>
+                <revnumber>2.5.2</revnumber>
+                <date>January 2019</date>
+                <revremark>The initial document released with the Yocto Project 2.5.2 Release.</revremark>
+            </revision>
+            <revision>
+                <revnumber>2.5.3</revnumber>
+                <date>&REL_MONTH_YEAR;</date>
+                <revremark>The initial document released with the Yocto Project 2.5.3 Release.</revremark>
+            </revision>
         </revhistory>
 
     <copyright>

documentation/poky.ent

@@ -1,19 +1,19 @@
-<!ENTITY DISTRO "2.5.1">
-<!ENTITY DISTRO_COMPRESSED "251">
+<!ENTITY DISTRO "2.5.3">
+<!ENTITY DISTRO_COMPRESSED "253">
 <!ENTITY DISTRO_NAME_NO_CAP "sumo">
 <!ENTITY DISTRO_NAME "Sumo">
 <!ENTITY DISTRO_NAME_NO_CAP_MINUS_ONE "rocko">
 <!ENTITY DISTRO_NAME_MINUS_ONE "Rocko">
-<!ENTITY YOCTO_DOC_VERSION "2.5.1">
-<!ENTITY YOCTO_DOC_VERSION_MINUS_ONE "2.4">
-<!ENTITY DISTRO_REL_TAG "yocto-2.5.1">
-<!ENTITY METAINTELVERSION "9.0">
-<!ENTITY REL_MONTH_YEAR "September 2018">
+<!ENTITY YOCTO_DOC_VERSION "2.5.3">
+<!ENTITY YOCTO_DOC_VERSION_MINUS_ONE "2.4.4">
+<!ENTITY DISTRO_REL_TAG "yocto-2.5.3">
+<!ENTITY METAINTELVERSION "9.2">
+<!ENTITY REL_MONTH_YEAR "March 2019">
 <!ENTITY META_INTEL_REL_TAG "&METAINTELVERSION;-&DISTRO_NAME_NO_CAP;-&YOCTO_DOC_VERSION;">
-<!ENTITY POKYVERSION "20.0.1">
-<!ENTITY POKYVERSION_COMPRESSED "2001">
+<!ENTITY POKYVERSION "20.0.3">
+<!ENTITY POKYVERSION_COMPRESSED "2003">
 <!ENTITY YOCTO_POKY "poky-&DISTRO_NAME_NO_CAP;-&POKYVERSION;">
-<!ENTITY COPYRIGHT_YEAR "2010-2018">
+<!ENTITY COPYRIGHT_YEAR "2010-2019">
 <!ENTITY YOCTO_DL_URL "http://downloads.yoctoproject.org">
 <!ENTITY YOCTO_HOME_URL "http://www.yoctoproject.org">
 <!ENTITY YOCTO_LISTS_URL "http://lists.yoctoproject.org">

documentation/profile-manual/profile-manual.xml

@@ -96,6 +96,16 @@
                 <date>September 2018</date>
                 <revremark>The initial document released with the Yocto Project 2.5.1 Release.</revremark>
             </revision>
+            <revision>
+                <revnumber>2.5.2</revnumber>
+                <date>January 2019</date>
+                <revremark>The initial document released with the Yocto Project 2.5.2 Release.</revremark>
+            </revision>
+            <revision>
+                <revnumber>2.5.3</revnumber>
+                <date>&REL_MONTH_YEAR;</date>
+                <revremark>The initial document released with the Yocto Project 2.5.3 Release.</revremark>
+            </revision>
         </revhistory>
 
     <copyright>

documentation/ref-manual/ref-manual.xml

@@ -127,6 +127,16 @@
                 <date>September 2018</date>
                 <revremark>The initial document released with the Yocto Project 2.5.1 Release.</revremark>
             </revision>
+            <revision>
+                <revnumber>2.5.2</revnumber>
+                <date>January 2019</date>
+                <revremark>The initial document released with the Yocto Project 2.5.2 Release.</revremark>
+            </revision>
+            <revision>
+                <revnumber>2.5.3</revnumber>
+                <date>&REL_MONTH_YEAR;</date>
+                <revremark>The initial document released with the Yocto Project 2.5.3 Release.</revremark>
+            </revision>
         </revhistory>
 
     <copyright>

documentation/ref-manual/ref-variables.xml

@@ -13301,7 +13301,7 @@ recipes-graphics/xorg-font/font-alias_1.0.3.bb:PR = "${INC_PR}.3"
                             </para></listitem>
                         <listitem><para><emphasis><filename>maxdate</filename> -</emphasis>
                             Apply the patch only if <filename>SRCDATE</filename>
-                            is not later than <filename>mindate</filename>.
+                            is not later than <filename>maxdate</filename>.
                             </para></listitem>
                         <listitem><para><emphasis><filename>minrev</filename> -</emphasis>
                             Apply the patch only if <filename>SRCREV</filename>
@@ -16412,7 +16412,7 @@ recipes-graphics/xorg-font/font-alias_1.0.3.bb:PR = "${INC_PR}.3"
      USERADD_PACKAGES = "${PN}"
                     </literallayout>
                     <note>
-                        If follows that if you are going to use the
+                        It follows that if you are going to use the
                         <filename>USERADD_PACKAGES</filename> variable,
                         you need to set one or more of the
                         <link linkend='var-USERADD_PARAM'><filename>USERADD_PARAM</filename></link>,

documentation/sdk-manual/sdk-manual.xml

@@ -61,6 +61,16 @@
                 <date>September 2018</date>
                 <revremark>The initial document released with the Yocto Project 2.5.1 Release.</revremark>
             </revision>
+            <revision>
+                <revnumber>2.5.2</revnumber>
+                <date>January 2019</date>
+                <revremark>The initial document released with the Yocto Project 2.5.2 Release.</revremark>
+            </revision>
+            <revision>
+                <revnumber>2.5.3</revnumber>
+                <date>&REL_MONTH_YEAR;</date>
+                <revremark>The initial document released with the Yocto Project 2.5.3 Release.</revremark>
+            </revision>
        </revhistory>
 
     <copyright>

documentation/toaster-manual/toaster-manual.xml

@@ -71,6 +71,16 @@
                 <date>September 2018</date>
                 <revremark>The initial document released with the Yocto Project 2.5.1 Release.</revremark>
             </revision>
+            <revision>
+                <revnumber>2.5.2</revnumber>
+                <date>January 2019</date>
+                <revremark>The initial document released with the Yocto Project 2.5.2 Release.</revremark>
+            </revision>
+            <revision>
+                <revnumber>2.5.3</revnumber>
+                <date>&REL_MONTH_YEAR;</date>
+                <revremark>The initial document released with the Yocto Project 2.5.3 Release.</revremark>
+            </revision>
        </revhistory>
 
     <copyright>

documentation/tools/mega-manual.sed

@@ -2,39 +2,39 @@
 # This style is for manual folders like "yocto-project-qs" and "poky-ref-manual".
 # This is the old way that did it.  Can't do that now that we have "bitbake-user-manual" strings
 # in the mega-manual.
-# s@"ulink" href="http://www.yoctoproject.org/docs/2.5.1/[a-z]*-[a-z]*-[a-z]*/[a-z]*-[a-z]*-[a-z]*.html#@"link" href="#@g
-s@"ulink" href="http://www.yoctoproject.org/docs/2.5.1/yocto-project-qs/yocto-project-qs.html#@"link" href="#@g
-s@"ulink" href="http://www.yoctoproject.org/docs/2.5.1/poky-ref-manual/poky-ref-manual.html#@"link" href="#@g
+# s@"ulink" href="http://www.yoctoproject.org/docs/2.5.2/[a-z]*-[a-z]*-[a-z]*/[a-z]*-[a-z]*-[a-z]*.html#@"link" href="#@g
+s@"ulink" href="http://www.yoctoproject.org/docs/2.5.2/yocto-project-qs/yocto-project-qs.html#@"link" href="#@g
+s@"ulink" href="http://www.yoctoproject.org/docs/2.5.2/poky-ref-manual/poky-ref-manual.html#@"link" href="#@g
 
 # Processes all other manuals (<word>-<word> style) except for the BitBake User Manual because
 # it is not included in the mega-manual.
 # This style is for manual folders that use two word, which is the standard now (e.g. "ref-manual").
 # This was the one-liner that worked before we introduced the BitBake User Manual, which is
 # not in the mega-manual.
-# s@"ulink" href="http://www.yoctoproject.org/docs/2.5.1/[a-z]*-[a-z]*/[a-z]*-[a-z]*.html#@"link" href="#@g
+# s@"ulink" href="http://www.yoctoproject.org/docs/2.5.2/[a-z]*-[a-z]*/[a-z]*-[a-z]*.html#@"link" href="#@g
 
-s@"ulink" href="http://www.yoctoproject.org/docs/2.5.1/sdk-manual/sdk-manual.html#@"link" href="#@g
-s@"ulink" href="http://www.yoctoproject.org/docs/2.5.1/bsp-guide/bsp-guide.html#@"link" href="#@g
-s@"ulink" href="http://www.yoctoproject.org/docs/2.5.1/dev-manual/dev-manual.html#@"link" href="#@g
-s@"ulink" href="http://www.yoctoproject.org/docs/2.5.1/overview-manual/overview-manual.html#@"link" href="#@g
-s@"ulink" href="http://www.yoctoproject.org/docs/2.5.1/brief-yoctoprojectqs/brief-yoctoprojectqs.html#@"link" href="#@g
-s@"ulink" href="http://www.yoctoproject.org/docs/2.5.1/kernel-dev/kernel-dev.html#@"link" href="#@g
-s@"ulink" href="http://www.yoctoproject.org/docs/2.5.1/profile-manual/profile-manual.html#@"link" href="#@g
-s@"ulink" href="http://www.yoctoproject.org/docs/2.5.1/ref-manual/ref-manual.html#@"link" href="#@g
-s@"ulink" href="http://www.yoctoproject.org/docs/2.5.1/toaster-manual/toaster-manual.html#@"link" href="#@g
+s@"ulink" href="http://www.yoctoproject.org/docs/2.5.2/sdk-manual/sdk-manual.html#@"link" href="#@g
+s@"ulink" href="http://www.yoctoproject.org/docs/2.5.2/bsp-guide/bsp-guide.html#@"link" href="#@g
+s@"ulink" href="http://www.yoctoproject.org/docs/2.5.2/dev-manual/dev-manual.html#@"link" href="#@g
+s@"ulink" href="http://www.yoctoproject.org/docs/2.5.2/overview-manual/overview-manual.html#@"link" href="#@g
+s@"ulink" href="http://www.yoctoproject.org/docs/2.5.2/brief-yoctoprojectqs/brief-yoctoprojectqs.html#@"link" href="#@g
+s@"ulink" href="http://www.yoctoproject.org/docs/2.5.2/kernel-dev/kernel-dev.html#@"link" href="#@g
+s@"ulink" href="http://www.yoctoproject.org/docs/2.5.2/profile-manual/profile-manual.html#@"link" href="#@g
+s@"ulink" href="http://www.yoctoproject.org/docs/2.5.2/ref-manual/ref-manual.html#@"link" href="#@g
+s@"ulink" href="http://www.yoctoproject.org/docs/2.5.2/toaster-manual/toaster-manual.html#@"link" href="#@g
 
 # Process cases where just an external manual is referenced without an id anchor
-s@<a class="ulink" href="http://www.yoctoproject.org/docs/2.5.1/brief-yoctoprojectqs/brief-yoctoprojectqs.html" target="_top">Yocto Project Quick Build</a>@Yocto Project Quick Build@g
-s@<a class="ulink" href="http://www.yoctoproject.org/docs/2.5.1/yocto-project-qs/yocto-project-qs.html" target="_top">Yocto Project Quick Start</a>@Yocto Project Quick Start@g
-s@<a class="ulink" href="http://www.yoctoproject.org/docs/2.5.1/dev-manual/dev-manual.html" target="_top">Yocto Project Development Tasks Manual</a>@Yocto Project Development Tasks Manual@g
-s@<a class="ulink" href="http://www.yoctoproject.org/docs/2.5.1/overview-manual/overview-manual.html" target="_top">Yocto Project Overview and Concepts Manual</a>@Yocto project Overview and Concepts Manual@g
-s@<a class="ulink" href="http://www.yoctoproject.org/docs/2.5.1/sdk-manual/sdk-manual.html" target="_top">Yocto Project Application Development and the Extensible Software Development Kit (eSDK)</a>@Yocto Project Application Development and the Extensible Software Development Kit (eSDK)@g
-s@<a class="ulink" href="http://www.yoctoproject.org/docs/2.5.1/bsp-guide/bsp-guide.html" target="_top">Yocto Project Board Support Package (BSP) Developer's Guide</a>@Yocto Project Board Support Package (BSP) Developer's Guide@g
-s@<a class="ulink" href="http://www.yoctoproject.org/docs/2.5.1/profile-manual/profile-manual.html" target="_top">Yocto Project Profiling and Tracing Manual</a>@Yocto Project Profiling and Tracing Manual@g
-s@<a class="ulink" href="http://www.yoctoproject.org/docs/2.5.1/kernel-dev/kernel-dev.html" target="_top">Yocto Project Linux Kernel Development Manual</a>@Yocto Project Linux Kernel Development Manual@g
-s@<a class="ulink" href="http://www.yoctoproject.org/docs/2.5.1/ref-manual/ref-manual.html" target="_top">Yocto Project Reference Manual</a>@Yocto Project Reference Manual@g
-s@<a class="ulink" href="http://www.yoctoproject.org/docs/2.5.1/toaster-manual/toaster-manual.html" target="_top">Toaster User Manual</a>@Toaster User Manual@g
+s@<a class="ulink" href="http://www.yoctoproject.org/docs/2.5.2/brief-yoctoprojectqs/brief-yoctoprojectqs.html" target="_top">Yocto Project Quick Build</a>@Yocto Project Quick Build@g
+s@<a class="ulink" href="http://www.yoctoproject.org/docs/2.5.2/yocto-project-qs/yocto-project-qs.html" target="_top">Yocto Project Quick Start</a>@Yocto Project Quick Start@g
+s@<a class="ulink" href="http://www.yoctoproject.org/docs/2.5.2/dev-manual/dev-manual.html" target="_top">Yocto Project Development Tasks Manual</a>@Yocto Project Development Tasks Manual@g
+s@<a class="ulink" href="http://www.yoctoproject.org/docs/2.5.2/overview-manual/overview-manual.html" target="_top">Yocto Project Overview and Concepts Manual</a>@Yocto project Overview and Concepts Manual@g
+s@<a class="ulink" href="http://www.yoctoproject.org/docs/2.5.2/sdk-manual/sdk-manual.html" target="_top">Yocto Project Application Development and the Extensible Software Development Kit (eSDK)</a>@Yocto Project Application Development and the Extensible Software Development Kit (eSDK)@g
+s@<a class="ulink" href="http://www.yoctoproject.org/docs/2.5.2/bsp-guide/bsp-guide.html" target="_top">Yocto Project Board Support Package (BSP) Developer's Guide</a>@Yocto Project Board Support Package (BSP) Developer's Guide@g
+s@<a class="ulink" href="http://www.yoctoproject.org/docs/2.5.2/profile-manual/profile-manual.html" target="_top">Yocto Project Profiling and Tracing Manual</a>@Yocto Project Profiling and Tracing Manual@g
+s@<a class="ulink" href="http://www.yoctoproject.org/docs/2.5.2/kernel-dev/kernel-dev.html" target="_top">Yocto Project Linux Kernel Development Manual</a>@Yocto Project Linux Kernel Development Manual@g
+s@<a class="ulink" href="http://www.yoctoproject.org/docs/2.5.2/ref-manual/ref-manual.html" target="_top">Yocto Project Reference Manual</a>@Yocto Project Reference Manual@g
+s@<a class="ulink" href="http://www.yoctoproject.org/docs/2.5.2/toaster-manual/toaster-manual.html" target="_top">Toaster User Manual</a>@Toaster User Manual@g
 
 # Process a single, rouge occurrence of a linked reference to the Mega-Manual.
-s@<a class="ulink" href="http://www.yoctoproject.org/docs/2.5.1/mega-manual/mega-manual.html" target="_top">Yocto Project Mega-Manual</a>@Yocto Project Mega-Manual@g
+s@<a class="ulink" href="http://www.yoctoproject.org/docs/2.5.2/mega-manual/mega-manual.html" target="_top">Yocto Project Mega-Manual</a>@Yocto Project Mega-Manual@g
 

meta-poky/conf/distro/poky.conf

@@ -1,6 +1,6 @@
 DISTRO = "poky"
 DISTRO_NAME = "Poky (Yocto Project Reference Distro)"
-DISTRO_VERSION = "2.5.2"
+DISTRO_VERSION = "2.5.3"
 DISTRO_CODENAME = "sumo"
 SDK_VENDOR = "-pokysdk"
 SDK_VERSION := "${@'${DISTRO_VERSION}'.replace('snapshot-${DATE}','snapshot')}"

meta-selftest/lib/oeqa/runtime/cases/dnf_runtime.py

@@ -27,7 +27,7 @@ class DnfSelftest(DnfTest):
         Expected: 1. Feeds were correctly set for dnf
                   2. Update recovers packages from host's repo
         Author: Humberto Ibarra <humberto.ibarra.lopez@intel.com>
-        Author: Alexander Kanavin <alexander.kanavin@intel.com>
+        Author: Alexander Kanavin <alex.kanavin@gmail.com>
         """
         # When we created an image, we had to supply fake ip and port
         # for the feeds. Now we can patch the real ones into the config file.

meta-selftest/recipes-test/delay/delay.bb

@@ -0,0 +1,12 @@
+SUMMARY = "Recipe with a fixed delay task"
+DESCRIPTION = "Contains a delay task to be used to for testing."
+LICENSE = "MIT"
+
+INHIBIT_DEFAULT_DEPS = "1"
+EXCLUDE_FROM_WORLD = "1"
+
+do_delay() {
+	sleep 5
+}
+do_delay[nostamp] = "1"
+addtask delay

meta/classes/archiver.bbclass

@@ -40,8 +40,11 @@ ARCHIVER_MODE[recipe] ?= "0"
 DEPLOY_DIR_SRC ?= "${DEPLOY_DIR}/sources"
 ARCHIVER_TOPDIR ?= "${WORKDIR}/deploy-sources"
 ARCHIVER_OUTDIR = "${ARCHIVER_TOPDIR}/${TARGET_SYS}/${PF}/"
+ARCHIVER_RPMTOPDIR ?= "${WORKDIR}/deploy-sources-rpm"
+ARCHIVER_RPMOUTDIR = "${ARCHIVER_RPMTOPDIR}/${TARGET_SYS}/${PF}/"
 ARCHIVER_WORKDIR = "${WORKDIR}/archiver-work/"
 
+
 do_dumpdata[dirs] = "${ARCHIVER_OUTDIR}"
 do_ar_recipe[dirs] = "${ARCHIVER_OUTDIR}"
 do_ar_original[dirs] = "${ARCHIVER_OUTDIR} ${ARCHIVER_WORKDIR}"
@@ -117,6 +120,9 @@ python () {
     if d.getVarFlag('ARCHIVER_MODE', 'srpm') == "1" and d.getVar('PACKAGES'):
         if "package_rpm" in d.getVar('PACKAGE_CLASSES'):
             d.appendVarFlag('do_deploy_archives', 'depends', ' %s:do_package_write_rpm' % pn)
+            d.appendVarFlag('do_package_write_rpm', 'dirs', ' ${ARCHIVER_RPMTOPDIR}')
+            d.appendVarFlag('do_package_write_rpm', 'sstate-inputdirs', ' ${ARCHIVER_RPMTOPDIR}')
+            d.appendVarFlag('do_package_write_rpm', 'sstate-outputdirs', ' ${DEPLOY_DIR_SRC}')
             if ar_dumpdata == "1":
                 d.appendVarFlag('do_package_write_rpm', 'depends', ' %s:do_dumpdata' % pn)
             if ar_recipe == "1":

meta/classes/image-buildinfo.bbclass

@@ -64,6 +64,8 @@ def buildinfo_target(d):
 
 # Write build information to target filesystem
 python buildinfo () {
+    if not d.getVar('IMAGE_BUILDINFO_FILE'):
+        return
     with open(d.expand('${IMAGE_ROOTFS}${IMAGE_BUILDINFO_FILE}'), 'w') as build:
         build.writelines((
             '''-----------------------

meta/classes/image_types.bbclass

@@ -239,6 +239,7 @@ EXTRA_IMAGECMD_ext4 ?= "-i 4096"
 EXTRA_IMAGECMD_btrfs ?= "-n 4096"
 EXTRA_IMAGECMD_f2fs ?= ""
 
+do_image_cpio[depends] += "cpio-native:do_populate_sysroot"
 do_image_jffs2[depends] += "mtd-utils-native:do_populate_sysroot"
 do_image_cramfs[depends] += "util-linux-native:do_populate_sysroot"
 do_image_ext2[depends] += "e2fsprogs-native:do_populate_sysroot"

meta/classes/kernel.bbclass

@@ -152,8 +152,8 @@ PACKAGES_DYNAMIC += "^${KERNEL_PACKAGE_NAME}-firmware-.*"
 export OS = "${TARGET_OS}"
 export CROSS_COMPILE = "${TARGET_PREFIX}"
 export KBUILD_BUILD_VERSION = "1"
-export KBUILD_BUILD_USER = "oe-user"
-export KBUILD_BUILD_HOST = "oe-host"
+export KBUILD_BUILD_USER ?= "oe-user"
+export KBUILD_BUILD_HOST ?= "oe-host"
 
 KERNEL_RELEASE ?= "${KERNEL_VERSION}"
 

meta/classes/multilib_script.bbclass

@@ -0,0 +1,32 @@
+#
+# Recipe needs to set MULTILIB_SCRIPTS in the form <pkgname>:<scriptname>, e.g.
+# MULTILIB_SCRIPTS = "${PN}-dev:${bindir}/file1 ${PN}:${base_bindir}/file2"
+# to indicate which script files to process from which packages.
+#
+
+inherit update-alternatives
+
+MULTILIB_SUFFIX = "${@d.getVar('base_libdir',1).split('/')[-1]}"
+
+PACKAGE_PREPROCESS_FUNCS += "multilibscript_rename"
+
+multilibscript_rename() {
+	:
+}
+
+python () {
+    # Do nothing if multilib isn't being used
+    if not d.getVar("MULTILIB_VARIANTS"):
+       return
+    # Do nothing for native/cross
+    if bb.data.inherits_class('native', d) or bb.data.inherits_class('cross', d):
+       return
+
+    for entry in (d.getVar("MULTILIB_SCRIPTS", False) or "").split():
+       pkg, script = entry.split(":")
+       scriptname = os.path.basename(script)
+       d.setVar("ALTERNATIVE_" + pkg, scriptname)
+       d.setVarFlag("ALTERNATIVE_LINK_NAME", scriptname, script)
+       d.setVarFlag("ALTERNATIVE_TARGET", scriptname, script + "-${MULTILIB_SUFFIX}")
+       d.appendVar("multilibscript_rename",  "\n	mv ${PKGD}" + script + " ${PKGD}" + script + "-${MULTILIB_SUFFIX}")
+}

meta/classes/package.bbclass

@@ -359,7 +359,8 @@ def append_source_info(file, sourcefile, d, fatal=True):
     # of rpm's debugedit, which was writing them out that way, and the code elsewhere
     # is still assuming that.
     debuglistoutput = '\0'.join(debugsources) + '\0'
-    open(sourcefile, 'a').write(debuglistoutput)
+    with open(sourcefile, 'a') as sf:
+        sf.write(debuglistoutput)
 
 
 def splitdebuginfo(file, debugfile, debugsrcdir, sourcefile, d):
@@ -1411,9 +1412,6 @@ fi
     pkgdest = d.getVar('PKGDEST')
     pkgdatadir = d.getVar('PKGDESTWORK')
 
-    # Take shared lock since we're only reading, not writing
-    lf = bb.utils.lockfile(d.expand("${PACKAGELOCK}"), True)
-
     data_file = pkgdatadir + d.expand("/${PN}" )
     f = open(data_file, 'w')
     f.write("PACKAGES: %s\n" % packages)
@@ -1518,7 +1516,6 @@ fi
     if bb.data.inherits_class('allarch', d) and not bb.data.inherits_class('packagegroup', d):
         write_extra_runtime_pkgs(global_variants, packages, pkgdatadir)
 
-    bb.utils.unlockfile(lf)
 }
 emit_pkgdata[dirs] = "${PKGDESTWORK}/runtime ${PKGDESTWORK}/runtime-reverse ${PKGDESTWORK}/runtime-rprovides"
 
@@ -1617,9 +1614,6 @@ python package_do_shlibs() {
 
     shlibswork_dir = d.getVar('SHLIBSWORKDIR')
 
-    # Take shared lock since we're only reading, not writing
-    lf = bb.utils.lockfile(d.expand("${PACKAGELOCK}"), True)
-
     def linux_so(file, needed, sonames, renames, pkgver):
         needs_ldconfig = False
         ldir = os.path.dirname(file).replace(pkgdest + "/" + pkg, '')
@@ -1732,7 +1726,11 @@ python package_do_shlibs() {
     use_ldconfig = bb.utils.contains('DISTRO_FEATURES', 'ldconfig', True, False, d)
 
     needed = {}
+
+    # Take shared lock since we're only reading, not writing
+    lf = bb.utils.lockfile(d.expand("${PACKAGELOCK}"), True)
     shlib_provider = oe.package.read_shlib_providers(d)
+    bb.utils.unlockfile(lf)
 
     for pkg in packages.split():
         private_libs = d.getVar('PRIVATE_LIBS_' + pkg) or d.getVar('PRIVATE_LIBS') or ""
@@ -1788,8 +1786,6 @@ python package_do_shlibs() {
             d.setVar('pkg_postinst_%s' % pkg, postinst)
         bb.debug(1, 'LIBNAMES: pkg %s sonames %s' % (pkg, sonames))
 
-    bb.utils.unlockfile(lf)
-
     assumed_libs = d.getVar('ASSUME_SHLIBS')
     if assumed_libs:
         libdir = d.getVar("libdir")
@@ -1902,9 +1898,6 @@ python package_do_pkgconfig () {
                             if hdr == 'Requires':
                                 pkgconfig_needed[pkg] += exp.replace(',', ' ').split()
 
-    # Take shared lock since we're only reading, not writing
-    lf = bb.utils.lockfile(d.expand("${PACKAGELOCK}"), True)
-
     for pkg in packages.split():
         pkgs_file = os.path.join(shlibswork_dir, pkg + ".pclist")
         if pkgconfig_provided[pkg] != []:
@@ -1913,6 +1906,9 @@ python package_do_pkgconfig () {
                 f.write('%s\n' % p)
             f.close()
 
+    # Take shared lock since we're only reading, not writing
+    lf = bb.utils.lockfile(d.expand("${PACKAGELOCK}"), True)
+
     # Go from least to most specific since the last one found wins
     for dir in reversed(shlibs_dirs):
         if not os.path.exists(dir):
@@ -1928,6 +1924,8 @@ python package_do_pkgconfig () {
                 for l in lines:
                     pkgconfig_provided[pkg].append(l.rstrip())
 
+    bb.utils.unlockfile(lf)
+
     for pkg in packages.split():
         deps = []
         for n in pkgconfig_needed[pkg]:
@@ -1945,8 +1943,6 @@ python package_do_pkgconfig () {
             for dep in deps:
                 fd.write(dep + '\n')
             fd.close()
-
-    bb.utils.unlockfile(lf)
 }
 
 def read_libdep_files(d):

meta/classes/package_rpm.bbclass

@@ -113,6 +113,10 @@ python write_specfile () {
             source_list = os.listdir(ar_outdir)
             source_number = 0
             for source in source_list:
+                # do_deploy_archives may have already run (from sstate) meaning a .src.rpm may already 
+                # exist in ARCHIVER_OUTDIR so skip if present.
+                if source.endswith(".src.rpm"):
+                    continue
                 # The rpmbuild doesn't need the root permission, but it needs
                 # to know the file's user and group name, the only user and
                 # group in fakeroot is "root" when working in fakeroot.
@@ -690,7 +694,7 @@ python do_package_rpm () {
     cmd = cmd + " --define '_tmppath " + workdir + "'"
     if d.getVarFlag('ARCHIVER_MODE', 'srpm') == '1' and bb.data.inherits_class('archiver', d):
         cmd = cmd + " --define '_sourcedir " + d.getVar('ARCHIVER_OUTDIR') + "'"
-        cmdsrpm = cmd + " --define '_srcrpmdir " + d.getVar('ARCHIVER_OUTDIR') + "'"
+        cmdsrpm = cmd + " --define '_srcrpmdir " + d.getVar('ARCHIVER_RPMOUTDIR') + "'"
         cmdsrpm = cmdsrpm + " -bs " + outspecfile
         # Build the .src.rpm
         d.setVar('SBUILDSPEC', cmdsrpm + "\n")

meta/classes/patch.bbclass

@@ -153,6 +153,7 @@ python patch_do_patch() {
 patch_do_patch[vardepsexclude] = "PATCHRESOLVE"
 
 addtask patch after do_unpack
+do_patch[umask] = "022"
 do_patch[dirs] = "${WORKDIR}"
 do_patch[depends] = "${PATCHDEPENDENCY}"
 

meta/classes/populate_sdk_ext.bbclass

@@ -325,8 +325,9 @@ python copy_buildsystem () {
             f.write('TCLIBCAPPEND = ""\n')
             f.write('DL_DIR = "${TOPDIR}/downloads"\n')
 
-            f.write('INHERIT += "%s"\n' % 'uninative')
-            f.write('UNINATIVE_CHECKSUM[%s] = "%s"\n\n' % (d.getVar('BUILD_ARCH'), uninative_checksum))
+            if bb.data.inherits_class('uninative', d):
+               f.write('INHERIT += "%s"\n' % 'uninative')
+               f.write('UNINATIVE_CHECKSUM[%s] = "%s"\n\n' % (d.getVar('BUILD_ARCH'), uninative_checksum))
             f.write('CONF_VERSION = "%s"\n\n' % d.getVar('CONF_VERSION', False))
 
             # Some classes are not suitable for SDK, remove them from INHERIT

meta/classes/testimage.bbclass

@@ -2,7 +2,7 @@
 #
 # Released under the MIT license (see COPYING.MIT)
 
-
+inherit metadata_scm
 # testimage.bbclass enables testing of qemu images using python unittests.
 # Most of the tests are commands run on target image over ssh.
 # To use it add testimage to global inherit and call your target image with -c testimage
@@ -26,6 +26,7 @@
 # TEST_LOG_DIR contains a command ssh log and may contain infromation about what command is running, output and return codes and for qemu a boot log till login.
 # Booting is handled by this class, and it's not a test in itself.
 # TEST_QEMUBOOT_TIMEOUT can be used to set the maximum time in seconds the launch code will wait for the login prompt.
+# TEST_QEMUPARAMS can be used to pass extra parameters to qemu, e.g. "-m 1024" for setting the amount of ram to 1 GB.
 
 TEST_LOG_DIR ?= "${WORKDIR}/testimage"
 
@@ -35,26 +36,13 @@ TEST_NEEDED_PACKAGES_DIR ?= "${WORKDIR}/testimage/packages"
 TEST_EXTRACTED_DIR ?= "${TEST_NEEDED_PACKAGES_DIR}/extracted"
 TEST_PACKAGED_DIR ?= "${TEST_NEEDED_PACKAGES_DIR}/packaged"
 
-RPMTESTSUITE = "${@bb.utils.contains('IMAGE_PKGTYPE', 'rpm', 'dnf rpm', '', d)}"
-SYSTEMDSUITE = "${@bb.utils.filter('DISTRO_FEATURES', 'systemd', d)}"
-MINTESTSUITE = "ping"
-NETTESTSUITE = "${MINTESTSUITE} ssh df date scp oe_syslog ${SYSTEMDSUITE}"
-DEVTESTSUITE = "gcc kernelmodule ldd"
+BASICTESTSUITE = "\
+    ping date df ssh scp python perl gi ptest parselogs \
+    logrotate connman systemd oe_syslog pam stap ldd xorg \
+    kernelmodule gcc buildcpio buildlzip buildgalculator \
+    dnf rpm opkg apt"
 
-DEFAULT_TEST_SUITES = "${MINTESTSUITE} auto"
-DEFAULT_TEST_SUITES_pn-core-image-minimal = "${MINTESTSUITE}"
-DEFAULT_TEST_SUITES_pn-core-image-minimal-dev = "${MINTESTSUITE}"
-DEFAULT_TEST_SUITES_pn-core-image-full-cmdline = "${NETTESTSUITE} perl python logrotate ptest"
-DEFAULT_TEST_SUITES_pn-core-image-x11 = "${MINTESTSUITE}"
-DEFAULT_TEST_SUITES_pn-core-image-lsb = "${NETTESTSUITE} pam parselogs ${RPMTESTSUITE} ptest"
-DEFAULT_TEST_SUITES_pn-core-image-sato = "${NETTESTSUITE} connman xorg parselogs ${RPMTESTSUITE} \
-    ${@bb.utils.contains('IMAGE_PKGTYPE', 'rpm', 'python', '', d)} ptest gi"
-DEFAULT_TEST_SUITES_pn-core-image-sato-sdk = "${NETTESTSUITE} buildcpio buildlzip buildgalculator \
-    connman ${DEVTESTSUITE} logrotate perl parselogs python ${RPMTESTSUITE} xorg ptest gi stap"
-DEFAULT_TEST_SUITES_pn-core-image-lsb-dev = "${NETTESTSUITE} pam perl python parselogs ${RPMTESTSUITE} ptest gi"
-DEFAULT_TEST_SUITES_pn-core-image-lsb-sdk = "${NETTESTSUITE} buildcpio buildlzip buildgalculator \
-    connman ${DEVTESTSUITE} logrotate pam parselogs perl python ${RPMTESTSUITE} ptest gi stap"
-DEFAULT_TEST_SUITES_pn-meta-toolchain = "auto"
+DEFAULT_TEST_SUITES = "${BASICTESTSUITE}"
 
 # aarch64 has no graphics
 DEFAULT_TEST_SUITES_remove_aarch64 = "xorg"
@@ -71,6 +59,7 @@ TEST_SUITES ?= "${DEFAULT_TEST_SUITES}"
 
 TEST_QEMUBOOT_TIMEOUT ?= "1000"
 TEST_TARGET ?= "qemu"
+TEST_QEMUPARAMS ?= ""
 
 TESTIMAGEDEPENDS = ""
 TESTIMAGEDEPENDS_qemuall = "qemu-native:do_populate_sysroot qemu-helper-native:do_populate_sysroot qemu-helper-native:do_addto_recipe_sysroot"
@@ -85,7 +74,7 @@ TESTIMAGEDEPENDS += "${@bb.utils.contains('IMAGE_PKGTYPE', 'rpm', 'createrepo-c-
 TESTIMAGELOCK = "${TMPDIR}/testimage.lock"
 TESTIMAGELOCK_qemuall = ""
 
-TESTIMAGE_DUMP_DIR ?= "/tmp/oe-saved-tests/"
+TESTIMAGE_DUMP_DIR ?= "${LOG_DIR}/runtime-hostdump/"
 
 TESTIMAGE_UPDATE_VARS ?= "DL_DIR WORKDIR DEPLOY_DIR"
 
@@ -132,6 +121,30 @@ def testimage_sanity(d):
         bb.fatal('When TEST_TARGET is set to "simpleremote" '
                  'TEST_TARGET_IP and TEST_SERVER_IP are needed too.')
 
+def get_testimage_configuration(d, test_type, machine):
+    import platform
+    from oeqa.utils.metadata import get_layers
+    configuration = {'TEST_TYPE': test_type,
+                    'MACHINE': machine,
+                    'DISTRO': d.getVar("DISTRO"),
+                    'IMAGE_BASENAME': d.getVar("IMAGE_BASENAME"),
+                    'IMAGE_PKGTYPE': d.getVar("IMAGE_PKGTYPE"),
+                    'STARTTIME': d.getVar("DATETIME"),
+                    'HOST_DISTRO': oe.lsb.distro_identifier().replace(' ', '-'),
+                    'LAYERS': get_layers(d.getVar("BBLAYERS"))}
+    return configuration
+get_testimage_configuration[vardepsexclude] = "DATETIME"
+
+def get_testimage_json_result_dir(d):
+    json_result_dir = os.path.join(d.getVar("LOG_DIR"), 'oeqa')
+    custom_json_result_dir = d.getVar("OEQA_JSON_RESULT_DIR")
+    if custom_json_result_dir:
+        json_result_dir = custom_json_result_dir
+    return json_result_dir
+
+def get_testimage_result_id(configuration):
+    return '%s_%s_%s_%s' % (configuration['TEST_TYPE'], configuration['IMAGE_BASENAME'], configuration['MACHINE'], configuration['STARTTIME'])
+
 def testimage_main(d):
     import os
     import json
@@ -185,12 +198,13 @@ def testimage_main(d):
     machine = d.getVar("MACHINE")
 
     # Get rootfs
-    fstypes = [fs for fs in d.getVar('IMAGE_FSTYPES').split(' ')
-                  if fs in supported_fstypes]
-    if not fstypes:
-        bb.fatal('Unsupported image type built. Add a comptible image to '
-                 'IMAGE_FSTYPES. Supported types: %s' %
-                 ', '.join(supported_fstypes))
+    fstypes = d.getVar('IMAGE_FSTYPES').split()
+    if d.getVar("TEST_TARGET") == "qemu":
+        fstypes = [fs for fs in fstypes if fs in supported_fstypes]
+        if not fstypes:
+            bb.fatal('Unsupported image type built. Add a comptible image to '
+                     'IMAGE_FSTYPES. Supported types: %s' %
+                     ', '.join(supported_fstypes))
     rootfs = '%s.%s' % (image_name, fstypes[0])
 
     # Get tmpdir (not really used, just for compatibility)
@@ -214,13 +228,11 @@ def testimage_main(d):
     boottime = int(d.getVar("TEST_QEMUBOOT_TIMEOUT"))
 
     # Get use_kvm
-    qemu_use_kvm = d.getVar("QEMU_USE_KVM")
-    if qemu_use_kvm and \
-       (oe.types.boolean(qemu_use_kvm) and 'x86' in machine or \
-        d.getVar('MACHINE') in qemu_use_kvm.split()):
-        kvm = True
-    else:
-        kvm = False
+    kvm = oe.types.qemu_use_kvm(d.getVar('QEMU_USE_KVM'), d.getVar('TARGET_ARCH'))
+
+    slirp = False
+    if d.getVar("QEMU_USE_SLIRP"):
+        slirp = True
 
     # TODO: We use the current implementatin of qemu runner because of
     # time constrains, qemu runner really needs a refactor too.
@@ -233,6 +245,8 @@ def testimage_main(d):
                       'boottime'    : boottime,
                       'bootlog'     : bootlog,
                       'kvm'         : kvm,
+                      'slirp'       : slirp,
+                      'dump_dir'    : d.getVar("TESTIMAGE_DUMP_DIR"),
                     }
 
     # TODO: Currently BBPATH is needed for custom loading of targets.
@@ -272,17 +286,12 @@ def testimage_main(d):
 
     package_extraction(d, tc.suites)
 
-    bootparams = None
-    if d.getVar('VIRTUAL-RUNTIME_init_manager', '') == 'systemd':
-        # Add systemd.log_level=debug to enable systemd debug logging
-        bootparams = 'systemd.log_target=console'
-
     results = None
     orig_sigterm_handler = signal.signal(signal.SIGTERM, sigterm_exception)
     try:
         # We need to check if runqemu ends unexpectedly
         # or if the worker send us a SIGTERM
-        tc.target.start(extra_bootparams=bootparams)
+        tc.target.start(params=d.getVar("TEST_QEMUPARAMS"))
         results = tc.runTests()
     except (RuntimeError, BlockingIOError) as err:
         if isinstance(err, RuntimeError):
@@ -299,7 +308,10 @@ def testimage_main(d):
     # Show results (if we have them)
     if not results:
         bb.fatal('%s - FAILED - tests were interrupted during execution' % pn, forcelog=True)
-    results.logDetails()
+    configuration = get_testimage_configuration(d, 'runtime', machine)
+    results.logDetails(get_testimage_json_result_dir(d),
+                       configuration,
+                       get_testimage_result_id(configuration))
     results.logSummary(pn)
     if not results.wasSuccessful():
         bb.fatal('%s - FAILED - check the task log and the ssh log' % pn, forcelog=True)
@@ -336,6 +348,7 @@ def create_index(arg):
     return None
 
 def create_rpm_index(d):
+    import glob
     # Index RPMs
     rpm_createrepo = bb.utils.which(os.getenv('PATH'), "createrepo_c")
     index_cmds = []
@@ -352,9 +365,13 @@ def create_rpm_index(d):
         lf = bb.utils.lockfile(lockfilename, False)
         oe.path.copyhardlinktree(rpm_dir, idx_path)
         # Full indexes overload a 256MB image so reduce the number of rpms
-        # in the feed. Filter to r* since we use the run-postinst packages and
-        # this leaves some allarch and machine arch packages too.
-        bb.utils.remove(idx_path + "*/[a-qs-z]*.rpm")
+        # in the feed by filtering to specific packages needed by the tests.
+        package_list = glob.glob(idx_path + "*/*.rpm")
+
+        for pkg in package_list:
+            if not os.path.basename(pkg).startswith(("rpm", "run-postinsts", "busybox", "bash", "update-alternatives", "libc6", "curl", "musl")):
+                bb.utils.remove(pkg)
+
         bb.utils.unlockfile(lf)
         cmd = '%s --update -q %s' % (rpm_createrepo, idx_path)
 

meta/classes/testsdk.bbclass

@@ -14,6 +14,30 @@
 #
 # where "<image-name>" is an image like core-image-sato.
 
+def get_sdk_configuration(d, test_type):
+    import platform
+    from oeqa.utils.metadata import get_layers
+    configuration = {'TEST_TYPE': test_type,
+                    'MACHINE': d.getVar("MACHINE"),
+                    'SDKMACHINE': d.getVar("SDKMACHINE"),
+                    'IMAGE_BASENAME': d.getVar("IMAGE_BASENAME"),
+                    'IMAGE_PKGTYPE': d.getVar("IMAGE_PKGTYPE"),
+                    'STARTTIME': d.getVar("DATETIME"),
+                    'HOST_DISTRO': oe.lsb.distro_identifier().replace(' ', '-'),
+                    'LAYERS': get_layers(d.getVar("BBLAYERS"))}
+    return configuration
+get_sdk_configuration[vardepsexclude] = "DATETIME"
+
+def get_sdk_json_result_dir(d):
+    json_result_dir = os.path.join(d.getVar("LOG_DIR"), 'oeqa')
+    custom_json_result_dir = d.getVar("OEQA_JSON_RESULT_DIR")
+    if custom_json_result_dir:
+        json_result_dir = custom_json_result_dir
+    return json_result_dir
+
+def get_sdk_result_id(configuration):
+    return '%s_%s_%s_%s_%s' % (configuration['TEST_TYPE'], configuration['IMAGE_BASENAME'], configuration['SDKMACHINE'], configuration['MACHINE'], configuration['STARTTIME'])
+
 def testsdk_main(d):
     import os
     import subprocess
@@ -24,8 +48,6 @@ def testsdk_main(d):
     from oeqa.sdk.context import OESDKTestContext, OESDKTestContextExecutor
     from oeqa.utils import make_logger_bitbake_compatible
 
-    bb.event.enable_threadlock()
-
     pn = d.getVar("PN")
     logger = make_logger_bitbake_compatible(logging.getLogger("BitBake"))
 
@@ -71,8 +93,10 @@ def testsdk_main(d):
 
         component = "%s %s" % (pn, OESDKTestContextExecutor.name)
         context_msg = "%s:%s" % (os.path.basename(tcname), os.path.basename(sdk_env))
-
-        result.logDetails()
+        configuration = get_sdk_configuration(d, 'sdk')
+        result.logDetails(get_sdk_json_result_dir(d),
+                          configuration,
+                          get_sdk_result_id(configuration))
         result.logSummary(component, context_msg)
 
         if not result.wasSuccessful():
@@ -99,8 +123,6 @@ def testsdkext_main(d):
     from oeqa.utils import avoid_paths_in_environ, make_logger_bitbake_compatible, subprocesstweak
     from oeqa.sdkext.context import OESDKExtTestContext, OESDKExtTestContextExecutor
 
-    bb.event.enable_threadlock()
-
     pn = d.getVar("PN")
     logger = make_logger_bitbake_compatible(logging.getLogger("BitBake"))
 
@@ -176,8 +198,10 @@ def testsdkext_main(d):
 
         component = "%s %s" % (pn, OESDKExtTestContextExecutor.name)
         context_msg = "%s:%s" % (os.path.basename(tcname), os.path.basename(sdk_env))
-
-        result.logDetails()
+        configuration = get_sdk_configuration(d, 'sdkext')
+        result.logDetails(get_sdk_json_result_dir(d),
+                          configuration,
+                          get_sdk_result_id(configuration))
         result.logSummary(component, context_msg)
 
         if not result.wasSuccessful():

meta/conf/distro/include/default-versions.inc

@@ -2,6 +2,6 @@
 # Default preferred versions
 #
 
-PREFERRED_VERSION_openssl = "1.0.%"
-PREFERRED_VERSION_openssl-native = "1.0.%"
-PREFERRED_VERSION_nativesdk-openssl = "1.0.%"
+PREFERRED_VERSION_openssl ?= "1.0.%"
+PREFERRED_VERSION_openssl-native ?= "1.0.%"
+PREFERRED_VERSION_nativesdk-openssl ?= "1.0.%"

meta/conf/distro/include/yocto-uninative.inc

@@ -6,9 +6,8 @@
 # to the distro running on the build machine.
 #
 
-UNINATIVE_MAXGLIBCVERSION = "2.28"
-
-UNINATIVE_URL ?= "http://downloads.yoctoproject.org/releases/uninative/2.3/"
-UNINATIVE_CHECKSUM[i686] ?= "44253cddbf629082568cea4fff59419106871a0cf81b4845b5d34e7014887b20"
-UNINATIVE_CHECKSUM[x86_64] ?= "c6954563dad3c95608117c6fc328099036c832bbd924ebf5fdccb622fc0a8684"
+UNINATIVE_MAXGLIBCVERSION = "2.29"
 
+UNINATIVE_URL ?= "http://downloads.yoctoproject.org/releases/uninative/2.4/"
+UNINATIVE_CHECKSUM[i686] ?= "fafacfc537a6ce2bd122bd16c146881ab5ac69bd575abf6cb68a0dd33fa70ea2"
+UNINATIVE_CHECKSUM[x86_64] ?= "06f91685b782f2ccfedf3070b3ba0fe4a5ba2f0766dad5c9d1642dccf95accd0"

meta/lib/oe/types.py

@@ -151,3 +151,27 @@ def path(value, relativeto='', normalize='true', mustexist='false'):
                 raise ValueError("{0}: {1}".format(value, os.strerror(errno.ENOENT)))
 
     return value
+
+def is_x86(arch):
+    """
+    Check whether arch is x86 or x86_64
+    """
+    if arch.startswith('x86_') or re.match('i.*86', arch):
+        return True
+    else:
+        return False
+
+def qemu_use_kvm(kvm, target_arch):
+    """
+    Enable kvm if target_arch == build_arch or both of them are x86 archs.
+    """
+
+    use_kvm = False
+    if kvm and boolean(kvm):
+        build_arch = os.uname()[4]
+        if is_x86(build_arch) and is_x86(target_arch):
+            use_kvm = True
+        elif build_arch == target_arch:
+            use_kvm = True
+    return use_kvm
+

meta/lib/oeqa/core/context.py

@@ -27,7 +27,6 @@ class OETestContext(object):
         self.logger = logger
         self._registry = {}
         self._registry['cases'] = collections.OrderedDict()
-        self._results = {}
 
     def _read_modules_from_manifest(self, manifest):
         if not os.path.exists(manifest):

meta/lib/oeqa/core/decorator/depends.py

@@ -3,7 +3,6 @@
 
 from unittest import SkipTest
 
-from oeqa.core.threaded import OETestRunnerThreaded
 from oeqa.core.exception import OEQADependency
 
 from . import OETestDiscover, registerDecorator
@@ -64,16 +63,10 @@ def _order_test_case_by_depends(cases, depends):
     return [cases[case_id] for case_id in cases_ordered]
 
 def _skipTestDependency(case, depends):
-    if isinstance(case.tc.runner, OETestRunnerThreaded):
-        import threading
-        results = case.tc._results[threading.get_ident()]
-    else:
-        results = case.tc._results
-
     skipReasons = ['errors', 'failures', 'skipped']
 
     for reason in skipReasons:
-        for test, _ in results[reason]:
+        for test, _ in getattr(case.tc.results, reason):
             if test.id() in depends:
                 raise SkipTest("Test case %s depends on %s and was in %s." \
                         % (case.id(), test.id(), reason))

meta/lib/oeqa/core/decorator/oetimeout.py

@@ -1,12 +1,8 @@
 # Copyright (C) 2016 Intel Corporation
 # Released under the MIT license (see COPYING.MIT)
 
-from . import OETestDecorator, registerDecorator
-
 import signal
-from threading import Timer
-
-from oeqa.core.threaded import OETestRunnerThreaded
+from . import OETestDecorator, registerDecorator
 from oeqa.core.exception import OEQATimeoutError
 
 @registerDecorator
@@ -14,32 +10,16 @@ class OETimeout(OETestDecorator):
     attrs = ('oetimeout',)
 
     def setUpDecorator(self):
-        self.logger.debug("Setting up a %d second(s) timeout" % self.oetimeout)
-
-        if isinstance(self.case.tc.runner, OETestRunnerThreaded):
-            self.timeouted = False
-            def _timeoutHandler():
-                self.timeouted = True
-
-            self.timer = Timer(self.oetimeout, _timeoutHandler)
-            self.timer.start()
-        else:
-            timeout = self.oetimeout
-            def _timeoutHandler(signum, frame):
-                raise OEQATimeoutError("Timed out after %s "
+        timeout = self.oetimeout
+        def _timeoutHandler(signum, frame):
+            raise OEQATimeoutError("Timed out after %s "
                     "seconds of execution" % timeout)
 
-            self.alarmSignal = signal.signal(signal.SIGALRM, _timeoutHandler)
-            signal.alarm(self.oetimeout)
+        self.logger.debug("Setting up a %d second(s) timeout" % self.oetimeout)
+        self.alarmSignal = signal.signal(signal.SIGALRM, _timeoutHandler)
+        signal.alarm(self.oetimeout)
 
     def tearDownDecorator(self):
-        if isinstance(self.case.tc.runner, OETestRunnerThreaded):
-            self.timer.cancel()
-            self.logger.debug("Removed Timer handler")
-            if self.timeouted:
-                raise OEQATimeoutError("Timed out after %s "
-                    "seconds of execution" % self.oetimeout)
-        else:
-            signal.alarm(0)
-            signal.signal(signal.SIGALRM, self.alarmSignal)
-            self.logger.debug("Removed SIGALRM handler")
+        signal.alarm(0)
+        signal.signal(signal.SIGALRM, self.alarmSignal)
+        self.logger.debug("Removed SIGALRM handler")

meta/lib/oeqa/core/loader.py

@@ -24,7 +24,7 @@ from oeqa.core.decorator import decoratorClasses, OETestDecorator, \
 # Generate the function definition because this differ across python versions
 # Python >= 3.4.4 uses tree parameters instead four but for example Python 3.5.3
 # ueses four parameters so isn't incremental.
-_failed_test_args = inspect.getargspec(unittest.loader._make_failed_test).args
+_failed_test_args = inspect.getfullargspec(unittest.loader._make_failed_test).args
 exec("""def _make_failed_test(%s): raise exception""" % ', '.join(_failed_test_args))
 unittest.loader._make_failed_test = _make_failed_test
 

meta/lib/oeqa/core/runner.py

@@ -6,17 +6,10 @@ import time
 import unittest
 import logging
 import re
+import json
 
-xmlEnabled = False
-try:
-    import xmlrunner
-    from xmlrunner.result import _XMLTestResult as _TestResult
-    from xmlrunner.runner import XMLTestRunner as _TestRunner
-    xmlEnabled = True
-except ImportError:
-    # use the base runner instead
-    from unittest import TextTestResult as _TestResult
-    from unittest import TextTestRunner as _TestRunner
+from unittest import TextTestResult as _TestResult
+from unittest import TextTestRunner as _TestRunner
 
 class OEStreamLogger(object):
     def __init__(self, logger):
@@ -42,8 +35,12 @@ class OETestResult(_TestResult):
     def __init__(self, tc, *args, **kwargs):
         super(OETestResult, self).__init__(*args, **kwargs)
 
+        self.successes = []
+
+        # Inject into tc so that TestDepends decorator can see results
+        tc.results = self
+
         self.tc = tc
-        self._tc_map_results()
 
     def startTest(self, test):
         # Allow us to trigger the testcase buffer mode on a per test basis
@@ -53,12 +50,6 @@ class OETestResult(_TestResult):
             self.buffer = test.buffer
         super(OETestResult, self).startTest(test)
 
-    def _tc_map_results(self):
-        self.tc._results['failures'] = self.failures
-        self.tc._results['errors'] = self.errors
-        self.tc._results['skipped'] = self.skipped
-        self.tc._results['expectedFailures'] = self.expectedFailures
-
     def logSummary(self, component, context_msg=''):
         elapsed_time = self.tc._run_end_time - self.tc._run_start_time
         self.tc.logger.info("SUMMARY:")
@@ -70,67 +61,60 @@ class OETestResult(_TestResult):
             msg = "%s - OK - All required tests passed" % component
         else:
             msg = "%s - FAIL - Required tests failed" % component
-        skipped = len(self.tc._results['skipped'])
-        if skipped: 
-            msg += " (skipped=%d)" % skipped
+        msg += " (successes=%d, skipped=%d, failures=%d, errors=%d)" % (len(self.successes), len(self.skipped), len(self.failures), len(self.errors))
         self.tc.logger.info(msg)
 
-    def _getDetailsNotPassed(self, case, type, desc):
-        found = False
+    def _getTestResultDetails(self, case):
+        result_types = {'failures': 'FAILED', 'errors': 'ERROR', 'skipped': 'SKIPPED',
+                        'expectedFailures': 'EXPECTEDFAIL', 'successes': 'PASSED'}
 
-        for (scase, msg) in self.tc._results[type]:
-            # XXX: When XML reporting is enabled scase is
-            # xmlrunner.result._TestInfo instance instead of
-            # string.
-            if xmlEnabled:
-                if case.id() == scase.test_id:
+        for rtype in result_types:
+            found = False
+            for (scase, msg) in getattr(self, rtype):
+                if case.id() == scase.id():
                     found = True
                     break
-                scase_str = scase.test_id
-            else:
-                if case == scase:
-                    found = True
-                    break
-                scase_str = str(scase)
+                scase_str = str(scase.id())
 
-            # When fails at module or class level the class name is passed as string
-            # so figure out to see if match
-            m = re.search("^setUpModule \((?P<module_name>.*)\)$", scase_str)
-            if m:
-                if case.__class__.__module__ == m.group('module_name'):
-                    found = True
-                    break
+                # When fails at module or class level the class name is passed as string
+                # so figure out to see if match
+                m = re.search("^setUpModule \((?P<module_name>.*)\)$", scase_str)
+                if m:
+                    if case.__class__.__module__ == m.group('module_name'):
+                        found = True
+                        break
 
-            m = re.search("^setUpClass \((?P<class_name>.*)\)$", scase_str)
-            if m:
-                class_name = "%s.%s" % (case.__class__.__module__,
-                        case.__class__.__name__)
+                m = re.search("^setUpClass \((?P<class_name>.*)\)$", scase_str)
+                if m:
+                    class_name = "%s.%s" % (case.__class__.__module__,
+                                            case.__class__.__name__)
 
-                if class_name == m.group('class_name'):
-                    found = True
-                    break
+                    if class_name == m.group('class_name'):
+                        found = True
+                        break
 
-        if found:
-            return (found, msg)
+            if found:
+                return result_types[rtype], msg
 
-        return (found, None)
+        return 'UNKNOWN', None
 
-    def logDetails(self):
+    def addSuccess(self, test):
+        #Added so we can keep track of successes too
+        self.successes.append((test, None))
+        super(OETestResult, self).addSuccess(test)
+
+    def logDetails(self, json_file_dir=None, configuration=None, result_id=None):
         self.tc.logger.info("RESULTS:")
+
+        result = {}
+        logs = {}
+        if hasattr(self.tc, "extraresults"):
+            result = self.tc.extraresults
+
         for case_name in self.tc._registry['cases']:
             case = self.tc._registry['cases'][case_name]
 
-            result_types = ['failures', 'errors', 'skipped', 'expectedFailures']
-            result_desc = ['FAILED', 'ERROR', 'SKIPPED', 'EXPECTEDFAIL']
-
-            fail = False
-            desc = None
-            for idx, name in enumerate(result_types):
-                (fail, msg) = self._getDetailsNotPassed(case, result_types[idx],
-                        result_desc[idx])
-                if fail:
-                    desc = result_desc[idx]
-                    break
+            (status, log) = self._getTestResultDetails(case)
 
             oeid = -1
             if hasattr(case, 'decorators'):
@@ -138,12 +122,27 @@ class OETestResult(_TestResult):
                     if hasattr(d, 'oeid'):
                         oeid = d.oeid
 
-            if fail:
-                self.tc.logger.info("RESULTS - %s - Testcase %s: %s" % (case.id(),
-                    oeid, desc))
+            if status not in logs:
+                logs[status] = []
+            logs[status].append("RESULTS - %s - Testcase %s: %s" % (case.id(), oeid, status))
+            if log:
+                result[case.id()] = {'status': status, 'log': log}
             else:
-                self.tc.logger.info("RESULTS - %s - Testcase %s: %s" % (case.id(),
-                    oeid, 'PASSED'))
+                result[case.id()] = {'status': status}
+
+        for i in ['PASSED', 'SKIPPED', 'EXPECTEDFAIL', 'ERROR', 'FAILED', 'UNKNOWN']:
+            if i not in logs:
+                continue
+            for l in logs[i]:
+                self.tc.logger.info(l)
+
+        if json_file_dir:
+            tresultjsonhelper = OETestResultJSONHelper()
+            tresultjsonhelper.dump_testresult_file(json_file_dir, configuration, result_id, result)
+
+    def wasSuccessful(self):
+        # Override as we unexpected successes aren't failures for us
+        return (len(self.failures) == len(self.errors) == 0)
 
 class OEListTestsResult(object):
     def wasSuccessful(self):
@@ -153,33 +152,14 @@ class OETestRunner(_TestRunner):
     streamLoggerClass = OEStreamLogger
 
     def __init__(self, tc, *args, **kwargs):
-        if xmlEnabled:
-            if not kwargs.get('output'):
-                kwargs['output'] = os.path.join(os.getcwd(),
-                        'TestResults_%s_%s' % (time.strftime("%Y%m%d%H%M%S"), os.getpid()))
-
         kwargs['stream'] = self.streamLoggerClass(tc.logger)
         super(OETestRunner, self).__init__(*args, **kwargs)
         self.tc = tc
         self.resultclass = OETestResult
 
-    # XXX: The unittest-xml-reporting package defines _make_result method instead
-    # of _makeResult standard on unittest.
-    if xmlEnabled:
-        def _make_result(self):
-            """
-            Creates a TestResult object which will be used to store
-            information about the executed tests.
-            """
-            # override in subclasses if necessary.
-            return self.resultclass(self.tc,
-                self.stream, self.descriptions, self.verbosity, self.elapsed_times
-            )
-    else:
-        def _makeResult(self):
-            return self.resultclass(self.tc, self.stream, self.descriptions,
-                    self.verbosity)
-
+    def _makeResult(self):
+        return self.resultclass(self.tc, self.stream, self.descriptions,
+                self.verbosity)
 
     def _walk_suite(self, suite, func):
         for obj in suite:
@@ -275,3 +255,29 @@ class OETestRunner(_TestRunner):
             self._list_tests_module(suite)
 
         return OEListTestsResult()
+
+class OETestResultJSONHelper(object):
+
+    testresult_filename = 'testresults.json'
+
+    def _get_existing_testresults_if_available(self, write_dir):
+        testresults = {}
+        file = os.path.join(write_dir, self.testresult_filename)
+        if os.path.exists(file):
+            with open(file, "r") as f:
+                testresults = json.load(f)
+        return testresults
+
+    def _write_file(self, write_dir, file_name, file_content):
+        file_path = os.path.join(write_dir, file_name)
+        with open(file_path, 'w') as the_file:
+            the_file.write(file_content)
+
+    def dump_testresult_file(self, write_dir, configuration, result_id, test_result):
+        bb.utils.mkdirhier(write_dir)
+        lf = bb.utils.lockfile(os.path.join(write_dir, 'jsontestresult.lock'))
+        test_results = self._get_existing_testresults_if_available(write_dir)
+        test_results[result_id] = {'configuration': configuration, 'result': test_result}
+        json_testresults = json.dumps(test_results, sort_keys=True, indent=4)
+        self._write_file(write_dir, self.testresult_filename, json_testresults)
+        bb.utils.unlockfile(lf)

meta/lib/oeqa/core/target/qemu.py

@@ -12,15 +12,14 @@ from oeqa.utils.qemurunner import QemuRunner
 supported_fstypes = ['ext3', 'ext4', 'cpio.gz', 'wic']
 
 class OEQemuTarget(OESSHTarget):
-    def __init__(self, logger, ip, server_ip, timeout=300, user='root',
+    def __init__(self, logger, server_ip, timeout=300, user='root',
             port=None, machine='', rootfs='', kernel='', kvm=False,
             dump_dir='', dump_host_cmds='', display='', bootlog='',
             tmpdir='', dir_image='', boottime=60, **kwargs):
 
-        super(OEQemuTarget, self).__init__(logger, ip, server_ip, timeout,
+        super(OEQemuTarget, self).__init__(logger, None, server_ip, timeout,
                 user, port)
 
-        self.ip = ip
         self.server_ip = server_ip
         self.machine = machine
         self.rootfs = rootfs

meta/lib/oeqa/core/tests/cases/loader/threaded/threaded.py

@@ -1,12 +0,0 @@
-# Copyright (C) 2017 Intel Corporation
-# Released under the MIT license (see COPYING.MIT)
-
-from oeqa.core.case import OETestCase
-
-class ThreadedTest(OETestCase):
-    def test_threaded_no_depends(self):
-        self.assertTrue(True, msg='How is this possible?')
-
-class ThreadedTest2(OETestCase):
-    def test_threaded_same_module(self):
-        self.assertTrue(True, msg='How is this possible?')

meta/lib/oeqa/core/tests/cases/loader/threaded/threaded_alone.py

@@ -1,8 +0,0 @@
-# Copyright (C) 2017 Intel Corporation
-# Released under the MIT license (see COPYING.MIT)
-
-from oeqa.core.case import OETestCase
-
-class ThreadedTestAlone(OETestCase):
-    def test_threaded_alone(self):
-        self.assertTrue(True, msg='How is this possible?')

meta/lib/oeqa/core/tests/cases/loader/threaded/threaded_depends.py

@@ -1,10 +0,0 @@
-# Copyright (C) 2017 Intel Corporation
-# Released under the MIT license (see COPYING.MIT)
-
-from oeqa.core.case import OETestCase
-from oeqa.core.decorator.depends import OETestDepends
-
-class ThreadedTest3(OETestCase):
-    @OETestDepends(['threaded.ThreadedTest.test_threaded_no_depends'])
-    def test_threaded_depends(self):
-        self.assertTrue(True, msg='How is this possible?')

meta/lib/oeqa/core/tests/cases/loader/threaded/threaded_module.py

@@ -1,12 +0,0 @@
-# Copyright (C) 2017 Intel Corporation
-# Released under the MIT license (see COPYING.MIT)
-
-from oeqa.core.case import OETestCase
-
-class ThreadedTestModule(OETestCase):
-    def test_threaded_module(self):
-        self.assertTrue(True, msg='How is this possible?')
-
-class ThreadedTestModule2(OETestCase):
-    def test_threaded_module2(self):
-        self.assertTrue(True, msg='How is this possible?')

meta/lib/oeqa/core/tests/common.py

@@ -33,13 +33,3 @@ class TestBase(unittest.TestCase):
         tc.loadTests(self.cases_path, modules=modules, tests=tests,
                      filters=filters)
         return tc
-
-    def _testLoaderThreaded(self, d={}, modules=[],
-            tests=[], filters={}):
-        from oeqa.core.threaded import OETestContextThreaded
-
-        tc = OETestContextThreaded(d, self.logger)
-        tc.loadTests(self.cases_path, modules=modules, tests=tests,
-                     filters=filters)
-
-        return tc

meta/lib/oeqa/core/tests/test_data.py

@@ -21,7 +21,7 @@ class TestData(TestBase):
 
         tc = self._testLoader(modules=self.modules)
         self.assertEqual(False, tc.runTests().wasSuccessful())
-        for test, data in tc._results['errors']:
+        for test, data in tc.errors:
             expect = False
             if expectedException in data:
                 expect = True
@@ -34,7 +34,7 @@ class TestData(TestBase):
 
         tc = self._testLoader(d=d, modules=self.modules)
         self.assertEqual(False, tc.runTests().wasSuccessful())
-        for test, data in tc._results['failures']:
+        for test, data in tc.failures:
             expect = False
             if expectedError in data:
                 expect = True

meta/lib/oeqa/core/tests/test_decorators.py

@@ -131,17 +131,5 @@ class TestTimeoutDecorator(TestBase):
         msg = "OETestTimeout didn't restore SIGALRM"
         self.assertIs(alarm_signal, signal.getsignal(signal.SIGALRM), msg=msg)
 
-    def test_timeout_thread(self):
-        tests = ['timeout.TimeoutTest.testTimeoutPass']
-        msg = 'Failed to run test using OETestTimeout'
-        tc = self._testLoaderThreaded(modules=self.modules, tests=tests)
-        self.assertTrue(tc.runTests().wasSuccessful(), msg=msg)
-
-    def test_timeout_threaded_fail(self):
-        tests = ['timeout.TimeoutTest.testTimeoutFail']
-        msg = "OETestTimeout test didn't timeout as expected"
-        tc = self._testLoaderThreaded(modules=self.modules, tests=tests)
-        self.assertFalse(tc.runTests().wasSuccessful(), msg=msg)
-
 if __name__ == '__main__':
     unittest.main()

meta/lib/oeqa/core/tests/test_loader.py

@@ -1,6 +1,6 @@
 #!/usr/bin/env python3
 
-# Copyright (C) 2016-2017 Intel Corporation
+# Copyright (C) 2016 Intel Corporation
 # Released under the MIT license (see COPYING.MIT)
 
 import os
@@ -82,33 +82,5 @@ class TestLoader(TestBase):
         msg = 'Expected modules from two different paths'
         self.assertEqual(modules, expected_modules, msg=msg)
 
-    def test_loader_threaded(self):
-        cases_path = self.cases_path
-
-        self.cases_path = [os.path.join(self.cases_path, 'loader', 'threaded')]
-
-        tc = self._testLoaderThreaded()
-        self.assertEqual(len(tc.suites), 3, "Expected to be 3 suites")
-
-        case_ids = ['threaded.ThreadedTest.test_threaded_no_depends',
-                'threaded.ThreadedTest2.test_threaded_same_module',
-                'threaded_depends.ThreadedTest3.test_threaded_depends']
-        for case in tc.suites[0]._tests:
-            self.assertEqual(case.id(),
-                    case_ids[tc.suites[0]._tests.index(case)])
-
-        case_ids = ['threaded_alone.ThreadedTestAlone.test_threaded_alone']
-        for case in tc.suites[1]._tests:
-            self.assertEqual(case.id(),
-                    case_ids[tc.suites[1]._tests.index(case)])
-
-        case_ids = ['threaded_module.ThreadedTestModule.test_threaded_module',
-                'threaded_module.ThreadedTestModule2.test_threaded_module2']
-        for case in tc.suites[2]._tests:
-            self.assertEqual(case.id(),
-                    case_ids[tc.suites[2]._tests.index(case)])
-
-        self.cases_path = cases_path
-
 if __name__ == '__main__':
     unittest.main()

meta/lib/oeqa/core/threaded.py

@@ -1,275 +0,0 @@
-# Copyright (C) 2017 Intel Corporation
-# Released under the MIT license (see COPYING.MIT)
-
-import threading
-import multiprocessing
-import queue
-import time
-
-from unittest.suite import TestSuite
-
-from oeqa.core.loader import OETestLoader
-from oeqa.core.runner import OEStreamLogger, OETestResult, OETestRunner
-from oeqa.core.context import OETestContext
-
-class OETestLoaderThreaded(OETestLoader):
-    def __init__(self, tc, module_paths, modules, tests, modules_required,
-            filters, process_num=0, *args, **kwargs):
-        super(OETestLoaderThreaded, self).__init__(tc, module_paths, modules,
-                tests, modules_required, filters, *args, **kwargs)
-
-        self.process_num = process_num
-
-    def discover(self):
-        suite = super(OETestLoaderThreaded, self).discover()
-
-        if self.process_num <= 0:
-            self.process_num = min(multiprocessing.cpu_count(),
-                    len(suite._tests))
-
-        suites = []
-        for _ in range(self.process_num):
-            suites.append(self.suiteClass())
-
-        def _search_for_module_idx(suites, case):
-            """
-                Cases in the same module needs to be run
-                in the same thread because PyUnit keeps track
-                of setUp{Module, Class,} and tearDown{Module, Class,}.
-            """
-
-            for idx in range(self.process_num):
-                suite = suites[idx]
-                for c in suite._tests:
-                    if case.__module__ == c.__module__:
-                        return idx
-
-            return -1
-
-        def _search_for_depend_idx(suites, depends):
-            """
-                Dependency cases needs to be run in the same
-                thread, because OEQA framework look at the state
-                of dependant test to figure out if skip or not.
-            """
-
-            for idx in range(self.process_num):
-                suite = suites[idx]
-
-                for case in suite._tests:
-                    if case.id() in depends:
-                        return idx
-            return -1
-
-        def _get_best_idx(suites):
-            sizes = [len(suite._tests) for suite in suites]
-            return sizes.index(min(sizes))
-
-        def _fill_suites(suite):
-            idx = -1
-            for case in suite:
-                if isinstance(case, TestSuite):
-                    _fill_suites(case)
-                else:
-                    idx = _search_for_module_idx(suites, case)
-
-                    depends = {}
-                    if 'depends' in self.tc._registry:
-                        depends = self.tc._registry['depends']
-
-                    if idx == -1 and case.id() in depends:
-                        case_depends = depends[case.id()] 
-                        idx = _search_for_depend_idx(suites, case_depends)
-
-                    if idx == -1:
-                        idx = _get_best_idx(suites)
-
-                    suites[idx].addTest(case)
-        _fill_suites(suite)
-
-        suites_tmp = suites
-        suites = []
-        for suite in suites_tmp:
-            if len(suite._tests) > 0:
-                suites.append(suite)
-
-        return suites
-
-class OEStreamLoggerThreaded(OEStreamLogger):
-    _lock = threading.Lock()
-    buffers = {}
-
-    def write(self, msg):
-        tid = threading.get_ident()
-
-        if not tid in self.buffers:
-            self.buffers[tid] = ""
-
-        if msg:
-            self.buffers[tid] += msg
-
-    def finish(self):
-        tid = threading.get_ident()
-        
-        self._lock.acquire()
-        self.logger.info('THREAD: %d' % tid)
-        self.logger.info('-' * 70)
-        for line in self.buffers[tid].split('\n'):
-            self.logger.info(line)
-        self._lock.release()
-
-class OETestResultThreadedInternal(OETestResult):
-    def _tc_map_results(self):
-        tid = threading.get_ident()
-        
-        # PyUnit generates a result for every test module run, test
-        # if the thread already has an entry to avoid lose the previous
-        # test module results.
-        if not tid in self.tc._results:
-            self.tc._results[tid] = {}
-            self.tc._results[tid]['failures'] = self.failures
-            self.tc._results[tid]['errors'] = self.errors
-            self.tc._results[tid]['skipped'] = self.skipped
-            self.tc._results[tid]['expectedFailures'] = self.expectedFailures
-
-class OETestResultThreaded(object):
-    _results = {}
-    _lock = threading.Lock()
-
-    def __init__(self, tc):
-        self.tc = tc
-
-    def _fill_tc_results(self):
-        tids = list(self.tc._results.keys())
-        fields = ['failures', 'errors', 'skipped', 'expectedFailures']
-
-        for tid in tids:
-            result = self.tc._results[tid]
-            for field in fields:
-                if not field in self.tc._results:
-                    self.tc._results[field] = []
-                self.tc._results[field].extend(result[field])
-
-    def addResult(self, result, run_start_time, run_end_time):
-        tid = threading.get_ident()
-
-        self._lock.acquire()
-        self._results[tid] = {}
-        self._results[tid]['result'] = result
-        self._results[tid]['run_start_time'] = run_start_time 
-        self._results[tid]['run_end_time'] = run_end_time 
-        self._results[tid]['result'] = result
-        self._lock.release()
-
-    def wasSuccessful(self):
-        wasSuccessful = True
-        for tid in self._results.keys():
-            wasSuccessful = wasSuccessful and \
-                    self._results[tid]['result'].wasSuccessful()
-        return wasSuccessful
-
-    def stop(self):
-        for tid in self._results.keys():
-            self._results[tid]['result'].stop()
-
-    def logSummary(self, component, context_msg=''):
-        elapsed_time = (self.tc._run_end_time - self.tc._run_start_time)
-
-        self.tc.logger.info("SUMMARY:")
-        self.tc.logger.info("%s (%s) - Ran %d tests in %.3fs" % (component,
-            context_msg, len(self.tc._registry['cases']), elapsed_time))
-        if self.wasSuccessful():
-            msg = "%s - OK - All required tests passed" % component
-        else:
-            msg = "%s - FAIL - Required tests failed" % component
-        self.tc.logger.info(msg)
-
-    def logDetails(self):
-        if list(self._results):
-            tid = list(self._results)[0]
-            result = self._results[tid]['result']
-            result.logDetails()
-
-class _Worker(threading.Thread):
-    """Thread executing tasks from a given tasks queue"""
-    def __init__(self, tasks, result, stream):
-        threading.Thread.__init__(self)
-        self.tasks = tasks
-
-        self.result = result
-        self.stream = stream
-
-    def run(self):
-        while True:
-            try:
-                func, args, kargs = self.tasks.get(block=False)
-            except queue.Empty:
-                break
-
-            try:
-                run_start_time = time.time()
-                rc = func(*args, **kargs)
-                run_end_time = time.time()
-                self.result.addResult(rc, run_start_time, run_end_time)
-                self.stream.finish()
-            except Exception as e:
-                print(e)
-            finally:
-                self.tasks.task_done()
-
-class _ThreadedPool:
-    """Pool of threads consuming tasks from a queue"""
-    def __init__(self, num_workers, num_tasks, stream=None, result=None):
-        self.tasks = queue.Queue(num_tasks)
-        self.workers = []
-
-        for _ in range(num_workers):
-            worker = _Worker(self.tasks, result, stream)
-            self.workers.append(worker)
-
-    def start(self):
-        for worker in self.workers:
-            worker.start()
-
-    def add_task(self, func, *args, **kargs):
-        """Add a task to the queue"""
-        self.tasks.put((func, args, kargs))
-
-    def wait_completion(self):
-        """Wait for completion of all the tasks in the queue"""
-        self.tasks.join()
-        for worker in self.workers:
-            worker.join()
-
-class OETestRunnerThreaded(OETestRunner):
-    streamLoggerClass = OEStreamLoggerThreaded
-
-    def __init__(self, tc, *args, **kwargs):
-        super(OETestRunnerThreaded, self).__init__(tc, *args, **kwargs)
-        self.resultclass = OETestResultThreadedInternal # XXX: XML reporting overrides at __init__
-
-    def run(self, suites):
-        result = OETestResultThreaded(self.tc)
-
-        pool = _ThreadedPool(len(suites), len(suites), stream=self.stream,
-                result=result)
-        for s in suites:
-            pool.add_task(super(OETestRunnerThreaded, self).run, s)
-        pool.start()
-        pool.wait_completion()
-        result._fill_tc_results()
-
-        return result
-
-class OETestContextThreaded(OETestContext):
-    loaderClass = OETestLoaderThreaded
-    runnerClass = OETestRunnerThreaded
-
-    def loadTests(self, module_paths, modules=[], tests=[],
-            modules_manifest="", modules_required=[], filters={}, process_num=0):
-        if modules_manifest:
-            modules = self._read_modules_from_manifest(modules_manifest)
-
-        self.loader = self.loaderClass(self, module_paths, modules, tests,
-                modules_required, filters, process_num)
-        self.suites = self.loader.discover()

meta/lib/oeqa/files/test.pl

@@ -1,2 +0,0 @@
-$a = 9.01e+21 - 9.01e+21 + 0.01;
-print ("the value of a is ", $a, "\n");

meta/lib/oeqa/files/testresults/testresults.json

@@ -0,0 +1,40 @@
+{
+    "runtime_core-image-minimal_qemuarm_20181225195701": {
+        "configuration": {
+            "DISTRO": "poky",
+            "HOST_DISTRO": "ubuntu-16.04",
+            "IMAGE_BASENAME": "core-image-minimal",
+            "IMAGE_PKGTYPE": "rpm",
+            "LAYERS": {
+                "meta": {
+                    "branch": "master",
+                    "commit": "801745d918e83f976c706f29669779f5b292ade3",
+                    "commit_count": 52782
+                },
+                "meta-poky": {
+                    "branch": "master",
+                    "commit": "801745d918e83f976c706f29669779f5b292ade3",
+                    "commit_count": 52782
+                },
+                "meta-yocto-bsp": {
+                    "branch": "master",
+                    "commit": "801745d918e83f976c706f29669779f5b292ade3",
+                    "commit_count": 52782
+                }
+            },
+            "MACHINE": "qemuarm",
+            "STARTTIME": "20181225195701",
+            "TEST_TYPE": "runtime"
+        },
+        "result": {
+            "apt.AptRepoTest.test_apt_install_from_repo": {
+                "log": "Test requires apt to be installed",
+                "status": "PASSED"
+            },
+            "buildcpio.BuildCpioTest.test_cpio": {
+                "log": "Test requires autoconf to be installed",
+                "status": "ERROR"
+            }            
+        }
+    }
+}

meta/lib/oeqa/runtime/cases/apt.py

@@ -18,7 +18,7 @@ class AptRepoTest(AptTest):
     @classmethod
     def setUpClass(cls):
         service_repo = os.path.join(cls.tc.td['DEPLOY_DIR_DEB'], 'all')
-        cls.repo_server = HTTPService(service_repo, cls.tc.target.server_ip)
+        cls.repo_server = HTTPService(service_repo, cls.tc.target.server_ip, logger=cls.tc.logger)
         cls.repo_server.start()
 
     @classmethod

meta/lib/oeqa/runtime/cases/buildcpio.py

@@ -1,7 +1,7 @@
 from oeqa.runtime.case import OERuntimeTestCase
 from oeqa.core.decorator.depends import OETestDepends
 from oeqa.core.decorator.oeid import OETestID
-from oeqa.core.decorator.data import skipIfNotFeature
+from oeqa.runtime.decorator.package import OEHasPackage
 
 from oeqa.runtime.utils.targetbuildproject import TargetBuildProject
 
@@ -13,17 +13,18 @@ class BuildCpioTest(OERuntimeTestCase):
         cls.project = TargetBuildProject(cls.tc.target,
                                          uri,
                                          dl_dir = cls.tc.td['DL_DIR'])
-        cls.project.download_archive()
 
     @classmethod
     def tearDownClass(cls):
         cls.project.clean()
 
     @OETestID(205)
-    @skipIfNotFeature('tools-sdk',
-                      'Test requires tools-sdk to be in IMAGE_FEATURES')
     @OETestDepends(['ssh.SSHTest.test_ssh'])
+    @OEHasPackage(['gcc'])
+    @OEHasPackage(['make'])
+    @OEHasPackage(['autoconf'])
     def test_cpio(self):
+        self.project.download_archive()
         self.project.run_configure()
         self.project.run_make()
         self.project.run_install()

meta/lib/oeqa/runtime/cases/buildgalculator.py

@@ -1,7 +1,7 @@
 from oeqa.runtime.case import OERuntimeTestCase
 from oeqa.core.decorator.depends import OETestDepends
 from oeqa.core.decorator.oeid import OETestID
-from oeqa.core.decorator.data import skipIfNotFeature
+from oeqa.runtime.decorator.package import OEHasPackage
 
 from oeqa.runtime.utils.targetbuildproject import TargetBuildProject
 
@@ -13,16 +13,17 @@ class GalculatorTest(OERuntimeTestCase):
         cls.project = TargetBuildProject(cls.tc.target,
                                          uri,
                                          dl_dir = cls.tc.td['DL_DIR'])
-        cls.project.download_archive()
 
     @classmethod
     def tearDownClass(cls):
         cls.project.clean()
 
     @OETestID(1526)
-    @skipIfNotFeature('tools-sdk',
-                      'Test requires tools-sdk to be in IMAGE_FEATURES')
     @OETestDepends(['ssh.SSHTest.test_ssh'])
+    @OEHasPackage(['gcc'])
+    @OEHasPackage(['make'])
+    @OEHasPackage(['autoconf'])
     def test_galculator(self):
+        self.project.download_archive()
         self.project.run_configure()
         self.project.run_make()

meta/lib/oeqa/runtime/cases/buildlzip.py

@@ -1,7 +1,7 @@
 from oeqa.runtime.case import OERuntimeTestCase
 from oeqa.core.decorator.depends import OETestDepends
 from oeqa.core.decorator.oeid import OETestID
-from oeqa.core.decorator.data import skipIfNotFeature
+from oeqa.runtime.decorator.package import OEHasPackage
 
 from oeqa.runtime.utils.targetbuildproject import TargetBuildProject
 
@@ -14,21 +14,19 @@ class BuildLzipTest(OERuntimeTestCase):
         cls.project = TargetBuildProject(cls.tc.target,
                                          uri,
                                          dl_dir = cls.tc.td['DL_DIR'])
-        cls.project.download_archive()
 
     @classmethod
     def tearDownClass(cls):
         cls.project.clean()
 
     @OETestID(206)
-    @skipIfNotFeature('tools-sdk',
-                      'Test requires tools-sdk to be in IMAGE_FEATURES')
     @OETestDepends(['ssh.SSHTest.test_ssh'])
+    @OEHasPackage(['gcc'])
+    @OEHasPackage(['make'])
+    @OEHasPackage(['autoconf'])
     def test_lzip(self):
+        self.project.download_archive()
         self.project.run_configure()
         self.project.run_make()
         self.project.run_install()
 
-    @classmethod
-    def tearDownClass(self):
-        self.project.clean()

meta/lib/oeqa/runtime/cases/date.py

@@ -3,6 +3,7 @@ import re
 from oeqa.runtime.case import OERuntimeTestCase
 from oeqa.core.decorator.depends import OETestDepends
 from oeqa.core.decorator.oeid import OETestID
+from oeqa.runtime.decorator.package import OEHasPackage
 
 class DateTest(OERuntimeTestCase):
 
@@ -18,6 +19,7 @@ class DateTest(OERuntimeTestCase):
 
     @OETestID(211)
     @OETestDepends(['ssh.SSHTest.test_ssh'])
+    @OEHasPackage(['coreutils', 'busybox'])
     def test_date(self):
         (status, output) = self.target.run('date +"%Y-%m-%d %T"')
         msg = 'Failed to get initial date, output: %s' % output

meta/lib/oeqa/runtime/cases/df.py

@@ -1,11 +1,13 @@
 from oeqa.runtime.case import OERuntimeTestCase
 from oeqa.core.decorator.depends import OETestDepends
 from oeqa.core.decorator.oeid import OETestID
+from oeqa.runtime.decorator.package import OEHasPackage
 
 class DfTest(OERuntimeTestCase):
 
     @OETestID(234)
     @OETestDepends(['ssh.SSHTest.test_ssh'])
+    @OEHasPackage(['coreutils', 'busybox'])
     def test_df(self):
         cmd = "df / | sed -n '2p' | awk '{print $4}'"
         (status,output) = self.target.run(cmd)

meta/lib/oeqa/runtime/cases/dnf.py

@@ -55,7 +55,7 @@ class DnfRepoTest(DnfTest):
     @classmethod
     def setUpClass(cls):
         cls.repo_server = HTTPService(os.path.join(cls.tc.td['WORKDIR'], 'oe-testimage-repo'),
-                                      cls.tc.target.server_ip)
+                                      cls.tc.target.server_ip, logger=cls.tc.logger)
         cls.repo_server.start()
 
     @classmethod
@@ -67,7 +67,8 @@ class DnfRepoTest(DnfTest):
         deploy_url = 'http://%s:%s/' %(self.target.server_ip, self.repo_server.port)
         cmdlinerepoopts = ["--repofrompath=oe-testimage-repo-%s,%s%s" %(arch, deploy_url, arch) for arch in pkgarchs]
 
-        self.dnf(" ".join(cmdlinerepoopts) + " --nogpgcheck " + command)
+        output = self.dnf(" ".join(cmdlinerepoopts) + " --nogpgcheck " + command)
+        return output
 
     @OETestDepends(['dnf.DnfBasicTest.test_dnf_help'])
     @OETestID(1744)
@@ -88,6 +89,9 @@ class DnfRepoTest(DnfTest):
     @OETestDepends(['dnf.DnfRepoTest.test_dnf_makecache'])
     @OETestID(1740)
     def test_dnf_install(self):
+        output = self.dnf_with_repo('list run-postinsts-dev')
+        if 'Installed Packages' in output:
+            self.dnf_with_repo('remove -y run-postinsts-dev')
         self.dnf_with_repo('install -y run-postinsts-dev')
 
     @OETestDepends(['dnf.DnfRepoTest.test_dnf_install'])
@@ -120,4 +124,44 @@ class DnfRepoTest(DnfTest):
     def test_dnf_reinstall(self):
         self.dnf_with_repo('reinstall -y run-postinsts-dev')
 
+    @OETestDepends(['dnf.DnfRepoTest.test_dnf_makecache'])
+    @OETestID(1771)
+    def test_dnf_installroot(self):
+        rootpath = '/home/root/chroot/test'
+        #Copy necessary files to avoid errors with not yet installed tools on
+        #installroot directory.
+        self.target.run('mkdir -p %s/etc' % rootpath, 1500)
+        self.target.run('mkdir -p %s/bin %s/sbin %s/usr/bin %s/usr/sbin' % (rootpath, rootpath, rootpath, rootpath), 1500)
+        self.target.run('mkdir -p %s/dev' % rootpath, 1500)
+        #Handle different architectures lib dirs
+        self.target.run('mkdir -p %s/lib' % rootpath, 1500)
+        self.target.run('mkdir -p %s/libx32' % rootpath, 1500)
+        self.target.run('mkdir -p %s/lib64' % rootpath, 1500)
+        self.target.run('cp /lib/libtinfo.so.5 %s/lib' % rootpath, 1500)
+        self.target.run('cp /libx32/libtinfo.so.5 %s/libx32' % rootpath, 1500)
+        self.target.run('cp /lib64/libtinfo.so.5 %s/lib64' % rootpath, 1500)
+        self.target.run('cp -r /etc/rpm %s/etc' % rootpath, 1500)
+        self.target.run('cp -r /etc/dnf %s/etc' % rootpath, 1500)
+        self.target.run('cp /bin/sh %s/bin' % rootpath, 1500)
+        self.target.run('mount -o bind /dev %s/dev/' % rootpath, 1500)
+        self.dnf_with_repo('install --installroot=%s -v -y --rpmverbosity=debug busybox run-postinsts' % rootpath)
+        status, output = self.target.run('test -e %s/var/cache/dnf' % rootpath, 1500)
+        self.assertEqual(0, status, output)
+        status, output = self.target.run('test -e %s/bin/busybox' % rootpath, 1500)
+        self.assertEqual(0, status, output)
 
+    @OETestDepends(['dnf.DnfRepoTest.test_dnf_makecache'])
+    @OETestID(1772)
+    def test_dnf_exclude(self):
+        excludepkg = 'curl-dev'
+        self.dnf_with_repo('install -y curl*')
+        self.dnf('list %s' % excludepkg, 0)
+        #Avoid remove dependencies to skip some errors on different archs and images
+        self.dnf_with_repo('remove --setopt=clean_requirements_on_remove=0 -y curl*')
+        #check curl-dev is not installed adter removing all curl occurrences
+        status, output = self.target.run('dnf list --installed | grep %s'% excludepkg, 1500)
+        self.assertEqual(1, status, "%s was not removed,  is listed as installed"%excludepkg)
+        self.dnf_with_repo('install -y --exclude=%s --exclude=curl-staticdev curl*' % excludepkg)
+        #check curl-dev is not installed after being excluded
+        status, output = self.target.run('dnf list --installed | grep %s'% excludepkg , 1500)
+        self.assertEqual(1, status, "%s was not excluded, is listed as installed"%excludepkg)

meta/lib/oeqa/runtime/cases/gcc.py

@@ -3,12 +3,12 @@ import os
 from oeqa.runtime.case import OERuntimeTestCase
 from oeqa.core.decorator.depends import OETestDepends
 from oeqa.core.decorator.oeid import OETestID
-from oeqa.core.decorator.data import skipIfNotFeature
+from oeqa.runtime.decorator.package import OEHasPackage
 
 class GccCompileTest(OERuntimeTestCase):
 
     @classmethod
-    def setUpClass(cls):
+    def setUp(cls):
         dst = '/tmp/'
         src = os.path.join(cls.tc.files_dir, 'test.c')
         cls.tc.target.copyTo(src, dst)
@@ -20,14 +20,13 @@ class GccCompileTest(OERuntimeTestCase):
         cls.tc.target.copyTo(src, dst)
 
     @classmethod
-    def tearDownClass(cls):
+    def tearDown(cls):
         files = '/tmp/test.c /tmp/test.o /tmp/test /tmp/testmakefile'
         cls.tc.target.run('rm %s' % files)
 
     @OETestID(203)
-    @skipIfNotFeature('tools-sdk',
-                      'Test requires tools-sdk to be in IMAGE_FEATURES')
     @OETestDepends(['ssh.SSHTest.test_ssh'])
+    @OEHasPackage(['gcc'])
     def test_gcc_compile(self):
         status, output = self.target.run('gcc /tmp/test.c -o /tmp/test -lm')
         msg = 'gcc compile failed, output: %s' % output
@@ -38,9 +37,8 @@ class GccCompileTest(OERuntimeTestCase):
         self.assertEqual(status, 0, msg=msg)
 
     @OETestID(200)
-    @skipIfNotFeature('tools-sdk',
-                      'Test requires tools-sdk to be in IMAGE_FEATURES')
     @OETestDepends(['ssh.SSHTest.test_ssh'])
+    @OEHasPackage(['g++'])
     def test_gpp_compile(self):
         status, output = self.target.run('g++ /tmp/test.c -o /tmp/test -lm')
         msg = 'g++ compile failed, output: %s' % output
@@ -51,9 +49,8 @@ class GccCompileTest(OERuntimeTestCase):
         self.assertEqual(status, 0, msg=msg)
 
     @OETestID(1142)
-    @skipIfNotFeature('tools-sdk',
-                      'Test requires tools-sdk to be in IMAGE_FEATURES')
     @OETestDepends(['ssh.SSHTest.test_ssh'])
+    @OEHasPackage(['g++'])
     def test_gpp2_compile(self):
         status, output = self.target.run('g++ /tmp/test.cpp -o /tmp/test -lm')
         msg = 'g++ compile failed, output: %s' % output
@@ -64,9 +61,9 @@ class GccCompileTest(OERuntimeTestCase):
         self.assertEqual(status, 0, msg=msg)
 
     @OETestID(204)
-    @skipIfNotFeature('tools-sdk',
-                      'Test requires tools-sdk to be in IMAGE_FEATURES')
     @OETestDepends(['ssh.SSHTest.test_ssh'])
+    @OEHasPackage(['gcc'])
+    @OEHasPackage(['make'])
     def test_make(self):
         status, output = self.target.run('cd /tmp; make -f testmakefile')
         msg = 'running make failed, output %s' % output

meta/lib/oeqa/runtime/cases/kernelmodule.py

@@ -4,11 +4,12 @@ from oeqa.runtime.case import OERuntimeTestCase
 from oeqa.core.decorator.depends import OETestDepends
 from oeqa.core.decorator.oeid import OETestID
 from oeqa.core.decorator.data import skipIfNotFeature
+from oeqa.runtime.decorator.package import OEHasPackage
 
 class KernelModuleTest(OERuntimeTestCase):
 
     @classmethod
-    def setUpClass(cls):
+    def setUp(cls):
         src = os.path.join(cls.tc.runtime_files_dir, 'hellomod.c')
         dst = '/tmp/hellomod.c'
         cls.tc.target.copyTo(src, dst)
@@ -18,7 +19,7 @@ class KernelModuleTest(OERuntimeTestCase):
         cls.tc.target.copyTo(src, dst)
 
     @classmethod
-    def tearDownClass(cls):
+    def tearDown(cls):
         files = '/tmp/Makefile /tmp/hellomod.c'
         cls.tc.target.run('rm %s' % files)
 
@@ -26,6 +27,9 @@ class KernelModuleTest(OERuntimeTestCase):
     @skipIfNotFeature('tools-sdk',
                       'Test requires tools-sdk to be in IMAGE_FEATURES')
     @OETestDepends(['gcc.GccCompileTest.test_gcc_compile'])
+    @OEHasPackage(['kernel-devsrc'])
+    @OEHasPackage(['make'])
+    @OEHasPackage(['gcc'])
     def test_kernel_module(self):
         cmds = [
             'cd /usr/src/kernel && make scripts prepare',

meta/lib/oeqa/runtime/cases/logrotate.py

@@ -8,9 +8,13 @@ from oeqa.runtime.decorator.package import OEHasPackage
 
 class LogrotateTest(OERuntimeTestCase):
 
+    @classmethod
+    def setUpClass(cls):
+        cls.tc.target.run('cp /etc/logrotate.d/wtmp $HOME/wtmp.oeqabak')
+
     @classmethod
     def tearDownClass(cls):
-        cls.tc.target.run('rm -rf $HOME/logrotate_dir')
+        cls.tc.target.run('mv -f $HOME/wtmp.oeqabak /etc/logrotate.d/wtmp && rm -rf $HOME/logrotate_dir')
 
     @OETestID(1544)
     @OETestDepends(['ssh.SSHTest.test_ssh'])

meta/lib/oeqa/runtime/cases/opkg.py

@@ -16,13 +16,13 @@ class OpkgTest(OERuntimeTestCase):
 class OpkgRepoTest(OpkgTest):
 
     @classmethod
-    def setUpClass(cls):
+    def setUp(cls):
         service_repo = os.path.join(cls.tc.td['DEPLOY_DIR_IPK'], 'all')
-        cls.repo_server = HTTPService(service_repo, cls.tc.target.server_ip)
+        cls.repo_server = HTTPService(service_repo, cls.tc.target.server_ip, logger=cls.tc.logger)
         cls.repo_server.start()
 
     @classmethod
-    def tearDownClass(cls):
+    def tearDown(cls):
         cls.repo_server.stop()
 
     def setup_source_config_for_package_install(self):

meta/lib/oeqa/runtime/cases/perl.py

@@ -6,32 +6,10 @@ from oeqa.core.decorator.oeid import OETestID
 from oeqa.runtime.decorator.package import OEHasPackage
 
 class PerlTest(OERuntimeTestCase):
-
-    @classmethod
-    def setUpClass(cls):
-        src = os.path.join(cls.tc.files_dir, 'test.pl')
-        dst = '/tmp/test.pl'
-        cls.tc.target.copyTo(src, dst)
-
-    @classmethod
-    def tearDownClass(cls):
-        dst = '/tmp/test.pl'
-        cls.tc.target.run('rm %s' % dst)
-
-    @OETestID(1141)
+    @OETestID(208)
     @OETestDepends(['ssh.SSHTest.test_ssh'])
     @OEHasPackage(['perl'])
-    def test_perl_exists(self):
-        status, output = self.target.run('which perl')
-        msg = 'Perl binary not in PATH or not on target.'
-        self.assertEqual(status, 0, msg=msg)
-
-    @OETestID(208)
-    @OETestDepends(['perl.PerlTest.test_perl_exists'])
     def test_perl_works(self):
-        status, output = self.target.run('perl /tmp/test.pl')
-        msg = 'Exit status was not 0. Output: %s' % output
-        self.assertEqual(status, 0, msg=msg)
-
-        msg = 'Incorrect output: %s' % output
-        self.assertEqual(output, "the value of a is 0.01", msg=msg)
+        status, output = self.target.run("perl -e '$_=\"Uryyb, jbeyq\"; tr/a-zA-Z/n-za-mN-ZA-M/;print'")
+        self.assertEqual(status, 0)
+        self.assertEqual(output, "Hello, world")

meta/lib/oeqa/runtime/cases/ptest.py

@@ -1,7 +1,11 @@
+import unittest
+import pprint
+
 from oeqa.runtime.case import OERuntimeTestCase
 from oeqa.core.decorator.depends import OETestDepends
 from oeqa.core.decorator.oeid import OETestID
 from oeqa.core.decorator.data import skipIfNotFeature
+from oeqa.runtime.decorator.package import OEHasPackage
 from oeqa.utils.logparser import Lparser, Result
 
 class PtestRunnerTest(OERuntimeTestCase):
@@ -49,6 +53,8 @@ class PtestRunnerTest(OERuntimeTestCase):
     @OETestID(1600)
     @skipIfNotFeature('ptest', 'Test requires ptest to be in DISTRO_FEATURES')
     @OETestDepends(['ssh.SSHTest.test_ssh'])
+    @OEHasPackage(['ptest-runner'])
+    @unittest.expectedFailure
     def test_ptestrunner(self):
         status, output = self.target.run('which ptest-runner', 0)
         if status != 0:
@@ -76,6 +82,11 @@ class PtestRunnerTest(OERuntimeTestCase):
         # status != 0 is OK since some ptest tests may fail
         self.assertTrue(status != 127, msg="Cannot execute ptest-runner!")
 
+        if not hasattr(self.tc, "extraresults"):
+            self.tc.extraresults = {}
+        extras = self.tc.extraresults
+        extras['ptestresult.rawlogs'] = {'log': output}
+
         # Parse and save results
         parse_result = self.parse_ptest(ptest_runner_log)
         parse_result.log_as_files(ptest_log_dir, test_status = ['pass','fail', 'skip'])
@@ -84,10 +95,18 @@ class PtestRunnerTest(OERuntimeTestCase):
             os.remove(ptest_log_dir_link)
         os.symlink(os.path.basename(ptest_log_dir), ptest_log_dir_link)
 
+        trans = str.maketrans("()", "__")
+        resmap = {'pass': 'PASSED', 'skip': 'SKIPPED', 'fail': 'FAILED'}
+        for section in parse_result.result_dict:
+            for test, result in parse_result.result_dict[section]:
+                testname = "ptestresult." + section + "." + "_".join(test.translate(trans).split())
+                extras[testname] = {'status': resmap[result]}
+
         failed_tests = {}
         for section in parse_result.result_dict:
-            failed_testcases = [ test for test, result in parse_result.result_dict[section] if result == 'fail' ]
+            failed_testcases = [ "_".join(test.translate(trans).split()) for test, result in parse_result.result_dict[section] if result == 'fail' ]
             if failed_testcases:
                 failed_tests[section] = failed_testcases
 
-        self.assertFalse(failed_tests, msg = "Failed ptests: %s" %(str(failed_tests)))
+        if failed_tests:
+            self.fail("Failed ptests:\n%s" % pprint.pformat(failed_tests))

meta/lib/oeqa/runtime/cases/python.py

@@ -1,43 +1,17 @@
-import os
-
 from oeqa.runtime.case import OERuntimeTestCase
 from oeqa.core.decorator.depends import OETestDepends
 from oeqa.core.decorator.oeid import OETestID
 from oeqa.runtime.decorator.package import OEHasPackage
 
 class PythonTest(OERuntimeTestCase):
-
-    @classmethod
-    def setUpClass(cls):
-        src = os.path.join(cls.tc.files_dir, 'test.py')
-        dst = '/tmp/test.py'
-        cls.tc.target.copyTo(src, dst)
-
-    @classmethod
-    def tearDownClass(cls):
-        dst = '/tmp/test.py'
-        cls.tc.target.run('rm %s' % dst)
-
-    @OETestID(1145)
-    @OETestDepends(['ssh.SSHTest.test_ssh'])
-    @OEHasPackage(['python-core'])
-    def test_python_exists(self):
-        status, output = self.target.run('which python')
-        msg = 'Python binary not in PATH or not on target.'
-        self.assertEqual(status, 0, msg=msg)
-
     @OETestID(965)
-    @OETestDepends(['python.PythonTest.test_python_exists'])
-    def test_python_stdout(self):
-        status, output = self.target.run('python /tmp/test.py')
+    @OETestDepends(['ssh.SSHTest.test_ssh'])
+    @OEHasPackage(['python3-core'])
+    def test_python3(self):
+        cmd = "python3 -c \"import codecs; print(codecs.encode('Uryyb, jbeyq', 'rot13'))\""
+        status, output = self.target.run(cmd)
         msg = 'Exit status was not 0. Output: %s' % output
         self.assertEqual(status, 0, msg=msg)
 
         msg = 'Incorrect output: %s' % output
-        self.assertEqual(output, "the value of a is 0.01", msg=msg)
-
-    @OETestID(1146)
-    @OETestDepends(['python.PythonTest.test_python_stdout'])
-    def test_python_testfile(self):
-        status, output = self.target.run('ls /tmp/testfile.python')
-        self.assertEqual(status, 0, msg='Python test file generate failed.')
+        self.assertEqual(output, "Hello, world", msg=msg)

meta/lib/oeqa/runtime/cases/rpm.py

@@ -10,11 +10,6 @@ from oeqa.core.utils.path import findFile
 
 class RpmBasicTest(OERuntimeTestCase):
 
-    @classmethod
-    def setUpClass(cls):
-        if cls.tc.td['PACKAGE_CLASSES'].split()[0] != 'package_rpm':
-            cls.skipTest('Tests require image to be build from rpm')
-
     @OETestID(960)
     @OEHasPackage(['rpm'])
     @OETestDepends(['ssh.SSHTest.test_ssh'])
@@ -26,6 +21,9 @@ class RpmBasicTest(OERuntimeTestCase):
     @OETestID(191)
     @OETestDepends(['rpm.RpmBasicTest.test_rpm_help'])
     def test_rpm_query(self):
+        status, output = self.target.run('ls /var/lib/rpm/')
+        if status != 0:
+            self.skipTest('No /var/lib/rpm on target')
         status, output = self.target.run('rpm -q rpm')
         msg = 'status and output: %s and %s' % (status, output)
         self.assertEqual(status, 0, msg=msg)
@@ -34,36 +32,31 @@ class RpmInstallRemoveTest(OERuntimeTestCase):
 
     @classmethod
     def setUpClass(cls):
-        if cls.tc.td['PACKAGE_CLASSES'].split()[0] != 'package_rpm':
-            cls.skipTest('Tests require image to be build from rpm')
-
         pkgarch = cls.td['TUNE_PKGARCH'].replace('-', '_')
         rpmdir = os.path.join(cls.tc.td['DEPLOY_DIR'], 'rpm', pkgarch)
-        # Pick rpm-doc as a test file to get installed, because it's small
+        # Pick base-passwd-doc as a test file to get installed, because it's small
         # and it will always be built for standard targets
-        rpm_doc = 'rpm-doc-*.%s.rpm' % pkgarch
+        rpm_doc = 'base-passwd-doc-*.%s.rpm' % pkgarch
+        if not os.path.exists(rpmdir):
+            return
         for f in fnmatch.filter(os.listdir(rpmdir), rpm_doc):
-            test_file = os.path.join(rpmdir, f)
-        dst = '/tmp/rpm-doc.rpm'
-        cls.tc.target.copyTo(test_file, dst)
-
-    @classmethod
-    def tearDownClass(cls):
-        dst = '/tmp/rpm-doc.rpm'
-        cls.tc.target.run('rm -f %s' % dst)
+            cls.test_file = os.path.join(rpmdir, f)
+        cls.dst = '/tmp/base-passwd-doc.rpm'
 
     @OETestID(192)
-    @OETestDepends(['rpm.RpmBasicTest.test_rpm_help'])
+    @OETestDepends(['rpm.RpmBasicTest.test_rpm_query'])
     def test_rpm_install(self):
-        status, output = self.target.run('rpm -ivh /tmp/rpm-doc.rpm')
-        msg = 'Failed to install rpm-doc package: %s' % output
+        self.tc.target.copyTo(self.test_file, self.dst)
+        status, output = self.target.run('rpm -ivh /tmp/base-passwd-doc.rpm')
+        msg = 'Failed to install base-passwd-doc package: %s' % output
         self.assertEqual(status, 0, msg=msg)
+        self.tc.target.run('rm -f %s' % self.dst)
 
     @OETestID(194)
     @OETestDepends(['rpm.RpmInstallRemoveTest.test_rpm_install'])
     def test_rpm_remove(self):
-        status,output = self.target.run('rpm -e rpm-doc')
-        msg = 'Failed to remove rpm-doc package: %s' % output
+        status,output = self.target.run('rpm -e base-passwd-doc')
+        msg = 'Failed to remove base-passwd-doc package: %s' % output
         self.assertEqual(status, 0, msg=msg)
 
     @OETestID(1096)
@@ -107,7 +100,7 @@ class RpmInstallRemoveTest(OERuntimeTestCase):
         Expected:    There should be some RPM prefixed entries in the above file.
         Product:     BSPs
         Author:      Alexandru Georgescu <alexandru.c.georgescu@intel.com>
-        Author:      Alexander Kanavin <alexander.kanavin@intel.com>
+        Author:      Alexander Kanavin <alex.kanavin@gmail.com>
         AutomatedBy: Daniel Istrate <daniel.alexandrux.istrate@intel.com>
         """
         db_files_cmd = 'ls /var/lib/rpm/__db.*'
@@ -118,19 +111,23 @@ class RpmInstallRemoveTest(OERuntimeTestCase):
         msg =  'Failed to find database files under /var/lib/rpm/ as __db.xxx'
         self.assertEqual(0, status, msg=msg)
 
+        self.tc.target.copyTo(self.test_file, self.dst)
+
         # Remove the package just in case
-        self.target.run('rpm -e rpm-doc')
+        self.target.run('rpm -e base-passwd-doc')
 
         # Install/Remove a package 10 times
         for i in range(10):
-            status, output = self.target.run('rpm -ivh /tmp/rpm-doc.rpm')
-            msg = 'Failed to install rpm-doc package. Reason: {}'.format(output)
+            status, output = self.target.run('rpm -ivh /tmp/base-passwd-doc.rpm')
+            msg = 'Failed to install base-passwd-doc package. Reason: {}'.format(output)
             self.assertEqual(0, status, msg=msg)
 
-            status, output = self.target.run('rpm -e rpm-doc')
-            msg = 'Failed to remove rpm-doc package. Reason: {}'.format(output)
+            status, output = self.target.run('rpm -e base-passwd-doc')
+            msg = 'Failed to remove base-passwd-doc package. Reason: {}'.format(output)
             self.assertEqual(0, status, msg=msg)
 
+        self.tc.target.run('rm -f %s' % self.dst)
+
         # if using systemd this should ensure all entries are flushed to /var
         status, output = self.target.run("journalctl --sync")
         # Get the amount of entries in the log file

meta/lib/oeqa/runtime/cases/scp.py

@@ -4,6 +4,7 @@ from tempfile import mkstemp
 from oeqa.runtime.case import OERuntimeTestCase
 from oeqa.core.decorator.depends import OETestDepends
 from oeqa.core.decorator.oeid import OETestID
+from oeqa.runtime.decorator.package import OEHasPackage
 
 class ScpTest(OERuntimeTestCase):
 
@@ -20,6 +21,7 @@ class ScpTest(OERuntimeTestCase):
 
     @OETestID(220)
     @OETestDepends(['ssh.SSHTest.test_ssh'])
+    @OEHasPackage(['openssh-scp', 'dropbear'])
     def test_scp_file(self):
         dst = '/tmp/test_scp_file'
 

meta/lib/oeqa/runtime/cases/ssh.py

@@ -1,11 +1,13 @@
 from oeqa.runtime.case import OERuntimeTestCase
 from oeqa.core.decorator.depends import OETestDepends
 from oeqa.core.decorator.oeid import OETestID
+from oeqa.runtime.decorator.package import OEHasPackage
 
 class SSHTest(OERuntimeTestCase):
 
     @OETestID(224)
     @OETestDepends(['ping.PingTest.test_ping'])
+    @OEHasPackage(['dropbear', 'openssh-sshd'])
     def test_ssh(self):
         (status, output) = self.target.run('uname -a')
         self.assertEqual(status, 0, msg='SSH Test failed: %s' % output)

meta/lib/oeqa/runtime/cases/stap.py

@@ -4,17 +4,18 @@ from oeqa.runtime.case import OERuntimeTestCase
 from oeqa.core.decorator.depends import OETestDepends
 from oeqa.core.decorator.oeid import OETestID
 from oeqa.core.decorator.data import skipIfNotFeature
+from oeqa.runtime.decorator.package import OEHasPackage
 
 class StapTest(OERuntimeTestCase):
 
     @classmethod
-    def setUpClass(cls):
+    def setUp(cls):
         src = os.path.join(cls.tc.runtime_files_dir, 'hello.stp')
         dst = '/tmp/hello.stp'
         cls.tc.target.copyTo(src, dst)
 
     @classmethod
-    def tearDownClass(cls):
+    def tearDown(cls):
         files = '/tmp/hello.stp'
         cls.tc.target.run('rm %s' % files)
 
@@ -22,6 +23,7 @@ class StapTest(OERuntimeTestCase):
     @skipIfNotFeature('tools-profile',
                       'Test requires tools-profile to be in IMAGE_FEATURES')
     @OETestDepends(['kernelmodule.KernelModuleTest.test_kernel_module'])
+    @OEHasPackage(['systemtap'])
     def test_stap(self):
         cmds = [
             'cd /usr/src/kernel && make scripts prepare',

meta/lib/oeqa/runtime/cases/xorg.py

@@ -2,6 +2,7 @@ from oeqa.runtime.case import OERuntimeTestCase
 from oeqa.core.decorator.depends import OETestDepends
 from oeqa.core.decorator.oeid import OETestID
 from oeqa.core.decorator.data import skipIfNotFeature
+from oeqa.runtime.decorator.package import OEHasPackage
 
 class XorgTest(OERuntimeTestCase):
 
@@ -9,6 +10,7 @@ class XorgTest(OERuntimeTestCase):
     @skipIfNotFeature('x11-base',
                       'Test requires x11 to be in IMAGE_FEATURES')
     @OETestDepends(['ssh.SSHTest.test_ssh'])
+    @OEHasPackage(['xserver-nodm-init'])
     def test_xorg_running(self):
         cmd ='%s | grep -v xinit | grep [X]org' % self.tc.target_cmds['ps']
         status, output = self.target.run(cmd)

meta/lib/oeqa/runtime/context.py

@@ -49,7 +49,6 @@ class OERuntimeTestContextExecutor(OETestContextExecutor):
     default_manifest = 'data/manifest'
     default_server_ip = '192.168.7.1'
     default_target_ip = '192.168.7.2'
-    default_host_dumper_dir = '/tmp/oe-saved-tests'
     default_extract_dir = 'packages/extracted'
 
     def register_commands(self, logger, subparsers):
@@ -71,9 +70,7 @@ class OERuntimeTestContextExecutor(OETestContextExecutor):
                 % self.default_server_ip)
 
         runtime_group.add_argument('--host-dumper-dir', action='store',
-                default=self.default_host_dumper_dir,
-                help="Directory where host status is dumped, if tests fails, default: %s" \
-                % self.default_host_dumper_dir)
+                help="Directory where host status is dumped, if tests fails")
 
         runtime_group.add_argument('--packages-manifest', action='store',
                 default=self.default_manifest,
@@ -101,7 +98,7 @@ class OERuntimeTestContextExecutor(OETestContextExecutor):
         if target_type == 'simpleremote':
             target = OESSHTarget(logger, target_ip, server_ip, **kwargs)
         elif target_type == 'qemu':
-            target = OEQemuTarget(logger, target_ip, server_ip, **kwargs)
+            target = OEQemuTarget(logger, server_ip, **kwargs)
         else:
             # XXX: This code uses the old naming convention for controllers and
             # targets, the idea it is to leave just targets as the controller

meta/lib/oeqa/sdk/cases/perl.py

@@ -1,8 +1,4 @@
-import os
-import shutil
 import unittest
-
-from oeqa.core.utils.path import remove_safe
 from oeqa.sdk.case import OESDKTestCase
 
 class PerlTest(OESDKTestCase):
@@ -12,17 +8,10 @@ class PerlTest(OESDKTestCase):
                 self.tc.hasHostPackage("perl-native")):
             raise unittest.SkipTest("No perl package in the SDK")
 
-        for f in ['test.pl']:
-            shutil.copyfile(os.path.join(self.tc.files_dir, f),
-                    os.path.join(self.tc.sdk_dir, f))
-        self.testfile = os.path.join(self.tc.sdk_dir, "test.pl")
-
-    def test_perl_exists(self):
-        self._run('which perl')
-
-    def test_perl_works(self):
-        self._run('perl %s' % self.testfile)
-
-    @classmethod
-    def tearDownClass(self):
-        remove_safe(self.testfile)
+    def test_perl(self):
+        try:
+            cmd = "perl -e '$_=\"Uryyb, jbeyq\"; tr/a-zA-Z/n-za-mN-ZA-M/;print'"
+            output = self._run(cmd)
+            self.assertEqual(output, "Hello, world")
+        except subprocess.CalledProcessError as e:
+            self.fail("Unexpected exit %d (output %s)" % (e.returncode, e.output))

meta/lib/oeqa/sdk/context.py

@@ -6,10 +6,9 @@ import sys
 import glob
 import re
 
-from oeqa.core.context import OETestContextExecutor
-from oeqa.core.threaded import OETestContextThreaded
+from oeqa.core.context import OETestContext, OETestContextExecutor
 
-class OESDKTestContext(OETestContextThreaded):
+class OESDKTestContext(OETestContext):
     sdk_files_dir = os.path.join(os.path.dirname(os.path.abspath(__file__)), "files")
 
     def __init__(self, td=None, logger=None, sdk_dir=None, sdk_env=None,

meta/lib/oeqa/selftest/case.py

@@ -12,6 +12,8 @@ import oeqa.utils.ftools as ftools
 from oeqa.utils.commands import runCmd, bitbake, get_bb_var
 from oeqa.core.case import OETestCase
 
+import bb.utils
+
 class OESelftestTestCase(OETestCase):
     def __init__(self, methodName="runTest"):
         self._extra_tear_down_commands = []
@@ -167,7 +169,7 @@ to ensure accurate results.")
         if self._track_for_cleanup:
             for path in self._track_for_cleanup:
                 if os.path.isdir(path):
-                    shutil.rmtree(path)
+                    bb.utils.remove(path, recurse=True)
                 if os.path.isfile(path):
                     os.remove(path)
             self._track_for_cleanup = []

meta/lib/oeqa/selftest/cases/buildoptions.py

@@ -38,10 +38,12 @@ class ImageOptionsTests(OESelftestTestCase):
         self.assertTrue(os.path.isfile(p), msg = "No ccache found (%s)" % p)
         self.write_config('INHERIT += "ccache"')
         self.add_command_to_tearDown('bitbake -c clean m4')
+        bitbake("m4 -c clean")
         bitbake("m4 -f -c compile")
         log_compile = os.path.join(get_bb_var("WORKDIR","m4"), "temp/log.do_compile")
-        res = runCmd("grep ccache %s" % log_compile, ignore_status=True)
-        self.assertEqual(0, res.status, msg="No match for ccache in m4 log.do_compile. For further details: %s" % log_compile)
+        with open(log_compile, "r") as f:
+            loglines = "".join(f.readlines())
+        self.assertIn("ccache", loglines, msg="No match for ccache in m4 log.do_compile. For further details: %s" % log_compile)
 
     @OETestID(1435)
     def test_read_only_image(self):
@@ -57,15 +59,15 @@ class DiskMonTest(OESelftestTestCase):
     @OETestID(277)
     def test_stoptask_behavior(self):
         self.write_config('BB_DISKMON_DIRS = "STOPTASKS,${TMPDIR},100000G,100K"')
-        res = bitbake("m4", ignore_status = True)
+        res = bitbake("delay -c delay", ignore_status = True)
         self.assertTrue('ERROR: No new tasks can be executed since the disk space monitor action is "STOPTASKS"!' in res.output, msg = "Tasks should have stopped. Disk monitor is set to STOPTASK: %s" % res.output)
         self.assertEqual(res.status, 1, msg = "bitbake reported exit code %s. It should have been 1. Bitbake output: %s" % (str(res.status), res.output))
         self.write_config('BB_DISKMON_DIRS = "ABORT,${TMPDIR},100000G,100K"')
-        res = bitbake("m4", ignore_status = True)
+        res = bitbake("delay -c delay", ignore_status = True)
         self.assertTrue('ERROR: Immediately abort since the disk space monitor action is "ABORT"!' in res.output, "Tasks should have been aborted immediatelly. Disk monitor is set to ABORT: %s" % res.output)
         self.assertEqual(res.status, 1, msg = "bitbake reported exit code %s. It should have been 1. Bitbake output: %s" % (str(res.status), res.output))
         self.write_config('BB_DISKMON_DIRS = "WARN,${TMPDIR},100000G,100K"')
-        res = bitbake("m4")
+        res = bitbake("delay -c delay")
         self.assertTrue('WARNING: The free space' in res.output, msg = "A warning should have been displayed for disk monitor is set to WARN: %s" %res.output)
 
 class SanityOptionsTest(OESelftestTestCase):

meta/lib/oeqa/selftest/cases/containerimage.py

@@ -39,6 +39,7 @@ class ContainerImageTests(OESelftestTestCase):
 IMAGE_FSTYPES = "container"
 PACKAGE_CLASSES = "package_ipk"
 IMAGE_FEATURES = ""
+IMAGE_BUILDINFO_FILE = ""
 """)
 
         bbvars = get_bb_vars(['bindir', 'sysconfdir', 'localstatedir',

meta/lib/oeqa/selftest/cases/devtool.py

@@ -11,6 +11,71 @@ from oeqa.utils.commands import runCmd, bitbake, get_bb_var, create_temp_layer
 from oeqa.utils.commands import get_bb_vars, runqemu, get_test_layer
 from oeqa.core.decorator.oeid import OETestID
 
+oldmetapath = None
+
+def setUpModule():
+    import bb.utils
+
+    global templayerdir
+    templayerdir = tempfile.mkdtemp(prefix='devtoolqa')
+    corecopydir = os.path.join(templayerdir, 'core-copy')
+    bblayers_conf = os.path.join(os.environ['BUILDDIR'], 'conf', 'bblayers.conf')
+    edited_layers = []
+
+    # We need to take a copy of the meta layer so we can modify it and not
+    # have any races against other tests that might be running in parallel
+    # however things like COREBASE mean that you can't just copy meta, you
+    # need the whole repository.
+    def bblayers_edit_cb(layerpath, canonical_layerpath):
+        global oldmetapath
+        if not canonical_layerpath.endswith('/'):
+            # This helps us match exactly when we're using this path later
+            canonical_layerpath += '/'
+        if not edited_layers and canonical_layerpath.endswith('/meta/'):
+            canonical_layerpath = os.path.realpath(canonical_layerpath) + '/'
+            edited_layers.append(layerpath)
+            oldmetapath = os.path.realpath(layerpath)
+            result = runCmd('git rev-parse --show-toplevel', cwd=canonical_layerpath)
+            oldreporoot = result.output.rstrip()
+            newmetapath = os.path.join(corecopydir, os.path.relpath(oldmetapath, oldreporoot))
+            runCmd('git clone %s %s' % (oldreporoot, corecopydir), cwd=templayerdir)
+            # Now we need to copy any modified files
+            # You might ask "why not just copy the entire tree instead of
+            # cloning and doing this?" - well, the problem with that is
+            # TMPDIR or an equally large subdirectory might exist
+            # under COREBASE and we don't want to copy that, so we have
+            # to be selective.
+            result = runCmd('git status --porcelain', cwd=oldreporoot)
+            for line in result.output.splitlines():
+                if line.startswith(' M ') or line.startswith('?? '):
+                    relpth = line.split()[1]
+                    pth = os.path.join(oldreporoot, relpth)
+                    if pth.startswith(canonical_layerpath):
+                        if relpth.endswith('/'):
+                            destdir = os.path.join(corecopydir, relpth)
+                            shutil.copytree(pth, destdir)
+                        else:
+                            destdir = os.path.join(corecopydir, os.path.dirname(relpth))
+                            bb.utils.mkdirhier(destdir)
+                            shutil.copy2(pth, destdir)
+            return newmetapath
+        else:
+            return layerpath
+    bb.utils.edit_bblayers_conf(bblayers_conf, None, None, bblayers_edit_cb)
+
+def tearDownModule():
+    if oldmetapath:
+        edited_layers = []
+        def bblayers_edit_cb(layerpath, canonical_layerpath):
+            if not edited_layers and canonical_layerpath.endswith('/meta'):
+                edited_layers.append(layerpath)
+                return oldmetapath
+            else:
+                return layerpath
+        bblayers_conf = os.path.join(os.environ['BUILDDIR'], 'conf', 'bblayers.conf')
+        bb.utils.edit_bblayers_conf(bblayers_conf, None, None, bblayers_edit_cb)
+    shutil.rmtree(templayerdir)
+
 class DevtoolBase(OESelftestTestCase):
 
     buffer = True

meta/lib/oeqa/selftest/cases/distrodata.py

@@ -21,7 +21,7 @@ class Distrodata(OESelftestTestCase):
         Summary:     Test that upstream version checks do not regress
         Expected:    Upstream version checks should succeed except for the recipes listed in the exception list.
         Product:     oe-core
-        Author:      Alexander Kanavin <alexander.kanavin@intel.com>
+        Author:      Alexander Kanavin <alex.kanavin@gmail.com>
         """
         checkpkg_result = open(os.path.join(get_bb_var("LOG_DIR"), "checkpkg.csv")).readlines()[1:]
         regressed_failures = [pkg_data[0] for pkg_data in [pkg_line.split('\t') for pkg_line in checkpkg_result] if pkg_data[11] == 'UNKNOWN_BROKEN']
@@ -46,7 +46,7 @@ but their recipes claim otherwise by setting UPSTREAM_VERSION_UNKNOWN. Please re
         Summary:     Test that oe-core recipes have a maintainer
         Expected:    All oe-core recipes (except a few special static/testing ones) should have a maintainer listed in maintainers.inc file.
         Product:     oe-core
-        Author:      Alexander Kanavin <alexander.kanavin@intel.com>
+        Author:      Alexander Kanavin <alex.kanavin@gmail.com>
         """
         def is_exception(pkg):
             exceptions = ["packagegroup-", "initramfs-", "systemd-machine-units", "target-sdk-provides-dummy"]

meta/lib/oeqa/selftest/cases/eSDK.py

@@ -70,11 +70,13 @@ CORE_IMAGE_EXTRA_INSTALL = "perl"
     @classmethod
     def setUpClass(cls):
         super(oeSDKExtSelfTest, cls).setUpClass()
-        cls.tmpdir_eSDKQA = tempfile.mkdtemp(prefix='eSDKQA')
-
-        sstate_dir = get_bb_var('SSTATE_DIR')
-
         cls.image = 'core-image-minimal'
+
+        bb_vars = get_bb_vars(['SSTATE_DIR', 'WORKDIR'], cls.image)
+        bb.utils.mkdirhier(bb_vars["WORKDIR"])
+        cls.tmpdirobj = tempfile.TemporaryDirectory(prefix="selftest-esdk-", dir=bb_vars["WORKDIR"])
+        cls.tmpdir_eSDKQA = cls.tmpdirobj.name
+
         oeSDKExtSelfTest.generate_eSDK(cls.image)
 
         # Install eSDK
@@ -87,14 +89,14 @@ CORE_IMAGE_EXTRA_INSTALL = "perl"
         sstate_config="""
 SDK_LOCAL_CONF_WHITELIST = "SSTATE_MIRRORS"
 SSTATE_MIRRORS =  "file://.* file://%s/PATH"
-            """ % sstate_dir
+            """ % bb_vars["SSTATE_DIR"]
         with open(os.path.join(cls.tmpdir_eSDKQA, 'conf', 'local.conf'), 'a+') as f:
             f.write(sstate_config)
 
     @classmethod
     def tearDownClass(cls):
-        shutil.rmtree(cls.tmpdir_eSDKQA, ignore_errors=True)
-        super(oeSDKExtSelfTest, cls).tearDownClass()
+        cls.tmpdirobj.cleanup()
+        super().tearDownClass()
 
     @OETestID(1602)
     def test_install_libraries_headers(self):

meta/lib/oeqa/selftest/cases/oelib/path.py

@@ -38,13 +38,6 @@ class TestRealPath(TestCase):
         ( "b/test", errno.ENOENT ),
     ]
 
-    def __del__(self):
-        try:
-            #os.system("tree -F %s" % self.tmpdir)
-            shutil.rmtree(self.tmpdir)
-        except:
-            pass
-
     def setUp(self):
         self.tmpdir = tempfile.mkdtemp(prefix = "oe-test_path")
         self.root = os.path.join(self.tmpdir, "R")
@@ -59,6 +52,9 @@ class TestRealPath(TestCase):
         for l in self.LINKS:
             os.symlink(l[1], os.path.join(self.root, l[0]))
 
+    def tearDown(self):
+        shutil.rmtree(self.tmpdir)
+
     def __realpath(self, file, use_physdir, assume_dir = True):
         return oe.path.realpath(os.path.join(self.root, file), self.root,
                                 use_physdir, assume_dir = assume_dir)

meta/lib/oeqa/selftest/cases/resulttooltests.py

@@ -0,0 +1,94 @@
+import os
+import sys
+basepath = os.path.abspath(os.path.dirname(__file__) + '/../../../../../')
+lib_path = basepath + '/scripts/lib'
+sys.path = sys.path + [lib_path]
+from resulttool.report import ResultsTextReport
+from resulttool import regression as regression
+from resulttool import resultutils as resultutils
+from oeqa.selftest.case import OESelftestTestCase
+
+class ResultToolTests(OESelftestTestCase):
+    base_results_data = {'base_result1': {'configuration': {"TEST_TYPE": "runtime",
+                                                            "TESTSERIES": "series1",
+                                                            "IMAGE_BASENAME": "image",
+                                                            "IMAGE_PKGTYPE": "ipk",
+                                                            "DISTRO": "mydistro",
+                                                            "MACHINE": "qemux86"},
+                                          'result': {}},
+                         'base_result2': {'configuration': {"TEST_TYPE": "runtime",
+                                                            "TESTSERIES": "series1",
+                                                            "IMAGE_BASENAME": "image",
+                                                            "IMAGE_PKGTYPE": "ipk",
+                                                            "DISTRO": "mydistro",
+                                                            "MACHINE": "qemux86-64"},
+                                          'result': {}}}
+    target_results_data = {'target_result1': {'configuration': {"TEST_TYPE": "runtime",
+                                                                "TESTSERIES": "series1",
+                                                                "IMAGE_BASENAME": "image",
+                                                                "IMAGE_PKGTYPE": "ipk",
+                                                                "DISTRO": "mydistro",
+                                                                "MACHINE": "qemux86"},
+                                          'result': {}},
+                           'target_result2': {'configuration': {"TEST_TYPE": "runtime",
+                                                                "TESTSERIES": "series1",
+                                                                "IMAGE_BASENAME": "image",
+                                                                "IMAGE_PKGTYPE": "ipk",
+                                                                "DISTRO": "mydistro",
+                                                                "MACHINE": "qemux86"},
+                                          'result': {}},
+                           'target_result3': {'configuration': {"TEST_TYPE": "runtime",
+                                                                "TESTSERIES": "series1",
+                                                                "IMAGE_BASENAME": "image",
+                                                                "IMAGE_PKGTYPE": "ipk",
+                                                                "DISTRO": "mydistro",
+                                                                "MACHINE": "qemux86-64"},
+                                          'result': {}}}
+
+    def test_report_can_aggregate_test_result(self):
+        result_data = {'result': {'test1': {'status': 'PASSED'},
+                                  'test2': {'status': 'PASSED'},
+                                  'test3': {'status': 'FAILED'},
+                                  'test4': {'status': 'ERROR'},
+                                  'test5': {'status': 'SKIPPED'}}}
+        report = ResultsTextReport()
+        result_report = report.get_aggregated_test_result(None, result_data)
+        self.assertTrue(result_report['passed'] == 2, msg="Passed count not correct:%s" % result_report['passed'])
+        self.assertTrue(result_report['failed'] == 2, msg="Failed count not correct:%s" % result_report['failed'])
+        self.assertTrue(result_report['skipped'] == 1, msg="Skipped count not correct:%s" % result_report['skipped'])
+
+    def test_regression_can_get_regression_base_target_pair(self):
+
+        results = {}
+        resultutils.append_resultsdata(results, ResultToolTests.base_results_data)
+        resultutils.append_resultsdata(results, ResultToolTests.target_results_data)
+        self.assertTrue('target_result1' in results['runtime/mydistro/qemux86/image'], msg="Pair not correct:%s" % results)
+        self.assertTrue('target_result3' in results['runtime/mydistro/qemux86-64/image'], msg="Pair not correct:%s" % results)
+
+    def test_regrresion_can_get_regression_result(self):
+        base_result_data = {'result': {'test1': {'status': 'PASSED'},
+                                       'test2': {'status': 'PASSED'},
+                                       'test3': {'status': 'FAILED'},
+                                       'test4': {'status': 'ERROR'},
+                                       'test5': {'status': 'SKIPPED'}}}
+        target_result_data = {'result': {'test1': {'status': 'PASSED'},
+                                         'test2': {'status': 'FAILED'},
+                                         'test3': {'status': 'PASSED'},
+                                         'test4': {'status': 'ERROR'},
+                                         'test5': {'status': 'SKIPPED'}}}
+        result, text = regression.compare_result(self.logger, "BaseTestRunName", "TargetTestRunName", base_result_data, target_result_data)
+        self.assertTrue(result['test2']['base'] == 'PASSED',
+                        msg="regression not correct:%s" % result['test2']['base'])
+        self.assertTrue(result['test2']['target'] == 'FAILED',
+                        msg="regression not correct:%s" % result['test2']['target'])
+        self.assertTrue(result['test3']['base'] == 'FAILED',
+                        msg="regression not correct:%s" % result['test3']['base'])
+        self.assertTrue(result['test3']['target'] == 'PASSED',
+                        msg="regression not correct:%s" % result['test3']['target'])
+
+    def test_merge_can_merged_results(self):
+        results = {}
+        resultutils.append_resultsdata(results, ResultToolTests.base_results_data, configmap=resultutils.flatten_map)
+        resultutils.append_resultsdata(results, ResultToolTests.target_results_data, configmap=resultutils.flatten_map)
+        self.assertEqual(len(results[''].keys()), 5, msg="Flattened results not correct %s" % str(results))
+

meta/lib/oeqa/selftest/cases/runqemu.py

@@ -5,6 +5,7 @@
 import re
 import tempfile
 import time
+import oe.types
 from oeqa.selftest.case import OESelftestTestCase
 from oeqa.utils.commands import bitbake, runqemu, get_bb_var, runCmd
 from oeqa.core.decorator.oeid import OETestID
@@ -24,6 +25,10 @@ class RunqemuTests(OESelftestTestCase):
         self.fstypes = "ext4 iso hddimg wic.vmdk wic.qcow2 wic.vdi"
         self.cmd_common = "runqemu nographic"
 
+        kvm = oe.types.qemu_use_kvm(get_bb_var('QEMU_USE_KVM'), 'x86_64')
+        if kvm:
+            self.cmd_common += " kvm"
+
         self.write_config(
 """
 MACHINE = "%s"
@@ -44,7 +49,8 @@ SYSLINUX_TIMEOUT = "10"
         """Test runqemu machine"""
         cmd = "%s %s" % (self.cmd_common, self.machine)
         with runqemu(self.recipe, ssh=False, launch_cmd=cmd) as qemu:
-            self.assertTrue(qemu.runner.logged, "Failed: %s" % cmd)
+            with open(qemu.qemurunnerlog) as f:
+                self.assertTrue(qemu.runner.logged, "Failed: %s, %s" % (cmd, f.read()))
 
     @OETestID(2002)
     def test_boot_machine_ext4(self):
@@ -52,7 +58,7 @@ SYSLINUX_TIMEOUT = "10"
         cmd = "%s %s ext4" % (self.cmd_common, self.machine)
         with runqemu(self.recipe, ssh=False, launch_cmd=cmd) as qemu:
             with open(qemu.qemurunnerlog) as f:
-                self.assertTrue('rootfs.ext4' in f.read(), "Failed: %s" % cmd)
+                self.assertIn('rootfs.ext4', f.read(), "Failed: %s" % cmd)
 
     @OETestID(2003)
     def test_boot_machine_iso(self):
@@ -60,14 +66,16 @@ SYSLINUX_TIMEOUT = "10"
         cmd = "%s %s iso" % (self.cmd_common, self.machine)
         with runqemu(self.recipe, ssh=False, launch_cmd=cmd) as qemu:
             with open(qemu.qemurunnerlog) as f:
-                self.assertTrue('media=cdrom' in f.read(), "Failed: %s" % cmd)
+                self.assertIn('media=cdrom', f.read(), "Failed: %s" % cmd)
 
     @OETestID(2004)
     def test_boot_recipe_image(self):
         """Test runqemu recipe-image"""
         cmd = "%s %s" % (self.cmd_common, self.recipe)
         with runqemu(self.recipe, ssh=False, launch_cmd=cmd) as qemu:
-            self.assertTrue(qemu.runner.logged, "Failed: %s" % cmd)
+            with open(qemu.qemurunnerlog) as f:
+                self.assertTrue(qemu.runner.logged, "Failed: %s, %s" % (cmd, f.read()))
+
 
     @OETestID(2005)
     def test_boot_recipe_image_vmdk(self):
@@ -75,7 +83,7 @@ SYSLINUX_TIMEOUT = "10"
         cmd = "%s %s wic.vmdk" % (self.cmd_common, self.recipe)
         with runqemu(self.recipe, ssh=False, launch_cmd=cmd) as qemu:
             with open(qemu.qemurunnerlog) as f:
-                self.assertTrue('format=vmdk' in f.read(), "Failed: %s" % cmd)
+                self.assertIn('format=vmdk', f.read(), "Failed: %s" % cmd)
 
     @OETestID(2006)
     def test_boot_recipe_image_vdi(self):
@@ -83,14 +91,16 @@ SYSLINUX_TIMEOUT = "10"
         cmd = "%s %s wic.vdi" % (self.cmd_common, self.recipe)
         with runqemu(self.recipe, ssh=False, launch_cmd=cmd) as qemu:
             with open(qemu.qemurunnerlog) as f:
-                self.assertTrue('format=vdi' in f.read(), "Failed: %s" % cmd)
+                self.assertIn('format=vdi', f.read(), "Failed: %s" % cmd)
 
     @OETestID(2007)
     def test_boot_deploy(self):
         """Test runqemu deploy_dir_image"""
         cmd = "%s %s" % (self.cmd_common, self.deploy_dir_image)
         with runqemu(self.recipe, ssh=False, launch_cmd=cmd) as qemu:
-            self.assertTrue(qemu.runner.logged, "Failed: %s" % cmd)
+            with open(qemu.qemurunnerlog) as f:
+                self.assertTrue(qemu.runner.logged, "Failed: %s, %s" % (cmd, f.read()))
+
 
     @OETestID(2008)
     def test_boot_deploy_hddimg(self):
@@ -98,7 +108,7 @@ SYSLINUX_TIMEOUT = "10"
         cmd = "%s %s hddimg" % (self.cmd_common, self.deploy_dir_image)
         with runqemu(self.recipe, ssh=False, launch_cmd=cmd) as qemu:
             with open(qemu.qemurunnerlog) as f:
-                self.assertTrue(re.search('file=.*.hddimg', f.read()), "Failed: %s" % cmd)
+                self.assertTrue(re.search('file=.*.hddimg', f.read()), "Failed: %s, %s" % (cmd, f.read()))
 
     @OETestID(2009)
     def test_boot_machine_slirp(self):
@@ -106,7 +116,7 @@ SYSLINUX_TIMEOUT = "10"
         cmd = "%s slirp %s" % (self.cmd_common, self.machine)
         with runqemu(self.recipe, ssh=False, launch_cmd=cmd) as qemu:
             with open(qemu.qemurunnerlog) as f:
-                self.assertTrue(' -netdev user' in f.read(), "Failed: %s" % cmd)
+                self.assertIn(' -netdev user', f.read(), "Failed: %s" % cmd)
 
     @OETestID(2009)
     def test_boot_machine_slirp_qcow2(self):
@@ -114,7 +124,7 @@ SYSLINUX_TIMEOUT = "10"
         cmd = "%s slirp wic.qcow2 %s" % (self.cmd_common, self.machine)
         with runqemu(self.recipe, ssh=False, launch_cmd=cmd) as qemu:
             with open(qemu.qemurunnerlog) as f:
-                self.assertTrue('format=qcow2' in f.read(), "Failed: %s" % cmd)
+                self.assertIn('format=qcow2', f.read(), "Failed: %s" % cmd)
 
     @OETestID(2010)
     def test_boot_qemu_boot(self):
@@ -125,7 +135,8 @@ SYSLINUX_TIMEOUT = "10"
             self.skipTest("%s not found" % qemuboot_conf)
         cmd = "%s %s" % (self.cmd_common, qemuboot_conf)
         with runqemu(self.recipe, ssh=False, launch_cmd=cmd) as qemu:
-            self.assertTrue(qemu.runner.logged, "Failed: %s" % cmd)
+            with open(qemu.qemurunnerlog) as f:
+                self.assertTrue(qemu.runner.logged, "Failed: %s, %s" % (cmd, f.read()))
 
     @OETestID(2011)
     def test_boot_rootfs(self):
@@ -136,7 +147,9 @@ SYSLINUX_TIMEOUT = "10"
             self.skipTest("%s not found" % rootfs)
         cmd = "%s %s" % (self.cmd_common, rootfs)
         with runqemu(self.recipe, ssh=False, launch_cmd=cmd) as qemu:
-            self.assertTrue(qemu.runner.logged, "Failed: %s" % cmd)
+            with open(qemu.qemurunnerlog) as f:
+                self.assertTrue(qemu.runner.logged, "Failed: %s, %s" % (cmd, f.read()))
+
 
 # This test was designed as a separate class to test that shutdown
 # command will shutdown qemu as expected on each qemu architecture

meta/lib/oeqa/selftest/cases/runtime_test.py

@@ -135,7 +135,7 @@ class TestImage(OESelftestTestCase):
         Summary: Check package feeds functionality for dnf
         Expected: 1. Check that remote package feeds can be accessed
         Product: oe-core
-        Author: Alexander Kanavin <alexander.kanavin@intel.com>
+        Author: Alexander Kanavin <alex.kanavin@gmail.com>
         """
         if get_bb_var('DISTRO') == 'poky-tiny':
             self.skipTest('core-image-full-cmdline not buildable for poky-tiny')
@@ -233,7 +233,7 @@ class Postinst(OESelftestTestCase):
         Expected:       The scriptlet failure is properly reported.
                         The file that is created after the error in the scriptlet is not present.
         Product: oe-core
-        Author: Alexander Kanavin <alexander.kanavin@intel.com>
+        Author: Alexander Kanavin <alex.kanavin@gmail.com>
         """
 
         import oe.path

meta/lib/oeqa/selftest/cases/signing.py

@@ -1,10 +1,12 @@
 from oeqa.selftest.case import OESelftestTestCase
 from oeqa.utils.commands import runCmd, bitbake, get_bb_var, get_bb_vars
 import os
+import oe
 import glob
 import re
 import shutil
 import tempfile
+from contextlib import contextmanager
 from oeqa.core.decorator.oeid import OETestID
 from oeqa.utils.ftools import write_file
 
@@ -15,23 +17,39 @@ class Signing(OESelftestTestCase):
     pub_key_path = ""
     secret_key_path = ""
 
-    @classmethod
-    def setUpClass(cls):
-        super(Signing, cls).setUpClass()
-        # Check that we can find the gpg binary and fail early if we can't
-        if not shutil.which("gpg"):
-            raise AssertionError("This test needs GnuPG")
+    def setup_gpg(self):
+        bitbake('gnupg-native -c addto_recipe_sysroot')
 
-        cls.gpg_dir = tempfile.mkdtemp(prefix="oeqa-signing-")
+        self.gpg_dir = tempfile.mkdtemp(prefix="oeqa-signing-")
+        self.track_for_cleanup(self.gpg_dir)
 
-        cls.pub_key_path = os.path.join(cls.testlayer_path, 'files', 'signing', "key.pub")
-        cls.secret_key_path = os.path.join(cls.testlayer_path, 'files', 'signing', "key.secret")
+        self.pub_key_path = os.path.join(self.testlayer_path, 'files', 'signing', "key.pub")
+        self.secret_key_path = os.path.join(self.testlayer_path, 'files', 'signing', "key.secret")
 
-        runCmd('gpg --batch --homedir %s --import %s %s' % (cls.gpg_dir, cls.pub_key_path, cls.secret_key_path))
+        nsysroot = get_bb_var("RECIPE_SYSROOT_NATIVE", "gnupg-native")
+        runCmd('gpg --batch --homedir %s --import %s %s' % (self.gpg_dir, self.pub_key_path, self.secret_key_path), native_sysroot=nsysroot)
+        return nsysroot + get_bb_var("bindir_native")
 
-    @classmethod
-    def tearDownClass(cls):
-        shutil.rmtree(cls.gpg_dir, ignore_errors=True)
+
+    @contextmanager
+    def create_new_builddir(self, builddir, newbuilddir):
+        bb.utils.mkdirhier(newbuilddir)
+        oe.path.copytree(builddir + "/conf", newbuilddir + "/conf")
+        oe.path.copytree(builddir + "/cache", newbuilddir + "/cache")
+
+        origenv = os.environ.copy()
+
+        for e in os.environ:
+            if builddir in os.environ[e]:
+                os.environ[e] = os.environ[e].replace(builddir, newbuilddir)
+
+        os.chdir(newbuilddir)
+        try:
+            yield
+        finally:
+            for e in origenv:
+                os.environ[e] = origenv[e]
+            os.chdir(builddir)
 
     @OETestID(1362)
     def test_signing_packages(self):
@@ -41,11 +59,13 @@ class Signing(OESelftestTestCase):
         Expected:    Images can be created from signed packages
         Product:     oe-core
         Author:      Daniel Istrate <daniel.alexandrux.istrate@intel.com>
-        Author:      Alexander Kanavin <alexander.kanavin@intel.com>
+        Author:      Alexander Kanavin <alex.kanavin@gmail.com>
         AutomatedBy: Daniel Istrate <daniel.alexandrux.istrate@intel.com>
         """
         import oe.packagedata
 
+        self.setup_gpg()
+
         package_classes = get_bb_var('PACKAGE_CLASSES')
         if 'package_rpm' not in package_classes:
             self.skipTest('This test requires RPM Packaging.')
@@ -108,11 +128,12 @@ class Signing(OESelftestTestCase):
 
         test_recipe = 'ed'
 
-        builddir = os.environ.get('BUILDDIR')
+        # Since we need gpg but we can't use gpg-native for sstate signatures, we 
+        # build gpg-native in our original builddir then run the tests in a second one.
+        builddir = os.environ.get('BUILDDIR') + "-testsign"
         sstatedir = os.path.join(builddir, 'test-sstate')
 
-        self.add_command_to_tearDown('bitbake -c clean %s' % test_recipe)
-        self.add_command_to_tearDown('rm -rf %s' % sstatedir)
+        nsysroot = self.setup_gpg()
 
         feature = 'SSTATE_SIG_KEY ?= "testuser"\n'
         feature += 'SSTATE_SIG_PASSPHRASE ?= "test123"\n'
@@ -124,19 +145,26 @@ class Signing(OESelftestTestCase):
 
         self.write_config(feature)
 
-        bitbake('-c clean %s' % test_recipe)
-        bitbake(test_recipe)
+        with self.create_new_builddir(os.environ['BUILDDIR'], builddir):
 
-        recipe_sig = glob.glob(sstatedir + '/*/*:ed:*_package.tgz.sig')
-        recipe_tgz = glob.glob(sstatedir + '/*/*:ed:*_package.tgz')
+            os.environ["PATH"] = nsysroot + ":" + os.environ["PATH"]
+            self.add_command_to_tearDown('bitbake -c clean %s' % test_recipe)
+            self.add_command_to_tearDown('rm -rf %s' % sstatedir)
+            self.add_command_to_tearDown('rm -rf %s' % builddir)
 
-        self.assertEqual(len(recipe_sig), 1, 'Failed to find .sig file.')
-        self.assertEqual(len(recipe_tgz), 1, 'Failed to find .tgz file.')
+            bitbake('-c clean %s' % test_recipe)
+            bitbake('-c populate_lic %s' % test_recipe)
 
-        ret = runCmd('gpg --homedir %s --verify %s %s' % (self.gpg_dir, recipe_sig[0], recipe_tgz[0]))
-        # gpg: Signature made Thu 22 Oct 2015 01:45:09 PM EEST using RSA key ID 61EEFB30
-        # gpg: Good signature from "testuser (nocomment) <testuser@email.com>"
-        self.assertIn('gpg: Good signature from', ret.output, 'Package signed incorrectly.')
+            recipe_sig = glob.glob(sstatedir + '/*/*:ed:*_populate_lic.tgz.sig')
+            recipe_tgz = glob.glob(sstatedir + '/*/*:ed:*_populate_lic.tgz')
+
+            self.assertEqual(len(recipe_sig), 1, 'Failed to find .sig file.')
+            self.assertEqual(len(recipe_tgz), 1, 'Failed to find .tgz file.')
+
+            ret = runCmd('gpg --homedir %s --verify %s %s' % (self.gpg_dir, recipe_sig[0], recipe_tgz[0]))
+            # gpg: Signature made Thu 22 Oct 2015 01:45:09 PM EEST using RSA key ID 61EEFB30
+            # gpg: Good signature from "testuser (nocomment) <testuser@email.com>"
+            self.assertIn('gpg: Good signature from', ret.output, 'Package signed incorrectly.')
 
 
 class LockedSignatures(OESelftestTestCase):

meta/lib/oeqa/selftest/context.py

@@ -5,12 +5,13 @@ import os
 import time
 import glob
 import sys
-import imp
+import importlib
 import signal
 from shutil import copyfile
 from random import choice
 
 import oeqa
+import oe
 
 from oeqa.core.context import OETestContext, OETestContextExecutor
 from oeqa.core.exception import OEQAPreRun, OEQATestNotFound
@@ -96,11 +97,17 @@ class OESelftestTestContextExecutor(OETestContextExecutor):
         return cases_paths
 
     def _process_args(self, logger, args):
-        args.output_log = '%s-results-%s.log' % (self.name,
-                time.strftime("%Y%m%d%H%M%S"))
+
+        args.test_start_time = time.strftime("%Y%m%d%H%M%S")
         args.test_data_file = None
         args.CASES_PATHS = None
 
+        bbvars = get_bb_vars()
+        logdir = os.environ.get("BUILDDIR")
+        if 'LOG_DIR' in bbvars:
+            logdir = bbvars['LOG_DIR']
+        args.output_log = logdir + '/%s-results-%s.log' % (self.name, args.test_start_time)
+
         super(OESelftestTestContextExecutor, self)._process_args(logger, args)
 
         if args.list_modules:
@@ -110,7 +117,7 @@ class OESelftestTestContextExecutor(OETestContextExecutor):
         elif args.list_tests:
             args.list_tests = 'name'
 
-        self.tc_kwargs['init']['td'] = get_bb_vars()
+        self.tc_kwargs['init']['td'] = bbvars
         self.tc_kwargs['init']['machines'] = self._get_available_machines()
 
         builddir = os.environ.get("BUILDDIR")
@@ -174,7 +181,7 @@ class OESelftestTestContextExecutor(OETestContextExecutor):
                     self.tc.logger.info("\t%s" % l)
 
                 sys.path.extend(layer_libdirs)
-                imp.reload(oeqa.selftest)
+                importlib.reload(oeqa.selftest)
 
         _check_required_env_variables(["BUILDDIR"])
         _check_presence_meta_selftest()
@@ -196,6 +203,28 @@ class OESelftestTestContextExecutor(OETestContextExecutor):
         self.tc.logger.info("Running bitbake -p")
         runCmd("bitbake -p")
 
+    def get_json_result_dir(self, args):
+        json_result_dir = os.path.join(self.tc.td["LOG_DIR"], 'oeqa')
+        if "OEQA_JSON_RESULT_DIR" in self.tc.td:
+            json_result_dir = self.tc.td["OEQA_JSON_RESULT_DIR"]
+
+        return json_result_dir
+
+    def get_configuration(self, args):
+        import platform
+        from oeqa.utils.metadata import metadata_from_bb
+        metadata = metadata_from_bb()
+        configuration = {'TEST_TYPE': 'oeselftest',
+                        'STARTTIME': args.test_start_time,
+                        'MACHINE': self.tc.td["MACHINE"],
+                        'HOST_DISTRO': oe.lsb.distro_identifier().replace(' ', '-'),
+                        'HOST_NAME': metadata['hostname'],
+                        'LAYERS': metadata['layers']}
+        return configuration
+
+    def get_result_id(self, configuration):
+        return '%s_%s_%s_%s' % (configuration['TEST_TYPE'], configuration['HOST_DISTRO'], configuration['MACHINE'], configuration['STARTTIME'])
+
     def _internal_run(self, logger, args):
         self.module_paths = self._get_cases_paths(
                 self.tc_kwargs['init']['td']['BBPATH'].split(':'))
@@ -212,7 +241,10 @@ class OESelftestTestContextExecutor(OETestContextExecutor):
         else:
             self._pre_run()
             rc = self.tc.runTests(**self.tc_kwargs['run'])
-            rc.logDetails()
+            configuration = self.get_configuration(args)
+            rc.logDetails(self.get_json_result_dir(args),
+                          configuration,
+                          self.get_result_id(configuration))
             rc.logSummary(self.name)
 
         return rc
@@ -270,7 +302,7 @@ class OESelftestTestContextExecutor(OETestContextExecutor):
 
             output_link = os.path.join(os.path.dirname(args.output_log),
                     "%s-results.log" % self.name)
-            if os.path.exists(output_link):
+            if os.path.lexists(output_link):
                 os.remove(output_link)
             os.symlink(args.output_log, output_link)
 

meta/lib/oeqa/targetcontrol.py

@@ -107,13 +107,9 @@ class QemuTarget(BaseTarget):
         dump_target_cmds = d.getVar("testimage_dump_target")
         dump_host_cmds = d.getVar("testimage_dump_host")
         dump_dir = d.getVar("TESTIMAGE_DUMP_DIR")
-        qemu_use_kvm = d.getVar("QEMU_USE_KVM")
-        if qemu_use_kvm and \
-           (oe.types.boolean(qemu_use_kvm) and "x86" in d.getVar("MACHINE") or \
-            d.getVar("MACHINE") in qemu_use_kvm.split()):
-            use_kvm = True
-        else:
-            use_kvm = False
+        if not dump_dir:
+            dump_dir = os.path.join(d.getVar('LOG_DIR'), 'runtime-hostdump')
+        use_kvm = oe.types.qemu_use_kvm(d.getVar('QEMU_USE_KVM'), d.getVar('TARGET_ARCH'))
 
         # Log QemuRunner log output to a file
         import oe.path
@@ -196,7 +192,7 @@ class QemuTarget(BaseTarget):
         else:
             raise bb.build.FuncFailed("%s - FAILED to re-start qemu - check the task log and the boot log" % self.pn)
 
-    def run_serial(self, command, timeout=5):
+    def run_serial(self, command, timeout=60):
         return self.runner.run_serial(command, timeout=timeout)
 
 

meta/lib/oeqa/utils/buildproject.py

@@ -17,16 +17,20 @@ class BuildProject(metaclass=ABCMeta):
         self.uri = uri
         self.archive = os.path.basename(uri)
         if not tmpdir:
-            tmpdir = tempfile.mkdtemp(prefix='buildproject')
+            self.tempdirobj = tempfile.TemporaryDirectory(prefix='buildproject-')
+            tmpdir = self.tempdirobj.name
         self.localarchive = os.path.join(tmpdir, self.archive)
         self.dl_dir = dl_dir
         if foldername:
             self.fname = foldername
         else:
             self.fname = re.sub(r'\.tar\.bz2$|\.tar\.gz$|\.tar\.xz$', '', self.archive)
+        self.needclean = False
 
     # Download self.archive to self.localarchive
     def _download_archive(self):
+
+        self.needclean = True
         if self.dl_dir and os.path.exists(os.path.join(self.dl_dir, self.archive)):
             shutil.copyfile(os.path.join(self.dl_dir, self.archive), self.localarchive)
             return
@@ -51,5 +55,7 @@ class BuildProject(metaclass=ABCMeta):
         return self._run('cd %s; make install %s' % (self.targetdir, install_args))
 
     def clean(self):
+        if not self.needclean:
+             return
         self._run('rm -rf %s' % self.targetdir)
         subprocess.check_call('rm -f %s' % self.localarchive, shell=True)

meta/lib/oeqa/utils/commands.py

@@ -146,6 +146,9 @@ class Command(object):
         # At this point we know that the process has closed stdout/stderr, so
         # it is safe and necessary to wait for the actual process completion.
         self.status = self.process.wait()
+        self.process.stdout.close()
+        if self.process.stderr:
+            self.process.stderr.close()
 
         self.log.debug("Command '%s' returned %d as exit code." % (self.cmd, self.status))
         # logging the complete output is insane
@@ -333,16 +336,20 @@ def runqemu(pn, ssh=True, runqemuparams='', image_fstype=None, launch_cmd=None,
         try:
             qemu.start(params=qemuparams, ssh=ssh, runqemuparams=runqemuparams, launch_cmd=launch_cmd, discard_writes=discard_writes)
         except bb.build.FuncFailed:
-            raise Exception('Failed to start QEMU - see the logs in %s' % logdir)
+            msg = 'Failed to start QEMU - see the logs in %s' % logdir
+            if os.path.exists(qemu.qemurunnerlog):
+                with open(qemu.qemurunnerlog, 'r') as f:
+                    msg = msg + "Qemurunner log output from %s:\n%s" % (qemu.qemurunnerlog, f.read())
+            raise Exception(msg)
 
         yield qemu
 
     finally:
+        targetlogger.removeHandler(handler)
         try:
             qemu.stop()
         except:
             pass
-    targetlogger.removeHandler(handler)
 
 def updateEnv(env_file):
     """