bitbake: daemonize: Always print any remaning UI events at exit

If there are events in the UI queue we want to print them regardless of
whether we're handling an exception or something like SystemExit.

This improves error messages for some other failure cases where currently
no logging would get printed and leave the user confused as to what went
wrong.

(Bitbake rev: 76c4f2c20216719736766e8ae7d089ccd061b71b)

Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>

bitbake/lib/bb/cooker.py

@@ -165,7 +165,7 @@ class BBCooker:
     Manages one bitbake build run
     """
 
-    def __init__(self, configuration, featureSet=None):
+    def __init__(self, configuration, featureSet=None, readypipe=None):
         self.recipecaches = None
         self.skiplist = {}
         self.featureset = CookerFeatures()
@@ -237,6 +237,10 @@ class BBCooker:
         # Let SIGHUP exit as SIGTERM
         signal.signal(signal.SIGHUP, self.sigterm_exception)
 
+        if readypipe:
+            os.write(readypipe, b"ready")
+            os.close(readypipe)
+
     def config_notifications(self, event):
         if event.maskname == "IN_Q_OVERFLOW":
             bb.warn("inotify event queue overflowed, invalidating caches.")
@@ -674,12 +678,12 @@ class BBCooker:
                     depend_tree["pn"][pn][ei] = vars(self.recipecaches[mc])[ei][taskfn]
 
 
+            dotname = "%s.%s" % (pn, bb.runqueue.taskname_from_tid(tid))
+            if not dotname in depend_tree["tdepends"]:
+                depend_tree["tdepends"][dotname] = []
             for dep in rq.rqdata.runtaskentries[tid].depends:
                 (depmc, depfn, deptaskname, deptaskfn) = bb.runqueue.split_tid_mcfn(dep)
                 deppn = self.recipecaches[mc].pkg_fn[deptaskfn]
-                dotname = "%s.%s" % (pn, bb.runqueue.taskname_from_tid(tid))
-                if not dotname in depend_tree["tdepends"]:
-                    depend_tree["tdepends"][dotname] = []
                 depend_tree["tdepends"][dotname].append("%s.%s" % (deppn, bb.runqueue.taskname_from_tid(dep)))
             if taskfn not in seen_fns:
                 seen_fns.append(taskfn)

bitbake/lib/bb/daemonize.py

@@ -1,49 +1,14 @@
 """
 Python Daemonizing helper
 
-Configurable daemon behaviors:
-
-    1.) The current working directory set to the "/" directory.
-    2.) The current file creation mode mask set to 0.
-    3.) Close all open files (1024). 
-    4.) Redirect standard I/O streams to "/dev/null".
-
-A failed call to fork() now raises an exception.
-
-References:
-    1) Advanced Programming in the Unix Environment: W. Richard Stevens
-	http://www.apuebook.com/apue3e.html
-    2) The Linux Programming Interface: Michael Kerrisk
-	http://man7.org/tlpi/index.html
-    3) Unix Programming Frequently Asked Questions:
-	http://www.faqs.org/faqs/unix-faq/programmer/faq/
-
-Modified to allow a function to be daemonized and return for 
-bitbake use by Richard Purdie
+Originally based on code Copyright (C) 2005 Chad J. Schroeder but now heavily modified
+to allow a function to be daemonized and return for bitbake use by Richard Purdie
 """
 
-__author__ = "Chad J. Schroeder"
-__copyright__ = "Copyright (C) 2005 Chad J. Schroeder"
-__version__ = "0.2"
-
-# Standard Python modules.
-import os                    # Miscellaneous OS interfaces.
-import sys                   # System-specific parameters and functions.
+import os
+import sys
 import io
-
-# Default daemon parameters.
-# File mode creation mask of the daemon.
-# For BitBake's children, we do want to inherit the parent umask.
-UMASK = None
-
-# Default maximum for the number of available file descriptors.
-MAXFD = 1024
-
-# The standard I/O file descriptors are redirected to /dev/null by default.
-if (hasattr(os, "devnull")):
-    REDIRECT_TO = os.devnull
-else:
-    REDIRECT_TO = "/dev/null"
+import traceback
 
 def createDaemon(function, logfile):
     """
@@ -66,36 +31,6 @@ def createDaemon(function, logfile):
         # leader of the new process group, we call os.setsid().  The process is
         # also guaranteed not to have a controlling terminal.
         os.setsid()
-
-        # Is ignoring SIGHUP necessary?
-        #
-        # It's often suggested that the SIGHUP signal should be ignored before
-        # the second fork to avoid premature termination of the process.  The
-        # reason is that when the first child terminates, all processes, e.g.
-        # the second child, in the orphaned group will be sent a SIGHUP.
-        #
-        # "However, as part of the session management system, there are exactly
-        # two cases where SIGHUP is sent on the death of a process:
-        #
-        #    1) When the process that dies is the session leader of a session that
-        #        is attached to a terminal device, SIGHUP is sent to all processes
-        #        in the foreground process group of that terminal device.
-        #    2) When the death of a process causes a process group to become
-        #        orphaned, and one or more processes in the orphaned group are
-        #        stopped, then SIGHUP and SIGCONT are sent to all members of the
-        #        orphaned group." [2]
-        #
-        # The first case can be ignored since the child is guaranteed not to have
-        # a controlling terminal.  The second case isn't so easy to dismiss.
-        # The process group is orphaned when the first child terminates and
-        # POSIX.1 requires that every STOPPED process in an orphaned process
-        # group be sent a SIGHUP signal followed by a SIGCONT signal.  Since the
-        # second child is not STOPPED though, we can safely forego ignoring the
-        # SIGHUP signal.  In any case, there are no ill-effects if it is ignored.
-        #
-        # import signal              # Set handlers for asynchronous events.
-        # signal.signal(signal.SIGHUP, signal.SIG_IGN)
-
         try:
             # Fork a second child and exit immediately to prevent zombies.  This
             # causes the second child process to be orphaned, making the init
@@ -109,77 +44,23 @@ def createDaemon(function, logfile):
         except OSError as e:
             raise Exception("%s [%d]" % (e.strerror, e.errno))
 
-        if (pid == 0):  # The second child.
-            # We probably don't want the file mode creation mask inherited from
-            # the parent, so we give the child complete control over permissions.
-            if UMASK is not None:
-                os.umask(UMASK)
-        else:
+        if (pid != 0):
             # Parent (the first child) of the second child.
+            # exit() or _exit()?
+            # _exit is like exit(), but it doesn't call any functions registered
+            # with atexit (and on_exit) or any registered signal handlers.  It also
+            # closes any open file descriptors.  Using exit() may cause all stdio
+            # streams to be flushed twice and any temporary files may be unexpectedly
+            # removed.  It's therefore recommended that child branches of a fork()
+            # and the parent branch(es) of a daemon use _exit().
             os._exit(0)
     else:
-        # exit() or _exit()?
-        # _exit is like exit(), but it doesn't call any functions registered
-        # with atexit (and on_exit) or any registered signal handlers.  It also
-        # closes any open file descriptors.  Using exit() may cause all stdio
-        # streams to be flushed twice and any temporary files may be unexpectedly
-        # removed.  It's therefore recommended that child branches of a fork()
-        # and the parent branch(es) of a daemon use _exit().
         os.waitpid(pid, 0)
         return
 
-    # Close all open file descriptors.  This prevents the child from keeping
-    # open any file descriptors inherited from the parent.  There is a variety
-    # of methods to accomplish this task.  Three are listed below.
-    #
-    # Try the system configuration variable, SC_OPEN_MAX, to obtain the maximum
-    # number of open file descriptors to close.  If it doesn't exist, use
-    # the default value (configurable).
-    #
-    # try:
-    #     maxfd = os.sysconf("SC_OPEN_MAX")
-    # except (AttributeError, ValueError):
-    #     maxfd = MAXFD
-    #
-    # OR
-    #
-    # if (os.sysconf_names.has_key("SC_OPEN_MAX")):
-    #     maxfd = os.sysconf("SC_OPEN_MAX")
-    # else:
-    #     maxfd = MAXFD
-    #
-    # OR
-    #
-    # Use the getrlimit method to retrieve the maximum file descriptor number
-    # that can be opened by this process.  If there is no limit on the
-    # resource, use the default value.
-    #
-    import resource             # Resource usage information.
-    maxfd = resource.getrlimit(resource.RLIMIT_NOFILE)[1]
-    if (maxfd == resource.RLIM_INFINITY):
-        maxfd = MAXFD
-  
-    # Iterate through and close all file descriptors.
-#    for fd in range(0, maxfd):
-#        try:
-#            os.close(fd)
-#        except OSError:        # ERROR, fd wasn't open to begin with (ignored)
-#            pass
+    # The second child.
 
-    # Redirect the standard I/O file descriptors to the specified file.  Since
-    # the daemon has no controlling terminal, most daemons redirect stdin,
-    # stdout, and stderr to /dev/null.  This is done to prevent side-effects
-    # from reads and writes to the standard I/O file descriptors.
-
-    # This call to open is guaranteed to return the lowest file descriptor,
-    # which will be 0 (stdin), since it was closed above.
-#    os.open(REDIRECT_TO, os.O_RDWR)    # standard input (0)
-
-    # Duplicate standard input to standard output and standard error.
-#    os.dup2(0, 1)                      # standard output (1)
-#    os.dup2(0, 2)                      # standard error (2)
-
-    # Replace those fds with our own
+    # Replace standard fds with our own
     si = open('/dev/null', 'r')
     os.dup2(si.fileno(), sys.stdin.fileno())
 
@@ -192,6 +73,10 @@ def createDaemon(function, logfile):
         sys.stdout = open(logfile, 'a+')
         sys.stderr = sys.stdout
 
-    function()
-
-    os._exit(0)
+    try:
+        function()
+    except Exception as e:
+        traceback.print_exc()
+    finally:
+        bb.event.print_ui_queue()
+        os._exit(0)

bitbake/lib/bb/event.py

@@ -149,23 +149,30 @@ def print_ui_queue():
 
         # First check to see if we have any proper messages
         msgprint = False
+        msgerrs = False
+
+        # Should we print to stderr?
+        for event in ui_queue[:]:
+            if isinstance(event, logging.LogRecord) and event.levelno >= logging.WARNING:
+                msgerrs = True
+                break
+
+        if msgerrs:
+            logger.addHandler(stderr)
+        else:
+            logger.addHandler(stdout)
+
         for event in ui_queue[:]:
             if isinstance(event, logging.LogRecord):
                 if event.levelno > logging.DEBUG:
-                    if event.levelno >= logging.WARNING:
-                        logger.addHandler(stderr)
-                    else:
-                        logger.addHandler(stdout)
                     logger.handle(event)
                     msgprint = True
-        if msgprint:
-            return
 
         # Nope, so just print all of the messages we have (including debug messages)
-        logger.addHandler(stdout)
-        for event in ui_queue[:]:
-            if isinstance(event, logging.LogRecord):
-                logger.handle(event)
+        if not msgprint:
+            for event in ui_queue[:]:
+                if isinstance(event, logging.LogRecord):
+                    logger.handle(event)
 
 def fire_ui_handlers(event, d):
     global _thread_lock

bitbake/lib/bb/fetch2/__init__.py

@@ -1726,9 +1726,8 @@ class Fetch(object):
             ret = try_mirrors(self, self.d, ud, mirrors, True)
             if not ret:
                 # Next try checking from the original uri, u
-                try:
-                    ret = m.checkstatus(self, ud, self.d)
-                except:
+                ret = m.checkstatus(self, ud, self.d)
+                if not ret:
                     # Finally, try checking uri, u, from MIRRORS
                     mirrors = mirror_from_string(self.d.getVar('MIRRORS'))
                     ret = try_mirrors(self, self.d, ud, mirrors, True)

bitbake/lib/bb/fetch2/wget.py

@@ -283,11 +283,6 @@ class Wget(FetchMethod):
             """
             http_error_403 = http_error_405
 
-            """
-            Some servers (e.g. FusionForge) returns 406 Not Acceptable when they
-            actually mean 405 Method Not Allowed.
-            """
-            http_error_406 = http_error_405
 
         class FixedHTTPRedirectHandler(urllib.request.HTTPRedirectHandler):
             """
@@ -316,7 +311,9 @@ class Wget(FetchMethod):
             uri = ud.url.split(";")[0]
             r = urllib.request.Request(uri)
             r.get_method = lambda: "HEAD"
-
+            # Some servers (FusionForge, as used on Alioth) require that the
+            # optional Accept header is set.
+            r.add_header("Accept", "*/*")
             def add_basic_auth(login_str, request):
                 '''Adds Basic auth to http request, pass in login:password as string'''
                 import base64

bitbake/lib/bb/main.py

@@ -277,7 +277,7 @@ class BitBakeConfigParameters(cookerdata.ConfigParameters):
 
         parser.add_option("-m", "--kill-server", action="store_true",
                           dest="kill_server", default=False,
-                          help="Terminate the bitbake server.")
+                          help="Terminate any running bitbake server.")
 
         parser.add_option("", "--observe-only", action="store_true",
                           dest="observe_only", default=False,
@@ -370,8 +370,10 @@ def bitbake_main(configParams, configuration):
     server_connection, ui_module = setup_bitbake(configParams, configuration)
     # No server connection
     if server_connection is None:
-        if configParams.status_only or configParams.kill_server:
+        if configParams.status_only:
             return 1
+        if configParams.kill_server:
+            return 0
 
     if not configParams.server_only:
         if configParams.status_only:

bitbake/lib/bb/runqueue.py

@@ -2372,7 +2372,7 @@ class RunQueueExecuteScenequeue(RunQueueExecute):
         self.rq.scenequeue_covered = self.scenequeue_covered
         self.rq.scenequeue_notcovered = self.scenequeue_notcovered
 
-        logger.debug(1, 'We can skip tasks %s', sorted(self.rq.scenequeue_covered))
+        logger.debug(1, 'We can skip tasks %s', "\n".join(sorted(self.rq.scenequeue_covered)))
 
         self.rq.state = runQueueRunInit
 

bitbake/lib/bb/server/process.py

@@ -100,7 +100,8 @@ class ProcessServer(multiprocessing.Process):
             else:
                 self.bitbake_lock.write("%s\n" % (os.getpid()))
             self.bitbake_lock.flush()
-        except:
+        except Exception as e:
+            print("Error writing to lock file: %s" % str(e))
             pass
 
         if self.cooker.configuration.profile:
@@ -132,20 +133,8 @@ class ProcessServer(multiprocessing.Process):
         fds = [self.sock]
         if self.xmlrpc:
             fds.append(self.xmlrpc)
+        print("Entering server connection loop")
         while not self.quit:
-            if self.command_channel in ready:
-                command = self.command_channel.get()
-                if command[0] == "terminateServer":
-                    self.quit = True
-                    continue
-                try:
-                    print("Running command %s" % command)
-                    self.command_channel_reply.send(self.cooker.command.runCommand(command))
-                except Exception as e:
-                   logger.exception('Exception in server main event loop running command %s (%s)' % (command, str(e)))
-
-            if self.xmlrpc in ready:
-                self.xmlrpc.handle_requests()
             if self.sock in ready:
                 self.controllersock, address = self.sock.accept()
                 if self.haveui:
@@ -194,6 +183,24 @@ class ProcessServer(multiprocessing.Process):
                 print("Server timeout, exiting.")
                 self.quit = True
 
+            if self.command_channel in ready:
+                try:
+                    command = self.command_channel.get()
+                except EOFError:
+                    # Client connection shutting down
+                    continue
+                if command[0] == "terminateServer":
+                    self.quit = True
+                    continue
+                try:
+                    print("Running command %s" % command)
+                    self.command_channel_reply.send(self.cooker.command.runCommand(command))
+                except Exception as e:
+                   logger.exception('Exception in server main event loop running command %s (%s)' % (command, str(e)))
+
+            if self.xmlrpc in ready:
+                self.xmlrpc.handle_requests()
+
             ready = self.idle_commands(.1, fds)
 
         print("Exiting")
@@ -303,19 +310,10 @@ class ServerCommunicator():
         self.recv = recv
 
     def runCommand(self, command):
-
         self.connection.send(command)
-        while True:
-            # don't let the user ctrl-c while we're waiting for a response
-            try:
-                for idx in range(0,4): # 0, 1, 2, 3
-                    if self.recv.poll(1):
-                        return self.recv.get()
-                    else:
-                        bb.note("Timeout while attempting to communicate with bitbake server, retrying...")
-                raise ProcessTimeout("Gave up; Too many tries: timeout while attempting to communicate with bitbake server")
-            except KeyboardInterrupt:
-                pass
+        if not self.recv.poll(5):
+            raise ProcessTimeout("Timeout while waiting for a reply from the bitbake server")
+        return self.recv.get()
 
     def updateFeatureSet(self, featureset):
         _, error = self.runCommand(["setFeatures", featureset])
@@ -336,12 +334,16 @@ class ServerCommunicator():
         return
 
 class BitBakeProcessServerConnection(object):
-    def __init__(self, ui_channel, recv, eq):
+    def __init__(self, ui_channel, recv, eq, sock):
         self.connection = ServerCommunicator(ui_channel, recv)
         self.events = eq
+        # Save sock so it doesn't get gc'd for the life of our connection
+        self.socket_connection = sock
 
     def terminate(self):
         self.socket_connection.close()
+        self.connection.connection.close()
+        self.connection.recv.close()
         return
 
 class BitBakeServer(object):
@@ -351,6 +353,7 @@ class BitBakeServer(object):
         self.featureset = featureset
         self.sockname = sockname
         self.bitbake_lock = lock
+        self.readypipe, self.readypipein = os.pipe()
 
         # Create server control socket
         if os.path.exists(sockname):
@@ -359,6 +362,8 @@ class BitBakeServer(object):
         self.sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
         # AF_UNIX has path length issues so chdir here to workaround
         cwd = os.getcwd()
+        logfile = os.path.join(cwd, "bitbake-cookerdaemon.log")
+
         try:
             os.chdir(os.path.dirname(sockname))
             self.sock.bind(os.path.basename(sockname))
@@ -367,11 +372,25 @@ class BitBakeServer(object):
         self.sock.listen(1)
 
         os.set_inheritable(self.sock.fileno(), True)
-        bb.daemonize.createDaemon(self._startServer, "bitbake-cookerdaemon.log")
+        bb.daemonize.createDaemon(self._startServer, logfile)
         self.sock.close()
         self.bitbake_lock.close()
 
+        ready = ConnectionReader(self.readypipe)
+        r = ready.wait(8)
+        if not r:
+            ready.close()
+            bb.error("Unable to start bitbake server")
+            if os.path.exists(logfile):
+                with open(logfile, "r") as f:
+                    logs=f.readlines()
+                bb.error("Last 10 lines of server log %s:\n%s" % (logfile, "".join(logs[-10:])))
+            raise SystemExit(1)
+        ready.close()
+        os.close(self.readypipein)
+
     def _startServer(self):
+        print("Starting bitbake server pid %d" % os.getpid())
         server = ProcessServer(self.bitbake_lock, self.sock, self.sockname)
         self.configuration.setServerRegIdleCallback(server.register_idle_function)
 
@@ -381,7 +400,7 @@ class BitBakeServer(object):
             if value:
                 setattr(self.configuration, "%s_server" % param, value)
 
-        self.cooker = bb.cooker.BBCooker(self.configuration, self.featureset)
+        self.cooker = bb.cooker.BBCooker(self.configuration, self.featureset, self.readypipein)
         server.cooker = self.cooker
         server.server_timeout = self.configuration.server_timeout
         server.xmlrpcinterface = self.configuration.xmlrpcinterface
@@ -400,7 +419,11 @@ def connectProcessServer(sockname, featureset):
     finally:
         os.chdir(cwd)
 
+    readfd = writefd = readfd1 = writefd1 = readfd2 = writefd2 = None
+    eq = command_chan_recv = command_chan = None
+
     try:
+
         # Send an fd for the remote to write events to
         readfd, writefd = os.pipe()
         eq = BBUIEventQueue(readfd)
@@ -413,13 +436,24 @@ def connectProcessServer(sockname, featureset):
 
         sendfds(sock, [writefd, readfd1, writefd2])
 
-        server_connection = BitBakeProcessServerConnection(command_chan, command_chan_recv, eq)
+        server_connection = BitBakeProcessServerConnection(command_chan, command_chan_recv, eq, sock)
+
+        # Close the ends of the pipes we won't use
+        for i in [writefd, readfd1, writefd2]:
+            os.close(i)
 
         server_connection.connection.updateFeatureSet(featureset)
 
-        # Save sock so it doesn't get gc'd for the life of our connection
-        server_connection.socket_connection = sock
-    except:
+    except (Exception, SystemExit) as e:
+        if command_chan_recv:
+            command_chan_recv.close()
+        if command_chan:
+            command_chan.close()
+        for i in [writefd, readfd1, writefd2]:
+            try:
+                os.close(i)
+            except OSError:
+                pass
         sock.close()
         raise
 
@@ -499,9 +533,14 @@ class BBUIEventQueue:
     def startCallbackHandler(self):
         bb.utils.set_process_name("UIEventQueue")
         while True:
-            self.reader.wait()
-            event = self.reader.get()
-            self.queue_event(event)
+            try:
+                self.reader.wait()
+                event = self.reader.get()
+                self.queue_event(event)
+            except EOFError:
+                # Easiest way to exit is to close the file descriptor to cause an exit
+                break
+        self.reader.close()
 
 class ConnectionReader(object):
 
@@ -523,6 +562,10 @@ class ConnectionReader(object):
     def fileno(self):
         return self.reader.fileno()
 
+    def close(self):
+        return self.reader.close()
+
+
 class ConnectionWriter(object):
 
     def __init__(self, fd):
@@ -536,3 +579,8 @@ class ConnectionWriter(object):
         with self.wlock:
             self.writer.send_bytes(obj)
 
+    def fileno(self):
+        return self.writer.fileno()
+
+    def close(self):
+        return self.writer.close()

bitbake/lib/bb/tests/fetch.py

@@ -1440,7 +1440,7 @@ class GitShallowTest(FetcherTest):
         def test_bitbake(self):
             self.git('remote add --mirror=fetch origin git://github.com/openembedded/bitbake', cwd=self.srcdir)
             self.git('config core.bare true', cwd=self.srcdir)
-            self.git('fetch --tags', cwd=self.srcdir)
+            self.git('fetch', cwd=self.srcdir)
 
             self.d.setVar('BB_GIT_SHALLOW_DEPTH', '0')
             # Note that the 1.10.0 tag is annotated, so this also tests

bitbake/lib/toaster/bldcontrol/bbcontroller.py

@@ -37,8 +37,8 @@ class BitbakeController(object):
     """
 
     def __init__(self, be):
-        import bb.server.xmlrpc
-        self.connection = bb.server.xmlrpc._create_server(be.bbaddress,
+        import bb.server.xmlrpcclient
+        self.connection = bb.server.xmlrpcclient._create_server(be.bbaddress,
                                                           int(be.bbport))[0]
 
     def _runCommand(self, command):

bitbake/lib/toaster/bldcontrol/localhostbecontroller.py

@@ -24,6 +24,7 @@ import os
 import sys
 import re
 import shutil
+import time
 from django.db import transaction
 from django.db.models import Q
 from bldcontrol.models import BuildEnvironment, BRLayer, BRVariable, BRTarget, BRBitbake
@@ -331,12 +332,22 @@ class LocalhostBEController(BuildEnvironmentController):
         bitbake = os.path.join(self.pokydirname, 'bitbake', 'bin', 'bitbake')
         toasterlayers = os.path.join(builddir,"conf/toaster-bblayers.conf")
         self._shellcmd('bash -c \"source %s %s; BITBAKE_UI="knotty" %s --read %s --read %s '
-                       '--server-only -t xmlrpc -B 0.0.0.0:0\"' % (oe_init,
+                       '--server-only -B 0.0.0.0:0\"' % (oe_init,
                        builddir, bitbake, confpath, toasterlayers), self.be.sourcedir)
 
         # read port number from bitbake.lock
         self.be.bbport = ""
         bblock = os.path.join(builddir, 'bitbake.lock')
+        # allow 10 seconds for bb lock file to appear but also be populated
+        for lock_check in range(10):
+            if not os.path.exists(bblock):
+                logger.debug("localhostbecontroller: waiting for bblock file to appear")
+                time.sleep(1)
+                continue
+            if 10 < os.stat(bblock).st_size:
+                break
+            logger.debug("localhostbecontroller: waiting for bblock content to appear")
+            time.sleep(1)
         with open(bblock) as fplock:
             for line in fplock:
                 if ":" in line:
@@ -365,10 +376,10 @@ class LocalhostBEController(BuildEnvironmentController):
         log = os.path.join(builddir, 'toaster_ui.log')
         local_bitbake = os.path.join(os.path.dirname(os.getenv('BBBASEDIR')),
                                      'bitbake')
-        self._shellcmd(['bash -c \"(TOASTER_BRBE="%s" BBSERVER="0.0.0.0:-1" '
+        self._shellcmd(['bash -c \"(TOASTER_BRBE="%s" BBSERVER="0.0.0.0:%s" '
                         '%s %s -u toasterui --token="" >>%s 2>&1;'
-                        'BITBAKE_UI="knotty" BBSERVER=0.0.0.0:-1 %s -m)&\"' \
-                        % (brbe, local_bitbake, bbtargets, log, bitbake)],
+                        'BITBAKE_UI="knotty" BBSERVER=0.0.0.0:%s %s -m)&\"' \
+                        % (brbe, self.be.bbport, local_bitbake, bbtargets, log, self.be.bbport, bitbake)],
                         builddir, nowait=True)
 
         logger.debug('localhostbecontroller: Build launched, exiting. '

bitbake/lib/toaster/orm/models.py

@@ -451,7 +451,7 @@ class Build(models.Model):
     recipes_to_parse = models.IntegerField(default=1)
 
     # number of recipes parsed so far for this build
-    recipes_parsed = models.IntegerField(default=0)
+    recipes_parsed = models.IntegerField(default=1)
 
     # number of repos to clone for this build
     repos_to_clone = models.IntegerField(default=1)

bitbake/lib/toaster/toastergui/templates/health.html

@@ -0,0 +1,6 @@
+<!DOCTYPE html>
+<html lang="en">
+  <head><title>Toaster Health</title></head>
+  <body>Ok</body>
+</html>
+

bitbake/lib/toaster/toastergui/urls.py

@@ -244,6 +244,11 @@ urlpatterns = [
         url(r'^mostrecentbuilds$', widgets.MostRecentBuildsView.as_view(),
             name='most_recent_builds'),
 
-          # default redirection
+        # JSON data for aggregators
+        url(r'^api/builds$', views.json_builds, name='json_builds'),
+        url(r'^api/building$', views.json_building, name='json_building'),
+        url(r'^api/build/(?P<build_id>\d+)$', views.json_build, name='json_build'),
+
+        # default redirection
         url(r'^$', RedirectView.as_view(url='landing', permanent=True)),
 ]

bitbake/lib/toaster/toastergui/views.py

@@ -35,7 +35,7 @@ from orm.models import BitbakeVersion, CustomImageRecipe
 from django.core.urlresolvers import reverse, resolve
 from django.core.exceptions import MultipleObjectsReturned, ObjectDoesNotExist
 from django.core.paginator import Paginator, EmptyPage, PageNotAnInteger
-from django.http import HttpResponseNotFound
+from django.http import HttpResponseNotFound, JsonResponse
 from django.utils import timezone
 from datetime import timedelta, datetime
 from toastergui.templatetags.projecttags import json as jsonfilter
@@ -1256,6 +1256,89 @@ def managedcontextprocessor(request):
     }
     return ret
 
+# REST-based API calls to return build/building status to external Toaster
+# managers and aggregators via JSON
+
+def _json_build_status(build_id,extend):
+    build_stat = None
+    try:
+        build = Build.objects.get( pk = build_id )
+        build_stat = {}
+        build_stat['id'] = build.id
+        build_stat['name'] = build.build_name
+        build_stat['machine'] = build.machine
+        build_stat['distro'] = build.distro
+        build_stat['start'] = build.started_on
+        # look up target name
+        target= Target.objects.get( build = build )
+        if target:
+            if target.task:
+                build_stat['target'] = '%s:%s' % (target.target,target.task)
+            else:
+                build_stat['target'] = '%s' % (target.target)
+        else:
+            build_stat['target'] = ''
+        # look up project name
+        project = Project.objects.get( build = build )
+        if project:
+            build_stat['project'] = project.name
+        else:
+            build_stat['project'] = ''
+        if Build.IN_PROGRESS == build.outcome:
+            now = timezone.now()
+            timediff = now - build.started_on
+            build_stat['seconds']='%.3f' % timediff.total_seconds()
+            build_stat['clone']='%d:%d' % (build.repos_cloned,build.repos_to_clone)
+            build_stat['parse']='%d:%d' % (build.recipes_parsed,build.recipes_to_parse)
+            tf = Task.objects.filter(build = build)
+            tfc = tf.count()
+            if tfc > 0:
+                tfd = tf.exclude(order__isnull=True).count()
+            else:
+                tfd = 0
+            build_stat['task']='%d:%d' % (tfd,tfc)
+        else:
+            build_stat['outcome'] = build.get_outcome_text()
+            timediff = build.completed_on - build.started_on
+            build_stat['seconds']='%.3f' % timediff.total_seconds()
+            build_stat['stop'] = build.completed_on
+            messages = LogMessage.objects.all().filter(build = build)
+            errors = len(messages.filter(level=LogMessage.ERROR) |
+                 messages.filter(level=LogMessage.EXCEPTION) |
+                 messages.filter(level=LogMessage.CRITICAL))
+            build_stat['errors'] = errors
+            warnings = len(messages.filter(level=LogMessage.WARNING))
+            build_stat['warnings'] = warnings
+        if extend:
+            build_stat['cooker_log'] = build.cooker_log_path
+    except Exception as e:
+        build_state = str(e)
+    return build_stat
+
+def json_builds(request):
+    build_table = []
+    builds = []
+    try:
+        builds = Build.objects.exclude(outcome=Build.IN_PROGRESS).order_by("-started_on")
+        for build in builds:
+            build_table.append(_json_build_status(build.id,False))
+    except Exception as e:
+        build_table = str(e)
+    return JsonResponse({'builds' : build_table, 'count' : len(builds)})
+
+def json_building(request):
+    build_table = []
+    builds = []
+    try:
+        builds = Build.objects.filter(outcome=Build.IN_PROGRESS).order_by("-started_on")
+        for build in builds:
+            build_table.append(_json_build_status(build.id,False))
+    except Exception as e:
+        build_table = str(e)
+    return JsonResponse({'building' : build_table, 'count' : len(builds)})
+
+def json_build(request,build_id):
+    return JsonResponse({'build' : _json_build_status(build_id,True)})
 
 
 import toastermain.settings
@@ -1694,3 +1777,4 @@ if True:
                 return render(request, "unavailable_artifact.html")
         except (ObjectDoesNotExist, IOError):
             return render(request, "unavailable_artifact.html")
+

bitbake/lib/toaster/toastermain/urls.py

@@ -20,9 +20,8 @@
 # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
 
 from django.conf.urls import patterns, include, url
-from django.views.generic import RedirectView
+from django.views.generic import RedirectView, TemplateView
 from django.views.decorators.cache import never_cache
-
 import bldcollector.views
 
 import logging
@@ -46,6 +45,8 @@ urlpatterns = [
     # in the future.
     url(r'^orm/eventfile$', bldcollector.views.eventfile),
 
+    url(r'^health$', TemplateView.as_view(template_name="health.html"), name='Toaster Health'),
+
     # if no application is selected, we have the magic toastergui app here
     url(r'^$', never_cache(RedirectView.as_view(url='/toastergui/', permanent=True))),
 ]

documentation/Makefile

@@ -143,7 +143,7 @@ endif
 ifeq ($(DOC),yocto-project-qs)
 XSLTOPTS = --xinclude
 ALLPREQ = html eclipse tarball
-TARFILES = yocto-project-qs.html qs-style.css figures/yocto-environment.png \
+TARFILES = yocto-project-qs.html qs-style.css \
            figures/yocto-project-transp.png \
            eclipse
 MANUALS = $(DOC)/$(DOC).html $(DOC)/eclipse
@@ -191,7 +191,7 @@ TARFILES = mega-manual.html mega-style.css figures/yocto-environment.png \
 	figures/wip.png
         else
 TARFILES = mega-manual.html mega-style.css figures/yocto-environment.png \
-        figures/building-an-image.png  \
+        figures/building-an-image.png figures/YP-flow-diagram.png \
 	figures/using-a-pre-built-image.png \
 	figures/poky-title.png figures/buildhistory.png \
         figures/buildhistory-web.png \
@@ -256,7 +256,7 @@ endif
 ifeq ($(DOC),ref-manual)
 XSLTOPTS = --xinclude
 ALLPREQ = html eclipse tarball
-TARFILES = ref-manual.html ref-style.css figures/poky-title.png \
+TARFILES = ref-manual.html ref-style.css figures/poky-title.png figures/YP-flow-diagram.png \
 	figures/buildhistory.png figures/buildhistory-web.png eclipse \
         figures/cross-development-toolchains.png figures/layer-input.png \
 	figures/package-feeds.png figures/source-input.png \

documentation/poky.ent

@@ -69,7 +69,7 @@
 <!ENTITY FEDORA_HOST_PACKAGES_ESSENTIAL "gawk make wget tar bzip2 gzip python3 unzip perl patch \
      diffutils diffstat git cpp gcc gcc-c++ glibc-devel texinfo chrpath \
      ccache perl-Data-Dumper perl-Text-ParseWords perl-Thread-Queue perl-bignum socat \
-     python3-pexpect findutils which file cpio python python3-pip xz which">
+     python3-pexpect findutils which file cpio python python3-pip xz">
 <!ENTITY OPENSUSE_HOST_PACKAGES_ESSENTIAL "python gcc gcc-c++ git chrpath make wget python-xml \
      diffstat makeinfo python-curses patch socat python3 python3-curses tar python3-pip \
      python3-pexpect xz which">
@@ -77,5 +77,5 @@
      $ sudo yum makecache
      $ sudo yum install gawk make wget tar bzip2 gzip python unzip perl patch \
      diffutils diffstat git cpp gcc gcc-c++ glibc-devel texinfo chrpath socat \
-     perl-Data-Dumper perl-Text-ParseWords perl-Thread-Queue python34-pip xz \
+     perl-Data-Dumper perl-Text-ParseWords perl-Thread-Queue python3-pip xz \
      which">

documentation/ref-manual/introduction.xml

@@ -25,12 +25,14 @@
         For introductory information on the Yocto Project, see the
         <ulink url='&YOCTO_HOME_URL;/ecosystem/yocto-project-backgrounders'>Yocto Project Backgrounders</ulink>
         on the
-        <ulink url='&YOCTO_HOME_URL;'>Yocto Project Website</ulink>.
+        <ulink url='&YOCTO_HOME_URL;'>Yocto Project Website</ulink> and the
+        "<link linkend='yp-intro'>Introducing the Yocto Project Development Environment</link>"
+        section.
     </para>
 
     <para>
-        You can find an introductory to using the Yocto Project by working
-        through the
+        If you want to use the Yocto Project to test run building an image
+        without having to understand concepts, work through the
         <ulink url='&YOCTO_DOCS_QS_URL;'>Yocto Project Quick Start</ulink>.
         You can find "how-to" information in the
         <ulink url='&YOCTO_DOCS_DEV_URL;'>Yocto Project Development Manual</ulink>.
@@ -43,8 +45,8 @@
     </para>
 </section>
 
-<section id='ref-yp-intro'>
-    <title>Yocto Project Introduction</title>
+<section id='yp-intro'>
+    <title>Introducing the Yocto Project Development Environment</title>
 
     <para>
         The Yocto Project is an open-source collaboration project whose
@@ -69,6 +71,68 @@
         is optimized for stylus-driven, low-resolution screens.
     </para>
 
+    <mediaobject>
+        <imageobject>
+            <imagedata fileref="figures/YP-flow-diagram.png"
+                format="PNG" align='center' width="8in"/>
+        </imageobject>
+    </mediaobject>
+
+    <para>
+        Here are some highlights for the Yocto Project:
+    </para>
+
+    <itemizedlist>
+        <listitem><para>
+            Provides a recent Linux kernel along with a set of system
+            commands and libraries suitable for the embedded
+            environment.
+            </para></listitem>
+        <listitem><para>
+            Makes available system components such as X11, GTK+, Qt,
+            Clutter, and SDL (among others) so you can create a rich user
+            experience on devices that have display hardware.
+            For devices that do not have a display or where you wish to
+            use alternative UI frameworks, these components need not be
+            installed.
+            </para></listitem>
+        <listitem><para>
+            Creates a focused and stable core compatible with the
+            OpenEmbedded project with which you can easily and reliably
+            build and develop.
+            </para></listitem>
+        <listitem><para>
+            Fully supports a wide range of hardware and device emulation
+            through the Quick EMUlator (QEMU).
+            </para></listitem>
+        <listitem><para>
+            Provides a layer mechanism that allows you to easily extend
+            the system, make customizations, and keep them organized.
+            </para></listitem>
+    </itemizedlist>
+
+    <para>
+        You can use the Yocto Project to generate images for many kinds
+        of devices.
+        As mentioned earlier, the Yocto Project supports creation of
+        reference images that you can boot within and emulate using QEMU.
+        The standard example machines target QEMU full-system
+        emulation for 32-bit and 64-bit variants of x86, ARM, MIPS, and
+        PowerPC architectures.
+        Beyond emulation, you can use the layer mechanism to extend
+        support to just about any platform that Linux can run on and that
+        a toolchain can target.
+    </para>
+
+    <para>
+        Another Yocto Project feature is the Sato reference User
+        Interface.
+        This optional UI that is based on GTK+ is intended for devices with
+        restricted screen sizes and is included as part of the
+        OpenEmbedded Core layer so that developers can test parts of the
+        software stack.
+    </para>
+
     <para>
         While the Yocto Project does not provide a strict testing framework,
         it does provide or generate for you artifacts that let you perform

documentation/yocto-project-qs/qs-style.css

@@ -730,6 +730,11 @@ div.navfooter {
 }
 
 
+.writernotes {
+  color: red;
+}
+
+
   /*********** /
  /  graphics  /
 / ***********/

documentation/yocto-project-qs/yocto-project-qs.xml

@@ -44,7 +44,6 @@
             </note>
         </legalnotice>
 
-
         <abstract>
             <imagedata fileref="figures/yocto-project-transp.png"
                         width="6in" depth="1in"
@@ -62,7 +61,7 @@
             on the OpenEmbedded (OE) project, which uses the
             <ulink url='&YOCTO_DOCS_REF_URL;#bitbake-term'>BitBake</ulink>
             tool, to construct complete Linux images.
-            The BitBake and OE components are combined together to form
+            The BitBake and OE components combine together to form
             a reference build host, historically known as
             <ulink url='&YOCTO_DOCS_REF_URL;#poky'>Poky</ulink>
             (<emphasis>Pah</emphasis>-kee).
@@ -74,8 +73,14 @@
             Linux images.
             Rather than go into great detail about the Yocto Project and its
             many capabilities, this quick start provides the minimal
-            information you need to try out the Yocto Project using a
-            supported Linux build host.
+            information you need to try out the Yocto Project using either a
+            supported Linux build host or a build host set up to use
+            <ulink url='https://git.yoctoproject.org/cgit/cgit.cgi/crops/about/'>CROPS</ulink>,
+            which leverages
+            <ulink url='https://www.docker.com/'>Docker Containers</ulink>.
+        </para>
+
+        <para>
             Reading and using the quick start should result in you having a
             basic understanding of what the Yocto Project is and how to use
             some of its core components.
@@ -100,6 +105,7 @@
                     <ulink url='http://www.yoctoproject.org/docs/2.4/yocto-project-qs/yocto-project-qs.html#building-an-image-for-hardware'>Building an Image for Hardware</ulink>
                     </para></listitem>
             </itemizedlist>
+<!--
             <note>
                 If you do not have a system that runs Linux and you want to give
                 the Yocto Project a test run, you might consider using the Yocto
@@ -111,18 +117,13 @@
                 <ulink url='https://www.yoctoproject.org/tools-resources/projects/build-appliance'>Yocto Project Build Appliance</ulink>
                 for more information.
             </note>
+-->
         </para>
 
         <para>
             For more detailed information on the Yocto Project, you can
             reference these resources:
             <itemizedlist>
-                <listitem><para>
-                    <emphasis>Yocto Project Backgrounders:</emphasis>
-                    The
-                    <ulink url='&YOCTO_HOME_URL;/ecosystem/yocto-project-backgrounders'>Yocto Project Backgrounder</ulink>
-                    provides introductory information on the Yocto Project.
-                    </para></listitem>
                 <listitem><para>
                     <emphasis>Website:</emphasis>
                     The
@@ -131,6 +132,19 @@
                     documentation, and access to a rich Yocto Project
                     Development Community into which you can tap.
                     </para></listitem>
+                <listitem><para>
+                    <emphasis>Yocto Project Backgrounders:</emphasis>
+                    The
+                    <ulink url='&YOCTO_HOME_URL;/ecosystem/yocto-project-backgrounders'>Yocto Project Backgrounder</ulink>
+                    provides introductory information on the Yocto Project.
+                    </para></listitem>
+                <listitem><para>
+                    <emphasis>Yocto Project Development Environment Overview:</emphasis>
+                    The
+                    "<ulink url='&YOCTO_DOCS_REF_URL;#yp-intro'>Introducing the Yocto Project Development Environment</ulink>"
+                    section presents an overview of the Yocto Project
+                    development environment.
+                    </para></listitem>
                 <listitem><para>
                     <emphasis>FAQs:</emphasis>
                     Lists commonly asked Yocto Project questions and answers.
@@ -161,243 +175,238 @@
         </para>
     </section>
 
-    <section id='yp-intro'>
-        <title>Introducing the Yocto Project Development Environment</title>
-
-        <para>
-            The Yocto Project through the OpenEmbedded build system provides an
-            open source development environment targeting the ARM, MIPS,
-            PowerPC, and x86 architectures for a variety of platforms
-            including x86-64 and emulated ones.
-            You can use components from the Yocto Project to design, develop,
-            build, debug, simulate, and test the complete software stack using
-            Linux, the X Window System, GTK+ frameworks, and Qt frameworks.
-        </para>
-
-        <mediaobject>
-            <imageobject>
-                <imagedata fileref="figures/yocto-environment.png"
-                    format="PNG" align='center' width="8in"/>
-            </imageobject>
-         </mediaobject>
-
-        <para>
-            Here are some highlights for the Yocto Project:
-        </para>
-
-        <itemizedlist>
-            <listitem><para>
-                Provides a recent Linux kernel along with a set of system
-                commands and libraries suitable for the embedded
-                environment.
-                </para></listitem>
-            <listitem><para>
-                Makes available system components such as X11, GTK+, Qt,
-                Clutter, and SDL (among others) so you can create a rich user
-                experience on devices that have display hardware.
-                For devices that do not have a display or where you wish to
-                use alternative UI frameworks, these components need not be
-                installed.
-                </para></listitem>
-            <listitem><para>
-                Creates a focused and stable core compatible with the
-                OpenEmbedded project with which you can easily and reliably
-                build and develop.
-                </para></listitem>
-            <listitem><para>
-                Fully supports a wide range of hardware and device emulation
-                through the Quick EMUlator (QEMU).
-                </para></listitem>
-            <listitem><para>
-                Provides a layer mechanism that allows you to easily extend
-                the system, make customizations, and keep them organized.
-                </para></listitem>
-        </itemizedlist>
-
-        <para>
-            You can use the Yocto Project to generate images for many kinds
-            of devices.
-            As mentioned earlier, the Yocto Project supports creation of
-            reference images that you can boot within and emulate using QEMU.
-            The standard example machines target QEMU full-system
-            emulation for 32-bit and 64-bit variants of x86, ARM, MIPS, and
-            PowerPC architectures.
-            Beyond emulation, you can use the layer mechanism to extend
-            support to just about any platform that Linux can run on and that
-            a toolchain can target.
-        </para>
-
-        <para>
-            Another Yocto Project feature is the Sato reference User
-            Interface.
-            This optional UI that is based on GTK+ is intended for devices with
-            restricted screen sizes and is included as part of the
-            OpenEmbedded Core layer so that developers can test parts of the
-            software stack.
-        </para>
-    </section>
-
     <section id='yp-resources'>
         <title>Setting Up to Use the Yocto Project</title>
 
         <para>
-            The following list shows what you need in order to use a
-            Linux-based build host to use the Yocto Project to build images:
+            Setting up to use the Yocto Project involves getting your build
+            host ready.
+            If you have a native Linux machine that runs a Yocto Project
+            supported distribution as described by the
+            "<ulink url='&YOCTO_DOCS_REF_URL;#detailed-supported-distros'>Supported Linux Distributions</ulink>"
+            section in the Yocto Project Reference Manual, you can prepare
+            that machine as your build host.
+            See the
+            "<link linkend='qs-native-linux-build-host'>Using a Native Linux Machine</link>"
+            section for more information.
         </para>
 
-        <itemizedlist>
-            <listitem><para><emphasis>Build Host</emphasis>
-                A build host with a minimum of 50 Gbytes of free disk
-                space that is running a supported Linux distribution (i.e.
-                recent releases of Fedora, openSUSE, CentOS, Debian, or
-                Ubuntu).
-                </para></listitem>
-            <listitem><para><emphasis>Build Host Packages</emphasis>
-                Appropriate packages installed on the build host.
-                </para></listitem>
-            <listitem><para><emphasis>The Yocto Project</emphasis>
-                A release of the Yocto Project.
-                </para></listitem>
-        </itemizedlist>
+        <para>
+            If you do not want to use the Yocto Project on a native Linux
+            machine, you can prepare your build host to use
+            <ulink url='https://git.yoctoproject.org/cgit/cgit.cgi/crops/about/'>CROPS</ulink>,
+            which leverages
+            <ulink url='https://www.docker.com/'>Docker Containers</ulink>.
+            You can set up a build host for Windows, Mac, and Linux
+            machines.
+            See the
+            "<link linkend='qs-crops-build-host'>Using CROPS and Containers</link>"
+            section for more information.
+        </para>
 
-        <section id='the-linux-distro'>
-            <title>The Linux Distribution</title>
+        <section id='qs-crops-build-host'>
+            <title>Using CROPS and Containers</title>
 
             <para>
-                The Yocto Project team verifies each release against recent
-                versions of the most popular Linux distributions that
-                provide stable releases.
-                In general, if you have the current release minus one of the
-                following distributions, you should have no problems.
-                <itemizedlist>
+                Follow these steps to get your build host set up with a
+                Poky container that you can use to complete the build
+                examples further down in the Quick Start:
+                <orderedlist>
                     <listitem><para>
-                        Ubuntu
+                        <emphasis>Reference the Procedure:</emphasis>
+                        Work through the first six steps of the procedure
+                        in the
+                        "<ulink url='&YOCTO_DOCS_DEV_URL;#setting-up-to-use-crops'>Setting Up to Use CROss PlatformS (CROPS)</ulink>"
+                        section of the Yocto Project Development Manual.
                         </para></listitem>
                     <listitem><para>
-                        Fedora
-                        </para></listitem>
-                    <listitem><para>
-                        openSUSE
-                        </para></listitem>
-                    <listitem><para>
-                        CentOS
-                        </para></listitem>
-                    <listitem><para>
-                        Debian
-                        </para></listitem>
-                </itemizedlist>
-                For a more detailed list of distributions that support the
-                Yocto Project, see the
-                "<ulink url='&YOCTO_DOCS_REF_URL;#detailed-supported-distros'>Supported Linux Distributions</ulink>"
-                section in the Yocto Project Reference Manual.
-            </para>
+                        <emphasis>Set Up the Poky Container to Use the Yocto Project:</emphasis>
+                        Go to
+                        <ulink url='https://github.com/crops/poky-container/blob/master/README.md'></ulink>
+                        and follow the directions to set up the Poky container
+                        on your build host.</para>
 
-            <para>
-                The OpenEmbedded build system should be able to run on any
-                modern distribution that has the following versions for
-                Git, tar, and Python.
-                <itemizedlist>
-                    <listitem><para>
-                        Git 1.8.3.1 or greater
+                        <para>Once you complete the setup instructions for your
+                        machine, you need to get a copy of the
+                        <filename>poky</filename> repository on your build
+                        host.
+                        See the
+                        "<link linkend='releases'>Yocto Project Release</link>"
+                        section to continue.
                         </para></listitem>
-                    <listitem><para>
-                        tar 1.27 or greater
-                        </para></listitem>
-                    <listitem><para>
-                        Python 3.4.0 or greater.
-                        </para></listitem>
-                </itemizedlist>
-                If your build host does not meet any of these three listed
-                version requirements, you can take steps to prepare the
-                system so that you can still use the Yocto Project.
-                See the
-                "<ulink url='&YOCTO_DOCS_REF_URL;#required-git-tar-and-python-versions'>Required Git, tar, and Python Versions</ulink>"
-                section in the Yocto Project Reference Manual for information.
+                </orderedlist>
             </para>
         </section>
 
-        <section id='packages'>
-            <title>The Build Host Packages</title>
+        <section id='qs-native-linux-build-host'>
+            <title>Using a Native Linux Machine</title>
 
             <para>
-                Required build host packages vary depending on your
-                build machine and what you want to do with the Yocto Project.
-                For example, if you want to build an image that can run
-                on QEMU in graphical mode (a minimal, basic build
-                requirement), then the build host package requirements
-                are different than if you want to build an image on a headless
-                system or build out the Yocto Project documentation set.
+                The following list shows what you need in order to use a
+                Linux-based build host to use the Yocto Project to build images:
             </para>
 
-            <para>
-                Collectively, the number of required packages is large
-                if you want to be able to cover all cases.
-                <note>
-                    In general, you need to have root access and then install
-                    the required packages.
-                    Thus, the commands in the following section may or may
-                    not work depending on whether or not your Linux
-                    distribution has <filename>sudo</filename> installed.
-                </note>
-            </para>
+            <itemizedlist>
+                <listitem><para><emphasis>Build Host</emphasis>
+                    A build host with a minimum of 50 Gbytes of free disk
+                    space that is running a supported Linux distribution (i.e.
+                    recent releases of Fedora, openSUSE, CentOS, Debian, or
+                    Ubuntu).
+                    </para></listitem>
+                <listitem><para><emphasis>Build Host Packages</emphasis>
+                    Appropriate packages installed on the build host.
+                    </para></listitem>
+            </itemizedlist>
 
-            <para>
-                The following list shows the required packages needed to build
-                an image that runs on QEMU in graphical mode (e.g. essential
-                plus graphics support).
-                For lists of required packages for other scenarios, see the
-                "<ulink url='&YOCTO_DOCS_REF_URL;#required-packages-for-the-host-development-system'>Required Packages for the Host Development System</ulink>"
-                section in the Yocto Project Reference Manual.
-                <itemizedlist>
-                    <listitem><para><emphasis>Ubuntu and Debian</emphasis>
-                        <literallayout class='monospaced'>
+            <section id='the-linux-distro'>
+                <title>The Linux Distribution</title>
+
+                <para>
+                    The Yocto Project team verifies each release against recent
+                    versions of the most popular Linux distributions that
+                    provide stable releases.
+                    In general, if you have the current release minus one of the
+                    following distributions, you should have no problems.
+                    <itemizedlist>
+                        <listitem><para>
+                            Ubuntu
+                            </para></listitem>
+                        <listitem><para>
+                            Fedora
+                            </para></listitem>
+                        <listitem><para>
+                            openSUSE
+                            </para></listitem>
+                        <listitem><para>
+                            CentOS
+                            </para></listitem>
+                        <listitem><para>
+                            Debian
+                            </para></listitem>
+                    </itemizedlist>
+                    For a more detailed list of distributions that support the
+                    Yocto Project, see the
+                    "<ulink url='&YOCTO_DOCS_REF_URL;#detailed-supported-distros'>Supported Linux Distributions</ulink>"
+                    section in the Yocto Project Reference Manual.
+                </para>
+
+                <para>
+                    The OpenEmbedded build system should be able to run on any
+                    modern distribution that has the following versions for
+                    Git, tar, and Python.
+                    <itemizedlist>
+                        <listitem><para>
+                            Git 1.8.3.1 or greater
+                            </para></listitem>
+                        <listitem><para>
+                            tar 1.27 or greater
+                            </para></listitem>
+                        <listitem><para>
+                            Python 3.4.0 or greater.
+                            </para></listitem>
+                    </itemizedlist>
+                    If your build host does not meet any of these three listed
+                    version requirements, you can take steps to prepare the
+                    system so that you can still use the Yocto Project.
+                    See the
+                    "<ulink url='&YOCTO_DOCS_REF_URL;#required-git-tar-and-python-versions'>Required Git, tar, and Python Versions</ulink>"
+                    section in the Yocto Project Reference Manual for information.
+                </para>
+            </section>
+
+            <section id='packages'>
+                <title>The Build Host Packages</title>
+
+                <para>
+                    Required build host packages vary depending on your
+                    build machine and what you want to do with the Yocto Project.
+                    For example, if you want to build an image that can run
+                    on QEMU in graphical mode (a minimal, basic build
+                    requirement), then the build host package requirements
+                    are different than if you want to build an image on a headless
+                    system or build out the Yocto Project documentation set.
+                </para>
+
+                <para>
+                    Collectively, the number of required packages is large
+                    if you want to be able to cover all cases.
+                    <note>
+                        In general, you need to have root access and then install
+                        the required packages.
+                        Thus, the commands in the following section may or may
+                        not work depending on whether or not your Linux
+                        distribution has <filename>sudo</filename> installed.
+                    </note>
+                </para>
+
+                <para>
+                    The following list shows the required packages needed to build
+                    an image that runs on QEMU in graphical mode (e.g. essential
+                    plus graphics support).
+                    For lists of required packages for other scenarios, see the
+                    "<ulink url='&YOCTO_DOCS_REF_URL;#required-packages-for-the-host-development-system'>Required Packages for the Host Development System</ulink>"
+                    section in the Yocto Project Reference Manual.
+                    <itemizedlist>
+                        <listitem><para><emphasis>Ubuntu and Debian</emphasis>
+                            <literallayout class='monospaced'>
      $ sudo apt-get install &UBUNTU_HOST_PACKAGES_ESSENTIAL; libsdl1.2-dev xterm
-                        </literallayout>
-                        </para></listitem>
-                    <listitem><para><emphasis>Fedora</emphasis>
-                        <literallayout class='monospaced'>
+                            </literallayout>
+                            </para></listitem>
+                        <listitem><para><emphasis>Fedora</emphasis>
+                            <literallayout class='monospaced'>
      $ sudo dnf install &FEDORA_HOST_PACKAGES_ESSENTIAL; SDL-devel xterm
-                        </literallayout>
-                        </para></listitem>
-                    <listitem><para><emphasis>OpenSUSE</emphasis>
-                        <literallayout class='monospaced'>
+                            </literallayout>
+                            </para></listitem>
+                        <listitem><para><emphasis>OpenSUSE</emphasis>
+                            <literallayout class='monospaced'>
      $ sudo zypper install &OPENSUSE_HOST_PACKAGES_ESSENTIAL; libSDL-devel xterm
-                        </literallayout>
-                        </para></listitem>
-                    <listitem><para><emphasis>CentOS</emphasis>
-                        <literallayout class='monospaced'>
+                            </literallayout>
+                            </para></listitem>
+                        <listitem><para><emphasis>CentOS</emphasis>
+                            <literallayout class='monospaced'>
      $ sudo yum install &CENTOS_HOST_PACKAGES_ESSENTIAL; SDL-devel xterm
-                        </literallayout>
-                        <note><title>Notes</title>
-                            <itemizedlist>
-                                <listitem><para>
-                                    CentOS 6.x users need to ensure that the
-                                    required versions of Git, tar and Python
-                                    are available.
-                                    For details, See the
-                                    "<ulink url='&YOCTO_DOCS_REF_URL;#required-git-tar-and-python-versions'>Required Git, tar, and Python Versions</ulink>"
-                                    section in the Yocto Project Reference
-                                    Manual for information.
-                                    </para></listitem>
-                                <listitem><para>
-                                    Extra Packages for Enterprise Linux
-                                    (i.e. <filename>epel-release</filename>)
-                                    is a collection of packages from Fedora
-                                    built on RHEL/CentOS for easy installation
-                                    of packages not included in enterprise
-                                    Linux by default.
-                                    You need to install these packages
-                                    separately.
-                                    </para></listitem>
-                                <listitem><para>
-                                    The <filename>makecache</filename> command
-                                    consumes additional Metadata from
-                                    <filename>epel-release</filename>.
-                                    </para></listitem>
-                            </itemizedlist>
-                        </note>
-                        </para></listitem>
-                </itemizedlist>
+                            </literallayout>
+                            <note><title>Notes</title>
+                                <itemizedlist>
+                                    <listitem><para>
+                                        CentOS 6.x users need to ensure that the
+                                        required versions of Git, tar and Python
+                                        are available.
+                                        For details, See the
+                                        "<ulink url='&YOCTO_DOCS_REF_URL;#required-git-tar-and-python-versions'>Required Git, tar, and Python Versions</ulink>"
+                                        section in the Yocto Project Reference
+                                        Manual for information.
+                                        </para></listitem>
+                                    <listitem><para>
+                                        Extra Packages for Enterprise Linux
+                                        (i.e. <filename>epel-release</filename>)
+                                        is a collection of packages from Fedora
+                                        built on RHEL/CentOS for easy installation
+                                        of packages not included in enterprise
+                                        Linux by default.
+                                        You need to install these packages
+                                        separately.
+                                        </para></listitem>
+                                    <listitem><para>
+                                        The <filename>makecache</filename> command
+                                        consumes additional Metadata from
+                                        <filename>epel-release</filename>.
+                                        </para></listitem>
+                                </itemizedlist>
+                            </note>
+                            </para></listitem>
+                    </itemizedlist>
+                </para>
+            </section>
+
+            <para>
+                Once you complete the setup instructions for your
+                machine, you need to get a copy of the
+                <filename>poky</filename> repository on your build
+                host.
+                Continue with the
+                "<link linkend='releases'>Yocto Project Release</link>"
+                section.
             </para>
         </section>
 
@@ -405,8 +414,9 @@
             <title>Yocto Project Release</title>
 
             <para>
-                The last requirement you need to meet before using the
-                Yocto Project is getting a Yocto Project release.
+                Now that your build host has the right packages (native
+                Linux machine) or you have the Poky container set up
+                (CROPS), you need to get a copy of the Yocto Project.
                 It is recommended that you get the latest Yocto Project release
                 by setting up (cloning in
                 <ulink url='&YOCTO_DOCS_REF_URL;#git'>Git</ulink> terms) a
@@ -417,9 +427,14 @@
             </para>
 
             <para>
-                Here is an example from an Ubuntu build host that clones the
-                <filename>poky</filename> repository and then checks out the
-                latest Yocto Project Release by tag
+                Here is an example from a native Linux machine that is
+                running Ubuntu.
+                <note>
+                    If your build host is using a Poky container, you can
+                    use the same Git commands.
+                </note>
+                The example clones the <filename>poky</filename> repository
+                and then checks out the latest Yocto Project Release by tag
                 (i.e. <filename>&DISTRO_REL_TAG;</filename>):
                 <literallayout class='monospaced'>
      $ git clone git://git.yoctoproject.org/poky
@@ -465,20 +480,21 @@
         <title>Building Images</title>
 
         <para>
-            Now that you have your system requirements in order, you can give
-            Yocto Project a try.
-            You can try out Yocto Project using either the command-line
-            interface or using Toaster, which uses a graphical user
-            interface.
-            If you want to try out the Yocto Project using a GUI, see the
-            <ulink url='&YOCTO_DOCS_TOAST_URL;'>Toaster User Manual</ulink>
-            for information on how to install and set up Toaster.
+            You are now ready to give the Yocto Project a try.
+            For this example, you will be using the command line to build
+            your images.
+            <note>
+                A graphical user interface to the Yocto Project is available
+                through Toaster.
+                See the
+                <ulink url='&YOCTO_DOCS_TOAST_URL;'>Toaster User Manual</ulink>
+                for more information.
+            </note>
         </para>
 
         <para>
-            To use the Yocto Project through the command-line interface,
-            finish this quick start, which presents steps that let you
-            do the following:
+            The remainder of this quick start steps you through the
+            following:
             <itemizedlist>
                 <listitem><para>
                     Build a <filename>qemux86</filename> reference image
@@ -538,7 +554,8 @@
 
             <para>
                 <orderedlist>
-                    <listitem><para><emphasis>Be Sure Your Build Host is Set Up:</emphasis>
+                    <listitem><para>
+                        <emphasis>Be Sure Your Build Host is Set Up:</emphasis>
                         The steps to build an image in this section depend on
                         your build host being properly set up.
                         Be sure you have worked through the requirements
@@ -546,7 +563,8 @@
                         "<link linkend='yp-resources'>Setting Up to Use the Yocto Project</link>"
                         section.
                         </para></listitem>
-                    <listitem><para><emphasis>Check Out Your Branch:</emphasis>
+                    <listitem><para>
+                        <emphasis>Check Out Your Branch:</emphasis>
                         Be sure you are in the
                         <ulink url='&YOCTO_DOCS_DEV_URL;#source-directory'>Source Directory</ulink>
                         (e.g. <filename>poky</filename>) and then check out
@@ -566,7 +584,8 @@
                         branch ensures you are using the latest files for
                         that release.
                         </para></listitem>
-                    <listitem><para><emphasis>Initialize the Build Environment:</emphasis>
+                    <listitem><para>
+                        <emphasis>Initialize the Build Environment:</emphasis>
                         Run the
                         <ulink url='&YOCTO_DOCS_REF_URL;#structure-core-script'><filename>&OE_INIT_FILE;</filename></ulink>
                         environment setup script to define the OpenEmbedded
@@ -591,7 +610,8 @@
                             setup script.
                         </note>
                         </para></listitem>
-                    <listitem><para><emphasis>Examine Your Local Configuration File:</emphasis>
+                    <listitem><para>
+                        <emphasis>Examine Your Local Configuration File:</emphasis>
                         When you set up the build environment, a local
                         configuration file named
                         <filename>local.conf</filename> becomes available in
@@ -645,7 +665,8 @@
                                 </para></listitem>
                         </itemizedlist>
                         </para></listitem>
-                    <listitem><para><emphasis>Start the Build:</emphasis>
+                    <listitem><para>
+                        <emphasis>Start the Build:</emphasis>
                         Continue with the following command to build an OS image
                         for the target, which is
                         <filename>core-image-sato</filename> in this example:
@@ -703,7 +724,8 @@
                         "<ulink url='&YOCTO_DOCS_REF_URL;#ref-images'>Images</ulink>"
                         chapter in the Yocto Project Reference Manual.
                         </para></listitem>
-                    <listitem><para><emphasis>Simulate Your Image Using QEMU:</emphasis>
+                    <listitem><para>
+                        <emphasis>Simulate Your Image Using QEMU:</emphasis>
                         Once this particular image is built, you can start QEMU
                         and run the image:
                         <literallayout class='monospaced'>
@@ -713,7 +735,8 @@
                         "<ulink url="&YOCTO_DOCS_DEV_URL;#dev-manual-qemu">Using the Quick EMUlator (QEMU)</ulink>"
                         chapter in the Yocto Project Development Manual.
                         </para></listitem>
-                    <listitem><para><emphasis>Exit QEMU:</emphasis>
+                    <listitem><para>
+                        <emphasis>Exit QEMU:</emphasis>
                         Exit QEMU by either clicking on the shutdown icon or by
                         typing <filename>Ctrl-C</filename> in the QEMU
                         transcript window from which you evoked QEMU.
@@ -743,7 +766,8 @@
 
             <para>
                 <orderedlist>
-                    <listitem><para><emphasis>Create a Local Copy of the
+                    <listitem><para>
+                        <emphasis>Create a Local Copy of the
                         <filename>meta-intel</filename> Repository:</emphasis>
                         Building an image for the MinnowBoard Turbot requires
                         the
@@ -792,7 +816,8 @@
                         "meta-intel-&DISTRO_NAME_NO_CAP;-&YOCTO_DOC_VERSION;"
                         in the above example.
                         </para></listitem>
-                    <listitem><para><emphasis>Configure the Build:</emphasis>
+                    <listitem><para>
+                        <emphasis>Configure the Build:</emphasis>
                         To configure the build, you edit the
                         <filename>bblayers.conf</filename> and
                         <filename>local.conf</filename> files, both of which are
@@ -897,7 +922,8 @@
      tmp/deploy/images/intel-corei7-64/core-image-base-intel-corei7-64.wic
                         </literallayout>
                         </para></listitem>
-                    <listitem><para><emphasis>Write the Image:</emphasis>
+                    <listitem><para>
+                        <emphasis>Write the Image:</emphasis>
                         You can write the image just built to a bootable media
                         (e.g. a USB key, SATA drive, SD card, etc.) using the
                         <filename>dd</filename> utility:
@@ -911,7 +937,8 @@
                         <filename>/dev/mmcblk0</filename>, which is most likely an
                         SD card).
                         </para></listitem>
-                    <listitem><para><emphasis>Boot the Hardware:</emphasis>
+                    <listitem><para>
+                        <emphasis>Boot the Hardware:</emphasis>
                         With the boot device provisioned, you can insert the
                         media into the MinnowBoard Turbot and boot the hardware.
                         The board should automatically detect the media and boot to
@@ -975,7 +1002,7 @@
                     <ulink url='&YOCTO_DOCS_SDK_URL;#sdk-extensible'>extensible SDK</ulink>,
                     which are used primarily for application development.
                     This manual also provides example workflows
-                    that use the popular <trademark class='trad'>Eclipse</trademark>
+                    that use the popular <trademark class='trade'>Eclipse</trademark>
                     development environment and that use <filename>devtool</filename>.
                     See the
                     "<ulink url='&YOCTO_DOCS_SDK_URL;#workflow-using-eclipse'>Workflow using Eclipse™</ulink>"

meta/classes/autotools.bbclass

@@ -141,7 +141,7 @@ ACLOCALEXTRAPATH_class-nativesdk = " -I ${STAGING_DATADIR_NATIVE}/aclocal/"
 
 python autotools_aclocals () {
     # Refresh variable with cache files
-    d.setVar("CONFIG_SITE", siteinfo_get_files(d, aclocalcache=True))
+    d.setVar("CONFIG_SITE", siteinfo_get_files(d, sysrootcache=True))
 }
 
 CONFIGURE_FILES = "${S}/configure.in ${S}/configure.ac ${S}/config.h.in ${S}/acinclude.m4 Makefile.am"

meta/classes/cml1.bbclass

@@ -26,8 +26,28 @@ python do_menuconfig() {
     except OSError:
         mtime = 0
 
-    oe_terminal("${SHELL} -c \"make %s; if [ \$? -ne 0 ]; then echo 'Command failed.'; printf 'Press any key to continue... '; read r; fi\"" % d.getVar('KCONFIG_CONFIG_COMMAND'),
+    # We need to know when the command completes but some terminals (including gnome-terminal
+    # and tmux) gives us no way to do this. We therefore write the pid to a temporal file
+    # then monitor the pid until it exits.
+    import tempfile
+    pidfile = tempfile.NamedTemporaryFile(delete = False).name
+    try:
+        oe_terminal("${SHELL} -c \"echo $$ > %s; make %s; if [ \$? -ne 0 ]; then echo 'Command failed.'; printf 'Press any key to continue... '; read r; fi\"" % (pidfile, d.getVar('KCONFIG_CONFIG_COMMAND')),
                 d.getVar('PN') + ' Configuration', d)
+        while os.stat(pidfile).st_size <= 0:
+            continue
+        with open(pidfile, "r") as f:
+            pid = int(f.readline())
+    finally:
+        os.unlink(pidfile)
+
+    import time
+    while True:
+        try:
+            os.kill(pid, 0)
+            time.sleep(0.1)
+        except OSError:
+            break
 
     # FIXME this check can be removed when the minimum bitbake version has been bumped
     if hasattr(bb.build, 'write_taint'):

meta/classes/grub-efi.bbclass

@@ -17,7 +17,6 @@
 # ${GRUB_ROOT} - grub's root device.
 
 do_bootimg[depends] += "${MLPREFIX}grub-efi:do_deploy"
-do_bootdirectdisk[depends] += "${MLPREFIX}grub-efi:do_deploy"
 
 GRUB_SERIAL ?= "console=ttyS0,115200"
 GRUB_CFG_VM = "${S}/grub_vm.cfg"

meta/classes/image-vm.bbclass

@@ -1,171 +0,0 @@
-# image-vm.bbclass
-# (loosly based off image-live.bbclass Copyright (C) 2004, Advanced Micro Devices, Inc.)
-#
-# Create an image which can be placed directly onto a harddisk using dd and then
-# booted.
-#
-# This uses syslinux. extlinux would have been nice but required the ext2/3
-# partition to be mounted. grub requires to run itself as part of the install
-# process.
-#
-# The end result is a 512 boot sector populated with an MBR and partition table
-# followed by an msdos fat16 partition containing syslinux and a linux kernel
-# completed by the ext2/3 rootfs.
-#
-# We have to push the msdos parition table size > 16MB so fat 16 is used as parted
-# won't touch fat12 partitions.
-
-inherit live-vm-common
-
-do_bootdirectdisk[depends] += "dosfstools-native:do_populate_sysroot \
-                               virtual/kernel:do_deploy \
-                               syslinux:do_populate_sysroot \
-                               syslinux-native:do_populate_sysroot \
-                               parted-native:do_populate_sysroot \
-                               mtools-native:do_populate_sysroot \
-                               ${PN}:do_image_${VM_ROOTFS_TYPE} \
-                               "
-
-IMAGE_TYPEDEP_vmdk = "${VM_ROOTFS_TYPE}"
-IMAGE_TYPEDEP_vdi = "${VM_ROOTFS_TYPE}"
-IMAGE_TYPEDEP_qcow2 = "${VM_ROOTFS_TYPE}"
-IMAGE_TYPEDEP_hdddirect = "${VM_ROOTFS_TYPE}"
-IMAGE_TYPES_MASKED += "vmdk vdi qcow2 hdddirect"
-
-VM_ROOTFS_TYPE ?= "ext4"
-ROOTFS ?= "${IMGDEPLOYDIR}/${IMAGE_LINK_NAME}.${VM_ROOTFS_TYPE}"
-
-# Used by bootloader
-LABELS_VM ?= "boot"
-ROOT_VM ?= "root=/dev/sda2"
-# Using an initramfs is optional. Enable it by setting INITRD_IMAGE_VM.
-INITRD_IMAGE_VM ?= ""
-INITRD_VM ?= "${@'${IMGDEPLOYDIR}/${INITRD_IMAGE_VM}-${MACHINE}.cpio.gz' if '${INITRD_IMAGE_VM}' else ''}"
-do_bootdirectdisk[depends] += "${@'${INITRD_IMAGE_VM}:do_image_complete' if '${INITRD_IMAGE_VM}' else ''}"
-
-BOOTDD_VOLUME_ID   ?= "boot"
-BOOTDD_EXTRA_SPACE ?= "16384"
-
-DISK_SIGNATURE ?= "${DISK_SIGNATURE_GENERATED}"
-DISK_SIGNATURE[vardepsexclude] = "DISK_SIGNATURE_GENERATED"
-
-build_boot_dd() {
-	HDDDIR="${S}/hdd/boot"
-	HDDIMG="${S}/hdd.image"
-	IMAGE=${IMGDEPLOYDIR}/${IMAGE_NAME}.hdddirect
-
-	populate_kernel $HDDDIR
-
-	if [ "${PCBIOS}" = "1" ]; then
-		syslinux_hddimg_populate $HDDDIR
-	fi
-	if [ "${EFI}" = "1" ]; then
-		efi_hddimg_populate $HDDDIR
-	fi
-
-	BLOCKS=`du -bks $HDDDIR | cut -f 1`
-	BLOCKS=`expr $BLOCKS + ${BOOTDD_EXTRA_SPACE}`
-
-	# Remove it since mkdosfs would fail when it exists
-	rm -f $HDDIMG
-	mkdosfs -n ${BOOTDD_VOLUME_ID} ${MKDOSFS_EXTRAOPTS} -C $HDDIMG $BLOCKS
-	mcopy -i $HDDIMG -s $HDDDIR/* ::/
-
-	if [ "${PCBIOS}" = "1" ]; then
-		syslinux_hdddirect_install $HDDIMG
-	fi	
-	chmod 644 $HDDIMG
-
-	ROOTFSBLOCKS=`du -Lbks ${ROOTFS} | cut -f 1`
-	TOTALSIZE=`expr $BLOCKS + $ROOTFSBLOCKS`
-	END1=`expr $BLOCKS \* 1024`
-	END2=`expr $END1 + 512`
-	END3=`expr \( $ROOTFSBLOCKS \* 1024 \) + $END1`
-
-	echo $ROOTFSBLOCKS $TOTALSIZE $END1 $END2 $END3
-	rm -rf $IMAGE
-	dd if=/dev/zero of=$IMAGE bs=1024 seek=$TOTALSIZE count=1
-
-	parted $IMAGE mklabel msdos
-	parted $IMAGE mkpart primary fat16 0 ${END1}B
-	parted $IMAGE unit B mkpart primary ext2 ${END2}B ${END3}B
-	parted $IMAGE set 1 boot on 
-
-	parted $IMAGE print
-
-	awk "BEGIN { printf \"$(echo ${DISK_SIGNATURE} | sed 's/\(..\)\(..\)\(..\)\(..\)/\\x\4\\x\3\\x\2\\x\1/')\" }" | \
-		dd of=$IMAGE bs=1 seek=440 conv=notrunc
-
-	OFFSET=`expr $END2 / 512`
-	if [ "${PCBIOS}" = "1" ]; then
-		dd if=${STAGING_DATADIR}/syslinux/mbr.bin of=$IMAGE conv=notrunc
-	fi
-
-	dd if=$HDDIMG of=$IMAGE conv=notrunc seek=1 bs=512
-	dd if=${ROOTFS} of=$IMAGE conv=notrunc seek=$OFFSET bs=512
-
-	cd ${IMGDEPLOYDIR}
-
-	ln -sf ${IMAGE_NAME}.hdddirect ${IMGDEPLOYDIR}/${IMAGE_LINK_NAME}.hdddirect
-} 
-
-python do_bootdirectdisk() {
-    validate_disk_signature(d)
-    set_live_vm_vars(d, 'VM')
-    if d.getVar("PCBIOS") == "1":
-        bb.build.exec_func('build_syslinux_cfg', d)
-    if d.getVar("EFI") == "1":
-        bb.build.exec_func('build_efi_cfg', d)
-    bb.build.exec_func('build_boot_dd', d)
-}
-
-def generate_disk_signature():
-    import uuid
-
-    signature = str(uuid.uuid4())[:8]
-
-    if signature != '00000000':
-        return signature
-    else:
-        return 'ffffffff'
-
-def validate_disk_signature(d):
-    import re
-
-    disk_signature = d.getVar("DISK_SIGNATURE")
-
-    if not re.match(r'^[0-9a-fA-F]{8}$', disk_signature):
-        bb.fatal("DISK_SIGNATURE '%s' must be an 8 digit hex string" % disk_signature)
-
-DISK_SIGNATURE_GENERATED := "${@generate_disk_signature()}"
-
-run_qemu_img (){
-    type="$1"
-    qemu-img convert -O $type ${IMGDEPLOYDIR}/${IMAGE_LINK_NAME}.hdddirect ${IMGDEPLOYDIR}/${IMAGE_NAME}.$type
-
-    ln -sf ${IMAGE_NAME}.$type ${IMGDEPLOYDIR}/${IMAGE_LINK_NAME}.$type
-}
-create_vmdk_image () {
-    run_qemu_img vmdk
-}
-
-create_vdi_image () {
-    run_qemu_img vdi
-}
-
-create_qcow2_image () {
-    run_qemu_img qcow2
-}
-
-python do_vmimg() {
-    if 'vmdk' in d.getVar('IMAGE_FSTYPES'):
-        bb.build.exec_func('create_vmdk_image', d)
-    if 'vdi' in d.getVar('IMAGE_FSTYPES'):
-        bb.build.exec_func('create_vdi_image', d)
-    if 'qcow2' in d.getVar('IMAGE_FSTYPES'):
-        bb.build.exec_func('create_qcow2_image', d)
-}
-
-addtask bootdirectdisk before do_vmimg
-addtask vmimg after do_bootdirectdisk before do_image_complete
-do_vmimg[depends] += "qemu-native:do_populate_sysroot"

meta/classes/image.bbclass

@@ -138,9 +138,6 @@ def build_live(d):
 IMAGE_TYPE_live = "${@build_live(d)}"
 inherit ${IMAGE_TYPE_live}
 
-IMAGE_TYPE_vm = '${@bb.utils.contains_any("IMAGE_FSTYPES", ["vmdk", "vdi", "qcow2", "hdddirect"], "image-vm", "", d)}'
-inherit ${IMAGE_TYPE_vm}
-
 IMAGE_TYPE_container = '${@bb.utils.contains("IMAGE_FSTYPES", "container", "image-container", "", d)}'
 inherit ${IMAGE_TYPE_container}
 
@@ -264,7 +261,7 @@ fakeroot python do_rootfs () {
 do_rootfs[dirs] = "${TOPDIR}"
 do_rootfs[cleandirs] += "${S} ${IMGDEPLOYDIR}"
 do_rootfs[umask] = "022"
-addtask rootfs before do_build after do_prepare_recipe_sysroot
+addtask rootfs after do_prepare_recipe_sysroot
 
 fakeroot python do_image () {
     from oe.utils import execute_pre_post_process
@@ -275,7 +272,7 @@ fakeroot python do_image () {
 }
 do_image[dirs] = "${TOPDIR}"
 do_image[umask] = "022"
-addtask do_image after do_rootfs before do_build
+addtask do_image after do_rootfs
 
 fakeroot python do_image_complete () {
     from oe.utils import execute_pre_post_process
@@ -292,6 +289,10 @@ do_image_complete[sstate-inputdirs] = "${IMGDEPLOYDIR}"
 do_image_complete[sstate-outputdirs] = "${DEPLOY_DIR_IMAGE}"
 do_image_complete[stamp-extra-info] = "${MACHINE}"
 addtask do_image_complete after do_image before do_build
+python do_image_complete_setscene () {
+    sstate_setscene(d)
+}
+addtask do_image_complete_setscene
 
 # Add image-level QA/sanity checks to IMAGE_QA_COMMANDS
 #
@@ -323,6 +324,15 @@ fakeroot python do_image_qa () {
 }
 addtask do_image_qa after do_image_complete before do_build
 
+SSTATETASKS += "do_image_qa"
+SSTATE_SKIP_CREATION_task-image-qa = '1'
+do_image_qa[sstate-inputdirs] = ""
+do_image_qa[sstate-outputdirs] = ""
+python do_image_qa_setscene () {
+    sstate_setscene(d)
+}
+addtask do_image_qa_setscene
+
 def setup_debugfs_variables(d):
     d.appendVar('IMAGE_ROOTFS', '-dbg')
     d.appendVar('IMAGE_LINK_NAME', '-dbg')

meta/classes/image_types.bbclass

@@ -119,7 +119,8 @@ IMAGE_CMD_squashfs-lzo = "mksquashfs ${IMAGE_ROOTFS} ${IMGDEPLOYDIR}/${IMAGE_NAM
 # In practice, it turned out to be not needed when creating archives and
 # required when extracting, but it seems prudent to use it in both cases.
 IMAGE_CMD_TAR ?= "tar"
-IMAGE_CMD_tar = "${IMAGE_CMD_TAR} -cvf ${IMGDEPLOYDIR}/${IMAGE_NAME}${IMAGE_NAME_SUFFIX}.tar -C ${IMAGE_ROOTFS} ."
+# ignore return code 1 "file changed as we read it" as other tasks(e.g. do_image_wic) may be hardlinking rootfs
+IMAGE_CMD_tar = "${IMAGE_CMD_TAR} -cf ${IMGDEPLOYDIR}/${IMAGE_NAME}${IMAGE_NAME_SUFFIX}.tar -C ${IMAGE_ROOTFS} . || [ $? -eq 1 ]"
 
 do_image_cpio[cleandirs] += "${WORKDIR}/cpio_append"
 IMAGE_CMD_cpio () {
@@ -254,10 +255,6 @@ IMAGE_TYPES = " \
     ubi ubifs multiubi \
     tar tar.gz tar.bz2 tar.xz tar.lz4 \
     cpio cpio.gz cpio.xz cpio.lzma cpio.lz4 \
-    vmdk \
-    vdi \
-    qcow2 \
-    hdddirect \
     elf \
     wic wic.gz wic.bz2 wic.lzma \
     container \
@@ -269,7 +266,7 @@ IMAGE_TYPES = " \
 # CONVERSION_CMD/DEPENDS.
 COMPRESSIONTYPES ?= ""
 
-CONVERSIONTYPES = "gz bz2 lzma xz lz4 lzo zip sum md5sum sha1sum sha224sum sha256sum sha384sum sha512sum bmap u-boot ${COMPRESSIONTYPES}"
+CONVERSIONTYPES = "gz bz2 lzma xz lz4 lzo zip sum md5sum sha1sum sha224sum sha256sum sha384sum sha512sum bmap u-boot vmdk vdi qcow2 ${COMPRESSIONTYPES}"
 CONVERSION_CMD_lzma = "lzma -k -f -7 ${IMAGE_NAME}${IMAGE_NAME_SUFFIX}.${type}"
 CONVERSION_CMD_gz = "gzip -f -9 -c ${IMAGE_NAME}${IMAGE_NAME_SUFFIX}.${type} > ${IMAGE_NAME}${IMAGE_NAME_SUFFIX}.${type}.gz"
 CONVERSION_CMD_bz2 = "pbzip2 -f -k ${IMAGE_NAME}${IMAGE_NAME_SUFFIX}.${type}"
@@ -287,6 +284,9 @@ CONVERSION_CMD_sha384sum = "sha384sum ${IMAGE_NAME}${IMAGE_NAME_SUFFIX}.${type}
 CONVERSION_CMD_sha512sum = "sha512sum ${IMAGE_NAME}${IMAGE_NAME_SUFFIX}.${type} > ${IMAGE_NAME}${IMAGE_NAME_SUFFIX}.${type}.sha512sum"
 CONVERSION_CMD_bmap = "bmaptool create ${IMAGE_NAME}${IMAGE_NAME_SUFFIX}.${type} -o ${IMAGE_NAME}${IMAGE_NAME_SUFFIX}.${type}.bmap"
 CONVERSION_CMD_u-boot = "mkimage -A ${UBOOT_ARCH} -O linux -T ramdisk -C none -n ${IMAGE_NAME} -d ${IMAGE_NAME}${IMAGE_NAME_SUFFIX}.${type} ${IMAGE_NAME}${IMAGE_NAME_SUFFIX}.${type}.u-boot"
+CONVERSION_CMD_vmdk = "qemu-img convert -O vmdk ${IMAGE_NAME}${IMAGE_NAME_SUFFIX}.${type} ${IMAGE_NAME}${IMAGE_NAME_SUFFIX}.${type}.vmdk"
+CONVERSION_CMD_vdi = "qemu-img convert -O vdi ${IMAGE_NAME}${IMAGE_NAME_SUFFIX}.${type} ${IMAGE_NAME}${IMAGE_NAME_SUFFIX}.${type}.vdi"
+CONVERSION_CMD_qcow2 = "qemu-img convert -O qcow2 ${IMAGE_NAME}${IMAGE_NAME_SUFFIX}.${type} ${IMAGE_NAME}${IMAGE_NAME_SUFFIX}.${type}.qcow2"
 CONVERSION_DEPENDS_lzma = "xz-native"
 CONVERSION_DEPENDS_gz = "pigz-native"
 CONVERSION_DEPENDS_bz2 = "pbzip2-native"
@@ -297,6 +297,9 @@ CONVERSION_DEPENDS_zip = "zip-native"
 CONVERSION_DEPENDS_sum = "mtd-utils-native"
 CONVERSION_DEPENDS_bmap = "bmap-tools-native"
 CONVERSION_DEPENDS_u-boot = "u-boot-mkimage-native"
+CONVERSION_DEPENDS_vmdk = "qemu-native"
+CONVERSION_DEPENDS_vdi = "qemu-native"
+CONVERSION_DEPENDS_qcow2 = "qemu-native"
 
 RUNNABLE_IMAGE_TYPES ?= "ext2 ext3 ext4"
 RUNNABLE_MACHINE_PATTERNS ?= "qemu"

meta/classes/license.bbclass

@@ -255,14 +255,9 @@ def get_boot_dependencies(d):
     """
 
     depends = []
-    boot_depends_string = ""
     taskdepdata = d.getVar("BB_TASKDEPDATA", False)
-    # Only bootimg and bootdirectdisk include the depends flag
-    boot_tasks = ["do_bootimg", "do_bootdirectdisk",]
-
-    for task in boot_tasks:
-        boot_depends_string = "%s %s" % (boot_depends_string,
-                d.getVarFlag(task, "depends") or "")
+    # Only bootimg includes the depends flag
+    boot_depends_string = d.getVarFlag("do_bootimg", "depends") or ""
     boot_depends = [dep.split(":")[0] for dep
                 in boot_depends_string.split()
                 if not dep.split(":")[0].endswith("-native")]

meta/classes/module.bbclass

@@ -1,6 +1,6 @@
 inherit module-base kernel-module-split pkgconfig
 
-addtask make_scripts after do_prepare_recipe_sysroot before do_compile
+addtask make_scripts after do_prepare_recipe_sysroot before do_configure
 do_make_scripts[lockfiles] = "${TMPDIR}/kernel-scripts.lock"
 do_make_scripts[depends] += "virtual/kernel:do_shared_workdir"
 

meta/classes/package.bbclass

@@ -879,6 +879,11 @@ python split_and_strip_files () {
         debugdir = "/.debug"
         debuglibdir = ""
         debugsrcdir = ""
+    elif d.getVar('PACKAGE_DEBUG_SPLIT_STYLE') == 'debug-with-srcpkg':
+        debugappend = ""
+        debugdir = "/.debug"
+        debuglibdir = ""
+        debugsrcdir = "/usr/src/debug"
     else:
         # Original OE-core, a.k.a. ".debug", style debug info
         debugappend = ""
@@ -1092,6 +1097,15 @@ python populate_packages () {
     
     autodebug = not (d.getVar("NOAUTOPACKAGEDEBUG") or False)
 
+    split_source_package = (d.getVar('PACKAGE_DEBUG_SPLIT_STYLE') == 'debug-with-srcpkg')
+
+    # If debug-with-srcpkg mode is enabled then the src package is added
+    # into the package list and the source directory as its main content
+    if split_source_package:
+        src_package_name = ('%s-src' % d.getVar('PN'))
+        packages += (' ' + src_package_name)
+        d.setVar('FILES_%s' % src_package_name, '/usr/src/debug')
+
     # Sanity check PACKAGES for duplicates
     # Sanity should be moved to sanity.bbclass once we have the infrastucture
     package_list = []
@@ -1100,7 +1114,12 @@ python populate_packages () {
         if pkg in package_list:
             msg = "%s is listed in PACKAGES multiple times, this leads to packaging errors." % pkg
             package_qa_handle_error("packages-list", msg, d)
-        elif autodebug and pkg.endswith("-dbg"):
+        # If debug-with-srcpkg mode is enabled then the src package will have
+        # priority over dbg package when assigning the files.
+        # This allows src package to include source files and remove them from dbg.
+        elif split_source_package and pkg.endswith("-src"):
+            package_list.insert(0, pkg)
+        elif autodebug and pkg.endswith("-dbg") and not split_source_package:
             package_list.insert(0, pkg)
         else:
             package_list.append(pkg)
@@ -1460,7 +1479,7 @@ python package_do_filedeps() {
     for pkg in packages.split():
         if d.getVar('SKIP_FILEDEPS_' + pkg) == '1':
             continue
-        if pkg.endswith('-dbg') or pkg.endswith('-doc') or pkg.find('-locale-') != -1 or pkg.find('-localedata-') != -1 or pkg.find('-gconv-') != -1 or pkg.find('-charmap-') != -1 or pkg.startswith('kernel-module-'):
+        if pkg.endswith('-dbg') or pkg.endswith('-doc') or pkg.find('-locale-') != -1 or pkg.find('-localedata-') != -1 or pkg.find('-gconv-') != -1 or pkg.find('-charmap-') != -1 or pkg.startswith('kernel-module-') or pkg.endswith('-src'):
             continue
         for files in chunks(pkgfiles[pkg], 100):
             pkglist.append((pkg, files, rpmdeps, pkgdest, magic))
@@ -1578,7 +1597,7 @@ python package_do_shlibs() {
                 combos.append("-".join(options[0:i]))
             return combos
 
-        if (file.endswith('.dylib') or file.endswith('.so')) and not pkg.endswith('-dev') and not pkg.endswith('-dbg'):
+        if (file.endswith('.dylib') or file.endswith('.so')) and not pkg.endswith('-dev') and not pkg.endswith('-dbg') and not pkg.endswith('-src'):
             # Drop suffix
             name = os.path.basename(file).rsplit(".",1)[0]
             # Find all combinations

meta/classes/rm_work.bbclass

@@ -35,22 +35,12 @@ do_rm_work () {
         fi
     done
 
-    cd ${WORKDIR}
-    for dir in *
-    do
-        # Retain only logs and other files in temp, safely ignore
-        # failures of removing pseudo folers on NFS2/3 server.
-        if [ $dir = 'pseudo' ]; then
-            rm -rf $dir 2> /dev/null || true
-        elif ! echo '${RM_WORK_EXCLUDE_ITEMS}' | grep -q -w "$dir"; then
-            rm -rf $dir
-        fi
-    done
-
     # Need to add pseudo back or subsqeuent work in this workdir
     # might fail since setscene may not rerun to recreate it
     mkdir -p ${WORKDIR}/pseudo/
 
+    excludes='${RM_WORK_EXCLUDE_ITEMS}'
+
     # Change normal stamps into setscene stamps as they better reflect the
     # fact WORKDIR is now empty
     # Also leave noexec stamps since setscene stamps don't cover them
@@ -71,7 +61,12 @@ do_rm_work () {
                 i=dummy
                 break
                 ;;
-            *do_rootfs*|*do_image*|*do_bootimg*|*do_bootdirectdisk*|*do_vmimg*|*do_write_qemuboot_conf*)
+            *do_image_complete*)
+                mv $i `echo $i | sed -e "s#${j}#${j}_setscene#"`
+                i=dummy
+                break
+                ;;
+            *do_rootfs*|*do_image*|*do_bootimg*|*do_write_qemuboot_conf*)
                 i=dummy
                 break
                 ;;
@@ -79,6 +74,12 @@ do_rm_work () {
                 i=dummy
                 break
                 ;;
+            *do_addto_recipe_sysroot*)
+                # Preserve recipe-sysroot-native if do_addto_recipe_sysroot has been used
+                excludes="$excludes recipe-sysroot-native"
+                i=dummy
+                break
+                ;;
             # We remove do_package entirely, including any
             # sstate version since otherwise we'd need to leave 'plaindirs' around
             # such as 'packages' and 'packages-split' and these can be large. No end
@@ -101,6 +102,18 @@ do_rm_work () {
         done
         rm -f $i
     done
+
+    cd ${WORKDIR}
+    for dir in *
+    do
+        # Retain only logs and other files in temp, safely ignore
+        # failures of removing pseudo folers on NFS2/3 server.
+        if [ $dir = 'pseudo' ]; then
+            rm -rf $dir 2> /dev/null || true
+        elif ! echo "$excludes" | grep -q -w "$dir"; then
+            rm -rf $dir
+        fi
+    done
 }
 do_rm_work_all () {
     :

meta/classes/rootfsdebugfiles.bbclass

@@ -15,6 +15,10 @@
 #    ROOTFS_DEBUG_FILES += "${TOPDIR}/conf/dropbear_rsa_host_key ${IMAGE_ROOTFS}/etc/dropbear/dropbear_rsa_host_key ;"
 # 2. Boot the image once, copy the dropbear_rsa_host_key from
 #    the device into your build conf directory.
+# 3. A optional parameter can be used to set file mode
+#    of the copied target, for instance:
+#    ROOTFS_DEBUG_FILES += "${TOPDIR}/conf/dropbear_rsa_host_key ${IMAGE_ROOTFS}/etc/dropbear/dropbear_rsa_host_key 0600;"
+#    in case they might be required to have a specific mode. (Shoundn't be too open, for example)
 #
 # Do not use for production images! It bypasses several
 # core build mechanisms (updating the image when one
@@ -27,10 +31,11 @@ ROOTFS_DEBUG_FILES[doc] = "Lists additional files or directories to be installed
 ROOTFS_POSTPROCESS_COMMAND += "rootfs_debug_files ;"
 rootfs_debug_files () {
    #!/bin/sh -e
-   echo "${ROOTFS_DEBUG_FILES}" | sed -e 's/;/\n/g' | while read source target; do
+   echo "${ROOTFS_DEBUG_FILES}" | sed -e 's/;/\n/g' | while read source target mode; do
       if [ -e "$source" ]; then
          mkdir -p $(dirname $target)
          cp -a $source $target
+         [ -n "$mode" ] && chmod $mode $target
       fi
    done
 }

meta/classes/siteinfo.bbclass

@@ -153,7 +153,7 @@ python () {
         bb.fatal("Please add your architecture to siteinfo.bbclass")
 }
 
-def siteinfo_get_files(d, aclocalcache = False):
+def siteinfo_get_files(d, sysrootcache = False):
     sitedata = siteinfo_data(d)
     sitefiles = ""
     for path in d.getVar("BBPATH").split(":"):
@@ -162,18 +162,11 @@ def siteinfo_get_files(d, aclocalcache = False):
             if os.path.exists(filename):
                 sitefiles += filename + " "
 
-    if not aclocalcache:
+    if not sysrootcache:
         return sitefiles
 
-    # Now check for siteconfig cache files in the directory setup by autotools.bbclass to
-    # avoid races.
-    #
-    # ACLOCALDIR may or may not exist so cache should only be set to True from autotools.bbclass
-    # after files have been copied into this location. To do otherwise risks parsing/signature
-    # issues and the directory being created/removed whilst this code executes. This can happen
-    # when a multilib recipe is parsed along with its base variant which may be running at the time
-    # causing rare but nasty failures
-    path_siteconfig = d.getVar('ACLOCALDIR')
+    # Now check for siteconfig cache files in sysroots
+    path_siteconfig = d.getVar('SITECONFIG_SYSROOTCACHE')
     if path_siteconfig and os.path.isdir(path_siteconfig):
         for i in os.listdir(path_siteconfig):
             if not i.endswith("_config"):

meta/classes/sstate.bbclass

@@ -346,8 +346,6 @@ def sstate_installpkgdir(ss, d):
         oe.path.remove(dir)
 
     for state in ss['dirs']:
-        if d.getVar('SSTATE_SKIP_CREATION') == '1':
-            continue
         prepdir(state[1])
         os.rename(sstateinst + state[0], state[1])
     sstate_install(ss, d)
@@ -596,8 +594,6 @@ def sstate_package(ss, d):
     for state in ss['dirs']:
         if not os.path.exists(state[1]):
             continue
-        if d.getVar('SSTATE_SKIP_CREATION') == '1':
-            continue
         srcbase = state[0].rstrip("/").rsplit('/', 1)[0]
         # Find and error for absolute symlinks. We could attempt to relocate but its not
         # clear where the symlink is relative to in this context. We could add that markup
@@ -625,6 +621,10 @@ def sstate_package(ss, d):
 
     d.setVar('SSTATE_BUILDDIR', sstatebuild)
     d.setVar('SSTATE_PKG', sstatepkg)
+    d.setVar('SSTATE_INSTDIR', sstatebuild)
+
+    if d.getVar('SSTATE_SKIP_CREATION') == '1':
+        return
 
     for f in (d.getVar('SSTATECREATEFUNCS') or '').split() + \
              ['sstate_create_package', 'sstate_sign_package'] + \
@@ -634,8 +634,6 @@ def sstate_package(ss, d):
 
     bb.siggen.dump_this_task(sstatepkg + ".siginfo", d)
 
-    d.setVar('SSTATE_INSTDIR', sstatebuild)
-
     return
 
 def pstaging_fetch(sstatefetch, sstatepkg, d):

meta/classes/systemd-boot.bbclass

@@ -7,10 +7,9 @@
 #                        maintenance.
 #
 # Set EFI_PROVIDER = "systemd-boot" to use systemd-boot on your live images instead of grub-efi
-# (images built by image-live.bbclass or image-vm.bbclass)
+# (images built by image-live.bbclass)
 
 do_bootimg[depends] += "${MLPREFIX}systemd-boot:do_deploy"
-do_bootdirectdisk[depends] += "${MLPREFIX}systemd-boot:do_deploy"
 
 EFIDIR = "/EFI/BOOT"
 

meta/classes/uninative.bbclass

@@ -49,6 +49,12 @@ python uninative_event_fetchloader() {
             localdata = bb.data.createCopy(d)
             localdata.setVar('FILESPATH', "")
             localdata.setVar('DL_DIR', tarballdir)
+            # Our games with path manipulation of DL_DIR mean standard PREMIRRORS don't work
+            # and we can't easily put 'chksum' into the url path from a url parameter with
+            # the current fetcher url handling
+            ownmirror = d.getVar('SOURCE_MIRROR_URL')
+            if ownmirror:
+                localdata.appendVar("PREMIRRORS", " ${UNINATIVE_URL}${UNINATIVE_TARBALL} ${SOURCE_MIRROR_URL}/uninative/%s/${UNINATIVE_TARBALL}" % chksum)
 
             srcuri = d.expand("${UNINATIVE_URL}${UNINATIVE_TARBALL};sha256sum=%s" % chksum)
             bb.note("Fetching uninative binary shim from %s" % srcuri)

meta/conf/distro/include/yocto-uninative.inc

@@ -6,6 +6,6 @@
 # to the distro running on the build machine.
 #
 
-UNINATIVE_URL ?= "http://downloads.yoctoproject.org/releases/uninative/1.6/"
-UNINATIVE_CHECKSUM[i686] ?= "d9daf66785834f888b6f50a1d37a8120815f8380e0e267d311e1dfaf4bb5404e"
-UNINATIVE_CHECKSUM[x86_64] ?= "2b4fffa308d9f19e0742a1a404ff42495fb50c165e5ca0458cedca157372691a"
+UNINATIVE_URL ?= "http://downloads.yoctoproject.org/releases/uninative/1.7/"
+UNINATIVE_CHECKSUM[i686] ?= "d7c341460035936c19d63fe02f354ef1bc993c62d694ae3a31458d1c6997f0c5"
+UNINATIVE_CHECKSUM[x86_64] ?= "ed033c868b87852b07957a4400f3b744c00aef5d6470346ea1a59b6d3e03075e"

meta/conf/documentation.conf

@@ -56,7 +56,6 @@ do_testsdk[doc] = "Installs an SDK and performs runtime tests on the tools insta
 do_uboot_mkimage[doc] = "Creates a uImage file from the kernel for the U-Boot bootloader"
 do_unpack[doc] = "Unpacks the source code into a working directory"
 do_validate_branches[doc] = "Ensures that the source/meta branches are on the locations specified by their SRCREV values for a linux-yocto style kernel"
-do_vmimg[doc] = "Creates an image for use with VMware or VirtualBox compatible virtual machine hosts (based on IMAGE_FSTYPES either .vmdk or .vdi)"
 
 # DESCRIPTIONS FOR VARIABLES #
 

meta/lib/oeqa/core/context.py

@@ -10,7 +10,7 @@ import collections
 
 from oeqa.core.loader import OETestLoader
 from oeqa.core.runner import OETestRunner
-from oeqa.core.exception import OEQAMissingManifest
+from oeqa.core.exception import OEQAMissingManifest, OEQATestNotFound
 
 class OETestContext(object):
     loaderClass = OETestLoader
@@ -139,6 +139,7 @@ class OETestContextExecutor(object):
 
         if args.run_tests:
             self.tc_kwargs['load']['modules'] = args.run_tests
+            self.tc_kwargs['load']['modules_required'] = args.run_tests
         else:
             self.tc_kwargs['load']['modules'] = []
 
@@ -151,7 +152,11 @@ class OETestContextExecutor(object):
         self._process_args(logger, args)
 
         self.tc = self._context_class(**self.tc_kwargs['init'])
-        self.tc.loadTests(self.module_paths, **self.tc_kwargs['load'])
+        try:
+            self.tc.loadTests(self.module_paths, **self.tc_kwargs['load'])
+        except OEQATestNotFound as ex:
+            logger.error(ex)
+            sys.exit(1)
 
         if args.list_tests:
             rc = self.tc.listTests(args.list_tests, **self.tc_kwargs['run'])

meta/lib/oeqa/core/exception.py

@@ -18,3 +18,6 @@ class OEQAMissingManifest(OEQAException):
 
 class OEQAPreRun(OEQAException):
     pass
+
+class OEQATestNotFound(OEQAException):
+    pass

meta/lib/oeqa/core/loader.py

@@ -9,6 +9,7 @@ import inspect
 from oeqa.core.utils.path import findFile
 from oeqa.core.utils.test import getSuiteModules, getCaseID
 
+from oeqa.core.exception import OEQATestNotFound
 from oeqa.core.case import OETestCase
 from oeqa.core.decorator import decoratorClasses, OETestDecorator, \
         OETestFilter, OETestDiscover
@@ -277,6 +278,28 @@ class OETestLoader(unittest.TestLoader):
 
         return self.suiteClass(suite)
 
+    def _required_modules_validation(self):
+        """
+            Search in Test context registry if a required
+            test is found, raise an exception when not found.
+        """
+
+        for module in self.modules_required:
+            found = False
+
+            # The module name is splitted to only compare the
+            # first part of a test case id.
+            comp_len = len(module.split('.'))
+            for case in self.tc._registry['cases']:
+                case_comp = '.'.join(case.split('.')[0:comp_len])
+                if module == case_comp:
+                    found = True
+                    break
+
+            if not found:
+                raise OEQATestNotFound("Not found %s in loaded test cases" % \
+                        module)
+
     def discover(self):
         big_suite = self.suiteClass()
         for path in self.module_paths:
@@ -291,6 +314,9 @@ class OETestLoader(unittest.TestLoader):
         for clss in discover_classes:
             cases = clss.discover(self.tc._registry)
 
+        if self.modules_required:
+            self._required_modules_validation()
+
         return self.suiteClass(cases) if cases else big_suite
 
     def _filterModule(self, module):

meta/lib/oeqa/core/runner.py

@@ -25,10 +25,14 @@ class OEStreamLogger(object):
 
     def write(self, msg):
         if len(msg) > 1 and msg[0] != '\n':
-            self.buffer += msg
-        else:
-            self.logger.log(logging.INFO, self.buffer.rstrip("\n"))
-            self.buffer = ""
+            if '...' in msg:
+                self.buffer += msg
+            elif self.buffer:
+                self.buffer += msg
+                self.logger.log(logging.INFO, self.buffer)
+                self.buffer = ""
+            else:
+                self.logger.log(logging.INFO, msg)
 
     def flush(self):
         for handler in self.logger.handlers:

meta/lib/oeqa/selftest/cases/imagefeatures.py

@@ -3,6 +3,7 @@ from oeqa.utils.commands import runCmd, bitbake, get_bb_var, runqemu
 from oeqa.core.decorator.oeid import OETestID
 from oeqa.utils.sshcontrol import SSHControl
 import os
+import json
 
 class ImageFeatures(OESelftestTestCase):
 
@@ -106,7 +107,7 @@ class ImageFeatures(OESelftestTestCase):
         Author:      Ed Bartosh <ed.bartosh@linux.intel.com>
         """
 
-        features = 'IMAGE_FSTYPES += " ext4 ext4.bmap"'
+        features = 'IMAGE_FSTYPES += " ext4 ext4.bmap ext4.bmap.gz"'
         self.write_config(features)
 
         image_name = 'core-image-minimal'
@@ -116,15 +117,85 @@ class ImageFeatures(OESelftestTestCase):
         link_name = get_bb_var('IMAGE_LINK_NAME', image_name)
         image_path = os.path.join(deploy_dir_image, "%s.ext4" % link_name)
         bmap_path = "%s.bmap" % image_path
+        gzip_path = "%s.gz" % bmap_path
 
-        # check if result image and bmap file are in deploy directory
+        # check if result image, bmap and bmap.gz files are in deploy directory
         self.assertTrue(os.path.exists(image_path))
         self.assertTrue(os.path.exists(bmap_path))
+        self.assertTrue(os.path.exists(gzip_path))
 
         # check if result image is sparse
         image_stat = os.stat(image_path)
         self.assertTrue(image_stat.st_size > image_stat.st_blocks * 512)
 
+        # check if the resulting gzip is valid
+        self.assertTrue(runCmd('gzip -t %s' % gzip_path))
+
+    def test_hypervisor_fmts(self):
+        """
+        Summary:     Check various hypervisor formats
+        Expected:    1. core-image-minimal can be built with vmdk, vdi and
+                        qcow2 support.
+                     2. qemu-img says each image has the expected format
+        Product:     oe-core
+        Author:      Tom Rini <trini@konsulko.com>
+        """
+
+        img_types = [ 'vmdk', 'vdi', 'qcow2' ]
+        features = ""
+        for itype in img_types:
+            features += 'IMAGE_FSTYPES += "wic.%s"\n' % itype
+        self.write_config(features)
+
+        image_name = 'core-image-minimal'
+        bitbake(image_name)
+
+        deploy_dir_image = get_bb_var('DEPLOY_DIR_IMAGE')
+        link_name = get_bb_var('IMAGE_LINK_NAME', image_name)
+        for itype in img_types:
+            image_path = os.path.join(deploy_dir_image, "%s.wic.%s" %
+                                      (link_name, itype))
+
+            # check if result image file is in deploy directory
+            self.assertTrue(os.path.exists(image_path))
+
+            # check if result image is vmdk
+            sysroot = get_bb_var('STAGING_DIR_NATIVE', 'core-image-minimal')
+            result = runCmd('qemu-img info --output json %s' % image_path,
+                            native_sysroot=sysroot)
+            self.assertTrue(json.loads(result.output).get('format') == itype)
+
+    def test_long_chain_conversion(self):
+        """
+        Summary:     Check for chaining many CONVERSION_CMDs together
+        Expected:    1. core-image-minimal can be built with
+                        ext4.bmap.gz.bz2.lzo.xz.u-boot and also create a
+                        sha256sum
+                     2. The above image has a valid sha256sum
+        Product:     oe-core
+        Author:      Tom Rini <trini@konsulko.com>
+        """
+
+        conv = "ext4.bmap.gz.bz2.lzo.xz.u-boot"
+        features = 'IMAGE_FSTYPES += "%s %s.sha256sum"' % (conv, conv)
+        self.write_config(features)
+
+        image_name = 'core-image-minimal'
+        bitbake(image_name)
+
+        deploy_dir_image = get_bb_var('DEPLOY_DIR_IMAGE')
+        link_name = get_bb_var('IMAGE_LINK_NAME', image_name)
+        image_path = os.path.join(deploy_dir_image, "%s.%s" %
+                                  (link_name, conv))
+
+        # check if resulting image is in the deploy directory
+        self.assertTrue(os.path.exists(image_path))
+        self.assertTrue(os.path.exists(image_path + ".sha256sum"))
+
+        # check if the resulting sha256sum agrees
+        self.assertTrue(runCmd('cd %s;sha256sum -c %s.%s.sha256sum' %
+                               (deploy_dir_image, link_name, conv)))
+
     def test_image_fstypes(self):
         """
         Summary:     Check if image of supported image fstypes can be built

meta/lib/oeqa/selftest/cases/runqemu.py

@@ -19,7 +19,7 @@ class RunqemuTests(OESelftestTestCase):
         super(RunqemuTests, self).setUpLocal()
         self.recipe = 'core-image-minimal'
         self.machine =  'qemux86-64'
-        self.fstypes = "ext4 iso hddimg vmdk qcow2 vdi"
+        self.fstypes = "ext4 iso hddimg wic.vmdk wic.qcow2 wic.vdi"
         self.cmd_common = "runqemu nographic"
 
         # Avoid emit the same record multiple times.
@@ -74,7 +74,7 @@ SYSLINUX_TIMEOUT = "10"
     @OETestID(2005)
     def test_boot_recipe_image_vmdk(self):
         """Test runqemu recipe-image vmdk"""
-        cmd = "%s %s vmdk" % (self.cmd_common, self.recipe)
+        cmd = "%s %s wic.vmdk" % (self.cmd_common, self.recipe)
         with runqemu(self.recipe, ssh=False, launch_cmd=cmd) as qemu:
             with open(qemu.qemurunnerlog) as f:
                 self.assertTrue('format=vmdk' in f.read(), "Failed: %s" % cmd)
@@ -82,7 +82,7 @@ SYSLINUX_TIMEOUT = "10"
     @OETestID(2006)
     def test_boot_recipe_image_vdi(self):
         """Test runqemu recipe-image vdi"""
-        cmd = "%s %s vdi" % (self.cmd_common, self.recipe)
+        cmd = "%s %s wic.vdi" % (self.cmd_common, self.recipe)
         with runqemu(self.recipe, ssh=False, launch_cmd=cmd) as qemu:
             with open(qemu.qemurunnerlog) as f:
                 self.assertTrue('format=vdi' in f.read(), "Failed: %s" % cmd)
@@ -113,7 +113,7 @@ SYSLINUX_TIMEOUT = "10"
     @OETestID(2009)
     def test_boot_machine_slirp_qcow2(self):
         """Test runqemu machine slirp qcow2"""
-        cmd = "%s slirp qcow2 %s" % (self.cmd_common, self.machine)
+        cmd = "%s slirp wic.qcow2 %s" % (self.cmd_common, self.machine)
         with runqemu(self.recipe, ssh=False, launch_cmd=cmd) as qemu:
             with open(qemu.qemurunnerlog) as f:
                 self.assertTrue('format=qcow2' in f.read(), "Failed: %s" % cmd)

meta/lib/oeqa/selftest/cases/wic.py

@@ -637,7 +637,7 @@ part /etc --source rootfs --ondisk mmcblk0 --fstype=ext4 --exclude-path bin/ --r
             cmd = "mount |grep '^/dev/' | cut -f1,3 -d ' '"
             status, output = qemu.run_serial(cmd)
             self.assertEqual(1, status, 'Failed to run command "%s": %s' % (cmd, output))
-            self.assertEqual(output, '/dev/root /\r\n/dev/sda3 /mnt')
+            self.assertEqual(output, '/dev/root /\r\n/dev/sda1 /boot\r\n/dev/sda3 /mnt')
 
     @only_for_arch(['i586', 'i686', 'x86_64'])
     @OETestID(1852)

meta/lib/oeqa/selftest/context.py

@@ -13,7 +13,7 @@ from random import choice
 import oeqa
 
 from oeqa.core.context import OETestContext, OETestContextExecutor
-from oeqa.core.exception import OEQAPreRun
+from oeqa.core.exception import OEQAPreRun, OEQATestNotFound
 
 from oeqa.utils.commands import runCmd, get_bb_vars, get_test_layer
 
@@ -196,7 +196,11 @@ class OESelftestTestContextExecutor(OETestContextExecutor):
                 self.tc_kwargs['init']['td']['BBPATH'].split(':'))
 
         self.tc = self._context_class(**self.tc_kwargs['init'])
-        self.tc.loadTests(self.module_paths, **self.tc_kwargs['load'])
+        try:
+            self.tc.loadTests(self.module_paths, **self.tc_kwargs['load'])
+        except OEQATestNotFound as ex:
+            logger.error(ex)
+            sys.exit(1)
 
         if args.list_tests:
             rc = self.tc.listTests(args.list_tests, **self.tc_kwargs['run'])

meta/recipes-core/images/build-appliance-image_15.0.0.bb

@@ -19,7 +19,7 @@ IMAGE_ROOTFS_EXTRA_SPACE = "41943040"
 APPEND += "rootfstype=ext4 quiet"
 
 DEPENDS = "zip-native python3-pip-native"
-IMAGE_FSTYPES = "vmdk"
+IMAGE_FSTYPES = "wic.vmdk"
 
 inherit core-image module-base setuptools3
 
@@ -120,7 +120,7 @@ create_bundle_files () {
 	cd ${WORKDIR}
 	mkdir -p Yocto_Build_Appliance
 	cp *.vmx* Yocto_Build_Appliance
-	ln -sf ${IMGDEPLOYDIR}/${IMAGE_NAME}.vmdk Yocto_Build_Appliance/Yocto_Build_Appliance.vmdk
+	ln -sf ${IMGDEPLOYDIR}/${IMAGE_NAME}${IMAGE_NAME_SUFFIX}.wic.vmdk Yocto_Build_Appliance/Yocto_Build_Appliance.vmdk
 	zip -r ${IMGDEPLOYDIR}/Yocto_Build_Appliance-${DATETIME}.zip Yocto_Build_Appliance
 	ln -sf Yocto_Build_Appliance-${DATETIME}.zip ${IMGDEPLOYDIR}/Yocto_Build_Appliance.zip
 }
@@ -130,4 +130,4 @@ python do_bundle_files() {
     bb.build.exec_func('create_bundle_files', d)
 }
 
-addtask bundle_files after do_vmimg before do_image_complete
+addtask bundle_files after do_image_wic before do_image_complete

meta/recipes-core/images/core-image-minimal-initramfs.bb

@@ -3,7 +3,7 @@ DESCRIPTION = "Small image capable of booting a device. The kernel includes \
 the Minimal RAM-based Initial Root Filesystem (initramfs), which finds the \
 first 'init' program more efficiently."
 
-PACKAGE_INSTALL = "initramfs-framework-base initramfs-module-udev initramfs-module-setup-live initramfs-module-install-efi ${VIRTUAL-RUNTIME_base-utils} udev base-passwd ${ROOTFS_BOOTSTRAP_INSTALL}"
+PACKAGE_INSTALL = "initramfs-live-boot initramfs-live-install initramfs-live-install-efi ${VIRTUAL-RUNTIME_base-utils} udev base-passwd ${ROOTFS_BOOTSTRAP_INSTALL}"
 
 # Do not pollute the initrd image with rootfs features
 IMAGE_FEATURES = ""

meta/recipes-core/ncurses/ncurses.inc

@@ -18,6 +18,8 @@ SRC_URI = "git://anonscm.debian.org/collab-maint/ncurses.git"
 EXTRA_AUTORECONF = "-I m4"
 CONFIG_SITE =+ "${WORKDIR}/config.cache"
 
+EXTRASITECONFIG = "CFLAGS='${CFLAGS} -I${SYSROOT_DESTDIR}${includedir}'"
+
 # Whether to enable separate widec libraries; must be 'true' or 'false'
 #
 # TODO: remove this variable when widec is supported in every setup?

meta/recipes-devtools/autoconf-archive/autoconf-archive_2016.09.16.bb

@@ -0,0 +1,14 @@
+SUMMARY = "a collection of freely re-usable Autoconf macros"
+HOMEPAGE = "http://www.gnu.org/software/autoconf-archive/"
+SECTION = "devel"
+LICENSE = "GPL-3.0-with-autoconf-exception"
+LIC_FILES_CHKSUM = "file://COPYING;md5=d32239bcb673463ab874e80d47fae504 \
+    file://COPYING.EXCEPTION;md5=fdef168ebff3bc2f13664c365a5fb515"
+
+SRC_URI = "${GNU_MIRROR}/${BPN}/${BPN}-${PV}.tar.xz"
+SRC_URI[md5sum] = "bf19d4cddce260b3c3e1d51d42509071"
+SRC_URI[sha256sum] = "e8f2efd235f842bad2f6938bf4a72240a5e5fcd248e8444335e63beb60fabd82"
+
+inherit autotools
+
+BBCLASSEXTEND = "native nativesdk"

meta/recipes-devtools/autoconf/autoconf.inc

@@ -1,4 +1,4 @@
-SUMMARY = "A GNU tool that procude shell scripts to automatically configure software"
+SUMMARY = "A GNU tool that produce shell scripts to automatically configure software"
 DESCRIPTION = "Autoconf is an extensible package of M4 macros that produce shell scripts to automatically \ 
 configure software source code packages. Autoconf creates a configuration script for a package from a template \
 file that lists the operating system features that the package can use, in the form of M4 macro calls."

meta/recipes-devtools/llvm/llvm/0001-llvm-TargetLibraryInfo-Undefine-libc-functions-if-th.patch

@@ -0,0 +1,93 @@
+From 28293e48cf1a52004c6a78de448718441f9e05f9 Mon Sep 17 00:00:00 2001
+From: Khem Raj <raj.khem@gmail.com>
+Date: Sat, 21 May 2016 00:33:20 +0000
+Subject: [PATCH 1/2] llvm: TargetLibraryInfo: Undefine libc functions if they
+ are macros
+
+musl defines some functions as macros and not inline functions
+if this is the case then make sure to undefine them
+
+Signed-off-by: Khem Raj <raj.khem@gmail.com>
+---
+Upstream-Status: Pending
+
+ include/llvm/Analysis/TargetLibraryInfo.def | 21 +++++++++++++++++++++
+ 1 file changed, 21 insertions(+)
+
+diff --git a/include/llvm/Analysis/TargetLibraryInfo.def b/include/llvm/Analysis/TargetLibraryInfo.def
+index 9cbe917c146..aff8419cf54 100644
+--- a/include/llvm/Analysis/TargetLibraryInfo.def
++++ b/include/llvm/Analysis/TargetLibraryInfo.def
+@@ -656,6 +656,9 @@ TLI_DEFINE_STRING_INTERNAL("fmodl")
+ TLI_DEFINE_ENUM_INTERNAL(fopen)
+ TLI_DEFINE_STRING_INTERNAL("fopen")
+ /// FILE *fopen64(const char *filename, const char *opentype)
++#ifdef fopen64
++#undef fopen64
++#endif
+ TLI_DEFINE_ENUM_INTERNAL(fopen64)
+ TLI_DEFINE_STRING_INTERNAL("fopen64")
+ /// int fprintf(FILE *stream, const char *format, ...);
+@@ -691,6 +694,9 @@ TLI_DEFINE_STRING_INTERNAL("fseek")
+ /// int fseeko(FILE *stream, off_t offset, int whence);
+ TLI_DEFINE_ENUM_INTERNAL(fseeko)
+ TLI_DEFINE_STRING_INTERNAL("fseeko")
++#ifdef fseeko64
++#undef fseeko64
++#endif
+ /// int fseeko64(FILE *stream, off64_t offset, int whence)
+ TLI_DEFINE_ENUM_INTERNAL(fseeko64)
+ TLI_DEFINE_STRING_INTERNAL("fseeko64")
+@@ -701,6 +707,9 @@ TLI_DEFINE_STRING_INTERNAL("fsetpos")
+ TLI_DEFINE_ENUM_INTERNAL(fstat)
+ TLI_DEFINE_STRING_INTERNAL("fstat")
+ /// int fstat64(int filedes, struct stat64 *buf)
++#ifdef fstat64
++#undef fstat64
++#endif
+ TLI_DEFINE_ENUM_INTERNAL(fstat64)
+ TLI_DEFINE_STRING_INTERNAL("fstat64")
+ /// int fstatvfs(int fildes, struct statvfs *buf);
+@@ -716,6 +725,9 @@ TLI_DEFINE_STRING_INTERNAL("ftell")
+ TLI_DEFINE_ENUM_INTERNAL(ftello)
+ TLI_DEFINE_STRING_INTERNAL("ftello")
+ /// off64_t ftello64(FILE *stream)
++#ifdef ftello64
++#undef ftello64
++#endif
+ TLI_DEFINE_ENUM_INTERNAL(ftello64)
+ TLI_DEFINE_STRING_INTERNAL("ftello64")
+ /// int ftrylockfile(FILE *file);
+@@ -836,6 +848,9 @@ TLI_DEFINE_STRING_INTERNAL("logl")
+ TLI_DEFINE_ENUM_INTERNAL(lstat)
+ TLI_DEFINE_STRING_INTERNAL("lstat")
+ /// int lstat64(const char *path, struct stat64 *buf);
++#ifdef lstat64
++#undef lstat64
++#endif
+ TLI_DEFINE_ENUM_INTERNAL(lstat64)
+ TLI_DEFINE_STRING_INTERNAL("lstat64")
+ /// void *malloc(size_t size);
+@@ -1055,6 +1070,9 @@ TLI_DEFINE_STRING_INTERNAL("sscanf")
+ TLI_DEFINE_ENUM_INTERNAL(stat)
+ TLI_DEFINE_STRING_INTERNAL("stat")
+ /// int stat64(const char *path, struct stat64 *buf);
++#ifdef stat64
++#undef stat64
++#endif
+ TLI_DEFINE_ENUM_INTERNAL(stat64)
+ TLI_DEFINE_STRING_INTERNAL("stat64")
+ /// int statvfs(const char *path, struct statvfs *buf);
+@@ -1184,6 +1202,9 @@ TLI_DEFINE_STRING_INTERNAL("times")
+ TLI_DEFINE_ENUM_INTERNAL(tmpfile)
+ TLI_DEFINE_STRING_INTERNAL("tmpfile")
+ /// FILE *tmpfile64(void)
++#ifdef tmpfile64
++#undef tmpfile64
++#endif
+ TLI_DEFINE_ENUM_INTERNAL(tmpfile64)
+ TLI_DEFINE_STRING_INTERNAL("tmpfile64")
+ /// int toascii(int c);
+-- 
+2.13.1
+

meta/recipes-devtools/llvm/llvm/0002-llvm-allow-env-override-of-exe-path.patch

@@ -0,0 +1,39 @@
+From d776487bac17650704614248d19d1e6b35775001 Mon Sep 17 00:00:00 2001
+From: Martin Kelly <mkelly@xevo.com>
+Date: Fri, 19 May 2017 00:22:57 -0700
+Subject: [PATCH 2/2] llvm: allow env override of exe path
+
+When using a native llvm-config from inside a sysroot, we need llvm-config to
+return the libraries, include directories, etc. from inside the sysroot rather
+than from the native sysroot. Thus provide an env override for calling
+llvm-config from a target sysroot.
+
+Signed-off-by: Martin Kelly <mkelly@xevo.com>
+Signed-off-by: Khem Raj <raj.khem@gmail.com>
+---
+Upstream-Status: Pending
+
+ tools/llvm-config/llvm-config.cpp | 7 +++++++
+ 1 file changed, 7 insertions(+)
+
+diff --git a/tools/llvm-config/llvm-config.cpp b/tools/llvm-config/llvm-config.cpp
+index 08b096afb05..d8d7742744e 100644
+--- a/tools/llvm-config/llvm-config.cpp
++++ b/tools/llvm-config/llvm-config.cpp
+@@ -225,6 +225,13 @@ Typical components:\n\
+ 
+ /// \brief Compute the path to the main executable.
+ std::string GetExecutablePath(const char *Argv0) {
++  // Hack for Yocto: we need to override the root path when we are using
++  // llvm-config from within a target sysroot.
++  const char *Sysroot = std::getenv("YOCTO_ALTERNATE_EXE_PATH");
++  if (Sysroot != nullptr) {
++    return Sysroot;
++  }
++
+   // This just needs to be some symbol in the binary; C++ doesn't
+   // allow taking the address of ::main however.
+   void *P = (void *)(intptr_t)GetExecutablePath;
+-- 
+2.13.1
+

meta/recipes-devtools/llvm/llvm_git.bb

@@ -0,0 +1,182 @@
+# Copyright (C) 2017 Khem Raj <raj.khem@gmail.com>
+# Released under the MIT license (see COPYING.MIT for the terms)
+
+DESCRIPTION = "The LLVM Compiler Infrastructure"
+HOMEPAGE = "http://llvm.org"
+LICENSE = "NCSA"
+SECTION = "devel"
+
+LIC_FILES_CHKSUM = "file://LICENSE.TXT;md5=e825e017edc35cfd58e26116e5251771"
+
+DEPENDS = "libffi libxml2-native zlib ninja-native llvm-native"
+
+DEPENDS_remove_toolchain-clang = "llvm-native"
+
+RDEPENDS_${PN}_append_class-target = " ncurses-terminfo"
+
+inherit perlnative pythonnative cmake pkgconfig
+
+PROVIDES += "llvm${PV}"
+
+LLVM_RELEASE = "${PV}"
+LLVM_DIR = "llvm${LLVM_RELEASE}"
+
+SRCREV = "9a5c333388cbb54a0ce3a67c4f539f5e590a089b"
+PV = "5.0"
+PATCH_VERSION = "0"
+SRC_URI = "git://github.com/llvm-mirror/llvm.git;branch=release_50;protocol=http \
+           file://0001-llvm-TargetLibraryInfo-Undefine-libc-functions-if-th.patch \
+           file://0002-llvm-allow-env-override-of-exe-path.patch \
+          "
+S = "${WORKDIR}/git"
+
+LLVM_INSTALL_DIR = "${WORKDIR}/llvm-install"
+def get_llvm_arch(bb, d, arch_var):
+    import re
+    a = d.getVar(arch_var, True)
+    if   re.match('(i.86|athlon|x86.64)$', a):         return 'X86'
+    elif re.match('arm$', a):                          return 'ARM'
+    elif re.match('armeb$', a):                        return 'ARM'
+    elif re.match('aarch64$', a):                      return 'AArch64'
+    elif re.match('aarch64_be$', a):                   return 'AArch64'
+    elif re.match('mips(isa|)(32|64|)(r6|)(el|)$', a): return 'Mips'
+    elif re.match('p(pc|owerpc)(|64)', a):             return 'PowerPC'
+    else:
+        bb.error("cannot map '%s' to a supported llvm architecture" % a)
+    return ""
+
+def get_llvm_target_arch(bb, d):
+    return get_llvm_arch(bb, d, 'TARGET_ARCH')
+#
+# Default to build all OE-Core supported target arches (user overridable).
+#
+LLVM_TARGETS ?= "${@get_llvm_target_arch(bb, d)}"
+LLVM_TARGETS_prepend_x86 = "AMDGPU;"
+LLVM_TARGETS_prepend_x86-64 = "AMDGPU;"
+
+EXTRA_OECMAKE += "-DLLVM_ENABLE_ASSERTIONS=OFF \
+                  -DLLVM_ENABLE_EXPENSIVE_CHECKS=OFF \
+                  -DLLVM_ENABLE_PIC=ON \
+                  -DLLVM_BINDINGS_LIST='' \
+                  -DLLVM_LINK_LLVM_DYLIB=ON \
+                  -DLLVM_ENABLE_FFI=ON \
+                  -DFFI_INCLUDE_DIR=$(pkg-config --variable=includedir libffi) \
+                  -DLLVM_OPTIMIZED_TABLEGEN=ON \
+                  -DLLVM_TARGETS_TO_BUILD="${LLVM_TARGETS}" \
+                  -G Ninja"
+
+EXTRA_OECMAKE_append_class-target = "\
+                  -DCMAKE_CROSSCOMPILING:BOOL=ON \
+                  -DLLVM_TABLEGEN=${STAGING_BINDIR_NATIVE}/llvm-tblgen \
+                 "
+
+EXTRA_OECMAKE_append_class-nativesdk = "\
+                  -DCMAKE_CROSSCOMPILING:BOOL=ON \
+                  -DLLVM_TABLEGEN=${STAGING_BINDIR_NATIVE}/llvm-tblgen \
+                 "
+
+do_configure_prepend() {
+# Fix paths in llvm-config
+	sed -i "s|sys::path::parent_path(CurrentPath))\.str()|sys::path::parent_path(sys::path::parent_path(CurrentPath))).str()|g" ${S}/tools/llvm-config/llvm-config.cpp
+	sed -ri "s#/(bin|include|lib)(/?\")#/\1/${LLVM_DIR}\2#g" ${S}/tools/llvm-config/llvm-config.cpp
+	sed -ri "s#lib/${LLVM_DIR}#${baselib}/${LLVM_DIR}#g" ${S}/tools/llvm-config/llvm-config.cpp
+}
+
+do_compile() {
+	NINJA_STATUS="[%p] " ninja -v ${PARALLEL_MAKE}
+}
+
+do_compile_class-native() {
+	NINJA_STATUS="[%p] " ninja -v ${PARALLEL_MAKE} llvm-tblgen
+}
+
+do_install() {
+	NINJA_STATUS="[%p] " DESTDIR=${LLVM_INSTALL_DIR} ninja -v install
+	install -D -m 0755 ${B}/NATIVE/bin/llvm-config ${D}${libdir}/${LLVM_DIR}/llvm-config-host
+
+	install -d ${D}${bindir}/${LLVM_DIR}
+	cp -r ${LLVM_INSTALL_DIR}${bindir}/* ${D}${bindir}/${LLVM_DIR}/
+
+	install -d ${D}${includedir}/${LLVM_DIR}
+	cp -r ${LLVM_INSTALL_DIR}${includedir}/* ${D}${includedir}/${LLVM_DIR}/
+
+	install -d ${D}${libdir}/${LLVM_DIR}
+
+	# The LLVM sources have "/lib" embedded and so we cannot completely rely on the ${libdir} variable
+	if [ -d ${LLVM_INSTALL_DIR}${libdir}/ ]; then
+		cp -r ${LLVM_INSTALL_DIR}${libdir}/* ${D}${libdir}/${LLVM_DIR}/
+	elif [ -d ${LLVM_INSTALL_DIR}${prefix}/lib ]; then
+		cp -r ${LLVM_INSTALL_DIR}${prefix}/lib/* ${D}${libdir}/${LLVM_DIR}/
+	elif [ -d ${LLVM_INSTALL_DIR}${prefix}/lib64 ]; then
+		cp -r ${LLVM_INSTALL_DIR}${prefix}/lib64/* ${D}${libdir}/${LLVM_DIR}/
+	fi
+
+	# Remove unnecessary cmake files
+	rm -rf ${D}${libdir}/${LLVM_DIR}/cmake
+
+	ln -s ${LLVM_DIR}/libLLVM-${PV}${SOLIBSDEV} ${D}${libdir}/libLLVM-${PV}${SOLIBSDEV}
+
+	# We'll have to delete the libLLVM.so due to multiple reasons...
+	rm -rf ${D}${libdir}/${LLVM_DIR}/libLLVM.so
+	rm -rf ${D}${libdir}/${LLVM_DIR}/libLTO.so
+}
+do_install_class-native() {
+	install -D -m 0755 ${B}/bin/llvm-tblgen ${D}${bindir}/llvm-tblgen
+}
+
+PACKAGES += "${PN}-bugpointpasses ${PN}-llvmhello"
+ALLOW_EMPTY_${PN} = "1"
+ALLOW_EMPTY_${PN}-staticdev = "1"
+FILES_${PN} = ""
+FILES_${PN}-staticdev = ""
+FILES_${PN}-dbg = " \
+    ${bindir}/${LLVM_DIR}/.debug \
+    ${libdir}/${LLVM_DIR}/.debug/BugpointPasses.so \
+    ${libdir}/${LLVM_DIR}/.debug/LLVMHello.so \
+    ${libdir}/${LLVM_DIR}/.debug/libLTO.so* \
+    ${libdir}/${LLVM_DIR}/.debug/llvm-config-host \
+    /usr/src/debug \
+"
+
+FILES_${PN}-dev = " \
+    ${bindir}/${LLVM_DIR} \
+    ${includedir}/${LLVM_DIR} \
+    ${libdir}/${LLVM_DIR}/llvm-config-host \
+"
+
+RRECOMMENDS_${PN}-dev += "${PN}-bugpointpasses ${PN}-llvmhello"
+
+FILES_${PN}-bugpointpasses = "\
+    ${libdir}/${LLVM_DIR}/BugpointPasses.so \
+"
+FILES_${PN} += "\
+    ${libdir}/${LLVM_DIR}/libLTO.so.* \
+"
+
+FILES_${PN}-llvmhello = "\
+    ${libdir}/${LLVM_DIR}/LLVMHello.so \
+"
+
+PACKAGES_DYNAMIC = "^libllvm${LLVM_RELEASE}-.*$"
+NOAUTOPACKAGEDEBUG = "1"
+
+INSANE_SKIP_${MLPREFIX}libllvm${LLVM_RELEASE}-llvm-${LLVM_RELEASE}.${PATCH_VERSION} += "dev-so"
+INSANE_SKIP_${MLPREFIX}libllvm${LLVM_RELEASE}-llvm-${LLVM_RELEASE} += "dev-so"
+INSANE_SKIP_${MLPREFIX}libllvm${LLVM_RELEASE}-llvm += "dev-so"
+
+python llvm_populate_packages() {
+    libdir = bb.data.expand('${libdir}', d)
+    libllvm_libdir = bb.data.expand('${libdir}/${LLVM_DIR}', d)
+    split_dbg_packages = do_split_packages(d, libllvm_libdir+'/.debug', '^lib(.*)\.so$', 'libllvm${LLVM_RELEASE}-%s-dbg', 'Split debug package for %s', allow_dirs=True)
+    split_packages = do_split_packages(d, libdir, '^lib(.*)\.so$', 'libllvm${LLVM_RELEASE}-%s', 'Split package for %s', allow_dirs=True, allow_links=True, recursive=True)
+    split_staticdev_packages = do_split_packages(d, libllvm_libdir, '^lib(.*)\.a$', 'libllvm${LLVM_RELEASE}-%s-staticdev', 'Split staticdev package for %s', allow_dirs=True)
+    if split_packages:
+        pn = d.getVar('PN', True)
+        d.appendVar('RDEPENDS_' + pn, ' '+' '.join(split_packages))
+        d.appendVar('RDEPENDS_' + pn + '-dbg', ' '+' '.join(split_dbg_packages))
+        d.appendVar('RDEPENDS_' + pn + '-staticdev', ' '+' '.join(split_staticdev_packages))
+}
+
+PACKAGESPLITFUNCS_prepend = "llvm_populate_packages "
+
+BBCLASSEXTEND = "native nativesdk"

meta/recipes-devtools/ninja/ninja_1.7.2.bb

@@ -0,0 +1,29 @@
+SUMMARY = "Ninja is a small build system with a focus on speed."
+HOMEPAGE = "http://martine.github.com/ninja/"
+LICENSE = "Apache-2.0"
+LIC_FILES_CHKSUM = "file://COPYING;md5=a81586a64ad4e476c791cda7e2f2c52e"
+
+DEPENDS = "re2c-native ninja-native"
+
+SRCREV = "717b7b4a31db6027207588c0fb89c3ead384747b"
+
+SRC_URI = "git://github.com/martine/ninja.git;branch=release"
+
+S = "${WORKDIR}/git"
+
+do_configure[noexec] = "1"
+
+do_compile_class-native() {
+	./configure.py --bootstrap
+}
+
+do_compile() {
+	./configure.py
+	ninja
+}
+
+do_install() {
+	install -D -m 0755  ${S}/ninja ${D}${bindir}/ninja
+}
+
+BBCLASSEXTEND = "native nativesdk"

meta/recipes-devtools/python/python-2.7-manifest.inc

@@ -281,7 +281,7 @@ RDEPENDS_${PN}-zlib="${PN}-core"
 FILES_${PN}-zlib="${libdir}/python2.7/lib-dynload/zlib.so "
 
 SUMMARY_${PN}-modules="All Python modules"
-RDEPENDS_${PN}-modules="${PN}-2to3 ${PN}-argparse ${PN}-audio ${PN}-bsddb ${PN}-codecs ${PN}-compile ${PN}-compiler ${PN}-compression ${PN}-contextlib ${PN}-core ${PN}-crypt ${PN}-ctypes ${PN}-curses ${PN}-datetime ${PN}-db ${PN}-debugger ${PN}-difflib ${PN}-distutils ${PN}-doctest ${PN}-email ${PN}-fcntl ${PN}-gdbm ${PN}-hotshot ${PN}-html ${PN}-idle ${PN}-image ${PN}-importlib ${PN}-io ${PN}-json ${PN}-lang ${PN}-logging ${PN}-mailbox ${PN}-math ${PN}-mime ${PN}-mmap ${PN}-multiprocessing ${PN}-netclient ${PN}-netserver ${PN}-numbers ${PN}-pickle ${PN}-pkgutil ${PN}-plistlib ${PN}-pprint ${PN}-profile ${PN}-pydoc ${PN}-re ${PN}-readline ${PN}-resource ${PN}-robotparser ${PN}-shell ${PN}-smtpd ${PN}-sqlite3 ${PN}-sqlite3-tests ${PN}-stringold ${PN}-subprocess ${PN}-syslog ${PN}-terminal ${PN}-tests ${PN}-textutils ${PN}-threading ${PN}-tkinter ${PN}-unittest ${PN}-unixadmin ${PN}-xml ${PN}-xmlrpc ${PN}-zlib  "
+RDEPENDS_${PN}-modules="${PN}-2to3 ${PN}-argparse ${PN}-audio ${PN}-bsddb ${PN}-codecs ${PN}-compile ${PN}-compiler ${PN}-compression ${PN}-contextlib ${PN}-core ${PN}-crypt ${PN}-ctypes ${PN}-curses ${PN}-datetime ${PN}-db ${PN}-debugger ${PN}-difflib ${PN}-distutils ${PN}-doctest ${PN}-email ${PN}-fcntl ${PN}-gdbm ${PN}-hotshot ${PN}-html ${PN}-idle ${PN}-image ${PN}-importlib ${PN}-io ${PN}-json ${PN}-lang ${PN}-logging ${PN}-mailbox ${PN}-math ${PN}-mime ${PN}-mmap ${PN}-multiprocessing ${PN}-netclient ${PN}-netserver ${PN}-numbers ${PN}-pickle ${PN}-pkgutil ${PN}-plistlib ${PN}-pprint ${PN}-profile ${PN}-pydoc ${PN}-re ${PN}-readline ${PN}-resource ${PN}-robotparser ${PN}-shell ${PN}-smtpd ${PN}-sqlite3 ${PN}-sqlite3-tests ${PN}-stringold ${PN}-subprocess ${PN}-syslog ${PN}-terminal ${PN}-textutils ${PN}-threading ${PN}-tkinter ${PN}-unittest ${PN}-unixadmin ${PN}-xml ${PN}-xmlrpc ${PN}-zlib  "
 ALLOW_EMPTY_${PN}-modules = "1"
 
 

meta/recipes-devtools/python/python-3.5-manifest.inc

@@ -277,7 +277,7 @@ RDEPENDS_${PN}-xmlrpc="${PN}-core ${PN}-xml ${PN}-netserver ${PN}-lang ${PN}-pyd
 FILES_${PN}-xmlrpc="${libdir}/python3.5/xmlrpclib.* ${libdir}/python3.5/__pycache__/xmlrpclib.* ${libdir}/python3.5/SimpleXMLRPCServer.* ${libdir}/python3.5/__pycache__/SimpleXMLRPCServer.* ${libdir}/python3.5/DocXMLRPCServer.* ${libdir}/python3.5/__pycache__/DocXMLRPCServer.* ${libdir}/python3.5/xmlrpc ${libdir}/python3.5/xmlrpc/__pycache__ "
 
 SUMMARY_${PN}-modules="All Python modules"
-RDEPENDS_${PN}-modules="${PN}-2to3 ${PN}-argparse ${PN}-asyncio ${PN}-audio ${PN}-codecs ${PN}-compile ${PN}-compression ${PN}-core ${PN}-crypt ${PN}-ctypes ${PN}-curses ${PN}-datetime ${PN}-db ${PN}-debugger ${PN}-difflib ${PN}-distutils ${PN}-doctest ${PN}-email ${PN}-enum ${PN}-fcntl ${PN}-gdbm ${PN}-html ${PN}-idle ${PN}-image ${PN}-importlib ${PN}-io ${PN}-json ${PN}-lang ${PN}-logging ${PN}-mailbox ${PN}-math ${PN}-mime ${PN}-mmap ${PN}-multiprocessing ${PN}-netclient ${PN}-netserver ${PN}-numbers ${PN}-pickle ${PN}-pkgutil ${PN}-pprint ${PN}-profile ${PN}-pydoc ${PN}-re ${PN}-readline ${PN}-reprlib ${PN}-resource ${PN}-selectors ${PN}-shell ${PN}-signal ${PN}-smtpd ${PN}-sqlite3 ${PN}-sqlite3-tests ${PN}-stringold ${PN}-subprocess ${PN}-syslog ${PN}-terminal ${PN}-tests ${PN}-textutils ${PN}-threading ${PN}-tkinter ${PN}-typing ${PN}-unittest ${PN}-unixadmin ${PN}-xml ${PN}-xmlrpc  "
+RDEPENDS_${PN}-modules="${PN}-2to3 ${PN}-argparse ${PN}-asyncio ${PN}-audio ${PN}-codecs ${PN}-compile ${PN}-compression ${PN}-core ${PN}-crypt ${PN}-ctypes ${PN}-curses ${PN}-datetime ${PN}-db ${PN}-debugger ${PN}-difflib ${PN}-distutils ${PN}-doctest ${PN}-email ${PN}-enum ${PN}-fcntl ${PN}-gdbm ${PN}-html ${PN}-idle ${PN}-image ${PN}-importlib ${PN}-io ${PN}-json ${PN}-lang ${PN}-logging ${PN}-mailbox ${PN}-math ${PN}-mime ${PN}-mmap ${PN}-multiprocessing ${PN}-netclient ${PN}-netserver ${PN}-numbers ${PN}-pickle ${PN}-pkgutil ${PN}-pprint ${PN}-profile ${PN}-pydoc ${PN}-re ${PN}-readline ${PN}-reprlib ${PN}-resource ${PN}-selectors ${PN}-shell ${PN}-signal ${PN}-smtpd ${PN}-sqlite3 ${PN}-sqlite3-tests ${PN}-stringold ${PN}-subprocess ${PN}-syslog ${PN}-terminal ${PN}-textutils ${PN}-threading ${PN}-tkinter ${PN}-typing ${PN}-unittest ${PN}-unixadmin ${PN}-xml ${PN}-xmlrpc  "
 ALLOW_EMPTY_${PN}-modules = "1"
 
 

meta/recipes-devtools/qemu/qemu-helper-native_1.0.bb

@@ -20,6 +20,5 @@ do_install() {
 	install tunctl ${D}${bindir}/
 }
 
-RM_WORK_EXCLUDE_ITEMS += "recipe-sysroot-native"
 DEPENDS += "qemu-native"
 addtask addto_recipe_sysroot after do_populate_sysroot before do_build

meta/recipes-graphics/mesa/files/0001-ac-fix-build-after-LLVM-5.0-SVN-r300718.patch

@@ -0,0 +1,40 @@
+From 9861437e58fdd0de01193a102608d34e5952953f Mon Sep 17 00:00:00 2001
+From: Christoph Haag <haagch+mesadev@frickel.club>
+Date: Thu, 20 Apr 2017 10:34:18 +0200
+Subject: [PATCH 1/2] ac: fix build after LLVM 5.0 SVN r300718
+MIME-Version: 1.0
+Content-Type: text/plain; charset=UTF-8
+Content-Transfer-Encoding: 8bit
+
+v2: previously getWithDereferenceableBytes() exists, but addAttr() doesn't take that type
+
+Signed-off-by: Christoph Haag <haagch+mesadev@frickel.club>
+Reviewed-by: Nicolai Hähnle <nicolai.haehnle@amd.com>
+Tested-and-reviewed-by: Mike Lothian <mike@fireburn.co.uk>
+---
+Upstream-Status: Backport
+
+ src/amd/common/ac_llvm_helper.cpp | 4 ++++
+ 1 file changed, 4 insertions(+)
+
+diff --git a/src/amd/common/ac_llvm_helper.cpp b/src/amd/common/ac_llvm_helper.cpp
+index d9ea4b1..11fa809 100644
+--- a/src/amd/common/ac_llvm_helper.cpp
++++ b/src/amd/common/ac_llvm_helper.cpp
+@@ -44,9 +44,13 @@ typedef AttributeSet AttributeList;
+ void ac_add_attr_dereferenceable(LLVMValueRef val, uint64_t bytes)
+ {
+    llvm::Argument *A = llvm::unwrap<llvm::Argument>(val);
++#if HAVE_LLVM < 0x0500
+    llvm::AttrBuilder B;
+    B.addDereferenceableAttr(bytes);
+    A->addAttr(llvm::AttributeList::get(A->getContext(), A->getArgNo() + 1,  B));
++#else
++   A->addAttr(llvm::Attribute::getWithDereferenceableBytes(A->getContext(), bytes));
++#endif
+ }
+ 
+ bool ac_is_sgpr_param(LLVMValueRef arg)
+-- 
+2.13.3
+

meta/recipes-graphics/mesa/files/0002-gallivm-Fix-build-against-LLVM-SVN-r302589.patch

@@ -0,0 +1,49 @@
+From a02a0dfda2712d30ad62b8f0421ec7b8244ba2cb Mon Sep 17 00:00:00 2001
+From: =?UTF-8?q?Michel=20D=C3=A4nzer?= <michel.daenzer@amd.com>
+Date: Wed, 10 May 2017 17:26:07 +0900
+Subject: [PATCH 2/2] gallivm: Fix build against LLVM SVN >= r302589
+MIME-Version: 1.0
+Content-Type: text/plain; charset=UTF-8
+Content-Transfer-Encoding: 8bit
+
+deregisterEHFrames doesn't take any parameters anymore.
+
+Reviewed-by: Vedran Miletić <vedran@miletic.net>
+Reviewed-by: Marek Olšák <marek.olsak@amd.com>
+---
+Upstream-Status: Backport
+
+ src/gallium/auxiliary/gallivm/lp_bld_misc.cpp | 12 +++++++++---
+ 1 file changed, 9 insertions(+), 3 deletions(-)
+
+diff --git a/src/gallium/auxiliary/gallivm/lp_bld_misc.cpp b/src/gallium/auxiliary/gallivm/lp_bld_misc.cpp
+index 2a388cb..0e4a531 100644
+--- a/src/gallium/auxiliary/gallivm/lp_bld_misc.cpp
++++ b/src/gallium/auxiliary/gallivm/lp_bld_misc.cpp
+@@ -342,14 +342,20 @@ class DelegatingJITMemoryManager : public BaseMemoryManager {
+       virtual void registerEHFrames(uint8_t *Addr, uint64_t LoadAddr, size_t Size) {
+          mgr()->registerEHFrames(Addr, LoadAddr, Size);
+       }
+-      virtual void deregisterEHFrames(uint8_t *Addr, uint64_t LoadAddr, size_t Size) {
+-         mgr()->deregisterEHFrames(Addr, LoadAddr, Size);
+-      }
+ #else
+       virtual void registerEHFrames(llvm::StringRef SectionData) {
+          mgr()->registerEHFrames(SectionData);
+       }
+ #endif
++#if HAVE_LLVM >= 0x0500
++      virtual void deregisterEHFrames() {
++         mgr()->deregisterEHFrames();
++      }
++#elif HAVE_LLVM >= 0x0304
++      virtual void deregisterEHFrames(uint8_t *Addr, uint64_t LoadAddr, size_t Size) {
++         mgr()->deregisterEHFrames(Addr, LoadAddr, Size);
++      }
++#endif
+       virtual void *getPointerToNamedFunction(const std::string &Name,
+                                               bool AbortOnFailure=true) {
+          return mgr()->getPointerToNamedFunction(Name, AbortOnFailure);
+-- 
+2.13.3
+

meta/recipes-graphics/mesa/files/llvm-config-version.patch

@@ -0,0 +1,32 @@
+When building llvm from git or svn it embeds the svn/git revision into internal version string
+
+$ /mnt/a/oe/build/tmp/work/corei7-64-bec-linux/mesa/2_17.1.5-r0/recipe-sysroot/usr/lib/llvm5.0/llvm-config-host --version
+5.0.0git-9a5c333388c
+
+We need to ignore everything after 5.0.0 which is what the cut cmd is doing
+
+Upstream-Status: Pending
+Signed-off-by: Khem Raj <raj.khem@gmail.com>
+
+Index: mesa-17.1.5/configure.ac
+===================================================================
+--- mesa-17.1.5.orig/configure.ac
++++ mesa-17.1.5/configure.ac
+@@ -967,7 +967,7 @@ strip_unwanted_llvm_flags() {
+ 
+ llvm_set_environment_variables() {
+     if test "x$LLVM_CONFIG" != xno; then
+-        LLVM_VERSION=`$LLVM_CONFIG --version | egrep -o '^[[0-9.]]+'`
++        LLVM_VERSION=`$LLVM_CONFIG --version | cut -c1-5`
+         LLVM_CPPFLAGS=`strip_unwanted_llvm_flags "$LLVM_CONFIG --cppflags"`
+         LLVM_INCLUDEDIR=`$LLVM_CONFIG --includedir`
+         LLVM_LIBDIR=`$LLVM_CONFIG --libdir`
+@@ -2560,7 +2560,7 @@ if test "x$enable_llvm" = xyes; then
+     dnl (See https://llvm.org/bugs/show_bug.cgi?id=6823)
+     if test "x$enable_llvm_shared_libs" = xyes; then
+         dnl We can't use $LLVM_VERSION because it has 'svn' stripped out,
+-        LLVM_SO_NAME=LLVM-`$LLVM_CONFIG --version`
++        LLVM_SO_NAME=LLVM-`$LLVM_CONFIG --version|cut -c1-5`
+         AS_IF([test -f "$LLVM_LIBDIR/lib$LLVM_SO_NAME.$IMP_LIB_EXT"], [llvm_have_one_so=yes])
+ 
+         if test "x$llvm_have_one_so" = xyes; then

meta/recipes-graphics/mesa/mesa.inc

@@ -25,8 +25,11 @@ REQUIRED_DISTRO_FEATURES = "opengl"
 PLATFORMS ??= "${@bb.utils.filter('PACKAGECONFIG', 'x11 wayland', d)} \
                ${@bb.utils.contains('PACKAGECONFIG', 'gbm', 'drm', '', d)}"
 
+export LLVM_CONFIG = "${STAGING_LIBDIR}/llvm${MESA_LLVM_RELEASE}/llvm-config-host"
+
 EXTRA_OECONF = "--enable-shared-glapi \
-                --with-llvm-prefix=${STAGING_BINDIR_NATIVE} \
+                --disable-opencl \
+                --with-llvm-prefix=${STAGING_LIBDIR}/llvm${MESA_LLVM_RELEASE} \
                 --with-platforms='${PLATFORMS}'"
 
 PACKAGECONFIG ??= "gbm egl gles dri \
@@ -68,7 +71,7 @@ GALLIUMDRIVERS_append_x86 = "${@bb.utils.contains('PACKAGECONFIG', 'gallium-llvm
 GALLIUMDRIVERS_append_x86-64 = "${@bb.utils.contains('PACKAGECONFIG', 'gallium-llvm', ',${GALLIUMDRIVERS_LLVM}', '', d)}"
 # keep --with-gallium-drivers separate, because when only one of gallium versions is enabled, other 2 were adding --without-gallium-drivers
 PACKAGECONFIG[gallium]      = "--enable-texture-float --with-gallium-drivers=${GALLIUMDRIVERS}, --without-gallium-drivers"
-MESA_LLVM_RELEASE ?= "3.3"
+MESA_LLVM_RELEASE ?= "5.0"
 PACKAGECONFIG[gallium-llvm] = "--enable-llvm --enable-llvm-shared-libs, --disable-llvm, llvm${MESA_LLVM_RELEASE} \
                                ${@'elfutils' if ${GALLIUMDRIVERS_LLVM33_ENABLED} else ''}"
 export WANT_LLVM_RELEASE = "${MESA_LLVM_RELEASE}"

meta/recipes-graphics/mesa/mesa_17.1.5.bb

@@ -7,6 +7,9 @@ SRC_URI = "https://mesa.freedesktop.org/archive/mesa-${PV}.tar.xz \
            file://0002-hardware-gloat.patch \
            file://0001-mapi-Only-install-khrplatform.h-with-EGL-or-GLES.patch \
            file://vulkan-mkdir.patch \
+           file://llvm-config-version.patch \
+           file://0001-ac-fix-build-after-LLVM-5.0-SVN-r300718.patch \
+           file://0002-gallivm-Fix-build-against-LLVM-SVN-r302589.patch \
 "
 SRC_URI[md5sum] = "6cf936fbcaadd98924298a7009e8265d"
 SRC_URI[sha256sum] = "378516b171712687aace4c7ea8b37c85895231d7a6d61e1e27362cf6034fded9"

meta/recipes-graphics/vulkan/assimp_4.0.0.bb

@@ -0,0 +1,21 @@
+DESCRIPTION = "Open Asset Import Library is a portable Open Source library to import \
+               various well-known 3D model formats in a uniform manner."
+HOMEPAGE = "http://www.assimp.org/"
+SECTION = "devel"
+
+LICENSE = "BSD-3-Clause"
+LIC_FILES_CHKSUM = "file://LICENSE;md5=2119edef0916b0bd511cb3c731076271"
+
+DEPENDS = "zlib"
+
+SRC_URI = "git://github.com/assimp/assimp.git"
+
+SRCREV = "52c8d06f5d6498afd66df983da348a6b112f1314"
+
+S = "${WORKDIR}/git"
+
+inherit cmake
+
+EXTRA_OECMAKE = "-DASSIMP_BUILD_ASSIMP_TOOLS=OFF -DASSIMP_BUILD_TESTS=OFF -DASSIMP_LIB_INSTALL_DIR=${baselib}"
+
+FILES_${PN}-dev += "${libdir}/cmake/"

meta/recipes-graphics/vulkan/assimp_git.bb

@@ -1,14 +0,0 @@
-DESCRIPTION = "Open Asset Import Library"
-LICENSE = "BSD-3-Clause"
-DEPENDS = "zlib"
-
-LIC_FILES_CHKSUM = "file://LICENSE;md5=2119edef0916b0bd511cb3c731076271 \
-                    file://code/Assimp.cpp;endline=41;md5=717f847b6e8f43c64cdbafcfea109923"
-
-SRC_URI = "git://github.com/assimp/assimp.git"
-SRCREV = "b38ba233f530fdb103d3ede3df5126121af78b10"
-S = "${WORKDIR}/git"
-
-inherit cmake
-EXTRA_OECMAKE = "-DASSIMP_BUILD_ASSIMP_TOOLS=OFF -DASSIMP_BUILD_TESTS=OFF"
-FILES_${PN}-dev += "${libdir}/cmake/assimp-3.3"

meta/recipes-kernel/systemtap/systemtap-native_git.bb

@@ -0,0 +1,6 @@
+
+require systemtap_git.bb
+
+inherit native
+
+addtask addto_recipe_sysroot after do_populate_sysroot before do_build

meta/recipes-kernel/systemtap/systemtap_git.bb

@@ -25,4 +25,4 @@ PACKAGECONFIG[python3-probes] = "--with-python3-probes,--without-python3-probes,
 
 inherit autotools gettext pkgconfig distutils3-base
 
-BBCLASSEXTEND = "native nativesdk"
+BBCLASSEXTEND = "nativesdk"

meta/recipes-multimedia/libpng/libpng_1.6.29.bb

@@ -12,7 +12,7 @@ SRC_URI = "${SOURCEFORGE_MIRROR}/project/${BPN}/${BPN}${LIBV}/${PV}/${BP}.tar.xz
 SRC_URI[md5sum] = "3245dbd76ea91e1437507357b858ec97"
 SRC_URI[sha256sum] = "4245b684e8fe829ebb76186327bb37ce5a639938b219882b53d64bd3cfc5f239"
 
-MIRRORS += "${SOURCEFORGE_MIRROR}/project/${BPN}/${BPN}${LIBV}/older-releases/${PV}"
+MIRRORS += "${SOURCEFORGE_MIRROR}/project/${BPN}/${BPN}${LIBV}/${PV}/ ${SOURCEFORGE_MIRROR}/project/${BPN}/${BPN}${LIBV}/older-releases/${PV}/"
 
 BINCONFIG = "${bindir}/libpng-config ${bindir}/libpng16-config"
 

meta/recipes-support/debianutils/debianutils_4.8.1.1.bb

@@ -37,6 +37,8 @@ RDEPENDS_${PN} += "${PN}-run-parts"
 
 ALTERNATIVE_PRIORITY="30"
 ALTERNATIVE_${PN} = "add-shell installkernel remove-shell savelog tempfile which"
+
+ALTERNATIVE_PRIORITY_${PN}-run-parts = "60"
 ALTERNATIVE_${PN}-run-parts = "run-parts"
 
 ALTERNATIVE_${PN}-doc = "which.1"

meta/recipes-support/libpcre/libpcre_8.41.bb

@@ -14,8 +14,8 @@ SRC_URI = "https://ftp.pcre.org/pub/pcre/pcre-${PV}.tar.bz2 \
            file://Makefile \
 "
 
-SRC_URI[md5sum] = "41a842bf7dcecd6634219336e2167d1d"
-SRC_URI[sha256sum] = "00e27a29ead4267e3de8111fcaa59b132d0533cdfdbdddf4b0604279acbcf4f4"
+SRC_URI[md5sum] = "c160d22723b1670447341b08c58981c1"
+SRC_URI[sha256sum] = "e62c7eac5ae7c0e7286db61ff82912e1c0b7a0c13706616e94a7dd729321b530"
 
 CVE_PRODUCT = "pcre"
 

meta/recipes-support/re2c/re2c/mkdir.patch

@@ -0,0 +1,36 @@
+Upstream-Status: Submitted (https://github.com/skvadrik/re2c/pull/191)
+Signed-off-by: Ross Burton <ross.burton@intel.com>
+
+From bccc10c60523f88c8f81413151cdcd612eb16198 Mon Sep 17 00:00:00 2001
+From: Ross Burton <ross.burton@intel.com>
+Date: Mon, 31 Jul 2017 15:43:41 +0100
+Subject: [PATCH] Makefile.am: create target directory before writing into it
+
+In some situations src/parse/ may not exist before a file is copied into the
+directory. Ensure that this doesn't happen by creating the directory first.
+---
+ re2c/Makefile.am | 1 +
+ 1 file changed, 1 insertion(+)
+
+diff --git a/re2c/Makefile.am b/re2c/Makefile.am
+index 3b3b2c5e..0707fc5a 100644
+--- a/Makefile.am
++++ b/Makefile.am
+@@ -187,6 +187,7 @@ CLEANFILES = \
+ 	$(DOC)
+ 
+ $(AUTOGEN_PARSER): $(CUSTOM_PARSER)
++	$(AM_V_at)$(MKDIR_P) $(dir $@)
+ 	$(AM_V_GEN) if test $(BISON) = "no"; \
+ 	then \
+ 		cp $(top_srcdir)/$(BOOTSTRAP_PARSER) $@ && \
+@@ -211,6 +212,7 @@ $(BOOTSTRAP_PARSER): $(CUSTOM_PARSER)
+ 		$(top_srcdir)/$(CUSTOM_PARSER);
+ 
+ .re.cc:
++	$(AM_V_at)$(MKDIR_P) $(dir $@)
+ 	$(AM_V_GEN) if test -x $(RE2C); \
+ 	then \
+ 		$(top_builddir)/$(RE2C) $(RE2CFLAGS) -o $@ $< && \
+-- 
+2.11.0

meta/recipes-support/re2c/re2c_0.16.bb

@@ -0,0 +1,15 @@
+SUMMARY = "Tool for writing very fast and very flexible scanners"
+HOMEPAGE = "http://re2c.sourceforge.net/"
+AUTHOR = "Marcus Börger <helly@users.sourceforge.net>"
+SECTION = "devel"
+LICENSE = "PD"
+LIC_FILES_CHKSUM = "file://README;beginline=146;md5=881056c9add17f8019ccd8c382ba963a"
+
+SRC_URI = "${SOURCEFORGE_MIRROR}/${BPN}/${BPN}-${PV}.tar.gz \
+           file://mkdir.patch"
+SRC_URI[md5sum] = "3bf508fabd52ed7334647d0ccb956e8d"
+SRC_URI[sha256sum] = "48c12564297641cceb5ff05aead57f28118db6277f31e2262437feba89069e84"
+
+BBCLASSEXTEND = "native"
+
+inherit autotools

scripts/contrib/python/generate-manifest-2.7.py

@@ -28,6 +28,7 @@ class MakefileMaker:
     def __init__( self, outfile, isNative ):
         """initialize"""
         self.packages = {}
+        self.excluded_pkgs = []
         self.targetPrefix = "${libdir}/python%s/" % VERSION[:3]
         self.isNative = isNative
         self.output = outfile
@@ -52,7 +53,7 @@ class MakefileMaker:
         self.out( """ """ )
         self.out( "" )
 
-    def addPackage( self, name, description, dependencies, filenames ):
+    def addPackage( self, name, description, dependencies, filenames, mod_exclude = False ):
         """add a package to the Makefile"""
         if type( filenames ) == type( "" ):
             filenames = filenames.split()
@@ -62,6 +63,8 @@ class MakefileMaker:
                 fullFilenames.append( "%s%s" % ( self.targetPrefix, filename ) )
             else:
                 fullFilenames.append( filename )
+        if mod_exclude:
+            self.excluded_pkgs.append( name )
         self.packages[name] = description, dependencies, fullFilenames
 
     def doBody( self ):
@@ -147,7 +150,7 @@ class MakefileMaker:
         line = 'RDEPENDS_${PN}-modules="'
 
         for name, data in sorted(self.packages.items()):
-            if name not in ['${PN}-dev', '${PN}-distutils-staticdev']:
+            if name not in ['${PN}-dev', '${PN}-distutils-staticdev'] and name not in self.excluded_pkgs:
                 line += "%s " % name
 
         self.out( "%s \"" % line )
@@ -382,7 +385,7 @@ if __name__ == "__main__":
     "pty.* tty.*" )
 
     m.addPackage( "${PN}-tests", "Python tests", "${PN}-core ${PN}-modules",
-    "test" ) # package
+    "test", True ) # package
 
     m.addPackage( "${PN}-threading", "Python threading & synchronization support", "${PN}-core ${PN}-lang",
     "_threading_local.* dummy_thread.* dummy_threading.* mutex.* threading.* Queue.*" )

scripts/contrib/python/generate-manifest-3.5.py

@@ -31,6 +31,7 @@ class MakefileMaker:
     def __init__( self, outfile, isNative ):
         """initialize"""
         self.packages = {}
+        self.excluded_pkgs = []
         self.targetPrefix = "${libdir}/python%s/" % VERSION[:3]
         self.isNative = isNative
         self.output = outfile
@@ -55,7 +56,7 @@ class MakefileMaker:
         self.out( """ """ )
         self.out( "" )
 
-    def addPackage( self, name, description, dependencies, filenames ):
+    def addPackage( self, name, description, dependencies, filenames, mod_exclude = False ):
         """add a package to the Makefile"""
         if type( filenames ) == type( "" ):
             filenames = filenames.split()
@@ -67,6 +68,8 @@ class MakefileMaker:
                                                  self.pycachePath( filename ) ) )
             else:
                 fullFilenames.append( filename )
+        if mod_exclude:
+            self.excluded_pkgs.append( name )
         self.packages[name] = description, dependencies, fullFilenames
 
     def pycachePath( self, filename ):
@@ -160,7 +163,7 @@ class MakefileMaker:
         line = 'RDEPENDS_${PN}-modules="'
 
         for name, data in sorted(self.packages.items()):
-            if name not in ['${PN}-dev', '${PN}-distutils-staticdev']:
+            if name not in ['${PN}-dev', '${PN}-distutils-staticdev'] and name not in self.excluded_pkgs:
                 line += "%s " % name
 
         self.out( "%s \"" % line )
@@ -401,7 +404,7 @@ if __name__ == "__main__":
     "pty.* tty.*" )
 
     m.addPackage( "${PN}-tests", "Python tests", "${PN}-core ${PN}-compression",
-    "test" ) # package
+    "test", True ) # package
 
     m.addPackage( "${PN}-threading", "Python threading & synchronization support", "${PN}-core ${PN}-lang",
     "_threading_local.* dummy_thread.* dummy_threading.* mutex.* threading.* queue.*" )

scripts/lib/bsp/help.py

@@ -818,6 +818,10 @@ yocto_layer_help_usage = """
 
 yocto_layer_create_usage = """
 
+ WARNING: this plugin will be removed starting 2.5 development in favour
+ of using 'bitbake-layers create-layer' script/plugin, offering a single
+ script to manage layers.
+
  Create a new generic Yocto layer
 
  usage: yocto-layer create <layer-name> [layer_priority]
@@ -845,6 +849,10 @@ yocto_layer_create_usage = """
 
 yocto_layer_create_help = """
 
+WARNING: this plugin will be removed starting 2.5 development in favour
+of using 'bitbake-layers create-layer' script/plugin, offering a single
+script to manage layers.
+
 NAME
     yocto-layer create - Create a new generic Yocto layer
 

scripts/lib/wic/canned-wks/common.wks.inc

@@ -1,3 +1,3 @@
 # This file is included into 3 canned wks files from this directory
 part /boot --source bootimg-pcbios --ondisk sda --label boot --active --align 1024
-part / --source rootfs --ondisk sda --fstype=ext4 --label platform --align 1024
+part / --source rootfs --use-uuid --fstype=ext4 --label platform --align 1024

scripts/lib/wic/canned-wks/directdisk-bootloader-config.cfg

@@ -12,16 +12,16 @@ DEFAULT Graphics console boot
 
 LABEL Graphics console boot
 KERNEL /vmlinuz
-APPEND label=boot root=/dev/sda2 rootwait
+APPEND label=boot rootwait
 
 LABEL Serial console boot
 KERNEL /vmlinuz
-APPEND label=boot root=/dev/sda2 rootwait console=ttyS0,115200
+APPEND label=boot rootwait console=ttyS0,115200
 
 LABEL Graphics console install
 KERNEL /vmlinuz
-APPEND label=install root=/dev/sda2 rootwait
+APPEND label=install rootwait
 
 LABEL Serial console install
 KERNEL /vmlinuz
-APPEND label=install root=/dev/sda2 rootwait console=ttyS0,115200
+APPEND label=install rootwait console=ttyS0,115200

scripts/lib/wic/canned-wks/qemux86-directdisk.wks

@@ -4,5 +4,5 @@
 
 include common.wks.inc
 
-bootloader  --timeout=0  --append="vga=0 uvesafb.mode_option=640x480-32 root=/dev/sda2 rw mem=256M ip=192.168.7.2::192.168.7.1:255.255.255.0 oprofile.timer=1 rootfstype=ext4 "
+bootloader  --timeout=0  --append="vga=0 uvesafb.mode_option=640x480-32 rw mem=256M ip=192.168.7.2::192.168.7.1:255.255.255.0 oprofile.timer=1 rootfstype=ext4 "
 

scripts/lib/wic/plugins/imager/direct.py

@@ -26,12 +26,15 @@
 
 import logging
 import os
+import random
 import shutil
 import tempfile
 import uuid
 
 from time import strftime
 
+from oe.path import copyhardlinktree
+
 from wic import WicError
 from wic.filemap import sparse_copy
 from wic.ksparser import KickStart, KickStartError
@@ -115,19 +118,25 @@ class DirectPlugin(ImagerPlugin):
             fstab_lines = fstab.readlines()
 
         if self._update_fstab(fstab_lines, self.parts):
-            shutil.copyfile(fstab_path, fstab_path + ".orig")
+            # copy rootfs dir to workdir to update fstab
+            # as rootfs can be used by other tasks and can't be modified
+            new_rootfs = os.path.realpath(os.path.join(self.workdir, "rootfs_copy"))
+            copyhardlinktree(image_rootfs, new_rootfs)
+            fstab_path = os.path.join(new_rootfs, 'etc/fstab')
+
+            os.unlink(fstab_path)
 
             with open(fstab_path, "w") as fstab:
                 fstab.writelines(fstab_lines)
 
-            return fstab_path
+            return new_rootfs
 
     def _update_fstab(self, fstab_lines, parts):
         """Assume partition order same as in wks"""
         updated = False
         for part in parts:
             if not part.realnum or not part.mountpoint \
-               or part.mountpoint in ("/", "/boot"):
+               or part.mountpoint == "/":
                 continue
 
             # mmc device partitions are named mmcblk0p1, mmcblk0p2..
@@ -156,7 +165,10 @@ class DirectPlugin(ImagerPlugin):
         filesystems from the artifacts directly and combine them into
         a partitioned image.
         """
-        fstab_path = self._write_fstab(self.rootfs_dir.get("ROOTFS_DIR"))
+        new_rootfs = self._write_fstab(self.rootfs_dir.get("ROOTFS_DIR"))
+        if new_rootfs:
+            # rootfs was copied to update fstab
+            self.rootfs_dir['ROOTFS_DIR'] = new_rootfs
 
         for part in self.parts:
             # get rootfs size from bitbake variable if it's not set in .ks file
@@ -172,12 +184,7 @@ class DirectPlugin(ImagerPlugin):
                     if rsize_bb:
                         part.size = int(round(float(rsize_bb)))
 
-        try:
-            self._image.prepare(self)
-        finally:
-            if fstab_path:
-                shutil.move(fstab_path + ".orig", fstab_path)
-
+        self._image.prepare(self)
         self._image.layout_partitions()
         self._image.create()
 
@@ -297,7 +304,7 @@ class PartitionedImage():
                           # all partitions (in bytes)
         self.ptable_format = ptable_format  # Partition table format
         # Disk system identifier
-        self.identifier = int.from_bytes(os.urandom(4), 'little')
+        self.identifier = random.SystemRandom().randint(1, 0xffffffff)
 
         self.partitions = partitions
         self.partimages = []

scripts/lib/wic/plugins/source/rootfs.py

@@ -48,7 +48,7 @@ class RootfsPlugin(SourcePlugin):
     @staticmethod
     def __get_rootfs_dir(rootfs_dir):
         if os.path.isdir(rootfs_dir):
-            return rootfs_dir
+            return os.path.realpath(rootfs_dir)
 
         image_rootfs_dir = get_bitbake_var("IMAGE_ROOTFS", rootfs_dir)
         if not os.path.isdir(image_rootfs_dir):
@@ -56,7 +56,7 @@ class RootfsPlugin(SourcePlugin):
                            "named %s has been found at %s, exiting." %
                            (rootfs_dir, image_rootfs_dir))
 
-        return image_rootfs_dir
+        return os.path.realpath(image_rootfs_dir)
 
     @classmethod
     def do_prepare_partition(cls, part, source_params, cr, cr_workdir,
@@ -81,20 +81,19 @@ class RootfsPlugin(SourcePlugin):
                 raise WicError("Couldn't find --rootfs-dir=%s connection or "
                                "it is not a valid path, exiting" % part.rootfs_dir)
 
-        real_rootfs_dir = cls.__get_rootfs_dir(rootfs_dir)
+        part.rootfs_dir = cls.__get_rootfs_dir(rootfs_dir)
 
+        new_rootfs = None
         # Handle excluded paths.
         if part.exclude_path is not None:
             # We need a new rootfs directory we can delete files from. Copy to
             # workdir.
-            new_rootfs = os.path.realpath(os.path.join(cr_workdir, "rootfs"))
+            new_rootfs = os.path.realpath(os.path.join(cr_workdir, "rootfs%d" % part.lineno))
 
             if os.path.lexists(new_rootfs):
                 shutil.rmtree(os.path.join(new_rootfs))
 
-            copyhardlinktree(real_rootfs_dir, new_rootfs)
-
-            real_rootfs_dir = new_rootfs
+            copyhardlinktree(part.rootfs_dir, new_rootfs)
 
             for orig_path in part.exclude_path:
                 path = orig_path
@@ -123,6 +122,5 @@ class RootfsPlugin(SourcePlugin):
                     # Delete whole directory.
                     shutil.rmtree(full_path)
 
-        part.rootfs_dir = real_rootfs_dir
         part.prepare_rootfs(cr_workdir, oe_builddir,
-                            real_rootfs_dir, native_sysroot)
+                            new_rootfs or part.rootfs_dir, native_sysroot)

scripts/runqemu

@@ -96,11 +96,11 @@ Examples:
   runqemu qemux86-64 core-image-sato ext4
   runqemu qemux86-64 wic-image-minimal wic
   runqemu path/to/bzImage-qemux86.bin path/to/nfsrootdir/ serial
-  runqemu qemux86 iso/hddimg/vmdk/qcow2/vdi/ramfs/cpio.gz...
+  runqemu qemux86 iso/hddimg/wic.vmdk/wic.qcow2/wic.vdi/ramfs/cpio.gz...
   runqemu qemux86 qemuparams="-m 256"
   runqemu qemux86 bootparams="psplash=false"
-  runqemu path/to/<image>-<machine>.vmdk
   runqemu path/to/<image>-<machine>.wic
+  runqemu path/to/<image>-<machine>.wic.vmdk
 """)
 
 def check_tun():
@@ -220,7 +220,8 @@ class BaseConfig(object):
         self.snapshot = False
         self.fstypes = ('ext2', 'ext3', 'ext4', 'jffs2', 'nfs', 'btrfs',
                         'cpio.gz', 'cpio', 'ramfs', 'tar.bz2', 'tar.gz')
-        self.vmtypes = ('hddimg', 'hdddirect', 'wic', 'vmdk', 'qcow2', 'vdi', 'iso')
+        self.vmtypes = ('hddimg', 'hdddirect', 'wic', 'wic.vmdk',
+                        'wic.qcow2', 'wic.vdi', 'iso')
         self.network_device = "-device e1000,netdev=net0,mac=@MAC@"
         # Use different mac section for tap and slirp to avoid
         # conflicts, e.g., when one is running with tap, the other is
@@ -987,6 +988,8 @@ class BaseConfig(object):
     def setup_rootfs(self):
         if self.get('QB_ROOTFS') == 'none':
             return
+        if 'wic.' in self.fstype:
+            self.fstype = self.fstype[4:]
         rootfs_format = self.fstype if self.fstype in ('vmdk', 'qcow2', 'vdi') else 'raw'
 
         qb_rootfs_opt = self.get('QB_ROOTFS_OPT')