buildhistory.bbclass: add ptest

The ptest log will be saved to buildhistory/ptest, we can easily get
the regression result between builds by:

$ git show HEAD ptest/pass.fail.skip.*

[YOCTO #11547]

(From OE-Core rev: dcb6cd19fb8c639cb844d116fb83827267f37421)

Signed-off-by: Robert Yang <liezhi.yang@windriver.com>
Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>

bitbake/bin/toaster

@@ -24,6 +24,14 @@ Usage: source toaster start|stop [webport=<address:port>] [noweb]
         [webport] Set the development server (default: localhost:8000)
 "
 
+custom_extention()
+{
+    custom_extension=$BBBASEDIR/lib/toaster/orm/fixtures/custom_toaster_append.sh
+    if [ -f $custom_extension ] ; then
+        $custom_extension $*
+    fi
+}
+
 databaseCheck()
 {
     retval=0
@@ -50,6 +58,11 @@ databaseCheck()
 webserverKillAll()
 {
     local pidfile
+    if [ -f ${BUILDDIR}/.toastermain.pid ] ; then
+        custom_extention web_stop_postpend
+    else
+        custom_extention noweb_stop_postpend
+    fi
     for pidfile in ${BUILDDIR}/.toastermain.pid ${BUILDDIR}/.runbuilds.pid; do
         if [ -f ${pidfile} ]; then
             pid=`cat ${pidfile}`
@@ -89,6 +102,7 @@ webserverStartAll()
     else
         echo "Toaster development webserver started at http://$ADDR_PORT"
         echo -e "\nYou can now run 'bitbake <target>' on the command line and monitor your build in Toaster.\nYou can also use a Toaster project to configure and run a build.\n"
+        custom_extention web_start_postpend $ADDR_PORT
     fi
 
     return $retval
@@ -168,7 +182,6 @@ fi
 unset OE_ROOT
 
 
-
 WEBSERVER=1
 ADDR_PORT="localhost:8000"
 unset CMD
@@ -241,6 +254,7 @@ fi
 echo "The system will $CMD."
 
 # Execute the commands
+custom_extention toaster_prepend $CMD $ADDR_PORT
 
 case $CMD in
     start )
@@ -262,9 +276,10 @@ case $CMD in
             if [ ! -f "$TOASTER_DIR/toaster.sqlite" ] ; then
                 if ! databaseCheck; then
                     echo "Failed ${CMD}."
-                  return 4
+                    return 4
                 fi
             fi
+            custom_extention noweb_start_postpend $ADDR_PORT
         fi
         if [ $WEBSERVER -gt 0 ] && ! webserverStartAll; then
             echo "Failed ${CMD}."
@@ -278,6 +293,7 @@ case $CMD in
         # set fail safe stop system on terminal exit
         trap stop_system SIGHUP
         echo "Successful ${CMD}."
+        custom_extention toaster_postpend $CMD $ADDR_PORT
         return 0
     ;;
     stop )
@@ -285,3 +301,5 @@ case $CMD in
         echo "Successful ${CMD}."
     ;;
 esac
+custom_extention toaster_postpend $CMD $ADDR_PORT
+

bitbake/lib/bb/command.py

@@ -78,6 +78,7 @@ class Command:
                 if not hasattr(command_method, 'readonly') or False == getattr(command_method, 'readonly'):
                     return None, "Not able to execute not readonly commands in readonly mode"
             try:
+                self.cooker.process_inotify_updates()
                 if getattr(command_method, 'needconfig', False):
                     self.cooker.updateCacheSync()
                 result = command_method(self, commandline)
@@ -98,6 +99,7 @@ class Command:
 
     def runAsyncCommand(self):
         try:
+            self.cooker.process_inotify_updates()
             if self.cooker.state in (bb.cooker.state.error, bb.cooker.state.shutdown, bb.cooker.state.forceshutdown):
                 # updateCache will trigger a shutdown of the parser
                 # and then raise BBHandledException triggering an exit

bitbake/lib/bb/cooker.py

@@ -165,7 +165,7 @@ class BBCooker:
     Manages one bitbake build run
     """
 
-    def __init__(self, configuration, featureSet=None):
+    def __init__(self, configuration, featureSet=None, readypipe=None):
         self.recipecaches = None
         self.skiplist = {}
         self.featureset = CookerFeatures()
@@ -189,7 +189,7 @@ class BBCooker:
 
         # If being called by something like tinfoil, we need to clean cached data
         # which may now be invalid
-        bb.parse.__mtime_cache = {}
+        bb.parse.clear_cache()
         bb.parse.BBHandler.cached_statements = {}
 
         self.ui_cmdline = None
@@ -205,15 +205,11 @@ class BBCooker:
 
         self.inotify_modified_files = []
 
-        def _process_inotify_updates(server, notifier_list, abort):
-            for n in notifier_list:
-                if n.check_events(timeout=0):
-                    # read notified events and enqeue them
-                    n.read_events()
-                    n.process_events()
+        def _process_inotify_updates(server, cooker, abort):
+            cooker.process_inotify_updates()
             return 1.0
 
-        self.configuration.server_register_idlecallback(_process_inotify_updates, [self.confignotifier, self.notifier])
+        self.configuration.server_register_idlecallback(_process_inotify_updates, self)
 
         # TOSTOP must not be set or our children will hang when they output
         try:
@@ -237,10 +233,23 @@ class BBCooker:
         # Let SIGHUP exit as SIGTERM
         signal.signal(signal.SIGHUP, self.sigterm_exception)
 
+        if readypipe:
+            os.write(readypipe, b"ready")
+            os.close(readypipe)
+
+    def process_inotify_updates(self):
+        for n in [self.confignotifier, self.notifier]:
+            if n.check_events(timeout=0):
+                # read notified events and enqeue them
+                n.read_events()
+                n.process_events()
+
     def config_notifications(self, event):
         if event.maskname == "IN_Q_OVERFLOW":
             bb.warn("inotify event queue overflowed, invalidating caches.")
+            self.parsecache_valid = False
             self.baseconfig_valid = False
+            bb.parse.clear_cache()
             return
         if not event.pathname in self.configwatcher.bbwatchedfiles:
             return
@@ -252,6 +261,10 @@ class BBCooker:
         if event.maskname == "IN_Q_OVERFLOW":
             bb.warn("inotify event queue overflowed, invalidating caches.")
             self.parsecache_valid = False
+            bb.parse.clear_cache()
+            return
+        if event.pathname.endswith("bitbake-cookerdaemon.log") \
+                or event.pathname.endswith("bitbake.lock"):
             return
         if not event.pathname in self.inotify_modified_files:
             self.inotify_modified_files.append(event.pathname)
@@ -674,12 +687,12 @@ class BBCooker:
                     depend_tree["pn"][pn][ei] = vars(self.recipecaches[mc])[ei][taskfn]
 
 
+            dotname = "%s.%s" % (pn, bb.runqueue.taskname_from_tid(tid))
+            if not dotname in depend_tree["tdepends"]:
+                depend_tree["tdepends"][dotname] = []
             for dep in rq.rqdata.runtaskentries[tid].depends:
                 (depmc, depfn, deptaskname, deptaskfn) = bb.runqueue.split_tid_mcfn(dep)
                 deppn = self.recipecaches[mc].pkg_fn[deptaskfn]
-                dotname = "%s.%s" % (pn, bb.runqueue.taskname_from_tid(tid))
-                if not dotname in depend_tree["tdepends"]:
-                    depend_tree["tdepends"][dotname] = []
                 depend_tree["tdepends"][dotname].append("%s.%s" % (deppn, bb.runqueue.taskname_from_tid(dep)))
             if taskfn not in seen_fns:
                 seen_fns.append(taskfn)
@@ -1188,7 +1201,7 @@ class BBCooker:
             bf = os.path.abspath(bf)
 
         self.collection = CookerCollectFiles(self.bbfile_config_priorities)
-        filelist, masked = self.collection.collect_bbfiles(self.data, self.data)
+        filelist, masked, searchdirs = self.collection.collect_bbfiles(self.data, self.data)
         try:
             os.stat(bf)
             bf = os.path.abspath(bf)
@@ -1464,6 +1477,7 @@ class BBCooker:
             self.updateCacheSync()
 
         if self.state != state.parsing and not self.parsecache_valid:
+            bb.parse.siggen.reset(self.data)
             self.parseConfiguration ()
             if CookerFeatures.SEND_SANITYEVENTS in self.featureset:
                 for mc in self.multiconfigs:
@@ -1477,7 +1491,11 @@ class BBCooker:
                     self.recipecaches[mc].ignored_dependencies.add(dep)
 
             self.collection = CookerCollectFiles(self.bbfile_config_priorities)
-            (filelist, masked) = self.collection.collect_bbfiles(self.data, self.data)
+            (filelist, masked, searchdirs) = self.collection.collect_bbfiles(self.data, self.data)
+
+            # Add inotify watches for directories searched for bb/bbappend files
+            for dirent in searchdirs:
+                self.add_filewatch([[dirent]])
 
             self.parser = CookerParser(self, filelist, masked)
             self.parsecache_valid = True
@@ -1649,6 +1667,18 @@ class CookerCollectFiles(object):
             collectlog.error("no recipe files to build, check your BBPATH and BBFILES?")
             bb.event.fire(CookerExit(), eventdata)
 
+        # We need to track where we look so that we can add inotify watches. There
+        # is no nice way to do this, this is horrid. We intercept the os.listdir()
+        # calls while we run glob().
+        origlistdir = os.listdir
+        searchdirs = []
+
+        def ourlistdir(d):
+            searchdirs.append(d)
+            return origlistdir(d)
+
+        os.listdir = ourlistdir
+
         # Can't use set here as order is important
         newfiles = []
         for f in files:
@@ -1666,6 +1696,8 @@ class CookerCollectFiles(object):
                     if g not in newfiles:
                         newfiles.append(g)
 
+        os.listdir = origlistdir
+
         bbmask = config.getVar('BBMASK')
 
         if bbmask:
@@ -1724,7 +1756,7 @@ class CookerCollectFiles(object):
                 topfile = bbfile_seen[base]
                 self.overlayed[topfile].append(f)
 
-        return (bbfiles, masked)
+        return (bbfiles, masked, searchdirs)
 
     def get_file_appends(self, fn):
         """

bitbake/lib/bb/daemonize.py

@@ -1,49 +1,14 @@
 """
 Python Daemonizing helper
 
-Configurable daemon behaviors:
-
-    1.) The current working directory set to the "/" directory.
-    2.) The current file creation mode mask set to 0.
-    3.) Close all open files (1024). 
-    4.) Redirect standard I/O streams to "/dev/null".
-
-A failed call to fork() now raises an exception.
-
-References:
-    1) Advanced Programming in the Unix Environment: W. Richard Stevens
-	http://www.apuebook.com/apue3e.html
-    2) The Linux Programming Interface: Michael Kerrisk
-	http://man7.org/tlpi/index.html
-    3) Unix Programming Frequently Asked Questions:
-	http://www.faqs.org/faqs/unix-faq/programmer/faq/
-
-Modified to allow a function to be daemonized and return for 
-bitbake use by Richard Purdie
+Originally based on code Copyright (C) 2005 Chad J. Schroeder but now heavily modified
+to allow a function to be daemonized and return for bitbake use by Richard Purdie
 """
 
-__author__ = "Chad J. Schroeder"
-__copyright__ = "Copyright (C) 2005 Chad J. Schroeder"
-__version__ = "0.2"
-
-# Standard Python modules.
-import os                    # Miscellaneous OS interfaces.
-import sys                   # System-specific parameters and functions.
+import os
+import sys
 import io
-
-# Default daemon parameters.
-# File mode creation mask of the daemon.
-# For BitBake's children, we do want to inherit the parent umask.
-UMASK = None
-
-# Default maximum for the number of available file descriptors.
-MAXFD = 1024
-
-# The standard I/O file descriptors are redirected to /dev/null by default.
-if (hasattr(os, "devnull")):
-    REDIRECT_TO = os.devnull
-else:
-    REDIRECT_TO = "/dev/null"
+import traceback
 
 def createDaemon(function, logfile):
     """
@@ -66,36 +31,6 @@ def createDaemon(function, logfile):
         # leader of the new process group, we call os.setsid().  The process is
         # also guaranteed not to have a controlling terminal.
         os.setsid()
-
-        # Is ignoring SIGHUP necessary?
-        #
-        # It's often suggested that the SIGHUP signal should be ignored before
-        # the second fork to avoid premature termination of the process.  The
-        # reason is that when the first child terminates, all processes, e.g.
-        # the second child, in the orphaned group will be sent a SIGHUP.
-        #
-        # "However, as part of the session management system, there are exactly
-        # two cases where SIGHUP is sent on the death of a process:
-        #
-        #    1) When the process that dies is the session leader of a session that
-        #        is attached to a terminal device, SIGHUP is sent to all processes
-        #        in the foreground process group of that terminal device.
-        #    2) When the death of a process causes a process group to become
-        #        orphaned, and one or more processes in the orphaned group are
-        #        stopped, then SIGHUP and SIGCONT are sent to all members of the
-        #        orphaned group." [2]
-        #
-        # The first case can be ignored since the child is guaranteed not to have
-        # a controlling terminal.  The second case isn't so easy to dismiss.
-        # The process group is orphaned when the first child terminates and
-        # POSIX.1 requires that every STOPPED process in an orphaned process
-        # group be sent a SIGHUP signal followed by a SIGCONT signal.  Since the
-        # second child is not STOPPED though, we can safely forego ignoring the
-        # SIGHUP signal.  In any case, there are no ill-effects if it is ignored.
-        #
-        # import signal              # Set handlers for asynchronous events.
-        # signal.signal(signal.SIGHUP, signal.SIG_IGN)
-
         try:
             # Fork a second child and exit immediately to prevent zombies.  This
             # causes the second child process to be orphaned, making the init
@@ -109,77 +44,23 @@ def createDaemon(function, logfile):
         except OSError as e:
             raise Exception("%s [%d]" % (e.strerror, e.errno))
 
-        if (pid == 0):  # The second child.
-            # We probably don't want the file mode creation mask inherited from
-            # the parent, so we give the child complete control over permissions.
-            if UMASK is not None:
-                os.umask(UMASK)
-        else:
+        if (pid != 0):
             # Parent (the first child) of the second child.
+            # exit() or _exit()?
+            # _exit is like exit(), but it doesn't call any functions registered
+            # with atexit (and on_exit) or any registered signal handlers.  It also
+            # closes any open file descriptors.  Using exit() may cause all stdio
+            # streams to be flushed twice and any temporary files may be unexpectedly
+            # removed.  It's therefore recommended that child branches of a fork()
+            # and the parent branch(es) of a daemon use _exit().
             os._exit(0)
     else:
-        # exit() or _exit()?
-        # _exit is like exit(), but it doesn't call any functions registered
-        # with atexit (and on_exit) or any registered signal handlers.  It also
-        # closes any open file descriptors.  Using exit() may cause all stdio
-        # streams to be flushed twice and any temporary files may be unexpectedly
-        # removed.  It's therefore recommended that child branches of a fork()
-        # and the parent branch(es) of a daemon use _exit().
         os.waitpid(pid, 0)
         return
 
-    # Close all open file descriptors.  This prevents the child from keeping
-    # open any file descriptors inherited from the parent.  There is a variety
-    # of methods to accomplish this task.  Three are listed below.
-    #
-    # Try the system configuration variable, SC_OPEN_MAX, to obtain the maximum
-    # number of open file descriptors to close.  If it doesn't exist, use
-    # the default value (configurable).
-    #
-    # try:
-    #     maxfd = os.sysconf("SC_OPEN_MAX")
-    # except (AttributeError, ValueError):
-    #     maxfd = MAXFD
-    #
-    # OR
-    #
-    # if (os.sysconf_names.has_key("SC_OPEN_MAX")):
-    #     maxfd = os.sysconf("SC_OPEN_MAX")
-    # else:
-    #     maxfd = MAXFD
-    #
-    # OR
-    #
-    # Use the getrlimit method to retrieve the maximum file descriptor number
-    # that can be opened by this process.  If there is no limit on the
-    # resource, use the default value.
-    #
-    import resource             # Resource usage information.
-    maxfd = resource.getrlimit(resource.RLIMIT_NOFILE)[1]
-    if (maxfd == resource.RLIM_INFINITY):
-        maxfd = MAXFD
-  
-    # Iterate through and close all file descriptors.
-#    for fd in range(0, maxfd):
-#        try:
-#            os.close(fd)
-#        except OSError:        # ERROR, fd wasn't open to begin with (ignored)
-#            pass
+    # The second child.
 
-    # Redirect the standard I/O file descriptors to the specified file.  Since
-    # the daemon has no controlling terminal, most daemons redirect stdin,
-    # stdout, and stderr to /dev/null.  This is done to prevent side-effects
-    # from reads and writes to the standard I/O file descriptors.
-
-    # This call to open is guaranteed to return the lowest file descriptor,
-    # which will be 0 (stdin), since it was closed above.
-#    os.open(REDIRECT_TO, os.O_RDWR)    # standard input (0)
-
-    # Duplicate standard input to standard output and standard error.
-#    os.dup2(0, 1)                      # standard output (1)
-#    os.dup2(0, 2)                      # standard error (2)
-
-    # Replace those fds with our own
+    # Replace standard fds with our own
     si = open('/dev/null', 'r')
     os.dup2(si.fileno(), sys.stdin.fileno())
 
@@ -192,6 +73,10 @@ def createDaemon(function, logfile):
         sys.stdout = open(logfile, 'a+')
         sys.stderr = sys.stdout
 
-    function()
-
-    os._exit(0)
+    try:
+        function()
+    except Exception as e:
+        traceback.print_exc()
+    finally:
+        bb.event.print_ui_queue()
+        os._exit(0)

bitbake/lib/bb/event.py

@@ -149,23 +149,30 @@ def print_ui_queue():
 
         # First check to see if we have any proper messages
         msgprint = False
+        msgerrs = False
+
+        # Should we print to stderr?
+        for event in ui_queue[:]:
+            if isinstance(event, logging.LogRecord) and event.levelno >= logging.WARNING:
+                msgerrs = True
+                break
+
+        if msgerrs:
+            logger.addHandler(stderr)
+        else:
+            logger.addHandler(stdout)
+
         for event in ui_queue[:]:
             if isinstance(event, logging.LogRecord):
                 if event.levelno > logging.DEBUG:
-                    if event.levelno >= logging.WARNING:
-                        logger.addHandler(stderr)
-                    else:
-                        logger.addHandler(stdout)
                     logger.handle(event)
                     msgprint = True
-        if msgprint:
-            return
 
         # Nope, so just print all of the messages we have (including debug messages)
-        logger.addHandler(stdout)
-        for event in ui_queue[:]:
-            if isinstance(event, logging.LogRecord):
-                logger.handle(event)
+        if not msgprint:
+            for event in ui_queue[:]:
+                if isinstance(event, logging.LogRecord):
+                    logger.handle(event)
 
 def fire_ui_handlers(event, d):
     global _thread_lock

bitbake/lib/bb/fetch2/__init__.py

@@ -411,8 +411,6 @@ def encodeurl(decoded):
 
     type, host, path, user, pswd, p = decoded
 
-    if not path:
-        raise MissingParameterError('path', "encoded from the data %s" % str(decoded))
     if not type:
         raise MissingParameterError('type', "encoded from the data %s" % str(decoded))
     url = '%s://' % type
@@ -423,10 +421,11 @@ def encodeurl(decoded):
         url += "@"
     if host and type != "file":
         url += "%s" % host
-    # Standardise path to ensure comparisons work
-    while '//' in path:
-        path = path.replace("//", "/")
-    url += "%s" % urllib.parse.quote(path)
+    if path:
+        # Standardise path to ensure comparisons work
+        while '//' in path:
+            path = path.replace("//", "/")
+        url += "%s" % urllib.parse.quote(path)
     if p:
         for parm in p:
             url += ";%s=%s" % (parm, p[parm])
@@ -1726,9 +1725,8 @@ class Fetch(object):
             ret = try_mirrors(self, self.d, ud, mirrors, True)
             if not ret:
                 # Next try checking from the original uri, u
-                try:
-                    ret = m.checkstatus(self, ud, self.d)
-                except:
+                ret = m.checkstatus(self, ud, self.d)
+                if not ret:
                     # Finally, try checking uri, u, from MIRRORS
                     mirrors = mirror_from_string(self.d.getVar('MIRRORS'))
                     ret = try_mirrors(self, self.d, ud, mirrors, True)

bitbake/lib/bb/fetch2/wget.py

@@ -283,11 +283,6 @@ class Wget(FetchMethod):
             """
             http_error_403 = http_error_405
 
-            """
-            Some servers (e.g. FusionForge) returns 406 Not Acceptable when they
-            actually mean 405 Method Not Allowed.
-            """
-            http_error_406 = http_error_405
 
         class FixedHTTPRedirectHandler(urllib.request.HTTPRedirectHandler):
             """
@@ -316,7 +311,9 @@ class Wget(FetchMethod):
             uri = ud.url.split(";")[0]
             r = urllib.request.Request(uri)
             r.get_method = lambda: "HEAD"
-
+            # Some servers (FusionForge, as used on Alioth) require that the
+            # optional Accept header is set.
+            r.add_header("Accept", "*/*")
             def add_basic_auth(login_str, request):
                 '''Adds Basic auth to http request, pass in login:password as string'''
                 import base64

bitbake/lib/bb/main.py

@@ -259,8 +259,10 @@ class BitBakeConfigParameters(cookerdata.ConfigParameters):
                           help="The name/address for the bitbake xmlrpc server to bind to.")
 
         parser.add_option("-T", "--idle-timeout", type=float, dest="server_timeout",
-                          default=float(os.environ.get("BB_SERVER_TIMEOUT", 0)) or None,
-                          help="Set timeout to unload bitbake server due to inactivity")
+                          default=os.getenv("BB_SERVER_TIMEOUT"),
+                          help="Set timeout to unload bitbake server due to inactivity, "
+                                "set to -1 means no unload, "
+                                "default: Environment variable BB_SERVER_TIMEOUT.")
 
         parser.add_option("", "--no-setscene", action="store_true",
                           dest="nosetscene", default=False,
@@ -277,7 +279,7 @@ class BitBakeConfigParameters(cookerdata.ConfigParameters):
 
         parser.add_option("-m", "--kill-server", action="store_true",
                           dest="kill_server", default=False,
-                          help="Terminate the bitbake server.")
+                          help="Terminate any running bitbake server.")
 
         parser.add_option("", "--observe-only", action="store_true",
                           dest="observe_only", default=False,
@@ -370,8 +372,10 @@ def bitbake_main(configParams, configuration):
     server_connection, ui_module = setup_bitbake(configParams, configuration)
     # No server connection
     if server_connection is None:
-        if configParams.status_only or configParams.kill_server:
+        if configParams.status_only:
             return 1
+        if configParams.kill_server:
+            return 0
 
     if not configParams.server_only:
         if configParams.status_only:
@@ -392,10 +396,10 @@ def bitbake_main(configParams, configuration):
 
     return 1
 
-def setup_bitbake(configParams, configuration, extrafeatures=None, setup_logging=True):
+def setup_bitbake(configParams, configuration, extrafeatures=None):
     # Ensure logging messages get sent to the UI as events
     handler = bb.event.LogHandler()
-    if setup_logging and not configParams.status_only:
+    if not configParams.status_only:
         # In status only mode there are no logs and no UI
         logger.addHandler(handler)
 

bitbake/lib/bb/parse/__init__.py

@@ -84,6 +84,10 @@ def update_cache(f):
         logger.debug(1, "Updating mtime cache for %s" % f)
         update_mtime(f)
 
+def clear_cache():
+    global __mtime_cache
+    __mtime_cache = {}
+
 def mark_dependency(d, f):
     if f.startswith('./'):
         f = "%s/%s" % (os.getcwd(), f[2:])

bitbake/lib/bb/runqueue.py

@@ -2372,7 +2372,7 @@ class RunQueueExecuteScenequeue(RunQueueExecute):
         self.rq.scenequeue_covered = self.scenequeue_covered
         self.rq.scenequeue_notcovered = self.scenequeue_notcovered
 
-        logger.debug(1, 'We can skip tasks %s', sorted(self.rq.scenequeue_covered))
+        logger.debug(1, 'We can skip tasks %s', "\n".join(sorted(self.rq.scenequeue_covered)))
 
         self.rq.state = runQueueRunInit
 

bitbake/lib/bb/server/process.py

@@ -100,7 +100,8 @@ class ProcessServer(multiprocessing.Process):
             else:
                 self.bitbake_lock.write("%s\n" % (os.getpid()))
             self.bitbake_lock.flush()
-        except:
+        except Exception as e:
+            print("Error writing to lock file: %s" % str(e))
             pass
 
         if self.cooker.configuration.profile:
@@ -132,20 +133,28 @@ class ProcessServer(multiprocessing.Process):
         fds = [self.sock]
         if self.xmlrpc:
             fds.append(self.xmlrpc)
-        while not self.quit:
-            if self.command_channel in ready:
-                command = self.command_channel.get()
-                if command[0] == "terminateServer":
-                    self.quit = True
-                    continue
-                try:
-                    print("Running command %s" % command)
-                    self.command_channel_reply.send(self.cooker.command.runCommand(command))
-                except Exception as e:
-                   logger.exception('Exception in server main event loop running command %s (%s)' % (command, str(e)))
+        print("Entering server connection loop")
 
-            if self.xmlrpc in ready:
-                self.xmlrpc.handle_requests()
+        def disconnect_client(self, fds):
+            if not self.haveui:
+                return
+            print("Disconnecting Client")
+            fds.remove(self.controllersock)
+            fds.remove(self.command_channel)
+            bb.event.unregister_UIHhandler(self.event_handle, True)
+            self.command_channel_reply.writer.close()
+            self.event_writer.writer.close()
+            del self.event_writer
+            self.controllersock.close()
+            self.controllersock = False
+            self.haveui = False
+            self.lastui = time.time()
+            self.cooker.clientComplete()
+            if self.timeout is None:
+                print("No timeout, exiting.")
+                self.quit = True
+
+        while not self.quit:
             if self.sock in ready:
                 self.controllersock, address = self.sock.accept()
                 if self.haveui:
@@ -175,25 +184,34 @@ class ProcessServer(multiprocessing.Process):
 
                     self.haveui = True
 
-                except EOFError:
-                    print("Disconnecting Client")
-                    fds.remove(self.controllersock)
-                    fds.remove(self.command_channel)
-                    bb.event.unregister_UIHhandler(self.event_handle, True)
-                    self.command_channel_reply.writer.close()
-                    self.event_writer.writer.close()
-                    del self.event_writer
-                    self.controllersock.close()
-                    self.haveui = False
-                    self.lastui = time.time()
-                    self.cooker.clientComplete()
-                    if self.timeout is None:
-                        print("No timeout, exiting.")
-                        self.quit = True
-            if not self.haveui and self.lastui and self.timeout and (self.lastui + self.timeout) < time.time():
+                except (EOFError, OSError):
+                    disconnect_client(self, fds)
+
+            if not self.timeout == -1.0 and not self.haveui and self.lastui and self.timeout and \
+                    (self.lastui + self.timeout) < time.time():
                 print("Server timeout, exiting.")
                 self.quit = True
 
+            if self.command_channel in ready:
+                try:
+                    command = self.command_channel.get()
+                except EOFError:
+                    # Client connection shutting down
+                    ready = []
+                    disconnect_client(self, fds)
+                    continue
+                if command[0] == "terminateServer":
+                    self.quit = True
+                    continue
+                try:
+                    print("Running command %s" % command)
+                    self.command_channel_reply.send(self.cooker.command.runCommand(command))
+                except Exception as e:
+                   logger.exception('Exception in server main event loop running command %s (%s)' % (command, str(e)))
+
+            if self.xmlrpc in ready:
+                self.xmlrpc.handle_requests()
+
             ready = self.idle_commands(.1, fds)
 
         print("Exiting")
@@ -294,7 +312,7 @@ class ProcessServer(multiprocessing.Process):
                 # Ignore EINTR
                 return []
         else:
-            return []
+            return select.select(fds,[],[],0)[0]
 
 
 class ServerCommunicator():
@@ -303,19 +321,10 @@ class ServerCommunicator():
         self.recv = recv
 
     def runCommand(self, command):
-
         self.connection.send(command)
-        while True:
-            # don't let the user ctrl-c while we're waiting for a response
-            try:
-                for idx in range(0,4): # 0, 1, 2, 3
-                    if self.recv.poll(1):
-                        return self.recv.get()
-                    else:
-                        bb.note("Timeout while attempting to communicate with bitbake server, retrying...")
-                raise ProcessTimeout("Gave up; Too many tries: timeout while attempting to communicate with bitbake server")
-            except KeyboardInterrupt:
-                pass
+        if not self.recv.poll(5):
+            raise ProcessTimeout("Timeout while waiting for a reply from the bitbake server")
+        return self.recv.get()
 
     def updateFeatureSet(self, featureset):
         _, error = self.runCommand(["setFeatures", featureset])
@@ -336,12 +345,16 @@ class ServerCommunicator():
         return
 
 class BitBakeProcessServerConnection(object):
-    def __init__(self, ui_channel, recv, eq):
+    def __init__(self, ui_channel, recv, eq, sock):
         self.connection = ServerCommunicator(ui_channel, recv)
         self.events = eq
+        # Save sock so it doesn't get gc'd for the life of our connection
+        self.socket_connection = sock
 
     def terminate(self):
         self.socket_connection.close()
+        self.connection.connection.close()
+        self.connection.recv.close()
         return
 
 class BitBakeServer(object):
@@ -351,6 +364,7 @@ class BitBakeServer(object):
         self.featureset = featureset
         self.sockname = sockname
         self.bitbake_lock = lock
+        self.readypipe, self.readypipein = os.pipe()
 
         # Create server control socket
         if os.path.exists(sockname):
@@ -359,6 +373,8 @@ class BitBakeServer(object):
         self.sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
         # AF_UNIX has path length issues so chdir here to workaround
         cwd = os.getcwd()
+        logfile = os.path.join(cwd, "bitbake-cookerdaemon.log")
+
         try:
             os.chdir(os.path.dirname(sockname))
             self.sock.bind(os.path.basename(sockname))
@@ -367,11 +383,25 @@ class BitBakeServer(object):
         self.sock.listen(1)
 
         os.set_inheritable(self.sock.fileno(), True)
-        bb.daemonize.createDaemon(self._startServer, "bitbake-cookerdaemon.log")
+        bb.daemonize.createDaemon(self._startServer, logfile)
         self.sock.close()
         self.bitbake_lock.close()
 
+        ready = ConnectionReader(self.readypipe)
+        r = ready.wait(30)
+        if not r:
+            ready.close()
+            bb.error("Unable to start bitbake server")
+            if os.path.exists(logfile):
+                with open(logfile, "r") as f:
+                    logs=f.readlines()
+                bb.error("Last 10 lines of server log %s:\n%s" % (logfile, "".join(logs[-10:])))
+            raise SystemExit(1)
+        ready.close()
+        os.close(self.readypipein)
+
     def _startServer(self):
+        print("Starting bitbake server pid %d" % os.getpid())
         server = ProcessServer(self.bitbake_lock, self.sock, self.sockname)
         self.configuration.setServerRegIdleCallback(server.register_idle_function)
 
@@ -381,7 +411,7 @@ class BitBakeServer(object):
             if value:
                 setattr(self.configuration, "%s_server" % param, value)
 
-        self.cooker = bb.cooker.BBCooker(self.configuration, self.featureset)
+        self.cooker = bb.cooker.BBCooker(self.configuration, self.featureset, self.readypipein)
         server.cooker = self.cooker
         server.server_timeout = self.configuration.server_timeout
         server.xmlrpcinterface = self.configuration.xmlrpcinterface
@@ -400,7 +430,11 @@ def connectProcessServer(sockname, featureset):
     finally:
         os.chdir(cwd)
 
+    readfd = writefd = readfd1 = writefd1 = readfd2 = writefd2 = None
+    eq = command_chan_recv = command_chan = None
+
     try:
+
         # Send an fd for the remote to write events to
         readfd, writefd = os.pipe()
         eq = BBUIEventQueue(readfd)
@@ -413,13 +447,24 @@ def connectProcessServer(sockname, featureset):
 
         sendfds(sock, [writefd, readfd1, writefd2])
 
-        server_connection = BitBakeProcessServerConnection(command_chan, command_chan_recv, eq)
+        server_connection = BitBakeProcessServerConnection(command_chan, command_chan_recv, eq, sock)
+
+        # Close the ends of the pipes we won't use
+        for i in [writefd, readfd1, writefd2]:
+            os.close(i)
 
         server_connection.connection.updateFeatureSet(featureset)
 
-        # Save sock so it doesn't get gc'd for the life of our connection
-        server_connection.socket_connection = sock
-    except:
+    except (Exception, SystemExit) as e:
+        if command_chan_recv:
+            command_chan_recv.close()
+        if command_chan:
+            command_chan.close()
+        for i in [writefd, readfd1, writefd2]:
+            try:
+                os.close(i)
+            except OSError:
+                pass
         sock.close()
         raise
 
@@ -499,9 +544,14 @@ class BBUIEventQueue:
     def startCallbackHandler(self):
         bb.utils.set_process_name("UIEventQueue")
         while True:
-            self.reader.wait()
-            event = self.reader.get()
-            self.queue_event(event)
+            try:
+                self.reader.wait()
+                event = self.reader.get()
+                self.queue_event(event)
+            except EOFError:
+                # Easiest way to exit is to close the file descriptor to cause an exit
+                break
+        self.reader.close()
 
 class ConnectionReader(object):
 
@@ -523,6 +573,10 @@ class ConnectionReader(object):
     def fileno(self):
         return self.reader.fileno()
 
+    def close(self):
+        return self.reader.close()
+
+
 class ConnectionWriter(object):
 
     def __init__(self, fd):
@@ -536,3 +590,8 @@ class ConnectionWriter(object):
         with self.wlock:
             self.writer.send_bytes(obj)
 
+    def fileno(self):
+        return self.writer.fileno()
+
+    def close(self):
+        return self.writer.close()

bitbake/lib/bb/siggen.py

@@ -94,6 +94,9 @@ class SignatureGeneratorBasic(SignatureGenerator):
         else:
             self.checksum_cache = None
 
+    def reset(self, data):
+        self.__init__(data)
+
     def init_rundepcheck(self, data):
         self.taskwhitelist = data.getVar("BB_HASHTASK_WHITELIST") or None
         if self.taskwhitelist:

bitbake/lib/bb/tests/fetch.py

@@ -1440,7 +1440,7 @@ class GitShallowTest(FetcherTest):
         def test_bitbake(self):
             self.git('remote add --mirror=fetch origin git://github.com/openembedded/bitbake', cwd=self.srcdir)
             self.git('config core.bare true', cwd=self.srcdir)
-            self.git('fetch --tags', cwd=self.srcdir)
+            self.git('fetch', cwd=self.srcdir)
 
             self.d.setVar('BB_GIT_SHALLOW_DEPTH', '0')
             # Note that the 1.10.0 tag is annotated, so this also tests

bitbake/lib/bb/tinfoil.py

@@ -383,8 +383,7 @@ class Tinfoil:
 
         self.server_connection, ui_module = setup_bitbake(config_params,
                             cookerconfig,
-                            extrafeatures,
-                            setup_logging=False)
+                            extrafeatures)
 
         self.ui_module = ui_module
 

bitbake/lib/bb/ui/taskexp.py

@@ -63,7 +63,9 @@ class PackageReverseDepView(Gtk.TreeView):
         self.current = None
         self.filter_model = model.filter_new()
         self.filter_model.set_visible_func(self._filter)
-        self.set_model(self.filter_model)
+        self.sort_model = self.filter_model.sort_new_with_model()
+        self.sort_model.set_sort_column_id(COL_DEP_PARENT, Gtk.SortType.ASCENDING)
+        self.set_model(self.sort_model)
         self.append_column(Gtk.TreeViewColumn(label, Gtk.CellRendererText(), text=COL_DEP_PARENT))
 
     def _filter(self, model, iter, data):

bitbake/lib/toaster/bldcontrol/bbcontroller.py

@@ -37,8 +37,8 @@ class BitbakeController(object):
     """
 
     def __init__(self, be):
-        import bb.server.xmlrpc
-        self.connection = bb.server.xmlrpc._create_server(be.bbaddress,
+        import bb.server.xmlrpcclient
+        self.connection = bb.server.xmlrpcclient._create_server(be.bbaddress,
                                                           int(be.bbport))[0]
 
     def _runCommand(self, command):

bitbake/lib/toaster/bldcontrol/localhostbecontroller.py

@@ -24,6 +24,7 @@ import os
 import sys
 import re
 import shutil
+import time
 from django.db import transaction
 from django.db.models import Q
 from bldcontrol.models import BuildEnvironment, BRLayer, BRVariable, BRTarget, BRBitbake
@@ -331,12 +332,22 @@ class LocalhostBEController(BuildEnvironmentController):
         bitbake = os.path.join(self.pokydirname, 'bitbake', 'bin', 'bitbake')
         toasterlayers = os.path.join(builddir,"conf/toaster-bblayers.conf")
         self._shellcmd('bash -c \"source %s %s; BITBAKE_UI="knotty" %s --read %s --read %s '
-                       '--server-only -t xmlrpc -B 0.0.0.0:0\"' % (oe_init,
+                       '--server-only -B 0.0.0.0:0\"' % (oe_init,
                        builddir, bitbake, confpath, toasterlayers), self.be.sourcedir)
 
         # read port number from bitbake.lock
         self.be.bbport = ""
         bblock = os.path.join(builddir, 'bitbake.lock')
+        # allow 10 seconds for bb lock file to appear but also be populated
+        for lock_check in range(10):
+            if not os.path.exists(bblock):
+                logger.debug("localhostbecontroller: waiting for bblock file to appear")
+                time.sleep(1)
+                continue
+            if 10 < os.stat(bblock).st_size:
+                break
+            logger.debug("localhostbecontroller: waiting for bblock content to appear")
+            time.sleep(1)
         with open(bblock) as fplock:
             for line in fplock:
                 if ":" in line:
@@ -365,10 +376,10 @@ class LocalhostBEController(BuildEnvironmentController):
         log = os.path.join(builddir, 'toaster_ui.log')
         local_bitbake = os.path.join(os.path.dirname(os.getenv('BBBASEDIR')),
                                      'bitbake')
-        self._shellcmd(['bash -c \"(TOASTER_BRBE="%s" BBSERVER="0.0.0.0:-1" '
+        self._shellcmd(['bash -c \"(TOASTER_BRBE="%s" BBSERVER="0.0.0.0:%s" '
                         '%s %s -u toasterui --token="" >>%s 2>&1;'
-                        'BITBAKE_UI="knotty" BBSERVER=0.0.0.0:-1 %s -m)&\"' \
-                        % (brbe, local_bitbake, bbtargets, log, bitbake)],
+                        'BITBAKE_UI="knotty" BBSERVER=0.0.0.0:%s %s -m)&\"' \
+                        % (brbe, self.be.bbport, local_bitbake, bbtargets, log, self.be.bbport, bitbake)],
                         builddir, nowait=True)
 
         logger.debug('localhostbecontroller: Build launched, exiting. '

bitbake/lib/toaster/bldcontrol/management/commands/checksettings.py

@@ -75,7 +75,10 @@ class Command(BaseCommand):
                         call_command("loaddata", "settings")
                         template_conf = os.environ.get("TEMPLATECONF", "")
 
-                        if "poky" in template_conf:
+                        if ToasterSetting.objects.filter(name='CUSTOM_XML_ONLY').count() > 0:
+                            # only use the custom settings
+                            pass
+                        elif "poky" in template_conf:
                             print("Loading poky configuration")
                             call_command("loaddata", "poky")
                         else:

bitbake/lib/toaster/orm/fixtures/custom_toaster_append.sh_sample

@@ -0,0 +1,49 @@
+#!/bin/bash
+
+# Copyright (C) 2017 Intel Corp.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License version 2 as
+# published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+# See the GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+
+# This is sample software. Rename it to 'custom_toaster_append.sh' and
+# enable the respective custom sections.
+
+verbose=0
+if [ $verbose -ne 0 ] ; then
+    echo "custom_toaster_append.sh:$*"
+fi
+
+if [ "toaster_prepend" = "$1" ] ; then
+    echo "Add custom actions here when Toaster script is started"
+fi
+
+if [ "web_start_postpend" = "$1" ] ; then
+    echo "Add custom actions here after Toaster web service is started"
+fi
+
+if [ "web_stop_postpend" = "$1" ] ; then
+    echo "Add custom actions here after Toaster web service is stopped"
+fi
+
+if [ "noweb_start_postpend" = "$1" ] ; then
+    echo "Add custom actions here after Toaster (no web) service is started"
+fi
+
+if [ "noweb_stop_postpend" = "$1" ] ; then
+    echo "Add custom actions here after Toaster (no web) service is stopped"
+fi
+
+if [ "toaster_postpend" = "$1" ] ; then
+    echo "Add custom actions here after Toaster script is done"
+fi
+

bitbake/lib/toaster/orm/management/commands/lsupdates.py

@@ -81,6 +81,8 @@ class Command(BaseCommand):
         os.system('setterm -cursor off')
 
         self.apiurl = DEFAULT_LAYERINDEX_SERVER
+        if ToasterSetting.objects.filter(name='CUSTOM_LAYERINDEX_SERVER').count() == 1:
+            self.apiurl = ToasterSetting.objects.get(name = 'CUSTOM_LAYERINDEX_SERVER').value
 
         assert self.apiurl is not None
         try:
@@ -92,7 +94,9 @@ class Command(BaseCommand):
 
         proxy_settings = os.environ.get("http_proxy", None)
 
-        def _get_json_response(apiurl=DEFAULT_LAYERINDEX_SERVER):
+        def _get_json_response(apiurl=None):
+            if None == apiurl:
+                apiurl=self.apiurl
             http_progress = Spinner()
             http_progress.start()
 

bitbake/lib/toaster/orm/models.py

@@ -451,7 +451,7 @@ class Build(models.Model):
     recipes_to_parse = models.IntegerField(default=1)
 
     # number of recipes parsed so far for this build
-    recipes_parsed = models.IntegerField(default=0)
+    recipes_parsed = models.IntegerField(default=1)
 
     # number of repos to clone for this build
     repos_to_clone = models.IntegerField(default=1)
@@ -1658,7 +1658,7 @@ class CustomImageRecipe(Recipe):
 
     def get_base_recipe_file(self):
         """Get the base recipe file path if it exists on the file system"""
-        path_schema_one = "%s/%s" % (self.base_recipe.layer_version.dirpath,
+        path_schema_one = "%s/%s" % (self.base_recipe.layer_version.local_path,
                                      self.base_recipe.file_path)
 
         path_schema_two = self.base_recipe.file_path

bitbake/lib/toaster/toastergui/api.py

@@ -18,6 +18,7 @@
 
 # Please run flake8 on this file before sending patches
 
+import os
 import re
 import logging
 import json
@@ -28,7 +29,7 @@ from orm.models import LayerVersionDependency, LayerSource, ProjectLayer
 from orm.models import Recipe, CustomImageRecipe, CustomImagePackage
 from orm.models import Layer, Target, Package, Package_Dependency
 from orm.models import ProjectVariable
-from bldcontrol.models import BuildRequest
+from bldcontrol.models import BuildRequest, BuildEnvironment
 from bldcontrol import bbcontroller
 
 from django.http import HttpResponse, JsonResponse
@@ -509,6 +510,27 @@ class XhrCustomRecipe(View):
                                    (tpackage.package.name, e))
                     pass
 
+        # pre-create layer directory structure, so that other builds
+        # are not blocked by this new recipe dependecy
+        # NOTE: this is parallel code to 'localhostbecontroller.py'
+        be = BuildEnvironment.objects.all()[0]
+        layerpath = os.path.join(be.builddir,
+                                 CustomImageRecipe.LAYER_NAME)
+        for name in ("conf", "recipes"):
+            path = os.path.join(layerpath, name)
+            if not os.path.isdir(path):
+                os.makedirs(path)
+        # pre-create layer.conf
+        config = os.path.join(layerpath, "conf", "layer.conf")
+        if not os.path.isfile(config):
+            with open(config, "w") as conf:
+                conf.write('BBPATH .= ":${LAYERDIR}"\nBBFILES += "${LAYERDIR}/recipes/*.bb"\n')
+        # pre-create new image's recipe file
+        recipe_path = os.path.join(layerpath, "recipes", "%s.bb" %
+                                   recipe.name)
+        with open(recipe_path, "w") as recipef:
+            recipef.write(recipe.generate_recipe_file_contents())
+
         return JsonResponse(
             {"error": "ok",
              "packages": recipe.get_all_packages().count(),
@@ -752,7 +774,6 @@ class XhrCustomRecipePackages(View):
                 return error_response("Package %s not found in excludes"
                                       " but was in included list" %
                                       package.name)
-
         else:
             recipe.appends_set.add(package)
             # Make sure that package is not in the excludes set
@@ -760,26 +781,27 @@ class XhrCustomRecipePackages(View):
                 recipe.excludes_set.remove(package)
             except:
                 pass
-            # Add the dependencies we think will be added to the recipe
-            # as a result of appending this package.
-            # TODO this should recurse down the entire deps tree
-            for dep in package.package_dependencies_source.all_depends():
-                try:
-                    cust_package = CustomImagePackage.objects.get(
-                        name=dep.depends_on.name)
 
-                    recipe.includes_set.add(cust_package)
-                    try:
-                        # When adding the pre-requisite package, make
-                        # sure it's not in the excluded list from a
-                        # prior removal.
-                        recipe.excludes_set.remove(cust_package)
-                    except package.DoesNotExist:
-                        # Don't care if the package had never been excluded
-                        pass
-                except:
-                    logger.warning("Could not add package's suggested"
-                                   "dependencies to the list")
+        # Add the dependencies we think will be added to the recipe
+        # as a result of appending this package.
+        # TODO this should recurse down the entire deps tree
+        for dep in package.package_dependencies_source.all_depends():
+            try:
+                cust_package = CustomImagePackage.objects.get(
+                    name=dep.depends_on.name)
+
+                recipe.includes_set.add(cust_package)
+                try:
+                    # When adding the pre-requisite package, make
+                    # sure it's not in the excluded list from a
+                    # prior removal.
+                    recipe.excludes_set.remove(cust_package)
+                except package.DoesNotExist:
+                    # Don't care if the package had never been excluded
+                    pass
+            except:
+                logger.warning("Could not add package's suggested"
+                               "dependencies to the list")
         return JsonResponse({"error": "ok"})
 
     def delete(self, request, *args, **kwargs):
@@ -797,22 +819,24 @@ class XhrCustomRecipePackages(View):
                 recipe.excludes_set.add(package)
             else:
                 recipe.appends_set.remove(package)
-                all_current_packages = recipe.get_all_packages()
 
-                reverse_deps_dictlist = self._get_all_dependents(
-                    package.pk,
-                    all_current_packages)
+            # remove dependencies as well
+            all_current_packages = recipe.get_all_packages()
 
-                ids = [entry['pk'] for entry in reverse_deps_dictlist]
-                reverse_deps = CustomImagePackage.objects.filter(id__in=ids)
-                for r in reverse_deps:
-                    try:
-                        if r.id in included_packages:
-                            recipe.excludes_set.add(r)
-                        else:
-                            recipe.appends_set.remove(r)
-                    except:
-                        pass
+            reverse_deps_dictlist = self._get_all_dependents(
+                package.pk,
+                all_current_packages)
+
+            ids = [entry['pk'] for entry in reverse_deps_dictlist]
+            reverse_deps = CustomImagePackage.objects.filter(id__in=ids)
+            for r in reverse_deps:
+                try:
+                    if r.id in included_packages:
+                        recipe.excludes_set.add(r)
+                    else:
+                        recipe.appends_set.remove(r)
+                except:
+                    pass
 
             return JsonResponse({"error": "ok"})
         except CustomImageRecipe.DoesNotExist:

bitbake/lib/toaster/toastergui/templates/health.html

@@ -0,0 +1,6 @@
+<!DOCTYPE html>
+<html lang="en">
+  <head><title>Toaster Health</title></head>
+  <body>Ok</body>
+</html>
+

bitbake/lib/toaster/toastergui/urls.py

@@ -244,6 +244,11 @@ urlpatterns = [
         url(r'^mostrecentbuilds$', widgets.MostRecentBuildsView.as_view(),
             name='most_recent_builds'),
 
-          # default redirection
+        # JSON data for aggregators
+        url(r'^api/builds$', views.json_builds, name='json_builds'),
+        url(r'^api/building$', views.json_building, name='json_building'),
+        url(r'^api/build/(?P<build_id>\d+)$', views.json_build, name='json_build'),
+
+        # default redirection
         url(r'^$', RedirectView.as_view(url='landing', permanent=True)),
 ]

bitbake/lib/toaster/toastergui/views.py

@@ -35,7 +35,7 @@ from orm.models import BitbakeVersion, CustomImageRecipe
 from django.core.urlresolvers import reverse, resolve
 from django.core.exceptions import MultipleObjectsReturned, ObjectDoesNotExist
 from django.core.paginator import Paginator, EmptyPage, PageNotAnInteger
-from django.http import HttpResponseNotFound
+from django.http import HttpResponseNotFound, JsonResponse
 from django.utils import timezone
 from datetime import timedelta, datetime
 from toastergui.templatetags.projecttags import json as jsonfilter
@@ -1256,6 +1256,89 @@ def managedcontextprocessor(request):
     }
     return ret
 
+# REST-based API calls to return build/building status to external Toaster
+# managers and aggregators via JSON
+
+def _json_build_status(build_id,extend):
+    build_stat = None
+    try:
+        build = Build.objects.get( pk = build_id )
+        build_stat = {}
+        build_stat['id'] = build.id
+        build_stat['name'] = build.build_name
+        build_stat['machine'] = build.machine
+        build_stat['distro'] = build.distro
+        build_stat['start'] = build.started_on
+        # look up target name
+        target= Target.objects.get( build = build )
+        if target:
+            if target.task:
+                build_stat['target'] = '%s:%s' % (target.target,target.task)
+            else:
+                build_stat['target'] = '%s' % (target.target)
+        else:
+            build_stat['target'] = ''
+        # look up project name
+        project = Project.objects.get( build = build )
+        if project:
+            build_stat['project'] = project.name
+        else:
+            build_stat['project'] = ''
+        if Build.IN_PROGRESS == build.outcome:
+            now = timezone.now()
+            timediff = now - build.started_on
+            build_stat['seconds']='%.3f' % timediff.total_seconds()
+            build_stat['clone']='%d:%d' % (build.repos_cloned,build.repos_to_clone)
+            build_stat['parse']='%d:%d' % (build.recipes_parsed,build.recipes_to_parse)
+            tf = Task.objects.filter(build = build)
+            tfc = tf.count()
+            if tfc > 0:
+                tfd = tf.exclude(order__isnull=True).count()
+            else:
+                tfd = 0
+            build_stat['task']='%d:%d' % (tfd,tfc)
+        else:
+            build_stat['outcome'] = build.get_outcome_text()
+            timediff = build.completed_on - build.started_on
+            build_stat['seconds']='%.3f' % timediff.total_seconds()
+            build_stat['stop'] = build.completed_on
+            messages = LogMessage.objects.all().filter(build = build)
+            errors = len(messages.filter(level=LogMessage.ERROR) |
+                 messages.filter(level=LogMessage.EXCEPTION) |
+                 messages.filter(level=LogMessage.CRITICAL))
+            build_stat['errors'] = errors
+            warnings = len(messages.filter(level=LogMessage.WARNING))
+            build_stat['warnings'] = warnings
+        if extend:
+            build_stat['cooker_log'] = build.cooker_log_path
+    except Exception as e:
+        build_state = str(e)
+    return build_stat
+
+def json_builds(request):
+    build_table = []
+    builds = []
+    try:
+        builds = Build.objects.exclude(outcome=Build.IN_PROGRESS).order_by("-started_on")
+        for build in builds:
+            build_table.append(_json_build_status(build.id,False))
+    except Exception as e:
+        build_table = str(e)
+    return JsonResponse({'builds' : build_table, 'count' : len(builds)})
+
+def json_building(request):
+    build_table = []
+    builds = []
+    try:
+        builds = Build.objects.filter(outcome=Build.IN_PROGRESS).order_by("-started_on")
+        for build in builds:
+            build_table.append(_json_build_status(build.id,False))
+    except Exception as e:
+        build_table = str(e)
+    return JsonResponse({'building' : build_table, 'count' : len(builds)})
+
+def json_build(request,build_id):
+    return JsonResponse({'build' : _json_build_status(build_id,True)})
 
 
 import toastermain.settings
@@ -1694,3 +1777,4 @@ if True:
                 return render(request, "unavailable_artifact.html")
         except (ObjectDoesNotExist, IOError):
             return render(request, "unavailable_artifact.html")
+

bitbake/lib/toaster/toastermain/urls.py

@@ -20,9 +20,8 @@
 # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
 
 from django.conf.urls import patterns, include, url
-from django.views.generic import RedirectView
+from django.views.generic import RedirectView, TemplateView
 from django.views.decorators.cache import never_cache
-
 import bldcollector.views
 
 import logging
@@ -46,6 +45,8 @@ urlpatterns = [
     # in the future.
     url(r'^orm/eventfile$', bldcollector.views.eventfile),
 
+    url(r'^health$', TemplateView.as_view(template_name="health.html"), name='Toaster Health'),
+
     # if no application is selected, we have the magic toastergui app here
     url(r'^$', never_cache(RedirectView.as_view(url='/toastergui/', permanent=True))),
 ]

documentation/Makefile

@@ -143,7 +143,7 @@ endif
 ifeq ($(DOC),yocto-project-qs)
 XSLTOPTS = --xinclude
 ALLPREQ = html eclipse tarball
-TARFILES = yocto-project-qs.html qs-style.css figures/yocto-environment.png \
+TARFILES = yocto-project-qs.html qs-style.css \
            figures/yocto-project-transp.png \
            eclipse
 MANUALS = $(DOC)/$(DOC).html $(DOC)/eclipse
@@ -191,7 +191,7 @@ TARFILES = mega-manual.html mega-style.css figures/yocto-environment.png \
 	figures/wip.png
         else
 TARFILES = mega-manual.html mega-style.css figures/yocto-environment.png \
-        figures/building-an-image.png  \
+        figures/building-an-image.png figures/YP-flow-diagram.png \
 	figures/using-a-pre-built-image.png \
 	figures/poky-title.png figures/buildhistory.png \
         figures/buildhistory-web.png \
@@ -256,7 +256,7 @@ endif
 ifeq ($(DOC),ref-manual)
 XSLTOPTS = --xinclude
 ALLPREQ = html eclipse tarball
-TARFILES = ref-manual.html ref-style.css figures/poky-title.png \
+TARFILES = ref-manual.html ref-style.css figures/poky-title.png figures/YP-flow-diagram.png \
 	figures/buildhistory.png figures/buildhistory-web.png eclipse \
         figures/cross-development-toolchains.png figures/layer-input.png \
 	figures/package-feeds.png figures/source-input.png \

documentation/bsp-guide/bsp.xml

@@ -1150,7 +1150,7 @@
                    <listitem><para>Create a <filename>.bbappend</filename>
                        file for the modified recipe.
                        For information on using append files, see the
-                       "<ulink url='&YOCTO_DOCS_DEV_URL;#using-bbappend-files'>Using .bbappend Files</ulink>"
+                       "<ulink url='&YOCTO_DOCS_DEV_URL;#using-bbappend-files'>Using .bbappend Files in Your Layer</ulink>"
                        section in the Yocto Project Development Manual.
                        </para></listitem>
                    <listitem><para>

documentation/dev-manual/dev-manual-common-tasks.xml

@@ -113,7 +113,7 @@
             </para>
 
             <para>
-                Follow these general steps to create your layer:
+                Follow these general steps to create your layer without the aid of a script:
                 <orderedlist>
                     <listitem><para><emphasis>Check Existing Layers:</emphasis>
                         Before creating a new layer, you should be sure someone
@@ -255,197 +255,182 @@
         </section>
 
         <section id='best-practices-to-follow-when-creating-layers'>
-            <title>Best Practices to Follow When Creating Layers</title>
+            <title>Following Best Practices When Creating Layers</title>
 
             <para>
                 To create layers that are easier to maintain and that will
                 not impact builds for other machines, you should consider the
-                information in the following sections.
-            </para>
-
-            <section id='avoid-overlaying-entire-recipes'>
-                <title>Avoid "Overlaying" Entire Recipes</title>
-
-                <para>
-                    Avoid "overlaying" entire recipes from other layers in your
-                    configuration.
-                    In other words, do not copy an entire recipe into your
-                    layer and then modify it.
-                    Rather, use an append file (<filename>.bbappend</filename>)
-                    to override
-                    only those parts of the original recipe you need to modify.
-                </para>
-            </section>
-
-            <section id='avoid-duplicating-include-files'>
-                <title>Avoid Duplicating Include Files</title>
-
-                <para>
-                    Avoid duplicating include files.
-                    Use append files (<filename>.bbappend</filename>)
-                    for each recipe
-                    that uses an include file.
-                    Or, if you are introducing a new recipe that requires
-                    the included file, use the path relative to the original
-                    layer directory to refer to the file.
-                    For example, use
-                    <filename>require recipes-core/</filename><replaceable>package</replaceable><filename>/</filename><replaceable>file</replaceable><filename>.inc</filename>
-                    instead of <filename>require </filename><replaceable>file</replaceable><filename>.inc</filename>.
-                    If you're finding you have to overlay the include file,
-                    it could indicate a deficiency in the include file in
-                    the layer to which it originally belongs.
-                    If this is the case, you should try to address that
-                    deficiency instead of overlaying the include file.
-                    For example, you could address this by getting the
-                    maintainer of the include file to add a variable or
-                    variables to make it easy to override the parts needing
-                    to be overridden.
-                </para>
-            </section>
-
-            <section id='structure-your-layers'>
-                <title>Structure Your Layers</title>
-
-                <para>
-                    Proper use of overrides within append files and placement
-                    of machine-specific files within your layer can ensure that
-                    a build is not using the wrong Metadata and negatively
-                    impacting a build for a different machine.
-                    Following are some examples:
-                    <itemizedlist>
-                        <listitem><para><emphasis>Modifying Variables to Support
-                            a Different Machine:</emphasis>
-                            Suppose you have a layer named
-                            <filename>meta-one</filename> that adds support
-                            for building machine "one".
-                            To do so, you use an append file named
-                            <filename>base-files.bbappend</filename> and
-                            create a dependency on "foo" by altering the
-                            <ulink url='&YOCTO_DOCS_REF_URL;#var-DEPENDS'><filename>DEPENDS</filename></ulink>
-                            variable:
-                            <literallayout class='monospaced'>
+                information in the following list:
+                <itemizedlist>
+                    <listitem><para>
+                        <emphasis>Avoid "Overlaying" Entire Recipes from Other Layers in Your Configuration:</emphasis>
+                        In other words, do not copy an entire recipe into your
+                        layer and then modify it.
+                        Rather, use an append file
+                        (<filename>.bbappend</filename>) to override only those
+                        parts of the original recipe you need to modify.
+                        </para></listitem>
+                    <listitem><para>
+                        <emphasis>Avoid Duplicating Include Files:</emphasis>
+                        Use append files (<filename>.bbappend</filename>)
+                        for each recipe that uses an include file.
+                        Or, if you are introducing a new recipe that requires
+                        the included file, use the path relative to the
+                        original layer directory to refer to the file.
+                        For example, use
+                        <filename>require recipes-core/</filename><replaceable>package</replaceable><filename>/</filename><replaceable>file</replaceable><filename>.inc</filename>
+                        instead of
+                        <filename>require </filename><replaceable>file</replaceable><filename>.inc</filename>.
+                        If you're finding you have to overlay the include file,
+                        it could indicate a deficiency in the include file in
+                        the layer to which it originally belongs.
+                        If this is the case, you should try to address that
+                        deficiency instead of overlaying the include file.
+                        For example, you could address this by getting the
+                        maintainer of the include file to add a variable or
+                        variables to make it easy to override the parts needing
+                        to be overridden.
+                        </para></listitem>
+                    <listitem><para>
+                        <emphasis>Structure Your Layers:</emphasis>
+                        Proper use of overrides within append files and
+                        placement of machine-specific files within your layer
+                        can ensure that a build is not using the wrong Metadata
+                        and negatively impacting a build for a different
+                        machine.
+                        Following are some examples:
+                        <itemizedlist>
+                            <listitem><para>
+                                <emphasis>Modify Variables to Support a
+                                Different Machine:</emphasis>
+                                Suppose you have a layer named
+                                <filename>meta-one</filename> that adds support
+                                for building machine "one".
+                                To do so, you use an append file named
+                                <filename>base-files.bbappend</filename> and
+                                create a dependency on "foo" by altering the
+                                <ulink url='&YOCTO_DOCS_REF_URL;#var-DEPENDS'><filename>DEPENDS</filename></ulink>
+                                variable:
+                                <literallayout class='monospaced'>
      DEPENDS = "foo"
-                            </literallayout>
-                            The dependency is created during any build that
-                            includes the layer
-                            <filename>meta-one</filename>.
-                            However, you might not want this dependency
-                            for all machines.
-                            For example, suppose you are building for
-                            machine "two" but your
-                            <filename>bblayers.conf</filename> file has the
-                            <filename>meta-one</filename> layer included.
-                            During the build, the
-                            <filename>base-files</filename> for machine
-                            "two" will also have the dependency on
-                            <filename>foo</filename>.</para>
-                            <para>To make sure your changes apply only when
-                            building machine "one", use a machine override
-                            with the <filename>DEPENDS</filename> statement:
-                            <literallayout class='monospaced'>
+                                </literallayout>
+                                The dependency is created during any build that
+                                includes the layer
+                                <filename>meta-one</filename>.
+                                However, you might not want this dependency
+                                for all machines.
+                                For example, suppose you are building for
+                                machine "two" but your
+                                <filename>bblayers.conf</filename> file has the
+                                <filename>meta-one</filename> layer included.
+                                During the build, the
+                                <filename>base-files</filename> for machine
+                                "two" will also have the dependency on
+                                <filename>foo</filename>.</para>
+                                <para>To make sure your changes apply only when
+                                building machine "one", use a machine override
+                                with the <filename>DEPENDS</filename> statement:
+                                <literallayout class='monospaced'>
      DEPENDS_one = "foo"
-                            </literallayout>
-                            You should follow the same strategy when using
-                            <filename>_append</filename> and
-                            <filename>_prepend</filename> operations:
-                            <literallayout class='monospaced'>
+                                </literallayout>
+                                You should follow the same strategy when using
+                                <filename>_append</filename> and
+                                <filename>_prepend</filename> operations:
+                                <literallayout class='monospaced'>
      DEPENDS_append_one = " foo"
      DEPENDS_prepend_one = "foo "
-                            </literallayout>
-                            As an actual example, here's a line from the recipe
-                            for gnutls, which adds dependencies on
-                            "argp-standalone" when building with the musl C
-                            library:
-                            <literallayout class='monospaced'>
+                                </literallayout>
+                                As an actual example, here's a line from the recipe
+                                for gnutls, which adds dependencies on
+                                "argp-standalone" when building with the musl C
+                                library:
+                                <literallayout class='monospaced'>
      DEPENDS_append_libc-musl = " argp-standalone"
-                            </literallayout>
-                            <note>
-                                Avoiding "+=" and "=+" and using
-                                machine-specific
-                                <filename>_append</filename>
-                                and <filename>_prepend</filename> operations
-                                is recommended as well.
-                            </note></para></listitem>
-                        <listitem><para><emphasis>Place Machine-Specific Files
-                            in Machine-Specific Locations:</emphasis>
-                            When you have a base recipe, such as
-                            <filename>base-files.bb</filename>, that
-                            contains a
-                            <ulink url='&YOCTO_DOCS_REF_URL;#var-SRC_URI'><filename>SRC_URI</filename></ulink>
-                            statement to a file, you can use an append file
-                            to cause the build to use your own version of
-                            the file.
-                            For example, an append file in your layer at
-                            <filename>meta-one/recipes-core/base-files/base-files.bbappend</filename>
-                            could extend
-                            <ulink url='&YOCTO_DOCS_REF_URL;#var-FILESPATH'><filename>FILESPATH</filename></ulink>
-                            using
-                            <ulink url='&YOCTO_DOCS_REF_URL;#var-FILESEXTRAPATHS'><filename>FILESEXTRAPATHS</filename></ulink>
-                            as follows:
-                            <literallayout class='monospaced'>
+                                </literallayout>
+                                <note>
+                                    Avoiding "+=" and "=+" and using
+                                    machine-specific
+                                    <filename>_append</filename>
+                                    and <filename>_prepend</filename> operations
+                                    is recommended as well.
+                                </note>
+                                </para></listitem>
+                            <listitem><para>
+                                <emphasis>Place Machine-Specific Files in
+                                Machine-Specific Locations:</emphasis>
+                                When you have a base recipe, such as
+                                <filename>base-files.bb</filename>, that
+                                contains a
+                                <ulink url='&YOCTO_DOCS_REF_URL;#var-SRC_URI'><filename>SRC_URI</filename></ulink>
+                                statement to a file, you can use an append file
+                                to cause the build to use your own version of
+                                the file.
+                                For example, an append file in your layer at
+                                <filename>meta-one/recipes-core/base-files/base-files.bbappend</filename>
+                                could extend
+                                <ulink url='&YOCTO_DOCS_REF_URL;#var-FILESPATH'><filename>FILESPATH</filename></ulink>
+                                using
+                                <ulink url='&YOCTO_DOCS_REF_URL;#var-FILESEXTRAPATHS'><filename>FILESEXTRAPATHS</filename></ulink>
+                                as follows:
+                                <literallayout class='monospaced'>
      FILESEXTRAPATHS_prepend := "${THISDIR}/${BPN}:"
-                            </literallayout>
-                            The build for machine "one" will pick up your
-                            machine-specific file as long as you have the
-                            file in
-                            <filename>meta-one/recipes-core/base-files/base-files/</filename>.
-                            However, if you are building for a different
-                            machine and the
-                            <filename>bblayers.conf</filename> file includes
-                            the <filename>meta-one</filename> layer and
-                            the location of your machine-specific file is
-                            the first location where that file is found
-                            according to <filename>FILESPATH</filename>,
-                            builds for all machines will also use that
-                            machine-specific file.</para>
-                            <para>You can make sure that a machine-specific
-                            file is used for a particular machine by putting
-                            the file in a subdirectory specific to the
-                            machine.
-                            For example, rather than placing the file in
-                            <filename>meta-one/recipes-core/base-files/base-files/</filename>
-                            as shown above, put it in
-                            <filename>meta-one/recipes-core/base-files/base-files/one/</filename>.
-                            Not only does this make sure the file is used
-                            only when building for machine "one", but the
-                            build process locates the file more quickly.</para>
-                            <para>In summary, you need to place all files
-                            referenced from <filename>SRC_URI</filename>
-                            in a machine-specific subdirectory within the
-                            layer in order to restrict those files to
-                            machine-specific builds.</para></listitem>
-                    </itemizedlist>
-                </para>
-            </section>
-
-            <section id='other-recommendations'>
-                <title>Other Recommendations</title>
-
-                <para>
-                    We also recommend the following:
-                    <itemizedlist>
-                        <listitem><para>If you want permission to use the
-                            Yocto Project Compatibility logo with your layer
-                            or application that uses your layer, perform the
-                            steps to apply for compatibility.
-                            See the
-                            "<link linkend='making-sure-your-layer-is-compatible-with-yocto-project'>Making Sure Your Layer is Compatible With Yocto Project</link>"
-                            section for more information.
-                            </para></listitem>
-                        <listitem><para>Store custom layers in a Git repository
-                            that uses the
-                            <filename>meta-<replaceable>layer_name</replaceable></filename> format.
-                            </para></listitem>
-                        <listitem><para>Clone the repository alongside other
-                            <filename>meta</filename> directories in the
-                            <ulink url='&YOCTO_DOCS_REF_URL;#source-directory'>Source Directory</ulink>.
-                            </para></listitem>
-                     </itemizedlist>
-                     Following these recommendations keeps your Source Directory and
-                     its configuration entirely inside the Yocto Project's core
-                     base.
-                </para>
-            </section>
+                                </literallayout>
+                                The build for machine "one" will pick up your
+                                machine-specific file as long as you have the
+                                file in
+                                <filename>meta-one/recipes-core/base-files/base-files/</filename>.
+                                However, if you are building for a different
+                                machine and the
+                                <filename>bblayers.conf</filename> file includes
+                                the <filename>meta-one</filename> layer and
+                                the location of your machine-specific file is
+                                the first location where that file is found
+                                according to <filename>FILESPATH</filename>,
+                                builds for all machines will also use that
+                                machine-specific file.</para>
+                                <para>You can make sure that a machine-specific
+                                file is used for a particular machine by putting
+                                the file in a subdirectory specific to the
+                                machine.
+                                For example, rather than placing the file in
+                                <filename>meta-one/recipes-core/base-files/base-files/</filename>
+                                as shown above, put it in
+                                <filename>meta-one/recipes-core/base-files/base-files/one/</filename>.
+                                Not only does this make sure the file is used
+                                only when building for machine "one", but the
+                                build process locates the file more quickly.</para>
+                                <para>In summary, you need to place all files
+                                referenced from <filename>SRC_URI</filename>
+                                in a machine-specific subdirectory within the
+                                layer in order to restrict those files to
+                                machine-specific builds.
+                                </para></listitem>
+                        </itemizedlist>
+                        </para></listitem>
+                    <listitem><para>
+                        <emphasis>Perform Steps to Apply for Yocto Project Compatibility:</emphasis>
+                        If you want permission to use the
+                        Yocto Project Compatibility logo with your layer
+                        or application that uses your layer, perform the
+                        steps to apply for compatibility.
+                        See the
+                        "<link linkend='making-sure-your-layer-is-compatible-with-yocto-project'>Making Sure Your Layer is Compatible With Yocto Project</link>"
+                        section for more information.
+                        </para></listitem>
+                    <listitem><para>
+                        <emphasis>Follow the Layer Naming Convention:</emphasis>
+                        Store custom layers in a Git repository that use the
+                        <filename>meta-<replaceable>layer_name</replaceable></filename>
+                        format.
+                        </para></listitem>
+                    <listitem><para>
+                        <emphasis>Group Your Layers Locally:</emphasis>
+                        Clone your repository alongside other cloned
+                        <filename>meta</filename> directories from the
+                        <ulink url='&YOCTO_DOCS_REF_URL;#source-directory'>Source Directory</ulink>.
+                        </para></listitem>
+                </itemizedlist>
+            </para>
         </section>
 
         <section id='making-sure-your-layer-is-compatible-with-yocto-project'>
@@ -686,37 +671,46 @@
         </section>
 
         <section id='using-bbappend-files'>
-            <title>Using .bbappend Files</title>
+            <title>Using .bbappend Files in Your Layer</title>
 
             <para>
-                Recipes used to append Metadata to other recipes are called
-                BitBake append files.
-                BitBake append files use the <filename>.bbappend</filename> file
-                type suffix, while the corresponding recipes to which Metadata
-                is being appended use the <filename>.bb</filename> file type
-                suffix.
+                A recipe that appends Metadata to another recipe is called a
+                BitBake append file.
+                A BitBake append file uses the <filename>.bbappend</filename>
+                file type suffix, while the corresponding recipe to which
+                Metadata is being appended uses the <filename>.bb</filename>
+                file type suffix.
             </para>
 
             <para>
-                A <filename>.bbappend</filename> file allows your layer to make
-                additions or changes to the content of another layer's recipe
-                without having to copy the other recipe into your layer.
+                You can use a <filename>.bbappend</filename> file in your
+                layer to make additions or changes to the content of another
+                layer's recipe without having to copy the other layer's
+                recipe into your layer.
                 Your <filename>.bbappend</filename> file resides in your layer,
                 while the main <filename>.bb</filename> recipe file to
                 which you are appending Metadata resides in a different layer.
             </para>
 
             <para>
-                Append files must have the same root names as their corresponding
-                recipes.
+                Being able to append information to an existing recipe not only
+                avoids duplication, but also automatically applies recipe
+                changes from a different layer into your layer.
+                If you were copying recipes, you would have to manually merge
+                changes as they occur.
+            </para>
+
+            <para>
+                When you create an append file, you must use the same root
+                name as the corresponding recipe file.
                 For example, the append file
                 <filename>someapp_&DISTRO;.bbappend</filename> must apply to
                 <filename>someapp_&DISTRO;.bb</filename>.
-                This means the original recipe and append file names are version
-                number-specific.
+                This means the original recipe and append file names are
+                version number-specific.
                 If the corresponding recipe is renamed to update to a newer
-                version, the corresponding <filename>.bbappend</filename> file must
-                be renamed (and possibly updated) as well.
+                version, you must also rename and possibly update
+                the corresponding <filename>.bbappend</filename> as well.
                 During the build process, BitBake displays an error on starting
                 if it detects a <filename>.bbappend</filename> file that does
                 not have a corresponding recipe with a matching name.
@@ -725,14 +719,6 @@
                 variable for information on how to handle this error.
             </para>
 
-            <para>
-                Being able to append information to an existing recipe not only
-                avoids duplication, but also automatically applies recipe
-                changes in a different layer to your layer.
-                If you were copying recipes, you would have to manually merge
-                changes as they occur.
-            </para>
-
             <para>
                 As an example, consider the main formfactor recipe and a
                 corresponding formfactor append file both from the
@@ -745,8 +731,7 @@
      SUMMARY = "Device formfactor information"
      SECTION = "base"
      LICENSE = "MIT"
-     LIC_FILES_CHKSUM = "file://${COREBASE}/LICENSE;md5=4d92cd373abda3937c2bc47fbc49d690 \
-                    file://${COREBASE}/meta/COPYING.MIT;md5=3da9cfbcb788c80a0384361b4de20420"
+     LIC_FILES_CHKSUM = "file://${COREBASE}/meta/COPYING.MIT;md5=3da9cfbcb788c80a0384361b4de20420"
      PR = "r45"
 
      SRC_URI = "file://config file://machconfig"
@@ -762,8 +747,7 @@
 	     if [ -s "${S}/machconfig" ]; then
 	             install -m 0644 ${S}/machconfig ${D}${sysconfdir}/formfactor/
 	     fi
-     }
-                </literallayout>
+     }                </literallayout>
                 In the main recipe, note the
                 <ulink url='&YOCTO_DOCS_REF_URL;#var-SRC_URI'><filename>SRC_URI</filename></ulink>
                 variable, which tells the OpenEmbedded build system where to
@@ -775,7 +759,8 @@
                 <filename>formfactor_0.0.bbappend</filename> and is from the
                 Raspberry Pi BSP Layer named
                 <filename>meta-raspberrypi</filename>.
-                The file is in <filename>recipes-bsp/formfactor</filename>:
+                The file is in the layer at
+                <filename>recipes-bsp/formfactor</filename>:
                 <literallayout class='monospaced'>
      FILESEXTRAPATHS_prepend := "${THISDIR}/${PN}:"
                 </literallayout>
@@ -795,12 +780,13 @@
             </para>
 
             <para>
-                The statement in this example extends the directories to include
+                The statement in this example extends the directories to
+                include
                 <filename>${</filename><ulink url='&YOCTO_DOCS_REF_URL;#var-THISDIR'><filename>THISDIR</filename></ulink><filename>}/${</filename><ulink url='&YOCTO_DOCS_REF_URL;#var-PN'><filename>PN</filename></ulink><filename>}</filename>,
                 which resolves to a directory named
                 <filename>formfactor</filename> in the same directory
                 in which the append file resides (i.e.
-                <filename>meta-raspberrypi/recipes-bsp/formfactor/formfactor</filename>.
+                <filename>meta-raspberrypi/recipes-bsp/formfactor</filename>.
                 This implies that you must have the supporting directory
                 structure set up that will contain any files or patches you
                 will be including from the layer.
@@ -808,8 +794,8 @@
 
             <para>
                 Using the immediate expansion assignment operator
-                <filename>:=</filename> is important because of the reference to
-                <filename>THISDIR</filename>.
+                <filename>:=</filename> is important because of the reference
+                to <filename>THISDIR</filename>.
                 The trailing colon character is important as it ensures that
                 items in the list remain colon-separated.
                 <note>
@@ -7144,7 +7130,7 @@ Some notes from Cal:
                         and <ulink url='&YOCTO_DOCS_REF_URL;#var-SRC_URI'><filename>SRC_URI</filename></ulink>
                         statements enable the OpenEmbedded build system to find the patch file.
                         For more information on using append files, see the
-                        "<link linkend='using-bbappend-files'>Using .bbappend Files</link>"
+                        "<link linkend='using-bbappend-files'>Using .bbappend Files in Your Layer</link>"
                         section.
                         </para></listitem>
                     <listitem><para><emphasis>Put the patch file in your layer</emphasis>:
@@ -7616,14 +7602,14 @@ Some notes from Cal:
                             on how to add recipes to your layer, see the
                             "<link linkend='creating-your-own-layer'>Creating Your Own Layer</link>"
                             and
-                            "<link linkend='best-practices-to-follow-when-creating-layers'>Best Practices to Follow When Creating Layers</link>"
+                            "<link linkend='best-practices-to-follow-when-creating-layers'>Following Best Practices When Creating Layers</link>"
                             sections.</para></listitem>
                         <listitem><para>Add any image recipes that are specific
                             to your distribution.</para></listitem>
                         <listitem><para>Add a <filename>psplash</filename>
                             append file for a branded splash screen.
                             For information on append files, see the
-                            "<link linkend='using-bbappend-files'>Using .bbappend Files</link>"
+                            "<link linkend='using-bbappend-files'>Using .bbappend Files in Your Layer</link>"
                             section.</para></listitem>
                         <listitem><para>Add any other append files to make
                             custom changes that are specific to individual

documentation/kernel-dev/kernel-dev-faq.xml

@@ -72,7 +72,7 @@
                         <filename>RDEPENDS_kernel-base</filename> to include or not
                         include "kernel-image".</para>
                         <para>See the
-                        "<ulink url='&YOCTO_DOCS_DEV_URL;#using-bbappend-files'>Using .bbappend Files</ulink>"
+                        "<ulink url='&YOCTO_DOCS_DEV_URL;#using-bbappend-files'>Using .bbappend Files in Your Layer</ulink>"
                         section in the Yocto Project Development Manual for information on
                         how to use an append file to override metadata.
                     </para>

documentation/poky.ent

@@ -69,7 +69,7 @@
 <!ENTITY FEDORA_HOST_PACKAGES_ESSENTIAL "gawk make wget tar bzip2 gzip python3 unzip perl patch \
      diffutils diffstat git cpp gcc gcc-c++ glibc-devel texinfo chrpath \
      ccache perl-Data-Dumper perl-Text-ParseWords perl-Thread-Queue perl-bignum socat \
-     python3-pexpect findutils which file cpio python python3-pip xz which">
+     python3-pexpect findutils which file cpio python python3-pip xz">
 <!ENTITY OPENSUSE_HOST_PACKAGES_ESSENTIAL "python gcc gcc-c++ git chrpath make wget python-xml \
      diffstat makeinfo python-curses patch socat python3 python3-curses tar python3-pip \
      python3-pexpect xz which">
@@ -77,5 +77,5 @@
      $ sudo yum makecache
      $ sudo yum install gawk make wget tar bzip2 gzip python unzip perl patch \
      diffutils diffstat git cpp gcc gcc-c++ glibc-devel texinfo chrpath socat \
-     perl-Data-Dumper perl-Text-ParseWords perl-Thread-Queue python34-pip xz \
+     perl-Data-Dumper perl-Text-ParseWords perl-Thread-Queue python3-pip xz \
      which">

documentation/ref-manual/introduction.xml

@@ -25,12 +25,14 @@
         For introductory information on the Yocto Project, see the
         <ulink url='&YOCTO_HOME_URL;/ecosystem/yocto-project-backgrounders'>Yocto Project Backgrounders</ulink>
         on the
-        <ulink url='&YOCTO_HOME_URL;'>Yocto Project Website</ulink>.
+        <ulink url='&YOCTO_HOME_URL;'>Yocto Project Website</ulink> and the
+        "<link linkend='yp-intro'>Introducing the Yocto Project Development Environment</link>"
+        section.
     </para>
 
     <para>
-        You can find an introductory to using the Yocto Project by working
-        through the
+        If you want to use the Yocto Project to test run building an image
+        without having to understand concepts, work through the
         <ulink url='&YOCTO_DOCS_QS_URL;'>Yocto Project Quick Start</ulink>.
         You can find "how-to" information in the
         <ulink url='&YOCTO_DOCS_DEV_URL;'>Yocto Project Development Manual</ulink>.
@@ -43,8 +45,8 @@
     </para>
 </section>
 
-<section id='ref-yp-intro'>
-    <title>Yocto Project Introduction</title>
+<section id='yp-intro'>
+    <title>Introducing the Yocto Project Development Environment</title>
 
     <para>
         The Yocto Project is an open-source collaboration project whose
@@ -69,6 +71,68 @@
         is optimized for stylus-driven, low-resolution screens.
     </para>
 
+    <mediaobject>
+        <imageobject>
+            <imagedata fileref="figures/YP-flow-diagram.png"
+                format="PNG" align='center' width="8in"/>
+        </imageobject>
+    </mediaobject>
+
+    <para>
+        Here are some highlights for the Yocto Project:
+    </para>
+
+    <itemizedlist>
+        <listitem><para>
+            Provides a recent Linux kernel along with a set of system
+            commands and libraries suitable for the embedded
+            environment.
+            </para></listitem>
+        <listitem><para>
+            Makes available system components such as X11, GTK+, Qt,
+            Clutter, and SDL (among others) so you can create a rich user
+            experience on devices that have display hardware.
+            For devices that do not have a display or where you wish to
+            use alternative UI frameworks, these components need not be
+            installed.
+            </para></listitem>
+        <listitem><para>
+            Creates a focused and stable core compatible with the
+            OpenEmbedded project with which you can easily and reliably
+            build and develop.
+            </para></listitem>
+        <listitem><para>
+            Fully supports a wide range of hardware and device emulation
+            through the Quick EMUlator (QEMU).
+            </para></listitem>
+        <listitem><para>
+            Provides a layer mechanism that allows you to easily extend
+            the system, make customizations, and keep them organized.
+            </para></listitem>
+    </itemizedlist>
+
+    <para>
+        You can use the Yocto Project to generate images for many kinds
+        of devices.
+        As mentioned earlier, the Yocto Project supports creation of
+        reference images that you can boot within and emulate using QEMU.
+        The standard example machines target QEMU full-system
+        emulation for 32-bit and 64-bit variants of x86, ARM, MIPS, and
+        PowerPC architectures.
+        Beyond emulation, you can use the layer mechanism to extend
+        support to just about any platform that Linux can run on and that
+        a toolchain can target.
+    </para>
+
+    <para>
+        Another Yocto Project feature is the Sato reference User
+        Interface.
+        This optional UI that is based on GTK+ is intended for devices with
+        restricted screen sizes and is included as part of the
+        OpenEmbedded Core layer so that developers can test parts of the
+        software stack.
+    </para>
+
     <para>
         While the Yocto Project does not provide a strict testing framework,
         it does provide or generate for you artifacts that let you perform
@@ -625,7 +689,7 @@
                 <para>Information in append files extends or overrides the
                 information in the similarly-named recipe file.
                 For an example of an append file in use, see the
-                "<ulink url='&YOCTO_DOCS_DEV_URL;#using-bbappend-files'>Using .bbappend Files</ulink>"
+                "<ulink url='&YOCTO_DOCS_DEV_URL;#using-bbappend-files'>Using .bbappend Files in Your Layer</ulink>"
                 section in the Yocto Project Development Manual.
                 <note>
                     Append files can also use wildcard patterns in their

documentation/yocto-project-qs/qs-style.css

@@ -730,6 +730,11 @@ div.navfooter {
 }
 
 
+.writernotes {
+  color: red;
+}
+
+
   /*********** /
  /  graphics  /
 / ***********/

documentation/yocto-project-qs/yocto-project-qs.xml

@@ -44,7 +44,6 @@
             </note>
         </legalnotice>
 
-
         <abstract>
             <imagedata fileref="figures/yocto-project-transp.png"
                         width="6in" depth="1in"
@@ -62,7 +61,7 @@
             on the OpenEmbedded (OE) project, which uses the
             <ulink url='&YOCTO_DOCS_REF_URL;#bitbake-term'>BitBake</ulink>
             tool, to construct complete Linux images.
-            The BitBake and OE components are combined together to form
+            The BitBake and OE components combine together to form
             a reference build host, historically known as
             <ulink url='&YOCTO_DOCS_REF_URL;#poky'>Poky</ulink>
             (<emphasis>Pah</emphasis>-kee).
@@ -74,8 +73,14 @@
             Linux images.
             Rather than go into great detail about the Yocto Project and its
             many capabilities, this quick start provides the minimal
-            information you need to try out the Yocto Project using a
-            supported Linux build host.
+            information you need to try out the Yocto Project using either a
+            supported Linux build host or a build host set up to use
+            <ulink url='https://git.yoctoproject.org/cgit/cgit.cgi/crops/about/'>CROPS</ulink>,
+            which leverages
+            <ulink url='https://www.docker.com/'>Docker Containers</ulink>.
+        </para>
+
+        <para>
             Reading and using the quick start should result in you having a
             basic understanding of what the Yocto Project is and how to use
             some of its core components.
@@ -100,6 +105,7 @@
                     <ulink url='http://www.yoctoproject.org/docs/2.4/yocto-project-qs/yocto-project-qs.html#building-an-image-for-hardware'>Building an Image for Hardware</ulink>
                     </para></listitem>
             </itemizedlist>
+<!--
             <note>
                 If you do not have a system that runs Linux and you want to give
                 the Yocto Project a test run, you might consider using the Yocto
@@ -111,18 +117,13 @@
                 <ulink url='https://www.yoctoproject.org/tools-resources/projects/build-appliance'>Yocto Project Build Appliance</ulink>
                 for more information.
             </note>
+-->
         </para>
 
         <para>
             For more detailed information on the Yocto Project, you can
             reference these resources:
             <itemizedlist>
-                <listitem><para>
-                    <emphasis>Yocto Project Backgrounders:</emphasis>
-                    The
-                    <ulink url='&YOCTO_HOME_URL;/ecosystem/yocto-project-backgrounders'>Yocto Project Backgrounder</ulink>
-                    provides introductory information on the Yocto Project.
-                    </para></listitem>
                 <listitem><para>
                     <emphasis>Website:</emphasis>
                     The
@@ -131,6 +132,19 @@
                     documentation, and access to a rich Yocto Project
                     Development Community into which you can tap.
                     </para></listitem>
+                <listitem><para>
+                    <emphasis>Yocto Project Backgrounders:</emphasis>
+                    The
+                    <ulink url='&YOCTO_HOME_URL;/ecosystem/yocto-project-backgrounders'>Yocto Project Backgrounder</ulink>
+                    provides introductory information on the Yocto Project.
+                    </para></listitem>
+                <listitem><para>
+                    <emphasis>Yocto Project Development Environment Overview:</emphasis>
+                    The
+                    "<ulink url='&YOCTO_DOCS_REF_URL;#yp-intro'>Introducing the Yocto Project Development Environment</ulink>"
+                    section presents an overview of the Yocto Project
+                    development environment.
+                    </para></listitem>
                 <listitem><para>
                     <emphasis>FAQs:</emphasis>
                     Lists commonly asked Yocto Project questions and answers.
@@ -161,243 +175,238 @@
         </para>
     </section>
 
-    <section id='yp-intro'>
-        <title>Introducing the Yocto Project Development Environment</title>
-
-        <para>
-            The Yocto Project through the OpenEmbedded build system provides an
-            open source development environment targeting the ARM, MIPS,
-            PowerPC, and x86 architectures for a variety of platforms
-            including x86-64 and emulated ones.
-            You can use components from the Yocto Project to design, develop,
-            build, debug, simulate, and test the complete software stack using
-            Linux, the X Window System, GTK+ frameworks, and Qt frameworks.
-        </para>
-
-        <mediaobject>
-            <imageobject>
-                <imagedata fileref="figures/yocto-environment.png"
-                    format="PNG" align='center' width="8in"/>
-            </imageobject>
-         </mediaobject>
-
-        <para>
-            Here are some highlights for the Yocto Project:
-        </para>
-
-        <itemizedlist>
-            <listitem><para>
-                Provides a recent Linux kernel along with a set of system
-                commands and libraries suitable for the embedded
-                environment.
-                </para></listitem>
-            <listitem><para>
-                Makes available system components such as X11, GTK+, Qt,
-                Clutter, and SDL (among others) so you can create a rich user
-                experience on devices that have display hardware.
-                For devices that do not have a display or where you wish to
-                use alternative UI frameworks, these components need not be
-                installed.
-                </para></listitem>
-            <listitem><para>
-                Creates a focused and stable core compatible with the
-                OpenEmbedded project with which you can easily and reliably
-                build and develop.
-                </para></listitem>
-            <listitem><para>
-                Fully supports a wide range of hardware and device emulation
-                through the Quick EMUlator (QEMU).
-                </para></listitem>
-            <listitem><para>
-                Provides a layer mechanism that allows you to easily extend
-                the system, make customizations, and keep them organized.
-                </para></listitem>
-        </itemizedlist>
-
-        <para>
-            You can use the Yocto Project to generate images for many kinds
-            of devices.
-            As mentioned earlier, the Yocto Project supports creation of
-            reference images that you can boot within and emulate using QEMU.
-            The standard example machines target QEMU full-system
-            emulation for 32-bit and 64-bit variants of x86, ARM, MIPS, and
-            PowerPC architectures.
-            Beyond emulation, you can use the layer mechanism to extend
-            support to just about any platform that Linux can run on and that
-            a toolchain can target.
-        </para>
-
-        <para>
-            Another Yocto Project feature is the Sato reference User
-            Interface.
-            This optional UI that is based on GTK+ is intended for devices with
-            restricted screen sizes and is included as part of the
-            OpenEmbedded Core layer so that developers can test parts of the
-            software stack.
-        </para>
-    </section>
-
     <section id='yp-resources'>
         <title>Setting Up to Use the Yocto Project</title>
 
         <para>
-            The following list shows what you need in order to use a
-            Linux-based build host to use the Yocto Project to build images:
+            Setting up to use the Yocto Project involves getting your build
+            host ready.
+            If you have a native Linux machine that runs a Yocto Project
+            supported distribution as described by the
+            "<ulink url='&YOCTO_DOCS_REF_URL;#detailed-supported-distros'>Supported Linux Distributions</ulink>"
+            section in the Yocto Project Reference Manual, you can prepare
+            that machine as your build host.
+            See the
+            "<link linkend='qs-native-linux-build-host'>Using a Native Linux Machine</link>"
+            section for more information.
         </para>
 
-        <itemizedlist>
-            <listitem><para><emphasis>Build Host</emphasis>
-                A build host with a minimum of 50 Gbytes of free disk
-                space that is running a supported Linux distribution (i.e.
-                recent releases of Fedora, openSUSE, CentOS, Debian, or
-                Ubuntu).
-                </para></listitem>
-            <listitem><para><emphasis>Build Host Packages</emphasis>
-                Appropriate packages installed on the build host.
-                </para></listitem>
-            <listitem><para><emphasis>The Yocto Project</emphasis>
-                A release of the Yocto Project.
-                </para></listitem>
-        </itemizedlist>
+        <para>
+            If you do not want to use the Yocto Project on a native Linux
+            machine, you can prepare your build host to use
+            <ulink url='https://git.yoctoproject.org/cgit/cgit.cgi/crops/about/'>CROPS</ulink>,
+            which leverages
+            <ulink url='https://www.docker.com/'>Docker Containers</ulink>.
+            You can set up a build host for Windows, Mac, and Linux
+            machines.
+            See the
+            "<link linkend='qs-crops-build-host'>Using CROPS and Containers</link>"
+            section for more information.
+        </para>
 
-        <section id='the-linux-distro'>
-            <title>The Linux Distribution</title>
+        <section id='qs-crops-build-host'>
+            <title>Using CROPS and Containers</title>
 
             <para>
-                The Yocto Project team verifies each release against recent
-                versions of the most popular Linux distributions that
-                provide stable releases.
-                In general, if you have the current release minus one of the
-                following distributions, you should have no problems.
-                <itemizedlist>
+                Follow these steps to get your build host set up with a
+                Poky container that you can use to complete the build
+                examples further down in the Quick Start:
+                <orderedlist>
                     <listitem><para>
-                        Ubuntu
+                        <emphasis>Set Up to use CROss PlatformS (CROPS):</emphasis>
+                        Work through the first six steps of the procedure
+                        in the
+                        "<ulink url='&YOCTO_DOCS_DEV_URL;#setting-up-to-use-crops'>Setting Up to Use CROss PlatformS (CROPS)</ulink>"
+                        section of the Yocto Project Development Manual.
                         </para></listitem>
                     <listitem><para>
-                        Fedora
-                        </para></listitem>
-                    <listitem><para>
-                        openSUSE
-                        </para></listitem>
-                    <listitem><para>
-                        CentOS
-                        </para></listitem>
-                    <listitem><para>
-                        Debian
-                        </para></listitem>
-                </itemizedlist>
-                For a more detailed list of distributions that support the
-                Yocto Project, see the
-                "<ulink url='&YOCTO_DOCS_REF_URL;#detailed-supported-distros'>Supported Linux Distributions</ulink>"
-                section in the Yocto Project Reference Manual.
-            </para>
+                        <emphasis>Set Up the Poky Container to Use the Yocto Project:</emphasis>
+                        Go to
+                        <ulink url='https://github.com/crops/poky-container/blob/master/README.md'></ulink>
+                        and follow the directions to set up the Poky container
+                        on your build host.</para>
 
-            <para>
-                The OpenEmbedded build system should be able to run on any
-                modern distribution that has the following versions for
-                Git, tar, and Python.
-                <itemizedlist>
-                    <listitem><para>
-                        Git 1.8.3.1 or greater
+                        <para>Once you complete the setup instructions for your
+                        machine, you need to get a copy of the
+                        <filename>poky</filename> repository on your build
+                        host.
+                        See the
+                        "<link linkend='releases'>Yocto Project Release</link>"
+                        section to continue.
                         </para></listitem>
-                    <listitem><para>
-                        tar 1.27 or greater
-                        </para></listitem>
-                    <listitem><para>
-                        Python 3.4.0 or greater.
-                        </para></listitem>
-                </itemizedlist>
-                If your build host does not meet any of these three listed
-                version requirements, you can take steps to prepare the
-                system so that you can still use the Yocto Project.
-                See the
-                "<ulink url='&YOCTO_DOCS_REF_URL;#required-git-tar-and-python-versions'>Required Git, tar, and Python Versions</ulink>"
-                section in the Yocto Project Reference Manual for information.
+                </orderedlist>
             </para>
         </section>
 
-        <section id='packages'>
-            <title>The Build Host Packages</title>
+        <section id='qs-native-linux-build-host'>
+            <title>Using a Native Linux Machine</title>
 
             <para>
-                Required build host packages vary depending on your
-                build machine and what you want to do with the Yocto Project.
-                For example, if you want to build an image that can run
-                on QEMU in graphical mode (a minimal, basic build
-                requirement), then the build host package requirements
-                are different than if you want to build an image on a headless
-                system or build out the Yocto Project documentation set.
+                The following list shows what you need in order to use a
+                Linux-based build host to use the Yocto Project to build images:
             </para>
 
-            <para>
-                Collectively, the number of required packages is large
-                if you want to be able to cover all cases.
-                <note>
-                    In general, you need to have root access and then install
-                    the required packages.
-                    Thus, the commands in the following section may or may
-                    not work depending on whether or not your Linux
-                    distribution has <filename>sudo</filename> installed.
-                </note>
-            </para>
+            <itemizedlist>
+                <listitem><para><emphasis>Build Host</emphasis>
+                    A build host with a minimum of 50 Gbytes of free disk
+                    space that is running a supported Linux distribution (i.e.
+                    recent releases of Fedora, openSUSE, CentOS, Debian, or
+                    Ubuntu).
+                    </para></listitem>
+                <listitem><para><emphasis>Build Host Packages</emphasis>
+                    Appropriate packages installed on the build host.
+                    </para></listitem>
+            </itemizedlist>
 
-            <para>
-                The following list shows the required packages needed to build
-                an image that runs on QEMU in graphical mode (e.g. essential
-                plus graphics support).
-                For lists of required packages for other scenarios, see the
-                "<ulink url='&YOCTO_DOCS_REF_URL;#required-packages-for-the-host-development-system'>Required Packages for the Host Development System</ulink>"
-                section in the Yocto Project Reference Manual.
-                <itemizedlist>
-                    <listitem><para><emphasis>Ubuntu and Debian</emphasis>
-                        <literallayout class='monospaced'>
+            <section id='the-linux-distro'>
+                <title>The Linux Distribution</title>
+
+                <para>
+                    The Yocto Project team verifies each release against recent
+                    versions of the most popular Linux distributions that
+                    provide stable releases.
+                    In general, if you have the current release minus one of the
+                    following distributions, you should have no problems.
+                    <itemizedlist>
+                        <listitem><para>
+                            Ubuntu
+                            </para></listitem>
+                        <listitem><para>
+                            Fedora
+                            </para></listitem>
+                        <listitem><para>
+                            openSUSE
+                            </para></listitem>
+                        <listitem><para>
+                            CentOS
+                            </para></listitem>
+                        <listitem><para>
+                            Debian
+                            </para></listitem>
+                    </itemizedlist>
+                    For a more detailed list of distributions that support the
+                    Yocto Project, see the
+                    "<ulink url='&YOCTO_DOCS_REF_URL;#detailed-supported-distros'>Supported Linux Distributions</ulink>"
+                    section in the Yocto Project Reference Manual.
+                </para>
+
+                <para>
+                    The OpenEmbedded build system should be able to run on any
+                    modern distribution that has the following versions for
+                    Git, tar, and Python.
+                    <itemizedlist>
+                        <listitem><para>
+                            Git 1.8.3.1 or greater
+                            </para></listitem>
+                        <listitem><para>
+                            tar 1.27 or greater
+                            </para></listitem>
+                        <listitem><para>
+                            Python 3.4.0 or greater.
+                            </para></listitem>
+                    </itemizedlist>
+                    If your build host does not meet any of these three listed
+                    version requirements, you can take steps to prepare the
+                    system so that you can still use the Yocto Project.
+                    See the
+                    "<ulink url='&YOCTO_DOCS_REF_URL;#required-git-tar-and-python-versions'>Required Git, tar, and Python Versions</ulink>"
+                    section in the Yocto Project Reference Manual for information.
+                </para>
+            </section>
+
+            <section id='packages'>
+                <title>The Build Host Packages</title>
+
+                <para>
+                    Required build host packages vary depending on your
+                    build machine and what you want to do with the Yocto Project.
+                    For example, if you want to build an image that can run
+                    on QEMU in graphical mode (a minimal, basic build
+                    requirement), then the build host package requirements
+                    are different than if you want to build an image on a headless
+                    system or build out the Yocto Project documentation set.
+                </para>
+
+                <para>
+                    Collectively, the number of required packages is large
+                    if you want to be able to cover all cases.
+                    <note>
+                        In general, you need to have root access and then install
+                        the required packages.
+                        Thus, the commands in the following section may or may
+                        not work depending on whether or not your Linux
+                        distribution has <filename>sudo</filename> installed.
+                    </note>
+                </para>
+
+                <para>
+                    The following list shows the required packages needed to build
+                    an image that runs on QEMU in graphical mode (e.g. essential
+                    plus graphics support).
+                    For lists of required packages for other scenarios, see the
+                    "<ulink url='&YOCTO_DOCS_REF_URL;#required-packages-for-the-host-development-system'>Required Packages for the Host Development System</ulink>"
+                    section in the Yocto Project Reference Manual.
+                    <itemizedlist>
+                        <listitem><para><emphasis>Ubuntu and Debian</emphasis>
+                            <literallayout class='monospaced'>
      $ sudo apt-get install &UBUNTU_HOST_PACKAGES_ESSENTIAL; libsdl1.2-dev xterm
-                        </literallayout>
-                        </para></listitem>
-                    <listitem><para><emphasis>Fedora</emphasis>
-                        <literallayout class='monospaced'>
+                            </literallayout>
+                            </para></listitem>
+                        <listitem><para><emphasis>Fedora</emphasis>
+                            <literallayout class='monospaced'>
      $ sudo dnf install &FEDORA_HOST_PACKAGES_ESSENTIAL; SDL-devel xterm
-                        </literallayout>
-                        </para></listitem>
-                    <listitem><para><emphasis>OpenSUSE</emphasis>
-                        <literallayout class='monospaced'>
+                            </literallayout>
+                            </para></listitem>
+                        <listitem><para><emphasis>OpenSUSE</emphasis>
+                            <literallayout class='monospaced'>
      $ sudo zypper install &OPENSUSE_HOST_PACKAGES_ESSENTIAL; libSDL-devel xterm
-                        </literallayout>
-                        </para></listitem>
-                    <listitem><para><emphasis>CentOS</emphasis>
-                        <literallayout class='monospaced'>
+                            </literallayout>
+                            </para></listitem>
+                        <listitem><para><emphasis>CentOS</emphasis>
+                            <literallayout class='monospaced'>
      $ sudo yum install &CENTOS_HOST_PACKAGES_ESSENTIAL; SDL-devel xterm
-                        </literallayout>
-                        <note><title>Notes</title>
-                            <itemizedlist>
-                                <listitem><para>
-                                    CentOS 6.x users need to ensure that the
-                                    required versions of Git, tar and Python
-                                    are available.
-                                    For details, See the
-                                    "<ulink url='&YOCTO_DOCS_REF_URL;#required-git-tar-and-python-versions'>Required Git, tar, and Python Versions</ulink>"
-                                    section in the Yocto Project Reference
-                                    Manual for information.
-                                    </para></listitem>
-                                <listitem><para>
-                                    Extra Packages for Enterprise Linux
-                                    (i.e. <filename>epel-release</filename>)
-                                    is a collection of packages from Fedora
-                                    built on RHEL/CentOS for easy installation
-                                    of packages not included in enterprise
-                                    Linux by default.
-                                    You need to install these packages
-                                    separately.
-                                    </para></listitem>
-                                <listitem><para>
-                                    The <filename>makecache</filename> command
-                                    consumes additional Metadata from
-                                    <filename>epel-release</filename>.
-                                    </para></listitem>
-                            </itemizedlist>
-                        </note>
-                        </para></listitem>
-                </itemizedlist>
+                            </literallayout>
+                            <note><title>Notes</title>
+                                <itemizedlist>
+                                    <listitem><para>
+                                        CentOS 6.x users need to ensure that the
+                                        required versions of Git, tar and Python
+                                        are available.
+                                        For details, See the
+                                        "<ulink url='&YOCTO_DOCS_REF_URL;#required-git-tar-and-python-versions'>Required Git, tar, and Python Versions</ulink>"
+                                        section in the Yocto Project Reference
+                                        Manual for information.
+                                        </para></listitem>
+                                    <listitem><para>
+                                        Extra Packages for Enterprise Linux
+                                        (i.e. <filename>epel-release</filename>)
+                                        is a collection of packages from Fedora
+                                        built on RHEL/CentOS for easy installation
+                                        of packages not included in enterprise
+                                        Linux by default.
+                                        You need to install these packages
+                                        separately.
+                                        </para></listitem>
+                                    <listitem><para>
+                                        The <filename>makecache</filename> command
+                                        consumes additional Metadata from
+                                        <filename>epel-release</filename>.
+                                        </para></listitem>
+                                </itemizedlist>
+                            </note>
+                            </para></listitem>
+                    </itemizedlist>
+                </para>
+            </section>
+
+            <para>
+                Once you complete the setup instructions for your
+                machine, you need to get a copy of the
+                <filename>poky</filename> repository on your build
+                host.
+                Continue with the
+                "<link linkend='releases'>Yocto Project Release</link>"
+                section.
             </para>
         </section>
 
@@ -405,8 +414,9 @@
             <title>Yocto Project Release</title>
 
             <para>
-                The last requirement you need to meet before using the
-                Yocto Project is getting a Yocto Project release.
+                Now that your build host has the right packages (native
+                Linux machine) or you have the Poky container set up
+                (CROPS), you need to get a copy of the Yocto Project.
                 It is recommended that you get the latest Yocto Project release
                 by setting up (cloning in
                 <ulink url='&YOCTO_DOCS_REF_URL;#git'>Git</ulink> terms) a
@@ -417,9 +427,14 @@
             </para>
 
             <para>
-                Here is an example from an Ubuntu build host that clones the
-                <filename>poky</filename> repository and then checks out the
-                latest Yocto Project Release by tag
+                Here is an example from a native Linux machine that is
+                running Ubuntu.
+                <note>
+                    If your build host is using a Poky container, you can
+                    use the same Git commands.
+                </note>
+                The example clones the <filename>poky</filename> repository
+                and then checks out the latest Yocto Project Release by tag
                 (i.e. <filename>&DISTRO_REL_TAG;</filename>):
                 <literallayout class='monospaced'>
      $ git clone git://git.yoctoproject.org/poky
@@ -465,20 +480,21 @@
         <title>Building Images</title>
 
         <para>
-            Now that you have your system requirements in order, you can give
-            Yocto Project a try.
-            You can try out Yocto Project using either the command-line
-            interface or using Toaster, which uses a graphical user
-            interface.
-            If you want to try out the Yocto Project using a GUI, see the
-            <ulink url='&YOCTO_DOCS_TOAST_URL;'>Toaster User Manual</ulink>
-            for information on how to install and set up Toaster.
+            You are now ready to give the Yocto Project a try.
+            For this example, you will be using the command line to build
+            your images.
+            <note>
+                A graphical user interface to the Yocto Project is available
+                through Toaster.
+                See the
+                <ulink url='&YOCTO_DOCS_TOAST_URL;'>Toaster User Manual</ulink>
+                for more information.
+            </note>
         </para>
 
         <para>
-            To use the Yocto Project through the command-line interface,
-            finish this quick start, which presents steps that let you
-            do the following:
+            The remainder of this quick start steps you through the
+            following:
             <itemizedlist>
                 <listitem><para>
                     Build a <filename>qemux86</filename> reference image
@@ -538,7 +554,8 @@
 
             <para>
                 <orderedlist>
-                    <listitem><para><emphasis>Be Sure Your Build Host is Set Up:</emphasis>
+                    <listitem><para>
+                        <emphasis>Be Sure Your Build Host is Set Up:</emphasis>
                         The steps to build an image in this section depend on
                         your build host being properly set up.
                         Be sure you have worked through the requirements
@@ -546,7 +563,8 @@
                         "<link linkend='yp-resources'>Setting Up to Use the Yocto Project</link>"
                         section.
                         </para></listitem>
-                    <listitem><para><emphasis>Check Out Your Branch:</emphasis>
+                    <listitem><para>
+                        <emphasis>Check Out Your Branch:</emphasis>
                         Be sure you are in the
                         <ulink url='&YOCTO_DOCS_DEV_URL;#source-directory'>Source Directory</ulink>
                         (e.g. <filename>poky</filename>) and then check out
@@ -566,7 +584,8 @@
                         branch ensures you are using the latest files for
                         that release.
                         </para></listitem>
-                    <listitem><para><emphasis>Initialize the Build Environment:</emphasis>
+                    <listitem><para>
+                        <emphasis>Initialize the Build Environment:</emphasis>
                         Run the
                         <ulink url='&YOCTO_DOCS_REF_URL;#structure-core-script'><filename>&OE_INIT_FILE;</filename></ulink>
                         environment setup script to define the OpenEmbedded
@@ -591,7 +610,8 @@
                             setup script.
                         </note>
                         </para></listitem>
-                    <listitem><para><emphasis>Examine Your Local Configuration File:</emphasis>
+                    <listitem><para>
+                        <emphasis>Examine Your Local Configuration File:</emphasis>
                         When you set up the build environment, a local
                         configuration file named
                         <filename>local.conf</filename> becomes available in
@@ -645,7 +665,8 @@
                                 </para></listitem>
                         </itemizedlist>
                         </para></listitem>
-                    <listitem><para><emphasis>Start the Build:</emphasis>
+                    <listitem><para>
+                        <emphasis>Start the Build:</emphasis>
                         Continue with the following command to build an OS image
                         for the target, which is
                         <filename>core-image-sato</filename> in this example:
@@ -703,7 +724,8 @@
                         "<ulink url='&YOCTO_DOCS_REF_URL;#ref-images'>Images</ulink>"
                         chapter in the Yocto Project Reference Manual.
                         </para></listitem>
-                    <listitem><para><emphasis>Simulate Your Image Using QEMU:</emphasis>
+                    <listitem><para>
+                        <emphasis>Simulate Your Image Using QEMU:</emphasis>
                         Once this particular image is built, you can start QEMU
                         and run the image:
                         <literallayout class='monospaced'>
@@ -713,7 +735,8 @@
                         "<ulink url="&YOCTO_DOCS_DEV_URL;#dev-manual-qemu">Using the Quick EMUlator (QEMU)</ulink>"
                         chapter in the Yocto Project Development Manual.
                         </para></listitem>
-                    <listitem><para><emphasis>Exit QEMU:</emphasis>
+                    <listitem><para>
+                        <emphasis>Exit QEMU:</emphasis>
                         Exit QEMU by either clicking on the shutdown icon or by
                         typing <filename>Ctrl-C</filename> in the QEMU
                         transcript window from which you evoked QEMU.
@@ -743,7 +766,8 @@
 
             <para>
                 <orderedlist>
-                    <listitem><para><emphasis>Create a Local Copy of the
+                    <listitem><para>
+                        <emphasis>Create a Local Copy of the
                         <filename>meta-intel</filename> Repository:</emphasis>
                         Building an image for the MinnowBoard Turbot requires
                         the
@@ -792,7 +816,8 @@
                         "meta-intel-&DISTRO_NAME_NO_CAP;-&YOCTO_DOC_VERSION;"
                         in the above example.
                         </para></listitem>
-                    <listitem><para><emphasis>Configure the Build:</emphasis>
+                    <listitem><para>
+                        <emphasis>Configure the Build:</emphasis>
                         To configure the build, you edit the
                         <filename>bblayers.conf</filename> and
                         <filename>local.conf</filename> files, both of which are
@@ -897,7 +922,8 @@
      tmp/deploy/images/intel-corei7-64/core-image-base-intel-corei7-64.wic
                         </literallayout>
                         </para></listitem>
-                    <listitem><para><emphasis>Write the Image:</emphasis>
+                    <listitem><para>
+                        <emphasis>Write the Image:</emphasis>
                         You can write the image just built to a bootable media
                         (e.g. a USB key, SATA drive, SD card, etc.) using the
                         <filename>dd</filename> utility:
@@ -911,7 +937,8 @@
                         <filename>/dev/mmcblk0</filename>, which is most likely an
                         SD card).
                         </para></listitem>
-                    <listitem><para><emphasis>Boot the Hardware:</emphasis>
+                    <listitem><para>
+                        <emphasis>Boot the Hardware:</emphasis>
                         With the boot device provisioned, you can insert the
                         media into the MinnowBoard Turbot and boot the hardware.
                         The board should automatically detect the media and boot to
@@ -975,7 +1002,7 @@
                     <ulink url='&YOCTO_DOCS_SDK_URL;#sdk-extensible'>extensible SDK</ulink>,
                     which are used primarily for application development.
                     This manual also provides example workflows
-                    that use the popular <trademark class='trad'>Eclipse</trademark>
+                    that use the popular <trademark class='trade'>Eclipse</trademark>
                     development environment and that use <filename>devtool</filename>.
                     See the
                     "<ulink url='&YOCTO_DOCS_SDK_URL;#workflow-using-eclipse'>Workflow using Eclipse™</ulink>"

meta-poky/conf/conf-notes.txt

@@ -1,3 +1,8 @@
+
+### Shell environment set up for builds. ###
+
+You can now run 'bitbake <target>'
+
 Common targets are:
     core-image-minimal
     core-image-sato

meta-poky/conf/distro/include/maintainers.inc

@@ -28,7 +28,9 @@
 # Please keep this list in alphabetical order.
 #
 RECIPE_MAINTAINER_pn-acl = "Chen Qi <Qi.Chen@windriver.com>"
+RECIPE_MAINTAINER_pn-acpica = "Fathi Boudra <fathi.boudra@linaro.org>"
 RECIPE_MAINTAINER_pn-acpid = "Maxin B. John <maxin.john@intel.com>"
+RECIPE_MAINTAINER_pn-acpitests = "Fathi Boudra <fathi.boudra@linaro.org>"
 RECIPE_MAINTAINER_pn-adwaita-icon-theme = "Jussi Kukkonen <jussi.kukkonen@intel.com>"
 RECIPE_MAINTAINER_pn-alsa-lib = "Tanu Kaskinen <tanuk@iki.fi>"
 RECIPE_MAINTAINER_pn-alsa-plugins = "Tanu Kaskinen <tanuk@iki.fi>"
@@ -80,6 +82,7 @@ RECIPE_MAINTAINER_pn-bsd-headers = "Khem Raj <raj.khem@gmail.com>"
 RECIPE_MAINTAINER_pn-btrfs-tools = "Alexander Kanavin <alexander.kanavin@intel.com>"
 RECIPE_MAINTAINER_pn-build-appliance-image = "Cristian Iorga <cristian.iorga@intel.com>"
 RECIPE_MAINTAINER_pn-build-compare = "Randy Witt <randy.e.witt@linux.intel.com>"
+RECIPE_MAINTAINER_pn-build-sysroots = "Richard Purdie <richard.purdie@linuxfoundation.org>"
 RECIPE_MAINTAINER_pn-builder = "Cristian Iorga <cristian.iorga@intel.com>"
 RECIPE_MAINTAINER_pn-buildtools-tarball = "Cristian Iorga <cristian.iorga@intel.com>"
 RECIPE_MAINTAINER_pn-busybox = "Armin Kuster <akuster808@gmail.com>"
@@ -109,7 +112,7 @@ RECIPE_MAINTAINER_pn-consolekit = "Chen Qi <Qi.Chen@windriver.com>"
 RECIPE_MAINTAINER_pn-coreutils = "Chen Qi <Qi.Chen@windriver.com>"
 RECIPE_MAINTAINER_pn-cpio = "Denys Dmytriyenko <denys@ti.com>"
 RECIPE_MAINTAINER_pn-cracklib = "Armin Kuster <akuster@mvista.com>"
-RECIPE_MAINTAINER_pn-createrepo = "Hongxu Jia <hongxu.jia@windriver.com>"
+RECIPE_MAINTAINER_pn-createrepo-c = "Alexander Kanavin <alexander.kanavin@intel.com>"
 RECIPE_MAINTAINER_pn-cronie = "Leonardo Sandoval <leonardo.sandoval.gonzalez@linux.intel.com>"
 RECIPE_MAINTAINER_pn-cross-localedef-native = "Khem Raj <raj.khem@gmail.com>"
 RECIPE_MAINTAINER_pn-cryptodev-linux = "Robert Yang <liezhi.yang@windriver.com>"
@@ -137,6 +140,7 @@ RECIPE_MAINTAINER_pn-distcc = "Hongxu Jia <hongxu.jia@windriver.com>"
 RECIPE_MAINTAINER_pn-distcc-config = "Dengke Du <dengke.du@windriver.com>"
 RECIPE_MAINTAINER_pn-dmidecode = "Alexander Kanavin <alexander.kanavin@intel.com>"
 RECIPE_MAINTAINER_pn-dmxproto = "Armin Kuster <akuster@mvista.com>"
+RECIPE_MAINTAINER_pn-dnf = "Alexander Kanavin <alexander.kanavin@intel.com>"
 RECIPE_MAINTAINER_pn-docbook-xml-dtd4 = "Dengke Du <dengke.du@windriver.com>"
 RECIPE_MAINTAINER_pn-docbook-xsl-stylesheets = "Dengke Du <dengke.du@windriver.com>"
 RECIPE_MAINTAINER_pn-dosfstools = "Dengke Du <dengke.du@windriver.com>"
@@ -216,8 +220,11 @@ RECIPE_MAINTAINER_pn-gnu-config = "Robert Yang <liezhi.yang@windriver.com>"
 RECIPE_MAINTAINER_pn-gnu-efi = "Robert Yang <liezhi.yang@windriver.com>"
 RECIPE_MAINTAINER_pn-gnupg = "Hongxu Jia <hongxu.jia@windriver.com>"
 RECIPE_MAINTAINER_pn-gnutls = "Armin Kuster <akuster@mvista.com>"
+RECIPE_MAINTAINER_pn_go = "Khem Raj <raj.khem@gmail.com>"
+RECIPE_MAINTAINER_pn-go-cross = "Khem Raj <raj.khem@gmail.com>"
+RECIPE_MAINTAINER_pn-go-helloworld = "Khem Raj <raj.khem@gmail.com>"
+RECIPE_MAINTAINER_pn-go-native = "Khem Raj <raj.khem@gmail.com>"
 RECIPE_MAINTAINER_pn-gobject-introspection = "Alexander Kanavin <alexander.kanavin@intel.com>"
-RECIPE_MAINTAINER_pn-go-native = "Armin Kuster <akuster808@gmail.com>"
 RECIPE_MAINTAINER_pn-gperf = "Alexander Kanavin <alexander.kanavin@intel.com>"
 RECIPE_MAINTAINER_pn-gpgme = "Hongxu Jia <hongxu.jia@windriver.com>"
 RECIPE_MAINTAINER_pn-gptfdisk = "Alexander Kanavin <alexander.kanavin@intel.com>"
@@ -289,6 +296,7 @@ RECIPE_MAINTAINER_pn-kexec-tools = "Armin Kuster <akuster@mvista.com>"
 RECIPE_MAINTAINER_pn-keymaps = "Alexander Kanavin <alexander.kanavin@intel.com>"
 RECIPE_MAINTAINER_pn-kmod = "Chen Qi <Qi.Chen@windriver.com>"
 RECIPE_MAINTAINER_pn-kmod-native = "Chen Qi <Qi.Chen@windriver.com>"
+RECIPE_MAINTAINER_pn-kmscube = "Carlos Rafael Giani <dv@pseudoterminal.org>"
 RECIPE_MAINTAINER_pn-l3afpad = "Jussi Kukkonen <jussi.kukkonen@intel.com>"
 RECIPE_MAINTAINER_pn-lame = "Tanu Kaskinen <tanuk@iki.fi>"
 RECIPE_MAINTAINER_pn-latencytop = "Alexander Kanavin <alexander.kanavin@intel.com>"
@@ -308,10 +316,12 @@ RECIPE_MAINTAINER_pn-libcap-ng = "Robert Yang <liezhi.yang@windriver.com>"
 RECIPE_MAINTAINER_pn-libcgroup = "Alexander Kanavin <alexander.kanavin@intel.com>"
 RECIPE_MAINTAINER_pn-libcheck = "Robert Yang <liezhi.yang@windriver.com>"
 RECIPE_MAINTAINER_pn-libclass-isa-perl = "Aníbal Limón <anibal.limon@linux.intel.com>"
+RECIPE_MAINTAINER_pn-libcomps = "Alexander Kanavin <alexander.kanavin@intel.com>"
 RECIPE_MAINTAINER_pn-libconvert-asn1-perl = "Aníbal Limón <anibal.limon@linux.intel.com>"
 RECIPE_MAINTAINER_pn-libcroco = "Jussi Kukkonen <jussi.kukkonen@intel.com>"
 RECIPE_MAINTAINER_pn-libdaemon = "Alexander Kanavin <alexander.kanavin@intel.com>"
 RECIPE_MAINTAINER_pn-libdmx = "Armin Kuster <akuster@mvista.com>"
+RECIPE_MAINTAINER_pn-libdnf = "Alexander Kanavin <alexander.kanavin@intel.com>"
 RECIPE_MAINTAINER_pn-libdrm = "Otavio Salvador <otavio.salvador@ossystems.com.br>"
 RECIPE_MAINTAINER_pn-libdumpvalue-perl = "Aníbal Limón <anibal.limon@linux.intel.com>"
 RECIPE_MAINTAINER_pn-libenv-perl = "Aníbal Limón <anibal.limon@linux.intel.com>"
@@ -351,6 +361,7 @@ RECIPE_MAINTAINER_pn-libnewt-python = "Hongxu Jia <hongxu.jia@windriver.com>"
 RECIPE_MAINTAINER_pn-libnfsidmap = "Maxin B. John <maxin.john@intel.com>"
 RECIPE_MAINTAINER_pn-libnl = "Alexander Kanavin <alexander.kanavin@intel.com>"
 RECIPE_MAINTAINER_pn-libnotify = "Jussi Kukkonen <jussi.kukkonen@intel.com>"
+RECIPE_MAINTAINER_pn-libnsl2 = "Khem Raj <raj.khem@gmail.com>"
 RECIPE_MAINTAINER_pn-libnss-mdns = "Alexander Kanavin <alexander.kanavin@intel.com>"
 RECIPE_MAINTAINER_pn-libogg = "Maxin B. John <maxin.john@intel.com>"
 RECIPE_MAINTAINER_pn-libomxil = "Maxin B. John <maxin.john@intel.com>"
@@ -366,6 +377,7 @@ RECIPE_MAINTAINER_pn-libpng12 = "Maxin B. John <maxin.john@intel.com>"
 RECIPE_MAINTAINER_pn-libpod-plainer-perl = "Aníbal Limón <anibal.limon@linux.intel.com>"
 RECIPE_MAINTAINER_pn-libproxy = "Maxin B. John <maxin.john@intel.com>"
 RECIPE_MAINTAINER_pn-libpthread-stubs = "Alexander Kanavin <alexander.kanavin@intel.com>"
+RECIPE_MAINTAINER_pn-librepo = "Alexander Kanavin <alexander.kanavin@intel.com>"
 RECIPE_MAINTAINER_pn-librsvg = "Jussi Kukkonen <jussi.kukkonen@intel.com>"
 RECIPE_MAINTAINER_pn-libsamplerate0 = "Tanu Kaskinen <tanuk@iki.fi>"
 RECIPE_MAINTAINER_pn-libsdl = "Robert Yang <liezhi.yang@windriver.com>"
@@ -536,6 +548,8 @@ RECIPE_MAINTAINER_pn-oprofile = "Alexander Kanavin <alexander.kanavin@intel.com>
 RECIPE_MAINTAINER_pn-orc = "Maxin B. John <maxin.john@intel.com>"
 RECIPE_MAINTAINER_pn-os-release = "Ross Burton <ross.burton@intel.com>"
 RECIPE_MAINTAINER_pn-ossp-uuid = "Alexander Kanavin <alexander.kanavin@intel.com>"
+RECIPE_MAINTAINER_pn-ovmf = "Ricardo Neri <ricardo.neri-calderon@linux.intel.com>"
+RECIPE_MAINTAINER_pn-ovmf-shell-image = "Ricardo Neri <ricardo.neri-calderon@linux.intel.com>"
 RECIPE_MAINTAINER_pn-p11-kit = "Alexander Kanavin <alexander.kanavin@intel.com>"
 RECIPE_MAINTAINER_pn-package-index = "Ross Burton <ross.burton@intel.com>"
 RECIPE_MAINTAINER_pn-pango = "Jussi Kukkonen <jussi.kukkonen@intel.com>"
@@ -606,6 +620,7 @@ RECIPE_MAINTAINER_pn-python3-distribute = "Jose Lamego <jose.a.lamego@linux.inte
 RECIPE_MAINTAINER_pn-python3-docutils = "Jose Lamego <jose.a.lamego@linux.intel.com>"
 RECIPE_MAINTAINER_pn-python3-git = "Jose Lamego <jose.a.lamego@linux.intel.com>"
 RECIPE_MAINTAINER_pn-python3-gitdb = "Jose Lamego <jose.a.lamego@linux.intel.com>"
+RECIPE_MAINTAINER_pn-python3-iniparse = "Jose Lamego <jose.a.lamego@linux.intel.com>"
 RECIPE_MAINTAINER_pn-python3-mako = "Jose Lamego <jose.a.lamego@linux.intel.com>"
 RECIPE_MAINTAINER_pn-python3-native = "Alejandro Hernandez <alejandro.hernandez@linux.intel.com>"
 RECIPE_MAINTAINER_pn-python3-nose = "Jose Lamego <jose.a.lamego@linux.intel.com>"
@@ -732,6 +747,7 @@ RECIPE_MAINTAINER_pn-weston = "Denys Dmytriyenko <denys@ti.com>"
 RECIPE_MAINTAINER_pn-weston-init = "Denys Dmytriyenko <denys@ti.com>"
 RECIPE_MAINTAINER_pn-wget = "Robert Yang <liezhi.yang@windriver.com>"
 RECIPE_MAINTAINER_pn-which = "Leonardo Sandoval <leonardo.sandoval.gonzalez@linux.intel.com>"
+RECIPE_MAINTAINER_pn-wic-tools = "Ed Bartosh <ed.bartosh@linux.intel.com>"
 RECIPE_MAINTAINER_pn-wireless-tools = "Changhyeok Bae <changhyeok.bae@lge.com>"
 RECIPE_MAINTAINER_pn-wpa-supplicant = "Changhyeok Bae <changhyeok.bae@lge.com>"
 RECIPE_MAINTAINER_pn-x11-common = "Armin Kuster <akuster@mvista.com>"

meta-poky/conf/distro/include/poky-world-exclude.inc

@@ -2,7 +2,3 @@
 # Things we exlude fromw world testing within the reference distro
 #
 
-# qwt from meta-qt4, has poky-lsb QA warnings, qt4 for lsb only
-EXCLUDE_FROM_WORLD_pn-qwt = "1"
-# python-pyqt from meta-qt4 requires sip from meta-oe
-EXCLUDE_FROM_WORLD_pn-python-pyqt = "1"

meta-poky/conf/distro/poky-lsb.conf

@@ -12,4 +12,4 @@ PREFERRED_PROVIDER_virtual/libx11 = "libx11"
 KERNEL_FEATURES_append_pn-linux-yocto = " features/nfsd/nfsd-enable.scc"
 
 # Use the LTSI Kernel for LSB Testing
-PREFERRED_VERSION_linux-yocto_linuxstdbase ?= "4.1%"
+PREFERRED_VERSION_linux-yocto_linuxstdbase ?= "4.9%"

meta-poky/conf/distro/poky.conf

@@ -15,13 +15,13 @@ DISTRO_VERSION[vardepsexclude] = "DATE"
 SDK_VERSION[vardepsexclude] = "DATE"
 
 # Override these in poky based distros
-POKY_DEFAULT_DISTRO_FEATURES = "largefile opengl ptest multiarch wayland"
+POKY_DEFAULT_DISTRO_FEATURES = "largefile opengl ptest multiarch wayland vulkan"
 POKY_DEFAULT_EXTRA_RDEPENDS = "packagegroup-core-boot"
 POKY_DEFAULT_EXTRA_RRECOMMENDS = "kernel-module-af-packet"
 
 DISTRO_FEATURES ?= "${DISTRO_FEATURES_DEFAULT} ${DISTRO_FEATURES_LIBC} ${POKY_DEFAULT_DISTRO_FEATURES}"
 
-PREFERRED_VERSION_linux-yocto ?= "4.10%"
+PREFERRED_VERSION_linux-yocto ?= "4.12%"
 
 SDK_NAME = "${DISTRO}-${TCLIBC}-${SDK_ARCH}-${IMAGE_BASENAME}-${TUNE_PKGARCH}"
 SDKPATH = "/opt/${DISTRO}/${SDK_VERSION}"
@@ -39,7 +39,7 @@ DISTRO_EXTRA_RDEPENDS_append_qemux86-64 = " ${POKYQEMUDEPS}"
 
 TCLIBCAPPEND = ""
 
-QEMU_TARGETS ?= "arm aarch64 i386 mips mipsel mips64 mips64el ppc x86_64"
+QEMU_TARGETS ?= "arm aarch64 i386 mips mipsel mips64 mips64el nios2 ppc x86_64"
 # Other QEMU_TARGETS "sh4"
 
 PREMIRRORS ??= "\

meta-selftest/files/distrodata/checkpkg_exceptions

@@ -1,27 +0,0 @@
-build-compare
-calibrateproto
-dbus-wait
-distcc
-fstests
-gnu-config
-gst-player
-kmscube
-libxcalibrate
-linux-firmware
-lsb
-mmc-utils
-ovmf
-piglit
-psplash
-puzzles
-rpm
-squashfs-tools
-tcf-agent
-unzip
-update-rc.d
-vulkan-demos
-x264
-xcursor-transparent-theme
-xinetd
-xvideo-tests
-zip

meta-selftest/files/signing/key.passphrase

@@ -0,0 +1 @@
+test123

meta-selftest/lib/oeqa/runtime/cases/dnf_runtime.py

@@ -6,6 +6,8 @@ class DnfSelftest(DnfTest):
 
     @classmethod
     def setUpClass(cls):
+        import tempfile
+        cls.temp_dir = tempfile.TemporaryDirectory(prefix="oeqa-remotefeeds-")
         cls.repo_server = HTTPService(os.path.join(cls.tc.td['WORKDIR'], 'oe-rootfs-repo'),
                                       cls.tc.target.server_ip)
         cls.repo_server.start()
@@ -13,6 +15,7 @@ class DnfSelftest(DnfTest):
     @classmethod
     def tearDownClass(cls):
         cls.repo_server.stop()
+        cls.temp_dir.cleanup()
 
     @OETestDepends(['dnf.DnfBasicTest.test_dnf_help'])
     def test_verify_package_feeds(self):
@@ -25,18 +28,20 @@ class DnfSelftest(DnfTest):
         """
         # When we created an image, we had to supply fake ip and port
         # for the feeds. Now we can patch the real ones into the config file.
-        import tempfile
-        temp_file = tempfile.TemporaryDirectory(prefix="oeqa-remotefeeds-").name
+        temp_file = os.path.join(self.temp_dir.name, 'tmp.repo')
         self.tc.target.copyFrom("/etc/yum.repos.d/oe-remote-repo.repo", temp_file)
         fixed_config = open(temp_file, "r").read().replace("bogus_ip", self.tc.target.server_ip).replace("bogus_port", str(self.repo_server.port))
-        open(temp_file, "w").write(fixed_config)
+        with open(temp_file, "w") as f:
+            f.write(fixed_config)
         self.tc.target.copyTo(temp_file, "/etc/yum.repos.d/oe-remote-repo.repo")
 
         import re
-        output_makecache = self.dnf('makecache')
+        # Use '-y' for non-interactive mode: automatically import the feed signing key
+        output_makecache = self.dnf('-vy makecache')
+        self.assertTrue(re.match(r".*Failed to synchronize cache", output_makecache, re.DOTALL) is None, msg = "dnf makecache failed to synchronize repo: %s" %(output_makecache))
         self.assertTrue(re.match(r".*Metadata cache created", output_makecache, re.DOTALL) is not None, msg = "dnf makecache failed: %s" %(output_makecache))
 
-        output_repoinfo = self.dnf('repoinfo')
+        output_repoinfo = self.dnf('-v repoinfo')
         matchobj = re.match(r".*Repo-pkgs\s*:\s*(?P<n_pkgs>[0-9]+)", output_repoinfo, re.DOTALL)
         self.assertTrue(matchobj is not None, msg = "Could not find the amount of packages in dnf repoinfo output: %s" %(output_repoinfo))
         self.assertTrue(int(matchobj.group('n_pkgs')) > 0, msg = "Amount of remote packages is not more than zero: %s\n" %(output_repoinfo))

meta-yocto-bsp/recipes-kernel/linux/linux-yocto_4.1.bbappend

@@ -1,26 +0,0 @@
-KBRANCH_genericx86  = "standard/base"
-KBRANCH_genericx86-64  = "standard/base"
-KBRANCH_edgerouter = "standard/edgerouter"
-KBRANCH_beaglebone = "standard/beaglebone"
-KBRANCH_mpc8315e-rdb = "standard/fsl-mpc8315e-rdb"
-
-KMACHINE_genericx86 ?= "common-pc"
-KMACHINE_genericx86-64 ?= "common-pc-64"
-
-SRCREV_machine_genericx86    ?= "f905fdd5da150ea809f847f00f3476220606c0ff"
-SRCREV_machine_genericx86-64 ?= "f905fdd5da150ea809f847f00f3476220606c0ff"
-SRCREV_machine_edgerouter ?= "e2bea2e96fcbbbb9b1efc3bc7505ac42e39bb5c0"
-SRCREV_machine_beaglebone ?= "243644adea596cda2ef1280a8e57075a1247089f"
-SRCREV_machine_mpc8315e-rdb ?= "e2bea2e96fcbbbb9b1efc3bc7505ac42e39bb5c0"
-
-COMPATIBLE_MACHINE_genericx86 = "genericx86"
-COMPATIBLE_MACHINE_genericx86-64 = "genericx86-64"
-COMPATIBLE_MACHINE_edgerouter = "edgerouter"
-COMPATIBLE_MACHINE_beaglebone = "beaglebone"
-COMPATIBLE_MACHINE_mpc8315e-rdb = "mpc8315e-rdb"
-
-LINUX_VERSION_genericx86 = "4.1.42"
-LINUX_VERSION_genericx86-64 = "4.1.42"
-LINUX_VERSION_edgerouter = "4.1.42"
-LINUX_VERSION_beaglebone = "4.1.42"
-LINUX_VERSION_mpc8315e-rdb = "4.1.42"

meta/classes/autotools.bbclass

@@ -141,7 +141,7 @@ ACLOCALEXTRAPATH_class-nativesdk = " -I ${STAGING_DATADIR_NATIVE}/aclocal/"
 
 python autotools_aclocals () {
     # Refresh variable with cache files
-    d.setVar("CONFIG_SITE", siteinfo_get_files(d, aclocalcache=True))
+    d.setVar("CONFIG_SITE", siteinfo_get_files(d, sysrootcache=True))
 }
 
 CONFIGURE_FILES = "${S}/configure.in ${S}/configure.ac ${S}/config.h.in ${S}/acinclude.m4 Makefile.am"

meta/classes/buildhistory.bbclass

@@ -824,6 +824,8 @@ python buildhistory_eventhandler() {
                 interrupted = getattr(e, '_interrupted', 0)
                 localdata.setVar('BUILDHISTORY_BUILD_INTERRUPTED', str(interrupted))
                 bb.build.exec_func("buildhistory_commit", localdata)
+            else:
+                bb.note("No commit since BUILDHISTORY_COMMIT != '1'")
 }
 
 addhandler buildhistory_eventhandler
@@ -910,3 +912,33 @@ def write_latest_srcrev(d, pkghistdir):
     else:
         if os.path.exists(srcrevfile):
             os.remove(srcrevfile)
+
+do_testimage[postfuncs] += "write_ptest_result"
+do_testimage[vardepsexclude] += "write_ptest_result"
+
+python write_ptest_result() {
+    write_latest_ptest_result(d, d.getVar('BUILDHISTORY_DIR'))
+}
+
+def write_latest_ptest_result(d, histdir):
+    import glob
+    import subprocess
+    test_log_dir = d.getVar('TEST_LOG_DIR')
+    input_ptest = os.path.join(test_log_dir, 'ptest_log')
+    output_ptest = os.path.join(histdir, 'ptest')
+    if os.path.exists(input_ptest):
+        try:
+            # Lock it avoid race issue
+            lock = bb.utils.lockfile(output_ptest + "/ptest.lock")
+            bb.utils.mkdirhier(output_ptest)
+            oe.path.copytree(input_ptest, output_ptest)
+            # Sort test result
+            for result in glob.glob('%s/pass.fail.*' % output_ptest):
+                bb.debug(1, 'Processing %s' % result)
+                cmd = ['sort', result, '-o', result]
+                bb.debug(1, 'Running %s' % cmd)
+                ret = subprocess.call(cmd)
+                if ret != 0:
+                    bb.error('Failed to run %s!' % cmd)
+        finally:
+            bb.utils.unlockfile(lock)

meta/classes/cross-canadian.bbclass

@@ -38,7 +38,7 @@ python () {
     extralibcs = [""]
     if "musl" in d.getVar("BASECANADIANEXTRAOS"):
         extralibcs.append("musl")
-    for variant in ["", "spe", "x32", "eabi", "n32"]:
+    for variant in ["", "spe", "x32", "eabi", "n32", "ilp32"]:
         for libc in extralibcs:
             entry = "linux"
             if variant and libc:

meta/classes/distrodata.bbclass

@@ -261,12 +261,44 @@ python do_checkpkg() {
         from bb.utils import vercmp_string
         from bb.fetch2 import FetchError, NoMethodError, decodeurl
 
-        """first check whether a uri is provided"""
-        src_uri = (d.getVar('SRC_URI') or '').split()
-        if src_uri:
-            uri_type, _, _, _, _, _ = decodeurl(src_uri[0])
-        else:
-            uri_type = "none"
+        def get_upstream_version_and_status():
+
+            # set if the upstream check fails reliably, e.g. absent git tags, or weird version format used on our or on upstream side.
+            upstream_version_unknown = localdata.getVar('UPSTREAM_VERSION_UNKNOWN')
+            # set if the upstream check cannot be reliably performed due to transient network failures, or server behaving weirdly. 
+            # This one should be used sparingly, as it completely excludes a recipe from upstream checking.
+            upstream_check_unreliable = localdata.getVar('UPSTREAM_CHECK_UNRELIABLE')
+
+            if upstream_check_unreliable == "1":
+                return "N/A", "CHECK_IS_UNRELIABLE"
+
+            try:
+                uv = oe.recipeutils.get_recipe_upstream_version(localdata)
+                pupver = uv['version'] if uv['version'] else "N/A"
+            except Exception as e:
+                pupver = "N/A"
+
+            if pupver == "N/A":
+                pstatus = "UNKNOWN" if upstream_version_unknown else "UNKNOWN_BROKEN"
+            else:
+                src_uri = (localdata.getVar('SRC_URI') or '').split()
+                if src_uri:
+                    uri_type, _, _, _, _, _ = decodeurl(src_uri[0])
+                else:
+                    uri_type = "none"
+                pv, _, _ = oe.recipeutils.get_recipe_pv_without_srcpv(pversion, uri_type)
+                upv, _, _ = oe.recipeutils.get_recipe_pv_without_srcpv(pupver, uri_type)
+
+                cmp = vercmp_string(pv, upv)
+                if cmp == -1:
+                    pstatus = "UPDATE" if not upstream_version_unknown else "KNOWN_BROKEN"
+                elif cmp == 0:
+                    pstatus = "MATCH" if not upstream_version_unknown else "KNOWN_BROKEN"
+                else:
+                    pstatus = "UNKNOWN" if upstream_version_unknown else "UNKNOWN_BROKEN"
+
+            return pupver, pstatus
+
 
         """initialize log files."""
         logpath = d.getVar('LOG_DIR')
@@ -313,34 +345,7 @@ python do_checkpkg() {
         psrcuri = localdata.getVar('SRC_URI')
         maintainer = localdata.getVar('RECIPE_MAINTAINER')
 
-        """ Get upstream version version """
-        pupver = ""
-        pstatus = ""
-
-        try:
-            uv = oe.recipeutils.get_recipe_upstream_version(localdata)
-
-            pupver = uv['version']
-        except Exception as e:
-            if e is FetchError:
-                pstatus = "ErrAccess"
-            elif e is NoMethodError:
-                pstatus = "ErrUnsupportedProto"
-            else:
-                pstatus = "ErrUnknown"
-
-        """Set upstream version status"""
-        if not pupver:
-            pupver = "N/A"
-        else:
-            pv, _, _ = oe.recipeutils.get_recipe_pv_without_srcpv(pversion, uri_type)
-            upv, _, _ = oe.recipeutils.get_recipe_pv_without_srcpv(pupver, uri_type)
-
-            cmp = vercmp_string(pv, upv)
-            if cmp == -1:
-                pstatus = "UPDATE"
-            elif cmp == 0:
-                pstatus = "MATCH"
+        pupver, pstatus = get_upstream_version_and_status()
 
         if psrcuri:
             psrcuri = psrcuri.split()[0]

meta/classes/externalsrc.bbclass

@@ -184,11 +184,19 @@ def srctree_hash_files(d, srcdir=None):
     import tempfile
 
     s_dir = srcdir or d.getVar('EXTERNALSRC')
-    git_dir = os.path.join(s_dir, '.git')
-    oe_hash_file = os.path.join(git_dir, 'oe-devtool-tree-sha1')
+    git_dir = None
+
+    try:
+        # git rev-parse returns the path relative to the current working
+        # directory
+        git_dir = os.path.join(s_dir,
+            subprocess.check_output(['git', 'rev-parse', '--git-dir'], cwd=s_dir).decode("utf-8").rstrip())
+    except subprocess.CalledProcessError:
+        pass
 
     ret = " "
-    if os.path.exists(git_dir):
+    if git_dir is not None:
+        oe_hash_file = os.path.join(git_dir, 'oe-devtool-tree-sha1')
         with tempfile.NamedTemporaryFile(prefix='oe-devtool-index') as tmp_index:
             # Clone index
             shutil.copyfile(os.path.join(git_dir, 'index'), tmp_index.name)

meta/classes/gettext.bbclass

@@ -17,3 +17,8 @@ DEPENDS_GETTEXT ??= "virtual/gettext gettext-native"
 
 BASEDEPENDS_append = " ${@gettext_dependencies(d)}"
 EXTRA_OECONF_append = " ${@gettext_oeconf(d)}"
+
+# Without this, msgfmt from gettext-native will not find ITS files
+# provided by target recipes (for example, polkit.its).
+GETTEXTDATADIRS_append_class-target = ":${STAGING_DATADIR}/gettext"
+export GETTEXTDATADIRS

meta/classes/goarch.bbclass

@@ -14,6 +14,7 @@ GO_BUILD_BINDIR = "${@['bin/${HOST_GOTUPLE}','bin'][d.getVar('BUILD_GOTUPLE',Tru
 # define here because everybody inherits this class
 #
 COMPATIBLE_HOST_linux-gnux32 = "null"
+COMPATIBLE_HOST_linux-muslx32 = "null"
 COMPATIBLE_HOST_powerpc = "null"
 COMPATIBLE_HOST_powerpc64 = "null"
 

meta/classes/grub-efi.bbclass

@@ -17,7 +17,6 @@
 # ${GRUB_ROOT} - grub's root device.
 
 do_bootimg[depends] += "${MLPREFIX}grub-efi:do_deploy"
-do_bootdirectdisk[depends] += "${MLPREFIX}grub-efi:do_deploy"
 
 GRUB_SERIAL ?= "console=ttyS0,115200"
 GRUB_CFG_VM = "${S}/grub_vm.cfg"

meta/classes/image-prelink.bbclass

@@ -1,6 +1,6 @@
 do_rootfs[depends] += "prelink-native:do_populate_sysroot"
 
-IMAGE_PREPROCESS_COMMAND += "prelink_setup; prelink_image; "
+IMAGE_PREPROCESS_COMMAND_append_libc-glibc = " prelink_setup; prelink_image; "
 
 python prelink_setup () {
     oe.utils.write_ld_so_conf(d)
@@ -36,7 +36,17 @@ prelink_image () {
 	dynamic_loader=$(linuxloader)
 
 	# prelink!
-	${STAGING_SBINDIR_NATIVE}/prelink --root ${IMAGE_ROOTFS} -amR -N -c ${sysconfdir}/prelink.conf --dynamic-linker $dynamic_loader
+	if [ "$BUILD_REPRODUCIBLE_BINARIES" = "1" ]; then
+		bbnote " prelink: BUILD_REPRODUCIBLE_BINARIES..."
+		if [ "$REPRODUCIBLE_TIMESTAMP_ROOTFS" = "" ]; then
+			export PRELINK_TIMESTAMP=`git log -1 --pretty=%ct `
+		else
+			export PRELINK_TIMESTAMP=$REPRODUCIBLE_TIMESTAMP_ROOTFS
+		fi
+		${STAGING_SBINDIR_NATIVE}/prelink --root ${IMAGE_ROOTFS} -am -N -c ${sysconfdir}/prelink.conf --dynamic-linker $dynamic_loader
+	else
+		${STAGING_SBINDIR_NATIVE}/prelink --root ${IMAGE_ROOTFS} -amR -N -c ${sysconfdir}/prelink.conf --dynamic-linker $dynamic_loader
+	fi
 
 	# Remove the prelink.conf if we had to add it.
 	if [ "$dummy_prelink_conf" = "true" ]; then

meta/classes/image-vm.bbclass

@@ -1,171 +0,0 @@
-# image-vm.bbclass
-# (loosly based off image-live.bbclass Copyright (C) 2004, Advanced Micro Devices, Inc.)
-#
-# Create an image which can be placed directly onto a harddisk using dd and then
-# booted.
-#
-# This uses syslinux. extlinux would have been nice but required the ext2/3
-# partition to be mounted. grub requires to run itself as part of the install
-# process.
-#
-# The end result is a 512 boot sector populated with an MBR and partition table
-# followed by an msdos fat16 partition containing syslinux and a linux kernel
-# completed by the ext2/3 rootfs.
-#
-# We have to push the msdos parition table size > 16MB so fat 16 is used as parted
-# won't touch fat12 partitions.
-
-inherit live-vm-common
-
-do_bootdirectdisk[depends] += "dosfstools-native:do_populate_sysroot \
-                               virtual/kernel:do_deploy \
-                               syslinux:do_populate_sysroot \
-                               syslinux-native:do_populate_sysroot \
-                               parted-native:do_populate_sysroot \
-                               mtools-native:do_populate_sysroot \
-                               ${PN}:do_image_${VM_ROOTFS_TYPE} \
-                               "
-
-IMAGE_TYPEDEP_vmdk = "${VM_ROOTFS_TYPE}"
-IMAGE_TYPEDEP_vdi = "${VM_ROOTFS_TYPE}"
-IMAGE_TYPEDEP_qcow2 = "${VM_ROOTFS_TYPE}"
-IMAGE_TYPEDEP_hdddirect = "${VM_ROOTFS_TYPE}"
-IMAGE_TYPES_MASKED += "vmdk vdi qcow2 hdddirect"
-
-VM_ROOTFS_TYPE ?= "ext4"
-ROOTFS ?= "${IMGDEPLOYDIR}/${IMAGE_LINK_NAME}.${VM_ROOTFS_TYPE}"
-
-# Used by bootloader
-LABELS_VM ?= "boot"
-ROOT_VM ?= "root=/dev/sda2"
-# Using an initramfs is optional. Enable it by setting INITRD_IMAGE_VM.
-INITRD_IMAGE_VM ?= ""
-INITRD_VM ?= "${@'${IMGDEPLOYDIR}/${INITRD_IMAGE_VM}-${MACHINE}.cpio.gz' if '${INITRD_IMAGE_VM}' else ''}"
-do_bootdirectdisk[depends] += "${@'${INITRD_IMAGE_VM}:do_image_complete' if '${INITRD_IMAGE_VM}' else ''}"
-
-BOOTDD_VOLUME_ID   ?= "boot"
-BOOTDD_EXTRA_SPACE ?= "16384"
-
-DISK_SIGNATURE ?= "${DISK_SIGNATURE_GENERATED}"
-DISK_SIGNATURE[vardepsexclude] = "DISK_SIGNATURE_GENERATED"
-
-build_boot_dd() {
-	HDDDIR="${S}/hdd/boot"
-	HDDIMG="${S}/hdd.image"
-	IMAGE=${IMGDEPLOYDIR}/${IMAGE_NAME}.hdddirect
-
-	populate_kernel $HDDDIR
-
-	if [ "${PCBIOS}" = "1" ]; then
-		syslinux_hddimg_populate $HDDDIR
-	fi
-	if [ "${EFI}" = "1" ]; then
-		efi_hddimg_populate $HDDDIR
-	fi
-
-	BLOCKS=`du -bks $HDDDIR | cut -f 1`
-	BLOCKS=`expr $BLOCKS + ${BOOTDD_EXTRA_SPACE}`
-
-	# Remove it since mkdosfs would fail when it exists
-	rm -f $HDDIMG
-	mkdosfs -n ${BOOTDD_VOLUME_ID} ${MKDOSFS_EXTRAOPTS} -C $HDDIMG $BLOCKS
-	mcopy -i $HDDIMG -s $HDDDIR/* ::/
-
-	if [ "${PCBIOS}" = "1" ]; then
-		syslinux_hdddirect_install $HDDIMG
-	fi	
-	chmod 644 $HDDIMG
-
-	ROOTFSBLOCKS=`du -Lbks ${ROOTFS} | cut -f 1`
-	TOTALSIZE=`expr $BLOCKS + $ROOTFSBLOCKS`
-	END1=`expr $BLOCKS \* 1024`
-	END2=`expr $END1 + 512`
-	END3=`expr \( $ROOTFSBLOCKS \* 1024 \) + $END1`
-
-	echo $ROOTFSBLOCKS $TOTALSIZE $END1 $END2 $END3
-	rm -rf $IMAGE
-	dd if=/dev/zero of=$IMAGE bs=1024 seek=$TOTALSIZE count=1
-
-	parted $IMAGE mklabel msdos
-	parted $IMAGE mkpart primary fat16 0 ${END1}B
-	parted $IMAGE unit B mkpart primary ext2 ${END2}B ${END3}B
-	parted $IMAGE set 1 boot on 
-
-	parted $IMAGE print
-
-	awk "BEGIN { printf \"$(echo ${DISK_SIGNATURE} | sed 's/\(..\)\(..\)\(..\)\(..\)/\\x\4\\x\3\\x\2\\x\1/')\" }" | \
-		dd of=$IMAGE bs=1 seek=440 conv=notrunc
-
-	OFFSET=`expr $END2 / 512`
-	if [ "${PCBIOS}" = "1" ]; then
-		dd if=${STAGING_DATADIR}/syslinux/mbr.bin of=$IMAGE conv=notrunc
-	fi
-
-	dd if=$HDDIMG of=$IMAGE conv=notrunc seek=1 bs=512
-	dd if=${ROOTFS} of=$IMAGE conv=notrunc seek=$OFFSET bs=512
-
-	cd ${IMGDEPLOYDIR}
-
-	ln -sf ${IMAGE_NAME}.hdddirect ${IMGDEPLOYDIR}/${IMAGE_LINK_NAME}.hdddirect
-} 
-
-python do_bootdirectdisk() {
-    validate_disk_signature(d)
-    set_live_vm_vars(d, 'VM')
-    if d.getVar("PCBIOS") == "1":
-        bb.build.exec_func('build_syslinux_cfg', d)
-    if d.getVar("EFI") == "1":
-        bb.build.exec_func('build_efi_cfg', d)
-    bb.build.exec_func('build_boot_dd', d)
-}
-
-def generate_disk_signature():
-    import uuid
-
-    signature = str(uuid.uuid4())[:8]
-
-    if signature != '00000000':
-        return signature
-    else:
-        return 'ffffffff'
-
-def validate_disk_signature(d):
-    import re
-
-    disk_signature = d.getVar("DISK_SIGNATURE")
-
-    if not re.match(r'^[0-9a-fA-F]{8}$', disk_signature):
-        bb.fatal("DISK_SIGNATURE '%s' must be an 8 digit hex string" % disk_signature)
-
-DISK_SIGNATURE_GENERATED := "${@generate_disk_signature()}"
-
-run_qemu_img (){
-    type="$1"
-    qemu-img convert -O $type ${IMGDEPLOYDIR}/${IMAGE_LINK_NAME}.hdddirect ${IMGDEPLOYDIR}/${IMAGE_NAME}.$type
-
-    ln -sf ${IMAGE_NAME}.$type ${IMGDEPLOYDIR}/${IMAGE_LINK_NAME}.$type
-}
-create_vmdk_image () {
-    run_qemu_img vmdk
-}
-
-create_vdi_image () {
-    run_qemu_img vdi
-}
-
-create_qcow2_image () {
-    run_qemu_img qcow2
-}
-
-python do_vmimg() {
-    if 'vmdk' in d.getVar('IMAGE_FSTYPES'):
-        bb.build.exec_func('create_vmdk_image', d)
-    if 'vdi' in d.getVar('IMAGE_FSTYPES'):
-        bb.build.exec_func('create_vdi_image', d)
-    if 'qcow2' in d.getVar('IMAGE_FSTYPES'):
-        bb.build.exec_func('create_qcow2_image', d)
-}
-
-addtask bootdirectdisk before do_vmimg
-addtask vmimg after do_bootdirectdisk before do_image_complete
-do_vmimg[depends] += "qemu-native:do_populate_sysroot"

meta/classes/image.bbclass

@@ -117,7 +117,7 @@ def rootfs_variables(d):
                  'IMAGE_ROOTFS_MAXSIZE','IMAGE_NAME','IMAGE_LINK_NAME','IMAGE_MANIFEST','DEPLOY_DIR_IMAGE','IMAGE_FSTYPES','IMAGE_INSTALL_COMPLEMENTARY','IMAGE_LINGUAS',
                  'MULTILIBRE_ALLOW_REP','MULTILIB_TEMP_ROOTFS','MULTILIB_VARIANTS','MULTILIBS','ALL_MULTILIB_PACKAGE_ARCHS','MULTILIB_GLOBAL_VARIANTS','BAD_RECOMMENDATIONS','NO_RECOMMENDATIONS',
                  'PACKAGE_ARCHS','PACKAGE_CLASSES','TARGET_VENDOR','TARGET_ARCH','TARGET_OS','OVERRIDES','BBEXTENDVARIANT','FEED_DEPLOYDIR_BASE_URI','INTERCEPT_DIR','USE_DEVFS',
-                 'CONVERSIONTYPES', 'IMAGE_GEN_DEBUGFS', 'ROOTFS_RO_UNNEEDED', 'IMGDEPLOYDIR', 'PACKAGE_EXCLUDE_COMPLEMENTARY']
+                 'CONVERSIONTYPES', 'IMAGE_GEN_DEBUGFS', 'ROOTFS_RO_UNNEEDED', 'IMGDEPLOYDIR', 'PACKAGE_EXCLUDE_COMPLEMENTARY', 'REPRODUCIBLE_TIMESTAMP_ROOTFS']
     variables.extend(rootfs_command_variables(d))
     variables.extend(variable_depends(d))
     return " ".join(variables)
@@ -138,9 +138,6 @@ def build_live(d):
 IMAGE_TYPE_live = "${@build_live(d)}"
 inherit ${IMAGE_TYPE_live}
 
-IMAGE_TYPE_vm = '${@bb.utils.contains_any("IMAGE_FSTYPES", ["vmdk", "vdi", "qcow2", "hdddirect"], "image-vm", "", d)}'
-inherit ${IMAGE_TYPE_vm}
-
 IMAGE_TYPE_container = '${@bb.utils.contains("IMAGE_FSTYPES", "container", "image-container", "", d)}'
 inherit ${IMAGE_TYPE_container}
 
@@ -174,10 +171,6 @@ python () {
     d.setVar('IMAGE_FEATURES', ' '.join(sorted(list(remain_features))))
 
     check_image_features(d)
-    initramfs_image = d.getVar('INITRAMFS_IMAGE') or ""
-    if initramfs_image != "":
-        d.appendVarFlag('do_build', 'depends', " %s:do_bundle_initramfs" %  d.getVar('PN'))
-        d.appendVarFlag('do_bundle_initramfs', 'depends', " %s:do_image_complete" % initramfs_image)
 }
 
 IMAGE_CLASSES += "image_types"
@@ -257,6 +250,7 @@ fakeroot python do_rootfs () {
     progress_reporter.next_stage()
 
     # generate rootfs
+    d.setVarFlag('REPRODUCIBLE_TIMESTAMP_ROOTFS', 'export', '1')
     create_rootfs(d, progress_reporter=progress_reporter, logcatcher=logcatcher)
 
     progress_reporter.finish()
@@ -264,18 +258,19 @@ fakeroot python do_rootfs () {
 do_rootfs[dirs] = "${TOPDIR}"
 do_rootfs[cleandirs] += "${S} ${IMGDEPLOYDIR}"
 do_rootfs[umask] = "022"
-addtask rootfs before do_build after do_prepare_recipe_sysroot
+addtask rootfs after do_prepare_recipe_sysroot
 
 fakeroot python do_image () {
     from oe.utils import execute_pre_post_process
 
+    d.setVarFlag('REPRODUCIBLE_TIMESTAMP_ROOTFS', 'export', '1')
     pre_process_cmds = d.getVar("IMAGE_PREPROCESS_COMMAND")
 
     execute_pre_post_process(d, pre_process_cmds)
 }
 do_image[dirs] = "${TOPDIR}"
 do_image[umask] = "022"
-addtask do_image after do_rootfs before do_build
+addtask do_image after do_rootfs
 
 fakeroot python do_image_complete () {
     from oe.utils import execute_pre_post_process
@@ -292,6 +287,10 @@ do_image_complete[sstate-inputdirs] = "${IMGDEPLOYDIR}"
 do_image_complete[sstate-outputdirs] = "${DEPLOY_DIR_IMAGE}"
 do_image_complete[stamp-extra-info] = "${MACHINE}"
 addtask do_image_complete after do_image before do_build
+python do_image_complete_setscene () {
+    sstate_setscene(d)
+}
+addtask do_image_complete_setscene
 
 # Add image-level QA/sanity checks to IMAGE_QA_COMMANDS
 #
@@ -323,6 +322,15 @@ fakeroot python do_image_qa () {
 }
 addtask do_image_qa after do_image_complete before do_build
 
+SSTATETASKS += "do_image_qa"
+SSTATE_SKIP_CREATION_task-image-qa = '1'
+do_image_qa[sstate-inputdirs] = ""
+do_image_qa[sstate-outputdirs] = ""
+python do_image_qa_setscene () {
+    sstate_setscene(d)
+}
+addtask do_image_qa_setscene
+
 def setup_debugfs_variables(d):
     d.appendVar('IMAGE_ROOTFS', '-dbg')
     d.appendVar('IMAGE_LINK_NAME', '-dbg')
@@ -427,6 +435,7 @@ python () {
         # date/time values. It will get expanded at execution time.
         # Similarly TMPDIR since otherwise we see QA stamp comparision problems
         localdata.delVar('DATETIME')
+        localdata.delVar('DATE')
         localdata.delVar('TMPDIR')
 
         image_cmd = localdata.getVar("IMAGE_CMD")
@@ -491,7 +500,7 @@ python () {
         d.prependVarFlag(task, 'postfuncs', ' create_symlinks')
         d.appendVarFlag(task, 'subimages', ' ' + ' '.join(subimages))
         d.appendVarFlag(task, 'vardeps', ' ' + ' '.join(vardeps))
-        d.appendVarFlag(task, 'vardepsexclude', 'DATETIME')
+        d.appendVarFlag(task, 'vardepsexclude', 'DATETIME DATE')
 
         bb.debug(2, "Adding task %s before %s, after %s" % (task, 'do_image_complete', after))
         bb.build.addtask(task, 'do_image_complete', after, d)
@@ -600,15 +609,6 @@ do_package_write_ipk[noexec] = "1"
 do_package_write_deb[noexec] = "1"
 do_package_write_rpm[noexec] = "1"
 
-# Allow the kernel to be repacked with the initramfs and boot image file as a single file
-do_bundle_initramfs[depends] += "virtual/kernel:do_bundle_initramfs"
-do_bundle_initramfs[nostamp] = "1"
-do_bundle_initramfs[noexec] = "1"
-do_bundle_initramfs () {
-	:
-}
-addtask bundle_initramfs after do_image_complete
-
 # Prepare the root links to point to the /usr counterparts.
 create_merged_usr_symlinks() {
     root="$1"
@@ -640,3 +640,15 @@ create_merged_usr_symlinks_sdk() {
 
 ROOTFS_PREPROCESS_COMMAND += "${@bb.utils.contains('DISTRO_FEATURES', 'usrmerge', 'create_merged_usr_symlinks_rootfs; ', '',d)}"
 POPULATE_SDK_PRE_TARGET_COMMAND += "${@bb.utils.contains('DISTRO_FEATURES', 'usrmerge', 'create_merged_usr_symlinks_sdk; ', '',d)}"
+
+reproducible_final_image_task () {
+    if [ "$BUILD_REPRODUCIBLE_BINARIES" = "1" ]; then
+        if [ "$REPRODUCIBLE_TIMESTAMP_ROOTFS" = "" ]; then
+            REPRODUCIBLE_TIMESTAMP_ROOTFS=`git log -1 --pretty=%ct`
+        fi
+        # Set mtime of all files to a reproducible value
+        bbnote "reproducible_final_image_task: mtime set to $REPRODUCIBLE_TIMESTAMP_ROOTFS"
+        find  ${IMAGE_ROOTFS} -exec touch -h  --date=@$REPRODUCIBLE_TIMESTAMP_ROOTFS {} \;
+    fi
+}
+IMAGE_PREPROCESS_COMMAND_append = " reproducible_final_image_task; "

meta/classes/image_types.bbclass

@@ -119,7 +119,8 @@ IMAGE_CMD_squashfs-lzo = "mksquashfs ${IMAGE_ROOTFS} ${IMGDEPLOYDIR}/${IMAGE_NAM
 # In practice, it turned out to be not needed when creating archives and
 # required when extracting, but it seems prudent to use it in both cases.
 IMAGE_CMD_TAR ?= "tar"
-IMAGE_CMD_tar = "${IMAGE_CMD_TAR} -cvf ${IMGDEPLOYDIR}/${IMAGE_NAME}${IMAGE_NAME_SUFFIX}.tar -C ${IMAGE_ROOTFS} ."
+# ignore return code 1 "file changed as we read it" as other tasks(e.g. do_image_wic) may be hardlinking rootfs
+IMAGE_CMD_tar = "${IMAGE_CMD_TAR} -cf ${IMGDEPLOYDIR}/${IMAGE_NAME}${IMAGE_NAME_SUFFIX}.tar -C ${IMAGE_ROOTFS} . || [ $? -eq 1 ]"
 
 do_image_cpio[cleandirs] += "${WORKDIR}/cpio_append"
 IMAGE_CMD_cpio () {
@@ -254,10 +255,6 @@ IMAGE_TYPES = " \
     ubi ubifs multiubi \
     tar tar.gz tar.bz2 tar.xz tar.lz4 \
     cpio cpio.gz cpio.xz cpio.lzma cpio.lz4 \
-    vmdk \
-    vdi \
-    qcow2 \
-    hdddirect \
     elf \
     wic wic.gz wic.bz2 wic.lzma \
     container \
@@ -269,7 +266,7 @@ IMAGE_TYPES = " \
 # CONVERSION_CMD/DEPENDS.
 COMPRESSIONTYPES ?= ""
 
-CONVERSIONTYPES = "gz bz2 lzma xz lz4 lzo zip sum md5sum sha1sum sha224sum sha256sum sha384sum sha512sum bmap u-boot ${COMPRESSIONTYPES}"
+CONVERSIONTYPES = "gz bz2 lzma xz lz4 lzo zip sum md5sum sha1sum sha224sum sha256sum sha384sum sha512sum bmap u-boot vmdk vdi qcow2 ${COMPRESSIONTYPES}"
 CONVERSION_CMD_lzma = "lzma -k -f -7 ${IMAGE_NAME}${IMAGE_NAME_SUFFIX}.${type}"
 CONVERSION_CMD_gz = "gzip -f -9 -c ${IMAGE_NAME}${IMAGE_NAME_SUFFIX}.${type} > ${IMAGE_NAME}${IMAGE_NAME_SUFFIX}.${type}.gz"
 CONVERSION_CMD_bz2 = "pbzip2 -f -k ${IMAGE_NAME}${IMAGE_NAME_SUFFIX}.${type}"
@@ -287,6 +284,9 @@ CONVERSION_CMD_sha384sum = "sha384sum ${IMAGE_NAME}${IMAGE_NAME_SUFFIX}.${type}
 CONVERSION_CMD_sha512sum = "sha512sum ${IMAGE_NAME}${IMAGE_NAME_SUFFIX}.${type} > ${IMAGE_NAME}${IMAGE_NAME_SUFFIX}.${type}.sha512sum"
 CONVERSION_CMD_bmap = "bmaptool create ${IMAGE_NAME}${IMAGE_NAME_SUFFIX}.${type} -o ${IMAGE_NAME}${IMAGE_NAME_SUFFIX}.${type}.bmap"
 CONVERSION_CMD_u-boot = "mkimage -A ${UBOOT_ARCH} -O linux -T ramdisk -C none -n ${IMAGE_NAME} -d ${IMAGE_NAME}${IMAGE_NAME_SUFFIX}.${type} ${IMAGE_NAME}${IMAGE_NAME_SUFFIX}.${type}.u-boot"
+CONVERSION_CMD_vmdk = "qemu-img convert -O vmdk ${IMAGE_NAME}${IMAGE_NAME_SUFFIX}.${type} ${IMAGE_NAME}${IMAGE_NAME_SUFFIX}.${type}.vmdk"
+CONVERSION_CMD_vdi = "qemu-img convert -O vdi ${IMAGE_NAME}${IMAGE_NAME_SUFFIX}.${type} ${IMAGE_NAME}${IMAGE_NAME_SUFFIX}.${type}.vdi"
+CONVERSION_CMD_qcow2 = "qemu-img convert -O qcow2 ${IMAGE_NAME}${IMAGE_NAME_SUFFIX}.${type} ${IMAGE_NAME}${IMAGE_NAME_SUFFIX}.${type}.qcow2"
 CONVERSION_DEPENDS_lzma = "xz-native"
 CONVERSION_DEPENDS_gz = "pigz-native"
 CONVERSION_DEPENDS_bz2 = "pbzip2-native"
@@ -297,6 +297,9 @@ CONVERSION_DEPENDS_zip = "zip-native"
 CONVERSION_DEPENDS_sum = "mtd-utils-native"
 CONVERSION_DEPENDS_bmap = "bmap-tools-native"
 CONVERSION_DEPENDS_u-boot = "u-boot-mkimage-native"
+CONVERSION_DEPENDS_vmdk = "qemu-native"
+CONVERSION_DEPENDS_vdi = "qemu-native"
+CONVERSION_DEPENDS_qcow2 = "qemu-native"
 
 RUNNABLE_IMAGE_TYPES ?= "ext2 ext3 ext4"
 RUNNABLE_MACHINE_PATTERNS ?= "qemu"

meta/classes/insane.bbclass

@@ -18,11 +18,6 @@
 #   files under exec_prefix
 
 
-# unsafe-references-in-binaries requires prelink-rtld from
-# prelink-native, but we don't want this DEPENDS for -native builds
-QADEPENDS = "prelink-native"
-QADEPENDS_class-native = ""
-QADEPENDS_class-nativesdk = ""
 QA_SANE = "True"
 
 # Elect whether a given type of error is a warning or error, they may
@@ -390,71 +385,6 @@ def package_qa_check_perm(path,name,d, elf, messages):
     """
     return
 
-QAPATHTEST[unsafe-references-in-scripts] = "package_qa_check_unsafe_references_in_scripts"
-def package_qa_check_unsafe_references_in_scripts(path, name, d, elf, messages):
-    """
-    Warn if scripts in base_[bindir|sbindir|libdir] reference files under exec_prefix
-    """
-    if unsafe_references_skippable(path, name, d):
-        return
-
-    if not elf:
-        import stat
-        import subprocess
-        pn = d.getVar('PN')
-
-        # Ensure we're checking an executable script
-        statinfo = os.stat(path)
-        if bool(statinfo.st_mode & stat.S_IXUSR):
-            # grep shell scripts for possible references to /exec_prefix/
-            exec_prefix = d.getVar('exec_prefix')
-            statement = "grep -e '%s/[^ :]\{1,\}/[^ :]\{1,\}' %s > /dev/null" % (exec_prefix, path)
-            if subprocess.call(statement, shell=True) == 0:
-                error_msg = pn + ": Found a reference to %s/ in %s" % (exec_prefix, path)
-                package_qa_handle_error("unsafe-references-in-scripts", error_msg, d)
-                error_msg = "Shell scripts in base_bindir and base_sbindir should not reference anything in exec_prefix"
-                package_qa_handle_error("unsafe-references-in-scripts", error_msg, d)
-
-def unsafe_references_skippable(path, name, d):
-    if bb.data.inherits_class('native', d) or bb.data.inherits_class('nativesdk', d):
-        return True
-
-    if "-dbg" in name or "-dev" in name:
-        return True
-
-    # Other package names to skip:
-    if name.startswith("kernel-module-"):
-        return True
-
-    # Skip symlinks
-    if os.path.islink(path):
-        return True
-
-    # Skip unusual rootfs layouts which make these tests irrelevant
-    exec_prefix = d.getVar('exec_prefix')
-    if exec_prefix == "":
-        return True
-
-    pkgdest = d.getVar('PKGDEST')
-    pkgdest = pkgdest + "/" + name
-    pkgdest = os.path.abspath(pkgdest)
-    base_bindir = pkgdest + d.getVar('base_bindir')
-    base_sbindir = pkgdest + d.getVar('base_sbindir')
-    base_libdir = pkgdest + d.getVar('base_libdir')
-    bindir = pkgdest + d.getVar('bindir')
-    sbindir = pkgdest + d.getVar('sbindir')
-    libdir = pkgdest + d.getVar('libdir')
-
-    if base_bindir == bindir and base_sbindir == sbindir and base_libdir == libdir:
-        return True
-
-    # Skip files not in base_[bindir|sbindir|libdir]
-    path = os.path.abspath(path)
-    if not (base_bindir in path or base_sbindir in path or base_libdir in path):
-        return True
-
-    return False
-
 QAPATHTEST[arch] = "package_qa_check_arch"
 def package_qa_check_arch(path,name,d, elf, messages):
     """
@@ -1153,8 +1083,6 @@ python do_package_qa () {
                continue
             if w in testmatrix and testmatrix[w] in g:
                 warnchecks.append(g[testmatrix[w]])
-            if w == 'unsafe-references-in-binaries':
-                oe.utils.write_ld_so_conf(d)
 
         errorchecks = []
         for e in (d.getVar("ERROR_QA") or "").split():
@@ -1162,8 +1090,6 @@ python do_package_qa () {
                continue
             if e in testmatrix and testmatrix[e] in g:
                 errorchecks.append(g[testmatrix[e]])
-            if e == 'unsafe-references-in-binaries':
-                oe.utils.write_ld_so_conf(d)
         return warnchecks, errorchecks
 
     for package in packages:

meta/classes/kernel-module-split.bbclass

@@ -120,6 +120,11 @@ python split_kernel_module_packages () {
                 rdepends[dep] = []
         d.setVar('RDEPENDS_' + pkg, bb.utils.join_deps(rdepends, commasep=False))
 
+        # let kernel modules runtime recommend kernel image
+        rrecommends = bb.utils.explode_dep_versions2(d.getVar('RRECOMMENDS_' + pkg) or "")
+        rrecommends['kernel-image'] = []
+        d.setVar('RRECOMMENDS_' + pkg, bb.utils.join_deps(rrecommends, commasep=False))
+
         # Avoid automatic -dev recommendations for modules ending with -dev.
         d.setVarFlag('RRECOMMENDS_' + pkg, 'nodeprrecs', 1)
 
@@ -138,7 +143,7 @@ python split_kernel_module_packages () {
     postinst = d.getVar('pkg_postinst_modules')
     postrm = d.getVar('pkg_postrm_modules')
 
-    modules = do_split_packages(d, root='${nonarch_base_libdir}/modules', file_regex=module_regex, output_pattern=module_pattern, description='%s kernel module', postinst=postinst, postrm=postrm, recursive=True, hook=frob_metadata, extra_depends='kernel-%s' % (d.getVar("KERNEL_VERSION")))
+    modules = do_split_packages(d, root='${nonarch_base_libdir}/modules', file_regex=module_regex, output_pattern=module_pattern, description='%s kernel module', postinst=postinst, postrm=postrm, recursive=True, hook=frob_metadata, extra_depends='')
     if modules:
         metapkg = d.getVar('KERNEL_MODULES_META_PACKAGE')
         d.appendVar('RDEPENDS_' + metapkg, ' '+' '.join(modules))

meta/classes/kernel-yocto.bbclass

@@ -143,6 +143,9 @@ do_kernel_metadata() {
 
 	# expand kernel features into their full path equivalents
 	bsp_definition=$(spp ${includes} --find -DKMACHINE=${KMACHINE} -DKTYPE=${LINUX_KERNEL_TYPE})
+	if [ $? -ne 0 ] || [ -z "${bsp_definition}" ]; then
+		bbfatal_log "Could not locate BSP definiton for ${KMACHINE}/${LINUX_KERNEL_TYPE}."
+	fi
 	meta_dir=$(kgit --meta)
 
 	# run1: pull all the configuration fragments, no matter where they come from

meta/classes/kernel.bbclass

@@ -255,8 +255,36 @@ python do_devshell_prepend () {
 
 addtask bundle_initramfs after do_install before do_deploy
 
+get_cc_option () {
+		# Check if KERNEL_CC supports the option "file-prefix-map".
+		# This option allows us to build images with __FILE__ values that do not
+		# contain the host build path.
+		if ${KERNEL_CC} -Q --help=joined | grep -q "\-ffile-prefix-map=<old=new>"; then
+			echo "-ffile-prefix-map=${S}=/kernel-source/"
+		fi
+}
+
 kernel_do_compile() {
 	unset CFLAGS CPPFLAGS CXXFLAGS LDFLAGS MACHINE
+	if [ "$BUILD_REPRODUCIBLE_BINARIES" = "1" ]; then
+		# kernel sources do not use do_unpack, so SOURCE_DATE_EPOCH may not
+		# be set....
+		if [ "$SOURCE_DATE_EPOCH" = "0" ]; then
+			olddir=`pwd`
+			cd ${S}
+			SOURCE_DATE_EPOCH=`git log  -1 --pretty=%ct`
+			# git repo not guaranteed, so fall back to REPRODUCIBLE_TIMESTAMP_ROOTFS
+			if [ $? -ne 0 ]; then
+				SOURCE_DATE_EPOCH=${REPRODUCIBLE_TIMESTAMP_ROOTFS}
+			fi
+			cd $olddir
+		fi
+
+		ts=`LC_ALL=C date -d @$SOURCE_DATE_EPOCH`
+		export KBUILD_BUILD_TIMESTAMP="$ts"
+		export KCONFIG_NOTIMESTAMP=1
+		bbnote "KBUILD_BUILD_TIMESTAMP: $ts"
+	fi
 	# The $use_alternate_initrd is only set from
 	# do_bundle_initramfs() This variable is specifically for the
 	# case where we are making a second pass at the kernel
@@ -270,20 +298,22 @@ kernel_do_compile() {
 		copy_initramfs
 		use_alternate_initrd=CONFIG_INITRAMFS_SOURCE=${B}/usr/${INITRAMFS_IMAGE_NAME}.cpio
 	fi
+	cc_extra=$(get_cc_option)
 	for typeformake in ${KERNEL_IMAGETYPE_FOR_MAKE} ; do
-		oe_runmake ${typeformake} CC="${KERNEL_CC}" LD="${KERNEL_LD}" ${KERNEL_EXTRA_ARGS} $use_alternate_initrd
+		oe_runmake ${typeformake} CC="${KERNEL_CC} $cc_extra " LD="${KERNEL_LD}" ${KERNEL_EXTRA_ARGS} $use_alternate_initrd
 	done
 	# vmlinux.gz is not built by kernel
 	if (echo "${KERNEL_IMAGETYPES}" | grep -wq "vmlinux\.gz"); then
 		mkdir -p "${KERNEL_OUTPUT_DIR}"
-		gzip -9c < ${B}/vmlinux > "${KERNEL_OUTPUT_DIR}/vmlinux.gz"
+		gzip -9cn < ${B}/vmlinux > "${KERNEL_OUTPUT_DIR}/vmlinux.gz"
 	fi
 }
 
 do_compile_kernelmodules() {
 	unset CFLAGS CPPFLAGS CXXFLAGS LDFLAGS MACHINE
 	if (grep -q -i -e '^CONFIG_MODULES=y$' ${B}/.config); then
-		oe_runmake -C ${B} ${PARALLEL_MAKE} modules CC="${KERNEL_CC}" LD="${KERNEL_LD}" ${KERNEL_EXTRA_ARGS}
+		cc_extra=$(get_cc_option)
+		oe_runmake -C ${B} ${PARALLEL_MAKE} modules CC="${KERNEL_CC} $cc_extra " LD="${KERNEL_LD}" ${KERNEL_EXTRA_ARGS}
 
 		# Module.symvers gets updated during the 
 		# building of the kernel modules. We need to

meta/classes/license.bbclass

@@ -255,14 +255,9 @@ def get_boot_dependencies(d):
     """
 
     depends = []
-    boot_depends_string = ""
     taskdepdata = d.getVar("BB_TASKDEPDATA", False)
-    # Only bootimg and bootdirectdisk include the depends flag
-    boot_tasks = ["do_bootimg", "do_bootdirectdisk",]
-
-    for task in boot_tasks:
-        boot_depends_string = "%s %s" % (boot_depends_string,
-                d.getVarFlag(task, "depends") or "")
+    # Only bootimg includes the depends flag
+    boot_depends_string = d.getVarFlag("do_bootimg", "depends") or ""
     boot_depends = [dep.split(":")[0] for dep
                 in boot_depends_string.split()
                 if not dep.split(":")[0].endswith("-native")]

meta/classes/module.bbclass

@@ -1,6 +1,6 @@
 inherit module-base kernel-module-split pkgconfig
 
-addtask make_scripts after do_prepare_recipe_sysroot before do_compile
+addtask make_scripts after do_prepare_recipe_sysroot before do_configure
 do_make_scripts[lockfiles] = "${TMPDIR}/kernel-scripts.lock"
 do_make_scripts[depends] += "virtual/kernel:do_shared_workdir"
 

meta/classes/package.bbclass

@@ -739,7 +739,7 @@ python fixup_perms () {
         bbpath = d.getVar('BBPATH')
         fs_perms_tables = d.getVar('FILESYSTEM_PERMS_TABLES')
         if not fs_perms_tables:
-            fs_perms_tables = 'files/fs-perms.txt'
+            fs_perms_tables = 'files/fs-perms.txt' if oe.types.boolean(d.getVar('VOLATILE_LOG_DIR', True)) else 'files/fs-perms-persistent-log.txt'
         for conf_file in fs_perms_tables.split():
             str += " %s" % bb.utils.which(bbpath, conf_file)
         return str
@@ -879,6 +879,11 @@ python split_and_strip_files () {
         debugdir = "/.debug"
         debuglibdir = ""
         debugsrcdir = ""
+    elif d.getVar('PACKAGE_DEBUG_SPLIT_STYLE') == 'debug-with-srcpkg':
+        debugappend = ""
+        debugdir = "/.debug"
+        debuglibdir = ""
+        debugsrcdir = "/usr/src/debug"
     else:
         # Original OE-core, a.k.a. ".debug", style debug info
         debugappend = ""
@@ -1092,6 +1097,15 @@ python populate_packages () {
     
     autodebug = not (d.getVar("NOAUTOPACKAGEDEBUG") or False)
 
+    split_source_package = (d.getVar('PACKAGE_DEBUG_SPLIT_STYLE') == 'debug-with-srcpkg')
+
+    # If debug-with-srcpkg mode is enabled then the src package is added
+    # into the package list and the source directory as its main content
+    if split_source_package:
+        src_package_name = ('%s-src' % d.getVar('PN'))
+        packages += (' ' + src_package_name)
+        d.setVar('FILES_%s' % src_package_name, '/usr/src/debug')
+
     # Sanity check PACKAGES for duplicates
     # Sanity should be moved to sanity.bbclass once we have the infrastucture
     package_list = []
@@ -1100,7 +1114,12 @@ python populate_packages () {
         if pkg in package_list:
             msg = "%s is listed in PACKAGES multiple times, this leads to packaging errors." % pkg
             package_qa_handle_error("packages-list", msg, d)
-        elif autodebug and pkg.endswith("-dbg"):
+        # If debug-with-srcpkg mode is enabled then the src package will have
+        # priority over dbg package when assigning the files.
+        # This allows src package to include source files and remove them from dbg.
+        elif split_source_package and pkg.endswith("-src"):
+            package_list.insert(0, pkg)
+        elif autodebug and pkg.endswith("-dbg") and not split_source_package:
             package_list.insert(0, pkg)
         else:
             package_list.append(pkg)
@@ -1434,7 +1453,7 @@ if [ x"$D" = "x" ]; then
 fi
 }
 
-RPMDEPS = "${STAGING_LIBDIR_NATIVE}/rpm/rpmdeps --rcfile ${STAGING_LIBDIR_NATIVE}/rpm/rpmrc  --macros ${STAGING_LIBDIR_NATIVE}/rpm/macros --define '_rpmconfigdir ${STAGING_LIBDIR_NATIVE}/rpm/'"
+RPMDEPS = "${STAGING_LIBDIR_NATIVE}/rpm/rpmdeps --alldeps"
 
 # Collect perfile run-time dependency metadata
 # Output:
@@ -1451,7 +1470,6 @@ python package_do_filedeps() {
     pkgdest = d.getVar('PKGDEST')
     packages = d.getVar('PACKAGES')
     rpmdeps = d.getVar('RPMDEPS')
-    magic = d.expand("${STAGING_DIR_NATIVE}${datadir_native}/misc/magic.mgc")
 
     def chunks(files, n):
         return [files[i:i+n] for i in range(0, len(files), n)]
@@ -1460,10 +1478,10 @@ python package_do_filedeps() {
     for pkg in packages.split():
         if d.getVar('SKIP_FILEDEPS_' + pkg) == '1':
             continue
-        if pkg.endswith('-dbg') or pkg.endswith('-doc') or pkg.find('-locale-') != -1 or pkg.find('-localedata-') != -1 or pkg.find('-gconv-') != -1 or pkg.find('-charmap-') != -1 or pkg.startswith('kernel-module-'):
+        if pkg.endswith('-dbg') or pkg.endswith('-doc') or pkg.find('-locale-') != -1 or pkg.find('-localedata-') != -1 or pkg.find('-gconv-') != -1 or pkg.find('-charmap-') != -1 or pkg.startswith('kernel-module-') or pkg.endswith('-src'):
             continue
         for files in chunks(pkgfiles[pkg], 100):
-            pkglist.append((pkg, files, rpmdeps, pkgdest, magic))
+            pkglist.append((pkg, files, rpmdeps, pkgdest))
 
     processed = oe.utils.multiprocess_exec( pkglist, oe.package.filedeprunner)
 
@@ -1578,7 +1596,7 @@ python package_do_shlibs() {
                 combos.append("-".join(options[0:i]))
             return combos
 
-        if (file.endswith('.dylib') or file.endswith('.so')) and not pkg.endswith('-dev') and not pkg.endswith('-dbg'):
+        if (file.endswith('.dylib') or file.endswith('.so')) and not pkg.endswith('-dev') and not pkg.endswith('-dbg') and not pkg.endswith('-src'):
             # Drop suffix
             name = os.path.basename(file).rsplit(".",1)[0]
             # Find all combinations

meta/classes/package_rpm.bbclass

@@ -7,10 +7,28 @@ RPMBUILD="rpmbuild"
 
 PKGWRITEDIRRPM = "${WORKDIR}/deploy-rpms"
 
-# Maintaining the perfile dependencies has singificant overhead when writing the 
+# Maintaining the perfile dependencies has singificant overhead when writing the
 # packages. When set, this value merges them for efficiency.
 MERGEPERFILEDEPS = "1"
 
+# Filter dependencies based on a provided function.
+def filter_deps(var, f):
+    import collections
+
+    depends_dict = bb.utils.explode_dep_versions2(var)
+    newdeps_dict = collections.OrderedDict()
+    for dep in depends_dict:
+        if f(dep):
+            newdeps_dict[dep] = depends_dict[dep]
+    return bb.utils.join_deps(newdeps_dict, commasep=False)
+
+# Filter out absolute paths (typically /bin/sh and /usr/bin/env) and any perl
+# dependencies for nativesdk packages.
+def filter_nativesdk_deps(srcname, var):
+    if var and srcname.startswith("nativesdk-"):
+        var = filter_deps(var, lambda dep: not dep.startswith('/') and dep != 'perl' and not dep.startswith('perl('))
+    return var
+
 # Construct per file dependencies file
 def write_rpm_perfiledata(srcname, d):
     workdir = d.getVar('WORKDIR')
@@ -26,7 +44,8 @@ def write_rpm_perfiledata(srcname, d):
             dependsflist = (d.getVar(dependsflist_key) or "")
             for dfile in dependsflist.split():
                 key = "FILE" + varname + "_" + dfile + "_" + pkg
-                depends_dict = bb.utils.explode_dep_versions(d.getVar(key) or "")
+                deps = filter_nativesdk_deps(srcname, d.getVar(key) or "")
+                depends_dict = bb.utils.explode_dep_versions(deps)
                 file = dfile.replace("@underscore@", "_")
                 file = file.replace("@closebrace@", "]")
                 file = file.replace("@openbrace@", "[")
@@ -359,6 +378,8 @@ python write_specfile () {
             splitrdepends = splitrdepends + " " + get_perfile('RDEPENDS', pkg, d)
             splitrprovides = splitrprovides + " " + get_perfile('RPROVIDES', pkg, d)
 
+        splitrdepends = filter_nativesdk_deps(srcname, splitrdepends)
+
         # Gather special src/first package data
         if srcname == splitname:
             srcrdepends    = splitrdepends

meta/classes/packagefeed-stability.bbclass

@@ -189,7 +189,7 @@ def package_compare_impl(pkgtype, d):
 
     # Remove all the old files and copy again if docopy
     if docopy:
-        bb.plain('Copying packages for recipe %s' % pn)
+        bb.note('Copying packages for recipe %s' % pn)
         pcmanifest = os.path.join(prepath, d.expand('pkg-compare-manifest-${MULTIMACH_TARGET_SYS}-${PN}'))
         try:
             with open(pcmanifest, 'r') as f:
@@ -224,7 +224,7 @@ def package_compare_impl(pkgtype, d):
                     shutil.copyfile(srcpath, destpath)
                 f.write('%s\n' % destpath)
     else:
-        bb.plain('Not copying packages for recipe %s' % pn)
+        bb.note('Not copying packages for recipe %s' % pn)
 
 do_cleansstate[postfuncs] += "pfs_cleanpkgs"
 python pfs_cleanpkgs () {

meta/classes/populate_sdk_base.bbclass

@@ -101,6 +101,26 @@ def populate_sdk_common(d):
     from oe.sdk import populate_sdk
     from oe.manifest import create_manifest, Manifest
 
+    # Handle package exclusions
+    excl_pkgs = (d.getVar("PACKAGE_EXCLUDE") or "").split()
+    inst_pkgs = (d.getVar("PACKAGE_INSTALL") or "").split()
+    inst_attempt_pkgs = (d.getVar("PACKAGE_INSTALL_ATTEMPTONLY") or "").split()
+
+    d.setVar('PACKAGE_INSTALL_ORIG', ' '.join(inst_pkgs))
+    d.setVar('PACKAGE_INSTALL_ATTEMPTONLY', ' '.join(inst_attempt_pkgs))
+
+    for pkg in excl_pkgs:
+        if pkg in inst_pkgs:
+            bb.warn("Package %s, set to be excluded, is in %s PACKAGE_INSTALL (%s).  It will be removed from the list." % (pkg, d.getVar('PN'), inst_pkgs))
+            inst_pkgs.remove(pkg)
+
+        if pkg in inst_attempt_pkgs:
+            bb.warn("Package %s, set to be excluded, is in %s PACKAGE_INSTALL_ATTEMPTONLY (%s).  It will be removed from the list." % (pkg, d.getVar('PN'), inst_pkgs))
+            inst_attempt_pkgs.remove(pkg)
+
+    d.setVar("PACKAGE_INSTALL", ' '.join(inst_pkgs))
+    d.setVar("PACKAGE_INSTALL_ATTEMPTONLY", ' '.join(inst_attempt_pkgs))
+
     pn = d.getVar('PN')
     runtime_mapping_rename("TOOLCHAIN_TARGET_TASK", pn, d)
     runtime_mapping_rename("TOOLCHAIN_TARGET_TASK_ATTEMPTONLY", pn, d)

meta/classes/qemuboot.bbclass

@@ -114,7 +114,8 @@ python do_write_qemuboot_conf() {
     with open(qemuboot, 'w') as f:
         cf.write(f)
 
-    if os.path.lexists(qemuboot_link):
-       os.remove(qemuboot_link)
-    os.symlink(os.path.basename(qemuboot), qemuboot_link)
+    if qemuboot_link != qemuboot:
+        if os.path.lexists(qemuboot_link):
+           os.remove(qemuboot_link)
+        os.symlink(os.path.basename(qemuboot), qemuboot_link)
 }

meta/classes/report-error.bbclass

@@ -29,6 +29,13 @@ python errorreport_handler () {
         import json
         import codecs
 
+        def nativelsb():
+            nativelsbstr = e.data.getVar("NATIVELSBSTRING")
+            # provide a bit more host info in case of uninative build
+            if e.data.getVar('UNINATIVE_URL') != 'unset':
+                return '/'.join([nativelsbstr, lsb_distro_identifier(e.data)])
+            return nativelsbstr
+
         logpath = e.data.getVar('ERR_REPORT_DIR')
         datafile = os.path.join(logpath, "error-report.txt")
 
@@ -38,7 +45,7 @@ python errorreport_handler () {
             machine = e.data.getVar("MACHINE")
             data['machine'] = machine
             data['build_sys'] = e.data.getVar("BUILD_SYS")
-            data['nativelsb'] = e.data.getVar("NATIVELSBSTRING")
+            data['nativelsb'] = nativelsb()
             data['distro'] = e.data.getVar("DISTRO")
             data['target_sys'] = e.data.getVar("TARGET_SYS")
             data['failures'] = []

meta/classes/rm_work.bbclass

@@ -35,22 +35,12 @@ do_rm_work () {
         fi
     done
 
-    cd ${WORKDIR}
-    for dir in *
-    do
-        # Retain only logs and other files in temp, safely ignore
-        # failures of removing pseudo folers on NFS2/3 server.
-        if [ $dir = 'pseudo' ]; then
-            rm -rf $dir 2> /dev/null || true
-        elif ! echo '${RM_WORK_EXCLUDE_ITEMS}' | grep -q -w "$dir"; then
-            rm -rf $dir
-        fi
-    done
-
     # Need to add pseudo back or subsqeuent work in this workdir
     # might fail since setscene may not rerun to recreate it
     mkdir -p ${WORKDIR}/pseudo/
 
+    excludes='${RM_WORK_EXCLUDE_ITEMS}'
+
     # Change normal stamps into setscene stamps as they better reflect the
     # fact WORKDIR is now empty
     # Also leave noexec stamps since setscene stamps don't cover them
@@ -71,7 +61,12 @@ do_rm_work () {
                 i=dummy
                 break
                 ;;
-            *do_rootfs*|*do_image*|*do_bootimg*|*do_bootdirectdisk*|*do_vmimg*|*do_write_qemuboot_conf*)
+            *do_image_complete*)
+                mv $i `echo $i | sed -e "s#do_image_complete#do_image_complete_setscene#"`
+                i=dummy
+                break
+                ;;
+            *do_rootfs*|*do_image*|*do_bootimg*|*do_write_qemuboot_conf*)
                 i=dummy
                 break
                 ;;
@@ -79,6 +74,12 @@ do_rm_work () {
                 i=dummy
                 break
                 ;;
+            *do_addto_recipe_sysroot*)
+                # Preserve recipe-sysroot-native if do_addto_recipe_sysroot has been used
+                excludes="$excludes recipe-sysroot-native"
+                i=dummy
+                break
+                ;;
             # We remove do_package entirely, including any
             # sstate version since otherwise we'd need to leave 'plaindirs' around
             # such as 'packages' and 'packages-split' and these can be large. No end
@@ -101,6 +102,18 @@ do_rm_work () {
         done
         rm -f $i
     done
+
+    cd ${WORKDIR}
+    for dir in *
+    do
+        # Retain only logs and other files in temp, safely ignore
+        # failures of removing pseudo folers on NFS2/3 server.
+        if [ $dir = 'pseudo' ]; then
+            rm -rf $dir 2> /dev/null || true
+        elif ! echo "$excludes" | grep -q -w "$dir"; then
+            rm -rf $dir
+        fi
+    done
 }
 do_rm_work_all () {
     :

meta/classes/rootfs-postcommands.bbclass

@@ -158,7 +158,10 @@ ssh_allow_empty_password () {
 	fi
 
 	if [ -d ${IMAGE_ROOTFS}${sysconfdir}/pam.d ] ; then
-		sed -i 's/nullok_secure/nullok/' ${IMAGE_ROOTFS}${sysconfdir}/pam.d/*
+		for f in `find ${IMAGE_ROOTFS}${sysconfdir}/pam.d/* -type f -exec test -e {} \; -print`
+		do
+			sed -i 's/nullok_secure/nullok/' $f
+		done
 	fi
 }
 
@@ -300,7 +303,8 @@ python write_image_test_data() {
     searchString = "%s/"%(d.getVar("TOPDIR")).replace("//","/")
     export2json(d, testdata,searchString=searchString,replaceString="")
 
-    if os.path.lexists(testdata_link):
-       os.remove(testdata_link)
-    os.symlink(os.path.basename(testdata), testdata_link)
+    if testdata_link != testdata:
+        if os.path.lexists(testdata_link):
+           os.remove(testdata_link)
+        os.symlink(os.path.basename(testdata), testdata_link)
 }

meta/classes/rootfsdebugfiles.bbclass

@@ -15,6 +15,10 @@
 #    ROOTFS_DEBUG_FILES += "${TOPDIR}/conf/dropbear_rsa_host_key ${IMAGE_ROOTFS}/etc/dropbear/dropbear_rsa_host_key ;"
 # 2. Boot the image once, copy the dropbear_rsa_host_key from
 #    the device into your build conf directory.
+# 3. A optional parameter can be used to set file mode
+#    of the copied target, for instance:
+#    ROOTFS_DEBUG_FILES += "${TOPDIR}/conf/dropbear_rsa_host_key ${IMAGE_ROOTFS}/etc/dropbear/dropbear_rsa_host_key 0600;"
+#    in case they might be required to have a specific mode. (Shoundn't be too open, for example)
 #
 # Do not use for production images! It bypasses several
 # core build mechanisms (updating the image when one
@@ -27,10 +31,11 @@ ROOTFS_DEBUG_FILES[doc] = "Lists additional files or directories to be installed
 ROOTFS_POSTPROCESS_COMMAND += "rootfs_debug_files ;"
 rootfs_debug_files () {
    #!/bin/sh -e
-   echo "${ROOTFS_DEBUG_FILES}" | sed -e 's/;/\n/g' | while read source target; do
+   echo "${ROOTFS_DEBUG_FILES}" | sed -e 's/;/\n/g' | while read source target mode; do
       if [ -e "$source" ]; then
          mkdir -p $(dirname $target)
          cp -a $source $target
+         [ -n "$mode" ] && chmod $mode $target
       fi
    done
 }

meta/classes/sign_package_feed.bbclass

@@ -28,6 +28,9 @@ PACKAGE_FEED_SIGN = '1'
 PACKAGE_FEED_GPG_BACKEND ?= 'local'
 PACKAGE_FEED_GPG_SIGNATURE_TYPE ?= 'ASC'
 
+# Make feed signing key to be present in rootfs
+FEATURE_PACKAGES_package-management_append = " signing-keys-packagefeed"
+
 python () {
     # Check sanity of configuration
     for var in ('PACKAGE_FEED_GPG_NAME', 'PACKAGE_FEED_GPG_PASSPHRASE_FILE'):

meta/classes/sign_rpm.bbclass

@@ -19,9 +19,12 @@
 # GPG_BIN
 #           Optional variable for specifying the gpg binary/wrapper to use for
 #           signing.
+# RPM_GPG_SIGN_CHUNK
+#           Optional variable indicating the number of packages used per gpg
+#           invocation
 # GPG_PATH
 #           Optional variable for specifying the gnupg "home" directory:
-#
+
 inherit sanity
 
 RPM_SIGN_PACKAGES='1'
@@ -29,6 +32,7 @@ RPM_SIGN_FILES ?= '0'
 RPM_GPG_BACKEND ?= 'local'
 # SHA-256 is used by default
 RPM_FILE_CHECKSUM_DIGEST ?= '8'
+RPM_GPG_SIGN_CHUNK ?= "${BB_NUMBER_THREADS}"
 
 
 python () {
@@ -56,6 +60,7 @@ python sign_rpm () {
                      d.getVar('RPM_GPG_NAME'),
                      d.getVar('RPM_GPG_PASSPHRASE'),
                      d.getVar('RPM_FILE_CHECKSUM_DIGEST'),
+                     int(d.getVar('RPM_GPG_SIGN_CHUNK')),
                      d.getVar('RPM_FSK_PATH'),
                      d.getVar('RPM_FSK_PASSWORD'))
 }

meta/classes/siteinfo.bbclass

@@ -153,7 +153,7 @@ python () {
         bb.fatal("Please add your architecture to siteinfo.bbclass")
 }
 
-def siteinfo_get_files(d, aclocalcache = False):
+def siteinfo_get_files(d, sysrootcache = False):
     sitedata = siteinfo_data(d)
     sitefiles = ""
     for path in d.getVar("BBPATH").split(":"):
@@ -162,18 +162,11 @@ def siteinfo_get_files(d, aclocalcache = False):
             if os.path.exists(filename):
                 sitefiles += filename + " "
 
-    if not aclocalcache:
+    if not sysrootcache:
         return sitefiles
 
-    # Now check for siteconfig cache files in the directory setup by autotools.bbclass to
-    # avoid races.
-    #
-    # ACLOCALDIR may or may not exist so cache should only be set to True from autotools.bbclass
-    # after files have been copied into this location. To do otherwise risks parsing/signature
-    # issues and the directory being created/removed whilst this code executes. This can happen
-    # when a multilib recipe is parsed along with its base variant which may be running at the time
-    # causing rare but nasty failures
-    path_siteconfig = d.getVar('ACLOCALDIR')
+    # Now check for siteconfig cache files in sysroots
+    path_siteconfig = d.getVar('SITECONFIG_SYSROOTCACHE')
     if path_siteconfig and os.path.isdir(path_siteconfig):
         for i in os.listdir(path_siteconfig):
             if not i.endswith("_config"):

meta/classes/sstate.bbclass

@@ -346,8 +346,6 @@ def sstate_installpkgdir(ss, d):
         oe.path.remove(dir)
 
     for state in ss['dirs']:
-        if d.getVar('SSTATE_SKIP_CREATION') == '1':
-            continue
         prepdir(state[1])
         os.rename(sstateinst + state[0], state[1])
     sstate_install(ss, d)
@@ -596,8 +594,6 @@ def sstate_package(ss, d):
     for state in ss['dirs']:
         if not os.path.exists(state[1]):
             continue
-        if d.getVar('SSTATE_SKIP_CREATION') == '1':
-            continue
         srcbase = state[0].rstrip("/").rsplit('/', 1)[0]
         # Find and error for absolute symlinks. We could attempt to relocate but its not
         # clear where the symlink is relative to in this context. We could add that markup
@@ -625,6 +621,10 @@ def sstate_package(ss, d):
 
     d.setVar('SSTATE_BUILDDIR', sstatebuild)
     d.setVar('SSTATE_PKG', sstatepkg)
+    d.setVar('SSTATE_INSTDIR', sstatebuild)
+
+    if d.getVar('SSTATE_SKIP_CREATION') == '1':
+        return
 
     for f in (d.getVar('SSTATECREATEFUNCS') or '').split() + \
              ['sstate_create_package', 'sstate_sign_package'] + \
@@ -634,8 +634,6 @@ def sstate_package(ss, d):
 
     bb.siggen.dump_this_task(sstatepkg + ".siginfo", d)
 
-    d.setVar('SSTATE_INSTDIR', sstatebuild)
-
     return
 
 def pstaging_fetch(sstatefetch, sstatepkg, d):

meta/classes/staging.bbclass

@@ -177,6 +177,7 @@ def staging_processfixme(fixme, target, recipesysroot, recipesysrootnative, d):
 def staging_populate_sysroot_dir(targetsysroot, nativesysroot, native, d):
     import glob
     import subprocess
+    import errno
 
     fixme = []
     postinsts = []

meta/classes/systemd-boot.bbclass

@@ -7,10 +7,9 @@
 #                        maintenance.
 #
 # Set EFI_PROVIDER = "systemd-boot" to use systemd-boot on your live images instead of grub-efi
-# (images built by image-live.bbclass or image-vm.bbclass)
+# (images built by image-live.bbclass)
 
 do_bootimg[depends] += "${MLPREFIX}systemd-boot:do_deploy"
-do_bootdirectdisk[depends] += "${MLPREFIX}systemd-boot:do_deploy"
 
 EFIDIR = "/EFI/BOOT"
 

meta/classes/testimage.bbclass

@@ -7,8 +7,8 @@
 # Most of the tests are commands run on target image over ssh.
 # To use it add testimage to global inherit and call your target image with -c testimage
 # You can try it out like this:
-# - first build a qemu core-image-sato
-# - add IMAGE_CLASSES += "testimage" in local.conf
+# - first add IMAGE_CLASSES += "testimage" in local.conf
+# - build a qemu core-image-sato
 # - then bitbake core-image-sato -c testimage. That will run a standard suite of tests.
 
 # You can set (or append to) TEST_SUITES in local.conf to select the tests

meta/classes/uninative.bbclass

@@ -49,6 +49,12 @@ python uninative_event_fetchloader() {
             localdata = bb.data.createCopy(d)
             localdata.setVar('FILESPATH', "")
             localdata.setVar('DL_DIR', tarballdir)
+            # Our games with path manipulation of DL_DIR mean standard PREMIRRORS don't work
+            # and we can't easily put 'chksum' into the url path from a url parameter with
+            # the current fetcher url handling
+            ownmirror = d.getVar('SOURCE_MIRROR_URL')
+            if ownmirror:
+                localdata.appendVar("PREMIRRORS", " ${UNINATIVE_URL}${UNINATIVE_TARBALL} ${SOURCE_MIRROR_URL}/uninative/%s/${UNINATIVE_TARBALL}" % chksum)
 
             srcuri = d.expand("${UNINATIVE_URL}${UNINATIVE_TARBALL};sha256sum=%s" % chksum)
             bb.note("Fetching uninative binary shim from %s" % srcuri)

meta/conf/bitbake.conf

@@ -85,6 +85,10 @@ USRBINPATH_class-nativesdk = "/usr/bin"
 # Root home directory
 ROOT_HOME ??= "/home/root"
 
+# If set to boolean true ('yes', 'y', 'true', 't', '1'), /var/log links to /var/volatile/log.
+# If set to boolean false ('no', 'n', 'false', 'f', '0'), /var/log is on persistent storage.
+VOLATILE_LOG_DIR ?= "yes"
+
 ##################################################################
 # Architecture-dependent build variables.
 ##################################################################
@@ -850,7 +854,7 @@ BB_HASHCONFIG_WHITELIST ?= "${BB_HASHBASE_WHITELIST} DATE TIME SSH_AGENT_PID \
     PARALLEL_MAKE BB_NUMBER_THREADS BB_ORIGENV BB_INVALIDCONF BBINCLUDED \
     GIT_PROXY_COMMAND ALL_PROXY all_proxy NO_PROXY no_proxy FTP_PROXY ftp_proxy \
     HTTP_PROXY http_proxy HTTPS_PROXY https_proxy SOCKS5_USER SOCKS5_PASSWD \
-    BB_SETSCENE_ENFORCE BB_CMDLINE"
+    BB_SETSCENE_ENFORCE BB_CMDLINE BB_SERVER_TIMEOUT"
 BB_SIGNATURE_EXCLUDE_FLAGS ?= "doc deps depends \
     lockfiles type vardepsexclude vardeps vardepvalue vardepvalueexclude \
     file-checksums python func task export unexport noexec nostamp dirs cleandirs \

meta/conf/conf-notes.txt

@@ -1,3 +1,8 @@
+
+### Shell environment set up for builds. ###
+
+You can now run 'bitbake <target>'
+
 Common targets are:
     core-image-minimal
     core-image-sato

meta/conf/distro/include/default-versions.inc

@@ -2,3 +2,6 @@
 # Default preferred versions
 #
 
+PREFERRED_VERSION_openssl = "1.0.%"
+PREFERRED_VERSION_openssl-native = "1.0.%"
+PREFERRED_VERSION_nativesdk-openssl = "1.0.%"

meta/conf/distro/include/no-static-libs.inc

@@ -25,6 +25,9 @@ DISABLE_STATIC_pn-openjade-native = ""
 DISABLE_STATIC_pn-openssl = ""
 DISABLE_STATIC_pn-openssl-native = ""
 DISABLE_STATIC_pn-nativesdk-openssl = ""
+DISABLE_STATIC_pn-openssl10 = ""
+DISABLE_STATIC_pn-openssl10-native = ""
+DISABLE_STATIC_pn-nativesdk-openssl10 = ""
 # libssp-static-dev included in build-appliance
 DISABLE_STATIC_pn-gcc-runtime = ""
 # libusb1-native is used to build static dfu-util-native

meta/conf/distro/include/tcmode-default.inc

@@ -24,10 +24,10 @@ PREFERRED_PROVIDER_virtual/gettext ??= "gettext"
 
 GCCVERSION ?= "7.%"
 SDKGCCVERSION ?= "${GCCVERSION}"
-BINUVERSION ?= "2.28%"
+BINUVERSION ?= "2.29%"
 GDBVERSION ?= "8.0%"
-GLIBCVERSION ?= "2.25%"
-LINUXLIBCVERSION ?= "4.10%"
+GLIBCVERSION ?= "2.26%"
+LINUXLIBCVERSION ?= "4.12%"
 
 PREFERRED_VERSION_gcc ?= "${GCCVERSION}"
 PREFERRED_VERSION_gcc-cross-${TARGET_ARCH} ?= "${GCCVERSION}"

meta/conf/distro/include/yocto-uninative.inc

@@ -6,6 +6,6 @@
 # to the distro running on the build machine.
 #
 
-UNINATIVE_URL ?= "http://downloads.yoctoproject.org/releases/uninative/1.6/"
-UNINATIVE_CHECKSUM[i686] ?= "d9daf66785834f888b6f50a1d37a8120815f8380e0e267d311e1dfaf4bb5404e"
-UNINATIVE_CHECKSUM[x86_64] ?= "2b4fffa308d9f19e0742a1a404ff42495fb50c165e5ca0458cedca157372691a"
+UNINATIVE_URL ?= "http://downloads.yoctoproject.org/releases/uninative/1.7/"
+UNINATIVE_CHECKSUM[i686] ?= "d7c341460035936c19d63fe02f354ef1bc993c62d694ae3a31458d1c6997f0c5"
+UNINATIVE_CHECKSUM[x86_64] ?= "ed033c868b87852b07957a4400f3b744c00aef5d6470346ea1a59b6d3e03075e"

meta/conf/documentation.conf

@@ -56,7 +56,6 @@ do_testsdk[doc] = "Installs an SDK and performs runtime tests on the tools insta
 do_uboot_mkimage[doc] = "Creates a uImage file from the kernel for the U-Boot bootloader"
 do_unpack[doc] = "Unpacks the source code into a working directory"
 do_validate_branches[doc] = "Ensures that the source/meta branches are on the locations specified by their SRCREV values for a linux-yocto style kernel"
-do_vmimg[doc] = "Creates an image for use with VMware or VirtualBox compatible virtual machine hosts (based on IMAGE_FSTYPES either .vmdk or .vdi)"
 
 # DESCRIPTIONS FOR VARIABLES #
 

meta/conf/machine-sdk/i586.conf

@@ -1,5 +1,4 @@
 SDK_ARCH = "i586"
 SDK_CC_ARCH = "-march=i586"
 ABIEXTENSION_class-nativesdk = ""
-SDK_OLDEST_KERNEL = "2.6.32"
 

meta/conf/machine-sdk/i686.conf

@@ -1,4 +1,3 @@
 SDK_ARCH = "i686"
 SDK_CC_ARCH = "-march=i686"
 ABIEXTENSION_class-nativesdk = ""
-SDK_OLDEST_KERNEL = "2.6.32"

meta/conf/machine-sdk/x86_64.conf

@@ -1,3 +1,2 @@
 SDK_ARCH = "x86_64"
 ABIEXTENSION_class-nativesdk = ""
-SDK_OLDEST_KERNEL = "2.6.32"

meta/conf/machine/include/qemuboot-mips.inc

@@ -4,5 +4,5 @@ QB_MEM = "-m 256"
 QB_MACHINE = "-machine malta"
 QB_KERNEL_CMDLINE_APPEND = "console=ttyS0 console=tty"
 # Add the 'virtio-rng-pci' device otherwise the guest may run out of entropy
-QB_OPT_APPEND = "-vga cirrus -show-cursor -usb -usbdevice tablet -device virtio-rng-pci"
+QB_OPT_APPEND = "-vga cirrus -show-cursor -usb -device usb-tablet -device virtio-rng-pci"
 QB_SYSTEM_NAME = "qemu-system-${TUNE_ARCH}"

meta/conf/machine/include/qemuboot-x86.inc

@@ -12,7 +12,7 @@ QB_AUDIO_DRV = "alsa"
 QB_AUDIO_OPT = "-soundhw ac97,es1370"
 QB_KERNEL_CMDLINE_APPEND = "vga=0 uvesafb.mode_option=${UVESA_MODE} oprofile.timer=1 uvesafb.task_timeout=-1"
 # Add the 'virtio-rng-pci' device otherwise the guest may run out of entropy
-QB_OPT_APPEND = "-vga vmware -show-cursor -usb -usbdevice tablet -device virtio-rng-pci"
+QB_OPT_APPEND = "-vga vmware -show-cursor -usb -device usb-tablet -device virtio-rng-pci"
 
 KERNEL_MODULE_AUTOLOAD += "uvesafb"
 KERNEL_MODULE_PROBECONF += "uvesafb"

meta/conf/machine/include/x86-base.inc

@@ -10,7 +10,8 @@ MACHINE_FEATURES += "screen keyboard pci usbhost ext2 ext3 x86 \
 
 MACHINE_EXTRA_RRECOMMENDS += "kernel-modules"
 
-IMAGE_FSTYPES ?= "hddimg"
+IMAGE_FSTYPES ?= "live"
+NOISO ?= "1"
 
 KERNEL_IMAGETYPE ?= "bzImage"
 
@@ -20,7 +21,7 @@ SERIAL_CONSOLE ?= "115200 ttyS0"
 # kernel-related variables
 #
 PREFERRED_PROVIDER_virtual/kernel ??= "linux-yocto"
-PREFERRED_VERSION_linux-yocto ??= "4.10%"
+PREFERRED_VERSION_linux-yocto ??= "4.12%"
 
 #
 # XSERVER subcomponents, used to build the XSERVER variable

meta/conf/machine/qemuarm.conf

@@ -15,6 +15,6 @@ QB_SYSTEM_NAME = "qemu-system-arm"
 QB_MACHINE = "-machine versatilepb"
 QB_KERNEL_CMDLINE_APPEND = "console=ttyAMA0,115200 console=tty"
 # Add the 'virtio-rng-pci' device otherwise the guest may run out of entropy
-QB_OPT_APPEND = "-show-cursor -usb -usbdevice tablet -device virtio-rng-pci"
-PREFERRED_VERSION_linux-yocto ??= "4.10%"
+QB_OPT_APPEND = "-show-cursor -usb -device usb-tablet -device virtio-rng-pci"
+PREFERRED_VERSION_linux-yocto ??= "4.12%"
 QB_DTB = "${@base_version_less_or_equal('PREFERRED_VERSION_linux-yocto', '4.7', '', 'zImage-versatile-pb.dtb', d)}"

meta/conf/machine/qemuppc.conf

@@ -17,5 +17,5 @@ QB_MACHINE = "-machine mac99"
 QB_CPU = "-cpu G4"
 QB_KERNEL_CMDLINE_APPEND = "console=tty console=ttyS0"
 # Add the 'virtio-rng-pci' device otherwise the guest may run out of entropy
-QB_OPT_APPEND = "-show-cursor -usb -usbdevice tablet -device virtio-rng-pci"
+QB_OPT_APPEND = "-show-cursor -usb -device usb-tablet -device virtio-rng-pci"
 QB_TAP_OPT = "-netdev tap,id=net0,ifname=@TAP@,script=no,downscript=no"

meta/conf/machine/qemux86-64.conf

@@ -25,7 +25,7 @@ XSERVER = "xserver-xorg \
            xserver-xorg-module-libint10 \
            "
 
-MACHINE_FEATURES += "x86"
+MACHINE_FEATURES += "x86 pci"
 
 MACHINE_ESSENTIAL_EXTRA_RDEPENDS += "v86d"
 

meta/conf/machine/qemux86.conf

@@ -24,7 +24,7 @@ XSERVER = "xserver-xorg \
            xserver-xorg-module-libint10 \
            "
 
-MACHINE_FEATURES += "x86"
+MACHINE_FEATURES += "x86 pci"
 
 MACHINE_ESSENTIAL_EXTRA_RDEPENDS += "v86d"
 

meta/files/common-licenses/pkgconf

@@ -0,0 +1,10 @@
+Copyright (c) 2011, 2012, 2013, 2014, 2015, 2016, 2017
+    pkgconf authors (see AUTHORS file in source directory).
+
+Permission to use, copy, modify, and/or distribute this software for any
+purpose with or without fee is hereby granted, provided that the above
+copyright notice and this permission notice appear in all copies.
+
+This software is provided 'as is' and without any warranty, express or
+implied.  In no event shall the authors be liable for any damages arising
+from the use of this software.

meta/files/fs-perms-persistent-log.txt

@@ -0,0 +1,66 @@
+# This file contains a list of files and directories with known permissions.
+# It is used by the packaging class to ensure that the permissions, owners and
+# group of listed files and directories are in sync across the system.
+#
+# The format of this file 
+#
+#<path>	<mode>	<uid>	<gid>	<walk>	<fmode>	<fuid>	<fgid>
+#
+# or
+#
+#<path> link <target>
+#
+# <path>: directory path
+# <mode>: mode for directory
+# <uid>:  uid for directory
+# <gid>:  gid for directory
+# <walk>: recursively walk the directory?  true or false
+# <fmode>: if walking, new mode for files
+# <fuid>:  if walking, new uid for files
+# <fgid>:  if walking, new gid for files
+# <target>: turn the directory into a symlink point to target
+#
+# in mode, uid or gid, a "-" means don't change any existing values
+#
+# /usr/src		0755	root	root	false	-	-	-
+# /usr/share/man	0755	root	root	true	0644	root	root
+
+# Note: all standard config directories are automatically assigned "0755 root root false - - -"
+
+# Documentation should always be corrected
+${mandir}		0755	root	root	true	0644	root	root
+${infodir}		0755	root	root	true	0644	root	root
+${docdir}		0755	root	root	true	0644	root	root
+${datadir}/gtk-doc	0755	root	root	true	0644	root	root
+
+# Fixup locales
+${datadir}/locale	0755	root	root	true	0644	root	root
+
+# Cleanup headers
+${includedir}		0755	root	root	true	0644	root	root
+${oldincludedir}	0755	root	root	true	0644	root	root
+
+# Cleanup debug src
+/usr/src/debug		0755	root	root	true	-	root	root
+
+# Items from base-files
+# Links
+${localstatedir}/run	link	/run
+${localstatedir}/lock	link	/run/lock
+${localstatedir}/tmp	link	volatile/tmp
+
+/home				0755	root	root	false - - -
+/srv				0755	root	root	false - - -
+${prefix}/src			0755	root	root	false - - -
+${localstatedir}/local		0755	root	root	false - - -
+
+# Special permissions from base-files
+# Set 1777
+/tmp				01777	root	root	false - - -
+${localstatedir}/volatile/tmp	01777	root	root	false - - -
+
+# Set 0700
+${ROOT_HOME}			0700	root	root	false - - -
+
+# Set 2775-lsb
+${localstatedir}/mail		02775	root	mail	false - - -