build-appliance-image: Update to master head revision

(From OE-Core rev: 303868c8976523342026817c6aa1a04ce0408c5d)

Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>

README.hardware

@@ -203,6 +203,75 @@ linux/arch/powerpc/boot/dts/mpc8315erdb.dts within the kernel source). If
 you have left them at the factory default then you shouldn't need to do
 anything here.
 
+Note: To boot from USB disk you need u-boot that supports 'ext2load usb'
+command. You need to setup TFTP server, load u-boot from there and
+flash it to NOR flash.
+
+Beware! Flashing bootloader is potentially dangerous operation that can
+brick your device if done incorrectly. Please, make sure you understand
+what below commands mean before executing them.
+
+Load the new u-boot.bin from TFTP server to memory address 200000
+=> tftp 200000 u-boot.bin
+
+Disable flash protection
+=> protect off all
+
+Erase the old u-boot from fe000000 to fe06ffff in NOR flash.
+The size is 0x70000 (458752 bytes)
+=> erase fe000000 fe06ffff
+
+Copy the new u-boot from address 200000 to fe000000
+the size is 0x70000. It has to be greater or equal to u-boot.bin size
+=> cp.b 200000 fe000000 70000
+
+Enable flash protection again
+=> protect on all
+
+Reset the board
+=> reset
+
+--- Booting from USB disk ---
+
+ 1. Flash partitioned image to the USB disk
+
+    # dd if=core-image-minimal-mpc8315e-rdb.wic of=/dev/sdb
+
+ 2. Plug USB disk into the MPC8315 board
+
+ 3. Connect the board's first serial port to your workstation and then start up
+    your favourite serial terminal so that you will be able to interact with
+    the serial console. If you don't have a favourite, picocom is suggested:
+
+  $ picocom /dev/ttyUSB0 -b 115200
+
+ 4. Power up or reset the board and press a key on the terminal when prompted
+    to get to the U-Boot command line
+
+ 5. Optional. Load the u-boot.bin from the USB disk:
+
+ => usb start
+ => ext2load usb 0:1 200000 u-boot.bin
+
+    and flash it to NOR flash as described above.
+
+ 6. Set fdtaddr and loadaddr. This is not necessary if you set them before.
+
+ => setenv fdtaddr a00000
+ => setenv loadaddr 1000000
+
+ 7. Load the kernel and dtb from first partition of the USB disk:
+
+ => usb start
+ => ext2load usb 0:1 $loadaddr uImage
+ => ext2load usb 0:1 $fdtaddr dtb
+
+ 8. Set bootargs and boot up the device
+
+ => setenv bootargs root=/dev/sdb2 rw rootwait console=ttyS0,115200
+ => bootm $loadaddr - $fdtaddr
+
+
 --- Booting from NFS root ---
 
 Load the kernel and dtb (device tree blob), and boot the system as follows:

bitbake/bin/bitbake

@@ -38,7 +38,7 @@ from bb.main import bitbake_main, BitBakeConfigParameters, BBMainException
 if sys.getfilesystemencoding() != "utf-8":
     sys.exit("Please use a locale setting which supports utf-8.\nPython can't change the filesystem locale after loading so we need a utf-8 when python starts or things won't work.")
 
-__version__ = "1.32.0"
+__version__ = "1.33.0"
 
 if __name__ == "__main__":
     if __version__ != bb.__version__:

bitbake/bin/bitbake-worker

@@ -136,7 +136,7 @@ def sigterm_handler(signum, frame):
     os.killpg(0, signal.SIGTERM)
     sys.exit()
 
-def fork_off_task(cfg, data, databuilder, workerdata, fn, task, taskname, appends, taskdepdata, extraconfigdata, quieterrors=False):
+def fork_off_task(cfg, data, databuilder, workerdata, fn, task, taskname, appends, taskdepdata, extraconfigdata, quieterrors=False, dry_run_exec=False):
     # We need to setup the environment BEFORE the fork, since
     # a fork() or exec*() activates PSEUDO...
 
@@ -152,8 +152,10 @@ def fork_off_task(cfg, data, databuilder, workerdata, fn, task, taskname, append
         except TypeError:
              umask = taskdep['umask'][taskname]
 
+    dry_run = cfg.dry_run or dry_run_exec
+
     # We can't use the fakeroot environment in a dry run as it possibly hasn't been built
-    if 'fakeroot' in taskdep and taskname in taskdep['fakeroot'] and not cfg.dry_run:
+    if 'fakeroot' in taskdep and taskname in taskdep['fakeroot'] and not dry_run:
         envvars = (workerdata["fakerootenv"][fn] or "").split()
         for key, value in (var.split('=') for var in envvars):
             envbackup[key] = os.environ.get(key)
@@ -220,6 +222,8 @@ def fork_off_task(cfg, data, databuilder, workerdata, fn, task, taskname, append
                 the_data = databuilder.mcdata[mc]
                 the_data.setVar("BB_WORKERCONTEXT", "1")
                 the_data.setVar("BB_TASKDEPDATA", taskdepdata)
+                if cfg.limited_deps:
+                    the_data.setVar("BB_LIMITEDDEPS", "1")
                 the_data.setVar("BUILDNAME", workerdata["buildname"])
                 the_data.setVar("DATE", workerdata["date"])
                 the_data.setVar("TIME", workerdata["time"])
@@ -263,7 +267,7 @@ def fork_off_task(cfg, data, databuilder, workerdata, fn, task, taskname, append
                     logger.critical(traceback.format_exc())
                 os._exit(1)
             try:
-                if cfg.dry_run:
+                if dry_run:
                     return 0
                 return bb.build.exec_task(fn, taskname, the_data, cfg.profile)
             except:
@@ -421,10 +425,10 @@ class BitbakeWorker(object):
         sys.exit(0)
 
     def handle_runtask(self, data):
-        fn, task, taskname, quieterrors, appends, taskdepdata = pickle.loads(data)
+        fn, task, taskname, quieterrors, appends, taskdepdata, dry_run_exec = pickle.loads(data)
         workerlog_write("Handling runtask %s %s %s\n" % (task, fn, taskname))
 
-        pid, pipein, pipeout = fork_off_task(self.cookercfg, self.data, self.databuilder, self.workerdata, fn, task, taskname, appends, taskdepdata, self.extraconfigdata, quieterrors)
+        pid, pipein, pipeout = fork_off_task(self.cookercfg, self.data, self.databuilder, self.workerdata, fn, task, taskname, appends, taskdepdata, self.extraconfigdata, quieterrors, dry_run_exec)
 
         self.build_pids[pid] = task
         self.build_pipes[pid] = runQueueWorkerPipe(pipein, pipeout)

bitbake/bin/toaster

@@ -158,6 +158,7 @@ unset OE_ROOT
 # 4) pid's we need to clean up on exit/shutdown
 # note: for future. in order to make this an arbitrary directory, we need to
 # make sure that the toaster.sqlite file doesn't default to `pwd` like it currently does.
+export BB_ENV_EXTRAWHITE="$BB_ENV_EXTRAWHITE TOASTER_DIR"
 export TOASTER_DIR=`pwd`
 
 WEBSERVER=1

bitbake/doc/bitbake-user-manual/bitbake-user-manual-metadata.xml

@@ -505,14 +505,14 @@
             <title>Unseting variables</title>
 
             <para>
-                It is possible to completely remove a variable or a variable flag 
+                It is possible to completely remove a variable or a variable flag
                 from BitBake's internal data dictionary by using the "unset" keyword.
                 Here is an example:
                 <literallayout class='monospaced'>
         unset DATE
         unset do_fetch[noexec]
                 </literallayout>
-                These two statements remove the <filename>DATE</filename> and the 
+                These two statements remove the <filename>DATE</filename> and the
                 <filename>do_fetch[noexec]</filename> flag.
             </para>
 
@@ -1984,128 +1984,194 @@
         <title>Events</title>
 
         <para>
-            BitBake allows installation of event handlers within
-            recipe and class files.
-            Events are triggered at certain points during operation,
-            such as the beginning of an operation against a given recipe
-            (<filename>*.bb</filename> file), the start of a given task,
-            task failure, task success, and so forth.
+            BitBake allows installation of event handlers within recipe
+            and class files.
+            Events are triggered at certain points during operation, such
+            as the beginning of operation against a given recipe
+            (i.e. <filename>*.bb</filename>), the start of a given task,
+            a task failure, a task success, and so forth.
             The intent is to make it easy to do things like email
-            notification on build failure.
+            notification on build failures.
         </para>
 
         <para>
-            Following is an example event handler that
-            prints the name of the event and the content of
-            the <filename>FILE</filename> variable:
+            Following is an example event handler that prints the name
+            of the event and the content of the
+            <filename>FILE</filename> variable:
             <literallayout class='monospaced'>
      addhandler myclass_eventhandler
      python myclass_eventhandler() {
          from bb.event import getName
-         from bb import data
          print("The name of the Event is %s" % getName(e))
-         print("The file we run for is %s" % data.getVar('FILE', e.data, True))
+         print("The file we run for is %s" % d.getVar('FILE'))
      }
+     myclass_eventhandler[eventmask] = "bb.event.BuildStarted bb.event.BuildCompleted"
             </literallayout>
-            This event handler gets called every time an event is
-            triggered.
-            A global variable "<filename>e</filename>" is defined and
-            "<filename>e.data</filename>" contains an instance of
-            "<filename>bb.data</filename>".
-            With the <filename>getName(e)</filename> method, one can get
+            In the previous example, an eventmask has been set so that
+            the handler only sees the "BuildStarted" and "BuildCompleted"
+            events.
+            This event handler gets called every time an event matching
+            the eventmask is triggered.
+            A global variable "e" is defined, which represents the current
+            event.
+            With the <filename>getName(e)</filename> method, you can get
             the name of the triggered event.
+            The global datastore is available as "d".
+            In legacy code, you might see "e.data" used to get the datastore.
+            However, realize that "e.data" is deprecated and you should use
+            "d" going forward.
         </para>
 
         <para>
-            Because you probably are only interested in a subset of events,
-            you would likely use the <filename>[eventmask]</filename> flag
-            for your event handler to be sure that only certain events
-            trigger the handler.
-            Given the previous example, suppose you only wanted the
-            <filename>bb.build.TaskFailed</filename> event to trigger that
-            event handler.
-            Use the flag as follows:
-            <literallayout class='monospaced'>
-     addhandler myclass_eventhandler
-     myclass_eventhandler[eventmask] = "bb.build.TaskFailed"
-     python myclass_eventhandler() {
-         from bb.event import getName
-         from bb import data
-         print("The name of the Event is %s" % getName(e))
-         print("The file we run for is %s" % data.getVar('FILE', e.data, True))
-     }
-            </literallayout>
+            The context of the datastore is appropriate to the event
+            in question.
+            For example, "BuildStarted" and "BuildCompleted" events run
+            before any tasks are executed so would be in the global
+            configuration datastore namespace.
+            No recipe-specific metadata exists in that namespace.
+            The "BuildStarted" and "buildCompleted" events also run in
+            the main cooker/server process rather than any worker context.
+            Thus, any changes made to the datastore would be seen by other
+            cooker/server events within the current build but not seen
+            outside of that build or in any worker context.
+            Task events run in the actual tasks in question consequently
+            have recipe-specific and task-specific contents.
+            These events run in the worker context and are discarded at
+            the end of task execution.
         </para>
 
         <para>
-            During a standard build, the following common events might occur:
+            During a standard build, the following common events might
+            occur.
+            The following events are the most common kinds of events that
+            most metadata might have an interest in viewing:
             <itemizedlist>
                 <listitem><para>
-                    <filename>bb.event.ConfigParsed()</filename>
+                    <filename>bb.event.ConfigParsed()</filename>:
+                    Fired when the base configuration; which consists of
+                    <filename>bitbake.conf</filename>,
+                    <filename>base.bbclass</filename> and any global
+                    <filename>INHERIT</filename> statements; has been parsed.
+                    You can see multiple such events when each of the
+                    workers parse the base configuration or if the server
+                    changes configuration and reparses.
+                    Any given datastore only has one such event executed
+                    against it, however.
+                    If
+                    <link linkende='var-BB_INVALIDCONF'><filename>BB_INVALIDCONF</filename></link>
+                    is set in the datastore by the event handler, the
+                    configuration is reparsed and a new event triggered,
+                    allowing the metadata to update configuration.
                     </para></listitem>
                 <listitem><para>
-                    <filename>bb.event.ParseStarted()</filename>
+                    <filename>bb.event.HeartbeatEvent()</filename>:
+                    Fires at regular time intervals of one second.
+                    You can configure the interval time using the
+                    <filename>BB_HEARTBEAT_EVENT</filename> variable.
+                    The event's "time" attribute is the
+                    <filename>time.time()</filename> value when the
+                    event is triggered.
+                    This event is useful for activities such as
+                    system state monitoring.
                     </para></listitem>
                 <listitem><para>
-                    <filename>bb.event.ParseProgress()</filename>
+                    <filename>bb.event.ParseStarted()</filename>:
+                    Fired when BitBake is about to start parsing recipes.
+                    This event's "total" attribute represents the number of
+                    recipes BitBake plans to parse.
                     </para></listitem>
                 <listitem><para>
-                    <filename>bb.event.ParseCompleted()</filename>
+                    <filename>bb.event.ParseProgress()</filename>:
+                    Fired as parsing progresses.
+                    This event's "current" attribute is the number of
+                    recipes parsed as well as the "total" attribute.
                     </para></listitem>
                 <listitem><para>
-                    <filename>bb.event.BuildStarted()</filename>
+                    <filename>bb.event.ParseCompleted()</filename>:
+                    Fired when parsing is complete.
+                    This event's "cached", "parsed", "skipped", "virtuals",
+                    "masked", and "errors" attributes provide statistics
+                    for the parsing results.
                     </para></listitem>
                 <listitem><para>
-                    <filename>bb.build.TaskStarted()</filename>
+                    <filename>bb.event.BuildStarted()</filename>:
+                    Fired when a new build starts.
                     </para></listitem>
                 <listitem><para>
-                    <filename>bb.build.TaskInvalid()</filename>
+                    <filename>bb.build.TaskStarted()</filename>:
+                    Fired when a task starts.
+                    This event's "taskfile" attribute points to the recipe
+                    from which the task originates.
+                    The "taskname" attribute, which is the task's name,
+                    includes the <filename>do_</filename> prefix, and the
+                    "logfile" attribute point to where the task's output is
+                    stored.
+                    Finally, the "time" attribute is the task's execution start
+                    time.
                     </para></listitem>
                 <listitem><para>
-                    <filename>bb.build.TaskFailedSilent()</filename>
+                    <filename>bb.build.TaskInvalid()</filename>:
+                    Fired if BitBake tries to execute a task that does not exist.
                     </para></listitem>
                 <listitem><para>
-                    <filename>bb.build.TaskFailed()</filename>
+                    <filename>bb.build.TaskFailedSilent()</filename>:
+                    Fired for setscene tasks that fail and should not be
+                    presented to the user verbosely.
                     </para></listitem>
                 <listitem><para>
-                    <filename>bb.build.TaskSucceeded()</filename>
+                    <filename>bb.build.TaskFailed()</filename>:
+                    Fired for normal tasks that fail.
                     </para></listitem>
                 <listitem><para>
-                    <filename>bb.event.BuildCompleted()</filename>
+                    <filename>bb.build.TaskSucceeded()</filename>:
+                    Fired when a task successfully completes.
                     </para></listitem>
                 <listitem><para>
-                    <filename>bb.cooker.CookerExit()</filename>
+                    <filename>bb.event.BuildCompleted()</filename>:
+                    Fired when a build finishes.
+                    </para></listitem>
+                <listitem><para>
+                    <filename>bb.cooker.CookerExit()</filename>:
+                    Fired when the BitBake server/cooker shuts down.
+                    This event is usually only seen by the UIs as a
+                    sign they should also shutdown.
                     </para></listitem>
             </itemizedlist>
-            Here is a list of other events that occur based on specific requests
-            to the server:
+        </para>
+
+        <para>
+            This next list of example events occur based on specific
+            requests to the server.
+            These events are often used to communicate larger pieces of
+            information from the BitBake server to other parts of
+            BitBake such as user interfaces:
             <itemizedlist>
                 <listitem><para>
                     <filename>bb.event.TreeDataPreparationStarted()</filename>
                     </para></listitem>
                 <listitem><para>
-                    <filename>bb.event.TreeDataPreparationProgress</filename>
+                    <filename>bb.event.TreeDataPreparationProgress()</filename>
                     </para></listitem>
                 <listitem><para>
-                    <filename>bb.event.TreeDataPreparationCompleted</filename>
+                    <filename>bb.event.TreeDataPreparationCompleted()</filename>
                     </para></listitem>
                 <listitem><para>
-                    <filename>bb.event.DepTreeGenerated</filename>
+                    <filename>bb.event.DepTreeGenerated()</filename>
                     </para></listitem>
                 <listitem><para>
-                    <filename>bb.event.CoreBaseFilesFound</filename>
+                    <filename>bb.event.CoreBaseFilesFound()</filename>
                     </para></listitem>
                 <listitem><para>
-                    <filename>bb.event.ConfigFilePathFound</filename>
+                    <filename>bb.event.ConfigFilePathFound()</filename>
                     </para></listitem>
                 <listitem><para>
-                    <filename>bb.event.FilesMatchingFound</filename>
+                    <filename>bb.event.FilesMatchingFound()</filename>
                     </para></listitem>
                 <listitem><para>
-                    <filename>bb.event.ConfigFilesFound</filename>
+                    <filename>bb.event.ConfigFilesFound()</filename>
                     </para></listitem>
                 <listitem><para>
-                    <filename>bb.event.TargetsTreeGenerated</filename>
+                    <filename>bb.event.TargetsTreeGenerated()</filename>
                     </para></listitem>
             </itemizedlist>
         </para>

bitbake/lib/bb/__init__.py

@@ -21,7 +21,7 @@
 # with this program; if not, write to the Free Software Foundation, Inc.,
 # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
 
-__version__ = "1.32.0"
+__version__ = "1.33.0"
 
 import sys
 if sys.version_info < (3, 4, 0):

bitbake/lib/bb/build.py

@@ -92,6 +92,7 @@ class TaskBase(event.Event):
     def __init__(self, t, logfile, d):
         self._task = t
         self._package = d.getVar("PF")
+        self._mc = d.getVar("BB_CURRENT_MC")
         self.taskfile = d.getVar("FILE")
         self.taskname = self._task
         self.logfile = logfile
@@ -194,13 +195,13 @@ def exec_func(func, d, dirs = None, pythonexception=False):
         oldcwd = None
 
     flags = d.getVarFlags(func)
-    cleandirs = flags.get('cleandirs')
+    cleandirs = flags.get('cleandirs') if flags else None
     if cleandirs:
         for cdir in d.expand(cleandirs).split():
             bb.utils.remove(cdir, True)
             bb.utils.mkdirhier(cdir)
 
-    if dirs is None:
+    if flags and dirs is None:
         dirs = flags.get('dirs')
         if dirs:
             dirs = d.expand(dirs).split()
@@ -861,3 +862,19 @@ def deltask(task, d):
         if task in deps:
             deps.remove(task)
             d.setVarFlag(bbtask, 'deps', deps)
+
+def preceedtask(task, with_recrdeptasks, d):
+    """
+    Returns a set of tasks in the current recipe which were specified as
+    precondition by the task itself ("after") or which listed themselves
+    as precondition ("before"). Preceeding tasks specified via the
+    "recrdeptask" are included in the result only if requested. Beware
+    that this may lead to the task itself being listed.
+    """
+    preceed = set()
+    preceed.update(d.getVarFlag(task, 'deps') or [])
+    if with_recrdeptasks:
+        recrdeptask = d.getVarFlag(task, 'recrdeptask')
+        if recrdeptask:
+            preceed.update(recrdeptask.split())
+    return preceed

bitbake/lib/bb/cooker.py

@@ -666,7 +666,37 @@ class BBCooker:
         if not task.startswith("do_"):
             task = "do_%s" % task
 
-        fulltargetlist = self.checkPackages(pkgs_to_build, task)
+        targetlist = self.checkPackages(pkgs_to_build, task)
+        fulltargetlist = []
+        defaulttask_implicit = ''
+        defaulttask_explicit = False
+        wildcard = False
+
+        # Wild card expansion:
+        # Replace string such as "multiconfig:*:bash"
+        # into "multiconfig:A:bash multiconfig:B:bash bash"
+        for k in targetlist:
+            if k.startswith("multiconfig:"):
+                if wildcard:
+                    bb.fatal('multiconfig conflict')
+                if k.split(":")[1] == "*":
+                    wildcard = True
+                    for mc in self.multiconfigs:
+                        if mc:
+                            fulltargetlist.append(k.replace('*', mc))
+                        # implicit default task
+                        else:
+                            defaulttask_implicit = k.split(":")[2]
+                else:
+                    fulltargetlist.append(k)
+            else:
+                defaulttask_explicit = True
+                fulltargetlist.append(k)
+
+        if not defaulttask_explicit and defaulttask_implicit != '':
+            fulltargetlist.append(defaulttask_implicit)
+
+        bb.debug(1,"Target list: %s" % (str(fulltargetlist)))
         taskdata = {}
         localdata = {}
 
@@ -1379,6 +1409,7 @@ class BBCooker:
         item = info_array[0].pn
         self.recipecaches[mc].ignored_dependencies = set()
         self.recipecaches[mc].bbfile_priority[fn] = 1
+        self.configuration.limited_deps = True
 
         # Remove external dependencies
         self.recipecaches[mc].task_deps[fn]['depends'] = {}
@@ -1674,6 +1705,15 @@ class BBCooker:
             pkgs_to_build.remove('universe')
             for mc in self.multiconfigs:
                 for t in self.recipecaches[mc].universe_target:
+                    if task:
+                        foundtask = False
+                        for provider_fn in self.recipecaches[mc].providers[t]:
+                            if task in self.recipecaches[mc].task_deps[provider_fn]['tasks']:
+                                foundtask = True
+                                break
+                        if not foundtask:
+                            bb.debug(1, "Skipping %s for universe tasks as task %s doesn't exist" % (t, task))
+                            continue
                     if mc:
                         t = "multiconfig:" + mc + ":" + t
                     pkgs_to_build.append(t)
@@ -1731,6 +1771,8 @@ class BBCooker:
 
         if self.parser:
             self.parser.shutdown(clean=not force, force=force)
+        self.notifier.stop()
+        self.confignotifier.stop()
 
     def finishcommand(self):
         self.state = state.initial

bitbake/lib/bb/cookerdata.py

@@ -147,6 +147,7 @@ class CookerConfiguration(object):
         self.interface = []
         self.writeeventlog = False
         self.server_only = False
+        self.limited_deps = False
 
         self.env = {}
 
@@ -289,7 +290,7 @@ class CookerDataBuilder(object):
 
             multiconfig = (self.data.getVar("BBMULTICONFIG") or "").split()
             for config in multiconfig:
-                mcdata = self.parseConfigurationFiles(['conf/multiconfig/%s.conf' % config] + self.prefiles, self.postfiles)
+                mcdata = self.parseConfigurationFiles(self.prefiles, self.postfiles, config)
                 bb.event.fire(bb.event.ConfigParsed(), mcdata)
                 self.mcdata[config] = mcdata
 
@@ -305,8 +306,9 @@ class CookerDataBuilder(object):
     def _findLayerConf(self, data):
         return findConfigFile("bblayers.conf", data)
 
-    def parseConfigurationFiles(self, prefiles, postfiles):
+    def parseConfigurationFiles(self, prefiles, postfiles, mc = "default"):
         data = bb.data.createCopy(self.basedata)
+        data.setVar("BB_CURRENT_MC", mc)
 
         # Parse files for loading *before* bitbake.conf and any includes
         for f in prefiles:

bitbake/lib/bb/data_smart.py

@@ -784,13 +784,14 @@ class DataSmart(MutableMapping):
                 if match:
                     removes.extend(self.expand(r).split())
 
-            filtered = filter(lambda v: v not in removes,
-                              value.split())
-            value = " ".join(filtered)
-            if expand and var in self.expand_cache:
-                 # We need to ensure the expand cache has the correct value
-                 # flag == "_content" here
-                self.expand_cache[var].value = value
+            if removes:
+                filtered = filter(lambda v: v not in removes,
+                                  value.split())
+                value = " ".join(filtered)
+                if expand and var in self.expand_cache:
+                    # We need to ensure the expand cache has the correct value
+                    # flag == "_content" here
+                    self.expand_cache[var].value = value
         return value
 
     def delVarFlag(self, var, flag, **loginfo):

bitbake/lib/bb/event.py

@@ -361,6 +361,17 @@ class RecipeEvent(Event):
 class RecipePreFinalise(RecipeEvent):
     """ Recipe Parsing Complete but not yet finialised"""
 
+class RecipeTaskPreProcess(RecipeEvent):
+    """
+    Recipe Tasks about to be finalised
+    The list of tasks should be final at this point and handlers
+    are only able to change interdependencies
+    """
+    def __init__(self, fn, tasklist):
+        self.fn = fn
+        self.tasklist = tasklist
+        Event.__init__(self)
+
 class RecipeParsed(RecipeEvent):
     """ Recipe Parsing Complete """
 
@@ -382,7 +393,7 @@ class StampUpdate(Event):
     targets = property(getTargets)
 
 class BuildBase(Event):
-    """Base class for bbmake run events"""
+    """Base class for bitbake build events"""
 
     def __init__(self, n, p, failures = 0):
         self._name = n
@@ -427,13 +438,13 @@ class BuildInit(BuildBase):
         BuildBase.__init__(self, name, p)
 
 class BuildStarted(BuildBase, OperationStarted):
-    """bbmake build run started"""
+    """Event when builds start"""
     def __init__(self, n, p, failures = 0):
         OperationStarted.__init__(self, "Building Started")
         BuildBase.__init__(self, n, p, failures)
 
 class BuildCompleted(BuildBase, OperationCompleted):
-    """bbmake build run completed"""
+    """Event when builds have completed"""
     def __init__(self, total, n, p, failures=0, interrupted=0):
         if not failures:
             OperationCompleted.__init__(self, total, "Building Succeeded")

bitbake/lib/bb/fetch2/__init__.py

@@ -819,6 +819,15 @@ def runfetchcmd(cmd, d, quiet=False, cleanup=None, log=None, workdir=None):
     if not cleanup:
         cleanup = []
 
+    # If PATH contains WORKDIR which contains PV which contains SRCPV we
+    # can end up in circular recursion here so give the option of breaking it
+    # in a data store copy.
+    try:
+        d.getVar("PV")
+    except bb.data_smart.ExpansionError:
+        d = bb.data.createCopy(d)
+        d.setVar("PV", "fetcheravoidrecurse")
+
     origenv = d.getVar("BB_ORIGENV", False)
     for var in exportvars:
         val = d.getVar(var) or (origenv and origenv.getVar(var))

bitbake/lib/bb/fetch2/gitsm.py

@@ -108,7 +108,7 @@ class GitSM(Git):
         os.rename(ud.clonedir, gitdir)
         runfetchcmd("sed " + gitdir + "/config -i -e 's/bare.*=.*true/bare = false/'", d)
         runfetchcmd(ud.basecmd + " reset --hard", d, workdir=tmpclonedir)
-        runfetchcmd(ud.basecmd + " checkout " + ud.revisions[ud.names[0]], d, workdir=tmpclonedir)
+        runfetchcmd(ud.basecmd + " checkout -f " + ud.revisions[ud.names[0]], d, workdir=tmpclonedir)
         runfetchcmd(ud.basecmd + " submodule update --init --recursive", d, workdir=tmpclonedir)
         self._set_relative_paths(tmpclonedir)
         runfetchcmd("sed " + gitdir + "/config -i -e 's/bare.*=.*false/bare = true/'", d, workdir=tmpclonedir)

bitbake/lib/bb/fetch2/wget.py

@@ -305,12 +305,24 @@ class Wget(FetchMethod):
             r = urllib.request.Request(uri)
             r.get_method = lambda: "HEAD"
 
-            if ud.user:
+            def add_basic_auth(login_str, request):
+                '''Adds Basic auth to http request, pass in login:password as string'''
                 import base64
-                encodeuser = base64.b64encode(ud.user.encode('utf-8')).decode("utf-8")
+                encodeuser = base64.b64encode(login_str.encode('utf-8')).decode("utf-8")
                 authheader =  "Basic %s" % encodeuser
                 r.add_header("Authorization", authheader)
 
+            if ud.user:
+                add_basic_auth(ud.user, r)
+
+            try:
+                import netrc, urllib.parse
+                n = netrc.netrc()
+                login, unused, password = n.authenticators(urllib.parse.urlparse(uri).hostname)
+                add_basic_auth("%s:%s" % (login, password), r)
+            except (TypeError, ImportError, IOError, netrc.NetrcParseError):
+                 pass
+
             opener.open(r)
         except urllib.error.URLError as e:
             if try_again:

bitbake/lib/bb/main.py

@@ -367,6 +367,7 @@ def start_server(servermodule, configParams, configuration, features):
         raise
     if not configParams.foreground:
         server.detach()
+        cooker.shutdown()
     cooker.lock.close()
     return server
 

bitbake/lib/bb/parse/ast.py

@@ -358,6 +358,7 @@ def finalize(fn, d, variant = None):
     bb.data.update_data(d)
 
     tasklist = d.getVar('__BBTASKS', False) or []
+    bb.event.fire(bb.event.RecipeTaskPreProcess(fn, list(tasklist)), d)
     bb.build.add_tasks(tasklist, d)
 
     bb.parse.siggen.finalise(fn, d, variant)

bitbake/lib/bb/persist_data.py

@@ -28,11 +28,7 @@ import sys
 import warnings
 from bb.compat import total_ordering
 from collections import Mapping
-
-try:
-    import sqlite3
-except ImportError:
-    from pysqlite2 import dbapi2 as sqlite3
+import sqlite3
 
 sqlversion = sqlite3.sqlite_version_info
 if sqlversion[0] < 3 or (sqlversion[0] == 3 and sqlversion[1] < 3):

bitbake/lib/bb/runqueue.py

@@ -36,6 +36,7 @@ from bb import msg, data, event
 from bb import monitordisk
 import subprocess
 import pickle
+from multiprocessing import Process
 
 bblogger = logging.getLogger("BitBake")
 logger = logging.getLogger("BitBake.RunQueue")
@@ -183,6 +184,18 @@ class RunQueueScheduler(object):
     def newbuilable(self, task):
         self.buildable.append(task)
 
+    def describe_task(self, taskid):
+        result = 'ID %s' % taskid
+        if self.rev_prio_map:
+            result = result + (' pri %d' % self.rev_prio_map[taskid])
+        return result
+
+    def dump_prio(self, comment):
+        bb.debug(3, '%s (most important first):\n%s' %
+                 (comment,
+                  '\n'.join(['%d. %s' % (index + 1, self.describe_task(taskid)) for
+                             index, taskid in enumerate(self.prio_map)])))
+
 class RunQueueSchedulerSpeed(RunQueueScheduler):
     """
     A scheduler optimised for speed. The priority map is sorted by task weight,
@@ -212,35 +225,100 @@ class RunQueueSchedulerSpeed(RunQueueScheduler):
 
 class RunQueueSchedulerCompletion(RunQueueSchedulerSpeed):
     """
-    A scheduler optimised to complete .bb files are quickly as possible. The
+    A scheduler optimised to complete .bb files as quickly as possible. The
     priority map is sorted by task weight, but then reordered so once a given
-    .bb file starts to build, it's completed as quickly as possible. This works
-    well where disk space is at a premium and classes like OE's rm_work are in
-    force.
+    .bb file starts to build, it's completed as quickly as possible by
+    running all tasks related to the same .bb file one after the after.
+    This works well where disk space is at a premium and classes like OE's
+    rm_work are in force.
     """
     name = "completion"
 
     def __init__(self, runqueue, rqdata):
-        RunQueueSchedulerSpeed.__init__(self, runqueue, rqdata)
+        super(RunQueueSchedulerCompletion, self).__init__(runqueue, rqdata)
 
-        #FIXME - whilst this groups all fns together it does not reorder the
-        #fn groups optimally.
+        # Extract list of tasks for each recipe, with tasks sorted
+        # ascending from "must run first" (typically do_fetch) to
+        # "runs last" (do_build). The speed scheduler prioritizes
+        # tasks that must run first before the ones that run later;
+        # this is what we depend on here.
+        task_lists = {}
+        for taskid in self.prio_map:
+            fn, taskname = taskid.rsplit(':', 1)
+            task_lists.setdefault(fn, []).append(taskname)
 
-        basemap = copy.deepcopy(self.prio_map)
-        self.prio_map = []
-        while (len(basemap) > 0):
-            entry = basemap.pop(0)
-            self.prio_map.append(entry)
-            fn = fn_from_tid(entry)
-            todel = []
-            for entry in basemap:
-                entry_fn = fn_from_tid(entry)
-                if entry_fn == fn:
-                    todel.append(basemap.index(entry))
-                    self.prio_map.append(entry)
-            todel.reverse()
-            for idx in todel:
-                del basemap[idx]
+        # Now unify the different task lists. The strategy is that
+        # common tasks get skipped and new ones get inserted after the
+        # preceeding common one(s) as they are found. Because task
+        # lists should differ only by their number of tasks, but not
+        # the ordering of the common tasks, this should result in a
+        # deterministic result that is a superset of the individual
+        # task ordering.
+        all_tasks = []
+        for recipe, new_tasks in task_lists.items():
+            index = 0
+            old_task = all_tasks[index] if index < len(all_tasks) else None
+            for new_task in new_tasks:
+                if old_task == new_task:
+                    # Common task, skip it. This is the fast-path which
+                    # avoids a full search.
+                    index += 1
+                    old_task = all_tasks[index] if index < len(all_tasks) else None
+                else:
+                    try:
+                        index = all_tasks.index(new_task)
+                        # Already present, just not at the current
+                        # place. We re-synchronized by changing the
+                        # index so that it matches again. Now
+                        # move on to the next existing task.
+                        index += 1
+                        old_task = all_tasks[index] if index < len(all_tasks) else None
+                    except ValueError:
+                        # Not present. Insert before old_task, which
+                        # remains the same (but gets shifted back).
+                        all_tasks.insert(index, new_task)
+                        index += 1
+        bb.debug(3, 'merged task list: %s'  % all_tasks)
+
+        # Now reverse the order so that tasks that finish the work on one
+        # recipe are considered more imporant (= come first). The ordering
+        # is now so that do_build is most important.
+        all_tasks.reverse()
+
+        # Group tasks of the same kind before tasks of less important
+        # kinds at the head of the queue (because earlier = lower
+        # priority number = runs earlier), while preserving the
+        # ordering by recipe. If recipe foo is more important than
+        # bar, then the goal is to work on foo's do_populate_sysroot
+        # before bar's do_populate_sysroot and on the more important
+        # tasks of foo before any of the less important tasks in any
+        # other recipe (if those other recipes are more important than
+        # foo).
+        #
+        # All of this only applies when tasks are runable. Explicit
+        # dependencies still override this ordering by priority.
+        #
+        # Here's an example why this priority re-ordering helps with
+        # minimizing disk usage. Consider a recipe foo with a higher
+        # priority than bar where foo DEPENDS on bar. Then the
+        # implicit rule (from base.bbclass) is that foo's do_configure
+        # depends on bar's do_populate_sysroot. This ensures that
+        # bar's do_populate_sysroot gets done first. Normally the
+        # tasks from foo would continue to run once that is done, and
+        # bar only gets completed and cleaned up later. By ordering
+        # bar's task that depend on bar's do_populate_sysroot before foo's
+        # do_configure, that problem gets avoided.
+        task_index = 0
+        self.dump_prio('original priorities')
+        for task in all_tasks:
+            for index in range(task_index, self.numTasks):
+                taskid = self.prio_map[index]
+                taskname = taskid.rsplit(':', 1)[1]
+                if taskname == task:
+                    del self.prio_map[index]
+                    self.prio_map.insert(task_index, taskid)
+                    task_index += 1
+        self.dump_prio('completion priorities')
 
 class RunTaskEntry(object):
     def __init__(self):
@@ -266,6 +344,7 @@ class RunQueueData:
         self.multi_provider_whitelist = (cfgData.getVar("MULTI_PROVIDER_WHITELIST") or "").split()
         self.setscenewhitelist = get_setscene_enforce_whitelist(cfgData)
         self.setscenewhitelist_checked = False
+        self.setscene_enforce = (cfgData.getVar('BB_SETSCENE_ENFORCE') == "1")
         self.init_progress_reporter = bb.progress.DummyMultiStageProcessProgressReporter()
 
         self.reset()
@@ -289,8 +368,8 @@ class RunQueueData:
         return tid + task_name_suffix
 
     def get_short_user_idstring(self, task, task_name_suffix = ""):
-        (mc, fn, taskname, _) = split_tid_mcfn(task)
-        pn = self.dataCaches[mc].pkg_fn[fn]
+        (mc, fn, taskname, taskfn) = split_tid_mcfn(task)
+        pn = self.dataCaches[mc].pkg_fn[taskfn]
         taskname = taskname_from_tid(task) + task_name_suffix
         return "%s:%s" % (pn, taskname)
 
@@ -565,6 +644,8 @@ class RunQueueData:
                 for (depname, idependtask) in irdepends:
                     if depname in taskData[mc].run_targets:
                         # Won't be in run_targets if ASSUME_PROVIDED
+                        if not taskData[mc].run_targets[depname]:
+                            continue
                         depdata = taskData[mc].run_targets[depname][0]
                         if depdata is not None:
                             t = depdata + ":" + idependtask
@@ -884,14 +965,14 @@ class RunQueueData:
         if not self.cooker.configuration.nosetscene:
             for tid in self.runtaskentries:
                 (mc, fn, taskname, _) = split_tid_mcfn(tid)
-                setscenetid = fn + ":" + taskname + "_setscene"
+                setscenetid = tid + "_setscene"
                 if setscenetid not in taskData[mc].taskentries:
                     continue
                 self.runq_setscene_tids.append(tid)
 
         def invalidate_task(tid, error_nostamp):
-            (mc, fn, taskname, _) = split_tid_mcfn(tid)
-            taskdep = self.dataCaches[mc].task_deps[fn]
+            (mc, fn, taskname, taskfn) = split_tid_mcfn(tid)
+            taskdep = self.dataCaches[mc].task_deps[taskfn]
             if fn + ":" + taskname not in taskData[mc].taskentries:
                 logger.warning("Task %s does not exist, invalidating this task will have no effect" % taskname)
             if 'nostamp' in taskdep and taskname in taskdep['nostamp']:
@@ -1003,8 +1084,9 @@ class RunQueue:
             magic = "decafbadbad"
         if fakeroot:
             magic = magic + "beef"
-            fakerootcmd = self.cfgData.getVar("FAKEROOTCMD")
-            fakerootenv = (self.cfgData.getVar("FAKEROOTBASEENV") or "").split()
+            mcdata = self.cooker.databuilder.mcdata[mc]
+            fakerootcmd = mcdata.getVar("FAKEROOTCMD")
+            fakerootenv = (mcdata.getVar("FAKEROOTBASEENV") or "").split()
             env = os.environ.copy()
             for key, value in (var.split('=') for var in fakerootenv):
                 env[key] = value
@@ -1066,10 +1148,9 @@ class RunQueue:
         for mc in self.rqdata.dataCaches:
             self.worker[mc] = self._start_worker(mc)
 
-    def start_fakeworker(self, rqexec):
-        if not self.fakeworker:
-            for mc in self.rqdata.dataCaches:
-                self.fakeworker[mc] = self._start_worker(mc, True, rqexec)
+    def start_fakeworker(self, rqexec, mc):
+        if not mc in self.fakeworker:
+            self.fakeworker[mc] = self._start_worker(mc, True, rqexec)
 
     def teardown_workers(self):
         self.teardown = True
@@ -1302,15 +1383,36 @@ class RunQueue:
         else:
             self.rqexe.finish()
 
-    def dump_signatures(self, options):
-        done = set()
-        bb.note("Reparsing files to collect dependency data")
+    def rq_dump_sigfn(self, fn, options):
         bb_cache = bb.cache.NoCache(self.cooker.databuilder)
+        the_data = bb_cache.loadDataFull(fn, self.cooker.collection.get_file_appends(fn))
+        siggen = bb.parse.siggen
+        dataCaches = self.rqdata.dataCaches
+        siggen.dump_sigfn(fn, dataCaches, options)
+
+    def dump_signatures(self, options):
+        fns = set()
+        bb.note("Reparsing files to collect dependency data")
+
         for tid in self.rqdata.runtaskentries:
             fn = fn_from_tid(tid)
-            if fn not in done:
-                the_data = bb_cache.loadDataFull(fn, self.cooker.collection.get_file_appends(fn))
-                done.add(fn)
+            fns.add(fn)
+
+        max_process = int(self.cfgData.getVar("BB_NUMBER_PARSE_THREADS") or os.cpu_count() or 1)
+        # We cannot use the real multiprocessing.Pool easily due to some local data
+        # that can't be pickled. This is a cheap multi-process solution.
+        launched = []
+        while fns:
+            if len(launched) < max_process:
+                p = Process(target=self.rq_dump_sigfn, args=(fns.pop(), options))
+                p.start()
+                launched.append(p)
+            for q in launched:
+                # The finished processes are joined when calling is_alive()
+                if not q.is_alive():
+                    launched.remove(q)
+        for p in launched:
+                p.join()
 
         bb.parse.siggen.dump_sigs(self.rqdata.dataCaches, options)
 
@@ -1337,7 +1439,7 @@ class RunQueue:
                 continue
 
             sq_fn.append(fn)
-            sq_hashfn.append(self.rqdata.dataCaches[mc].hashfn[fn])
+            sq_hashfn.append(self.rqdata.dataCaches[mc].hashfn[taskfn])
             sq_hash.append(self.rqdata.runtaskentries[tid].hash)
             sq_taskname.append(taskname)
             sq_task.append(tid)
@@ -1417,8 +1519,8 @@ class RunQueue:
 
 
         for tid in invalidtasks:
-            (mc, fn, taskname, _) = split_tid_mcfn(tid)
-            pn = self.rqdata.dataCaches[mc].pkg_fn[fn]
+            (mc, fn, taskname, taskfn) = split_tid_mcfn(tid)
+            pn = self.rqdata.dataCaches[mc].pkg_fn[taskfn]
             h = self.rqdata.runtaskentries[tid].hash
             matches = bb.siggen.find_siginfo(pn, taskname, [], self.cfgData)
             match = None
@@ -1521,8 +1623,8 @@ class RunQueueExecute:
         taskdata = {}
         taskdeps.add(task)
         for dep in taskdeps:
-            (mc, fn, taskname, _) = split_tid_mcfn(dep)
-            pn = self.rqdata.dataCaches[mc].pkg_fn[fn]
+            (mc, fn, taskname, taskfn) = split_tid_mcfn(dep)
+            pn = self.rqdata.dataCaches[mc].pkg_fn[taskfn]
             taskdata[dep] = [pn, taskname, fn]
         call = self.rq.depvalidate + "(task, taskdata, notneeded, d)"
         locs = { "task" : task, "taskdata" : taskdata, "notneeded" : self.scenequeue_notneeded, "d" : self.cooker.data }
@@ -1722,7 +1824,7 @@ class RunQueueExecuteTasks(RunQueueExecute):
 
             # Check tasks that are going to run against the whitelist
             def check_norun_task(tid, showerror=False):
-                (mc, fn, taskname, _) = split_tid_mcfn(tid)
+                (mc, fn, taskname, taskfn) = split_tid_mcfn(tid)
                 # Ignore covered tasks
                 if tid in self.rq.scenequeue_covered:
                     return False
@@ -1730,11 +1832,11 @@ class RunQueueExecuteTasks(RunQueueExecute):
                 if self.rq.check_stamp_task(tid, taskname, cache=self.stampcache):
                     return False
                 # Ignore noexec tasks
-                taskdep = self.rqdata.dataCaches[mc].task_deps[fn]
+                taskdep = self.rqdata.dataCaches[mc].task_deps[taskfn]
                 if 'noexec' in taskdep and taskname in taskdep['noexec']:
                     return False
 
-                pn = self.rqdata.dataCaches[mc].pkg_fn[fn]
+                pn = self.rqdata.dataCaches[mc].pkg_fn[taskfn]
                 if not check_setscene_enforce_whitelist(pn, taskname, self.rqdata.setscenewhitelist):
                     if showerror:
                         if tid in self.rqdata.runq_setscene_tids:
@@ -1790,7 +1892,7 @@ class RunQueueExecuteTasks(RunQueueExecute):
                 bb.event.fire(startevent, self.cfgData)
                 self.runq_running.add(task)
                 self.stats.taskActive()
-                if not self.cooker.configuration.dry_run:
+                if not (self.cooker.configuration.dry_run or self.rqdata.setscene_enforce):
                     bb.build.make_stamp(taskname, self.rqdata.dataCaches[mc], taskfn)
                 self.task_complete(task)
                 return True
@@ -1801,19 +1903,19 @@ class RunQueueExecuteTasks(RunQueueExecute):
             taskdepdata = self.build_taskdepdata(task)
 
             taskdep = self.rqdata.dataCaches[mc].task_deps[taskfn]
-            if 'fakeroot' in taskdep and taskname in taskdep['fakeroot'] and not self.cooker.configuration.dry_run:
-                if not self.rq.fakeworker:
+            if 'fakeroot' in taskdep and taskname in taskdep['fakeroot'] and not (self.cooker.configuration.dry_run or self.rqdata.setscene_enforce):
+                if not mc in self.rq.fakeworker:
                     try:
-                        self.rq.start_fakeworker(self)
+                        self.rq.start_fakeworker(self, mc)
                     except OSError as exc:
                         logger.critical("Failed to spawn fakeroot worker to run %s: %s" % (task, str(exc)))
                         self.rq.state = runQueueFailed
                         self.stats.taskFailed()
                         return True
-                self.rq.fakeworker[mc].process.stdin.write(b"<runtask>" + pickle.dumps((taskfn, task, taskname, False, self.cooker.collection.get_file_appends(fn), taskdepdata)) + b"</runtask>")
+                self.rq.fakeworker[mc].process.stdin.write(b"<runtask>" + pickle.dumps((taskfn, task, taskname, False, self.cooker.collection.get_file_appends(fn), taskdepdata, self.rqdata.setscene_enforce)) + b"</runtask>")
                 self.rq.fakeworker[mc].process.stdin.flush()
             else:
-                self.rq.worker[mc].process.stdin.write(b"<runtask>" + pickle.dumps((taskfn, task, taskname, False, self.cooker.collection.get_file_appends(taskfn), taskdepdata)) + b"</runtask>")
+                self.rq.worker[mc].process.stdin.write(b"<runtask>" + pickle.dumps((taskfn, task, taskname, False, self.cooker.collection.get_file_appends(taskfn), taskdepdata, self.rqdata.setscene_enforce)) + b"</runtask>")
                 self.rq.worker[mc].process.stdin.flush()
 
             self.build_stamps[task] = bb.build.stampfile(taskname, self.rqdata.dataCaches[mc], taskfn, noextra=True)
@@ -1884,6 +1986,7 @@ class RunQueueExecuteScenequeue(RunQueueExecute):
         sq_revdeps_new = {}
         sq_revdeps_squash = {}
         self.sq_harddeps = {}
+        self.stamps = {}
 
         # We need to construct a dependency graph for the setscene functions. Intermediate
         # dependencies between the setscene tasks only complicate the code. This code
@@ -1994,9 +2097,10 @@ class RunQueueExecuteScenequeue(RunQueueExecute):
         # e.g. do_sometask_setscene[depends] = "targetname:do_someothertask_setscene"
         # Note that anything explicitly depended upon will have its reverse dependencies removed to avoid circular dependencies
         for tid in self.rqdata.runq_setscene_tids:
-                (mc, fn, taskname, _) = split_tid_mcfn(tid)
-                realtid = fn + ":" + taskname + "_setscene"
+                (mc, fn, taskname, taskfn) = split_tid_mcfn(tid)
+                realtid = tid + "_setscene"
                 idepends = self.rqdata.taskData[mc].taskentries[realtid].idepends
+                self.stamps[tid] = bb.build.stampfile(taskname + "_setscene", self.rqdata.dataCaches[mc], taskfn, noextra=True)
                 for (depname, idependtask) in idepends:
 
                     if depname not in self.rqdata.taskData[mc].build_targets:
@@ -2060,7 +2164,7 @@ class RunQueueExecuteScenequeue(RunQueueExecute):
             for tid in self.sq_revdeps:
                 (mc, fn, taskname, taskfn) = split_tid_mcfn(tid)
 
-                taskdep = self.rqdata.dataCaches[mc].task_deps[fn]
+                taskdep = self.rqdata.dataCaches[mc].task_deps[taskfn]
 
                 if 'noexec' in taskdep and taskname in taskdep['noexec']:
                     noexec.append(tid)
@@ -2081,7 +2185,7 @@ class RunQueueExecuteScenequeue(RunQueueExecute):
                     continue
 
                 sq_fn.append(fn)
-                sq_hashfn.append(self.rqdata.dataCaches[mc].hashfn[fn])
+                sq_hashfn.append(self.rqdata.dataCaches[mc].hashfn[taskfn])
                 sq_hash.append(self.rqdata.runtaskentries[tid].hash)
                 sq_taskname.append(taskname)
                 sq_task.append(tid)
@@ -2129,8 +2233,8 @@ class RunQueueExecuteScenequeue(RunQueueExecute):
     def check_taskfail(self, task):
         if self.rqdata.setscenewhitelist:
             realtask = task.split('_setscene')[0]
-            (mc, fn, taskname, _) = split_tid_mcfn(realtask)
-            pn = self.rqdata.dataCaches[mc].pkg_fn[fn]
+            (mc, fn, taskname, taskfn) = split_tid_mcfn(realtask)
+            pn = self.rqdata.dataCaches[mc].pkg_fn[taskfn]
             if not check_setscene_enforce_whitelist(pn, taskname, self.rqdata.setscenewhitelist):
                 logger.error('Task %s.%s failed' % (pn, taskname + "_setscene"))
                 self.rq.state = runQueueCleanUp
@@ -2173,7 +2277,7 @@ class RunQueueExecuteScenequeue(RunQueueExecute):
         if self.stats.active < self.number_tasks:
             # Find the next setscene to run
             for nexttask in self.rqdata.runq_setscene_tids:
-                if nexttask in self.runq_buildable and nexttask not in self.runq_running:
+                if nexttask in self.runq_buildable and nexttask not in self.runq_running and self.stamps[nexttask] not in self.build_stamps.values():
                     if nexttask in self.unskippable:
                         logger.debug(2, "Setscene task %s is unskippable" % nexttask)
                     if nexttask not in self.unskippable and len(self.sq_revdeps[nexttask]) > 0 and self.sq_revdeps[nexttask].issubset(self.scenequeue_covered) and self.check_dependencies(nexttask, self.sq_revdeps[nexttask], True):
@@ -2217,14 +2321,16 @@ class RunQueueExecuteScenequeue(RunQueueExecute):
 
             taskdep = self.rqdata.dataCaches[mc].task_deps[taskfn]
             if 'fakeroot' in taskdep and taskname in taskdep['fakeroot'] and not self.cooker.configuration.dry_run:
-                if not self.rq.fakeworker:
-                    self.rq.start_fakeworker(self)
-                self.rq.fakeworker[mc].process.stdin.write(b"<runtask>" + pickle.dumps((taskfn, task, taskname, True, self.cooker.collection.get_file_appends(taskfn), taskdepdata)) + b"</runtask>")
+                if not mc in self.rq.fakeworker:
+                    self.rq.start_fakeworker(self, mc)
+                self.rq.fakeworker[mc].process.stdin.write(b"<runtask>" + pickle.dumps((taskfn, task, taskname, True, self.cooker.collection.get_file_appends(taskfn), taskdepdata, False)) + b"</runtask>")
                 self.rq.fakeworker[mc].process.stdin.flush()
             else:
-                self.rq.worker[mc].process.stdin.write(b"<runtask>" + pickle.dumps((taskfn, task, taskname, True, self.cooker.collection.get_file_appends(taskfn), taskdepdata)) + b"</runtask>")
+                self.rq.worker[mc].process.stdin.write(b"<runtask>" + pickle.dumps((taskfn, task, taskname, True, self.cooker.collection.get_file_appends(taskfn), taskdepdata, False)) + b"</runtask>")
                 self.rq.worker[mc].process.stdin.flush()
 
+            self.build_stamps[task] = bb.build.stampfile(taskname, self.rqdata.dataCaches[mc], taskfn, noextra=True)
+            self.build_stamps2.append(self.build_stamps[task])
             self.runq_running.add(task)
             self.stats.taskActive()
             if self.stats.active < self.number_tasks:
@@ -2260,7 +2366,7 @@ class RunQueueExecuteScenequeue(RunQueueExecute):
         def getsetscenedeps(tid):
             deps = set()
             (mc, fn, taskname, _) = split_tid_mcfn(tid)
-            realtid = fn + ":" + taskname + "_setscene"
+            realtid = tid + "_setscene"
             idepends = self.rqdata.taskData[mc].taskentries[realtid].idepends
             for (depname, idependtask) in idepends:
                 if depname not in self.rqdata.taskData[mc].build_targets:

bitbake/lib/bb/server/process.py

@@ -233,6 +233,9 @@ class BitBakeProcessServerConnection(BitBakeBaseServerConnection):
         self.ui_channel.close()
         self.event_queue.close()
         self.event_queue.setexit()
+        # XXX: Call explicity close in _writer to avoid
+        # fd leakage because isn't called on Queue.close()
+        self.event_queue._writer.close()
 
 # Wrap Queue to provide API which isn't server implementation specific
 class ProcessEventQueue(multiprocessing.queues.Queue):
@@ -264,7 +267,6 @@ class ProcessEventQueue(multiprocessing.queues.Queue):
                 sys.exit(1)
             return None
 
-
 class BitBakeServer(BitBakeBaseServer):
     def initServer(self, single_use=True):
         # establish communication channels.  We use bidirectional pipes for

bitbake/lib/bb/siggen.py

@@ -307,8 +307,8 @@ class SignatureGeneratorBasic(SignatureGenerator):
                 pass
             raise err
 
-    def dump_sigs(self, dataCaches, options):
-        for fn in self.taskdeps:
+    def dump_sigfn(self, fn, dataCaches, options):
+        if fn in self.taskdeps:
             for task in self.taskdeps[fn]:
                 tid = fn + ":" + task
                 (mc, _, _) = bb.runqueue.split_tid(tid)
@@ -352,9 +352,14 @@ def dump_this_task(outfile, d):
     bb.parse.siggen.dump_sigtask(fn, task, outfile, "customfile:" + referencestamp)
 
 def clean_basepath(a):
+    mc = None
+    if a.startswith("multiconfig:"):
+        _, mc, a = a.split(":", 2)
     b = a.rsplit("/", 2)[1] + a.rsplit("/", 2)[2]
     if a.startswith("virtual:"):
         b = b + ":" + a.rsplit(":", 1)[0]
+    if mc:
+        b = b + ":multiconfig:" + mc
     return b
 
 def clean_basepaths(a):

bitbake/lib/bb/ui/uihelper.py

@@ -32,7 +32,10 @@ class BBUIHelper:
 
     def eventHandler(self, event):
         if isinstance(event, bb.build.TaskStarted):
-            self.running_tasks[event.pid] = { 'title' : "%s %s" % (event._package, event._task), 'starttime' : time.time() }
+            if event._mc != "default":
+                self.running_tasks[event.pid] = { 'title' : "mc:%s:%s %s" % (event._mc, event._package, event._task), 'starttime' : time.time() }
+            else:
+                self.running_tasks[event.pid] = { 'title' : "%s %s" % (event._package, event._task), 'starttime' : time.time() }
             self.running_pids.append(event.pid)
             self.needUpdate = True
         elif isinstance(event, bb.build.TaskSucceeded):

bitbake/lib/bb/utils.py

@@ -523,12 +523,8 @@ def md5_file(filename):
     """
     Return the hex string representation of the MD5 checksum of filename.
     """
-    try:
-        import hashlib
-        m = hashlib.md5()
-    except ImportError:
-        import md5
-        m = md5.new()
+    import hashlib
+    m = hashlib.md5()
 
     with open(filename, "rb") as f:
         for line in f:
@@ -538,14 +534,9 @@ def md5_file(filename):
 def sha256_file(filename):
     """
     Return the hex string representation of the 256-bit SHA checksum of
-    filename.  On Python 2.4 this will return None, so callers will need to
-    handle that by either skipping SHA checks, or running a standalone sha256sum
-    binary.
+    filename.
     """
-    try:
-        import hashlib
-    except ImportError:
-        return None
+    import hashlib
 
     s = hashlib.sha256()
     with open(filename, "rb") as f:
@@ -557,10 +548,7 @@ def sha1_file(filename):
     """
     Return the hex string representation of the SHA1 checksum of the filename
     """
-    try:
-        import hashlib
-    except ImportError:
-        return None
+    import hashlib
 
     s = hashlib.sha1()
     with open(filename, "rb") as f:

bitbake/lib/bblayers/query.py

@@ -242,17 +242,22 @@ skipped recipes will also be listed, with a " (skipped)" suffix.
 
 Lists recipes with the bbappends that apply to them as subitems.
 """
-
-        logger.plain('=== Appended recipes ===')
+        if args.pnspec:
+            logger.plain('=== Matched appended recipes ===')
+        else:
+            logger.plain('=== Appended recipes ===')
 
         pnlist = list(self.tinfoil.cooker_data.pkg_pn.keys())
         pnlist.sort()
         appends = False
         for pn in pnlist:
+            if args.pnspec and pn != args.pnspec:
+                continue
+
             if self.show_appends_for_pn(pn):
                 appends = True
 
-        if self.show_appends_for_skipped():
+        if not args.pnspec and self.show_appends_for_skipped():
             appends = True
 
         if not appends:
@@ -477,7 +482,8 @@ NOTE: .bbappend files can impact the dependencies.
         parser_show_recipes.add_argument('-i', '--inherits', help='only list recipes that inherit the named class', metavar='CLASS', default='')
         parser_show_recipes.add_argument('pnspec', nargs='?', help='optional recipe name specification (wildcards allowed, enclose in quotes to avoid shell expansion)')
 
-        self.add_command(sp, 'show-appends', self.do_show_appends)
+        parser_show_appends = self.add_command(sp, 'show-appends', self.do_show_appends)
+        parser_show_appends.add_argument('pnspec', nargs='?', help='optional recipe name specification (wildcards allowed, enclose in quotes to avoid shell expansion)')
 
         parser_show_cross_depends = self.add_command(sp, 'show-cross-depends', self.do_show_cross_depends)
         parser_show_cross_depends.add_argument('-f', '--filenames', help='show full file path', action='store_true')

bitbake/lib/prserv/serv.py

@@ -5,12 +5,7 @@ import threading
 import queue
 import socket
 import io
-
-try:
-    import sqlite3
-except ImportError:
-    from pysqlite2 import dbapi2 as sqlite3
-
+import sqlite3
 import bb.server.xmlrpc
 import prserv
 import prserv.db
@@ -242,12 +237,25 @@ class PRServer(SimpleXMLRPCServer):
 
         sys.stdout.flush()
         sys.stderr.flush()
+
+        # We could be called from a python thread with io.StringIO as
+        # stdout/stderr or it could be 'real' unix fd forking where we need
+        # to physically close the fds to prevent the program launching us from
+        # potentially hanging on a pipe. Handle both cases.
         si = open('/dev/null', 'r')
+        try:
+            os.dup2(si.fileno(),sys.stdin.fileno())
+        except (AttributeError, io.UnsupportedOperation):
+            sys.stdin = si
         so = open(self.logfile, 'a+')
-        se = so
-        os.dup2(si.fileno(),sys.stdin.fileno())
-        os.dup2(so.fileno(),sys.stdout.fileno())
-        os.dup2(se.fileno(),sys.stderr.fileno())
+        try:
+            os.dup2(so.fileno(),sys.stdout.fileno())
+        except (AttributeError, io.UnsupportedOperation):
+            sys.stdout = so
+        try:
+            os.dup2(so.fileno(),sys.stderr.fileno())
+        except (AttributeError, io.UnsupportedOperation):
+            sys.stderr = so
 
         # Clear out all log handlers prior to the fork() to avoid calling
         # event handlers not part of the PRserver

documentation/dev-manual/dev-manual-newbie.xml

@@ -1394,83 +1394,182 @@
         can be reviewed and merged by the appropriate maintainer.
     </para>
 
-    <para>
-        Before submitting any change, be sure to find out who you should be
-        notifying.
-        Several methods exist through which you find out who you should be copying
-        or notifying:
-        <itemizedlist>
-            <listitem><para><emphasis>Maintenance File:</emphasis>
-                Examine the <filename>maintainers.inc</filename> file, which is
-                located in the
-                <link linkend='source-directory'>Source Directory</link>
-                at <filename>meta-poky/conf/distro/include</filename>, to
-                see who is responsible for code.
-                </para></listitem>
-            <listitem><para><emphasis>Board Support Package (BSP) README Files:</emphasis>
-                For BSP maintainers of supported BSPs, you can examine
-                individual BSP <filename>README</filename> files.
-                In addition, some layers (such as the <filename>meta-intel</filename> layer),
-                include a <filename>MAINTAINERS</filename> file which contains
-                a list of all supported BSP maintainers for that layer.
-                </para></listitem>
-            <listitem><para><emphasis>Search by File:</emphasis>
-                Using <link linkend='git'>Git</link>, you can enter the
-                following command to bring up a short list of all commits
-                against a specific file:
-                <literallayout class='monospaced'>
+    <section id='submit-change-overview'>
+        <title>Overview</title>
+
+        <para>
+            The Yocto Project uses a mailing list and patch-based workflow
+            that is similar to the Linux kernel but contains important
+            differences.
+            In general, a mailing list exists through which you can submit
+            patches.
+            The specific mailing list you need to use depends on the
+            location of the code you are changing.
+            Each component (e.g. layer) should have a
+            <filename>README</filename> file that indicates where to send
+            the changes and which process to follow.
+        </para>
+
+        <para>
+            You can send the patch to the mailing list using whichever approach
+            you feel comfortable with to generate the patch.
+            Once sent, the patch is usually reviewed by the community at large.
+            If somebody has concerns with the patch, they will usually voice
+            their concern over the mailing list.
+            If a patch does not receive any negative reviews, the maintainer of
+            the affected layer typically takes the patch, tests it, and then
+            based on successful testing, merges the patch.
+        </para>
+
+        <para>
+            Specific to OpenEmbedded-Core, two commonly used testing trees
+            exist:
+            <itemizedlist>
+                <listitem><para>
+                    <emphasis>"ross/mut" branch:</emphasis>
+                    The "mut" (master-under-test) tree
+                    exists in the <filename>poky-contrib</filename> repository
+                    in the
+                    <ulink url='&YOCTO_GIT_URL;'>Yocto Project source repositories</ulink>.
+                    </para></listitem>
+                <listitem><para>
+                    <emphasis>"master-next" branch:</emphasis>
+                    This branch is part of the main
+                    "poky" repository in the Yocto Project source repositories.
+                    </para></listitem>
+            </itemizedlist>
+            Maintainers use these branches to test submissions prior to merging
+            patches.
+            Thus, you can get an idea of the status of a patch based on
+            whether the patch has been merged into one of these branches.
+        </para>
+
+        <para>
+            This system is imperfect and patches can sometimes get lost in the
+            flow.
+            Asking about the status of a patch is reasonable if the patch
+            has been idle for a while with no feedback.
+            The Yocto Project does have plans to use
+            <ulink url='https://en.wikipedia.org/wiki/Patchwork_(software)'>Patchwork</ulink>
+            to track the status of patches and also to automatically preview
+            patches.
+        </para>
+
+        <para>
+            The following sections provide general instructions for both
+            pushing changes upstream and for submitting changes as patches.
+        </para>
+    </section>
+
+    <section id='submit-change-submissions-to-poky'>
+        <title>Submissions to Poky</title>
+
+        <para>
+            The "poky" repository, which is the Yocto Project's reference build
+            environment, is a hybrid repository that contains several
+            individual pieces (e.g. BitBake, OpenEmbedded-Core, meta-yocto,
+            documentation, and so forth) built using the combo-layer tool.
+            The upstream location used for submitting changes varies by
+            component:
+            <itemizedlist>
+                <listitem><para>
+                    <emphasis>Core Metadata:</emphasis>
+                    Send your patch to the
+                    <ulink url='http://lists.openembedded.org/mailman/listinfo/openembedded-core'>openembedded-core</ulink>
+                    mailing list.  For example, a change to anything under
+                    the <filename>meta</filename> or
+                    <filename>scripts</filename> directories should be sent
+                    to this mailing list.
+                    </para></listitem>
+                <listitem><para>
+                    <emphasis>BitBake:</emphasis>
+                    For changes to BitBake (i.e. anything under the
+                    <filename>bitbake</filename> directory), send your patch
+                    to the
+                    <ulink url='http://lists.openembedded.org/mailman/listinfo/bitbake-devel'>bitbake-devel</ulink>
+                    mailing list.
+                    </para></listitem>
+                <listitem><para>
+                    <emphasis>"meta-yocto-bsp" and "meta-poky" trees:</emphasis>
+                    These trees are
+                    part of the "meta-yocto" repository in the Yocto Project
+                    source repositories.
+                    Use the
+                    <ulink url='https://lists.yoctoproject.org/listinfo/poky'>poky</ulink>
+                    mailing list.
+                    </para></listitem>
+            </itemizedlist>
+        </para>
+    </section>
+
+    <section id='submit-change-submissions-to-other-layers'>
+        <title>Submissions to Other Layers</title>
+
+        <para>
+            For changes to other layers hosted in the Yocto Project source
+            repositories (i.e. <filename>yoctoproject.org</filename>), tools,
+            and the Yocto Project documentation, use the
+            <ulink url='https://lists.yoctoproject.org/listinfo/yocto'>Yocto Project</ulink>
+            general mailing list.
+            <note>
+                Sometimes a layer's documentation specifies to use a
+                particular mailing list.
+                If so, use that list.
+            </note>
+            For additional recipes that do not fit into the core Metadata, you
+            should determine which layer the recipe should go into and submit
+            the change in the manner recommended by the documentation (e.g.
+            the <filename>README</filename> file) supplied with the layer.
+            If in doubt, please ask on the Yocto general mailing list or on
+            the openembedded-devel mailing list.
+        </para>
+    </section>
+
+    <section id='submit-change-patch-submission-details'>
+        <title>Patch Submission Details</title>
+
+        <para>
+            When submitting any change, you can check who you should be
+            notifying.
+            Use either of these methods to find out:
+            <itemizedlist>
+                <listitem><para>
+                    <emphasis>Maintenance File:</emphasis>
+                    Examine the <filename>maintainers.inc</filename> file, which is
+                    located in the
+                    <link linkend='source-directory'>Source Directory</link>
+                    at <filename>meta-poky/conf/distro/include</filename>, to
+                    see who is responsible for code.
+                    </para></listitem>
+                <listitem><para>
+                    <emphasis>Search by File:</emphasis>
+                    Using <link linkend='git'>Git</link>, you can enter the
+                    following command to bring up a short list of all commits
+                    against a specific file:
+                    <literallayout class='monospaced'>
      git shortlog -- <replaceable>filename</replaceable>
-                </literallayout>
-                Just provide the name of the file for which you are interested.
-                The information returned is not ordered by history but does
-                include a list of all committers grouped by name.
-                From the list, you can see who is responsible for the bulk of
-                the changes against the file.
-                </para></listitem>
-        </itemizedlist>
-    </para>
+                    </literallayout>
+                    Just provide the name of the file for which you are interested.
+                    The information returned is not ordered by history but does
+                    include a list of all committers grouped by name.
+                    From the list, you can see who is responsible for the bulk of
+                    the changes against the file.
+                    </para></listitem>
+            </itemizedlist>
+        </para>
 
-    <para>
-        For a list of the Yocto Project and related mailing lists, see the
-        "<ulink url='&YOCTO_DOCS_REF_URL;#resources-mailinglist'>Mailing lists</ulink>" section in
-        the Yocto Project Reference Manual.
-    </para>
+        <para>
+            For a list of the Yocto Project and related mailing lists, see the
+            "<ulink url='&YOCTO_DOCS_REF_URL;#resources-mailinglist'>Mailing lists</ulink>"
+            section in the Yocto Project Reference Manual.
+        </para>
 
-    <para>
-        Here is some guidance on which mailing list to use for what type of change:
-        <itemizedlist>
-            <listitem><para>For changes to the core
-                <link linkend='metadata'>Metadata</link>, send your patch to the
-                <ulink url='&OE_LISTS_URL;/listinfo/openembedded-core'>openembedded-core</ulink> mailing list.
-                For example, a change to anything under the <filename>meta</filename> or
-                <filename>scripts</filename> directories
-                should be sent to this mailing list.</para></listitem>
-            <listitem><para>For changes to BitBake (anything under the <filename>bitbake</filename>
-                directory), send your patch to the
-                <ulink url='&OE_LISTS_URL;/listinfo/bitbake-devel'>bitbake-devel</ulink> mailing list.</para></listitem>
-            <listitem><para>For changes to <filename>meta-poky</filename>, send your patch to the
-                <ulink url='&YOCTO_LISTS_URL;/listinfo/poky'>poky</ulink> mailing list.</para></listitem>
-            <listitem><para>For changes to other layers hosted on
-                <filename>yoctoproject.org</filename> (unless the
-                layer's documentation specifies otherwise), tools, and Yocto Project
-                documentation, use the
-                <ulink url='&YOCTO_LISTS_URL;/listinfo/yocto'>yocto</ulink> mailing list.</para></listitem>
-            <listitem><para>For additional recipes that do not fit into the core Metadata,
-                you should determine which layer the recipe should go into and submit the
-                change in the manner recommended by the documentation (e.g. README) supplied
-                with the layer. If in doubt, please ask on the
-                <ulink url='&YOCTO_LISTS_URL;/listinfo/yocto'>yocto</ulink> or
-                <ulink url='&OE_LISTS_URL;/listinfo/openembedded-devel'>openembedded-devel</ulink>
-                mailing lists.</para></listitem>
-        </itemizedlist>
-    </para>
-
-    <para>
-        When you send a patch, be sure to include a "Signed-off-by:"
-        line in the same style as required by the Linux kernel.
-        Adding this line signifies that you, the submitter, have agreed to the Developer's Certificate of Origin 1.1
-        as follows:
-        <literallayout class='monospaced'>
+        <para>
+            When you send a patch, be sure to include a "Signed-off-by:"
+            line in the same style as required by the Linux kernel.
+            Adding this line signifies that you, the submitter, have agreed
+            to the Developer's Certificate of Origin 1.1 as follows:
+            <literallayout class='monospaced'>
      Developer's Certificate of Origin 1.1
 
      By making a contribution to this project, I certify that:
@@ -1496,68 +1595,75 @@
          personal information I submit with it, including my sign-off) is
          maintained indefinitely and may be redistributed consistent with
          this project or the open source license(s) involved.
-        </literallayout>
-    </para>
+            </literallayout>
+        </para>
 
-    <para>
-        In a collaborative environment, it is necessary to have some sort of standard
-        or method through which you submit changes.
-        Otherwise, things could get quite chaotic.
-        One general practice to follow is to make small, controlled changes.
-        Keeping changes small and isolated aids review, makes merging/rebasing easier
-        and keeps the change history clean when anyone needs to refer to it in future.
-    </para>
+        <para>
+            In a collaborative environment, it is necessary to have some sort
+            of standard or method through which you submit changes.
+            Otherwise, things could get quite chaotic.
+            One general practice to follow is to make small, controlled changes.
+            Keeping changes small and isolated aids review, makes
+            merging/rebasing easier and keeps the change history clean should
+            anyone need to refer to it in future.
+        </para>
 
-    <para>
-        When you make a commit, you must follow certain standards established by the
-        OpenEmbedded and Yocto Project development teams.
-        For each commit, you must provide a single-line summary of the change and you
-        should almost always provide a more detailed description of what you did (i.e.
-        the body of the commit message).
-        The only exceptions for not providing a detailed description would be if your
-        change is a simple, self-explanatory change that needs no further description
-        beyond the summary.
-        Here are the guidelines for composing a commit message:
-        <itemizedlist>
-            <listitem><para>Provide a single-line, short summary of the change.
-                This summary is typically viewable in the "shortlist" of changes.
-                Thus, providing something short and descriptive that gives the reader
-                a summary of the change is useful when viewing a list of many commits.
-                This short description should be prefixed by the recipe name (if changing a recipe), or
-                else the short form path to the file being changed.
-                </para></listitem>
-            <listitem><para>For the body of the commit message, provide detailed information
-                that describes what you changed, why you made the change, and the approach
-                you used. It may also be helpful if you mention how you tested the change.
-                Provide as much detail as you can in the body of the commit message.
-                </para></listitem>
-            <listitem><para>
-                If the change addresses a specific bug or issue that is
-                associated with a bug-tracking ID, include a reference to that
-                ID in your detailed description.
-                For example, the Yocto Project uses a specific convention for
-                bug references - any commit that addresses a specific bug should
-                use the following form for the detailed description:
-                <literallayout class='monospaced'>
+        <para>
+            When you make a commit, you must follow certain standards
+            established by the OpenEmbedded and Yocto Project development teams.
+            For each commit, you must provide a single-line summary of the
+            change and you should almost always provide a more detailed
+            description of what you did (i.e. the body of the commit message).
+            The only exceptions for not providing a detailed description would
+            be if your change is a simple, self-explanatory change that needs
+            no further description beyond the summary.
+            Here are the guidelines for composing a commit message:
+            <itemizedlist>
+                <listitem><para>
+                    Provide a single-line, short summary of the change.
+                    This summary is typically viewable in the "shortlist" of
+                    changes.
+                    Thus, providing something short and descriptive that
+                    gives the reader a summary of the change is useful when
+                    viewing a list of many commits.
+                    You should prefix this short description with the recipe
+                    name (if changing a recipe), or else with the short form
+                    path to the file being changed.
+                    </para></listitem>
+                <listitem><para>
+                    For the body of the commit message, provide detailed
+                    information that describes what you changed, why you made
+                    the change, and the approach you used.
+                    It might also be helpful if you mention how you tested
+                    the change.
+                    Provide as much detail as you can in the body of the
+                    commit message.
+                    </para></listitem>
+                <listitem><para>
+                    If the change addresses a specific bug or issue that is
+                    associated with a bug-tracking ID, include a reference
+                    to that ID in your detailed description.
+                    For example, the Yocto Project uses a specific convention
+                    for bug references - any commit that addresses a specific
+                    bug should use the following form for the detailed
+                    description:
+                    <literallayout class='monospaced'>
      Fixes [YOCTO #<replaceable>bug-id</replaceable>]
 
      <replaceable>detailed description of change</replaceable>
-                </literallayout></para></listitem>
+                    </literallayout>
+                    </para></listitem>
                 Where <replaceable>bug-id</replaceable> is replaced with the
                 specific bug ID from the Yocto Project Bugzilla instance.
-        </itemizedlist>
-    </para>
+            </itemizedlist>
+        </para>
 
-    <para>
-        You can find more guidance on creating well-formed commit messages at this OpenEmbedded
-        wiki page:
-        <ulink url='&OE_HOME_URL;/wiki/Commit_Patch_Message_Guidelines'></ulink>.
-    </para>
-
-    <para>
-        The next two sections describe general instructions for both pushing
-        changes upstream and for submitting changes as patches.
-    </para>
+        <para>
+            You can find more guidance on creating well-formed commit messages
+            at this OpenEmbedded wiki page:
+            <ulink url='&OE_HOME_URL;/wiki/Commit_Patch_Message_Guidelines'></ulink>.
+        </para>
+    </section>
 
     <section id='pushing-a-change-upstream'>
         <title>Using Scripts to Push a Change Upstream and Request a Pull</title>

documentation/dev-manual/dev-manual-start.xml

@@ -328,6 +328,96 @@
     </para>
 </section>
 
+<section id='flashing-images-using-bmaptool'>
+    <title>Flashing Images Using <filename>bmaptool</filename></title>
+
+    <para>
+        An easy way to flash an image to a bootable device is to use
+        <filename>bmaptool</filename>, which is integrated into the
+        OpenEmbedded build system.
+    </para>
+
+    <para>
+        Following, is an example that shows how to flash a Wic image.
+        <note>
+            You can use <filename>bmaptool</filename> to flash any
+            type of image.
+        </note>
+        Use these steps to flash an image using
+        <filename>bmaptool</filename>:
+        <note>
+            Unless you are able to install the
+            <filename>bmap-tools</filename> package as mentioned in the note
+            in the second bullet of step 3 further down, you will need to build
+            <filename>bmaptool</filename> before using it.
+            Build the tool using the following command:
+            <literallayout class='monospaced'>
+     $ bitbake bmap-tools-native
+            </literallayout>
+        </note>
+        <orderedlist>
+            <listitem><para>
+                Add the following to your <filename>local.conf</filename>
+                file:
+                <literallayout class='monospaced'>
+     IMAGE_FSTYPES += "wic wic.bmap"
+                </literallayout>
+                </para></listitem>
+            <listitem><para>
+                Either have your image ready (pre-built) or take the step
+                build the image:
+                <literallayout class='monospaced'>
+     $ bitbake <replaceable>image</replaceable>
+                </literallayout>
+                </para></listitem>
+            <listitem><para>
+                Flash the image to the media by using
+                <filename>bmaptool</filename> depending on your particular
+                setup:
+                <itemizedlist>
+                    <listitem><para>
+                        If you have write access to the media,
+                        use this command form:
+                        <literallayout class='monospaced'>
+     $ oe-run-native bmaptool copy ./tmp/deploy/images/qemux86-64/core-image-minimal-<replaceable>machine</replaceable>.wic /dev/sd<replaceable>X</replaceable>
+                        </literallayout>
+                        </para></listitem>
+                    <listitem><para>
+                        If you do not have write access to
+                        the media, use the following
+                        commands:
+                        <literallayout class='monospaced'>
+     $ sudo bash
+     $ PATH=tmp/sysroots/x86_64-linux/usr/bin/ bmaptool copy ./tmp/deploy/images/qemux86-64/core-image-minimal-<replaceable>machine</replaceable>.wic /dev/sd<replaceable>X</replaceable>
+                        </literallayout>
+                        <note>
+                            If you are using Ubuntu or Debian distributions,
+                            you can install the
+                            <filename>bmap-tools</filename> package using the
+                            following command and then use the tool
+                            without specifying
+                            <filename>PATH</filename> even from the
+                            root account:
+                            <literallayout class='monospaced'>
+     $ sudo apt-get install bmap-tools
+                            </literallayout>
+                        </note>
+                        </para></listitem>
+                </itemizedlist>
+                </para></listitem>
+        </orderedlist>
+    </para>
+
+    <para>
+        For help on the <filename>bmaptool</filename> command, use either of
+        the following commands:
+        <literallayout class='monospaced'>
+     $ bmaptool --help
+     $ oe-run-native bmaptool --help
+        </literallayout>
+    </para>
+</section>
+
 <section id='using-pre-built-binaries-and-qemu'>
     <title>Using Pre-Built Binaries and QEMU</title>
 
@@ -352,6 +442,11 @@
         <ulink url='&YOCTO_DOCS_REF_URL;#var-SDKMACHINE'><filename>SDKMACHINE</filename></ulink>).
         If you are not using an SDK type image, you need to separately download
         and install the stand-alone Yocto Project cross-toolchain tarball.
+        See the
+        "<ulink url='&YOCTO_DOCS_SDK_URL;#sdk-appendix-obtain'>Obtaining the SDK</ulink>"
+        appendix in the Yocto Project Software Development Kit (SDK)
+        Developer's Guide for more information on locating and installing
+        cross-toolchains.
     </para>
 
     <para>

documentation/poky.ent

@@ -61,13 +61,13 @@
 <!ENTITY OE_INIT_PATH "&YOCTO_POKY;/oe-init-build-env">
 <!ENTITY OE_INIT_FILE "oe-init-build-env">
 <!ENTITY UBUNTU_HOST_PACKAGES_ESSENTIAL "gawk wget git-core diffstat unzip texinfo gcc-multilib \
-     build-essential chrpath socat">
+     build-essential chrpath socat cpio python python3 pip3 pexpect">
 <!ENTITY FEDORA_HOST_PACKAGES_ESSENTIAL "gawk make wget tar bzip2 gzip python3 unzip perl patch \
      diffutils diffstat git cpp gcc gcc-c++ glibc-devel texinfo chrpath \
      ccache perl-Data-Dumper perl-Text-ParseWords perl-Thread-Queue perl-bignum socat \
-     findutils which">
+     findutils which file cpio python pip3 pexpect">
 <!ENTITY OPENSUSE_HOST_PACKAGES_ESSENTIAL "python gcc gcc-c++ git chrpath make wget python-xml \
-     diffstat makeinfo python-curses patch socat">
+     diffstat makeinfo python-curses patch socat python3 python3-curses tar pip3 pexpect">
 <!ENTITY CENTOS_HOST_PACKAGES_ESSENTIAL "gawk make wget tar bzip2 gzip python unzip perl patch \
      diffutils diffstat git cpp gcc gcc-c++ glibc-devel texinfo chrpath socat \
-     perl-Data-Dumper perl-Text-ParseWords perl-Thread-Queue">
+     perl-Data-Dumper perl-Text-ParseWords perl-Thread-Queue pip3 pexpect">

documentation/ref-manual/ref-classes.xml

@@ -1873,11 +1873,22 @@
     </para>
 
     <para>
-        This means that each built kernel module is packaged separately and inter-module
-        dependencies are created by parsing the <filename>modinfo</filename> output.
-        If all modules are required, then installing the <filename>kernel-modules</filename>
-        package installs all packages with modules and various other kernel packages
-        such as <filename>kernel-vmlinux</filename>.
+        This means that each built kernel module is packaged separately and
+        inter-module dependencies are created by parsing the
+        <filename>modinfo</filename> output.
+        If all modules are required, then installing the
+        <filename>kernel-modules</filename> package installs all packages with
+        modules and various other kernel packages such as
+        <filename>kernel-vmlinux</filename>.
+    </para>
+
+    <para>
+        The <filename>kernel</filename> class contains logic that allows
+        you to embed an initial RAM filesystem (initramfs) image when
+        you build the kernel image.
+        For information on how to build an initramfs, see the
+        "<ulink url='&YOCTO_DOCS_DEV_URL;#building-an-initramfs-image'>Building an Initial RAM Filesystem (initramfs) Image</ulink>"
+        section in the Yocto Project Development Manual.
     </para>
 
     <para>

documentation/ref-manual/ref-features.xml

@@ -297,6 +297,12 @@
                     Enables logging postinstall script runs to
                     the <filename>/var/log/postinstall.log</filename> file
                     on first boot of the image on the target system.
+                    <note>
+                        To make the <filename>/var/log</filename> directory
+                        on the target persistent, use the
+                        <ulink url='&YOCTO_DOCS_REF_URL;#var-VOLATILE_LOG_DIR'><filename>VOLATILE_LOG_DIR</filename></ulink>
+                        variable by setting it to "no".
+                    </note>
                     </para></listitem>
                 <listitem><para><emphasis>ptest-pkgs:</emphasis>
                     Installs ptest packages for all ptest-enabled recipes.

documentation/ref-manual/ref-images.xml

@@ -54,9 +54,6 @@
             <listitem><para><filename>core-image-clutter</filename>:
                 An image with support for the Open GL-based toolkit Clutter, which enables development of
                 rich and animated graphical user interfaces.</para></listitem>
-            <listitem><para><filename>core-image-directfb</filename>:
-                An image that uses <filename>directfb</filename> instead of X11.
-                </para></listitem>
             <listitem><para><filename>core-image-full-cmdline</filename>:
                 A console-only image with more full-featured Linux system
                 functionality installed.</para></listitem>

documentation/ref-manual/ref-variables.xml

@@ -37,7 +37,7 @@
        <link linkend='var-S'>S</link>
        <link linkend='var-T'>T</link>
        <link linkend='var-UBOOT_CONFIG'>U</link>
-<!--               <link linkend='var-glossary-v'>V</link> -->
+       <link linkend='var-VOLATILE_LOG_DIR'>V</link>
        <link linkend='var-WARN_QA'>W</link>
        <link linkend='var-XSERVER'>X</link>
 <!--               <link linkend='var-glossary-y'>Y</link> -->
@@ -2273,12 +2273,13 @@
 
         <glossentry id='var-CONFIG_INITRAMFS_SOURCE'><glossterm>CONFIG_INITRAMFS_SOURCE</glossterm>
             <info>
-                CONFIG_INITRAMFS_SOURCE[doc] = "Identifies the initial RAM disk (initramfs) source files. The OpenEmbedded build system receives and uses this kernel Kconfig variable as an environment variable."
+                CONFIG_INITRAMFS_SOURCE[doc] = "Identifies the initial RAM filesystem (initramfs) source files. The OpenEmbedded build system receives and uses this kernel Kconfig variable as an environment variable."
             </info>
             <glossdef>
                 <para role="glossdeffirst">
 <!--                <para role="glossdeffirst"><imagedata fileref="figures/define-generic.png" /> -->
-                    Identifies the initial RAM disk (initramfs) source files.
+                    Identifies the initial RAM filesystem (initramfs) source
+                    files.
                     The OpenEmbedded build system receives and uses
                     this kernel Kconfig variable as an environment variable.
                     By default, the variable is set to null ("").
@@ -2304,6 +2305,12 @@
                     If you specify multiple directories and files, the
                     initramfs image will be the aggregate of all of them.
                 </para>
+
+                <para>
+                    For information on creating an initramfs, see the
+                    "<ulink url='&YOCTO_DOCS_DEV_URL;#building-an-initramfs-image'>Building an Initial RAM Filesystem (initramfs) Image</ulink>"
+                    section in the Yocto Project Development Manual.
+                </para>
             </glossdef>
         </glossentry>
 
@@ -5405,9 +5412,12 @@
                         variable to specify packages for installation.
                         Instead, use the
                         <link linkend='var-PACKAGE_INSTALL'><filename>PACKAGE_INSTALL</filename></link>
-                        variable, which allows the initial RAM disk (initramfs)
-                        recipe to use a fixed set of packages and not be
-                        affected by <filename>IMAGE_INSTALL</filename>.
+                        variable, which allows the initial RAM filesystem
+                        (initramfs) recipe to use a fixed set of packages and
+                        not be affected by <filename>IMAGE_INSTALL</filename>.
+                        For information on creating an initramfs, see the
+                        "<ulink url='&YOCTO_DOCS_DEV_URL;#building-an-initramfs-image'>Building an Initial RAM Filesystem (initramfs) Image</ulink>"
+                        section in the Yocto Project Development Manual.
                     </note>
                 </para>
 
@@ -6133,13 +6143,13 @@ recipes-graphics/xorg-font/font-alias_1.0.3.bb:PR = "${INC_PR}.3"
 
         <glossentry id='var-INITRAMFS_FSTYPES'><glossterm>INITRAMFS_FSTYPES</glossterm>
             <info>
-                INITRAMFS_FSTYPES[doc] = "Defines the format for the output image of an initial RAM disk (initramfs), which is used during boot."
+                INITRAMFS_FSTYPES[doc] = "Defines the format for the output image of an initial RAM filesystem (initramfs), which is used during boot."
             </info>
             <glossdef>
                 <para role="glossdeffirst">
 <!--                <para role="glossdeffirst"><imagedata fileref="figures/define-generic.png" /> -->
                     Defines the format for the output image of an initial
-                    RAM disk (initramfs), which is used during boot.
+                    RAM filesystem (initramfs), which is used during boot.
                     Supported formats are the same as those supported by the
                     <link linkend='var-IMAGE_FSTYPES'><filename>IMAGE_FSTYPES</filename></link>
                     variable.
@@ -6152,7 +6162,7 @@ recipes-graphics/xorg-font/font-alias_1.0.3.bb:PR = "${INC_PR}.3"
                     <ulink url='&YOCTO_DOCS_DEV_URL;#source-directory'>Source Directory</ulink>,
                     is "cpio.gz".
                     The Linux kernel's initramfs mechanism, as opposed to the
-                    initial RAM disk
+                    initial RAM filesystem
                     <ulink url='https://en.wikipedia.org/wiki/Initrd'>initrd</ulink>
                     mechanism, expects an optionally compressed cpio
                     archive.
@@ -6162,7 +6172,7 @@ recipes-graphics/xorg-font/font-alias_1.0.3.bb:PR = "${INC_PR}.3"
 
         <glossentry id='var-INITRAMFS_IMAGE'><glossterm>INITRAMFS_IMAGE</glossterm>
             <info>
-                INITRAMFS_IMAGE[doc] = "Specifies the PROVIDES name of an image recipe that is used to build an initial RAM disk (initramfs) image."
+                INITRAMFS_IMAGE[doc] = "Specifies the PROVIDES name of an image recipe that is used to build an initial RAM filesystem (initramfs) image."
             </info>
             <glossdef>
                 <para role="glossdeffirst">
@@ -6170,7 +6180,7 @@ recipes-graphics/xorg-font/font-alias_1.0.3.bb:PR = "${INC_PR}.3"
                     Specifies the
                     <link linkend='var-PROVIDES'><filename>PROVIDES</filename></link>
                     name of an image recipe that is used to build an initial
-                    RAM disk (initramfs) image.
+                    RAM filesystem (initramfs) image.
                     An initramfs provides a temporary root filesystem used for
                     early system initialization (e.g. loading of modules
                     needed to locate and mount the "real" root filesystem).
@@ -6211,17 +6221,21 @@ recipes-graphics/xorg-font/font-alias_1.0.3.bb:PR = "${INC_PR}.3"
                 </para>
 
                 <para>
-                    Finally, for more information you can also see the
+                    For more information, you can also see the
                     <link linkend='var-INITRAMFS_IMAGE_BUNDLE'><filename>INITRAMFS_IMAGE_BUNDLE</filename></link>
                     variable, which allows the generated image to be bundled
                     inside the kernel image.
+                    Additionally, for information on creating an initramfs, see
+                    the
+                    "<ulink url='&YOCTO_DOCS_DEV_URL;#building-an-initramfs-image'>Building an Initial RAM Filesystem (initramfs) Image</ulink>"
+                    section in the Yocto Project Development Manual.
                 </para>
             </glossdef>
         </glossentry>
 
         <glossentry id='var-INITRAMFS_IMAGE_BUNDLE'><glossterm>INITRAMFS_IMAGE_BUNDLE</glossterm>
             <info>
-                INITRAMFS_IMAGE_BUNDLE[doc] = "Controls whether or not the image recipe specified by INITRAMFS_IMAGE is run through an extra pass (do_bundle_initramfs) during kernel compilation in order to build a single binary that contains both the kernel image and the initial RAM disk (initramfs)."
+                INITRAMFS_IMAGE_BUNDLE[doc] = "Controls whether or not the image recipe specified by INITRAMFS_IMAGE is run through an extra pass (do_bundle_initramfs) during kernel compilation in order to build a single binary that contains both the kernel image and the initial RAM filesystem (initramfs)."
             </info>
             <glossdef>
                 <para role="glossdeffirst">
@@ -6231,8 +6245,8 @@ recipes-graphics/xorg-font/font-alias_1.0.3.bb:PR = "${INC_PR}.3"
                     is run through an extra pass
                     (<link linkend='ref-tasks-bundle_initramfs'><filename>do_bundle_initramfs</filename></link>)
                     during kernel compilation in order to build a single binary
-                    that contains both the kernel image and the initial RAM disk
-                    (initramfs).
+                    that contains both the kernel image and the initial RAM
+                    filesystem (initramfs) image.
                     This makes use of the
                     <link linkend='var-CONFIG_INITRAMFS_SOURCE'><filename>CONFIG_INITRAMFS_SOURCE</filename></link>
                     kernel feature.
@@ -6279,6 +6293,9 @@ recipes-graphics/xorg-font/font-alias_1.0.3.bb:PR = "${INC_PR}.3"
                     See the
                     <ulink url='&YOCTO_GIT_URL;/cgit/cgit.cgi/poky/tree/meta-poky/conf/local.conf.sample.extended'><filename>local.conf.sample.extended</filename></ulink>
                     file for additional information.
+                    Also, for information on creating an initramfs, see the
+                    "<ulink url='&YOCTO_DOCS_DEV_URL;#building-an-initramfs-image'>Building an Initial RAM Filesystem (initramfs) Image</ulink>"
+                    section in the Yocto Project Development Manual.
                 </para>
             </glossdef>
         </glossentry>
@@ -9105,9 +9122,12 @@ recipes-graphics/xorg-font/font-alias_1.0.3.bb:PR = "${INC_PR}.3"
                     the
                     <link linkend='images-core-image-minimal-initramfs'><filename>core-image-minimal-initramfs</filename></link>
                     image.
-                    When working with an initial RAM disk (initramfs)
+                    When working with an initial RAM filesystem (initramfs)
                     image, use the <filename>PACKAGE_INSTALL</filename>
                     variable.
+                    For information on creating an initramfs, see the
+                    "<ulink url='&YOCTO_DOCS_DEV_URL;#building-an-initramfs-image'>Building an Initial RAM Filesystem (initramfs) Image</ulink>"
+                    section in the Yocto Project Development Manual.
                 </para>
             </glossdef>
         </glossentry>
@@ -15524,8 +15544,30 @@ recipes-graphics/xorg-font/font-alias_1.0.3.bb:PR = "${INC_PR}.3"
 
     </glossdiv>
 
-<!--            <glossdiv id='var-glossary-v'><title>V</title>-->
-<!--            </glossdiv>-->
+    <glossdiv id='var-glossary-v'><title>V</title>
+
+        <glossentry id='var-VOLATILE_LOG_DIR'><glossterm>VOLATILE_LOG_DIR</glossterm>
+            <info>
+               VOLATILE_LOG_DIR[doc] = "Specifies the persistence of the target's /var/log directory, which is used to house postinstall target log files."
+            </info>
+            <glossdef>
+                <para role="glossdeffirst">
+<!--                <para role="glossdeffirst"><imagedata fileref="figures/define-generic.png" /> -->
+                    Specifies the persistence of the target's
+                    <filename>/var/log</filename> directory, which is used to
+                    house postinstall target log files.
+                </para>
+
+                <para>
+                    By default, <filename>VOLATILE_LOG_DIR</filename> is set
+                    to "yes", which means the file is not persistent.
+                    You can override this setting by setting the
+                    variable to "no" to make the log directory persistent.
+                </para>
+            </glossdef>
+        </glossentry>
+
+    </glossdiv>
 
     <glossdiv id='var-glossary-w'><title>W</title>
 
@@ -15548,6 +15590,26 @@ recipes-graphics/xorg-font/font-alias_1.0.3.bb:PR = "${INC_PR}.3"
             </glossdef>
         </glossentry>
 
+        <glossentry id='var-WKS_FILE'><glossterm>WKS_FILE</glossterm>
+            <info>
+               WKS_FILE[doc] = "Specifies the name of the wic kickstart file."
+            </info>
+            <glossdef>
+                <para role="glossdeffirst">
+                    Specifies the location of the Wic
+                    kickstart file that is used by the OpenEmbedded build
+                    system to create a partitioned image
+                    (<replaceable>image</replaceable><filename>.wic</filename>).
+                    For information on how to create a
+                    partitioned image, see the
+                    "<ulink url='&YOCTO_DOCS_DEV_URL;#creating-wic-images-oe'>Creating Partitioned Images</ulink>"
+                    section.
+                    For details on the kickstart file format, see the
+                    "<ulink url='&YOCTO_DOCS_DEV_URL;#openembedded-kickstart-wks-reference'>OpenEmbedded Kickstart (<filename>.wks</filename>) Reference</ulink>.
+                </para>
+            </glossdef>
+        </glossentry>
+
         <glossentry id='var-WORKDIR'><glossterm>WORKDIR</glossterm>
             <info>
                WORKDIR[doc] = "The pathname of the working directory in which the OpenEmbedded build system builds a recipe. This directory is located within the TMPDIR directory structure and changes as different packages are built."

documentation/ref-manual/technical-details.xml

@@ -1445,14 +1445,23 @@
 
             <para>
                 The <filename>LIC_FILES_CHKSUM</filename>
-                variable contains checksums of the license text in the source code for the recipe.
-                Following is an example of how to specify <filename>LIC_FILES_CHKSUM</filename>:
+                variable contains checksums of the license text in the source
+                code for the recipe.
+                Following is an example of how to specify
+                <filename>LIC_FILES_CHKSUM</filename>:
                 <literallayout class='monospaced'>
      LIC_FILES_CHKSUM = "file://COPYING;md5=xxxx \
                          file://licfile1.txt;beginline=5;endline=29;md5=yyyy \
                          file://licfile2.txt;endline=50;md5=zzzz \
                          ..."
                 </literallayout>
+                <note>
+                    When using "beginline" and "endline", realize that line
+                    numbering begins with one and not zero.
+                    Also, the lines included are inclusive (e.g. lines five
+                    through 29 in the previous example for
+                    <filename>licfile1.txt</filename>.
+                </note>
             </para>
 
             <para>
@@ -1474,7 +1483,8 @@
 
             <para>
                 The first line locates a file in
-                <filename>${S}/src/ls.c</filename>.
+                <filename>${S}/src/ls.c</filename> and isolates lines five
+                through 16 as license text.
                 The second line refers to a file in
                 <filename><link linkend='var-WORKDIR'>WORKDIR</link></filename>.
             </para>

documentation/ref-manual/usingpoky.xml

@@ -69,6 +69,39 @@
             <literallayout class='monospaced'>
      $ bitbake <replaceable>target</replaceable>
             </literallayout>
+            <note>
+                <para>
+                    If you experience a build error due to resources
+                    temporarily being unavailable and it appears you
+                    should not be having this issue, it might be due
+                    to the combination of a 4.3+ Linux kernel and
+                    <filename>systemd</filename> version 228+
+                    (i.e. see this
+                    <ulink url='http://unix.stackexchange.com/questions/253903/creating-threads-fails-with-resource-temporarily-unavailable-with-4-3-kernel'>link</ulink>
+                    for information).
+                </para>
+
+                <para>
+                    To work around this issue, you can try either
+                    of the following:
+                    <itemizedlist>
+                        <listitem><para>
+                            Try the build again.
+                            </para></listitem>
+                        <listitem><para>
+                            Modify the "DefaultTasksMax"
+                            <filename>systemd</filename> parameter
+                            by uncommenting it and setting it to
+                            "infinity".
+                            You can find this parameter in the
+                            <filename>system.conf</filename> file
+                            located in
+                            <filename>/etc/systemd</filename>
+                            on most systems.
+                            </para></listitem>
+                    </itemizedlist>
+                </para>
+            </note>
         </para>
 
         <para>
@@ -391,14 +424,6 @@
                     <link linkend='var-ASSUME_PROVIDED'><filename>ASSUME_PROVIDED</filename></link>
                     are not listed.
                     </para></listitem>
-                <listitem><para>
-                    <filename>pn-depends.dot</filename>: A graph showing
-                    dependencies between build-time targets (recipes).
-                    </para></listitem>
-                <listitem><para>
-                    <filename>package-depends.dot</filename>: A graph showing
-                    known dependencies between runtime targets.
-                    </para></listitem>
                 <listitem><para>
                     <filename>task-depends.dot</filename>: A graph showing
                     dependencies between tasks.

documentation/yocto-project-qs/yocto-project-qs.xml

@@ -16,11 +16,26 @@
                 Permission is granted to copy, distribute and/or modify this document under
                 the terms of the <ulink type="http" url="http://creativecommons.org/licenses/by-sa/2.0/uk/">Creative Commons Attribution-Share Alike 2.0 UK: England &amp; Wales</ulink> as published by Creative Commons.
             </para>
-            <note>
-                For the latest version of this manual associated with this
-                Yocto Project release, see the
-                <ulink url='&YOCTO_DOCS_QS_URL;'>Yocto Project Quick Start</ulink>
-                from the Yocto Project website.
+            <note><title>Manual Notes</title>
+                <itemizedlist>
+                    <listitem><para>
+                        For the latest version of the Yocto Project Quick
+                        Start associated with this Yocto Project release,
+                        see the Yocto Project Quick Start from the
+                        <ulink url='&YOCTO_HOME_URL;'>Yocto Project website</ulink>.
+                        </para></listitem>
+                    <listitem><para>
+                        Later releases of the Yocto Project might exist and the
+                        associated Yocto Project Quick Start for any later
+                        release can be found at
+                        <ulink url='&YOCTO_HOME_URL;/documentation'></ulink>.
+                        </para></listitem>
+                    <listitem><para>
+                        For an in-development version of the Yocto Project
+                        Quick Start, see
+                        <ulink url='&YOCTO_DOCS_URL;/latest/yocto-project-qs/yocto-project-qs.html'></ulink>.
+                        </para></listitem>
+                </itemizedlist>
             </note>
         </legalnotice>
 
@@ -312,7 +327,8 @@
                         </para></listitem>
                     <listitem><para><emphasis>OpenSUSE</emphasis>
                         <literallayout class='monospaced'>
-     $ sudo zypper install &OPENSUSE_HOST_PACKAGES_ESSENTIAL; libSDL-devel xterm
+     $ sudo zypper install &OPENSUSE_HOST_PACKAGES_ESSENTIAL; \
+     libSDL-devel xterm
                         </literallayout>
                         </para></listitem>
                     <listitem><para><emphasis>CentOS</emphasis>
@@ -566,14 +582,47 @@
                         <note>
                             Depending on the number of processors and cores, the
                             amount of RAM, the speed of your Internet connection
-                            and other factors, the build process could take several
-                            hours the first time you run it.
+                            and other factors, the build process could take
+                            several hours the first time you run it.
                             Subsequent builds run much faster since parts of the
                             build are cached.
                         </note>
                         <literallayout class='monospaced'>
      $ bitbake core-image-sato
                         </literallayout>
+                        <note>
+                            <para>
+                                If you experience a build error due to resources
+                                temporarily being unavailable and it appears you
+                                should not be having this issue, it might be due
+                                to the combination of a 4.3+ Linux kernel and
+                                <filename>systemd</filename> version 228+
+                                (i.e. see this
+                                <ulink url='http://unix.stackexchange.com/questions/253903/creating-threads-fails-with-resource-temporarily-unavailable-with-4-3-kernel'>link</ulink>
+                                for information).
+                            </para>
+
+                            <para>
+                                To work around this issue, you can try either
+                                of the following:
+                                <itemizedlist>
+                                    <listitem><para>
+                                        Try the build again.
+                                        </para></listitem>
+                                    <listitem><para>
+                                        Modify the "DefaultTasksMax"
+                                        <filename>systemd</filename> parameter
+                                        by uncommenting it and setting it to
+                                        "infinity".
+                                        You can find this parameter in the
+                                        <filename>system.conf</filename> file
+                                        located in
+                                        <filename>/etc/systemd</filename>
+                                        on most systems.
+                                        </para></listitem>
+                                </itemizedlist>
+                            </para>
+                        </note>
                         For information on using the
                         <filename>bitbake</filename> command, see the
                         "<ulink url='&YOCTO_DOCS_REF_URL;#usingpoky-components-bitbake'>BitBake</ulink>"
@@ -724,6 +773,39 @@
                         <literallayout class='monospaced'>
      $ bitbake core-image-base
                         </literallayout>
+                        <note>
+                            <para>
+                                If you experience a build error due to resources
+                                temporarily being unavailable and it appears you
+                                should not be having this issue, it might be due
+                                to the combination of a 4.3+ Linux kernel and
+                                <filename>systemd</filename> version 228+
+                                (i.e. see this
+                                <ulink url='http://unix.stackexchange.com/questions/253903/creating-threads-fails-with-resource-temporarily-unavailable-with-4-3-kernel'>link</ulink>
+                                for information).
+                            </para>
+
+                            <para>
+                                To work around this issue, you can try either
+                                of the following:
+                                <itemizedlist>
+                                    <listitem><para>
+                                        Try the build again.
+                                        </para></listitem>
+                                    <listitem><para>
+                                        Modify the "DefaultTasksMax"
+                                        <filename>systemd</filename> parameter
+                                        by uncommenting it and setting it to
+                                        "infinity".
+                                        You can find this parameter in the
+                                        <filename>system.conf</filename> file
+                                        located in
+                                        <filename>/etc/systemd</filename>
+                                        on most systems.
+                                        </para></listitem>
+                                </itemizedlist>
+                            </para>
+                        </note>
                         Once the build completes, the resulting console-only image
                         is located in the Build Directory here:
                         <literallayout class='monospaced'>

meta-poky/conf/distro/include/maintainers.inc

@@ -584,7 +584,7 @@ RECIPE_MAINTAINER_pn-pax = "Hongxu Jia <hongxu.jia@windriver.com>"
 RECIPE_MAINTAINER_pn-pax-utils = "Hongxu Jia <hongxu.jia@windriver.com>"
 RECIPE_MAINTAINER_pn-pbzip2 = "Alexander Kanavin <alexander.kanavin@intel.com>"
 RECIPE_MAINTAINER_pn-pciutils = "Chen Qi <Qi.Chen@windriver.com>"
-RECIPE_MAINTAINER_pn-pcmanfm = "Edwin Plauchu <edwin.plauchu.camacho@linux.intel.com>"
+RECIPE_MAINTAINER_pn-pcmanfm = "Alexander Kanavin <alexander.kanavin@intel.com>"
 RECIPE_MAINTAINER_pn-pcmciautils = "Robert Yang <liezhi.yang@windriver.com>"
 RECIPE_MAINTAINER_pn-perf = "Bruce Ashfield <bruce.ashfield@windriver.com>"
 RECIPE_MAINTAINER_pn-perl = "Aníbal Limón <anibal.limon@linux.intel.com>"

meta-poky/conf/distro/poky.conf

@@ -1,7 +1,7 @@
 DISTRO = "poky"
 DISTRO_NAME = "Poky (Yocto Project Reference Distro)"
-DISTRO_VERSION = "2.2"
-DISTRO_CODENAME = "morty"
+DISTRO_VERSION = "2.2+snapshot-${DATE}"
+DISTRO_CODENAME = "master"
 SDK_VENDOR = "-pokysdk"
 SDK_VERSION := "${@'${DISTRO_VERSION}'.replace('snapshot-${DATE}','snapshot')}"
 
@@ -79,9 +79,11 @@ SANITY_TESTED_DISTROS ?= " \
             ubuntu-15.04 \n \
             ubuntu-15.10 \n \
             ubuntu-16.04 \n \
+            ubuntu-16.10 \n \
             fedora-22 \n \
             fedora-23 \n \
             fedora-24 \n \
+            fedora-25 \n \
             centos-7 \n \
             debian-8 \n \
             opensuse-13.2 \n \
@@ -93,7 +95,7 @@ SANITY_TESTED_DISTROS ?= " \
 # that breaks the format and have been previously discussed on the mailing list 
 # with general agreement from the core team.
 #
-OELAYOUT_ABI = "11"
+OELAYOUT_ABI = "12"
 
 # add poky sanity bbclass
 INHERIT += "poky-sanity"

meta-selftest/lib/oeqa/runtime/cases/selftest.py

@@ -1,13 +1,9 @@
-import os
+from oeqa.runtime.case import OERuntimeTestCase
+from oeqa.core.decorator.depends import OETestDepends
 
-from oeqa.oetest import oeRuntimeTest, skipModule
-from oeqa.utils.commands import runCmd
-from oeqa.utils.decorators import *
+class Selftest(OERuntimeTestCase):
 
-class Selftest(oeRuntimeTest):
-
-    @skipUnlessPassed("test_ssh")
-    @tag("selftest_package_install")
+    @OETestDepends(['ssh.SSHTest.test_ssh'])
     def test_install_package(self):
         """
         Summary: Check basic package installation functionality.
@@ -21,8 +17,7 @@ class Selftest(oeRuntimeTest):
         (status, output) = self.target.run("socat -V")
         self.assertEqual(status, 0, msg="socat is not installed")
 
-    @skipUnlessPassed("test_install_package")
-    @tag("selftest_package_install")
+    @OETestDepends(['selftest.Selftest.test_install_package'])
     def test_verify_unistall(self):
         """
         Summary: Check basic package installation functionality.
@@ -34,22 +29,3 @@ class Selftest(oeRuntimeTest):
 
         (status, output) = self.target.run("socat -V")
         self.assertNotEqual(status, 0, msg="socat is still installed")
-
-    @tag("selftest_sdk")
-    def test_sdk(self):
-
-        result = runCmd("env -0")
-        sdk_path = search_sdk_path(result.output)
-        self.assertTrue(sdk_path, msg="Can't find SDK path")
-
-        tar_cmd = os.path.join(sdk_path, "tar")
-        result = runCmd("%s --help" % tar_cmd)
-
-def search_sdk_path(env):
-    for line in env.split("\0"):
-        (key, _, value) = line.partition("=")
-        if key == "PATH":
-            for path in value.split(":"):
-                if "pokysdk" in path:
-                    return path
-    return ""

meta-selftest/recipes-test/aspell/aspell_0.0.0.1.bb

@@ -14,6 +14,8 @@ SRC_URI = "${GNU_MIRROR}/aspell/aspell-${PV}.tar.gz"
 SRC_URI[md5sum] = "e66a9c9af6a60dc46134fdacf6ce97d7"
 SRC_URI[sha256sum] = "f52583a83a63633701c5f71db3dc40aab87b7f76b29723aeb27941eff42df6e1"
 
+EXCLUDE_FROM_WORLD = "1"
+
 PACKAGECONFIG ??= ""
 PACKAGECONFIG[curses] = "--enable-curses,--disable-curses,ncurses"
 

meta-selftest/recipes-test/devtool/devtool-test-localonly.bb

@@ -4,3 +4,4 @@ INHIBIT_DEFAULT_DEPS = "1"
 SRC_URI = "file://file1 \
            file://file2"
 
+EXCLUDE_FROM_WORLD = "1"

meta-selftest/recipes-test/devtool/devtool-test-patch-gz.bb

@@ -14,3 +14,4 @@ SRC_URI[sha256sum] = "67c2fc94a7ecedbaae0d1837e82e93d1d98f4a6d759828860e552119af
 
 inherit autotools pkgconfig
 
+EXCLUDE_FROM_WORLD = "1"

meta-selftest/recipes-test/devtool/devtool-test-subdir.bb

@@ -5,3 +5,5 @@ SRC_URI = "file://devtool-test-subdir.tar.gz \
            file://testfile;subdir=${BPN}"
 
 S = "${WORKDIR}/${BPN}"
+
+EXCLUDE_FROM_WORLD = "1"

meta-selftest/recipes-test/devtool/devtool-upgrade-test1_1.5.3.bb

@@ -12,5 +12,7 @@ PR = "r5"
 
 S = "${WORKDIR}/pv-${PV}"
 
+EXCLUDE_FROM_WORLD = "1"
+
 inherit autotools
 

meta-selftest/recipes-test/devtool/devtool-upgrade-test1_1.5.3.bb.upgraded

@@ -10,5 +10,7 @@ SRC_URI[sha256sum] = "9dd45391806b0ed215abee4c5ac1597d018c386fe9c1f5afd2f6bc3b07
 
 S = "${WORKDIR}/pv-${PV}"
 
+EXCLUDE_FROM_WORLD = "1"
+
 inherit autotools
 

meta-selftest/recipes-test/devtool/devtool-upgrade-test2_git.bb

@@ -15,4 +15,6 @@ SRC_URI = "git://git.yoctoproject.org/dbus-wait"
 
 S = "${WORKDIR}/git"
 
+EXCLUDE_FROM_WORLD = "1"
+
 inherit autotools pkgconfig

meta-selftest/recipes-test/devtool/devtool-upgrade-test2_git.bb.upgraded

@@ -14,4 +14,6 @@ SRC_URI = "git://git.yoctoproject.org/dbus-wait"
 
 S = "${WORKDIR}/git"
 
+EXCLUDE_FROM_WORLD = "1"
+
 inherit autotools pkgconfig

meta-selftest/recipes-test/recipetool/selftest-recipetool-appendfile.bb

@@ -15,6 +15,8 @@ SRC_URI = "file://installscript.sh \
            file://selftest-replaceme-inst-todir-globfile \
            file://selftest-replaceme-inst-func"
 
+EXCLUDE_FROM_WORLD = "1"
+
 install_extrafunc() {
 	install -m 0644 ${WORKDIR}/selftest-replaceme-inst-func ${D}${datadir}/selftest-replaceme-inst-func
 }

meta-skeleton/recipes-skeleton/useradd/useradd-example.bb

@@ -14,6 +14,8 @@ S = "${WORKDIR}"
 
 PACKAGES =+ "${PN}-user3"
 
+EXCLUDE_FROM_WORLD = "1"
+
 inherit useradd
 
 # You must set USERADD_PACKAGES when you inherit useradd. This

meta-yocto-bsp/conf/machine/beaglebone.conf

@@ -16,7 +16,7 @@ include conf/machine/include/tune-cortexa8.inc
 
 IMAGE_FSTYPES += "tar.bz2 jffs2 wic wic.bmap"
 EXTRA_IMAGECMD_jffs2 = "-lnp "
-WKS_FILE = "sdimage-bootpart.wks"
+WKS_FILE ?= "beaglebone.wks"
 IMAGE_INSTALL_append = " kernel-devicetree kernel-image-zimage"
 do_image_wic[depends] += "mtools-native:do_populate_sysroot dosfstools-native:do_populate_sysroot"
 
@@ -31,7 +31,7 @@ KERNEL_EXTRA_ARGS += "LOADADDR=${UBOOT_ENTRYPOINT}"
 
 SPL_BINARY = "MLO"
 UBOOT_SUFFIX = "img"
-UBOOT_MACHINE = "am335x_evm_config"
+UBOOT_MACHINE = "am335x_boneblack_config"
 UBOOT_ENTRYPOINT = "0x80008000"
 UBOOT_LOADADDRESS = "0x80008000"
 

meta-yocto-bsp/conf/machine/edgerouter.conf

@@ -21,6 +21,6 @@ MACHINE_EXTRA_RRECOMMENDS = " kernel-modules"
 IMAGE_FSTYPES ?= "jffs2 tar.bz2 wic wic.bmap"
 JFFS2_ERASEBLOCK = "0x10000"
 
-WKS_FILE ?= "sdimage-bootpart.wks"
+WKS_FILE ?= "edgerouter.wks"
 IMAGE_BOOT_FILES ?= "vmlinux;vmlinux.64"
 do_image_wic[depends] += "mtools-native:do_populate_sysroot dosfstools-native:do_populate_sysroot"

meta-yocto-bsp/conf/machine/include/genericx86-common.inc

@@ -21,6 +21,6 @@ GLIBC_ADDONS = "nptl"
 EXTRA_OECONF_append_pn-matchbox-panel-2 = " --with-battery=acpi"
 
 IMAGE_FSTYPES += "wic wic.bmap"
-WKS_FILE ?= "mkefidisk.wks"
+WKS_FILE ?= "genericx86.wks"
 do_image_wic[depends] += "gptfdisk-native:do_populate_sysroot"
 do_image_wic[recrdeptask] += "do_bootimg"

meta-yocto-bsp/conf/machine/mpc8315e-rdb.conf

@@ -29,3 +29,7 @@ MACHINE_EXTRA_RRECOMMENDS = " kernel-modules"
 
 IMAGE_FSTYPES ?= "jffs2 tar.bz2"
 JFFS2_ERASEBLOCK = "0x4000"
+
+IMAGE_FSTYPES += "wic"
+WKS_FILE ?= 'mpc8315e-rdb.wks'
+IMAGE_BOOT_FILES ?= "u-boot.bin uImage uImage-mpc8315erdb.dtb;dtb"

meta-yocto-bsp/lib/oeqa/selftest/gummiboot.py

@@ -1,83 +0,0 @@
-from oeqa.selftest.base import oeSelfTest
-from oeqa.utils.commands import runCmd, bitbake, get_bb_var, runqemu
-from oeqa.utils.decorators import testcase
-import re
-import os
-import sys
-import logging
-
-
-class Gummiboot(oeSelfTest):
-
-    def _common_setup(self):
-        """
-        Common setup for test cases: 1101, 1103
-        """
-
-        # Set EFI_PROVIDER = "gummiboot" and MACHINE = "genericx86-64" in conf/local.conf
-        features = 'EFI_PROVIDER = "gummiboot"\n'
-        features += 'MACHINE = "genericx86-64"'
-        self.append_config(features)
-
-    def _common_build(self):
-        """
-        Common build for test cases: 1101, 1103
-        """
-
-        # Build a genericx86-64/efi gummiboot image
-        bitbake('syslinux syslinux-native parted-native dosfstools-native mtools-native core-image-minimal')
-
-
-    @testcase(1101)
-    def test_efi_gummiboot_images_can_be_built(self):
-        """
-        Summary:     Check if efi/gummiboot images can be built
-        Expected:    1. File gummibootx64.efi should be available in build/tmp/deploy/images/genericx86-64
-                     2. Efi/gummiboot images can be built
-        Product:     oe-core
-        Author:      Ionut Chisanovici <ionutx.chisanovici@intel.com>
-        AutomatedBy: Daniel Istrate <daniel.alexandrux.istrate@intel.com>
-        """
-
-        # We'd use DEPLOY_DIR_IMAGE here, except that we need its value for
-        # MACHINE="genericx86-64 which is probably not the one configured
-        gummibootfile = os.path.join(get_bb_var('DEPLOY_DIR'), 'images', 'genericx86-64', 'gummibootx64.efi')
-
-        self._common_setup()
-
-        # Ensure we're actually testing that this gets built and not that
-        # it was around from an earlier build
-        bitbake('-c cleansstate gummiboot')
-        runCmd('rm -f %s' % gummibootfile)
-
-        self._common_build()
-
-        found = os.path.isfile(gummibootfile)
-        self.assertTrue(found, 'Gummiboot file %s not found' % gummibootfile)
-
-    @testcase(1103)
-    def test_wic_command_can_create_efi_gummiboot_installation_images(self):
-        """
-        Summary:     Check that wic command can create efi/gummiboot installation images
-        Expected:    A .direct file in folder /var/tmp/wic/ must be created.
-        Product:     oe-core
-        Author:      Ionut Chisanovici <ionutx.chisanovici@intel.com>
-        AutomatedBy: Daniel Istrate <daniel.alexandrux.istrate@intel.com>
-        """
-
-        self._common_setup()
-        self._common_build()
-
-        # Create efi/gummiboot installation images
-        wic_create_cmd = 'wic create mkgummidisk -e core-image-minimal'
-        result = runCmd(wic_create_cmd)
-
-        # Find file written by wic from output
-        res = re.search('(/var/tmp/wic/.*\.direct)', result.output)
-        if res:
-            direct_file = res.group(1)
-            # Check it actually exists
-            if not os.path.exists(direct_file):
-                self.fail('wic reported direct file "%s" does not exist; wic output:\n%s' % (direct_file, result.output))
-        else:
-            self.fail('No .direct file reported in wic output:\n%s' % result.output)

meta-yocto-bsp/wic/beaglebone.wks

@@ -0,0 +1,6 @@
+# short-description: Create SD card image for Beaglebone
+# long-description: Creates a partitioned SD card image for Beaglebone.
+# Boot files are located in the first vfat partition.
+
+part /boot --source bootimg-partition --ondisk mmcblk --fstype=vfat --label boot --active --align 4 --size 16
+part / --source rootfs --ondisk mmcblk --fstype=ext4 --label root --align 4

meta-yocto-bsp/wic/edgerouter.wks

@@ -0,0 +1,4 @@
+# short-description: Create SD card image for Edgerouter
+# long-description: Create a partitioned SD card image for MIPS64 Edgerouter reference hardware.
+part /boot --source bootimg-partition --ondisk sda --fstype=vfat --label boot --active --align 4 --size 16
+part / --source rootfs --ondisk sda --fstype=ext4 --label root --align 4

meta-yocto-bsp/wic/genericx86.wks

@@ -0,0 +1,7 @@
+# short-description: Create an EFI disk image for genericx86*
+# long-description: Creates a partitioned EFI disk image for genericx86* machines
+part /boot --source bootimg-efi --sourceparams="loader=grub-efi" --ondisk sda --label msdos --active --align 1024
+part / --source rootfs --ondisk sda --fstype=ext4 --label platform --align 1024 --use-uuid
+part swap --ondisk sda --size 44 --label swap1 --fstype=swap
+
+bootloader --ptable gpt --timeout=5 --append="rootfstype=ext4 console=ttyS0,115200 console=tty0"

meta-yocto-bsp/wic/mpc8315e-rdb.wks

@@ -0,0 +1,4 @@
+# short-description: Create SD card image for MPC8315E-RDB
+# long-description: Create a partitioned SD card image for Freescale MPC8315E-RDB reference hardware.
+part /boot --source bootimg-partition --ondisk sdb --fstype=ext3 --label boot
+part / --source rootfs --ondisk sdb --fstype=ext3 --label root

meta/classes/allarch.bbclass

@@ -2,10 +2,6 @@
 # This class is used for architecture independent recipes/data files (usually scripts)
 #
 
-# Expand STAGING_DIR_HOST since for cross-canadian/native/nativesdk, this will
-# point elsewhere after these changes.
-STAGING_DIR_HOST := "${STAGING_DIR_HOST}"
-
 PACKAGE_ARCH = "all"
 
 python () {
@@ -25,13 +21,16 @@ python () {
         d.setVar("TARGET_AS_ARCH", "none")
         d.setVar("TARGET_FPU", "")
         d.setVar("TARGET_PREFIX", "")
-        d.setVar("PACKAGE_EXTRA_ARCHS", "")
+        # Expand PACKAGE_EXTRA_ARCHS since the staging code needs this
+        # (this removes any dependencies from the hash perspective)
+        d.setVar("PACKAGE_EXTRA_ARCHS", d.getVar("PACKAGE_EXTRA_ARCHS"))
         d.setVar("SDK_ARCH", "none")
         d.setVar("SDK_CC_ARCH", "none")
         d.setVar("TARGET_CPPFLAGS", "none")
         d.setVar("TARGET_CFLAGS", "none")
         d.setVar("TARGET_CXXFLAGS", "none")
         d.setVar("TARGET_LDFLAGS", "none")
+        d.setVar("POPULATESYSROOTDEPS", "")
 
         # Avoid this being unnecessarily different due to nuances of
         # the target machine that aren't important for "all" arch

meta/classes/autotools.bbclass

@@ -27,7 +27,7 @@ inherit siteinfo
 # results for autoconf tests we cannot run at build time.
 export CONFIG_SITE = "${@siteinfo_get_files(d)}"
 
-acpaths = "default"
+acpaths ?= "default"
 EXTRA_AUTORECONF = "--exclude=autopoint"
 
 export lt_cv_sys_lib_dlsearch_path_spec = "${libdir} ${base_libdir}"
@@ -131,129 +131,15 @@ EXTRACONFFUNCS ??= ""
 
 EXTRA_OECONF_append = " ${PACKAGECONFIG_CONFARGS}"
 
-do_configure[prefuncs] += "autotools_preconfigure autotools_copy_aclocals ${EXTRACONFFUNCS}"
+do_configure[prefuncs] += "autotools_preconfigure autotools_aclocals ${EXTRACONFFUNCS}"
 do_configure[postfuncs] += "autotools_postconfigure"
 
-ACLOCALDIR = "${WORKDIR}/aclocal-copy"
-
-python autotools_copy_aclocals () {
-    import copy
-
-    s = d.getVar("AUTOTOOLS_SCRIPT_PATH")
-    if not os.path.exists(s + "/configure.in") and not os.path.exists(s + "/configure.ac"):
-        if not d.getVar("AUTOTOOLS_COPYACLOCAL", False):
-            return
-
-    taskdepdata = d.getVar("BB_TASKDEPDATA", False)
-    #bb.warn(str(taskdepdata))
-    pn = d.getVar("PN")
-    aclocaldir = d.getVar("ACLOCALDIR")
-    oe.path.remove(aclocaldir)
-    bb.utils.mkdirhier(aclocaldir)
-    start = None
-    configuredeps = []
-    # Detect bitbake -b usage
-    # Everything but quilt-native would have dependencies
-    nodeps = (pn != "quilt-native")
-
-    for dep in taskdepdata:
-        data = taskdepdata[dep]
-        if data[1] == "do_configure" and data[0] == pn:
-            start = dep
-        if not nodeps and start:
-            break
-        if nodeps and data[0] != pn:
-            nodeps = False
-    if start is None:
-        bb.fatal("Couldn't find ourself in BB_TASKDEPDATA?")
-
-    # We need to figure out which m4 files we need to expose to this do_configure task.
-    # This needs to match what would get restored from sstate, which is controlled 
-    # ultimately by calls from bitbake to setscene_depvalid().
-    # That function expects a setscene dependency tree. We build a dependency tree 
-    # condensed to do_populate_sysroot -> do_populate_sysroot dependencies, similar to 
-    # that used by setscene tasks. We can then call into setscene_depvalid() and decide
-    # which dependencies we can "see" and should expose the m4 files for.
-    setscenedeps = copy.deepcopy(taskdepdata)
-
-    start = set([start])
-
-    # Create collapsed do_populate_sysroot -> do_populate_sysroot tree
-    for dep in taskdepdata:
-        data = setscenedeps[dep]        
-        if data[1] != "do_populate_sysroot":
-            for dep2 in setscenedeps:
-                data2 = setscenedeps[dep2]
-                if dep in data2[3]:
-                    data2[3].update(setscenedeps[dep][3])
-                    data2[3].remove(dep)
-            if dep in start:
-                start.update(setscenedeps[dep][3])
-                start.remove(dep)
-            del setscenedeps[dep]
-
-    # Remove circular references
-    for dep in setscenedeps:
-        if dep in setscenedeps[dep][3]:
-            setscenedeps[dep][3].remove(dep)
-
-    # Direct dependencies should be present and can be depended upon
-    for dep in start:
-        configuredeps.append(setscenedeps[dep][0])
-
-    # Call into setscene_depvalid for each sub-dependency and only copy m4 files
-    # for ones that would be restored from sstate.
-    done = list(start)
-    next = list(start)
-    while next:
-        new = []
-        for dep in next:
-            data = setscenedeps[dep]
-            for datadep in data[3]:
-                if datadep in done:
-                    continue
-                taskdeps = {}
-                taskdeps[dep] = setscenedeps[dep][:2]
-                taskdeps[datadep] = setscenedeps[datadep][:2]
-                retval = setscene_depvalid(datadep, taskdeps, [], d)
-                if retval:
-                    bb.note("Skipping setscene dependency %s for m4 macro copying" % datadep)
-                    continue
-                done.append(datadep)
-                new.append(datadep)
-                configuredeps.append(setscenedeps[datadep][0])
-        next = new
-
-    cp = []
-    if nodeps:
-        bb.warn("autotools: Unable to find task dependencies, -b being used? Pulling in all m4 files")
-        for l in [d.expand("${STAGING_DATADIR_NATIVE}/aclocal/"), d.expand("${STAGING_DATADIR}/aclocal/")]:
-            cp.extend(os.path.join(l, f) for f in os.listdir(l))
-
-    for c in configuredeps:
-        if c.endswith("-native"):
-            manifest = d.expand("${SSTATE_MANIFESTS}/manifest-${BUILD_ARCH}-%s.populate_sysroot" % c)
-        elif c.startswith("nativesdk-"):
-            manifest = d.expand("${SSTATE_MANIFESTS}/manifest-${SDK_ARCH}_${SDK_OS}-%s.populate_sysroot" % c)
-        elif "-cross-" in c or "-crosssdk" in c:
-            continue
-        else:
-            manifest = d.expand("${SSTATE_MANIFESTS}/manifest-${MACHINE}-%s.populate_sysroot" % c)
-        try:
-            f = open(manifest, "r")
-            for l in f:
-                if "/aclocal/" in l and l.strip().endswith(".m4"):
-                    cp.append(l.strip())
-                elif "config_site.d/" in l:
-                    cp.append(l.strip())
-        except:
-            bb.warn("%s not found" % manifest)
-
-    for c in cp:
-        t = os.path.join(aclocaldir, os.path.basename(c))
-        if not os.path.exists(t):
-            os.symlink(c, t)
+ACLOCALDIR = "${STAGING_DATADIR}/aclocal"
+ACLOCALEXTRAPATH = ""
+ACLOCALEXTRAPATH_class-target = " -I ${STAGING_DATADIR_NATIVE}/aclocal/"
+ACLOCALEXTRAPATH_class-nativesdk = " -I ${STAGING_DATADIR_NATIVE}/aclocal/"
 
+python autotools_aclocals () {
     # Refresh variable with cache files
     d.setVar("CONFIG_SITE", siteinfo_get_files(d, aclocalcache=True))
 }
@@ -279,6 +165,7 @@ autotools_do_configure() {
 	if [ -e ${AUTOTOOLS_SCRIPT_PATH}/configure.in -o -e ${AUTOTOOLS_SCRIPT_PATH}/configure.ac ]; then
 		olddir=`pwd`
 		cd ${AUTOTOOLS_SCRIPT_PATH}
+		mkdir -p ${ACLOCALDIR}
 		ACLOCAL="aclocal --system-acdir=${ACLOCALDIR}/"
 		if [ x"${acpaths}" = xdefault ]; then
 			acpaths=
@@ -289,6 +176,7 @@ autotools_do_configure() {
 		else
 			acpaths="${acpaths}"
 		fi
+		acpaths="$acpaths ${ACLOCALEXTRAPATH}"
 		AUTOV=`automake --version | sed -e '1{s/.* //;s/\.[0-9]\+$//};q'`
 		automake --version
 		echo "AUTOV is $AUTOV"

meta/classes/base.bbclass

@@ -284,7 +284,7 @@ CLEANBROKEN = "0"
 
 addtask configure after do_patch
 do_configure[dirs] = "${B}"
-do_configure[deptask] = "do_populate_sysroot"
+do_prepare_recipe_sysroot[deptask] = "do_populate_sysroot"
 base_do_configure() {
 	if [ -n "${CONFIGURESTAMPFILE}" -a -e "${CONFIGURESTAMPFILE}" ]; then
 		if [ "`cat ${CONFIGURESTAMPFILE}`" != "${BB_TASKHASH}" ]; then

meta/classes/binconfig-disabled.bbclass

@@ -15,6 +15,7 @@ do_install_append () {
 		echo "echo 'ERROR: $x should not be used, use an alternative such as pkg-config' >&2" >> ${D}$x
 		echo "echo '--should-not-have-used-$x'" >> ${D}$x
 		echo "exit 1" >> ${D}$x
+		chmod +x ${D}$x
 	done
 }
 

meta/classes/blacklist.bbclass

@@ -38,7 +38,7 @@ python blacklist_multilib_eventhandler() {
 }
 
 python () {
-    blacklist = d.getVarFlag('PNBLACKLIST', d.getVar('PN'), True)
+    blacklist = d.getVarFlag('PNBLACKLIST', d.getVar('PN'))
 
     if blacklist:
         raise bb.parse.SkipPackage("Recipe is blacklisted: %s" % (blacklist))

meta/classes/bugzilla.bbclass

@@ -117,7 +117,7 @@ python bugzilla_eventhandler() {
         compon  = data.getVar("BUGZILLA_COMPONENT")
         version = data.getVar("BUGZILLA_VERSION")
 
-        proxy   = data.getVar('http_proxy', True )
+        proxy   = data.getVar('http_proxy')
         if (proxy):
             import urllib2
             s, u, p, hostport = urllib2._parse_proxy(proxy)
@@ -140,7 +140,7 @@ python bugzilla_eventhandler() {
                                                            "pv"      : data.getVar("PV"),
                                                            }
         log_file = glob.glob("%s/log.%s.*" % (event.data.getVar('T'), event.task))
-        text     = "The %s step in %s failed at %s for machine %s" % (e.task, data.getVar("PN"), data.getVar('DATETIME'), data.getVar( 'MACHINE', True ) )
+        text     = "The %s step in %s failed at %s for machine %s" % (e.task, data.getVar("PN"), data.getVar('DATETIME'), data.getVar('MACHINE') )
         if len(log_file) != 0:
             print >> debug_file, "Adding log file %s" % log_file[0]
             file = open(log_file[0], 'r')

meta/classes/buildstats.bbclass

@@ -197,7 +197,7 @@ python runqueue_stats () {
     # are available that we need to find the output directory.
     # The persistent SystemStats is stored in the datastore and
     # closed when the build is done.
-    system_stats = d.getVar('_buildstats_system_stats')
+    system_stats = d.getVar('_buildstats_system_stats', False)
     if not system_stats and isinstance(e, (bb.runqueue.sceneQueueTaskStarted, bb.runqueue.runQueueTaskStarted)):
         system_stats = buildstats.SystemStats(d)
         d.setVar('_buildstats_system_stats', system_stats)

meta/classes/ccache.bbclass

@@ -2,5 +2,8 @@ CCACHE = "${@bb.utils.which(d.getVar('PATH'), 'ccache') and 'ccache '}"
 export CCACHE_DIR ?= "${TMPDIR}/ccache/${MULTIMACH_HOST_SYS}/${PN}"
 CCACHE_DISABLE[unexport] = "1"
 
+DEPENDS_append_class-target = " ccache-native"
+DEPENDS[vardepvalueexclude] = " ccache-native"
+
 do_configure[dirs] =+ "${CCACHE_DIR}"
 do_kernel_configme[dirs] =+ "${CCACHE_DIR}"

meta/classes/cml1.bbclass

@@ -27,7 +27,7 @@ python do_menuconfig() {
         mtime = 0
 
     oe_terminal("${SHELL} -c \"make %s; if [ \$? -ne 0 ]; then echo 'Command failed.'; printf 'Press any key to continue... '; read r; fi\"" % d.getVar('KCONFIG_CONFIG_COMMAND'),
-                d.getVar('PN', True ) + ' Configuration', d)
+                d.getVar('PN') + ' Configuration', d)
 
     # FIXME this check can be removed when the minimum bitbake version has been bumped
     if hasattr(bb.build, 'write_taint'):

meta/classes/compress_doc.bbclass

@@ -249,7 +249,7 @@ python compress_doc_updatealternatives () {
                     d.setVarFlag('ALTERNATIVE_TARGET', new_name, new_target)
                 elif d.getVar('ALTERNATIVE_TARGET_%s' % pkg):
                     d.setVar('ALTERNATIVE_TARGET_%s' % pkg, new_target)
-                elif d.getVar('ALTERNATIVE_TARGET', old_name, True):
+                elif d.getVar('ALTERNATIVE_TARGET'):
                     d.setVar('ALTERNATIVE_TARGET', new_target)
 
                 new_names.append(new_name)

meta/classes/cross-canadian.bbclass

@@ -100,9 +100,9 @@ MULTIMACH_TARGET_SYS = "${PACKAGE_ARCH}${HOST_VENDOR}-${HOST_OS}"
 
 INHIBIT_DEFAULT_DEPS = "1"
 
-STAGING_DIR_HOST = "${STAGING_DIR}/${HOST_ARCH}-${SDKPKGSUFFIX}${HOST_VENDOR}-${HOST_OS}"
+STAGING_DIR_HOST = "${RECIPE_SYSROOT}"
 
-TOOLCHAIN_OPTIONS = " --sysroot=${STAGING_DIR}/${HOST_ARCH}-${SDKPKGSUFFIX}${HOST_VENDOR}-${HOST_OS}"
+TOOLCHAIN_OPTIONS = " --sysroot=${RECIPE_SYSROOT}"
 
 PATH_append = ":${TMPDIR}/sysroots/${HOST_ARCH}/${bindir_cross}"
 PKGHIST_DIR = "${TMPDIR}/pkghistory/${HOST_ARCH}-${SDKPKGSUFFIX}${HOST_VENDOR}-${HOST_OS}/"
@@ -169,6 +169,7 @@ USE_NLS = "${SDKUSE_NLS}"
 # and not any particular tune that is enabled.
 TARGET_ARCH[vardepsexclude] = "TUNE_ARCH"
 
+PKGDATA_DIR = "${TMPDIR}/pkgdata/${SDK_SYS}"
 # If MLPREFIX is set by multilib code, shlibs
 # points to the wrong place so force it
 SHLIBSDIRS = "${PKGDATA_DIR}/nativesdk-shlibs2"

meta/classes/cross.bbclass

@@ -19,11 +19,11 @@ HOST_AS_ARCH = "${BUILD_AS_ARCH}"
 
 export lt_cv_sys_lib_dlsearch_path_spec = "${libdir} ${base_libdir} /lib /lib64 /usr/lib /usr/lib64"
 
-STAGING_DIR_HOST = "${STAGING_DIR}/${HOST_ARCH}${HOST_VENDOR}-${HOST_OS}"
+STAGING_DIR_HOST = "${RECIPE_SYSROOT_NATIVE}"
 
 PACKAGE_ARCH = "${BUILD_ARCH}"
 
-MULTIMACH_TARGET_SYS = "${PACKAGE_ARCH}${BUILD_VENDOR}-${BUILD_OS}"
+MULTIMACH_TARGET_SYS = "${BUILD_ARCH}${BUILD_VENDOR}-${BUILD_OS}"
 
 export PKG_CONFIG_DIR = "${exec_prefix}/lib/pkgconfig"
 export PKG_CONFIG_SYSROOT_DIR = ""
@@ -38,6 +38,10 @@ TOOLCHAIN_OPTIONS = ""
 
 DEPENDS_GETTEXT = "gettext-native"
 
+# This class encodes staging paths into its scripts data so can only be
+# reused if we manipulate the paths.
+SSTATE_SCAN_CMD ?= "${SSTATE_SCAN_CMD_NATIVE}"
+
 # Path mangling needed by the cross packaging
 # Note that we use := here to ensure that libdir and includedir are
 # target paths.

meta/classes/crosssdk.bbclass

@@ -8,9 +8,12 @@ python () {
 	d.setVar('TUNE_PKGARCH', d.getVar('SDK_ARCH'))
 }
 
-STAGING_DIR_TARGET = "${STAGING_DIR}/${SDK_ARCH}-${SDKPKGSUFFIX}${SDK_VENDOR}-${SDK_OS}"
 STAGING_BINDIR_TOOLCHAIN = "${STAGING_DIR_NATIVE}${bindir_native}/${TARGET_ARCH}${TARGET_VENDOR}-${TARGET_OS}"
 
+# This class encodes staging paths into its scripts data so can only be
+# reused if we manipulate the paths.
+SSTATE_SCAN_CMD ?= "${SSTATE_SCAN_CMD_NATIVE}"
+
 TARGET_ARCH = "${SDK_ARCH}"
 TARGET_VENDOR = "${SDK_VENDOR}"
 TARGET_OS = "${SDK_OS}"

meta/classes/distrodata.bbclass

@@ -392,6 +392,7 @@ python distro_check_eventhandler() {
 
 addtask distro_check
 do_distro_check[nostamp] = "1"
+do_distro_check[vardepsexclude] += "DATETIME"
 python do_distro_check() {
     """checks if the package is present in other public Linux distros"""
     import oe.distro_check as dc
@@ -475,5 +476,3 @@ do_checklicenseall[nostamp] = "1"
 do_checklicenseall() {
         :
 }
-
-

meta/classes/externalsrc.bbclass

@@ -4,7 +4,7 @@
 # Copyright (C) 2009 Chris Larson <clarson@kergoth.com>
 # Released under the MIT license (see COPYING.MIT for the terms)
 #
-# externalsrc.bbclass enables use of an existing source tree, usually external to 
+# externalsrc.bbclass enables use of an existing source tree, usually external to
 # the build system to build a piece of software rather than the usual fetch/unpack/patch
 # process.
 #
@@ -108,6 +108,10 @@ python () {
         # We don't want the workdir to go away
         d.appendVar('RM_WORK_EXCLUDE', ' ' + d.getVar('PN'))
 
+        bb.build.addtask('do_buildclean',
+                         'do_clean' if d.getVar('S') == d.getVar('B') else None,
+                         None, d)
+
         # If B=S the same builddir is used even for different architectures.
         # Thus, use a shared CONFIGURESTAMPFILE and STAMP directory so that
         # change of do_configure task hash is correctly detected and stamps are
@@ -116,14 +120,17 @@ python () {
             configstamp = '${TMPDIR}/work-shared/${PN}/${EXTENDPE}${PV}-${PR}/configure.sstate'
             d.setVar('CONFIGURESTAMPFILE', configstamp)
             d.setVar('STAMP', '${STAMPS_DIR}/work-shared/${PN}/${EXTENDPE}${PV}-${PR}')
+            d.setVar('STAMPCLEAN', '${STAMPS_DIR}/work-shared/${PN}/*-*')
 }
 
 python externalsrc_configure_prefunc() {
+    s_dir = d.getVar('S')
     # Create desired symlinks
     symlinks = (d.getVar('EXTERNALSRC_SYMLINKS') or '').split()
+    newlinks = []
     for symlink in symlinks:
         symsplit = symlink.split(':', 1)
-        lnkfile = os.path.join(d.getVar('S'), symsplit[0])
+        lnkfile = os.path.join(s_dir, symsplit[0])
         target = d.expand(symsplit[1])
         if len(symsplit) > 1:
             if os.path.islink(lnkfile):
@@ -135,6 +142,19 @@ python externalsrc_configure_prefunc() {
                 # File/dir exists with same name as link, just leave it alone
                 continue
             os.symlink(target, lnkfile)
+            newlinks.append(symsplit[0])
+    # Hide the symlinks from git
+    try:
+        git_exclude_file = os.path.join(s_dir, '.git/info/exclude')
+        if os.path.exists(git_exclude_file):
+            with open(git_exclude_file, 'r+') as efile:
+                elines = efile.readlines()
+                for link in newlinks:
+                    if link in elines or '/'+link in elines:
+                        continue
+                    efile.write('/' + link + '\n')
+    except IOError as ioe:
+        bb.note('Failed to hide EXTERNALSRC_SYMLINKS from git')
 }
 
 python externalsrc_compile_prefunc() {
@@ -142,12 +162,23 @@ python externalsrc_compile_prefunc() {
     bb.plain('NOTE: %s: compiling from external source tree %s' % (d.getVar('PN'), d.getVar('EXTERNALSRC')))
 }
 
-def srctree_hash_files(d):
+do_buildclean[dirs] = "${S} ${B}"
+do_buildclean[nostamp] = "1"
+do_buildclean[doc] = "Call 'make clean' or equivalent in ${B}"
+externalsrc_do_buildclean() {
+	if [ -e Makefile -o -e makefile -o -e GNUmakefile ]; then
+		oe_runmake clean || die "make failed"
+	else
+		bbnote "nothing to do - no makefile found"
+	fi
+}
+
+def srctree_hash_files(d, srcdir=None):
     import shutil
     import subprocess
     import tempfile
 
-    s_dir = d.getVar('EXTERNALSRC')
+    s_dir = srcdir or d.getVar('EXTERNALSRC')
     git_dir = os.path.join(s_dir, '.git')
     oe_hash_file = os.path.join(git_dir, 'oe-devtool-tree-sha1')
 
@@ -165,7 +196,7 @@ def srctree_hash_files(d):
             fobj.write(sha1)
         ret = oe_hash_file + ':True'
     else:
-        ret = d.getVar('EXTERNALSRC') + '/*:True'
+        ret = s_dir + '/*:True'
     return ret
 
 def srctree_configure_hash_files(d):
@@ -188,3 +219,5 @@ def srctree_configure_hash_files(d):
                 if f in search_files:
                     out_items.append('%s:True' % os.path.join(root, f))
     return ' '.join(out_items)
+
+EXPORT_FUNCTIONS do_buildclean

meta/classes/extrausers.bbclass

@@ -63,3 +63,7 @@ set_user_group () {
 		remaining=`echo $remaining | cut -d ';' -f2-`
 	done
 }
+
+USERADDEXTENSION ?= ""
+
+inherit ${USERADDEXTENSION}

meta/classes/fontcache.bbclass

@@ -3,7 +3,7 @@
 # packages.
 #
 
-DEPENDS += "qemu-native"
+PACKAGE_WRITE_DEPS += "qemu-native"
 inherit qemu
 
 FONT_PACKAGES ??= "${PN}"

meta/classes/fs-uuid.bbclass

@@ -13,7 +13,7 @@ def get_rootfs_uuid(d):
     bb.fatal('Could not determine filesystem UUID of %s' % rootfs)
 
 # Replace the special <<uuid-of-rootfs>> inside a string (like the
-# root= APPEND string in a syslinux.cfg or gummiboot entry) with the
+# root= APPEND string in a syslinux.cfg or systemd-boot entry) with the
 # actual UUID of the rootfs. Does nothing if the special string
 # is not used.
 def replace_rootfs_uuid(d, string):

meta/classes/gconf.bbclass

@@ -1,4 +1,5 @@
-DEPENDS += "gconf gconf-native"
+DEPENDS += "gconf"
+PACKAGE_WRITE_DEPS += "gconf-native"
 
 # These are for when gconftool is used natively and the prefix isn't necessarily
 # the sysroot.  TODO: replicate the postinst logic for -native packages going

meta/classes/gio-module-cache.bbclass

@@ -1,4 +1,4 @@
-DEPENDS += "qemu-native"
+PACKAGE_WRITE_DEPS += "qemu-native"
 inherit qemu
 
 GIO_MODULE_PACKAGES ??= "${PN}"

meta/classes/gobject-introspection.bbclass

@@ -17,7 +17,7 @@ UNKNOWN_CONFIGURE_WHITELIST_append = " --enable-introspection --disable-introspe
 
 # Generating introspection data depends on a combination of native and target
 # introspection tools, and qemu to run the target tools.
-DEPENDS_append_class-target = " gobject-introspection gobject-introspection-native qemu-native"
+DEPENDS_append_class-target = " gobject-introspection gobject-introspection-native qemu-native prelink-native"
 
 # Even though introspection is disabled on -native, gobject-introspection package is still
 # needed for m4 macros.

meta/classes/grub-efi.bbclass

@@ -40,13 +40,15 @@ efi_populate() {
 
 	install -d ${DEST}${EFIDIR}
 
-	GRUB_IMAGE="bootia32.efi"
+	GRUB_IMAGE="grub-efi-bootia32.efi"
+	DEST_IMAGE="bootia32.efi"
 	if [ "${TARGET_ARCH}" = "x86_64" ]; then
-		GRUB_IMAGE="bootx64.efi"
+		GRUB_IMAGE="grub-efi-bootx64.efi"
+		DEST_IMAGE="bootx64.efi"
 	fi
-	install -m 0644 ${DEPLOY_DIR_IMAGE}/${GRUB_IMAGE} ${DEST}${EFIDIR}
+	install -m 0644 ${DEPLOY_DIR_IMAGE}/${GRUB_IMAGE} ${DEST}${EFIDIR}/${DEST_IMAGE}
 	EFIPATH=$(echo "${EFIDIR}" | sed 's/\//\\/g')
-	printf 'fs0:%s\%s\n' "$EFIPATH" "$GRUB_IMAGE" >${DEST}/startup.nsh
+	printf 'fs0:%s\%s\n' "$EFIPATH" "$DEST_IMAGE" >${DEST}/startup.nsh
 
 	install -m 0644 ${GRUB_CFG} ${DEST}${EFIDIR}/grub.cfg
 }

meta/classes/gsettings.bbclass

@@ -7,12 +7,13 @@
 
 # TODO use a trigger so that this runs once per package operation run
 
-DEPENDS += "glib-2.0-native"
 
 RDEPENDS_${PN} += "glib-2.0-utils"
 
 FILES_${PN} += "${datadir}/glib-2.0/schemas"
 
+PACKAGE_WRITE_DEPS += "glib-2.0-native"
+
 gsettings_postinstrm () {
 	glib-compile-schemas $D${datadir}/glib-2.0/schemas
 }

meta/classes/gtk-icon-cache.bbclass

@@ -2,6 +2,8 @@ FILES_${PN} += "${datadir}/icons/hicolor"
 
 DEPENDS += "${@['hicolor-icon-theme', '']['${BPN}' == 'hicolor-icon-theme']} gtk-icon-utils-native"
 
+PACKAGE_WRITE_DEPS += "gtk-icon-utils-native gdk-pixbuf-native"
+
 gtk_icon_cache_postinst() {
 if [ "x$D" != "x" ]; then
 	$INTERCEPT_DIR/postinst_intercept update_icon_cache ${PKG} \

meta/classes/gtk-immodules-cache.bbclass

@@ -2,7 +2,7 @@
 #
 # Usage: Set GTKIMMODULES_PACKAGES to the packages that needs to update the inputmethod modules
 
-DEPENDS =+ "qemu-native"
+PACKAGE_WRITE_DEPS += "qemu-native"
 
 inherit qemu
 

meta/classes/gummiboot.bbclass

@@ -1,121 +0,0 @@
-# Copyright (C) 2014 Intel Corporation
-#
-# Released under the MIT license (see COPYING.MIT)
-
-# gummiboot.bbclass - equivalent of grub-efi.bbclass
-# Set EFI_PROVIDER = "gummiboot" to use gummiboot on your live images instead of grub-efi
-# (images built by image-live.bbclass or image-vm.bbclass)
-
-do_bootimg[depends] += "${MLPREFIX}gummiboot:do_deploy"
-do_bootdirectdisk[depends] += "${MLPREFIX}gummiboot:do_deploy"
-
-EFIDIR = "/EFI/BOOT"
-
-GUMMIBOOT_CFG ?= "${S}/loader.conf"
-GUMMIBOOT_ENTRIES ?= ""
-GUMMIBOOT_TIMEOUT ?= "10"
-
-# Need UUID utility code.
-inherit fs-uuid
-
-efi_populate() {
-        DEST=$1
-
-        EFI_IMAGE="gummibootia32.efi"
-        DEST_EFI_IMAGE="bootia32.efi"
-        if [ "${TARGET_ARCH}" = "x86_64" ]; then
-            EFI_IMAGE="gummibootx64.efi"
-            DEST_EFI_IMAGE="bootx64.efi"
-        fi
-
-        install -d ${DEST}${EFIDIR}
-        # gummiboot requires these paths for configuration files
-        # they are not customizable so no point in new vars
-        install -d ${DEST}/loader
-        install -d ${DEST}/loader/entries
-        install -m 0644 ${DEPLOY_DIR_IMAGE}/${EFI_IMAGE} ${DEST}${EFIDIR}/${DEST_EFI_IMAGE}
-        EFIPATH=$(echo "${EFIDIR}" | sed 's/\//\\/g')
-        printf 'fs0:%s\%s\n' "$EFIPATH" "$DEST_EFI_IMAGE" >${DEST}/startup.nsh
-        install -m 0644 ${GUMMIBOOT_CFG} ${DEST}/loader/loader.conf
-        for i in ${GUMMIBOOT_ENTRIES}; do
-            install -m 0644 ${i} ${DEST}/loader/entries
-        done
-}
-
-efi_iso_populate() {
-        iso_dir=$1
-        efi_populate $iso_dir
-        mkdir -p ${EFIIMGDIR}/${EFIDIR}
-        cp $iso_dir/${EFIDIR}/* ${EFIIMGDIR}${EFIDIR}
-        cp $iso_dir/vmlinuz ${EFIIMGDIR}
-        EFIPATH=$(echo "${EFIDIR}" | sed 's/\//\\/g')
-        echo "fs0:${EFIPATH}\\${DEST_EFI_IMAGE}" > ${EFIIMGDIR}/startup.nsh
-        if [ -f "$iso_dir/initrd" ] ; then
-            cp $iso_dir/initrd ${EFIIMGDIR}
-        fi
-}
-
-efi_hddimg_populate() {
-        efi_populate $1
-}
-
-python build_efi_cfg() {
-    s = d.getVar("S", True)
-    labels = d.getVar('LABELS', True)
-    if not labels:
-        bb.debug(1, "LABELS not defined, nothing to do")
-        return
-
-    if labels == []:
-        bb.debug(1, "No labels, nothing to do")
-        return
-
-    cfile = d.getVar('GUMMIBOOT_CFG', True)
-    try:
-         cfgfile = open(cfile, 'w')
-    except OSError:
-        bb.fatal('Unable to open %s' % cfile)
-
-    cfgfile.write('# Automatically created by OE\n')
-    cfgfile.write('default %s\n' % (labels.split()[0]))
-    timeout = d.getVar('GUMMIBOOT_TIMEOUT', True)
-    if timeout:
-        cfgfile.write('timeout %s\n' % timeout)
-    else:
-        cfgfile.write('timeout 10\n')
-    cfgfile.close()
-
-    for label in labels.split():
-        localdata = d.createCopy()
-
-        overrides = localdata.getVar('OVERRIDES', True)
-        if not overrides:
-            bb.fatal('OVERRIDES not defined')
-
-        entryfile = "%s/%s.conf" % (s, label)
-        d.appendVar("GUMMIBOOT_ENTRIES", " " + entryfile)
-        try:
-            entrycfg = open(entryfile, "w")
-        except OSError:
-            bb.fatal('Unable to open %s' % entryfile)
-        localdata.setVar('OVERRIDES', label + ':' + overrides)
-        bb.data.update_data(localdata)
-
-        entrycfg.write('title %s\n' % label)
-        entrycfg.write('linux /vmlinuz\n')
-
-        append = localdata.getVar('APPEND', True)
-        initrd = localdata.getVar('INITRD', True)
-
-        if initrd:
-            entrycfg.write('initrd /initrd\n')
-        lb = label
-        if label == "install":
-            lb = "install-efi"
-        entrycfg.write('options LABEL=%s ' % lb)
-        if append:
-            append = replace_rootfs_uuid(d, append)
-            entrycfg.write('%s' % append)
-        entrycfg.write('\n')
-        entrycfg.close()
-}

meta/classes/gzipnative.bbclass

@@ -1,5 +0,0 @@
-EXTRANATIVEPATH += "pigz-native gzip-native"
-DEPENDS += "gzip-native"
-
-# tar may get run by do_unpack or do_populate_lic which could call gzip
-do_unpack[depends] += "gzip-native:do_populate_sysroot"

meta/classes/image-wic.bbclass

@@ -0,0 +1,120 @@
+# The WICVARS variable is used to define list of bitbake variables used in wic code
+# variables from this list is written to <image>.env file
+WICVARS ?= "\
+           BBLAYERS IMGDEPLOYDIR DEPLOY_DIR_IMAGE FAKEROOTCMD IMAGE_BASENAME IMAGE_BOOT_FILES \
+           IMAGE_LINK_NAME IMAGE_ROOTFS INITRAMFS_FSTYPES INITRD INITRD_LIVE ISODIR RECIPE_SYSROOT_NATIVE \
+           ROOTFS_SIZE STAGING_DATADIR STAGING_DIR STAGING_LIBDIR TARGET_SYS TRANSLATED_TARGET_ARCH"
+
+WKS_FILE ??= "${IMAGE_BASENAME}.${MACHINE}.wks"
+WKS_FILES ?= "${WKS_FILE} ${IMAGE_BASENAME}.wks"
+WKS_SEARCH_PATH ?= "${THISDIR}:${@':'.join('%s/wic' % p for p in '${BBPATH}'.split(':'))}:${@':'.join('%s/scripts/lib/wic/canned-wks' % l for l in '${BBPATH}:${COREBASE}'.split(':'))}"
+WKS_FULL_PATH = "${@wks_search('${WKS_FILES}'.split(), '${WKS_SEARCH_PATH}') or ''}"
+
+def wks_search(files, search_path):
+    for f in files:
+        if os.path.isabs(f):
+            if os.path.exists(f):
+                return f
+        else:
+            searched = bb.utils.which(search_path, f)
+            if searched:
+                return searched
+
+WIC_CREATE_EXTRA_ARGS ?= ""
+
+IMAGE_CMD_wic () {
+	out="${IMGDEPLOYDIR}/${IMAGE_NAME}"
+	wks="${WKS_FULL_PATH}"
+	if [ -z "$wks" ]; then
+		bbfatal "No kickstart files from WKS_FILES were found: ${WKS_FILES}. Please set WKS_FILE or WKS_FILES appropriately."
+	fi
+
+	BUILDDIR="${TOPDIR}" wic create "$wks" --vars "${STAGING_DIR}/${MACHINE}/imgdata/" -e "${IMAGE_BASENAME}" -o "$out/" ${WIC_CREATE_EXTRA_ARGS}
+	mv "$out/$(basename "${wks%.wks}")"*.direct "$out${IMAGE_NAME_SUFFIX}.wic"
+	rm -rf "$out/"
+}
+IMAGE_CMD_wic[vardepsexclude] = "WKS_FULL_PATH WKS_FILES"
+
+# Rebuild when the wks file or vars in WICVARS change
+USING_WIC = "${@bb.utils.contains_any('IMAGE_FSTYPES', 'wic ' + ' '.join('wic.%s' % c for c in '${CONVERSIONTYPES}'.split()), '1', '', d)}"
+WKS_FILE_CHECKSUM = "${@'${WKS_FULL_PATH}:%s' % os.path.exists('${WKS_FULL_PATH}') if '${USING_WIC}' else ''}"
+do_image_wic[file-checksums] += "${WKS_FILE_CHECKSUM}"
+do_image_wic[depends] += "wic-tools:do_build"
+
+python () {
+    if d.getVar('USING_WIC') and 'do_bootimg' in d:
+        bb.build.addtask('do_image_wic', '', 'do_bootimg', d)
+}
+
+python do_write_wks_template () {
+    """Write out expanded template contents to WKS_FULL_PATH."""
+    import re
+
+    template_body = d.getVar('_WKS_TEMPLATE')
+
+    # Remove any remnant variable references left behind by the expansion
+    # due to undefined variables
+    expand_var_regexp = re.compile(r"\${[^{}@\n\t :]+}")
+    while True:
+        new_body = re.sub(expand_var_regexp, '', template_body)
+        if new_body == template_body:
+            break
+        else:
+            template_body = new_body
+
+    wks_file = d.getVar('WKS_FULL_PATH')
+    with open(wks_file, 'w') as f:
+        f.write(template_body)
+}
+
+python () {
+    if d.getVar('USING_WIC'):
+        wks_file_u = d.getVar('WKS_FULL_PATH', False)
+        wks_file = d.expand(wks_file_u)
+        base, ext = os.path.splitext(wks_file)
+        if ext == '.in' and os.path.exists(wks_file):
+            wks_out_file = os.path.join(d.getVar('WORKDIR'), os.path.basename(base))
+            d.setVar('WKS_FULL_PATH', wks_out_file)
+            d.setVar('WKS_TEMPLATE_PATH', wks_file_u)
+            d.setVar('WKS_FILE_CHECKSUM', '${WKS_TEMPLATE_PATH}:True')
+
+            # We need to re-parse each time the file changes, and bitbake
+            # needs to be told about that explicitly.
+            bb.parse.mark_dependency(d, wks_file)
+
+            try:
+                with open(wks_file, 'r') as f:
+                    body = f.read()
+            except (IOError, OSError) as exc:
+                pass
+            else:
+                # Previously, I used expandWithRefs to get the dependency list
+                # and add it to WICVARS, but there's no point re-parsing the
+                # file in process_wks_template as well, so just put it in
+                # a variable and let the metadata deal with the deps.
+                d.setVar('_WKS_TEMPLATE', body)
+                bb.build.addtask('do_write_wks_template', 'do_image_wic', None, d)
+}
+
+#
+# Write environment variables used by wic
+# to tmp/sysroots/<machine>/imgdata/<image>.env
+#
+python do_rootfs_wicenv () {
+    wicvars = d.getVar('WICVARS')
+    if not wicvars:
+        return
+
+    stdir = d.getVar('STAGING_DIR')
+    outdir = os.path.join(stdir, d.getVar('MACHINE'), 'imgdata')
+    bb.utils.mkdirhier(outdir)
+    basename = d.getVar('IMAGE_BASENAME')
+    with open(os.path.join(outdir, basename) + '.env', 'w') as envf:
+        for var in wicvars.split():
+            value = d.getVar(var)
+            if value:
+                envf.write('%s="%s"\n' % (var, value.strip()))
+}
+addtask do_rootfs_wicenv after do_image before do_image_wic
+do_rootfs_wicenv[vardeps] += "${WICVARS}"
+do_rootfs_wicenv[prefuncs] = 'set_image_size'

meta/classes/image.bbclass

@@ -9,11 +9,9 @@ TOOLCHAIN_TARGET_TASK += "${PACKAGE_INSTALL}"
 TOOLCHAIN_TARGET_TASK_ATTEMPTONLY += "${PACKAGE_INSTALL_ATTEMPTONLY}"
 POPULATE_SDK_POST_TARGET_COMMAND += "rootfs_sysroot_relativelinks; "
 
-inherit gzipnative
-
 LICENSE = "MIT"
 PACKAGES = ""
-DEPENDS += "${MLPREFIX}qemuwrapper-cross ${MLPREFIX}depmodwrapper-cross"
+DEPENDS += "${MLPREFIX}qemuwrapper-cross depmodwrapper-cross"
 RDEPENDS += "${PACKAGE_INSTALL} ${LINGUAS_INSTALL}"
 RRECOMMENDS += "${PACKAGE_INSTALL_ATTEMPTONLY}"
 
@@ -160,7 +158,7 @@ python () {
     deps = ""
     for dep in (d.getVar('EXTRA_IMAGEDEPENDS') or "").split():
         deps += " %s:do_populate_sysroot" % dep
-    d.appendVarFlag('do_build', 'depends', deps)
+    d.appendVarFlag('do_image_complete', 'depends', deps)
 
     #process IMAGE_FEATURES, we must do this before runtime_mapping_rename
     #Check for replaces image features
@@ -189,6 +187,8 @@ python () {
 IMAGE_CLASSES += "image_types"
 inherit ${IMAGE_CLASSES}
 
+inherit image-wic
+
 IMAGE_POSTPROCESS_COMMAND ?= ""
 
 # some default locales
@@ -270,7 +270,7 @@ fakeroot python do_rootfs () {
 do_rootfs[dirs] = "${TOPDIR}"
 do_rootfs[cleandirs] += "${S} ${IMGDEPLOYDIR}"
 do_rootfs[umask] = "022"
-addtask rootfs before do_build
+addtask rootfs before do_build after do_prepare_recipe_sysroot
 
 fakeroot python do_image () {
     from oe.utils import execute_pre_post_process
@@ -329,29 +329,6 @@ fakeroot python do_image_qa () {
 }
 addtask do_image_qa after do_image_complete before do_build
 
-#
-# Write environment variables used by wic
-# to tmp/sysroots/<machine>/imgdata/<image>.env
-#
-python do_rootfs_wicenv () {
-    wicvars = d.getVar('WICVARS')
-    if not wicvars:
-        return
-
-    stdir = d.getVar('STAGING_DIR_TARGET')
-    outdir = os.path.join(stdir, 'imgdata')
-    bb.utils.mkdirhier(outdir)
-    basename = d.getVar('IMAGE_BASENAME')
-    with open(os.path.join(outdir, basename) + '.env', 'w') as envf:
-        for var in wicvars.split():
-            value = d.getVar(var)
-            if value:
-                envf.write('%s="%s"\n' % (var, value.strip()))
-}
-addtask do_rootfs_wicenv after do_image before do_image_wic
-do_rootfs_wicenv[vardeps] += "${WICVARS}"
-do_rootfs_wicenv[prefuncs] = 'set_image_size'
-
 def setup_debugfs_variables(d):
     d.appendVar('IMAGE_ROOTFS', '-dbg')
     d.appendVar('IMAGE_LINK_NAME', '-dbg')
@@ -621,9 +598,9 @@ do_patch[noexec] = "1"
 do_configure[noexec] = "1"
 do_compile[noexec] = "1"
 do_install[noexec] = "1"
-do_populate_sysroot[noexec] = "1"
+deltask do_populate_sysroot
 do_package[noexec] = "1"
-do_package_qa[noexec] = "1"
+deltask do_package_qa
 do_packagedata[noexec] = "1"
 do_package_write_ipk[noexec] = "1"
 do_package_write_deb[noexec] = "1"

meta/classes/image_types.bbclass

@@ -17,17 +17,25 @@ def imagetypes_getdepends(d):
                 d += ":do_populate_sysroot"
             deps.add(d)
 
+    # Take a type in the form of foo.bar.car and split it into the items
+    # needed for the image deps "foo", and the conversion deps ["bar", "car"]
+    def split_types(typestring):
+        types = typestring.split(".")
+        return types[0], types[1:]
+
     fstypes = set((d.getVar('IMAGE_FSTYPES') or "").split())
     fstypes |= set((d.getVar('IMAGE_FSTYPES_DEBUGFS') or "").split())
 
     deps = set()
     for typestring in fstypes:
-        types = typestring.split(".")
-        basetype, resttypes = types[0], types[1:]
-
+        basetype, resttypes = split_types(typestring)
         adddep(d.getVar('IMAGE_DEPENDS_%s' % basetype) , deps)
+
         for typedepends in (d.getVar("IMAGE_TYPEDEP_%s" % basetype) or "").split():
+            base, rest = split_types(typedepends)
+            resttypes += rest
             adddep(d.getVar('IMAGE_DEPENDS_%s' % typedepends) , deps)
+
         for ctype in resttypes:
             adddep(d.getVar("CONVERSION_DEPENDS_%s" % ctype), deps)
             adddep(d.getVar("COMPRESS_DEPENDS_%s" % ctype), deps)
@@ -184,92 +192,6 @@ IMAGE_CMD_ubi () {
 
 IMAGE_CMD_ubifs = "mkfs.ubifs -r ${IMAGE_ROOTFS} -o ${IMGDEPLOYDIR}/${IMAGE_NAME}${IMAGE_NAME_SUFFIX}.ubifs ${MKUBIFS_ARGS}"
 
-WKS_FILE ??= "${IMAGE_BASENAME}.${MACHINE}.wks"
-WKS_FILES ?= "${WKS_FILE} ${IMAGE_BASENAME}.wks"
-WKS_SEARCH_PATH ?= "${THISDIR}:${@':'.join('%s/scripts/lib/wic/canned-wks' % l for l in '${BBPATH}:${COREBASE}'.split(':'))}"
-WKS_FULL_PATH = "${@wks_search('${WKS_FILES}'.split(), '${WKS_SEARCH_PATH}') or ''}"
-
-def wks_search(files, search_path):
-    for f in files:
-        if os.path.isabs(f):
-            if os.path.exists(f):
-                return f
-        else:
-            searched = bb.utils.which(search_path, f)
-            if searched:
-                return searched
-
-WIC_CREATE_EXTRA_ARGS ?= ""
-
-IMAGE_CMD_wic () {
-	out="${IMGDEPLOYDIR}/${IMAGE_NAME}"
-	wks="${WKS_FULL_PATH}"
-	if [ -z "$wks" ]; then
-		bbfatal "No kickstart files from WKS_FILES were found: ${WKS_FILES}. Please set WKS_FILE or WKS_FILES appropriately."
-	fi
-
-	BUILDDIR="${TOPDIR}" wic create "$wks" --vars "${STAGING_DIR_TARGET}/imgdata/" -e "${IMAGE_BASENAME}" -o "$out/" ${WIC_CREATE_EXTRA_ARGS}
-	mv "$out/build/$(basename "${wks%.wks}")"*.direct "$out${IMAGE_NAME_SUFFIX}.wic"
-	rm -rf "$out/"
-}
-IMAGE_CMD_wic[vardepsexclude] = "WKS_FULL_PATH WKS_FILES"
-
-# Rebuild when the wks file or vars in WICVARS change
-USING_WIC = "${@bb.utils.contains_any('IMAGE_FSTYPES', 'wic ' + ' '.join('wic.%s' % c for c in '${CONVERSIONTYPES}'.split()), '1', '', d)}"
-WKS_FILE_CHECKSUM = "${@'${WKS_FULL_PATH}:%s' % os.path.exists('${WKS_FULL_PATH}') if '${USING_WIC}' else ''}"
-do_image_wic[file-checksums] += "${WKS_FILE_CHECKSUM}"
-
-python () {
-    if d.getVar('USING_WIC') and 'do_bootimg' in d:
-        bb.build.addtask('do_image_wic', '', 'do_bootimg', d)
-}
-
-python do_write_wks_template () {
-    """Write out expanded template contents to WKS_FULL_PATH."""
-    import re
-
-    template_body = d.getVar('_WKS_TEMPLATE')
-
-    # Remove any remnant variable references left behind by the expansion
-    # due to undefined variables
-    expand_var_regexp = re.compile(r"\${[^{}@\n\t :]+}")
-    while True:
-        new_body = re.sub(expand_var_regexp, '', template_body)
-        if new_body == template_body:
-            break
-        else:
-            template_body = new_body
-
-    wks_file = d.getVar('WKS_FULL_PATH')
-    with open(wks_file, 'w') as f:
-        f.write(template_body)
-}
-
-python () {
-    if d.getVar('USING_WIC'):
-        wks_file_u = d.getVar('WKS_FULL_PATH', False)
-        wks_file = d.expand(wks_file_u)
-        base, ext = os.path.splitext(wks_file)
-        if ext == '.in' and os.path.exists(wks_file):
-            wks_out_file = os.path.join(d.getVar('WORKDIR'), os.path.basename(base))
-            d.setVar('WKS_FULL_PATH', wks_out_file)
-            d.setVar('WKS_TEMPLATE_PATH', wks_file_u)
-            d.setVar('WKS_FILE_CHECKSUM', '${WKS_TEMPLATE_PATH}:True')
-
-            try:
-                with open(wks_file, 'r') as f:
-                    body = f.read()
-            except (IOError, OSError) as exc:
-                pass
-            else:
-                # Previously, I used expandWithRefs to get the dependency list
-                # and add it to WICVARS, but there's no point re-parsing the
-                # file in process_wks_template as well, so just put it in
-                # a variable and let the metadata deal with the deps.
-                d.setVar('_WKS_TEMPLATE', body)
-                bb.build.addtask('do_write_wks_template', 'do_image_wic', None, d)
-}
-
 EXTRA_IMAGECMD = ""
 
 inherit siteinfo
@@ -346,7 +268,7 @@ CONVERSION_CMD_sha384sum = "sha384sum ${IMAGE_NAME}${IMAGE_NAME_SUFFIX}.${type}
 CONVERSION_CMD_sha512sum = "sha512sum ${IMAGE_NAME}${IMAGE_NAME_SUFFIX}.${type} > ${IMAGE_NAME}${IMAGE_NAME_SUFFIX}.${type}.sha512sum"
 CONVERSION_CMD_bmap = "bmaptool create ${IMAGE_NAME}${IMAGE_NAME_SUFFIX}.${type} -o ${IMAGE_NAME}${IMAGE_NAME_SUFFIX}.${type}.bmap"
 CONVERSION_DEPENDS_lzma = "xz-native"
-CONVERSION_DEPENDS_gz = ""
+CONVERSION_DEPENDS_gz = "pigz-native"
 CONVERSION_DEPENDS_bz2 = "pbzip2-native"
 CONVERSION_DEPENDS_xz = "xz-native"
 CONVERSION_DEPENDS_lz4 = "lz4-native"
@@ -366,7 +288,3 @@ IMAGE_EXTENSION_live = "hddimg iso"
 # The IMAGE_TYPES_MASKED variable will be used to mask out from the IMAGE_FSTYPES,
 # images that will not be built at do_rootfs time: vmdk, vdi, qcow2, hdddirect, hddimg, iso, etc.
 IMAGE_TYPES_MASKED ?= ""
-
-# The WICVARS variable is used to define list of bitbake variables used in wic code
-# variables from this list is written to <image>.env file
-WICVARS ?= "BBLAYERS IMGDEPLOYDIR DEPLOY_DIR_IMAGE HDDDIR IMAGE_BASENAME IMAGE_BOOT_FILES IMAGE_LINK_NAME IMAGE_ROOTFS INITRAMFS_FSTYPES INITRD ISODIR MACHINE_ARCH ROOTFS_SIZE STAGING_DATADIR STAGING_DIR_NATIVE STAGING_LIBDIR TARGET_SYS"

meta/classes/insane.bbclass

@@ -174,6 +174,8 @@ def package_qa_get_machine_dict(d):
             "linux-gnun32" :       {
                         "mips64":       ( 8,     0,    0,          False,         32),
                         "mips64el":     ( 8,     0,    0,          True,          32),
+                        "mipsisa64r6":  ( 8,     0,    0,          False,         32),
+                        "mipsisa64r6el":( 8,     0,    0,          True,          32),
                       },
         }
 
@@ -334,7 +336,7 @@ def package_qa_check_libdir(d):
     import re
 
     pkgdest = d.getVar('PKGDEST')
-    base_libdir = d.getVar("base_libdir",True) + os.sep
+    base_libdir = d.getVar("base_libdir") + os.sep
     libdir = d.getVar("libdir") + os.sep
     libexecdir = d.getVar("libexecdir") + os.sep
     exec_prefix = d.getVar("exec_prefix") + os.sep
@@ -566,7 +568,7 @@ def package_qa_check_desktop(path, name, d, elf, messages):
     Run all desktop files through desktop-file-validate.
     """
     if path.endswith(".desktop"):
-        desktop_file_validate = os.path.join(d.getVar('STAGING_BINDIR_NATIVE',True),'desktop-file-validate')
+        desktop_file_validate = os.path.join(d.getVar('STAGING_BINDIR_NATIVE'),'desktop-file-validate')
         output = os.popen("%s %s" % (desktop_file_validate, path))
         # This only produces output on errors
         for l in output:
@@ -738,17 +740,21 @@ python populate_lic_qa_checksum() {
 
         if (not beginline) and (not endline):
             md5chksum = bb.utils.md5_file(srclicfile)
+            with open(srclicfile, 'rb') as f:
+                license = f.read()
         else:
             fi = open(srclicfile, 'rb')
             fo = tempfile.NamedTemporaryFile(mode='wb', prefix='poky.', suffix='.tmp', delete=False)
             tmplicfile = fo.name;
             lineno = 0
             linesout = 0
+            license = []
             for line in fi:
                 lineno += 1
                 if (lineno >= beginline):
                     if ((lineno <= endline) or not endline):
                         fo.write(line)
+                        license.append(line)
                         linesout += 1
                     else:
                         break
@@ -756,6 +762,7 @@ python populate_lic_qa_checksum() {
             fo.close()
             fi.close()
             md5chksum = bb.utils.md5_file(tmplicfile)
+            license = b''.join(license)
             os.unlink(tmplicfile)
 
         if recipemd5 == md5chksum:
@@ -764,6 +771,30 @@ python populate_lic_qa_checksum() {
             if recipemd5:
                 msg = pn + ": The LIC_FILES_CHKSUM does not match for " + url
                 msg = msg + "\n" + pn + ": The new md5 checksum is " + md5chksum
+                try:
+                    license_lines = license.decode('utf-8').split('\n')
+                except:
+                    # License text might not be valid UTF-8, in which
+                    # case we don't know how to include it in our output
+                    # and have to skip it.
+                    pass
+                else:
+                    max_lines = int(d.getVar('QA_MAX_LICENSE_LINES') or 20)
+                    if not license_lines or license_lines[-1] != '':
+                        # Ensure that our license text ends with a line break
+                        # (will be added with join() below).
+                        license_lines.append('')
+                    remove = len(license_lines) - max_lines
+                    if remove > 0:
+                        start = max_lines // 2
+                        end = start + remove - 1
+                        del license_lines[start:end]
+                        license_lines.insert(start, '...')
+                    msg = msg + "\n" + pn + ": Here is the selected license text:" + \
+                          "\n" + \
+                          "{:v^70}".format(" beginline=%d " % beginline if beginline else "") + \
+                          "\n" + "\n".join(license_lines) + \
+                          "{:^^70}".format(" endline=%d " % endline if endline else "")
                 if beginline:
                     if endline:
                         srcfiledesc = "%s (lines %d through to %d)" % (srclicfile, beginline, endline)
@@ -796,6 +827,7 @@ def package_qa_check_staged(path,d):
     sane = True
     tmpdir = d.getVar('TMPDIR')
     workdir = os.path.join(tmpdir, "work")
+    recipesysroot = d.getVar("RECIPE_SYSROOT")
 
     if bb.data.inherits_class("native", d) or bb.data.inherits_class("cross", d):
         pkgconfigcheck = workdir
@@ -811,12 +843,14 @@ def package_qa_check_staged(path,d):
             if file.endswith(".la"):
                 with open(path) as f:
                     file_content = f.read()
+                    file_content = file_content.replace(recipesysroot, "")
                     if workdir in file_content:
                         error_msg = "%s failed sanity test (workdir) in path %s" % (file,root)
                         sane = package_qa_handle_error("la", error_msg, d)
             elif file.endswith(".pc"):
                 with open(path) as f:
                     file_content = f.read()
+                    file_content = file_content.replace(recipesysroot, "")
                     if pkgconfigcheck in file_content:
                         error_msg = "%s failed sanity test (tmpdir) in path %s" % (file,root)
                         sane = package_qa_handle_error("pkgconfig", error_msg, d)

meta/classes/kernel-fitimage.bbclass

@@ -26,7 +26,7 @@ python __anonymous () {
         # Verified boot will sign the fitImage and append the public key to
         # U-boot dtb. We ensure the U-Boot dtb is deployed before assembling
         # the fitImage:
-        if d.getVar('UBOOT_SIGN_ENABLE'):
+        if d.getVar('UBOOT_SIGN_ENABLE') == "1":
             uboot_pn = d.getVar('PREFERRED_PROVIDER_u-boot') or 'u-boot'
             d.appendVarFlag('do_assemble_fitimage', 'depends', ' %s:do_deploy' % uboot_pn)
 }
@@ -178,17 +178,45 @@ EOF
 fitimage_emit_section_ramdisk() {
 
 	ramdisk_csum="sha1"
+	ramdisk_ctype="none"
+	ramdisk_loadline=""
+	ramdisk_entryline=""
+
+	if [ -n "${UBOOT_RD_LOADADDRESS}" ]; then
+		ramdisk_loadline="load = <${UBOOT_RD_LOADADDRESS}>;"
+	fi
+	if [ -n "${UBOOT_RD_ENTRYPOINT}" ]; then
+		ramdisk_entryline="entry = <${UBOOT_RD_ENTRYPOINT}>;"
+	fi
+
+	case $3 in
+		*.gz)
+			ramdisk_ctype="gzip"
+			;;
+		*.bz2)
+			ramdisk_ctype="bzip2"
+			;;
+		*.lzma)
+			ramdisk_ctype="lzma"
+			;;
+		*.lzo)
+			ramdisk_ctype="lzo"
+			;;
+		*.lz4)
+			ramdisk_ctype="lz4"
+			;;
+	esac
 
 	cat << EOF >> ${1}
                 ramdisk@${2} {
-                        description = "ramdisk image";
+                        description = "${INITRAMFS_IMAGE}";
                         data = /incbin/("${3}");
                         type = "ramdisk";
                         arch = "${UBOOT_ARCH}";
                         os = "linux";
-                        compression = "none";
-                        load = <${UBOOT_RD_LOADADDRESS}>;
-                        entry = <${UBOOT_RD_ENTRYPOINT}>;
+                        compression = "${ramdisk_ctype}";
+                        ${ramdisk_loadline}
+                        ${ramdisk_entryline}
                         hash@1 {
                                 algo = "${ramdisk_csum}";
                         };
@@ -332,8 +360,15 @@ fitimage_assemble() {
 	# Step 4: Prepare a ramdisk section.
 	#
 	if [ "x${ramdiskcount}" = "x1" ] ; then
-		copy_initramfs
-		fitimage_emit_section_ramdisk ${1} "${ramdiskcount}" usr/${INITRAMFS_IMAGE}-${MACHINE}.cpio
+		# Find and use the first initramfs image archive type we find
+		for img in cpio.lz4 cpio.lzo cpio.lzma cpio.xz cpio.gz cpio; do
+			initramfs_path="${DEPLOY_DIR_IMAGE}/${INITRAMFS_IMAGE}-${MACHINE}.${img}"
+			echo "Using $initramfs_path"
+			if [ -e "${initramfs_path}" ]; then
+				fitimage_emit_section_ramdisk ${1} "${ramdiskcount}" "${initramfs_path}"
+				break
+			fi
+		done
 	fi
 
 	fitimage_emit_section_maint ${1} sectend
@@ -410,11 +445,11 @@ kernel_do_deploy_append() {
 
 		if [ -n "${INITRAMFS_IMAGE}" ]; then
 			echo "Copying fit-image-${INITRAMFS_IMAGE}.its source file..."
-			its_initramfs_base_name="${KERNEL_IMAGETYPE}-its-${INITRAMFS_IMAGE}-${PV}-${PR}-${MACHINE}-${DATETIME}"
-			its_initramfs_symlink_name=${KERNEL_IMAGETYPE}-its-${INITRAMFS_IMAGE}-${MACHINE}
+			its_initramfs_base_name="fitImage-its-${INITRAMFS_IMAGE}-${PV}-${PR}-${MACHINE}-${DATETIME}"
+			its_initramfs_symlink_name=fitImage-its-${INITRAMFS_IMAGE}-${MACHINE}
 			install -m 0644 fit-image-${INITRAMFS_IMAGE}.its ${DEPLOYDIR}/${its_initramfs_base_name}.its
-			fit_initramfs_base_name="${KERNEL_IMAGETYPE}-${INITRAMFS_IMAGE}-${PV}-${PR}-${MACHINE}-${DATETIME}"
-			fit_initramfs_symlink_name=${KERNEL_IMAGETYPE}-${INITRAMFS_IMAGE}-${MACHINE}
+			fit_initramfs_base_name="fitImage-${INITRAMFS_IMAGE}-${PV}-${PR}-${MACHINE}-${DATETIME}"
+			fit_initramfs_symlink_name=fitImage-${INITRAMFS_IMAGE}-${MACHINE}
 			install -m 0644 arch/${ARCH}/boot/fitImage-${INITRAMFS_IMAGE} ${DEPLOYDIR}/${fit_initramfs_base_name}.bin
 		fi
 

meta/classes/kernel-module-split.bbclass

@@ -22,6 +22,8 @@ if [ x"$D" = "x" ]; then
 fi
 }
 
+PACKAGE_WRITE_DEPS += "kmod-native depmodwrapper-cross"
+
 do_install_append() {
 	install -d ${D}${sysconfdir}/modules-load.d/ ${D}${sysconfdir}/modprobe.d/
 }
@@ -31,6 +33,7 @@ PACKAGESPLITFUNCS_prepend = "split_kernel_module_packages "
 KERNEL_MODULES_META_PACKAGE ?= "kernel-modules"
 
 KERNEL_MODULE_PACKAGE_PREFIX ?= ""
+KERNEL_MODULE_PROVIDE_VIRTUAL ?= "1"
 
 python split_kernel_module_packages () {
     import re
@@ -119,10 +122,16 @@ python split_kernel_module_packages () {
         # Avoid automatic -dev recommendations for modules ending with -dev.
         d.setVarFlag('RRECOMMENDS_' + pkg, 'nodeprrecs', 1)
 
+        # Provide virtual package without postfix
+        providevirt = d.getVar('KERNEL_MODULE_PROVIDE_VIRTUAL')
+        if providevirt == "1":
+           postfix = format.split('%s')[1]
+           d.setVar('RPROVIDES_' + pkg, pkg.replace(postfix, ''))
+
     module_regex = '^(.*)\.k?o$'
 
     module_pattern_prefix = d.getVar('KERNEL_MODULE_PACKAGE_PREFIX')
-    module_pattern = module_pattern_prefix + 'kernel-module-%s'
+    module_pattern = module_pattern_prefix + 'kernel-module-%s-' + d.getVar("KERNEL_VERSION")
 
     postinst = d.getVar('pkg_postinst_modules')
     postrm = d.getVar('pkg_postrm_modules')

meta/classes/kernel-yocto.bbclass

@@ -286,7 +286,7 @@ python do_kernel_configcheck() {
     # if KMETA isn't set globally by a recipe using this routine, we need to
     # set the default to 'meta'. Otherwise, kconf_check is not passed a valid
     # meta-series for processing
-    kmeta = d.getVar( "KMETA", True ) or "meta"
+    kmeta = d.getVar("KMETA") or "meta"
     if not os.path.exists(kmeta):
         kmeta = "." + kmeta
 
@@ -298,8 +298,8 @@ python do_kernel_configcheck() {
     cmd = d.expand("cd ${S}; kconf_check --report -o ${S}/%s/cfg/ ${B}/.config ${S} %s" % (kmeta,configs))
     ret, result = oe.utils.getstatusoutput("%s%s" % (pathprefix, cmd))
 
-    config_check_visibility = int(d.getVar( "KCONF_AUDIT_LEVEL", True ) or 0)
-    bsp_check_visibility = int(d.getVar( "KCONF_BSP_AUDIT_LEVEL", True ) or 0)
+    config_check_visibility = int(d.getVar("KCONF_AUDIT_LEVEL") or 0)
+    bsp_check_visibility = int(d.getVar("KCONF_BSP_AUDIT_LEVEL") or 0)
 
     # if config check visibility is non-zero, report dropped configuration values
     mismatch_file = d.expand("${S}/%s/cfg/mismatch.txt" % kmeta)

meta/classes/kernel.bbclass

@@ -1,7 +1,8 @@
 inherit linux-kernel-base kernel-module-split
 
 PROVIDES += "virtual/kernel"
-DEPENDS += "virtual/${TARGET_PREFIX}binutils virtual/${TARGET_PREFIX}gcc kmod-native depmodwrapper-cross bc-native lzop-native"
+DEPENDS += "virtual/${TARGET_PREFIX}binutils virtual/${TARGET_PREFIX}gcc kmod-native bc-native lzop-native"
+PACKAGE_WRITE_DEPS += "depmodwrapper-cross virtual/update-alternatives-native"
 
 S = "${STAGING_KERNEL_DIR}"
 B = "${WORKDIR}/build"
@@ -45,10 +46,11 @@ python __anonymous () {
 
     for type in types.split():
         typelower = type.lower()
+        imagedest = d.getVar('KERNEL_IMAGEDEST')
 
         d.appendVar('PACKAGES', ' ' + 'kernel-image-' + typelower)
 
-        d.setVar('FILES_kernel-image-' + typelower, '/boot/' + type + '-${KERNEL_VERSION_NAME}')
+        d.setVar('FILES_kernel-image-' + typelower, '/' + imagedest + '/' + type + '-${KERNEL_VERSION_NAME}')
 
         d.appendVar('RDEPENDS_kernel-image', ' ' + 'kernel-image-' + typelower)
 
@@ -56,9 +58,8 @@ python __anonymous () {
 
         d.setVar('ALLOW_EMPTY_kernel-image-' + typelower, '1')
 
-        imagedest = d.getVar('KERNEL_IMAGEDEST')
         priority = d.getVar('KERNEL_PRIORITY')
-        postinst = '#!/bin/sh\n' + 'update-alternatives --install /' + imagedest + '/' + type + ' ' + type + ' ' + '/' + imagedest + '/' + type + '-${KERNEL_VERSION_NAME} ' + priority + ' || true' + '\n'
+        postinst = '#!/bin/sh\n' + 'update-alternatives --install /' + imagedest + '/' + type + ' ' + type + ' ' + type + '-${KERNEL_VERSION_NAME} ' + priority + ' || true' + '\n'
         d.setVar('pkg_postinst_kernel-image-' + typelower, postinst)
 
         postrm = '#!/bin/sh\n' + 'update-alternatives --remove' + ' ' + type + ' ' + type + '-${KERNEL_VERSION_NAME} || true' + '\n'

meta/classes/kernelsrc.bbclass

@@ -1,7 +1,7 @@
 S = "${STAGING_KERNEL_DIR}"
-do_fetch[noexec] = "1"
-do_unpack[depends] += "virtual/kernel:do_patch"
-do_unpack[noexec] = "1"
+deltask do_fetch
+deltask do_unpack
+do_patch[depends] += "virtual/kernel:do_patch"
 do_patch[noexec] = "1"
 do_package[depends] += "virtual/kernel:do_populate_sysroot"
 KERNEL_VERSION = "${@get_kernelversion_file("${STAGING_KERNEL_BUILDDIR}")}"

meta/classes/license.bbclass

@@ -117,7 +117,7 @@ def write_license_files(d, license_manifest, pkg_dic):
     copy_lic_manifest = d.getVar('COPY_LIC_MANIFEST')
     copy_lic_dirs = d.getVar('COPY_LIC_DIRS')
     if copy_lic_manifest == "1":
-        rootfs_license_dir = os.path.join(d.getVar('IMAGE_ROOTFS', 'True'), 
+        rootfs_license_dir = os.path.join(d.getVar('IMAGE_ROOTFS'), 
                                 'usr', 'share', 'common-licenses')
         bb.utils.mkdirhier(rootfs_license_dir)
         rootfs_license_manifest = os.path.join(rootfs_license_dir,
@@ -516,7 +516,7 @@ def canonical_license(d, license):
     """
     lic = d.getVarFlag('SPDXLICENSEMAP', license) or ""
     if not lic and license.endswith('+'):
-        lic = d.getVarFlag('SPDXLICENSEMAP', license.rstrip('+'), True)
+        lic = d.getVarFlag('SPDXLICENSEMAP', license.rstrip('+'))
         if lic:
             lic += '+'
     return lic or license

meta/classes/mime.bbclass

@@ -1,4 +1,5 @@
-DEPENDS += "shared-mime-info-native shared-mime-info"
+DEPENDS += "shared-mime-info"
+PACKAGE_WRITE_DEPS += "shared-mime-info-native"
 
 mime_postinst() {
 if [ "$1" = configure ]; then

meta/classes/mirrors.bbclass

@@ -27,7 +27,7 @@ ${GNUPG_MIRROR}	ftp://mirrors.dotsrc.org/gcrypt \n \
 ftp://dante.ctan.org/tex-archive ftp://ftp.fu-berlin.de/tex/CTAN \n \
 ftp://dante.ctan.org/tex-archive http://sunsite.sut.ac.jp/pub/archives/ctan/ \n \
 ftp://dante.ctan.org/tex-archive http://ctan.unsw.edu.au/ \n \
-ftp://ftp.gnutls.org/gcrypt/gnutls ${GNUPG_MIRROR} \n \
+ftp://ftp.gnutls.org/gcrypt/gnutls ${GNUPG_MIRROR}/gnutls \n \
 http://ftp.info-zip.org/pub/infozip/src/ http://mirror.switch.ch/ftp/mirror/infozip/src/ \n \
 http://ftp.info-zip.org/pub/infozip/src/ ftp://sunsite.icm.edu.pl/pub/unix/archiving/info-zip/src/ \n \
 ftp://lsof.itap.purdue.edu/pub/tools/unix/lsof/  ftp://ftp.cerias.purdue.edu/pub/tools/unix/sysutils/lsof/ \n \

meta/classes/module.bbclass

@@ -1,12 +1,13 @@
 inherit module-base kernel-module-split
 
-addtask make_scripts after do_patch before do_compile
+addtask make_scripts after do_prepare_recipe_sysroot before do_compile
 do_make_scripts[lockfiles] = "${TMPDIR}/kernel-scripts.lock"
 do_make_scripts[depends] += "virtual/kernel:do_shared_workdir"
 
 EXTRA_OEMAKE += "KERNEL_SRC=${STAGING_KERNEL_DIR}"
 
 MODULES_INSTALL_TARGET ?= "modules_install"
+MODULES_MODULE_SYMVERS_LOCATION ?= ""
 
 python __anonymous () {
     depends = d.getVar('DEPENDS')
@@ -35,10 +36,17 @@ module_do_install() {
 	           O=${STAGING_KERNEL_BUILDDIR} \
 	           ${MODULES_INSTALL_TARGET}
 
-	install -d -m0755 ${D}${includedir}/${BPN}
-	cp -a --no-preserve=ownership ${B}/Module.symvers ${D}${includedir}/${BPN}
-	# it doesn't actually seem to matter which path is specified here
-	sed -e 's:${B}/::g' -i ${D}${includedir}/${BPN}/Module.symvers
+	if [ ! -e "${B}/${MODULES_MODULE_SYMVERS_LOCATION}/Module.symvers" ] ; then
+		bbwarn "Module.symvers not found in ${B}/${MODULES_MODULE_SYMVERS_LOCATION}"
+		bbwarn "Please consider setting MODULES_MODULE_SYMVERS_LOCATION to a"
+		bbwarn "directory below B to get correct inter-module dependencies"
+	else
+		install -Dm0644 "${B}/${MODULES_MODULE_SYMVERS_LOCATION}"/Module.symvers ${D}${includedir}/${BPN}/Module.symvers
+		# Module.symvers contains absolute path to the build directory.
+		# While it doesn't actually seem to matter which path is specified,
+		# clear them out to avoid confusion
+		sed -e 's:${B}/::g' -i ${D}${includedir}/${BPN}/Module.symvers
+	fi
 }
 
 EXPORT_FUNCTIONS do_compile do_install

meta/classes/multilib.bbclass

@@ -119,7 +119,7 @@ PACKAGEFUNCS_append = " do_package_qa_multilib"
 python do_package_qa_multilib() {
 
     def check_mlprefix(pkg, var, mlprefix):
-        values = bb.utils.explode_deps(d.getVar('%s_%s' % (var, pkg), True) or d.getVar(var) or "")
+        values = bb.utils.explode_deps(d.getVar('%s_%s' % (var, pkg)) or d.getVar(var) or "")
         candidates = []
         for i in values:
             if i.startswith('virtual/'):

meta/classes/multilib_global.bbclass

@@ -137,13 +137,11 @@ addhandler multilib_virtclass_handler_vendor
 multilib_virtclass_handler_vendor[eventmask] = "bb.event.ConfigParsed"
 
 python multilib_virtclass_handler_global () {
-    if not e.data:
+    variant = e.data.getVar("BBEXTENDVARIANT")
+    if variant:
         return
 
-    variant = e.data.getVar("BBEXTENDVARIANT")
-
-    if isinstance(e, bb.event.RecipeParsed) and not variant:
-        if bb.data.inherits_class('kernel', e.data) or \
+    if bb.data.inherits_class('kernel', e.data) or \
             bb.data.inherits_class('module-base', e.data) or \
             (bb.data.inherits_class('allarch', e.data) and\
              not bb.data.inherits_class('packagegroup', e.data)):
@@ -177,5 +175,5 @@ python multilib_virtclass_handler_global () {
 }
 
 addhandler multilib_virtclass_handler_global
-multilib_virtclass_handler_global[eventmask] = "bb.event.RecipePreFinalise bb.event.RecipeParsed"
+multilib_virtclass_handler_global[eventmask] = "bb.event.RecipeParsed"