linux-yocto: Update genericx86* SRCREVs for linux-yocto 4.4

(From meta-yocto rev: 6b55634430188b37ccf19b264502270a64ce4d58)

Signed-off-by: Alejandro Hernandez <alejandro.hernandez@linux.intel.com>
Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>

bitbake/bin/bitbake-worker

@@ -436,8 +436,7 @@ class BitbakeWorker(object):
             self.build_pipes[pipe].read()
 
 try:
-    sys.stdin = sys.stdin.detach()
-    worker = BitbakeWorker(sys.stdin)
+    worker = BitbakeWorker(os.fdopen(sys.stdin.fileno(), 'rb'))
     if not profiling:
         worker.serve()
     else:

bitbake/bin/toaster-eventreplay

@@ -21,154 +21,106 @@
 # with this program; if not, write to the Free Software Foundation, Inc.,
 # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
 
+"""
+This command takes a filename as a single parameter. The filename is read
+as a build eventlog, and the ToasterUI is used to process events in the file
+and log data in the database
+"""
 
-# This command takes a filename as a single parameter. The filename is read
-# as a build eventlog, and the ToasterUI is used to process events in the file
-# and log data in the database
-
-from __future__ import print_function
 import os
-import sys, logging
+import sys
+import json
+import pickle
+import codecs
+
+from collections import namedtuple
 
 # mangle syspath to allow easy import of modules
-sys.path.insert(0, os.path.join(os.path.dirname(os.path.dirname(os.path.abspath(__file__))),
-                                'lib'))
-
+from os.path import join, dirname, abspath
+sys.path.insert(0, join(dirname(dirname(abspath(__file__))), 'lib'))
 
 import bb.cooker
 from bb.ui import toasterui
-import sys
-import logging
 
-import json, pickle
+class EventPlayer:
+    """Emulate a connection to a bitbake server."""
 
+    def __init__(self, eventfile, variables):
+        self.eventfile = eventfile
+        self.variables = variables
+        self.eventmask = []
 
-class FileReadEventsServerConnection():
-    """  Emulates a connection to a bitbake server that feeds
-        events coming actually read from a saved log file.
-    """
+    def waitEvent(self, _timeout):
+        """Read event from the file."""
+        line = self.eventfile.readline().strip()
+        if not line:
+            return
+        try:
+            event_str = json.loads(line)['vars'].encode('utf-8')
+            event = pickle.loads(codecs.decode(event_str, 'base64'))
+            event_name = "%s.%s" % (event.__module__, event.__class__.__name__)
+            if event_name not in self.eventmask:
+                return
+            return event
+        except ValueError as err:
+            print("Failed loading ", line)
+            raise err
 
-    class MockConnection():
-        """ fill-in for the proxy to the server. we just return generic data
+    def runCommand(self, command_line):
+        """Emulate running a command on the server."""
+        name = command_line[0]
+
+        if name == "getVariable":
+            var_name = command_line[1]
+            variable = self.variables.get(var_name)
+            if variable:
+                return variable['v'], None
+            return None, "Missing variable %s" % var_name
+
+        elif name == "getAllKeysWithFlags":
+            dump = {}
+            flaglist = command_line[1]
+            for key, val in self.variables.items():
+                try:
+                    if not key.startswith("__"):
+                        dump[key] = {
+                            'v': val['v'],
+                            'history' : val['history'],
+                        }
+                        for flag in flaglist:
+                            dump[key][flag] = val[flag]
+                except Exception as err:
+                    print(err)
+            return (dump, None)
+
+        elif name == 'setEventMask':
+            self.eventmask = command_line[-1]
+            return True, None
+
+        else:
+            raise Exception("Command %s not implemented" % command_line[0])
+
+    def getEventHandle(self):
         """
-        def __init__(self, sc):
-            self._sc = sc
+        This method is called by toasterui.
+        The return value is passed to self.runCommand but not used there.
+        """
+        pass
 
-        def runCommand(self, commandArray):
-            """ emulates running a command on the server; only read-only commands are accepted """
-            command_name = commandArray[0]
+def main(argv):
+    with open(argv[-1]) as eventfile:
+        # load variables from the first line
+        variables = json.loads(eventfile.readline().strip())['allvariables']
 
-            if command_name == "getVariable":
-                if commandArray[1] in self._sc._variables:
-                    return (self._sc._variables[commandArray[1]]['v'], None)
-                return (None, "Missing variable")
-
-            elif command_name == "getAllKeysWithFlags":
-                dump = {}
-                flaglist = commandArray[1]
-                for k in self._sc._variables.keys():
-                    try:
-                        if not k.startswith("__"):
-                            v = self._sc._variables[k]['v']
-                            dump[k] = {
-                                'v' : v ,
-                                'history' : self._sc._variables[k]['history'],
-                            }
-                            for d in flaglist:
-                                dump[k][d] = self._sc._variables[k][d]
-                    except Exception as e:
-                        print(e)
-                return (dump, None)
-            else:
-                raise Exception("Command %s not implemented" % commandArray[0])
-
-        def terminateServer(self):
-            """ do not do anything """
-            pass
-
-
-
-    class EventReader():
-        def __init__(self, sc):
-            self._sc = sc
-            self.firstraise = 0
-
-        def _create_event(self, line):
-            def _import_class(name):
-                assert len(name) > 0
-                assert "." in name, name
-
-                components = name.strip().split(".")
-                modulename = ".".join(components[:-1])
-                moduleklass = components[-1]
-
-                module = __import__(modulename, fromlist=[str(moduleklass)])
-                return getattr(module, moduleklass)
-
-            # we build a toaster event out of current event log line
-            try:
-                event_data = json.loads(line.strip())
-                event_class = _import_class(event_data['class'])
-                event_object = pickle.loads(json.loads(event_data['vars']))
-            except ValueError as e:
-                print("Failed loading ", line)
-                raise e
-
-            if not isinstance(event_object, event_class):
-                raise Exception("Error loading objects %s class %s ", event_object, event_class)
-
-            return event_object
-
-        def waitEvent(self, timeout):
-
-            nextline = self._sc._eventfile.readline()
-            if len(nextline) == 0:
-                # the build data ended, while toasterui still waits for events.
-                # this happens when the server was abruptly stopped, so we simulate this
-                self.firstraise += 1
-                if self.firstraise == 1:
-                    raise KeyboardInterrupt()
-                else:
-                    return None
-            else:
-                self._sc.lineno += 1
-            return self._create_event(nextline)
-
-
-    def _readVariables(self, variableline):
-        self._variables = json.loads(variableline.strip())['allvariables']
-
-
-    def __init__(self, file_name):
-        self.connection = FileReadEventsServerConnection.MockConnection(self)
-        self._eventfile = open(file_name, "r")
-
-        # we expect to have the variable dump at the start of the file
-        self.lineno = 1
-        self._readVariables(self._eventfile.readline())
-
-        self.events = FileReadEventsServerConnection.EventReader(self)
-
-
-
-
-
-class MockConfigParameters():
-    """ stand-in for cookerdata.ConfigParameters; as we don't really config a cooker, this
-        serves just to supply needed interfaces for the toaster ui to work """
-    def __init__(self):
-        self.observe_only = True            # we can only read files
+        params = namedtuple('ConfigParams', ['observe_only'])(True)
+        player = EventPlayer(eventfile, variables)
 
+        return toasterui.main(player, player, params)
 
 # run toaster ui on our mock bitbake class
 if __name__ == "__main__":
-    if len(sys.argv) < 2:
-        print("Usage: %s event.log " % sys.argv[0])
+    if len(sys.argv) != 2:
+        print("Usage: %s <event file>" % os.path.basename(sys.argv[0]))
         sys.exit(1)
 
-    file_name = sys.argv[-1]
-    mock_connection = FileReadEventsServerConnection(file_name)
-    configParams = MockConfigParameters()
-
-    # run the main program and set exit code to the returned value
-    sys.exit(toasterui.main(mock_connection.connection, mock_connection.events, configParams))
+    sys.exit(main(sys.argv))

bitbake/doc/bitbake-user-manual/bitbake-user-manual-fetching.xml

@@ -670,8 +670,8 @@
                             The <filename>module</filename> and <filename>vob</filename>
                             options are combined to create the <filename>load</filename> rule in
                             the view config spec.
-                            As an example, consider the <filename>vob</filename> and 
-                            <filename>module</filename> values from the 
+                            As an example, consider the <filename>vob</filename> and
+                            <filename>module</filename> values from the
                             <filename>SRC_URI</filename> statement at the start of this section.
                             Combining those values results in the following:
                             <literallayout class='monospaced'>
@@ -716,6 +716,68 @@
             </para>
         </section>
 
+        <section id='perforce-fetcher'>
+            <title>Perforce Fetcher (<filename>p4://</filename>)</title>
+
+            <para>
+                This fetcher submodule fetches code from the
+                <ulink url='https://www.perforce.com/'>Perforce</ulink>
+                source control system.
+                The executable used is specified by
+                <filename>FETCHCMD_p4</filename>, which defaults
+                to "p4".
+                The fetcher's temporary working directory is set by
+                <link linkend='var-P4DIR'><filename>P4DIR</filename></link>,
+                which defaults to "DL_DIR/p4".
+            </para>
+
+            <para>
+                To use this fetcher, make sure your recipe has proper
+                <link linkend='var-SRC_URI'><filename>SRC_URI</filename></link>,
+                <link linkend='var-SRCREV'><filename>SRCREV</filename></link>, and
+                <link linkend='var-PV'><filename>PV</filename></link> values.
+                The p4 executable is able to use the config file defined by your
+                system's <filename>P4CONFIG</filename> environment variable in
+                order to define the Perforce server URL and port, username, and
+                password if you do not wish to keep those values in a recipe
+                itself.
+                If you choose not to use <filename>P4CONFIG</filename>,
+                or to explicitly set variables that <filename>P4CONFIG</filename>
+                can contain, you can specify the <filename>P4PORT</filename> value,
+                which is the server's URL and port number, and you can
+                specify a username and password directly in your recipe within
+                <filename>SRC_URI</filename>.
+            </para>
+
+            <para>
+                Here is an example that relies on <filename>P4CONFIG</filename>
+                to specify the server URL and port, username, and password, and
+                fetches the Head Revision:
+                <literallayout class='monospaced'>
+    SRC_URI = "p4://example-depot/main/source/..."
+    SRCREV = "${AUTOREV}"
+    PV = "p4-${SRCPV}"
+    S = "${WORKDIR}/p4"
+                </literallayout>
+            </para>
+
+            <para>
+                Here is an example that specifies the server URL and port,
+                username, and password, and fetches a Revision based on a Label:
+                <literallayout class='monospaced'>
+    P4PORT = "tcp:p4server.example.net:1666"
+    SRC_URI = "p4://user:passwd@example-depot/main/source/..."
+    SRCREV = "release-1.0"
+    PV = "p4-${SRCPV}"
+    S = "${WORKDIR}/p4"
+                </literallayout>
+                <note>
+                    You should always set <filename>S</filename>
+                    to <filename>"${WORKDIR}/p4"</filename> in your recipe.
+                </note>
+            </para>
+        </section>
+
         <section id='other-fetchers'>
             <title>Other Fetchers</title>
 
@@ -725,9 +787,6 @@
                     <listitem><para>
                         Bazaar (<filename>bzr://</filename>)
                         </para></listitem>
-                    <listitem><para>
-                        Perforce (<filename>p4://</filename>)
-                        </para></listitem>
                     <listitem><para>
                         Trees using Git Annex (<filename>gitannex://</filename>)
                         </para></listitem>

bitbake/doc/bitbake-user-manual/bitbake-user-manual-intro.xml

@@ -634,6 +634,44 @@
                 </para>
             </section>
 
+            <section id='executing-a-list-of-task-and-recipe-combinations'>
+                <title>Executing a List of Task and Recipe Combinations</title>
+
+                <para>
+                    The BitBake command line supports specifying different
+                    tasks for individual targets when you specify multiple
+                    targets.
+                    For example, suppose you had two targets (or recipes)
+                    <filename>myfirstrecipe</filename> and
+                    <filename>mysecondrecipe</filename> and you needed
+                    BitBake to run <filename>taskA</filename> for the first
+                    recipe and <filename>taskB</filename> for the second
+                    recipe:
+                    <literallayout class='monospaced'>
+     $ bitbake myfirstrecipe recipe:do_taskA mysecondrecipe recipe:do_taskB
+                    </literallayout>
+                </para>
+            </section>
+
+            <section id='executing-a-list-of-task-and-recipe-combinations'>
+                <title>Executing a List of Task and Recipe Combinations</title>
+
+                <para>
+                    The BitBake command line supports specifying different
+                    tasks for individual targets when you specify multiple
+                    targets.
+                    For example, suppose you had two targets (or recipes)
+                    <filename>myfirstrecipe</filename> and
+                    <filename>mysecondrecipe</filename> and you needed
+                    BitBake to run <filename>taskA</filename> for the first
+                    recipe and <filename>taskB</filename> for the second
+                    recipe:
+                    <literallayout class='monospaced'>
+     $ bitbake myfirstrecipe recipe:do_taskA mysecondrecipe recipe:do_taskB
+                    </literallayout>
+                </para>
+            </section>
+
             <section id='generating-dependency-graphs'>
                 <title>Generating Dependency Graphs</title>
 

bitbake/doc/bitbake-user-manual/bitbake-user-manual-metadata.xml

@@ -1624,15 +1624,32 @@
         <title>Dependencies</title>
 
         <para>
-            To allow for efficient operation given multiple processes
-            executing in parallel, BitBake handles dependencies at
-            the task level.
-            BitBake supports a robust method to handle these dependencies.
-        </para>
+            To allow for efficient parallel processing, BitBake handles
+            dependencies at the task level.
+            Dependencies can exist both between tasks within a single recipe
+            and between tasks in different recipes.
+            Following are examples of each:
+            <itemizedlist>
+                <listitem><para>For tasks within a single recipe, a
+                    recipe's <filename>do_configure</filename>
+                    task might need to complete before its
+                    <filename>do_compile</filename> task can run.
+                    </para></listitem>
+                <listitem><para>For tasks in different recipes, one
+                    recipe's <filename>do_configure</filename>
+                    task might require another recipe's
+                    <filename>do_populate_sysroot</filename>
+                    task to finish first such that the libraries and headers
+                    provided by the other recipe are available.
+                    </para></listitem>
+             </itemizedlist>
+         </para>
 
-        <para>
-            This section describes several types of dependency mechanisms.
-        </para>
+         <para>
+             This section describes several ways to declare dependencies.
+             Remember, even though dependencies are declared in different ways, they
+             are all simply dependencies between tasks.
+         </para>
 
         <section id='dependencies-internal-to-the-bb-file'>
             <title>Dependencies Internal to the <filename>.bb</filename> File</title>
@@ -1653,6 +1670,40 @@
                 task.
                 And, the <filename>do_build</filename> depends on the completion
                 of the <filename>printdate</filename> task.
+                <note>
+                    Recipes are built by having their
+                    <filename>do_build</filename> (not to be confused with
+                    <filename>do_compile</filename>) tasks executed.
+                    For a task to run when a recipe is built, the task must
+                    therefore be a direct or indirect dependency of
+                    <filename>do_build</filename>.
+                    For illustration, here are some examples:
+                    <itemizedlist>
+                        <listitem><para>
+                            The directive
+                            <filename>addtask mytask before do_build</filename>
+                            causes <filename>mytask</filename> to run when the
+                            recipe is built.
+                            In this example, <filename>mytask</filename> is run
+                            at an unspecified time relative to other tasks within
+                            the recipe, since <filename>after</filename> is not used.
+                            </para></listitem>
+                        <listitem><para>
+                            The directive
+                            <filename>addtask mytask after do_configure</filename>
+                            by itself does not cause <filename>mytask</filename>
+                            to run when the recipe is built.
+                            The task can still be run manually using the following:
+                            <literallayout class='monospaced'>
+     $ bitbake <replaceable>recipe</replaceable> -c mytask
+                            </literallayout>
+                            <filename>mytask</filename> could also be declared as
+                            a dependency of some other task.
+                            Regardless, the task is run after
+                            <filename>do_configure</filename>.
+                            </para></listitem>
+                    </itemizedlist>
+                </note>
             </para>
         </section>
 
@@ -1721,6 +1772,20 @@
                 and added.
             </para>
 
+            <para>
+                The "recrdeptask" flag is most commonly used in high-level
+                recipes that need to wait for some task to finish "globally".
+                For example, <filename>image.bbclass</filename> has the following:
+                <literallayout class='monospaced'>
+     do_rootfs[recrdeptask] += "do_packagedata"
+                </literallayout>
+                This statement says that the <filename>do_packagedata</filename>
+                task of the current recipe and all recipes reachable
+                (by way of dependencies) from the
+                image recipe must run before the <filename>do_rootfs</filename>
+                task can run.
+            </para>
+
             <para>
                 You might want to not only have BitBake look for
                 dependencies of those tasks, but also have BitBake look
@@ -1785,9 +1850,9 @@
                     </thead>
                     <tbody>
                         <row>
-                            <entry align="left"><filename>d.getVar("X", expand=False)</filename></entry>
+                            <entry align="left"><filename>d.getVar("X", expand)</filename></entry>
                             <entry align="left">Returns the value of variable "X".
-                                Using "expand=True" expands the value.</entry>
+                            Using "expand=True" expands the value.</entry>
                         </row>
                         <row>
                             <entry align="left"><filename>d.setVar("X", "value")</filename></entry>
@@ -1810,9 +1875,9 @@
                             <entry align="left">Renames the variable "X" to "Y".</entry>
                         </row>
                         <row>
-                            <entry align="left"><filename>d.getVarFlag("X", flag, expand=False)</filename></entry>
-                            <entry align="left">Gets then named flag from the variable "X".
-                                Using "expand=True" expands the named flag.</entry>
+                            <entry align="left"><filename>d.getVarFlag("X", flag, expand)</filename></entry>
+                            <entry align="left">Returns the value of variable "X".
+                                Using "expand=True" expands the value.</entry>
                         </row>
                         <row>
                             <entry align="left"><filename>d.setVarFlag("X", flag, "value")</filename></entry>

bitbake/doc/bitbake-user-manual/bitbake-user-manual-ref-variables.xml

@@ -52,7 +52,7 @@
        <link linkend='var-MIRRORS'>M</link>
 <!--               <link linkend='var-glossary-n'>N</link> -->
        <link linkend='var-OVERRIDES'>O</link>
-       <link linkend='var-PACKAGES'>P</link>
+       <link linkend='var-P4DIR'>P</link>
 <!--       <link linkend='var-QMAKE_PROFILES'>Q</link> -->
        <link linkend='var-RDEPENDS'>R</link>
        <link linkend='var-SECTION'>S</link>
@@ -986,6 +986,29 @@
      BBCLASSEXTEND =+ "native nativesdk"
      BBCLASSEXTEND =+ "multilib:<replaceable>multilib_name</replaceable>"
                     </literallayout>
+                    <note>
+                        <para>
+                        Internally, the <filename>BBCLASSEXTEND</filename>
+                        mechanism generates recipe variants by rewriting
+                        variable values and applying overrides such as
+                        <filename>_class-native</filename>.
+                        For example, to generate a native version of a recipe,
+                        a
+                        <link linkend='var-DEPENDS'><filename>DEPENDS</filename></link>
+                        on "foo" is rewritten to a <filename>DEPENDS</filename>
+                        on "foo-native".
+                        </para>
+
+                        <para>
+                        Even when using <filename>BBCLASSEXTEND</filename>, the
+                        recipe is only parsed once.
+                        Parsing once adds some limitations.
+                        For example, it is not possible to
+                        include a different file depending on the variant,
+                        since <filename>include</filename> statements are
+                        processed when the recipe is parsed.
+                        </para>
+                    </note>
                 </para>
              </glossdef>
         </glossentry>
@@ -994,7 +1017,7 @@
             <glossdef>
                 <para>
                     Sets the BitBake debug output level to a specific value
-                    as incremented by the <filename>-d</filename> command line
+                    as incremented by the <filename>-D</filename> command line
                     option.
                     <note>
                         You must set this variable in the external environment
@@ -1748,6 +1771,15 @@
 
     <glossdiv id='var-glossary-p'><title>P</title>
 
+        <glossentry id='var-P4DIR'><glossterm>P4DIR</glossterm>
+            <glossdef>
+                <para>
+                    The directory in which a local copy of a Perforce depot
+                    is stored when it is fetched.
+                </para>
+            </glossdef>
+        </glossentry>
+
         <glossentry id='var-PACKAGES'><glossterm>PACKAGES</glossterm>
             <glossdef>
                 <para>The list of packages the recipe creates.

bitbake/lib/bb/build.py

@@ -35,6 +35,7 @@ import stat
 import bb
 import bb.msg
 import bb.process
+import bb.progress
 from bb import data, event, utils
 
 bblogger = logging.getLogger('BitBake')
@@ -137,6 +138,25 @@ class TaskInvalid(TaskBase):
         super(TaskInvalid, self).__init__(task, None, metadata)
         self._message = "No such task '%s'" % task
 
+class TaskProgress(event.Event):
+    """
+    Task made some progress that could be reported to the user, usually in
+    the form of a progress bar or similar.
+    NOTE: this class does not inherit from TaskBase since it doesn't need
+    to - it's fired within the task context itself, so we don't have any of
+    the context information that you do in the case of the other events.
+    The event PID can be used to determine which task it came from.
+    The progress value is normally 0-100, but can also be negative
+    indicating that progress has been made but we aren't able to determine
+    how much.
+    The rate is optional, this is simply an extra string to display to the
+    user if specified.
+    """
+    def __init__(self, progress, rate=None):
+        self.progress = progress
+        self.rate = rate
+        event.Event.__init__(self)
+
 
 class LogTee(object):
     def __init__(self, logger, outfile):
@@ -168,6 +188,11 @@ class LogTee(object):
 def exec_func(func, d, dirs = None, pythonexception=False):
     """Execute a BB 'function'"""
 
+    try:
+        oldcwd = os.getcwd()
+    except:
+        oldcwd = None
+
     body = d.getVar(func, False)
     if not body:
         if body is None:
@@ -191,9 +216,7 @@ def exec_func(func, d, dirs = None, pythonexception=False):
             bb.utils.mkdirhier(adir)
         adir = dirs[-1]
     else:
-        adir = d.getVar('B', True)
-        bb.utils.mkdirhier(adir)
-
+        adir = None
     ispython = flags.get('python')
 
     lockflag = flags.get('lockfiles')
@@ -237,6 +260,13 @@ def exec_func(func, d, dirs = None, pythonexception=False):
         else:
             exec_func_shell(func, d, runfile, cwd=adir)
 
+    if oldcwd and os.getcwd() != oldcwd:
+        try:
+            bb.warn("Task %s changed cwd to %s" % (func, os.getcwd()))
+            os.chdir(oldcwd)
+        except:
+            pass
+
 _functionfmt = """
 {function}(d)
 """
@@ -252,7 +282,8 @@ def exec_func_python(func, d, runfile, cwd=None, pythonexception=False):
     if cwd:
         try:
             olddir = os.getcwd()
-        except OSError:
+        except OSError as e:
+            bb.warn("%s: Cannot get cwd: %s" % (func, e))
             olddir = None
         os.chdir(cwd)
 
@@ -278,8 +309,8 @@ def exec_func_python(func, d, runfile, cwd=None, pythonexception=False):
         if cwd and olddir:
             try:
                 os.chdir(olddir)
-            except OSError:
-                pass
+            except OSError as e:
+                bb.warn("%s: Cannot restore cwd %s: %s" % (func, olddir, e))
 
 def shell_trap_code():
     return '''#!/bin/sh\n
@@ -340,11 +371,28 @@ exit $ret
     else:
         logfile = sys.stdout
 
+    progress = d.getVarFlag(func, 'progress', True)
+    if progress:
+        if progress == 'percent':
+            # Use default regex
+            logfile = bb.progress.BasicProgressHandler(d, outfile=logfile)
+        elif progress.startswith('percent:'):
+            # Use specified regex
+            logfile = bb.progress.BasicProgressHandler(d, regex=progress.split(':', 1)[1], outfile=logfile)
+        elif progress.startswith('outof:'):
+            # Use specified regex
+            logfile = bb.progress.OutOfProgressHandler(d, regex=progress.split(':', 1)[1], outfile=logfile)
+        else:
+            bb.warn('%s: invalid task progress varflag value "%s", ignoring' % (func, progress))
+
     def readfifo(data):
         lines = data.split(b'\0')
         for line in lines:
+            # Just skip empty commands
+            if not line:
+                continue
             splitval = line.split(b' ', 1)
-            cmd = splitval[0]
+            cmd = splitval[0].decode("utf-8")
             if len(splitval) > 1:
                 value = splitval[1].decode("utf-8")
             else:
@@ -368,7 +416,8 @@ exit $ret
                 level = int(splitval[0])
                 value = splitval[1]
                 bb.debug(level, value)
-
+            else:
+                bb.warn("Unrecognised command '%s' on FIFO" % cmd)
     tempdir = d.getVar('T', True)
     fifopath = os.path.join(tempdir, 'fifo.%s' % os.getpid())
     if os.path.exists(fifopath):
@@ -778,6 +827,7 @@ def deltask(task, d):
     bbtasks = d.getVar('__BBTASKS', False) or []
     if task in bbtasks:
         bbtasks.remove(task)
+        d.delVarFlag(task, 'task')
         d.setVar('__BBTASKS', bbtasks)
 
     d.delVarFlag(task, 'deps')

bitbake/lib/bb/cooker.py

@@ -744,7 +744,7 @@ class BBCooker:
             for dep in rq.rqdata.runtaskentries[tid].depends:
                 depfn = bb.runqueue.fn_from_tid(dep)
                 deppn = self.recipecache.pkg_fn[depfn]
-                dotname = "%s.%s" % (pn, bb.runqueue.taskname_from_tid(dep))
+                dotname = "%s.%s" % (pn, bb.runqueue.taskname_from_tid(tid))
                 if not dotname in depend_tree["tdepends"]:
                     depend_tree["tdepends"][dotname] = []
                 depend_tree["tdepends"][dotname].append("%s.%s" % (deppn, bb.runqueue.taskname_from_tid(dep)))

bitbake/lib/bb/event.py

@@ -647,6 +647,33 @@ class MetadataEvent(Event):
         self.type = eventtype
         self._localdata = eventdata
 
+class ProcessStarted(Event):
+    """
+    Generic process started event (usually part of the initial startup)
+    where further progress events will be delivered
+    """
+    def __init__(self, processname, total):
+        Event.__init__(self)
+        self.processname = processname
+        self.total = total
+
+class ProcessProgress(Event):
+    """
+    Generic process progress event (usually part of the initial startup)
+    """
+    def __init__(self, processname, progress):
+        Event.__init__(self)
+        self.processname = processname
+        self.progress = progress
+
+class ProcessFinished(Event):
+    """
+    Generic process finished event (usually part of the initial startup)
+    """
+    def __init__(self, processname):
+        Event.__init__(self)
+        self.processname = processname
+
 class SanityCheck(Event):
     """
     Event to run sanity checks, either raise errors or generate events as return status.

bitbake/lib/bb/fetch2/__init__.py

@@ -779,7 +779,7 @@ def localpath(url, d):
     fetcher = bb.fetch2.Fetch([url], d)
     return fetcher.localpath(url)
 
-def runfetchcmd(cmd, d, quiet=False, cleanup=None):
+def runfetchcmd(cmd, d, quiet=False, cleanup=None, log=None):
     """
     Run cmd returning the command output
     Raise an error if interrupted or cmd fails
@@ -821,7 +821,7 @@ def runfetchcmd(cmd, d, quiet=False, cleanup=None):
     error_message = ""
 
     try:
-        (output, errors) = bb.process.run(cmd, shell=True, stderr=subprocess.PIPE)
+        (output, errors) = bb.process.run(cmd, log=log, shell=True, stderr=subprocess.PIPE)
         success = True
     except bb.process.NotFoundError as e:
         error_message = "Fetch command %s" % (e.command)

bitbake/lib/bb/fetch2/git.py

@@ -71,11 +71,53 @@ import os
 import re
 import bb
 import errno
+import bb.progress
 from   bb    import data
 from   bb.fetch2 import FetchMethod
 from   bb.fetch2 import runfetchcmd
 from   bb.fetch2 import logger
 
+
+class GitProgressHandler(bb.progress.LineFilterProgressHandler):
+    """Extract progress information from git output"""
+    def __init__(self, d):
+        self._buffer = ''
+        self._count = 0
+        super(GitProgressHandler, self).__init__(d)
+        # Send an initial progress event so the bar gets shown
+        self._fire_progress(-1)
+
+    def write(self, string):
+        self._buffer += string
+        stages = ['Counting objects', 'Compressing objects', 'Receiving objects', 'Resolving deltas']
+        stage_weights = [0.2, 0.05, 0.5, 0.25]
+        stagenum = 0
+        for i, stage in reversed(list(enumerate(stages))):
+            if stage in self._buffer:
+                stagenum = i
+                self._buffer = ''
+                break
+        self._status = stages[stagenum]
+        percs = re.findall(r'(\d+)%', string)
+        if percs:
+            progress = int(round((int(percs[-1]) * stage_weights[stagenum]) + (sum(stage_weights[:stagenum]) * 100)))
+            rates = re.findall(r'([\d.]+ [a-zA-Z]*/s+)', string)
+            if rates:
+                rate = rates[-1]
+            else:
+                rate = None
+            self.update(progress, rate)
+        else:
+            if stagenum == 0:
+                percs = re.findall(r': (\d+)', string)
+                if percs:
+                    count = int(percs[-1])
+                    if count > self._count:
+                        self._count = count
+                        self._fire_progress(-count)
+        super(GitProgressHandler, self).write(string)
+
+
 class Git(FetchMethod):
     """Class to fetch a module or modules from git repositories"""
     def init(self, d):
@@ -196,10 +238,11 @@ class Git(FetchMethod):
             # We do this since git will use a "-l" option automatically for local urls where possible
             if repourl.startswith("file://"):
                 repourl = repourl[7:]
-            clone_cmd = "%s clone --bare --mirror %s %s" % (ud.basecmd, repourl, ud.clonedir)
+            clone_cmd = "LANG=C %s clone --bare --mirror %s %s --progress" % (ud.basecmd, repourl, ud.clonedir)
             if ud.proto.lower() != 'file':
                 bb.fetch2.check_network_access(d, clone_cmd)
-            runfetchcmd(clone_cmd, d)
+            progresshandler = GitProgressHandler(d)
+            runfetchcmd(clone_cmd, d, log=progresshandler)
 
         os.chdir(ud.clonedir)
         # Update the checkout if needed
@@ -214,10 +257,11 @@ class Git(FetchMethod):
                 logger.debug(1, "No Origin")
 
             runfetchcmd("%s remote add --mirror=fetch origin %s" % (ud.basecmd, repourl), d)
-            fetch_cmd = "%s fetch -f --prune %s refs/*:refs/*" % (ud.basecmd, repourl)
+            fetch_cmd = "LANG=C %s fetch -f --prune --progress %s refs/*:refs/*" % (ud.basecmd, repourl)
             if ud.proto.lower() != 'file':
                 bb.fetch2.check_network_access(d, fetch_cmd, ud.url)
-            runfetchcmd(fetch_cmd, d)
+            progresshandler = GitProgressHandler(d)
+            runfetchcmd(fetch_cmd, d, log=progresshandler)
             runfetchcmd("%s prune-packed" % ud.basecmd, d)
             runfetchcmd("%s pack-redundant --all | xargs -r rm" % ud.basecmd, d)
             try:

bitbake/lib/bb/fetch2/wget.py

@@ -31,6 +31,7 @@ import subprocess
 import os
 import logging
 import bb
+import bb.progress
 import urllib.request, urllib.parse, urllib.error
 from   bb import data
 from   bb.fetch2 import FetchMethod
@@ -41,6 +42,27 @@ from   bb.utils import export_proxies
 from   bs4 import BeautifulSoup
 from   bs4 import SoupStrainer
 
+class WgetProgressHandler(bb.progress.LineFilterProgressHandler):
+    """
+    Extract progress information from wget output.
+    Note: relies on --progress=dot (with -v or without -q/-nv) being
+    specified on the wget command line.
+    """
+    def __init__(self, d):
+        super(WgetProgressHandler, self).__init__(d)
+        # Send an initial progress event so the bar gets shown
+        self._fire_progress(0)
+
+    def writeline(self, line):
+        percs = re.findall(r'(\d+)%\s+([\d.]+[A-Z])', line)
+        if percs:
+            progress = int(percs[-1][0])
+            rate = percs[-1][1] + '/s'
+            self.update(progress, rate)
+            return False
+        return True
+
+
 class Wget(FetchMethod):
     """Class to fetch urls via 'wget'"""
     def supports(self, ud, d):
@@ -66,13 +88,15 @@ class Wget(FetchMethod):
         if not ud.localfile:
             ud.localfile = data.expand(urllib.parse.unquote(ud.host + ud.path).replace("/", "."), d)
 
-        self.basecmd = d.getVar("FETCHCMD_wget", True) or "/usr/bin/env wget -t 2 -T 30 -nv --passive-ftp --no-check-certificate"
+        self.basecmd = d.getVar("FETCHCMD_wget", True) or "/usr/bin/env wget -t 2 -T 30 --passive-ftp --no-check-certificate"
 
     def _runwget(self, ud, d, command, quiet):
 
+        progresshandler = WgetProgressHandler(d)
+
         logger.debug(2, "Fetching %s using command '%s'" % (ud.url, command))
         bb.fetch2.check_network_access(d, command)
-        runfetchcmd(command, d, quiet)
+        runfetchcmd(command + ' --progress=dot -v', d, quiet, log=progresshandler)
 
     def download(self, ud, d):
         """Fetch urls"""
@@ -104,7 +128,7 @@ class Wget(FetchMethod):
 
         return True
 
-    def checkstatus(self, fetch, ud, d):
+    def checkstatus(self, fetch, ud, d, try_again=True):
         import urllib.request, urllib.error, urllib.parse, socket, http.client
         from urllib.response import addinfourl
         from bb.fetch2 import FetchConnectionCache
@@ -278,9 +302,13 @@ class Wget(FetchMethod):
             r.get_method = lambda: "HEAD"
             opener.open(r)
         except urllib.error.URLError as e:
-            # debug for now to avoid spamming the logs in e.g. remote sstate searches
-            logger.debug(2, "checkstatus() urlopen failed: %s" % e)
-            return False
+            if try_again:
+                logger.debug(2, "checkstatus: trying again")
+                return self.checkstatus(fetch, ud, d, False)
+            else:
+                # debug for now to avoid spamming the logs in e.g. remote sstate searches
+                logger.debug(2, "checkstatus() urlopen failed: %s" % e)
+                return False
         return True
 
     def _parse_path(self, regex, s):

bitbake/lib/bb/main.py

@@ -179,6 +179,9 @@ class BitBakeConfigParameters(cookerdata.ConfigParameters):
         parser.add_option("-D", "--debug", action="count", dest="debug", default=0,
                           help="Increase the debug level. You can specify this more than once.")
 
+        parser.add_option("-q", "--quiet", action="store_true", dest="quiet", default=False,
+                          help="Output less log message data to the terminal.")
+
         parser.add_option("-n", "--dry-run", action="store_true", dest="dry_run", default=False,
                           help="Don't execute, just go through the motions.")
 
@@ -244,9 +247,16 @@ class BitBakeConfigParameters(cookerdata.ConfigParameters):
                           help="Run bitbake without a UI, only starting a server "
                                "(cooker) process.")
 
+        parser.add_option("", "--foreground", action="store_true",
+                          help="Run bitbake server in foreground.")
+
         parser.add_option("-B", "--bind", action="store", dest="bind", default=False,
                           help="The name/address for the bitbake server to bind to.")
 
+        parser.add_option("-T", "--idle-timeout", type=int,
+                          default=int(os.environ.get("BBTIMEOUT", "0")),
+                          help="Set timeout to unload bitbake server due to inactivity")
+
         parser.add_option("", "--no-setscene", action="store_true",
                           dest="nosetscene", default=False,
                           help="Do not run any setscene tasks. sstate will be ignored and "
@@ -279,6 +289,12 @@ class BitBakeConfigParameters(cookerdata.ConfigParameters):
 
         options, targets = parser.parse_args(argv)
 
+        if options.quiet and options.verbose:
+            parser.error("options --quiet and --verbose are mutually exclusive")
+
+        if options.quiet and options.debug:
+            parser.error("options --quiet and --debug are mutually exclusive")
+
         # use configuration files from environment variables
         if "BBPRECONF" in os.environ:
             options.prefile.append(os.environ["BBPRECONF"])
@@ -294,8 +310,10 @@ class BitBakeConfigParameters(cookerdata.ConfigParameters):
 
         # if BBSERVER says to autodetect, let's do that
         if options.remote_server:
-            [host, port] = options.remote_server.split(":", 2)
-            port = int(port)
+            port = -1
+            if options.remote_server != 'autostart':
+                host, port = options.remote_server.split(":", 2)
+                port = int(port)
             # use automatic port if port set to -1, means read it from
             # the bitbake.lock file; this is a bit tricky, but we always expect
             # to be in the base of the build directory if we need to have a
@@ -312,17 +330,19 @@ class BitBakeConfigParameters(cookerdata.ConfigParameters):
                     lf.close()
                     options.remote_server = remotedef
                 except Exception as e:
-                    raise BBMainException("Failed to read bitbake.lock (%s), invalid port" % str(e))
+                    if options.remote_server != 'autostart':
+                        raise BBMainException("Failed to read bitbake.lock (%s), invalid port" % str(e))
 
         return options, targets[1:]
 
 
 def start_server(servermodule, configParams, configuration, features):
     server = servermodule.BitBakeServer()
-    single_use = not configParams.server_only
+    single_use = not configParams.server_only and os.getenv('BBSERVER') != 'autostart'
     if configParams.bind:
         (host, port) = configParams.bind.split(':')
-        server.initServer((host, int(port)), single_use)
+        server.initServer((host, int(port)), single_use=single_use,
+                          idle_timeout=configParams.idle_timeout)
         configuration.interface = [server.serverImpl.host, server.serverImpl.port]
     else:
         server.initServer(single_use=single_use)
@@ -345,7 +365,8 @@ def start_server(servermodule, configParams, configuration, features):
             if isinstance(event, logging.LogRecord):
                 logger.handle(event)
         raise
-    server.detach()
+    if not configParams.foreground:
+        server.detach()
     cooker.lock.close()
     return server
 
@@ -386,6 +407,10 @@ def bitbake_main(configParams, configuration):
                                   ("the BBSERVER environment variable" if "BBSERVER" in os.environ \
                                    else "the '--remote-server' option"))
 
+    elif configParams.foreground:
+        raise BBMainException("FATAL: The '--foreground' option can only be used "
+                              "with --server-only.\n")
+
     if configParams.bind and configParams.servertype != "xmlrpc":
         raise BBMainException("FATAL: If '-B' or '--bind' is defined, we must "
                               "set the servertype as 'xmlrpc'.\n")
@@ -436,6 +461,14 @@ def bitbake_main(configParams, configuration):
         server = start_server(servermodule, configParams, configuration, featureset)
         bb.event.ui_queue = []
     else:
+        if os.getenv('BBSERVER') == 'autostart':
+            if configParams.remote_server == 'autostart' or \
+               not servermodule.check_connection(configParams.remote_server, timeout=2):
+                configParams.bind = 'localhost:0'
+                srv = start_server(servermodule, configParams, configuration, featureset)
+                configParams.remote_server = '%s:%d' % tuple(configuration.interface)
+                bb.event.ui_queue = []
+
         # we start a stub server that is actually a XMLRPClient that connects to a real server
         server = servermodule.BitBakeXMLRPCClient(configParams.observe_only,
                                                   configParams.xmlrpctoken)
@@ -475,6 +508,8 @@ def bitbake_main(configParams, configuration):
     else:
         print("Bitbake server address: %s, server port: %s" % (server.serverImpl.host,
                                                                server.serverImpl.port))
+        if configParams.foreground:
+            server.serverImpl.serve_forever()
         return 0
 
     return 1

bitbake/lib/bb/msg.py

@@ -182,9 +182,12 @@ def constructLogOptions():
         debug_domains["BitBake.%s" % domainarg] = logging.DEBUG - dlevel + 1
     return level, debug_domains
 
-def addDefaultlogFilter(handler, cls = BBLogFilter):
+def addDefaultlogFilter(handler, cls = BBLogFilter, forcelevel=None):
     level, debug_domains = constructLogOptions()
 
+    if forcelevel is not None:
+        level = forcelevel
+
     cls(handler, level, debug_domains)
 
 #

bitbake/lib/bb/progress.py

@@ -0,0 +1,273 @@
+"""
+BitBake progress handling code
+"""
+
+# Copyright (C) 2016 Intel Corporation
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License version 2 as
+# published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License along
+# with this program; if not, write to the Free Software Foundation, Inc.,
+# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+import sys
+import re
+import time
+import inspect
+import bb.event
+import bb.build
+
+class ProgressHandler(object):
+    """
+    Base class that can pretend to be a file object well enough to be
+    used to build objects to intercept console output and determine the
+    progress of some operation.
+    """
+    def __init__(self, d, outfile=None):
+        self._progress = 0
+        self._data = d
+        self._lastevent = 0
+        if outfile:
+            self._outfile = outfile
+        else:
+            self._outfile = sys.stdout
+
+    def _fire_progress(self, taskprogress, rate=None):
+        """Internal function to fire the progress event"""
+        bb.event.fire(bb.build.TaskProgress(taskprogress, rate), self._data)
+
+    def write(self, string):
+        self._outfile.write(string)
+
+    def flush(self):
+        self._outfile.flush()
+
+    def update(self, progress, rate=None):
+        ts = time.time()
+        if progress > 100:
+            progress = 100
+        if progress != self._progress or self._lastevent + 1 < ts:
+            self._fire_progress(progress, rate)
+            self._lastevent = ts
+            self._progress = progress
+
+class LineFilterProgressHandler(ProgressHandler):
+    """
+    A ProgressHandler variant that provides the ability to filter out
+    the lines if they contain progress information. Additionally, it
+    filters out anything before the last line feed on a line. This can
+    be used to keep the logs clean of output that we've only enabled for
+    getting progress, assuming that that can be done on a per-line
+    basis.
+    """
+    def __init__(self, d, outfile=None):
+        self._linebuffer = ''
+        super(LineFilterProgressHandler, self).__init__(d, outfile)
+
+    def write(self, string):
+        self._linebuffer += string
+        while True:
+            breakpos = self._linebuffer.find('\n') + 1
+            if breakpos == 0:
+                break
+            line = self._linebuffer[:breakpos]
+            self._linebuffer = self._linebuffer[breakpos:]
+            # Drop any line feeds and anything that precedes them
+            lbreakpos = line.rfind('\r') + 1
+            if lbreakpos:
+                line = line[lbreakpos:]
+            if self.writeline(line):
+                super(LineFilterProgressHandler, self).write(line)
+
+    def writeline(self, line):
+        return True
+
+class BasicProgressHandler(ProgressHandler):
+    def __init__(self, d, regex=r'(\d+)%', outfile=None):
+        super(BasicProgressHandler, self).__init__(d, outfile)
+        self._regex = re.compile(regex)
+        # Send an initial progress event so the bar gets shown
+        self._fire_progress(0)
+
+    def write(self, string):
+        percs = self._regex.findall(string)
+        if percs:
+            progress = int(percs[-1])
+            self.update(progress)
+        super(BasicProgressHandler, self).write(string)
+
+class OutOfProgressHandler(ProgressHandler):
+    def __init__(self, d, regex, outfile=None):
+        super(OutOfProgressHandler, self).__init__(d, outfile)
+        self._regex = re.compile(regex)
+        # Send an initial progress event so the bar gets shown
+        self._fire_progress(0)
+
+    def write(self, string):
+        nums = self._regex.findall(string)
+        if nums:
+            progress = (float(nums[-1][0]) / float(nums[-1][1])) * 100
+            self.update(progress)
+        super(OutOfProgressHandler, self).write(string)
+
+class MultiStageProgressReporter(object):
+    """
+    Class which allows reporting progress without the caller
+    having to know where they are in the overall sequence. Useful
+    for tasks made up of python code spread across multiple
+    classes / functions - the progress reporter object can
+    be passed around or stored at the object level and calls
+    to next_stage() and update() made whereever needed.
+    """
+    def __init__(self, d, stage_weights, debug=False):
+        """
+        Initialise the progress reporter.
+
+        Parameters:
+        * d: the datastore (needed for firing the events)
+        * stage_weights: a list of weight values, one for each stage.
+          The value is scaled internally so you only need to specify
+          values relative to other values in the list, so if there
+          are two stages and the first takes 2s and the second takes
+          10s you would specify [2, 10] (or [1, 5], it doesn't matter).
+        * debug: specify True (and ensure you call finish() at the end)
+          in order to show a printout of the calculated stage weights
+          based on timing each stage. Use this to determine what the
+          weights should be when you're not sure.
+        """
+        self._data = d
+        total = sum(stage_weights)
+        self._stage_weights = [float(x)/total for x in stage_weights]
+        self._stage = -1
+        self._base_progress = 0
+        # Send an initial progress event so the bar gets shown
+        self._fire_progress(0)
+        self._debug = debug
+        self._finished = False
+        if self._debug:
+            self._last_time = time.time()
+            self._stage_times = []
+            self._stage_total = None
+            self._callers = []
+
+    def _fire_progress(self, taskprogress):
+        bb.event.fire(bb.build.TaskProgress(taskprogress), self._data)
+
+    def next_stage(self, stage_total=None):
+        """
+        Move to the next stage.
+        Parameters:
+        * stage_total: optional total for progress within the stage,
+          see update() for details
+        NOTE: you need to call this before the first stage.
+        """
+        self._stage += 1
+        self._stage_total = stage_total
+        if self._stage == 0:
+            # First stage
+            if self._debug:
+                self._last_time = time.time()
+        else:
+            if self._stage < len(self._stage_weights):
+                self._base_progress = sum(self._stage_weights[:self._stage]) * 100
+                if self._debug:
+                    currtime = time.time()
+                    self._stage_times.append(currtime - self._last_time)
+                    self._last_time = currtime
+                    self._callers.append(inspect.getouterframes(inspect.currentframe())[1])
+            elif not self._debug:
+                bb.warn('ProgressReporter: current stage beyond declared number of stages')
+                self._base_progress = 100
+            self._fire_progress(self._base_progress)
+
+    def update(self, stage_progress):
+        """
+        Update progress within the current stage.
+        Parameters:
+        * stage_progress: progress value within the stage. If stage_total
+          was specified when next_stage() was last called, then this
+          value is considered to be out of stage_total, otherwise it should
+          be a percentage value from 0 to 100.
+        """
+        if self._stage_total:
+            stage_progress = (float(stage_progress) / self._stage_total) * 100
+        if self._stage < 0:
+            bb.warn('ProgressReporter: update called before first call to next_stage()')
+        elif self._stage < len(self._stage_weights):
+            progress = self._base_progress + (stage_progress * self._stage_weights[self._stage])
+        else:
+            progress = self._base_progress
+        if progress > 100:
+            progress = 100
+        self._fire_progress(progress)
+
+    def finish(self):
+        if self._finished:
+            return
+        self._finished = True
+        if self._debug:
+            import math
+            self._stage_times.append(time.time() - self._last_time)
+            mintime = max(min(self._stage_times), 0.01)
+            self._callers.append(None)
+            stage_weights = [int(math.ceil(x / mintime)) for x in self._stage_times]
+            bb.warn('Stage weights: %s' % stage_weights)
+            out = []
+            for stage_weight, caller in zip(stage_weights, self._callers):
+                if caller:
+                    out.append('Up to %s:%d: %d' % (caller[1], caller[2], stage_weight))
+                else:
+                    out.append('Up to finish: %d' % stage_weight)
+            bb.warn('Stage times:\n  %s' % '\n  '.join(out))
+
+class MultiStageProcessProgressReporter(MultiStageProgressReporter):
+    """
+    Version of MultiStageProgressReporter intended for use with
+    standalone processes (such as preparing the runqueue)
+    """
+    def __init__(self, d, processname, stage_weights, debug=False):
+        self._processname = processname
+        MultiStageProgressReporter.__init__(self, d, stage_weights, debug)
+
+    def start(self):
+        bb.event.fire(bb.event.ProcessStarted(self._processname, 100), self._data)
+
+    def _fire_progress(self, taskprogress):
+        if taskprogress == 0:
+            self.start()
+            return
+        bb.event.fire(bb.event.ProcessProgress(self._processname, taskprogress), self._data)
+
+    def finish(self):
+        MultiStageProgressReporter.finish(self)
+        bb.event.fire(bb.event.ProcessFinished(self._processname), self._data)
+
+class DummyMultiStageProcessProgressReporter(MultiStageProgressReporter):
+    """
+    MultiStageProcessProgressReporter that takes the calls and does nothing
+    with them (to avoid a bunch of "if progress_reporter:" checks)
+    """
+    def __init__(self):
+        MultiStageProcessProgressReporter.__init__(self, "", None, [])
+
+    def _fire_progress(self, taskprogress, rate=None):
+        pass
+
+    def start(self):
+        pass
+
+    def next_stage(self, stage_total=None):
+        pass
+
+    def update(self, stage_progress):
+        pass
+
+    def finish(self):
+        pass

bitbake/lib/bb/runqueue.py

@@ -240,6 +240,8 @@ class RunQueueData:
 
         self.stampwhitelist = cfgData.getVar("BB_STAMP_WHITELIST", True) or ""
         self.multi_provider_whitelist = (cfgData.getVar("MULTI_PROVIDER_WHITELIST", True) or "").split()
+        self.setscenewhitelist = get_setscene_enforce_whitelist(cfgData)
+        self.init_progress_reporter = bb.progress.DummyMultiStageProcessProgressReporter()
 
         self.reset()
 
@@ -431,7 +433,8 @@ class RunQueueData:
             # Nothing to do
             return 0
 
-        logger.info("Preparing RunQueue")
+        self.init_progress_reporter.start()
+        self.init_progress_reporter.next_stage()
 
         # Step A - Work out a list of tasks to run
         #
@@ -561,8 +564,9 @@ class RunQueueData:
         # e.g. do_sometask[recrdeptask] = "do_someothertask"
         # (makes sure sometask runs after someothertask of all DEPENDS, RDEPENDS and intertask dependencies, recursively)
         # We need to do this separately since we need all of runtaskentries[*].depends to be complete before this is processed
+        self.init_progress_reporter.next_stage(len(recursivetasks))
         extradeps = {}
-        for tid in recursivetasks:
+        for taskcounter, tid in enumerate(recursivetasks):
             extradeps[tid] = set(self.runtaskentries[tid].depends)
 
             tasknames = recursivetasks[tid]
@@ -584,6 +588,7 @@ class RunQueueData:
             if tid in recursiveitasks:
                 for dep in recursiveitasks[tid]:
                     generate_recdeps(dep)
+            self.init_progress_reporter.update(taskcounter)
 
         # Remove circular references so that do_a[recrdeptask] = "do_a do_b" can work
         for tid in recursivetasks:
@@ -599,6 +604,8 @@ class RunQueueData:
                 logger.debug(2, "Task %s contains self reference!", tid)
                 self.runtaskentries[tid].depends.remove(tid)
 
+        self.init_progress_reporter.next_stage()
+
         # Step B - Mark all active tasks
         #
         # Start with the tasks we were asked to run and mark all dependencies
@@ -663,6 +670,8 @@ class RunQueueData:
             else:
                 mark_active(tid, 1)
 
+        self.init_progress_reporter.next_stage()
+
         # Step C - Prune all inactive tasks
         #
         # Once all active tasks are marked, prune the ones we don't need.
@@ -673,6 +682,8 @@ class RunQueueData:
                 del self.runtaskentries[tid]
                 delcount += 1
 
+        self.init_progress_reporter.next_stage()
+
         #
         # Step D - Sanity checks and computation
         #
@@ -688,11 +699,15 @@ class RunQueueData:
 
         logger.verbose("Assign Weightings")
 
+        self.init_progress_reporter.next_stage()
+
         # Generate a list of reverse dependencies to ease future calculations
         for tid in self.runtaskentries:
             for dep in self.runtaskentries[tid].depends:
                 self.runtaskentries[dep].revdeps.add(tid)
 
+        self.init_progress_reporter.next_stage()
+
         # Identify tasks at the end of dependency chains
         # Error on circular dependency loops (length two)
         endpoints = []
@@ -708,10 +723,14 @@ class RunQueueData:
 
         logger.verbose("Compute totals (have %s endpoint(s))", len(endpoints))
 
+        self.init_progress_reporter.next_stage()
+
         # Calculate task weights
         # Check of higher length circular dependencies
         self.runq_weight = self.calculate_task_weights(endpoints)
 
+        self.init_progress_reporter.next_stage()
+
         # Sanity Check - Check for multiple tasks building the same provider
         prov_list = {}
         seen_fn = []
@@ -803,6 +822,8 @@ class RunQueueData:
                 else:
                     logger.error(msg)
 
+        self.init_progress_reporter.next_stage()
+
         # Create a whitelist usable by the stamp checks
         stampfnwhitelist = []
         for entry in self.stampwhitelist.split():
@@ -812,6 +833,8 @@ class RunQueueData:
             stampfnwhitelist.append(fn)
         self.stampfnwhitelist = stampfnwhitelist
 
+        self.init_progress_reporter.next_stage()
+
         # Iterate over the task list looking for tasks with a 'setscene' function
         self.runq_setscene_tids = []
         if not self.cooker.configuration.nosetscene:
@@ -836,6 +859,8 @@ class RunQueueData:
                 logger.verbose("Invalidate task %s, %s", taskname, fn)
                 bb.parse.siggen.invalidate_task(taskname, self.dataCache, fn)
 
+        self.init_progress_reporter.next_stage()
+
         # Invalidate task if force mode active
         if self.cooker.configuration.force:
             for (fn, target) in self.target_pairs:
@@ -849,6 +874,8 @@ class RunQueueData:
                         st = "do_%s" % st
                     invalidate_task(fn, st, True)
 
+        self.init_progress_reporter.next_stage()
+
         # Create and print to the logs a virtual/xxxx -> PN (fn) table
         virtmap = taskData.get_providermap(prefix="virtual/")
         virtpnmap = {}
@@ -858,6 +885,8 @@ class RunQueueData:
         if hasattr(bb.parse.siggen, "tasks_resolved"):
             bb.parse.siggen.tasks_resolved(virtmap, virtpnmap, self.dataCache)
 
+        self.init_progress_reporter.next_stage()
+
         # Iterate over the task list and call into the siggen code
         dealtwith = set()
         todeal = set(self.runtaskentries)
@@ -1095,14 +1124,25 @@ class RunQueue:
 
         if self.state is runQueuePrepare:
             self.rqexe = RunQueueExecuteDummy(self)
+            # NOTE: if you add, remove or significantly refactor the stages of this
+            # process then you should recalculate the weightings here. This is quite
+            # easy to do - just change the next line temporarily to pass debug=True as
+            # the last parameter and you'll get a printout of the weightings as well
+            # as a map to the lines where next_stage() was called. Of course this isn't
+            # critical, but it helps to keep the progress reporting accurate.
+            self.rqdata.init_progress_reporter = bb.progress.MultiStageProcessProgressReporter(self.cooker.data,
+                                                            "Initialising tasks",
+                                                            [43, 967, 4, 3, 1, 5, 3, 7, 13, 1, 2, 1, 1, 246, 35, 1, 38, 1, 35, 2, 338, 204, 142, 3, 3, 37, 244])
             if self.rqdata.prepare() == 0:
                 self.state = runQueueComplete
             else:
                 self.state = runQueueSceneInit
+                self.rqdata.init_progress_reporter.next_stage()
 
                 # we are ready to run,  emit dependency info to any UI or class which
                 # needs it
                 depgraph = self.cooker.buildDependTree(self, self.rqdata.taskData)
+                self.rqdata.init_progress_reporter.next_stage()
                 bb.event.fire(bb.event.DepTreeGenerated(depgraph), self.cooker.data)
 
         if self.state is runQueueSceneInit:
@@ -1115,7 +1155,9 @@ class RunQueue:
                     self.write_diffscenetasks(invalidtasks)
                 self.state = runQueueComplete
             else:
+                self.rqdata.init_progress_reporter.next_stage()
                 self.start_worker()
+                self.rqdata.init_progress_reporter.next_stage()
                 self.rqexe = RunQueueExecuteScenequeue(self)
 
         if self.state in [runQueueSceneRun, runQueueRunning, runQueueCleanUp]:
@@ -1128,6 +1170,8 @@ class RunQueue:
             if self.cooker.configuration.setsceneonly:
                 self.state = runQueueComplete
             else:
+                # Just in case we didn't setscene
+                self.rqdata.init_progress_reporter.finish()
                 logger.info("Executing RunQueue Tasks")
                 self.rqexe = RunQueueExecuteTasks(self)
                 self.state = runQueueRunning
@@ -1175,8 +1219,8 @@ class RunQueue:
                 pass
             self.state = runQueueComplete
             raise
-        except:
-            logger.error("An uncaught exception occured in runqueue, please see the failure below:")
+        except Exception as err:
+            logger.exception("An uncaught exception occurred in runqueue")
             try:
                 self.teardown_workers()
             except:
@@ -1596,8 +1640,51 @@ class RunQueueExecuteTasks(RunQueueExecute):
         Run the tasks in a queue prepared by rqdata.prepare()
         """
 
-        self.rq.read_workers()
+        if self.rqdata.setscenewhitelist:
+            # Check tasks that are going to run against the whitelist
+            def check_norun_task(tid, showerror=False):
+                fn = fn_from_tid(tid)
+                taskname = taskname_from_tid(tid)
+                # Ignore covered tasks
+                if tid in self.rq.scenequeue_covered:
+                    return False
+                # Ignore stamped tasks
+                if self.rq.check_stamp_task(tid, taskname, cache=self.stampcache):
+                    return False
+                # Ignore noexec tasks
+                taskdep = self.rqdata.dataCache.task_deps[fn]
+                if 'noexec' in taskdep and taskname in taskdep['noexec']:
+                    return False
 
+                pn = self.rqdata.dataCache.pkg_fn[fn]
+                if not check_setscene_enforce_whitelist(pn, taskname, self.rqdata.setscenewhitelist):
+                    if showerror:
+                        if tid in self.rqdata.runq_setscene_tids:
+                            logger.error('Task %s.%s attempted to execute unexpectedly and should have been setscened' % (pn, taskname))
+                        else:
+                            logger.error('Task %s.%s attempted to execute unexpectedly' % (pn, taskname))
+                    return True
+                return False
+            # Look to see if any tasks that we think shouldn't run are going to
+            unexpected = False
+            for tid in self.rqdata.runtaskentries:
+                if check_norun_task(tid):
+                    unexpected = True
+                    break
+            if unexpected:
+                # Run through the tasks in the rough order they'd have executed and print errors
+                # (since the order can be useful - usually missing sstate for the last few tasks
+                # is the cause of the problem)
+                task = self.sched.next()
+                while task is not None:
+                    check_norun_task(task, showerror=True)
+                    self.task_skip(task, 'Setscene enforcement check')
+                    task = self.sched.next()
+
+                self.rq.state = runQueueCleanUp
+                return True
+
+        self.rq.read_workers()
 
         if self.stats.total == 0:
             # nothing to do
@@ -1725,6 +1812,8 @@ class RunQueueExecuteScenequeue(RunQueueExecute):
         # therefore aims to collapse the huge runqueue dependency tree into a smaller one
         # only containing the setscene functions.
 
+        self.rqdata.init_progress_reporter.next_stage()
+
         # First process the chains up to the first setscene task.
         endpoints = {}
         for tid in self.rqdata.runtaskentries:
@@ -1734,6 +1823,8 @@ class RunQueueExecuteScenequeue(RunQueueExecute):
                 #bb.warn("Added endpoint %s" % (tid))
                 endpoints[tid] = set()
 
+        self.rqdata.init_progress_reporter.next_stage()
+
         # Secondly process the chains between setscene tasks.
         for tid in self.rqdata.runq_setscene_tids:
             #bb.warn("Added endpoint 2 %s" % (tid))
@@ -1743,6 +1834,8 @@ class RunQueueExecuteScenequeue(RunQueueExecute):
                     #bb.warn("  Added endpoint 3 %s" % (dep))
                     endpoints[dep].add(tid)
 
+        self.rqdata.init_progress_reporter.next_stage()
+
         def process_endpoints(endpoints):
             newendpoints = {}
             for point, task in endpoints.items():
@@ -1767,6 +1860,8 @@ class RunQueueExecuteScenequeue(RunQueueExecute):
 
         process_endpoints(endpoints)
 
+        self.rqdata.init_progress_reporter.next_stage()
+
         # Build a list of setscene tasks which are "unskippable"
         # These are direct endpoints referenced by the build
         endpoints2 = {}
@@ -1803,7 +1898,9 @@ class RunQueueExecuteScenequeue(RunQueueExecute):
             if sq_revdeps_new2[tid]:
                 self.unskippable.append(tid)
 
-        for tid in self.rqdata.runtaskentries:
+        self.rqdata.init_progress_reporter.next_stage(len(self.rqdata.runtaskentries))
+
+        for taskcounter, tid in enumerate(self.rqdata.runtaskentries):
             if tid in self.rqdata.runq_setscene_tids:
                 deps = set()
                 for dep in sq_revdeps_new[tid]:
@@ -1811,6 +1908,9 @@ class RunQueueExecuteScenequeue(RunQueueExecute):
                 sq_revdeps_squash[tid] = deps
             elif len(sq_revdeps_new[tid]) != 0:
                 bb.msg.fatal("RunQueue", "Something went badly wrong during scenequeue generation, aborting. Please report this problem.")
+            self.rqdata.init_progress_reporter.update(taskcounter)
+
+        self.rqdata.init_progress_reporter.next_stage()
 
         # Resolve setscene inter-task dependencies
         # e.g. do_sometask_setscene[depends] = "targetname:do_someothertask_setscene"
@@ -1838,10 +1938,14 @@ class RunQueueExecuteScenequeue(RunQueueExecute):
                     # Have to zero this to avoid circular dependencies
                     sq_revdeps_squash[deptid] = set()
 
+        self.rqdata.init_progress_reporter.next_stage()
+
         for task in self.sq_harddeps:
              for dep in self.sq_harddeps[task]:
                  sq_revdeps_squash[dep].add(task)
 
+        self.rqdata.init_progress_reporter.next_stage()
+
         #for tid in sq_revdeps_squash:
         #    for dep in sq_revdeps_squash[tid]:
         #        data = data + "\n   %s" % dep
@@ -1857,6 +1961,8 @@ class RunQueueExecuteScenequeue(RunQueueExecute):
             for dep in self.sq_revdeps[tid]:
                 self.sq_deps[dep].add(tid)
 
+        self.rqdata.init_progress_reporter.next_stage()
+
         for tid in self.sq_revdeps:
             if len(self.sq_revdeps[tid]) == 0:
                 self.runq_buildable.add(tid)
@@ -1912,6 +2018,8 @@ class RunQueueExecuteScenequeue(RunQueueExecute):
                     logger.debug(2, 'No package found, so skipping setscene task %s', tid)
                     self.outrightfail.append(tid)
 
+        self.rqdata.init_progress_reporter.finish()
+
         logger.info('Executing SetScene Tasks')
 
         self.rq.state = runQueueSceneRun
@@ -1940,6 +2048,16 @@ class RunQueueExecuteScenequeue(RunQueueExecute):
         self.scenequeue_covered.add(task)
         self.scenequeue_updatecounters(task)
 
+    def check_taskfail(self, task):
+        if self.rqdata.setscenewhitelist:
+            realtask = task.split('_setscene')[0]
+            fn = fn_from_tid(realtask)
+            taskname = taskname_from_tid(realtask)
+            pn = self.rqdata.dataCache.pkg_fn[fn]
+            if not check_setscene_enforce_whitelist(pn, taskname, self.rqdata.setscenewhitelist):
+                logger.error('Task %s.%s failed' % (pn, taskname + "_setscene"))
+                self.rq.state = runQueueCleanUp
+
     def task_complete(self, task):
         self.stats.taskCompleted()
         bb.event.fire(sceneQueueTaskCompleted(task, self.stats, self.rq), self.cfgData)
@@ -1950,6 +2068,7 @@ class RunQueueExecuteScenequeue(RunQueueExecute):
         bb.event.fire(sceneQueueTaskFailed(task, self.stats, result, self), self.cfgData)
         self.scenequeue_notcovered.add(task)
         self.scenequeue_updatecounters(task, True)
+        self.check_taskfail(task)
 
     def task_failoutright(self, task):
         self.runq_running.add(task)
@@ -2231,3 +2350,27 @@ class runQueuePipe():
         if len(self.queue) > 0:
             print("Warning, worker left partial message: %s" % self.queue)
         self.input.close()
+
+def get_setscene_enforce_whitelist(d):
+    if d.getVar('BB_SETSCENE_ENFORCE', True) != '1':
+        return None
+    whitelist = (d.getVar("BB_SETSCENE_ENFORCE_WHITELIST", True) or "").split()
+    outlist = []
+    for item in whitelist[:]:
+        if item.startswith('%:'):
+            for target in sys.argv[1:]:
+                if not target.startswith('-'):
+                    outlist.append(target.split(':')[0] + ':' + item.split(':')[1])
+        else:
+            outlist.append(item)
+    return outlist
+
+def check_setscene_enforce_whitelist(pn, taskname, whitelist):
+    import fnmatch
+    if whitelist:
+        item = '%s:%s' % (pn, taskname)
+        for whitelist_item in whitelist:
+            if fnmatch.fnmatch(item, whitelist_item):
+                return True
+        return False
+    return True

bitbake/lib/bb/server/xmlrpc.py

@@ -85,6 +85,23 @@ def _create_server(host, port, timeout = 60):
     s = xmlrpc.client.ServerProxy("http://%s:%d/" % (host, port), transport=t, allow_none=True, use_builtin_types=True)
     return s, t
 
+def check_connection(remote, timeout):
+    try:
+        host, port = remote.split(":")
+        port = int(port)
+    except Exception as e:
+        bb.warn("Failed to read remote definition (%s)" % str(e))
+        raise e
+
+    server, _transport = _create_server(host, port, timeout)
+    try:
+        ret, err =  server.runCommand(['getVariable', 'TOPDIR'])
+        if err or not ret:
+            return False
+    except ConnectionError:
+        return False
+    return True
+
 class BitBakeServerCommands():
 
     def __init__(self, server):
@@ -188,7 +205,7 @@ class XMLRPCServer(SimpleXMLRPCServer, BaseImplServer):
     # remove this when you're done with debugging
     # allow_reuse_address = True
 
-    def __init__(self, interface, single_use=False):
+    def __init__(self, interface, single_use=False, idle_timeout=0):
         """
         Constructor
         """
@@ -206,6 +223,10 @@ class XMLRPCServer(SimpleXMLRPCServer, BaseImplServer):
         self.commands = BitBakeServerCommands(self)
         self.autoregister_all_functions(self.commands, "")
         self.interface = interface
+        self.time = time.time()
+        self.idle_timeout = idle_timeout
+        if idle_timeout:
+            self.register_idle_function(self.handle_idle_timeout, self)
 
     def addcooker(self, cooker):
         BaseImplServer.addcooker(self, cooker)
@@ -221,6 +242,12 @@ class XMLRPCServer(SimpleXMLRPCServer, BaseImplServer):
             if name.startswith(prefix):
                 self.register_function(method, name[len(prefix):])
 
+    def handle_idle_timeout(self, server, data, abort):
+        if not abort:
+            if time.time() - server.time > server.idle_timeout:
+                server.quit = True
+                print("Server idle timeout expired")
+        return []
 
     def serve_forever(self):
         # Start the actual XMLRPC server
@@ -263,6 +290,8 @@ class XMLRPCServer(SimpleXMLRPCServer, BaseImplServer):
             try:
                 fd_sets = select.select(fds, [], [], socktimeout)
                 if fd_sets[0] and self in fd_sets[0]:
+                    if self.idle_timeout:
+                        self.time = time.time()
                     self._handle_request_noblock()
             except IOError:
                 # we ignore interrupted calls
@@ -334,9 +363,10 @@ class BitBakeXMLRPCServerConnection(BitBakeBaseServerConnection):
             pass
 
 class BitBakeServer(BitBakeBaseServer):
-    def initServer(self, interface = ("localhost", 0), single_use = False):
+    def initServer(self, interface = ("localhost", 0),
+                   single_use = False, idle_timeout=0):
         self.interface = interface
-        self.serverImpl = XMLRPCServer(interface, single_use)
+        self.serverImpl = XMLRPCServer(interface, single_use, idle_timeout)
 
     def detach(self):
         daemonize.createDaemon(self.serverImpl.serve_forever, "bitbake-cookerdaemon.log")

bitbake/lib/bb/ui/buildinfohelper.py

@@ -37,7 +37,7 @@ os.environ["DJANGO_SETTINGS_MODULE"] =\
 django.setup()
 
 from orm.models import Build, Task, Recipe, Layer_Version, Layer, Target, LogMessage, HelpText
-from orm.models import Target_Image_File, BuildArtifact
+from orm.models import Target_Image_File, TargetKernelFile, TargetSDKFile
 from orm.models import Variable, VariableHistory
 from orm.models import Package, Package_File, Target_Installed_Package, Target_File
 from orm.models import Task_Dependency, Package_Dependency
@@ -121,11 +121,27 @@ class ORMWrapper(object):
 
         return vars(self)[dictname][key]
 
+    def get_similar_target_with_image_files(self, target):
+        """
+        Get a Target object "similar" to target; i.e. with the same target
+        name ('core-image-minimal' etc.) and machine.
+        """
+        return target.get_similar_target_with_image_files()
+
+    def get_similar_target_with_sdk_files(self, target):
+        return target.get_similar_target_with_sdk_files()
+
+    def clone_image_artifacts(self, target_from, target_to):
+        target_to.clone_image_artifacts_from(target_from)
+
+    def clone_sdk_artifacts(self, target_from, target_to):
+        target_to.clone_sdk_artifacts_from(target_from)
+
     def _timestamp_to_datetime(self, secs):
         """
         Convert timestamp in seconds to Python datetime
         """
-        return datetime(1970, 1, 1) + timedelta(seconds=secs)
+        return timezone.make_aware(datetime(1970, 1, 1) + timedelta(seconds=secs))
 
     # pylint: disable=no-self-use
     # we disable detection of no self use in functions because the methods actually work on the object
@@ -194,6 +210,10 @@ class ORMWrapper(object):
 
     @staticmethod
     def get_or_create_targets(target_info):
+        """
+        NB get_or_create() is used here because for Toaster-triggered builds,
+        we already created the targets when the build was triggered.
+        """
         result = []
         for target in target_info['targets']:
             task = ''
@@ -203,13 +223,10 @@ class ORMWrapper(object):
                 task = task[3:]
             if task == 'build':
                 task = ''
-            obj, created = Target.objects.get_or_create(build=target_info['build'],
-                                                        target=target)
-            if created:
-                obj.is_image = False
-                if task:
-                    obj.task = task
-                obj.save()
+
+            obj, _ = Target.objects.get_or_create(build=target_info['build'],
+                                                  target=target,
+                                                  task=task)
             result.append(obj)
         return result
 
@@ -239,6 +256,10 @@ class ORMWrapper(object):
         target.license_manifest_path = license_manifest_path
         target.save()
 
+    def update_target_set_package_manifest(self, target, package_manifest_path):
+        target.package_manifest_path = package_manifest_path
+        target.save()
+
     def update_task_object(self, build, task_name, recipe_name, task_stats):
         """
         Find the task for build which matches the recipe and task name
@@ -674,20 +695,23 @@ class ORMWrapper(object):
             logger.warning("buildinfohelper: target_package_info could not identify recipes: \n%s", errormsg)
 
     def save_target_image_file_information(self, target_obj, file_name, file_size):
-        Target_Image_File.objects.create( target = target_obj,
-                            file_name = file_name,
-                            file_size = file_size)
+        Target_Image_File.objects.create(target=target_obj,
+            file_name=file_name, file_size=file_size)
 
-    def save_artifact_information(self, build_obj, file_name, file_size):
-        # we skip the image files from other builds
-        if Target_Image_File.objects.filter(file_name = file_name).count() > 0:
-            return
+    def save_target_kernel_file(self, target_obj, file_name, file_size):
+        """
+        Save kernel file (bzImage, modules*) information for a Target target_obj.
+        """
+        TargetKernelFile.objects.create(target=target_obj,
+            file_name=file_name, file_size=file_size)
 
-        # do not update artifacts found in other builds
-        if BuildArtifact.objects.filter(file_name = file_name).count() > 0:
-            return
-
-        BuildArtifact.objects.create(build = build_obj, file_name = file_name, file_size = file_size)
+    def save_target_sdk_file(self, target_obj, file_name, file_size):
+        """
+        Save SDK artifacts to the database, associating them with a
+        Target object.
+        """
+        TargetSDKFile.objects.create(target=target_obj, file_name=file_name,
+            file_size=file_size)
 
     def create_logmessage(self, log_information):
         assert 'build' in log_information
@@ -855,6 +879,11 @@ class BuildInfoHelper(object):
         Keeps in memory all data that needs matching before writing it to the database
     """
 
+    # tasks which produce image files; note we include '', as we set
+    # the task for a target to '' (i.e. 'build') if no target is
+    # explicitly defined
+    IMAGE_GENERATING_TASKS = ['', 'build', 'image', 'populate_sdk_ext']
+
     # pylint: disable=protected-access
     # the code will look into the protected variables of the event; no easy way around this
     # pylint: disable=bad-continuation
@@ -1061,36 +1090,10 @@ class BuildInfoHelper(object):
 
         return self.brbe
 
-
-    def update_target_image_file(self, event):
-        evdata = BuildInfoHelper._get_data_from_event(event)
-
-        for t in self.internal_state['targets']:
-            if t.is_image == True:
-                output_files = list(evdata.keys())
-                for output in output_files:
-                    if t.target in output and 'rootfs' in output and not output.endswith(".manifest"):
-                        self.orm_wrapper.save_target_image_file_information(t, output, evdata[output])
-
-    def update_artifact_image_file(self, event):
-        evdata = BuildInfoHelper._get_data_from_event(event)
-        for artifact_path in evdata.keys():
-            self.orm_wrapper.save_artifact_information(self.internal_state['build'], artifact_path, evdata[artifact_path])
-
     def update_build_information(self, event, errors, warnings, taskfailures):
         if 'build' in self.internal_state:
             self.orm_wrapper.update_build_object(self.internal_state['build'], errors, warnings, taskfailures)
 
-
-    def store_license_manifest_path(self, event):
-        deploy_dir = BuildInfoHelper._get_data_from_event(event)['deploy_dir']
-        image_name = BuildInfoHelper._get_data_from_event(event)['image_name']
-        path = deploy_dir + "/licenses/" + image_name + "/license.manifest"
-        for target in self.internal_state['targets']:
-            if target.target in image_name:
-                self.orm_wrapper.update_target_set_license_manifest(target, path)
-
-
     def store_started_task(self, event):
         assert isinstance(event, (bb.runqueue.sceneQueueTaskStarted, bb.runqueue.runQueueTaskStarted, bb.runqueue.runQueueTaskSkipped))
         assert 'taskfile' in vars(event)
@@ -1241,11 +1244,16 @@ class BuildInfoHelper(object):
                     logger.warning("KeyError in save_target_package_information"
                                    "%s ", e)
 
-                try:
-                    self.orm_wrapper.save_target_file_information(self.internal_state['build'], target, filedata)
-                except KeyError as e:
-                    logger.warning("KeyError in save_target_file_information"
-                                   "%s ", e)
+                # only try to find files in the image if the task for this
+                # target is one which produces image files; otherwise, the old
+                # list of files in the files-in-image.txt file will be
+                # appended to the target even if it didn't produce any images
+                if target.task in BuildInfoHelper.IMAGE_GENERATING_TASKS:
+                    try:
+                        self.orm_wrapper.save_target_file_information(self.internal_state['build'], target, filedata)
+                    except KeyError as e:
+                        logger.warning("KeyError in save_target_file_information"
+                                       "%s ", e)
 
 
 
@@ -1506,6 +1514,304 @@ class BuildInfoHelper(object):
 
         self.orm_wrapper.create_logmessage(log_information)
 
+    def _get_filenames_from_image_license(self, image_license_manifest_path):
+        """
+        Find the FILES line in the image_license.manifest file,
+        which has the basenames of the bzImage and modules files
+        in this format:
+        FILES: bzImage--4.4.11+git0+3a5f494784_53e84104c5-r0-qemux86-20160603165040.bin modules--4.4.11+git0+3a5f494784_53e84104c5-r0-qemux86-20160603165040.tgz
+        """
+        files = []
+        with open(image_license_manifest_path) as image_license:
+            for line in image_license:
+                if line.startswith('FILES'):
+                    files_str = line.split(':')[1].strip()
+                    files_str = re.sub(r' {2,}', ' ', files_str)
+                    files = files_str.split(' ')
+        return files
+
+    def _endswith(self, str_to_test, endings):
+        """
+        Returns True if str ends with one of the strings in the list
+        endings, False otherwise
+        """
+        endswith = False
+        for ending in endings:
+            if str_to_test.endswith(ending):
+                endswith = True
+                break
+        return endswith
+
+    def _get_image_files(self, deploy_dir_image, image_name, image_file_extensions):
+        """
+        Find files in deploy_dir_image whose basename starts with the
+        string image_name and ends with one of the strings in
+        image_file_extensions.
+
+        Returns a list of file dictionaries like
+
+        [
+            {
+                'path': '/path/to/image/file',
+                'size': <file size in bytes>
+            }
+        ]
+        """
+        image_files = []
+
+        for dirpath, _, filenames in os.walk(deploy_dir_image):
+            for filename in filenames:
+                if filename.startswith(image_name) and \
+                self._endswith(filename, image_file_extensions):
+                    image_file_path = os.path.join(dirpath, filename)
+                    image_file_size = os.stat(image_file_path).st_size
+
+                    image_files.append({
+                        'path': image_file_path,
+                        'size': image_file_size
+                    })
+
+        return image_files
+
+    def scan_image_artifacts(self):
+        """
+        Scan for built image artifacts in DEPLOY_DIR_IMAGE and associate them
+        with a Target object in self.internal_state['targets'].
+
+        We have two situations to handle:
+
+        1. This is the first time a target + machine has been built, so
+        add files from the DEPLOY_DIR_IMAGE to the target.
+
+        OR
+
+        2. There are no new files for the target (they were already produced by
+        a previous build), so copy them from the most recent previous build with
+        the same target, task and machine.
+        """
+        deploy_dir_image = \
+            self.server.runCommand(['getVariable', 'DEPLOY_DIR_IMAGE'])[0]
+
+        # if there's no DEPLOY_DIR_IMAGE, there aren't going to be
+        # any image artifacts, so we can return immediately
+        if not deploy_dir_image:
+            return
+
+        buildname = self.server.runCommand(['getVariable', 'BUILDNAME'])[0]
+        machine = self.server.runCommand(['getVariable', 'MACHINE'])[0]
+        image_name = self.server.runCommand(['getVariable', 'IMAGE_NAME'])[0]
+
+        # location of the manifest files for this build;
+        # note that this file is only produced if an image is produced
+        license_directory = \
+            self.server.runCommand(['getVariable', 'LICENSE_DIRECTORY'])[0]
+
+        # file name extensions for image files
+        image_file_extensions_unique = {}
+        image_fstypes = self.server.runCommand(
+            ['getVariable', 'IMAGE_FSTYPES'])[0]
+        if image_fstypes != None:
+            image_types_str = image_fstypes.strip()
+            image_file_extensions = re.sub(r' {2,}', ' ', image_types_str)
+            image_file_extensions_unique = set(image_file_extensions.split(' '))
+
+        targets = self.internal_state['targets']
+
+        # filter out anything which isn't an image target
+        image_targets = [target for target in targets if target.is_image]
+
+        for image_target in image_targets:
+            # this is set to True if we find at least one file relating to
+            # this target; if this remains False after the scan, we copy the
+            # files from the most-recent Target with the same target + machine
+            # onto this Target instead
+            has_files = False
+
+            # we construct this because by the time we reach
+            # BuildCompleted, this has reset to
+            # 'defaultpkgname-<MACHINE>-<BUILDNAME>';
+            # we need to change it to
+            # <TARGET>-<MACHINE>-<BUILDNAME>
+            real_image_name = re.sub(r'^defaultpkgname', image_target.target,
+                image_name)
+
+            image_license_manifest_path = os.path.join(
+                license_directory,
+                real_image_name,
+                'image_license.manifest')
+
+            image_package_manifest_path = os.path.join(
+                license_directory,
+                real_image_name,
+                'image_license.manifest')
+
+            # if image_license.manifest exists, we can read the names of bzImage
+            # and modules files for this build from it, then look for them
+            # in the DEPLOY_DIR_IMAGE; note that this file is only produced
+            # if an image file was produced
+            if os.path.isfile(image_license_manifest_path):
+                has_files = True
+
+                basenames = self._get_filenames_from_image_license(
+                    image_license_manifest_path)
+
+                for basename in basenames:
+                    artifact_path = os.path.join(deploy_dir_image, basename)
+                    artifact_size = os.stat(artifact_path).st_size
+
+                    # note that the artifact will only be saved against this
+                    # build if it hasn't been already
+                    self.orm_wrapper.save_target_kernel_file(image_target,
+                        artifact_path, artifact_size)
+
+                # store the license manifest path on the target
+                # (this file is also created any time an image file is created)
+                license_manifest_path = os.path.join(license_directory,
+                    real_image_name, 'license.manifest')
+
+                self.orm_wrapper.update_target_set_license_manifest(
+                    image_target, license_manifest_path)
+
+                # store the package manifest path on the target (this file
+                # is created any time an image file is created)
+                package_manifest_path = os.path.join(deploy_dir_image,
+                    real_image_name + '.rootfs.manifest')
+
+                if os.path.exists(package_manifest_path):
+                    self.orm_wrapper.update_target_set_package_manifest(
+                        image_target, package_manifest_path)
+
+            # scan the directory for image files relating to this build
+            # (via real_image_name); note that we don't have to set
+            # has_files = True, as searching for the license manifest file
+            # will already have set it to true if at least one image file was
+            # produced; note that the real_image_name includes BUILDNAME, which
+            # in turn includes a timestamp; so if no files were produced for
+            # this timestamp (i.e. the build reused existing image files already
+            # in the directory), no files will be recorded against this target
+            image_files = self._get_image_files(deploy_dir_image,
+                real_image_name, image_file_extensions_unique)
+
+            for image_file in image_files:
+                self.orm_wrapper.save_target_image_file_information(
+                    image_target, image_file['path'], image_file['size'])
+
+            if not has_files:
+                # copy image files and build artifacts from the
+                # most-recently-built Target with the
+                # same target + machine as this Target; also copy the license
+                # manifest path, as that is not treated as an artifact and needs
+                # to be set separately
+                similar_target = \
+                    self.orm_wrapper.get_similar_target_with_image_files(
+                        image_target)
+
+                if similar_target:
+                    logger.info('image artifacts for target %s cloned from ' \
+                        'target %s' % (image_target.pk, similar_target.pk))
+                    self.orm_wrapper.clone_image_artifacts(similar_target,
+                        image_target)
+
+    def _get_sdk_targets(self):
+        """
+        Return targets which could generate SDK artifacts, i.e.
+        "do_populate_sdk" and "do_populate_sdk_ext".
+        """
+        return [target for target in self.internal_state['targets'] \
+            if target.task in ['populate_sdk', 'populate_sdk_ext']]
+
+    def scan_sdk_artifacts(self, event):
+        """
+        Note that we have to intercept an SDKArtifactInfo event from
+        toaster.bbclass (via toasterui) to get hold of the SDK variables we
+        need to be able to scan for files accurately: this is because
+        variables like TOOLCHAIN_OUTPUTNAME have reset to None by the time
+        BuildCompleted is fired by bitbake, so we have to get those values
+        while the build is still in progress.
+
+        For populate_sdk_ext, this runs twice, with two different
+        TOOLCHAIN_OUTPUTNAME settings, each of which will capture some of the
+        files in the SDK output directory.
+        """
+        sdk_vars = BuildInfoHelper._get_data_from_event(event)
+        toolchain_outputname = sdk_vars['TOOLCHAIN_OUTPUTNAME']
+
+        # targets which might have created SDK artifacts
+        sdk_targets = self._get_sdk_targets()
+
+        # location of SDK artifacts
+        tmpdir = self.server.runCommand(['getVariable', 'TMPDIR'])[0]
+        sdk_dir = os.path.join(tmpdir, 'deploy', 'sdk')
+
+        # all files in the SDK directory
+        artifacts = []
+        for dir_path, _, filenames in os.walk(sdk_dir):
+            for filename in filenames:
+                full_path = os.path.join(dir_path, filename)
+                if not os.path.islink(full_path):
+                    artifacts.append(full_path)
+
+        for sdk_target in sdk_targets:
+            # find files in the SDK directory which haven't already been
+            # recorded against a Target and whose basename matches
+            # TOOLCHAIN_OUTPUTNAME
+            for artifact_path in artifacts:
+                basename = os.path.basename(artifact_path)
+
+                toolchain_match = basename.startswith(toolchain_outputname)
+
+                # files which match the name of the target which produced them;
+                # for example,
+                # poky-glibc-x86_64-core-image-sato-i586-toolchain-ext-2.1+snapshot.sh
+                target_match = re.search(sdk_target.target, basename)
+
+                # targets which produce "*-nativesdk-*" files
+                is_ext_sdk_target = sdk_target.task in \
+                    ['do_populate_sdk_ext', 'populate_sdk_ext']
+
+                # SDK files which don't match the target name, i.e.
+                # x86_64-nativesdk-libc.*
+                # poky-glibc-x86_64-buildtools-tarball-i586-buildtools-nativesdk-standalone-2.1+snapshot*
+                is_ext_sdk_file = re.search('-nativesdk-', basename)
+
+                file_from_target = (toolchain_match and target_match) or \
+                    (is_ext_sdk_target and is_ext_sdk_file)
+
+                if file_from_target:
+                    # don't record the file if it's already been added to this
+                    # target
+                    matching_files = TargetSDKFile.objects.filter(
+                        target=sdk_target, file_name=artifact_path)
+
+                    if matching_files.count() == 0:
+                        artifact_size = os.stat(artifact_path).st_size
+
+                        self.orm_wrapper.save_target_sdk_file(
+                            sdk_target, artifact_path, artifact_size)
+
+    def clone_required_sdk_artifacts(self):
+        """
+        If an SDK target doesn't have any SDK artifacts, this means that
+        the postfuncs of populate_sdk or populate_sdk_ext didn't fire, which
+        in turn means that the targets of this build didn't generate any new
+        artifacts.
+
+        In this case, clone SDK artifacts for targets in the current build
+        from existing targets for this build.
+        """
+        sdk_targets = self._get_sdk_targets()
+        for sdk_target in sdk_targets:
+            # only clone for SDK targets which have no TargetSDKFiles yet
+            if sdk_target.targetsdkfile_set.all().count() == 0:
+                similar_target = \
+                    self.orm_wrapper.get_similar_target_with_sdk_files(
+                        sdk_target)
+                if similar_target:
+                    logger.info('SDK artifacts for target %s cloned from ' \
+                        'target %s' % (sdk_target.pk, similar_target.pk))
+                    self.orm_wrapper.clone_sdk_artifacts(similar_target,
+                        sdk_target)
+
     def close(self, errorcode):
         if self.brbe is not None:
             self._store_build_done(errorcode)

bitbake/lib/bb/ui/knotty.py

@@ -40,10 +40,13 @@ logger = logging.getLogger("BitBake")
 interactive = sys.stdout.isatty()
 
 class BBProgress(progressbar.ProgressBar):
-    def __init__(self, msg, maxval):
+    def __init__(self, msg, maxval, widgets=None):
         self.msg = msg
-        widgets = [progressbar.Percentage(), ' ', progressbar.Bar(), ' ',
-           progressbar.ETA()]
+        self.extrapos = -1
+        if not widgets:
+            widgets = [progressbar.Percentage(), ' ', progressbar.Bar(), ' ',
+            progressbar.ETA()]
+            self.extrapos = 4
 
         try:
             self._resize_default = signal.getsignal(signal.SIGWINCH)
@@ -51,23 +54,44 @@ class BBProgress(progressbar.ProgressBar):
             self._resize_default = None
         progressbar.ProgressBar.__init__(self, maxval, [self.msg + ": "] + widgets, fd=sys.stdout)
 
-    def _handle_resize(self, signum, frame):
+    def _handle_resize(self, signum=None, frame=None):
         progressbar.ProgressBar._handle_resize(self, signum, frame)
         if self._resize_default:
             self._resize_default(signum, frame)
+
     def finish(self):
         progressbar.ProgressBar.finish(self)
         if self._resize_default:
             signal.signal(signal.SIGWINCH, self._resize_default)
 
+    def setmessage(self, msg):
+        self.msg = msg
+        self.widgets[0] = msg
+
+    def setextra(self, extra):
+        if extra:
+            extrastr = str(extra)
+            if extrastr[0] != ' ':
+                extrastr = ' ' + extrastr
+            if extrastr[-1] != ' ':
+                extrastr += ' '
+        else:
+            extrastr = ' '
+        self.widgets[self.extrapos] = extrastr
+
+    def _need_update(self):
+        # We always want the bar to print when update() is called
+        return True
+
 class NonInteractiveProgress(object):
     fobj = sys.stdout
 
     def __init__(self, msg, maxval):
         self.msg = msg
         self.maxval = maxval
+        self.finished = False
 
-    def start(self):
+    def start(self, update=True):
         self.fobj.write("%s..." % self.msg)
         self.fobj.flush()
         return self
@@ -76,8 +100,11 @@ class NonInteractiveProgress(object):
         pass
 
     def finish(self):
+        if self.finished:
+            return
         self.fobj.write("done.\n")
         self.fobj.flush()
+        self.finished = True
 
 def new_progress(msg, maxval):
     if interactive:
@@ -134,7 +161,7 @@ class TerminalFilter(object):
                 cr = (25, 80)
         return cr
 
-    def __init__(self, main, helper, console, errconsole, format):
+    def __init__(self, main, helper, console, errconsole, format, quiet):
         self.main = main
         self.helper = helper
         self.cuu = None
@@ -142,6 +169,7 @@ class TerminalFilter(object):
         self.interactive = sys.stdout.isatty()
         self.footer_present = False
         self.lastpids = []
+        self.quiet = quiet
 
         if not self.interactive:
             return
@@ -181,6 +209,8 @@ class TerminalFilter(object):
         console.addFilter(InteractConsoleLogFilter(self, format))
         errconsole.addFilter(InteractConsoleLogFilter(self, format))
 
+        self.main_progress = None
+
     def clearFooter(self):
         if self.footer_present:
             lines = self.footer_present
@@ -195,28 +225,69 @@ class TerminalFilter(object):
         activetasks = self.helper.running_tasks
         failedtasks = self.helper.failed_tasks
         runningpids = self.helper.running_pids
-        if self.footer_present and (self.lastcount == self.helper.tasknumber_current) and (self.lastpids == runningpids):
+        if self.footer_present and not self.helper.needUpdate:
             return
+        self.helper.needUpdate = False
         if self.footer_present:
             self.clearFooter()
         if (not self.helper.tasknumber_total or self.helper.tasknumber_current == self.helper.tasknumber_total) and not len(activetasks):
             return
         tasks = []
         for t in runningpids:
-            tasks.append("%s (pid %s)" % (activetasks[t]["title"], t))
+            progress = activetasks[t].get("progress", None)
+            if progress is not None:
+                pbar = activetasks[t].get("progressbar", None)
+                rate = activetasks[t].get("rate", None)
+                start_time = activetasks[t].get("starttime", None)
+                if not pbar or pbar.bouncing != (progress < 0):
+                    if progress < 0:
+                        pbar = BBProgress("0: %s (pid %s) " % (activetasks[t]["title"], t), 100, widgets=[progressbar.BouncingSlider()])
+                        pbar.bouncing = True
+                    else:
+                        pbar = BBProgress("0: %s (pid %s) " % (activetasks[t]["title"], t), 100)
+                        pbar.bouncing = False
+                    activetasks[t]["progressbar"] = pbar
+                tasks.append((pbar, progress, rate, start_time))
+            else:
+                tasks.append("%s (pid %s)" % (activetasks[t]["title"], t))
 
         if self.main.shutdown:
             content = "Waiting for %s running tasks to finish:" % len(activetasks)
-        elif not len(activetasks):
-            content = "No currently running tasks (%s of %s)" % (self.helper.tasknumber_current, self.helper.tasknumber_total)
-        else:
-            content = "Currently %s running tasks (%s of %s):" % (len(activetasks), self.helper.tasknumber_current, self.helper.tasknumber_total)
-        print(content)
-        lines = 1 + int(len(content) / (self.columns + 1))
-        for tasknum, task in enumerate(tasks[:(self.rows - 2)]):
-            content = "%s: %s" % (tasknum, task)
             print(content)
-            lines = lines + 1 + int(len(content) / (self.columns + 1))
+        else:
+            if not len(activetasks):
+                content = "No currently running tasks (%s of %s)" % (self.helper.tasknumber_current, self.helper.tasknumber_total)
+            else:
+                content = "Currently %2s running tasks (%s of %s)" % (len(activetasks), self.helper.tasknumber_current, self.helper.tasknumber_total)
+            maxtask = self.helper.tasknumber_total + 1
+            if not self.main_progress or self.main_progress.maxval != maxtask:
+                widgets = [' ', progressbar.Percentage(), ' ', progressbar.Bar()]
+                self.main_progress = BBProgress("Running tasks", maxtask, widgets=widgets)
+                self.main_progress.start(False)
+            self.main_progress.setmessage(content)
+            self.main_progress.update(self.helper.tasknumber_current)
+            print('')
+        lines = 1 + int(len(content) / (self.columns + 1))
+        if not self.quiet:
+            for tasknum, task in enumerate(tasks[:(self.rows - 2)]):
+                if isinstance(task, tuple):
+                    pbar, progress, rate, start_time = task
+                    if not pbar.start_time:
+                        pbar.start(False)
+                        if start_time:
+                            pbar.start_time = start_time
+                    pbar.setmessage('%s:%s' % (tasknum, pbar.msg.split(':', 1)[1]))
+                    if progress > -1:
+                        pbar.setextra(rate)
+                        output = pbar.update(progress)
+                    else:
+                        output = pbar.update(1)
+                    if not output or (len(output) <= pbar.term_width):
+                        print('')
+                else:
+                    content = "%s: %s" % (tasknum, task)
+                    print(content)
+                lines = lines + 1 + int(len(content) / (self.columns + 1))
         self.footer_present = lines
         self.lastpids = runningpids[:]
         self.lastcount = self.helper.tasknumber_current
@@ -249,7 +320,8 @@ _evt_list = [ "bb.runqueue.runQueueExitWait", "bb.event.LogExecTTY", "logging.Lo
               "bb.command.CommandExit", "bb.command.CommandCompleted",  "bb.cooker.CookerExit",
               "bb.event.MultipleProviders", "bb.event.NoProvider", "bb.runqueue.sceneQueueTaskStarted",
               "bb.runqueue.runQueueTaskStarted", "bb.runqueue.runQueueTaskFailed", "bb.runqueue.sceneQueueTaskFailed",
-              "bb.event.BuildBase", "bb.build.TaskStarted", "bb.build.TaskSucceeded", "bb.build.TaskFailedSilent"]
+              "bb.event.BuildBase", "bb.build.TaskStarted", "bb.build.TaskSucceeded", "bb.build.TaskFailedSilent",
+              "bb.build.TaskProgress", "bb.event.ProcessStarted", "bb.event.ProcessProgress", "bb.event.ProcessFinished"]
 
 def main(server, eventHandler, params, tf = TerminalFilter):
 
@@ -266,7 +338,10 @@ def main(server, eventHandler, params, tf = TerminalFilter):
     errconsole = logging.StreamHandler(sys.stderr)
     format_str = "%(levelname)s: %(message)s"
     format = bb.msg.BBLogFormatter(format_str)
-    bb.msg.addDefaultlogFilter(console, bb.msg.BBLogFilterStdOut)
+    if params.options.quiet:
+        bb.msg.addDefaultlogFilter(console, bb.msg.BBLogFilterStdOut, bb.msg.BBLogFormatter.WARNING)
+    else:
+        bb.msg.addDefaultlogFilter(console, bb.msg.BBLogFilterStdOut)
     bb.msg.addDefaultlogFilter(errconsole, bb.msg.BBLogFilterStdErr)
     console.setFormatter(format)
     errconsole.setFormatter(format)
@@ -287,6 +362,12 @@ def main(server, eventHandler, params, tf = TerminalFilter):
         bb.msg.addDefaultlogFilter(consolelog)
         consolelog.setFormatter(conlogformat)
         logger.addHandler(consolelog)
+        loglink = os.path.join(os.path.dirname(consolelogfile), 'console-latest.log')
+        bb.utils.remove(loglink)
+        try:
+           os.symlink(os.path.basename(consolelogfile), loglink)
+        except OSError:
+           pass
 
     llevel, debug_domains = bb.msg.constructLogOptions()
     server.runCommand(["setEventMask", server.getEventHandle(), llevel, debug_domains, _evt_list])
@@ -323,7 +404,7 @@ def main(server, eventHandler, params, tf = TerminalFilter):
     warnings = 0
     taskfailures = []
 
-    termfilter = tf(main, helper, console, errconsole, format)
+    termfilter = tf(main, helper, console, errconsole, format, params.options.quiet)
     atexit.register(termfilter.finish)
 
     while True:
@@ -415,15 +496,19 @@ def main(server, eventHandler, params, tf = TerminalFilter):
                 parseprogress = new_progress("Parsing recipes", event.total).start()
                 continue
             if isinstance(event, bb.event.ParseProgress):
-                parseprogress.update(event.current)
+                if parseprogress:
+                    parseprogress.update(event.current)
+                else:
+                    bb.warn("Got ParseProgress event for parsing that never started?")
                 continue
             if isinstance(event, bb.event.ParseCompleted):
                 if not parseprogress:
                     continue
-
                 parseprogress.finish()
-                print(("Parsing of %d .bb files complete (%d cached, %d parsed). %d targets, %d skipped, %d masked, %d errors."
-                    % ( event.total, event.cached, event.parsed, event.virtuals, event.skipped, event.masked, event.errors)))
+                pasreprogress = None
+                if not params.options.quiet:
+                    print(("Parsing of %d .bb files complete (%d cached, %d parsed). %d targets, %d skipped, %d masked, %d errors."
+                        % ( event.total, event.cached, event.parsed, event.virtuals, event.skipped, event.masked, event.errors)))
                 continue
 
             if isinstance(event, bb.event.CacheLoadStarted):
@@ -434,7 +519,8 @@ def main(server, eventHandler, params, tf = TerminalFilter):
                 continue
             if isinstance(event, bb.event.CacheLoadCompleted):
                 cacheprogress.finish()
-                print("Loaded %d entries from dependency cache." % event.num_entries)
+                if not params.options.quiet:
+                    print("Loaded %d entries from dependency cache." % event.num_entries)
                 continue
 
             if isinstance(event, bb.command.CommandFailed):
@@ -518,6 +604,22 @@ def main(server, eventHandler, params, tf = TerminalFilter):
             if isinstance(event, bb.event.DepTreeGenerated):
                 continue
 
+            if isinstance(event, bb.event.ProcessStarted):
+                parseprogress = new_progress(event.processname, event.total)
+                parseprogress.start(False)
+                continue
+            if isinstance(event, bb.event.ProcessProgress):
+                if parseprogress:
+                    parseprogress.update(event.progress)
+                else:
+                    bb.warn("Got ProcessProgress event for someting that never started?")
+                continue
+            if isinstance(event, bb.event.ProcessFinished):
+                if parseprogress:
+                    parseprogress.finish()
+                parseprogress = None
+                continue
+
             # ignore
             if isinstance(event, (bb.event.BuildBase,
                                   bb.event.MetadataEvent,
@@ -529,7 +631,8 @@ def main(server, eventHandler, params, tf = TerminalFilter):
                                   bb.event.OperationStarted,
                                   bb.event.OperationCompleted,
                                   bb.event.OperationProgress,
-                                  bb.event.DiskFull)):
+                                  bb.event.DiskFull,
+                                  bb.build.TaskProgress)):
                 continue
 
             logger.error("Unknown event: %s", event)
@@ -582,7 +685,7 @@ def main(server, eventHandler, params, tf = TerminalFilter):
         if return_value and errors:
             summary += pluralise("\nSummary: There was %s ERROR message shown, returning a non-zero exit code.",
                                  "\nSummary: There were %s ERROR messages shown, returning a non-zero exit code.", errors)
-        if summary:
+        if summary and not params.options.quiet:
             print(summary)
 
         if interrupted:

bitbake/lib/bb/ui/toasterui.py

@@ -363,6 +363,9 @@ def main(server, eventHandler, params):
                     errors += 1
                     errorcode = 1
                     logger.error("Command execution failed: %s", event.error)
+                elif isinstance(event, bb.event.BuildCompleted):
+                    buildinfohelper.scan_image_artifacts()
+                    buildinfohelper.clone_required_sdk_artifacts()
 
                 # turn off logging to the current build log
                 _close_build_log(build_log)
@@ -410,12 +413,8 @@ def main(server, eventHandler, params):
                     buildinfohelper.store_target_package_data(event)
                 elif event.type == "MissedSstate":
                     buildinfohelper.store_missed_state_tasks(event)
-                elif event.type == "ImageFileSize":
-                    buildinfohelper.update_target_image_file(event)
-                elif event.type == "ArtifactFileSize":
-                    buildinfohelper.update_artifact_image_file(event)
-                elif event.type == "LicenseManifestPath":
-                    buildinfohelper.store_license_manifest_path(event)
+                elif event.type == "SDKArtifactInfo":
+                    buildinfohelper.scan_sdk_artifacts(event)
                 elif event.type == "SetBRBE":
                     buildinfohelper.brbe = buildinfohelper._get_data_from_event(event)
                 elif event.type == "OSErrorException":

bitbake/lib/bb/ui/uihelper.py

@@ -18,6 +18,7 @@
 # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
 
 import bb.build
+import time
 
 class BBUIHelper:
     def __init__(self):
@@ -31,7 +32,7 @@ class BBUIHelper:
 
     def eventHandler(self, event):
         if isinstance(event, bb.build.TaskStarted):
-            self.running_tasks[event.pid] = { 'title' : "%s %s" % (event._package, event._task) }
+            self.running_tasks[event.pid] = { 'title' : "%s %s" % (event._package, event._task), 'starttime' : time.time() }
             self.running_pids.append(event.pid)
             self.needUpdate = True
         if isinstance(event, bb.build.TaskSucceeded):
@@ -52,6 +53,11 @@ class BBUIHelper:
             self.tasknumber_current = event.stats.completed + event.stats.active + event.stats.failed + 1
             self.tasknumber_total = event.stats.total
             self.needUpdate = True
+        if isinstance(event, bb.build.TaskProgress):
+            if event.pid > 0:
+                self.running_tasks[event.pid]['progress'] = event.progress
+                self.running_tasks[event.pid]['rate'] = event.rate
+                self.needUpdate = True
 
     def getTasks(self):
         self.needUpdate = False

bitbake/lib/bb/utils.py

@@ -250,6 +250,7 @@ def explode_dep_versions2(s):
         if not (i in r and r[i]):
             r[lastdep] = []
 
+    r = collections.OrderedDict(sorted(r.items(), key=lambda x: x[0]))
     return r
 
 def explode_dep_versions(s):
@@ -574,6 +575,7 @@ def preserved_envvars_exported():
         'TERM',
         'USER',
         'LC_ALL',
+        'BBSERVER',
     ]
 
 def preserved_envvars():
@@ -598,7 +600,6 @@ def filter_environment(good_vars):
             continue
 
         removed_vars[key] = os.environ[key]
-        os.unsetenv(key)
         del os.environ[key]
 
     # If we spawn a python process, we need to have a UTF-8 locale, else python's file
@@ -1444,9 +1445,8 @@ def set_process_name(name):
     # This is nice to have for debugging, not essential
     try:
         libc = cdll.LoadLibrary('libc.so.6')
-        buff = create_string_buffer(len(name)+1)
-        buff.value = name
-        libc.prctl(15, byref(buff), 0, 0, 0)
+        buf = create_string_buffer(bytes(name, 'utf-8'))
+        libc.prctl(15, byref(buf), 0, 0, 0)
     except:
         pass
 

bitbake/lib/progressbar.py

@@ -1,384 +0,0 @@
-#!/usr/bin/env python
-# -*- coding: iso-8859-1 -*-
-#
-# progressbar  - Text progressbar library for python.
-# Copyright (c) 2005 Nilton Volpato
-#
-# This library is free software; you can redistribute it and/or
-# modify it under the terms of the GNU Lesser General Public
-# License as published by the Free Software Foundation; either
-# version 2.1 of the License, or (at your option) any later version.
-#
-# This library is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-# Lesser General Public License for more details.
-#
-# You should have received a copy of the GNU Lesser General Public
-# License along with this library; if not, write to the Free Software
-# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
-
-
-"""Text progressbar library for python.
-
-This library provides a text mode progressbar. This is typically used
-to display the progress of a long running operation, providing a
-visual clue that processing is underway.
-
-The ProgressBar class manages the progress, and the format of the line
-is given by a number of widgets. A widget is an object that may
-display diferently depending on the state of the progress. There are
-three types of widget:
-- a string, which always shows itself;
-- a ProgressBarWidget, which may return a diferent value every time
-it's update method is called; and
-- a ProgressBarWidgetHFill, which is like ProgressBarWidget, except it
-expands to fill the remaining width of the line.
-
-The progressbar module is very easy to use, yet very powerful. And
-automatically supports features like auto-resizing when available.
-"""
-
-from __future__ import division
-
-__author__ = "Nilton Volpato"
-__author_email__ = "first-name dot last-name @ gmail.com"
-__date__ = "2006-05-07"
-__version__ = "2.3-dev"
-
-import sys, time, os
-from array import array
-try:
-    from fcntl import ioctl
-    import termios
-except ImportError:
-    pass
-import signal
-try:
-    basestring
-except NameError:
-    basestring = (str,)
-
-class ProgressBarWidget(object):
-    """This is an element of ProgressBar formatting.
-
-    The ProgressBar object will call it's update value when an update
-    is needed. It's size may change between call, but the results will
-    not be good if the size changes drastically and repeatedly.
-    """
-    def update(self, pbar):
-        """Returns the string representing the widget.
-
-        The parameter pbar is a reference to the calling ProgressBar,
-        where one can access attributes of the class for knowing how
-        the update must be made.
-
-        At least this function must be overriden."""
-        pass
-
-class ProgressBarWidgetHFill(object):
-    """This is a variable width element of ProgressBar formatting.
-
-    The ProgressBar object will call it's update value, informing the
-    width this object must the made. This is like TeX \\hfill, it will
-    expand to fill the line. You can use more than one in the same
-    line, and they will all have the same width, and together will
-    fill the line.
-    """
-    def update(self, pbar, width):
-        """Returns the string representing the widget.
-
-        The parameter pbar is a reference to the calling ProgressBar,
-        where one can access attributes of the class for knowing how
-        the update must be made. The parameter width is the total
-        horizontal width the widget must have.
-
-        At least this function must be overriden."""
-        pass
-
-
-class ETA(ProgressBarWidget):
-    "Widget for the Estimated Time of Arrival"
-    def format_time(self, seconds):
-        return time.strftime('%H:%M:%S', time.gmtime(seconds))
-    def update(self, pbar):
-        if pbar.currval == 0:
-            return 'ETA:  --:--:--'
-        elif pbar.finished:
-            return 'Time: %s' % self.format_time(pbar.seconds_elapsed)
-        else:
-            elapsed = pbar.seconds_elapsed
-            eta = elapsed * pbar.maxval / pbar.currval - elapsed
-            return 'ETA:  %s' % self.format_time(eta)
-
-class FileTransferSpeed(ProgressBarWidget):
-    "Widget for showing the transfer speed (useful for file transfers)."
-    def __init__(self, unit='B'):
-        self.unit = unit
-        self.fmt = '%6.2f %s'
-        self.prefixes = ['', 'K', 'M', 'G', 'T', 'P']
-    def update(self, pbar):
-        if pbar.seconds_elapsed < 2e-6:#== 0:
-            bps = 0.0
-        else:
-            bps = pbar.currval / pbar.seconds_elapsed
-        spd = bps
-        for u in self.prefixes:
-            if spd < 1000:
-                break
-            spd /= 1000
-        return self.fmt % (spd, u + self.unit + '/s')
-
-class RotatingMarker(ProgressBarWidget):
-    "A rotating marker for filling the bar of progress."
-    def __init__(self, markers='|/-\\'):
-        self.markers = markers
-        self.curmark = -1
-    def update(self, pbar):
-        if pbar.finished:
-            return self.markers[0]
-        self.curmark = (self.curmark + 1) % len(self.markers)
-        return self.markers[self.curmark]
-
-class Percentage(ProgressBarWidget):
-    "Just the percentage done."
-    def update(self, pbar):
-        return '%3d%%' % pbar.percentage()
-
-class SimpleProgress(ProgressBarWidget):
-    "Returns what is already done and the total, e.g.: '5 of 47'"
-    def __init__(self, sep=' of '):
-        self.sep = sep
-    def update(self, pbar):
-        return '%d%s%d' % (pbar.currval, self.sep, pbar.maxval)
-
-class Bar(ProgressBarWidgetHFill):
-    "The bar of progress. It will stretch to fill the line."
-    def __init__(self, marker='#', left='|', right='|'):
-        self.marker = marker
-        self.left = left
-        self.right = right
-    def _format_marker(self, pbar):
-        if isinstance(self.marker, basestring):
-            return self.marker
-        else:
-            return self.marker.update(pbar)
-    def update(self, pbar, width):
-        percent = pbar.percentage()
-        cwidth = width - len(self.left) - len(self.right)
-        marked_width = int(percent * cwidth // 100)
-        m = self._format_marker(pbar)
-        bar = (self.left + (m * marked_width).ljust(cwidth) + self.right)
-        return bar
-
-class ReverseBar(Bar):
-    "The reverse bar of progress, or bar of regress. :)"
-    def update(self, pbar, width):
-        percent = pbar.percentage()
-        cwidth = width - len(self.left) - len(self.right)
-        marked_width = int(percent * cwidth // 100)
-        m = self._format_marker(pbar)
-        bar = (self.left + (m*marked_width).rjust(cwidth) + self.right)
-        return bar
-
-default_widgets = [Percentage(), ' ', Bar()]
-class ProgressBar(object):
-    """This is the ProgressBar class, it updates and prints the bar.
-
-    A common way of using it is like:
-    >>> pbar = ProgressBar().start()
-    >>> for i in xrange(100):
-    ...    # do something
-    ...    pbar.update(i+1)
-    ...
-    >>> pbar.finish()
-
-    You can also use a progressbar as an iterator:
-    >>> progress = ProgressBar()
-    >>> for i in progress(some_iterable):
-    ...    # do something
-    ...
-
-    But anything you want to do is possible (well, almost anything).
-    You can supply different widgets of any type in any order. And you
-    can even write your own widgets! There are many widgets already
-    shipped and you should experiment with them.
-
-    The term_width parameter must be an integer or None. In the latter case
-    it will try to guess it, if it fails it will default to 80 columns.
-
-    When implementing a widget update method you may access any
-    attribute or function of the ProgressBar object calling the
-    widget's update method. The most important attributes you would
-    like to access are:
-    - currval: current value of the progress, 0 <= currval <= maxval
-    - maxval: maximum (and final) value of the progress
-    - finished: True if the bar has finished (reached 100%), False o/w
-    - start_time: the time when start() method of ProgressBar was called
-    - seconds_elapsed: seconds elapsed since start_time
-    - percentage(): percentage of the progress [0..100]. This is a method.
-
-    The attributes above are unlikely to change between different versions,
-    the other ones may change or cease to exist without notice, so try to rely
-    only on the ones documented above if you are extending the progress bar.
-    """
-
-    __slots__ = ('currval', 'fd', 'finished', 'last_update_time', 'maxval',
-                 'next_update', 'num_intervals', 'seconds_elapsed',
-                 'signal_set', 'start_time', 'term_width', 'update_interval',
-                 'widgets', '_iterable')
-
-    _DEFAULT_MAXVAL = 100
-
-    def __init__(self, maxval=None, widgets=default_widgets, term_width=None,
-                 fd=sys.stderr):
-        self.maxval = maxval
-        self.widgets = widgets
-        self.fd = fd
-        self.signal_set = False
-        if term_width is not None:
-            self.term_width = term_width
-        else:
-            try:
-                self._handle_resize(None, None)
-                signal.signal(signal.SIGWINCH, self._handle_resize)
-                self.signal_set = True
-            except (SystemExit, KeyboardInterrupt):
-                raise
-            except:
-                self.term_width = int(os.environ.get('COLUMNS', 80)) - 1
-
-        self.currval = 0
-        self.finished = False
-        self.start_time = None
-        self.last_update_time = None
-        self.seconds_elapsed = 0
-        self._iterable = None
-
-    def __call__(self, iterable):
-        try:
-            self.maxval = len(iterable)
-        except TypeError:
-            # If the iterable has no length, then rely on the value provided
-            # by the user, otherwise fail.
-            if not (isinstance(self.maxval, (int, long)) and self.maxval > 0):
-                raise RuntimeError('Could not determine maxval from iterable. '
-                                   'You must explicitly provide a maxval.')
-        self._iterable = iter(iterable)
-        self.start()
-        return self
-
-    def __iter__(self):
-        return self
-
-    def next(self):
-        try:
-            next = self._iterable.next()
-            self.update(self.currval + 1)
-            return next
-        except StopIteration:
-            self.finish()
-            raise
-
-    def _handle_resize(self, signum, frame):
-        h, w = array('h', ioctl(self.fd, termios.TIOCGWINSZ, '\0' * 8))[:2]
-        self.term_width = w
-
-    def percentage(self):
-        "Returns the percentage of the progress."
-        return self.currval * 100.0 / self.maxval
-
-    def _format_widgets(self):
-        r = []
-        hfill_inds = []
-        num_hfill = 0
-        currwidth = 0
-        for i, w in enumerate(self.widgets):
-            if isinstance(w, ProgressBarWidgetHFill):
-                r.append(w)
-                hfill_inds.append(i)
-                num_hfill += 1
-            elif isinstance(w, basestring):
-                r.append(w)
-                currwidth += len(w)
-            else:
-                weval = w.update(self)
-                currwidth += len(weval)
-                r.append(weval)
-        for iw in hfill_inds:
-            widget_width = int((self.term_width - currwidth) // num_hfill)
-            r[iw] = r[iw].update(self, widget_width)
-        return r
-
-    def _format_line(self):
-        return ''.join(self._format_widgets()).ljust(self.term_width)
-
-    def _next_update(self):
-        return int((int(self.num_intervals *
-                        (self.currval / self.maxval)) + 1) *
-                   self.update_interval)
-
-    def _need_update(self):
-        """Returns true when the progressbar should print an updated line.
-
-        You can override this method if you want finer grained control over
-        updates.
-
-        The current implementation is optimized to be as fast as possible and
-        as economical as possible in the number of updates. However, depending
-        on your usage you may want to do more updates. For instance, if your
-        progressbar stays in the same percentage for a long time, and you want
-        to update other widgets, like ETA, then you could return True after
-        some time has passed with no updates.
-
-        Ideally you could call self._format_line() and see if it's different
-        from the previous _format_line() call, but calling _format_line() takes
-        around 20 times more time than calling this implementation of
-        _need_update().
-        """
-        return self.currval >= self.next_update
-
-    def update(self, value):
-        "Updates the progress bar to a new value."
-        assert 0 <= value <= self.maxval, '0 <= %d <= %d' % (value, self.maxval)
-        self.currval = value
-        if not self._need_update():
-            return
-        if self.start_time is None:
-            raise RuntimeError('You must call start() before calling update()')
-        now = time.time()
-        self.seconds_elapsed = now - self.start_time
-        self.next_update = self._next_update()
-        self.fd.write(self._format_line() + '\r')
-        self.last_update_time = now
-
-    def start(self):
-        """Starts measuring time, and prints the bar at 0%.
-
-        It returns self so you can use it like this:
-        >>> pbar = ProgressBar().start()
-        >>> for i in xrange(100):
-        ...    # do something
-        ...    pbar.update(i+1)
-        ...
-        >>> pbar.finish()
-        """
-        if self.maxval is None:
-            self.maxval = self._DEFAULT_MAXVAL
-        assert self.maxval > 0
-
-        self.num_intervals = max(100, self.term_width)
-        self.update_interval = self.maxval / self.num_intervals
-        self.next_update = 0
-
-        self.start_time = self.last_update_time = time.time()
-        self.update(0)
-        return self
-
-    def finish(self):
-        """Used to tell the progress is finished."""
-        self.finished = True
-        self.update(self.maxval)
-        self.fd.write('\n')
-        if self.signal_set:
-            signal.signal(signal.SIGWINCH, signal.SIG_DFL)

bitbake/lib/progressbar/LICENSE.txt

@@ -0,0 +1,52 @@
+You can redistribute and/or modify this library under the terms of the
+GNU LGPL license or BSD license (or both).
+
+---
+
+progressbar - Text progress bar library for python.
+Copyright (C) 2005 Nilton Volpato
+
+This library is free software; you can redistribute it and/or
+modify it under the terms of the GNU Lesser General Public
+License as published by the Free Software Foundation; either
+version 2.1 of the License, or (at your option) any later version.
+
+This library is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+Lesser General Public License for more details.
+
+You should have received a copy of the GNU Lesser General Public
+License along with this library; if not, write to the Free Software
+Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
+
+---
+
+progressbar - Text progress bar library for python
+Copyright (c) 2008 Nilton Volpato
+
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are met:
+
+ a. Redistributions of source code must retain the above copyright notice,
+    this list of conditions and the following disclaimer.
+ b. Redistributions in binary form must reproduce the above copyright
+    notice, this list of conditions and the following disclaimer in the
+    documentation and/or other materials provided with the distribution.
+ c. Neither the name of the author nor the names of its contributors
+    may be used to endorse or promote products derived from this software
+    without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR
+ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+DAMAGE.

bitbake/lib/progressbar/__init__.py

@@ -0,0 +1,49 @@
+#!/usr/bin/python
+# -*- coding: utf-8 -*-
+#
+# progressbar  - Text progress bar library for Python.
+# Copyright (c) 2005 Nilton Volpato
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation; either
+# version 2.1 of the License, or (at your option) any later version.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
+
+"""Text progress bar library for Python.
+
+A text progress bar is typically used to display the progress of a long
+running operation, providing a visual cue that processing is underway.
+
+The ProgressBar class manages the current progress, and the format of the line
+is given by a number of widgets. A widget is an object that may display
+differently depending on the state of the progress bar. There are three types
+of widgets:
+ - a string, which always shows itself
+
+ - a ProgressBarWidget, which may return a different value every time its
+   update method is called
+
+ - a ProgressBarWidgetHFill, which is like ProgressBarWidget, except it
+   expands to fill the remaining width of the line.
+
+The progressbar module is very easy to use, yet very powerful. It will also
+automatically enable features like auto-resizing when the system supports it.
+"""
+
+__author__ = 'Nilton Volpato'
+__author_email__ = 'first-name dot last-name @ gmail.com'
+__date__ = '2011-05-14'
+__version__ = '2.3'
+
+from .compat import *
+from .widgets import *
+from .progressbar import *

bitbake/lib/progressbar/compat.py

@@ -0,0 +1,44 @@
+# -*- coding: utf-8 -*-
+#
+# progressbar  - Text progress bar library for Python.
+# Copyright (c) 2005 Nilton Volpato
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation; either
+# version 2.1 of the License, or (at your option) any later version.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
+
+"""Compatibility methods and classes for the progressbar module."""
+
+
+# Python 3.x (and backports) use a modified iterator syntax
+# This will allow 2.x to behave with 3.x iterators
+try:
+  next
+except NameError:
+    def next(iter):
+        try:
+            # Try new style iterators
+            return iter.__next__()
+        except AttributeError:
+            # Fallback in case of a "native" iterator
+            return iter.next()
+
+
+# Python < 2.5 does not have "any"
+try:
+  any
+except NameError:
+   def any(iterator):
+      for item in iterator:
+         if item: return True
+      return False

bitbake/lib/progressbar/progressbar.py

@@ -0,0 +1,315 @@
+# -*- coding: utf-8 -*-
+#
+# progressbar  - Text progress bar library for Python.
+# Copyright (c) 2005 Nilton Volpato
+#
+# (With some small changes after importing into BitBake)
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation; either
+# version 2.1 of the License, or (at your option) any later version.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
+
+"""Main ProgressBar class."""
+
+from __future__ import division
+
+import math
+import os
+import signal
+import sys
+import time
+
+try:
+    from fcntl import ioctl
+    from array import array
+    import termios
+except ImportError:
+    pass
+
+from .compat import *  # for: any, next
+from . import widgets
+
+
+class UnknownLength: pass
+
+
+class ProgressBar(object):
+    """The ProgressBar class which updates and prints the bar.
+
+    A common way of using it is like:
+    >>> pbar = ProgressBar().start()
+    >>> for i in range(100):
+    ...    # do something
+    ...    pbar.update(i+1)
+    ...
+    >>> pbar.finish()
+
+    You can also use a ProgressBar as an iterator:
+    >>> progress = ProgressBar()
+    >>> for i in progress(some_iterable):
+    ...    # do something
+    ...
+
+    Since the progress bar is incredibly customizable you can specify
+    different widgets of any type in any order. You can even write your own
+    widgets! However, since there are already a good number of widgets you
+    should probably play around with them before moving on to create your own
+    widgets.
+
+    The term_width parameter represents the current terminal width. If the
+    parameter is set to an integer then the progress bar will use that,
+    otherwise it will attempt to determine the terminal width falling back to
+    80 columns if the width cannot be determined.
+
+    When implementing a widget's update method you are passed a reference to
+    the current progress bar. As a result, you have access to the
+    ProgressBar's methods and attributes. Although there is nothing preventing
+    you from changing the ProgressBar you should treat it as read only.
+
+    Useful methods and attributes include (Public API):
+     - currval: current progress (0 <= currval <= maxval)
+     - maxval: maximum (and final) value
+     - finished: True if the bar has finished (reached 100%)
+     - start_time: the time when start() method of ProgressBar was called
+     - seconds_elapsed: seconds elapsed since start_time and last call to
+                        update
+     - percentage(): progress in percent [0..100]
+    """
+
+    __slots__ = ('currval', 'fd', 'finished', 'last_update_time',
+                 'left_justify', 'maxval', 'next_update', 'num_intervals',
+                 'poll', 'seconds_elapsed', 'signal_set', 'start_time',
+                 'term_width', 'update_interval', 'widgets', '_time_sensitive',
+                 '__iterable')
+
+    _DEFAULT_MAXVAL = 100
+    _DEFAULT_TERMSIZE = 80
+    _DEFAULT_WIDGETS = [widgets.Percentage(), ' ', widgets.Bar()]
+
+    def __init__(self, maxval=None, widgets=None, term_width=None, poll=1,
+                 left_justify=True, fd=sys.stderr):
+        """Initializes a progress bar with sane defaults."""
+
+        # Don't share a reference with any other progress bars
+        if widgets is None:
+            widgets = list(self._DEFAULT_WIDGETS)
+
+        self.maxval = maxval
+        self.widgets = widgets
+        self.fd = fd
+        self.left_justify = left_justify
+
+        self.signal_set = False
+        if term_width is not None:
+            self.term_width = term_width
+        else:
+            try:
+                self._handle_resize(None, None)
+                signal.signal(signal.SIGWINCH, self._handle_resize)
+                self.signal_set = True
+            except (SystemExit, KeyboardInterrupt): raise
+            except Exception as e:
+                print("DEBUG 5 %s" % e)
+                self.term_width = self._env_size()
+
+        self.__iterable = None
+        self._update_widgets()
+        self.currval = 0
+        self.finished = False
+        self.last_update_time = None
+        self.poll = poll
+        self.seconds_elapsed = 0
+        self.start_time = None
+        self.update_interval = 1
+        self.next_update = 0
+
+
+    def __call__(self, iterable):
+        """Use a ProgressBar to iterate through an iterable."""
+
+        try:
+            self.maxval = len(iterable)
+        except:
+            if self.maxval is None:
+                self.maxval = UnknownLength
+
+        self.__iterable = iter(iterable)
+        return self
+
+
+    def __iter__(self):
+        return self
+
+
+    def __next__(self):
+        try:
+            value = next(self.__iterable)
+            if self.start_time is None:
+                self.start()
+            else:
+                self.update(self.currval + 1)
+            return value
+        except StopIteration:
+            if self.start_time is None:
+                self.start()
+            self.finish()
+            raise
+
+
+    # Create an alias so that Python 2.x won't complain about not being
+    # an iterator.
+    next = __next__
+
+
+    def _env_size(self):
+        """Tries to find the term_width from the environment."""
+
+        return int(os.environ.get('COLUMNS', self._DEFAULT_TERMSIZE)) - 1
+
+
+    def _handle_resize(self, signum=None, frame=None):
+        """Tries to catch resize signals sent from the terminal."""
+
+        h, w = array('h', ioctl(self.fd, termios.TIOCGWINSZ, '\0' * 8))[:2]
+        self.term_width = w
+
+
+    def percentage(self):
+        """Returns the progress as a percentage."""
+        if self.currval >= self.maxval:
+            return 100.0
+        return (self.currval * 100.0 / self.maxval) if self.maxval else 100.00
+
+    percent = property(percentage)
+
+
+    def _format_widgets(self):
+        result = []
+        expanding = []
+        width = self.term_width
+
+        for index, widget in enumerate(self.widgets):
+            if isinstance(widget, widgets.WidgetHFill):
+                result.append(widget)
+                expanding.insert(0, index)
+            else:
+                widget = widgets.format_updatable(widget, self)
+                result.append(widget)
+                width -= len(widget)
+
+        count = len(expanding)
+        while count:
+            portion = max(int(math.ceil(width * 1. / count)), 0)
+            index = expanding.pop()
+            count -= 1
+
+            widget = result[index].update(self, portion)
+            width -= len(widget)
+            result[index] = widget
+
+        return result
+
+
+    def _format_line(self):
+        """Joins the widgets and justifies the line."""
+
+        widgets = ''.join(self._format_widgets())
+
+        if self.left_justify: return widgets.ljust(self.term_width)
+        else: return widgets.rjust(self.term_width)
+
+
+    def _need_update(self):
+        """Returns whether the ProgressBar should redraw the line."""
+        if self.currval >= self.next_update or self.finished: return True
+
+        delta = time.time() - self.last_update_time
+        return self._time_sensitive and delta > self.poll
+
+
+    def _update_widgets(self):
+        """Checks all widgets for the time sensitive bit."""
+
+        self._time_sensitive = any(getattr(w, 'TIME_SENSITIVE', False)
+                                    for w in self.widgets)
+
+
+    def update(self, value=None):
+        """Updates the ProgressBar to a new value."""
+
+        if value is not None and value is not UnknownLength:
+            if (self.maxval is not UnknownLength
+                and not 0 <= value <= self.maxval):
+
+                raise ValueError('Value out of range')
+
+            self.currval = value
+
+
+        if not self._need_update(): return
+        if self.start_time is None:
+            raise RuntimeError('You must call "start" before calling "update"')
+
+        now = time.time()
+        self.seconds_elapsed = now - self.start_time
+        self.next_update = self.currval + self.update_interval
+        output = self._format_line()
+        self.fd.write(output + '\r')
+        self.fd.flush()
+        self.last_update_time = now
+        return output
+
+
+    def start(self, update=True):
+        """Starts measuring time, and prints the bar at 0%.
+
+        It returns self so you can use it like this:
+        >>> pbar = ProgressBar().start()
+        >>> for i in range(100):
+        ...    # do something
+        ...    pbar.update(i+1)
+        ...
+        >>> pbar.finish()
+        """
+
+        if self.maxval is None:
+            self.maxval = self._DEFAULT_MAXVAL
+
+        self.num_intervals = max(100, self.term_width)
+        self.next_update = 0
+
+        if self.maxval is not UnknownLength:
+            if self.maxval < 0: raise ValueError('Value out of range')
+            self.update_interval = self.maxval / self.num_intervals
+
+
+        self.start_time = time.time()
+        if update:
+            self.last_update_time = self.start_time
+            self.update(0)
+        else:
+            self.last_update_time = 0
+
+        return self
+
+
+    def finish(self):
+        """Puts the ProgressBar bar in the finished state."""
+
+        if self.finished:
+            return
+        self.finished = True
+        self.update(self.maxval)
+        self.fd.write('\n')
+        if self.signal_set:
+            signal.signal(signal.SIGWINCH, signal.SIG_DFL)

bitbake/lib/progressbar/widgets.py

@@ -0,0 +1,391 @@
+# -*- coding: utf-8 -*-
+#
+# progressbar  - Text progress bar library for Python.
+# Copyright (c) 2005 Nilton Volpato
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation; either
+# version 2.1 of the License, or (at your option) any later version.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
+
+"""Default ProgressBar widgets."""
+
+from __future__ import division
+
+import datetime
+import math
+
+try:
+    from abc import ABCMeta, abstractmethod
+except ImportError:
+    AbstractWidget = object
+    abstractmethod = lambda fn: fn
+else:
+    AbstractWidget = ABCMeta('AbstractWidget', (object,), {})
+
+
+def format_updatable(updatable, pbar):
+    if hasattr(updatable, 'update'): return updatable.update(pbar)
+    else: return updatable
+
+
+class Widget(AbstractWidget):
+    """The base class for all widgets.
+
+    The ProgressBar will call the widget's update value when the widget should
+    be updated. The widget's size may change between calls, but the widget may
+    display incorrectly if the size changes drastically and repeatedly.
+
+    The boolean TIME_SENSITIVE informs the ProgressBar that it should be
+    updated more often because it is time sensitive.
+    """
+
+    TIME_SENSITIVE = False
+    __slots__ = ()
+
+    @abstractmethod
+    def update(self, pbar):
+        """Updates the widget.
+
+        pbar - a reference to the calling ProgressBar
+        """
+
+
+class WidgetHFill(Widget):
+    """The base class for all variable width widgets.
+
+    This widget is much like the \\hfill command in TeX, it will expand to
+    fill the line. You can use more than one in the same line, and they will
+    all have the same width, and together will fill the line.
+    """
+
+    @abstractmethod
+    def update(self, pbar, width):
+        """Updates the widget providing the total width the widget must fill.
+
+        pbar - a reference to the calling ProgressBar
+        width - The total width the widget must fill
+        """
+
+
+class Timer(Widget):
+    """Widget which displays the elapsed seconds."""
+
+    __slots__ = ('format_string',)
+    TIME_SENSITIVE = True
+
+    def __init__(self, format='Elapsed Time: %s'):
+        self.format_string = format
+
+    @staticmethod
+    def format_time(seconds):
+        """Formats time as the string "HH:MM:SS"."""
+
+        return str(datetime.timedelta(seconds=int(seconds)))
+
+
+    def update(self, pbar):
+        """Updates the widget to show the elapsed time."""
+
+        return self.format_string % self.format_time(pbar.seconds_elapsed)
+
+
+class ETA(Timer):
+    """Widget which attempts to estimate the time of arrival."""
+
+    TIME_SENSITIVE = True
+
+    def update(self, pbar):
+        """Updates the widget to show the ETA or total time when finished."""
+
+        if pbar.currval == 0:
+            return 'ETA:  --:--:--'
+        elif pbar.finished:
+            return 'Time: %s' % self.format_time(pbar.seconds_elapsed)
+        else:
+            elapsed = pbar.seconds_elapsed
+            eta = elapsed * pbar.maxval / pbar.currval - elapsed
+            return 'ETA:  %s' % self.format_time(eta)
+
+
+class AdaptiveETA(Timer):
+    """Widget which attempts to estimate the time of arrival.
+
+    Uses a weighted average of two estimates:
+      1) ETA based on the total progress and time elapsed so far
+      2) ETA based on the progress as per the last 10 update reports
+
+    The weight depends on the current progress so that to begin with the
+    total progress is used and at the end only the most recent progress is
+    used.
+    """
+
+    TIME_SENSITIVE = True
+    NUM_SAMPLES = 10
+
+    def _update_samples(self, currval, elapsed):
+        sample = (currval, elapsed)
+        if not hasattr(self, 'samples'):
+            self.samples = [sample] * (self.NUM_SAMPLES + 1)
+        else:
+            self.samples.append(sample)
+        return self.samples.pop(0)
+
+    def _eta(self, maxval, currval, elapsed):
+        return elapsed * maxval / float(currval) - elapsed
+
+    def update(self, pbar):
+        """Updates the widget to show the ETA or total time when finished."""
+        if pbar.currval == 0:
+            return 'ETA:  --:--:--'
+        elif pbar.finished:
+            return 'Time: %s' % self.format_time(pbar.seconds_elapsed)
+        else:
+            elapsed = pbar.seconds_elapsed
+            currval1, elapsed1 = self._update_samples(pbar.currval, elapsed)
+            eta = self._eta(pbar.maxval, pbar.currval, elapsed)
+            if pbar.currval > currval1:
+                etasamp = self._eta(pbar.maxval - currval1,
+                                    pbar.currval - currval1,
+                                    elapsed - elapsed1)
+                weight = (pbar.currval / float(pbar.maxval)) ** 0.5
+                eta = (1 - weight) * eta + weight * etasamp
+            return 'ETA:  %s' % self.format_time(eta)
+
+
+class FileTransferSpeed(Widget):
+    """Widget for showing the transfer speed (useful for file transfers)."""
+
+    FORMAT = '%6.2f %s%s/s'
+    PREFIXES = ' kMGTPEZY'
+    __slots__ = ('unit',)
+
+    def __init__(self, unit='B'):
+        self.unit = unit
+
+    def update(self, pbar):
+        """Updates the widget with the current SI prefixed speed."""
+
+        if pbar.seconds_elapsed < 2e-6 or pbar.currval < 2e-6: # =~ 0
+            scaled = power = 0
+        else:
+            speed = pbar.currval / pbar.seconds_elapsed
+            power = int(math.log(speed, 1000))
+            scaled = speed / 1000.**power
+
+        return self.FORMAT % (scaled, self.PREFIXES[power], self.unit)
+
+
+class AnimatedMarker(Widget):
+    """An animated marker for the progress bar which defaults to appear as if
+    it were rotating.
+    """
+
+    __slots__ = ('markers', 'curmark')
+
+    def __init__(self, markers='|/-\\'):
+        self.markers = markers
+        self.curmark = -1
+
+    def update(self, pbar):
+        """Updates the widget to show the next marker or the first marker when
+        finished"""
+
+        if pbar.finished: return self.markers[0]
+
+        self.curmark = (self.curmark + 1) % len(self.markers)
+        return self.markers[self.curmark]
+
+# Alias for backwards compatibility
+RotatingMarker = AnimatedMarker
+
+
+class Counter(Widget):
+    """Displays the current count."""
+
+    __slots__ = ('format_string',)
+
+    def __init__(self, format='%d'):
+        self.format_string = format
+
+    def update(self, pbar):
+        return self.format_string % pbar.currval
+
+
+class Percentage(Widget):
+    """Displays the current percentage as a number with a percent sign."""
+
+    def update(self, pbar):
+        return '%3d%%' % pbar.percentage()
+
+
+class FormatLabel(Timer):
+    """Displays a formatted label."""
+
+    mapping = {
+        'elapsed': ('seconds_elapsed', Timer.format_time),
+        'finished': ('finished', None),
+        'last_update': ('last_update_time', None),
+        'max': ('maxval', None),
+        'seconds': ('seconds_elapsed', None),
+        'start': ('start_time', None),
+        'value': ('currval', None)
+    }
+
+    __slots__ = ('format_string',)
+    def __init__(self, format):
+        self.format_string = format
+
+    def update(self, pbar):
+        context = {}
+        for name, (key, transform) in self.mapping.items():
+            try:
+                value = getattr(pbar, key)
+
+                if transform is None:
+                   context[name] = value
+                else:
+                   context[name] = transform(value)
+            except: pass
+
+        return self.format_string % context
+
+
+class SimpleProgress(Widget):
+    """Returns progress as a count of the total (e.g.: "5 of 47")."""
+
+    __slots__ = ('sep',)
+
+    def __init__(self, sep=' of '):
+        self.sep = sep
+
+    def update(self, pbar):
+        return '%d%s%d' % (pbar.currval, self.sep, pbar.maxval)
+
+
+class Bar(WidgetHFill):
+    """A progress bar which stretches to fill the line."""
+
+    __slots__ = ('marker', 'left', 'right', 'fill', 'fill_left')
+
+    def __init__(self, marker='#', left='|', right='|', fill=' ',
+                 fill_left=True):
+        """Creates a customizable progress bar.
+
+        marker - string or updatable object to use as a marker
+        left - string or updatable object to use as a left border
+        right - string or updatable object to use as a right border
+        fill - character to use for the empty part of the progress bar
+        fill_left - whether to fill from the left or the right
+        """
+        self.marker = marker
+        self.left = left
+        self.right = right
+        self.fill = fill
+        self.fill_left = fill_left
+
+
+    def update(self, pbar, width):
+        """Updates the progress bar and its subcomponents."""
+
+        left, marked, right = (format_updatable(i, pbar) for i in
+                               (self.left, self.marker, self.right))
+
+        width -= len(left) + len(right)
+        # Marked must *always* have length of 1
+        if pbar.maxval:
+          marked *= int(pbar.currval / pbar.maxval * width)
+        else:
+          marked = ''
+
+        if self.fill_left:
+            return '%s%s%s' % (left, marked.ljust(width, self.fill), right)
+        else:
+            return '%s%s%s' % (left, marked.rjust(width, self.fill), right)
+
+
+class ReverseBar(Bar):
+    """A bar which has a marker which bounces from side to side."""
+
+    def __init__(self, marker='#', left='|', right='|', fill=' ',
+                 fill_left=False):
+        """Creates a customizable progress bar.
+
+        marker - string or updatable object to use as a marker
+        left - string or updatable object to use as a left border
+        right - string or updatable object to use as a right border
+        fill - character to use for the empty part of the progress bar
+        fill_left - whether to fill from the left or the right
+        """
+        self.marker = marker
+        self.left = left
+        self.right = right
+        self.fill = fill
+        self.fill_left = fill_left
+
+
+class BouncingBar(Bar):
+    def update(self, pbar, width):
+        """Updates the progress bar and its subcomponents."""
+
+        left, marker, right = (format_updatable(i, pbar) for i in
+                               (self.left, self.marker, self.right))
+
+        width -= len(left) + len(right)
+
+        if pbar.finished: return '%s%s%s' % (left, width * marker, right)
+
+        position = int(pbar.currval % (width * 2 - 1))
+        if position > width: position = width * 2 - position
+        lpad = self.fill * (position - 1)
+        rpad = self.fill * (width - len(marker) - len(lpad))
+
+        # Swap if we want to bounce the other way
+        if not self.fill_left: rpad, lpad = lpad, rpad
+
+        return '%s%s%s%s%s' % (left, lpad, marker, rpad, right)
+
+
+class BouncingSlider(Bar):
+    """
+    A slider that bounces back and forth in response to update() calls
+    without reference to the actual value. Based on a combination of
+    BouncingBar from a newer version of this module and RotatingMarker.
+    """
+    def __init__(self, marker='<=>'):
+        self.curmark = -1
+        self.forward = True
+        Bar.__init__(self, marker=marker)
+    def update(self, pbar, width):
+        left, marker, right = (format_updatable(i, pbar) for i in
+                               (self.left, self.marker, self.right))
+
+        width -= len(left) + len(right)
+        if width < 0:
+            return ''
+
+        if pbar.finished: return '%s%s%s' % (left, width * '=', right)
+
+        self.curmark = self.curmark + 1
+        position = int(self.curmark % (width * 2 - 1))
+        if position + len(marker) > width:
+            self.forward = not self.forward
+            self.curmark = 1
+            position = 1
+        lpad = ' ' * (position - 1)
+        rpad = ' ' * (width - len(marker) - len(lpad))
+
+        if not self.forward:
+            temp = lpad
+            lpad = rpad
+            rpad = temp
+        return '%s%s%s%s%s' % (left, lpad, marker, rpad, right)

bitbake/lib/toaster/orm/migrations/0008_refactor_artifact_models.py

@@ -0,0 +1,39 @@
+# -*- coding: utf-8 -*-
+from __future__ import unicode_literals
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('orm', '0007_auto_20160523_1446'),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name='TargetKernelFile',
+            fields=[
+                ('id', models.AutoField(auto_created=True, primary_key=True, verbose_name='ID', serialize=False)),
+                ('file_name', models.FilePathField()),
+                ('file_size', models.IntegerField()),
+                ('target', models.ForeignKey(to='orm.Target')),
+            ],
+        ),
+        migrations.CreateModel(
+            name='TargetSDKFile',
+            fields=[
+                ('id', models.AutoField(auto_created=True, primary_key=True, verbose_name='ID', serialize=False)),
+                ('file_name', models.FilePathField()),
+                ('file_size', models.IntegerField()),
+                ('target', models.ForeignKey(to='orm.Target')),
+            ],
+        ),
+        migrations.RemoveField(
+            model_name='buildartifact',
+            name='build',
+        ),
+        migrations.DeleteModel(
+            name='BuildArtifact',
+        ),
+    ]

bitbake/lib/toaster/orm/migrations/0009_target_package_manifest_path.py

@@ -0,0 +1,19 @@
+# -*- coding: utf-8 -*-
+from __future__ import unicode_literals
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('orm', '0008_refactor_artifact_models'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='target',
+            name='package_manifest_path',
+            field=models.CharField(null=True, max_length=500),
+        ),
+    ]

bitbake/lib/toaster/orm/models.py

@@ -22,7 +22,7 @@
 from __future__ import unicode_literals
 
 from django.db import models, IntegrityError
-from django.db.models import F, Q, Avg, Max, Sum
+from django.db.models import F, Q, Avg, Max, Sum, Count
 from django.utils import timezone
 from django.utils.encoding import force_bytes
 
@@ -436,58 +436,62 @@ class Build(models.Model):
             eta += ((eta - self.started_on)*(100-completeper))/completeper
         return eta
 
+    def has_images(self):
+        """
+        Returns True if at least one of the targets for this build has an
+        image file associated with it, False otherwise
+        """
+        targets = Target.objects.filter(build_id=self.id)
+        has_images = False
+        for target in targets:
+            if target.has_images():
+                has_images = True
+                break
+        return has_images
+
+    def has_image_recipes(self):
+        """
+        Returns True if a build has any targets which were built from
+        image recipes.
+        """
+        image_recipes = self.get_image_recipes()
+        return len(image_recipes) > 0
+
     def get_image_file_extensions(self):
         """
-        Get list of file name extensions for images produced by this build
+        Get string of file name extensions for images produced by this build;
+        note that this is the actual list of extensions stored on Target objects
+        for this build, and not the value of IMAGE_FSTYPES.
+
+        Returns comma-separated string, e.g. "vmdk, ext4"
         """
-        targets = Target.objects.filter(build_id = self.id)
         extensions = []
 
-        # pattern to match against file path for building extension string
-        pattern = re.compile('\.([^\.]+?)$')
-
+        targets = Target.objects.filter(build_id = self.id)
         for target in targets:
-            if (not target.is_image):
+            if not target.is_image:
                 continue
 
-            target_image_files = Target_Image_File.objects.filter(target_id = target.id)
+            target_image_files = Target_Image_File.objects.filter(
+                target_id=target.id)
 
             for target_image_file in target_image_files:
-                file_name = os.path.basename(target_image_file.file_name)
-                suffix = ''
+                extensions.append(target_image_file.suffix)
 
-                continue_matching = True
-
-                # incrementally extract the suffix from the file path,
-                # checking it against the list of valid suffixes at each
-                # step; if the path is stripped of all potential suffix
-                # parts without matching a valid suffix, this returns all
-                # characters after the first '.' in the file name
-                while continue_matching:
-                    matches = pattern.search(file_name)
-
-                    if None == matches:
-                        continue_matching = False
-                        suffix = re.sub('^\.', '', suffix)
-                        continue
-                    else:
-                        suffix = matches.group(1) + suffix
-
-                    if suffix in Target_Image_File.SUFFIXES:
-                        continue_matching = False
-                        continue
-                    else:
-                        # reduce the file name and try to find the next
-                        # segment from the path which might be part
-                        # of the suffix
-                        file_name = re.sub('.' + matches.group(1), '', file_name)
-                        suffix = '.' + suffix
-
-                if not suffix in extensions:
-                    extensions.append(suffix)
+        extensions = list(set(extensions))
+        extensions.sort()
 
         return ', '.join(extensions)
 
+    def get_image_fstypes(self):
+        """
+        Get the IMAGE_FSTYPES variable value for this build as a de-duplicated
+        list of image file suffixes.
+        """
+        image_fstypes = Variable.objects.get(
+            build=self, variable_name='IMAGE_FSTYPES').variable_value
+        return list(set(re.split(r' {1,}', image_fstypes)))
+
     def get_sorted_target_list(self):
         tgts = Target.objects.filter(build_id = self.id).order_by( 'target' );
         return( tgts );
@@ -600,28 +604,6 @@ class Build(models.Model):
     def __str__(self):
         return "%d %s %s" % (self.id, self.project, ",".join([t.target for t in self.target_set.all()]))
 
-
-# an Artifact is anything that results from a Build, and may be of interest to the user, and is not stored elsewhere
-class BuildArtifact(models.Model):
-    build = models.ForeignKey(Build)
-    file_name = models.FilePathField()
-    file_size = models.IntegerField()
-
-    def get_local_file_name(self):
-        try:
-            deploydir = Variable.objects.get(build = self.build, variable_name="DEPLOY_DIR").variable_value
-            return  self.file_name[len(deploydir)+1:]
-        except:
-            raise
-
-        return self.file_name
-
-    def get_basename(self):
-        return os.path.basename(self.file_name)
-
-    def is_available(self):
-        return self.build.buildrequest.environment.has_artifact(self.file_name)
-
 class ProjectTarget(models.Model):
     project = models.ForeignKey(Project)
     target = models.CharField(max_length=100)
@@ -635,6 +617,7 @@ class Target(models.Model):
     is_image = models.BooleanField(default = False)
     image_size = models.IntegerField(default=0)
     license_manifest_path = models.CharField(max_length=500, null=True)
+    package_manifest_path = models.CharField(max_length=500, null=True)
 
     def package_count(self):
         return Target_Installed_Package.objects.filter(target_id__exact=self.id).count()
@@ -642,6 +625,171 @@ class Target(models.Model):
     def __unicode__(self):
         return self.target
 
+    def get_similar_targets(self):
+        """
+        Get target sfor the same machine, task and target name
+        (e.g. 'core-image-minimal') from a successful build for this project
+        (but excluding this target).
+
+        Note that we only look for targets built by this project because
+        projects can have different configurations from each other, and put
+        their artifacts in different directories.
+
+        The possibility of error when retrieving candidate targets
+        is minimised by the fact that bitbake will rebuild artifacts if MACHINE
+        (or various other variables) change. In this case, there is no need to
+        clone artifacts from another target, as those artifacts will have
+        been re-generated for this target anyway.
+        """
+        query = ~Q(pk=self.pk) & \
+            Q(target=self.target) & \
+            Q(build__machine=self.build.machine) & \
+            Q(build__outcome=Build.SUCCEEDED) & \
+            Q(build__project=self.build.project)
+
+        return Target.objects.filter(query)
+
+    def get_similar_target_with_image_files(self):
+        """
+        Get the most recent similar target with Target_Image_Files associated
+        with it, for the purpose of cloning those files onto this target.
+        """
+        similar_target = None
+
+        candidates = self.get_similar_targets()
+        if candidates.count() == 0:
+            return similar_target
+
+        task_subquery = Q(task=self.task)
+
+        # we can look for a 'build' task if this task is a 'populate_sdk_ext'
+        # task, as the latter also creates images; and vice versa; note that
+        # 'build' targets can have their task set to '';
+        # also note that 'populate_sdk' does not produce image files
+        image_tasks = [
+            '', # aka 'build'
+            'build',
+            'image',
+            'populate_sdk_ext'
+        ]
+        if self.task in image_tasks:
+            task_subquery = Q(task__in=image_tasks)
+
+        # annotate with the count of files, to exclude any targets which
+        # don't have associated files
+        candidates = candidates.annotate(num_files=Count('target_image_file'))
+
+        query = task_subquery & Q(num_files__gt=0)
+
+        candidates = candidates.filter(query)
+
+        if candidates.count() > 0:
+            candidates.order_by('build__completed_on')
+            similar_target = candidates.last()
+
+        return similar_target
+
+    def get_similar_target_with_sdk_files(self):
+        """
+        Get the most recent similar target with TargetSDKFiles associated
+        with it, for the purpose of cloning those files onto this target.
+        """
+        similar_target = None
+
+        candidates = self.get_similar_targets()
+        if candidates.count() == 0:
+            return similar_target
+
+        # annotate with the count of files, to exclude any targets which
+        # don't have associated files
+        candidates = candidates.annotate(num_files=Count('targetsdkfile'))
+
+        query = Q(task=self.task) & Q(num_files__gt=0)
+
+        candidates = candidates.filter(query)
+
+        if candidates.count() > 0:
+            candidates.order_by('build__completed_on')
+            similar_target = candidates.last()
+
+        return similar_target
+
+    def clone_image_artifacts_from(self, target):
+        """
+        Make clones of the Target_Image_Files and TargetKernelFile objects
+        associated with Target target, then associate them with this target.
+
+        Note that for Target_Image_Files, we only want files from the previous
+        build whose suffix matches one of the suffixes defined in this
+        target's build's IMAGE_FSTYPES configuration variable. This prevents the
+        Target_Image_File object for an ext4 image being associated with a
+        target for a project which didn't produce an ext4 image (for example).
+
+        Also sets the license_manifest_path and package_manifest_path
+        of this target to the same path as that of target being cloned from, as
+        the manifests are also build artifacts but are treated differently.
+        """
+
+        image_fstypes = self.build.get_image_fstypes()
+
+        # filter out any image files whose suffixes aren't in the
+        # IMAGE_FSTYPES suffixes variable for this target's build
+        image_files = [target_image_file \
+            for target_image_file in target.target_image_file_set.all() \
+            if target_image_file.suffix in image_fstypes]
+
+        for image_file in image_files:
+            image_file.pk = None
+            image_file.target = self
+            image_file.save()
+
+        kernel_files = target.targetkernelfile_set.all()
+        for kernel_file in kernel_files:
+            kernel_file.pk = None
+            kernel_file.target = self
+            kernel_file.save()
+
+        self.license_manifest_path = target.license_manifest_path
+        self.package_manifest_path = target.package_manifest_path
+        self.save()
+
+    def clone_sdk_artifacts_from(self, target):
+        """
+        Clone TargetSDKFile objects from target and associate them with this
+        target.
+        """
+        sdk_files = target.targetsdkfile_set.all()
+        for sdk_file in sdk_files:
+            sdk_file.pk = None
+            sdk_file.target = self
+            sdk_file.save()
+
+    def has_images(self):
+        """
+        Returns True if this target has one or more image files attached to it.
+        """
+        return self.target_image_file_set.all().count() > 0
+
+# kernel artifacts for a target: bzImage and modules*
+class TargetKernelFile(models.Model):
+    target = models.ForeignKey(Target)
+    file_name = models.FilePathField()
+    file_size = models.IntegerField()
+
+    @property
+    def basename(self):
+        return os.path.basename(self.file_name)
+
+# SDK artifacts for a target: sh and manifest files
+class TargetSDKFile(models.Model):
+    target = models.ForeignKey(Target)
+    file_name = models.FilePathField()
+    file_size = models.IntegerField()
+
+    @property
+    def basename(self):
+        return os.path.basename(self.file_name)
+
 class Target_Image_File(models.Model):
     # valid suffixes for image files produced by a build
     SUFFIXES = {
@@ -658,6 +806,13 @@ class Target_Image_File(models.Model):
 
     @property
     def suffix(self):
+        """
+        Suffix for image file, minus leading "."
+        """
+        for suffix in Target_Image_File.SUFFIXES:
+            if self.file_name.endswith(suffix):
+                return suffix
+
         filename, suffix = os.path.splitext(self.file_name)
         suffix = suffix.lstrip('.')
         return suffix

bitbake/lib/toaster/tests/browser/selenium_helpers.py

@@ -27,179 +27,8 @@
 Helper methods for creating Toaster Selenium tests which run within
 the context of Django unit tests.
 """
-
-import os
-import time
-
 from django.contrib.staticfiles.testing import StaticLiveServerTestCase
-from selenium import webdriver
-from selenium.webdriver.support.ui import WebDriverWait
-from selenium.common.exceptions import NoSuchElementException, \
-        StaleElementReferenceException, TimeoutException
+from tests.browser.selenium_helpers_base import SeleniumTestCaseBase
 
-def create_selenium_driver(browser='chrome'):
-    # set default browser string based on env (if available)
-    env_browser = os.environ.get('TOASTER_TESTS_BROWSER')
-    if env_browser:
-        browser = env_browser
-
-    if browser == 'chrome':
-        return webdriver.Chrome(
-            service_args=["--verbose", "--log-path=selenium.log"]
-        )
-    elif browser == 'firefox':
-        return webdriver.Firefox()
-    elif browser == 'ie':
-        return webdriver.Ie()
-    elif browser == 'phantomjs':
-        return webdriver.PhantomJS()
-    else:
-        msg = 'Selenium driver for browser %s is not available' % browser
-        raise RuntimeError(msg)
-
-class Wait(WebDriverWait):
-    """
-    Subclass of WebDriverWait with predetermined timeout and poll
-    frequency. Also deals with a wider variety of exceptions.
-    """
-    _TIMEOUT = 10
-    _POLL_FREQUENCY = 0.5
-
-    def __init__(self, driver):
-        super(Wait, self).__init__(driver, self._TIMEOUT, self._POLL_FREQUENCY)
-
-    def until(self, method, message=''):
-        """
-        Calls the method provided with the driver as an argument until the
-        return value is not False.
-        """
-
-        end_time = time.time() + self._timeout
-        while True:
-            try:
-                value = method(self._driver)
-                if value:
-                    return value
-            except NoSuchElementException:
-                pass
-            except StaleElementReferenceException:
-                pass
-
-            time.sleep(self._poll)
-            if time.time() > end_time:
-                break
-
-        raise TimeoutException(message)
-
-    def until_not(self, method, message=''):
-        """
-        Calls the method provided with the driver as an argument until the
-        return value is False.
-        """
-
-        end_time = time.time() + self._timeout
-        while True:
-            try:
-                value = method(self._driver)
-                if not value:
-                    return value
-            except NoSuchElementException:
-                return True
-            except StaleElementReferenceException:
-                pass
-
-            time.sleep(self._poll)
-            if time.time() > end_time:
-                break
-
-        raise TimeoutException(message)
-
-class SeleniumTestCase(StaticLiveServerTestCase):
-    """
-    NB StaticLiveServerTestCase is used as the base test case so that
-    static files are served correctly in a Selenium test run context; see
-    https://docs.djangoproject.com/en/1.9/ref/contrib/staticfiles/#specialized-test-case-to-support-live-testing
-    """
-
-    @classmethod
-    def setUpClass(cls):
-        """ Create a webdriver driver at the class level """
-
-        super(SeleniumTestCase, cls).setUpClass()
-
-        # instantiate the Selenium webdriver once for all the test methods
-        # in this test case
-        cls.driver = create_selenium_driver()
-        cls.driver.maximize_window()
-
-    @classmethod
-    def tearDownClass(cls):
-        """ Clean up webdriver driver """
-
-        cls.driver.quit()
-        super(SeleniumTestCase, cls).tearDownClass()
-
-    def get(self, url):
-        """
-        Selenium requires absolute URLs, so convert Django URLs returned
-        by resolve() or similar to absolute ones and get using the
-        webdriver instance.
-
-        url: a relative URL
-        """
-        abs_url = '%s%s' % (self.live_server_url, url)
-        self.driver.get(abs_url)
-
-    def find(self, selector):
-        """ Find single element by CSS selector """
-        return self.driver.find_element_by_css_selector(selector)
-
-    def find_all(self, selector):
-        """ Find all elements matching CSS selector """
-        return self.driver.find_elements_by_css_selector(selector)
-
-    def focused_element(self):
-        """ Return the element which currently has focus on the page """
-        return self.driver.switch_to.active_element
-
-    def wait_until_present(self, selector):
-        """ Wait until element matching CSS selector is on the page """
-        is_present = lambda driver: self.find(selector)
-        msg = 'An element matching "%s" should be on the page' % selector
-        element = Wait(self.driver).until(is_present, msg)
-        return element
-
-    def wait_until_visible(self, selector):
-        """ Wait until element matching CSS selector is visible on the page """
-        is_visible = lambda driver: self.find(selector).is_displayed()
-        msg = 'An element matching "%s" should be visible' % selector
-        Wait(self.driver).until(is_visible, msg)
-        return self.find(selector)
-
-    def wait_until_focused(self, selector):
-        """ Wait until element matching CSS selector has focus """
-        is_focused = \
-            lambda driver: self.find(selector) == self.focused_element()
-        msg = 'An element matching "%s" should be focused' % selector
-        Wait(self.driver).until(is_focused, msg)
-        return self.find(selector)
-
-    def enter_text(self, selector, value):
-        """ Insert text into element matching selector """
-        # note that keyup events don't occur until the element is clicked
-        # (in the case of <input type="text"...>, for example), so simulate
-        # user clicking the element before inserting text into it
-        field = self.click(selector)
-
-        field.send_keys(value)
-        return field
-
-    def click(self, selector):
-        """ Click on element which matches CSS selector """
-        element = self.wait_until_visible(selector)
-        element.click()
-        return element
-
-    def get_page_source(self):
-        """ Get raw HTML for the current page """
-        return self.driver.page_source
+class SeleniumTestCase(SeleniumTestCaseBase, StaticLiveServerTestCase):
+    pass

bitbake/lib/toaster/tests/browser/test_builddashboard_page.py

@@ -48,6 +48,10 @@ class TestBuildDashboardPage(SeleniumTestCase):
                                            started_on=now,
                                            completed_on=now)
 
+        self.build3 = Build.objects.create(project=project,
+                                           started_on=now,
+                                           completed_on=now)
+
         # exception
         msg1 = 'an exception was thrown'
         self.exception_message = LogMessage.objects.create(
@@ -71,6 +75,11 @@ class TestBuildDashboardPage(SeleniumTestCase):
             layer=layer, build=self.build1
         )
 
+        # non-image recipes related to a build, for testing the new custom
+        # image button
+        layer_version2 = Layer_Version.objects.create(layer=layer,
+            build=self.build3)
+
         # image recipes
         self.image_recipe1 = Recipe.objects.create(
             name='recipeA',
@@ -248,3 +257,14 @@ class TestBuildDashboardPage(SeleniumTestCase):
         ]
 
         self._check_labels_in_modal(modal, expected_recipes)
+
+    def test_new_custom_image_button_no_image(self):
+        """
+        Check that a build which builds non-image recipes doesn't show
+        the new custom image button on the dashboard.
+        """
+        self._get_build_dashboard(self.build3)
+        selector = '[data-role="new-custom-image-trigger"]'
+        self.assertFalse(self.element_exists(selector),
+            'new custom image button should not show for builds which ' \
+            'don\'t have any image recipes')

bitbake/lib/toaster/tests/browser/test_builddashboard_page_artifacts.py

@@ -0,0 +1,217 @@
+#! /usr/bin/env python
+# ex:ts=4:sw=4:sts=4:et
+# -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*-
+#
+# BitBake Toaster Implementation
+#
+# Copyright (C) 2013-2016 Intel Corporation
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License version 2 as
+# published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License along
+# with this program; if not, write to the Free Software Foundation, Inc.,
+# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+from django.core.urlresolvers import reverse
+from django.utils import timezone
+
+from tests.browser.selenium_helpers import SeleniumTestCase
+
+from orm.models import Project, Release, BitbakeVersion, Build, Target, Package
+from orm.models import Target_Image_File, TargetSDKFile, TargetKernelFile
+from orm.models import Target_Installed_Package
+
+class TestBuildDashboardPageArtifacts(SeleniumTestCase):
+    """ Tests for artifacts on the build dashboard /build/X """
+
+    def setUp(self):
+        bbv = BitbakeVersion.objects.create(name='bbv1', giturl='/tmp/',
+                                            branch='master', dirpath="")
+        release = Release.objects.create(name='release1',
+                                         bitbake_version=bbv)
+        self.project = Project.objects.create_project(name='test project',
+                                                      release=release)
+
+    def _get_build_dashboard(self, build):
+        """
+        Navigate to the build dashboard for build
+        """
+        url = reverse('builddashboard', args=(build.id,))
+        self.get(url)
+
+    def _has_build_artifacts_heading(self):
+        """
+        Check whether the "Build artifacts" heading is visible (True if it
+        is, False otherwise).
+        """
+        return self.element_exists('[data-heading="build-artifacts"]')
+
+    def _has_images_menu_option(self):
+        """
+        Try to get the "Images" list element from the left-hand menu in the
+        build dashboard, and return True if it is present, False otherwise.
+        """
+        return self.element_exists('li.nav-header[data-menu-heading="images"]')
+
+    def test_no_artifacts(self):
+        """
+        If a build produced no artifacts, the artifacts heading and images
+        menu option shouldn't show.
+        """
+        now = timezone.now()
+        build = Build.objects.create(project=self.project,
+            started_on=now, completed_on=now, outcome=Build.SUCCEEDED)
+
+        Target.objects.create(is_image=False, build=build, task='',
+            target='mpfr-native')
+
+        self._get_build_dashboard(build)
+
+        # check build artifacts heading
+        msg = 'Build artifacts heading should not be displayed for non-image' \
+            'builds'
+        self.assertFalse(self._has_build_artifacts_heading(), msg)
+
+        # check "Images" option in left-hand menu (should not be there)
+        msg = 'Images option should not be shown in left-hand menu'
+        self.assertFalse(self._has_images_menu_option(), msg)
+
+    def test_sdk_artifacts(self):
+        """
+        If a build produced SDK artifacts, they should be shown, but the section
+        for image files and the images menu option should be hidden.
+
+        The packages count and size should also be hidden.
+        """
+        now = timezone.now()
+        build = Build.objects.create(project=self.project,
+            started_on=now, completed_on=timezone.now(),
+            outcome=Build.SUCCEEDED)
+
+        target = Target.objects.create(is_image=True, build=build,
+            task='populate_sdk', target='core-image-minimal')
+
+        sdk_file1 = TargetSDKFile.objects.create(target=target,
+            file_size=100000,
+            file_name='/home/foo/core-image-minimal.toolchain.sh')
+
+        sdk_file2 = TargetSDKFile.objects.create(target=target,
+            file_size=120000,
+            file_name='/home/foo/x86_64.toolchain.sh')
+
+        self._get_build_dashboard(build)
+
+        # check build artifacts heading
+        msg = 'Build artifacts heading should be displayed for SDK ' \
+            'builds which generate artifacts'
+        self.assertTrue(self._has_build_artifacts_heading(), msg)
+
+        # check "Images" option in left-hand menu (should not be there)
+        msg = 'Images option should not be shown in left-hand menu for ' \
+            'builds which didn\'t generate an image file'
+        self.assertFalse(self._has_images_menu_option(), msg)
+
+        # check links to SDK artifacts
+        sdk_artifact_links = self.find_all('[data-links="sdk-artifacts"] li')
+        self.assertEqual(len(sdk_artifact_links), 2,
+            'should be links to 2 SDK artifacts')
+
+        # package count and size should not be visible, no link on
+        # target name
+        selector = '[data-value="target-package-count"]'
+        self.assertFalse(self.element_exists(selector),
+            'package count should not be shown for non-image builds')
+
+        selector = '[data-value="target-package-size"]'
+        self.assertFalse(self.element_exists(selector),
+            'package size should not be shown for non-image builds')
+
+        selector = '[data-link="target-packages"]'
+        self.assertFalse(self.element_exists(selector),
+            'link to target packages should not be on target heading')
+
+    def test_image_artifacts(self):
+        """
+        If a build produced image files, kernel artifacts, and manifests,
+        they should all be shown, as well as the image link in the left-hand
+        menu.
+
+        The packages count and size should be shown, with a link to the
+        package display page.
+        """
+        now = timezone.now()
+        build = Build.objects.create(project=self.project,
+            started_on=now, completed_on=timezone.now(),
+            outcome=Build.SUCCEEDED)
+
+        target = Target.objects.create(is_image=True, build=build,
+            task='', target='core-image-minimal',
+            license_manifest_path='/home/foo/license.manifest',
+            package_manifest_path='/home/foo/package.manifest')
+
+        image_file = Target_Image_File.objects.create(target=target,
+            file_name='/home/foo/core-image-minimal.ext4', file_size=9000)
+
+        kernel_file1 = TargetKernelFile.objects.create(target=target,
+            file_name='/home/foo/bzImage', file_size=2000)
+
+        kernel_file2 = TargetKernelFile.objects.create(target=target,
+            file_name='/home/foo/bzImage', file_size=2000)
+
+        package = Package.objects.create(build=build, name='foo', size=1024,
+            installed_name='foo1')
+        installed_package = Target_Installed_Package.objects.create(
+            target=target, package=package)
+
+        self._get_build_dashboard(build)
+
+        # check build artifacts heading
+        msg = 'Build artifacts heading should be displayed for image ' \
+            'builds'
+        self.assertTrue(self._has_build_artifacts_heading(), msg)
+
+        # check "Images" option in left-hand menu (should be there)
+        msg = 'Images option should be shown in left-hand menu for image builds'
+        self.assertTrue(self._has_images_menu_option(), msg)
+
+        # check link to image file
+        selector = '[data-links="image-artifacts"] li'
+        self.assertTrue(self.element_exists(selector),
+            'should be a link to the image file (selector %s)' % selector)
+
+        # check links to kernel artifacts
+        kernel_artifact_links = \
+            self.find_all('[data-links="kernel-artifacts"] li')
+        self.assertEqual(len(kernel_artifact_links), 2,
+            'should be links to 2 kernel artifacts')
+
+        # check manifest links
+        selector = 'a[data-link="license-manifest"]'
+        self.assertTrue(self.element_exists(selector),
+            'should be a link to the license manifest (selector %s)' % selector)
+
+        selector = 'a[data-link="package-manifest"]'
+        self.assertTrue(self.element_exists(selector),
+            'should be a link to the package manifest (selector %s)' % selector)
+
+        # check package count and size, link on target name
+        selector = '[data-value="target-package-count"]'
+        element = self.find(selector)
+        self.assertEquals(element.text, '1',
+            'package count should be shown for image builds')
+
+        selector = '[data-value="target-package-size"]'
+        element = self.find(selector)
+        self.assertEquals(element.text, '1.0 KB',
+            'package size should be shown for image builds')
+
+        selector = '[data-link="target-packages"]'
+        self.assertTrue(self.element_exists(selector),
+            'link to target packages should be on target heading')

bitbake/lib/toaster/tests/browser/test_layerdetails_page.py

@@ -0,0 +1,190 @@
+#! /usr/bin/env python
+# ex:ts=4:sw=4:sts=4:et
+# -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*-
+#
+# BitBake Toaster Implementation
+#
+# Copyright (C) 2013-2016 Intel Corporation
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License version 2 as
+# published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License along
+# with this program; if not, write to the Free Software Foundation, Inc.,
+# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+from django.core.urlresolvers import reverse
+from tests.browser.selenium_helpers import SeleniumTestCase
+
+from orm.models import Layer, Layer_Version, Project, LayerSource, Release
+from orm.models import BitbakeVersion
+
+from selenium.webdriver.support import expected_conditions as EC
+from selenium.webdriver.support.ui import WebDriverWait
+from selenium.webdriver.common.by import By
+
+
+class TestLayerDetailsPage(SeleniumTestCase):
+    """ Test layerdetails page works correctly """
+
+    def __init__(self, *args, **kwargs):
+        super(TestLayerDetailsPage, self).__init__(*args, **kwargs)
+
+        self.initial_values = None
+        self.url = None
+        self.imported_layer_version = None
+
+    def setUp(self):
+        release = Release.objects.create(
+            name='baz',
+            bitbake_version=BitbakeVersion.objects.create(name='v1')
+        )
+
+        # project to add new custom images to
+        self.project = Project.objects.create(name='foo', release=release)
+
+        layer_source = LayerSource.objects.create(
+            sourcetype=LayerSource.TYPE_IMPORTED)
+
+        name = "meta-imported"
+        vcs_url = "git://example.com/meta-imported"
+        subdir = "/layer"
+        gitrev = "d33d"
+        summary = "A imported layer"
+        description = "This was imported"
+
+        imported_layer = Layer.objects.create(name=name,
+                                              vcs_url=vcs_url,
+                                              summary=summary,
+                                              description=description)
+
+        self.imported_layer_version = Layer_Version.objects.create(
+            layer=imported_layer,
+            layer_source=layer_source,
+            branch=gitrev,
+            commit=gitrev,
+            dirpath=subdir,
+            project=self.project)
+
+        self.initial_values = [name, vcs_url, subdir, gitrev, summary,
+                               description]
+        self.url = reverse('layerdetails',
+                           args=(self.project.pk,
+                                 self.imported_layer_version.pk))
+
+    def test_edit_layerdetails(self):
+        """ Edit all the editable fields for the layer refresh the page and
+        check that the new values exist"""
+
+        self.get(self.url)
+
+        self.click("#add-remove-layer-btn")
+
+        # Open every edit box
+        for btn in self.find_all("dd .glyphicon-edit"):
+            btn.click()
+
+        self.wait_until_visible("dd input")
+
+        # Edit each value
+        for inputs in self.find_all("dd input[type=text]") + \
+                self.find_all("dd textarea"):
+            # ignore the tt inputs (twitter typeahead input)
+            if "tt-" in inputs.get_attribute("class"):
+                continue
+
+            value = inputs.get_attribute("value")
+
+            self.assertTrue(value in self.initial_values,
+                            "Expecting any of \"%s\"but got \"%s\"" %
+                            (self.initial_values, value))
+
+            inputs.send_keys("-edited")
+
+        for save_btn in self.find_all(".change-btn"):
+            save_btn.click()
+
+        # Refresh the page to see if the new values are returned
+        self.get(self.url)
+
+        new_values = ["%s-edited" % old_val
+                      for old_val in self.initial_values]
+
+        for inputs in self.find_all("dd input[type=text]") + \
+                self.find_all("dd textarea"):
+            # ignore the tt inputs (twitter typeahead input)
+            if "tt-" in inputs.get_attribute("class"):
+                continue
+
+            value = inputs.get_attribute("value")
+
+            self.assertTrue(value in new_values,
+                            "Expecting any of \"%s\"but got \"%s\"" %
+                            (self.initial_values, value))
+
+    def test_delete_layer(self):
+        """ Delete the layer """
+
+        self.get(self.url)
+
+        # Wait for the tables to load to avoid a race condition where the
+        # toaster tables have made an async request. If the layer is deleted
+        # before the request finishes it will cause an exception and fail this
+        # test.
+        wait = WebDriverWait(self.driver, 30)
+
+        wait.until(EC.text_to_be_present_in_element(
+            (By.CLASS_NAME,
+             "table-count-recipestable"), "0"))
+
+        wait.until(EC.text_to_be_present_in_element(
+            (By.CLASS_NAME,
+             "table-count-machinestable"), "0"))
+
+        self.click('a[data-target="#delete-layer-modal"]')
+        self.wait_until_visible("#delete-layer-modal")
+        self.click("#layer-delete-confirmed")
+
+        notification = self.wait_until_visible("#change-notification-msg")
+        expected_text = "You have deleted 1 layer from your project: %s" % \
+            self.imported_layer_version.layer.name
+
+        self.assertTrue(expected_text in notification.text,
+                        "Expected notification text \"%s\" not found instead"
+                        "it was \"%s\"" %
+                        (expected_text, notification.text))
+
+    def test_addrm_to_project(self):
+        self.get(self.url)
+
+        # Add the layer
+        self.click("#add-remove-layer-btn")
+
+        notification = self.wait_until_visible("#change-notification-msg")
+
+        expected_text = "You have added 1 layer to your project: %s" % \
+            self.imported_layer_version.layer.name
+
+        self.assertTrue(expected_text in notification.text,
+                        "Expected notification text %s not found was "
+                        " \"%s\" instead" %
+                        (expected_text, notification.text))
+
+        # Remove the layer
+        self.click("#add-remove-layer-btn")
+
+        notification = self.wait_until_visible("#change-notification-msg")
+
+        expected_text = "You have removed 1 layer from your project: %s" % \
+            self.imported_layer_version.layer.name
+
+        self.assertTrue(expected_text in notification.text,
+                        "Expected notification text %s not found was "
+                        " \"%s\" instead" %
+                        (expected_text, notification.text))

bitbake/lib/toaster/tests/browser/test_project_config_page.py

@@ -31,6 +31,9 @@ class TestProjectConfigsPage(SeleniumTestCase):
     """ Test data at /project/X/builds is displayed correctly """
 
     PROJECT_NAME = 'test project'
+    INVALID_PATH_START_TEXT = 'The directory path should either start with a /'
+    INVALID_PATH_CHAR_TEXT = 'The directory path cannot include spaces or ' \
+        'any of these characters'
 
     def setUp(self):
         bbv = BitbakeVersion.objects.create(name='bbv1', giturl='/tmp/',
@@ -116,3 +119,113 @@ class TestProjectConfigsPage(SeleniumTestCase):
                checkbox.click()
                self.assertTrue(("cpio" not in element.text),
                                "Image still present in the textbox")
+
+    def test_set_download_dir(self):
+        """
+        Validate the allowed and disallowed types in the directory field for
+        DL_DIR
+        """
+
+        ProjectVariable.objects.get_or_create(project=self.project1,
+            name='DL_DIR')
+        url = reverse('projectconf', args=(self.project1.id,))
+        self.get(url)
+
+        # activate the input to edit download dir
+        self.click('#change-dl_dir-icon')
+        self.wait_until_visible('#new-dl_dir')
+
+        # downloads dir path doesn't start with / or ${...}
+        self.enter_text('#new-dl_dir', 'home/foo')
+        element = self.wait_until_visible('#hintError-initialChar-dl_dir')
+
+        msg = 'downloads directory path starts with invalid character but ' \
+            'treated as valid'
+        self.assertTrue((self.INVALID_PATH_START_TEXT in element.text), msg)
+
+        # downloads dir path has a space
+        self.driver.find_element_by_id('new-dl_dir').clear()
+        self.enter_text('#new-dl_dir', '/foo/bar a')
+
+        element = self.wait_until_visible('#hintError-dl_dir')
+        msg = 'downloads directory path characters invalid but treated as valid'
+        self.assertTrue((self.INVALID_PATH_CHAR_TEXT in element.text), msg)
+
+        # downloads dir path starts with ${...} but has a space
+        self.driver.find_element_by_id('new-dl_dir').clear()
+        self.enter_text('#new-dl_dir', '${TOPDIR}/down foo')
+
+        element = self.wait_until_visible('#hintError-dl_dir')
+        msg = 'downloads directory path characters invalid but treated as valid'
+        self.assertTrue((self.INVALID_PATH_CHAR_TEXT in element.text), msg)
+
+        # downloads dir path starts with /
+        self.driver.find_element_by_id('new-dl_dir').clear()
+        self.enter_text('#new-dl_dir', '/bar/foo')
+
+        hidden_element = self.driver.find_element_by_id('hintError-dl_dir')
+        self.assertEqual(hidden_element.is_displayed(), False,
+            'downloads directory path valid but treated as invalid')
+
+        # downloads dir path starts with ${...}
+        self.driver.find_element_by_id('new-dl_dir').clear()
+        self.enter_text('#new-dl_dir', '${TOPDIR}/down')
+
+        hidden_element = self.driver.find_element_by_id('hintError-dl_dir')
+        self.assertEqual(hidden_element.is_displayed(), False,
+            'downloads directory path valid but treated as invalid')
+
+    def test_set_sstate_dir(self):
+        """
+        Validate the allowed and disallowed types in the directory field for
+        SSTATE_DIR
+        """
+
+        ProjectVariable.objects.get_or_create(project=self.project1,
+            name='SSTATE_DIR')
+        url = reverse('projectconf', args=(self.project1.id,))
+        self.get(url)
+
+        self.click('#change-sstate_dir-icon')
+
+        self.wait_until_visible('#new-sstate_dir')
+
+        # path doesn't start with / or ${...}
+        self.enter_text('#new-sstate_dir', 'home/foo')
+        element = self.wait_until_visible('#hintError-initialChar-sstate_dir')
+
+        msg = 'sstate directory path starts with invalid character but ' \
+            'treated as valid'
+        self.assertTrue((self.INVALID_PATH_START_TEXT in element.text), msg)
+
+        # path has a space
+        self.driver.find_element_by_id('new-sstate_dir').clear()
+        self.enter_text('#new-sstate_dir', '/foo/bar a')
+
+        element = self.wait_until_visible('#hintError-sstate_dir')
+        msg = 'sstate directory path characters invalid but treated as valid'
+        self.assertTrue((self.INVALID_PATH_CHAR_TEXT in element.text), msg)
+
+        # path starts with ${...} but has a space
+        self.driver.find_element_by_id('new-sstate_dir').clear()
+        self.enter_text('#new-sstate_dir', '${TOPDIR}/down foo')
+
+        element = self.wait_until_visible('#hintError-sstate_dir')
+        msg = 'sstate directory path characters invalid but treated as valid'
+        self.assertTrue((self.INVALID_PATH_CHAR_TEXT in element.text), msg)
+
+        # path starts with /
+        self.driver.find_element_by_id('new-sstate_dir').clear()
+        self.enter_text('#new-sstate_dir', '/bar/foo')
+
+        hidden_element = self.driver.find_element_by_id('hintError-sstate_dir')
+        self.assertEqual(hidden_element.is_displayed(), False,
+            'sstate directory path valid but treated as invalid')
+
+        # paths starts with ${...}
+        self.driver.find_element_by_id('new-sstate_dir').clear()
+        self.enter_text('#new-sstate_dir', '${TOPDIR}/down')
+
+        hidden_element = self.driver.find_element_by_id('hintError-sstate_dir')
+        self.assertEqual(hidden_element.is_displayed(), False,
+            'sstate directory path valid but treated as invalid')

bitbake/lib/toaster/toastergui/api.py

@@ -20,11 +20,17 @@
 # Temporary home for the UI's misc API
 import re
 
-from orm.models import Project, ProjectTarget, Build
+from orm.models import Project, ProjectTarget, Build, Layer_Version
+from orm.models import LayerVersionDependency, LayerSource, ProjectLayer
 from bldcontrol.models import BuildRequest
 from bldcontrol import bbcontroller
 from django.http import HttpResponse, JsonResponse
 from django.views.generic import View
+from django.core.urlresolvers import reverse
+
+
+def error_response(error):
+    return JsonResponse({"error": error})
 
 
 class XhrBuildRequest(View):
@@ -78,20 +84,24 @@ class XhrBuildRequest(View):
                     br.save()
 
                 except BuildRequest.DoesNotExist:
-                    return JsonResponse({'error':'No such build id %s' % i})
+                    return error_response('No such build id %s' % i)
 
-            return JsonResponse({'error': 'ok'})
+            return error_response('ok')
 
         if 'buildDelete' in request.POST:
             for i in request.POST['buildDelete'].strip().split(" "):
                 try:
-                    BuildRequest.objects.select_for_update().get(project = project, pk = i, state__lte = BuildRequest.REQ_DELETED).delete()
+                    BuildRequest.objects.select_for_update().get(
+                        project=project,
+                        pk=i,
+                        state__lte=BuildRequest.REQ_DELETED).delete()
+
                 except BuildRequest.DoesNotExist:
                     pass
-            return JsonResponse({'error': 'ok' })
+            return error_response("ok")
 
         if 'targets' in request.POST:
-            ProjectTarget.objects.filter(project = project).delete()
+            ProjectTarget.objects.filter(project=project).delete()
             s = str(request.POST['targets'])
             for t in re.sub(r'[;%|"]', '', s).split(" "):
                 if ":" in t:
@@ -99,13 +109,104 @@ class XhrBuildRequest(View):
                 else:
                     target = t
                     task = ""
-                ProjectTarget.objects.create(project = project,
-                                             target = target,
-                                             task = task)
+                ProjectTarget.objects.create(project=project,
+                                             target=target,
+                                             task=task)
             project.schedule_build()
 
-            return JsonResponse({'error': 'ok' })
+            return error_response('ok')
 
         response = HttpResponse()
         response.status_code = 500
         return response
+
+
+class XhrLayer(View):
+    """ Get and Update Layer information """
+
+    def post(self, request, *args, **kwargs):
+        """
+          Update a layer
+
+          Entry point: /xhr_layer/<layerversion_id>
+          Method: POST
+
+          Args:
+              vcs_url, dirpath, commit, up_branch, summary, description
+
+              add_dep = append a layerversion_id as a dependency
+              rm_dep = remove a layerversion_id as a depedency
+          Returns:
+              {"error": "ok"}
+            or
+              {"error": <error message>}
+        """
+
+        try:
+            # We currently only allow Imported layers to be edited
+            layer_version = Layer_Version.objects.get(
+                id=kwargs['layerversion_id'],
+                project=kwargs['pid'],
+                layer_source__sourcetype=LayerSource.TYPE_IMPORTED)
+
+        except Layer_Version.DoesNotExist:
+            return error_response("Cannot find imported layer to update")
+
+        if "vcs_url" in request.POST:
+            layer_version.layer.vcs_url = request.POST["vcs_url"]
+        if "dirpath" in request.POST:
+            layer_version.dirpath = request.POST["dirpath"]
+        if "commit" in request.POST:
+            layer_version.commit = request.POST["commit"]
+            layer_version.branch = request.POST["commit"]
+        if "up_branch" in request.POST:
+            layer_version.up_branch_id = int(request.POST["up_branch"])
+        if "summary" in request.POST:
+            layer_version.layer.summary = request.POST["summary"]
+        if "description" in request.POST:
+            layer_version.layer.description = request.POST["description"]
+
+        if "add_dep" in request.POST:
+            lvd = LayerVersionDependency(
+                layer_version=layer_version,
+                depends_on_id=request.POST["add_dep"])
+            lvd.save()
+
+        if "rm_dep" in request.POST:
+            rm_dep = LayerVersionDependency.objects.get(
+                layer_version=layer_version,
+                depends_on_id=request.POST["rm_dep"])
+            rm_dep.delete()
+
+        try:
+            layer_version.layer.save()
+            layer_version.save()
+        except Exception as e:
+            return error_response("Could not update layer version entry: %s"
+                                  % e)
+
+        return JsonResponse({"error": "ok"})
+
+    def delete(self, request, *args, **kwargs):
+        try:
+            # We currently only allow Imported layers to be deleted
+            layer_version = Layer_Version.objects.get(
+                id=kwargs['layerversion_id'],
+                project=kwargs['pid'],
+                layer_source__sourcetype=LayerSource.TYPE_IMPORTED)
+        except Layer_Version.DoesNotExist:
+            return error_response("Cannot find imported layer to delete")
+
+        try:
+            ProjectLayer.objects.get(project=kwargs['pid'],
+                                     layercommit=layer_version).delete()
+        except ProjectLayer.DoesNotExist:
+            pass
+
+        layer_version.layer.delete()
+        layer_version.delete()
+
+        return JsonResponse({
+            "error": "ok",
+            "redirect": reverse('project', args=(kwargs['pid'],))
+        })

bitbake/lib/toaster/toastergui/static/js/layerBtn.js

@@ -55,8 +55,8 @@ function layerBtnsInit() {
     });
   });
 
-  $(".build-recipe-btn").unbind('click');
-  $(".build-recipe-btn").click(function(e){
+  $("td .build-recipe-btn").unbind('click');
+  $("td .build-recipe-btn").click(function(e){
     e.preventDefault();
     var recipe = $(this).data('recipe-name');
 

bitbake/lib/toaster/toastergui/static/js/layerdetails.js

@@ -397,6 +397,26 @@ function layerDetailsPageInit (ctx) {
     $(this).parents("form").submit();
   });
 
+  $("#layer-delete-confirmed").click(function(){
+    $.cookie("layer-deleted", ctx.layerVersion.name, { path: '/'});
+
+    $.ajax({
+        type: "DELETE",
+        url: ctx.xhrUpdateLayerUrl,
+        headers: { 'X-CSRFToken' : $.cookie('csrftoken')},
+        success: function(data) {
+          if (data.error != "ok") {
+            console.warn(data.error);
+          } else {
+            window.location = data.redirect + "?notify=layer-deleted";
+          }
+        },
+        error: function(data) {
+          console.warn("Call failed");
+          console.warn(data);
+        }
+    });
+  });
 
   layerDepsList.find(".glyphicon-trash").click(layerDepRemoveClick);
   layerDepsList.find("a").tooltip();

bitbake/lib/toaster/toastergui/static/js/projectpage.js

@@ -58,12 +58,30 @@ function projectPageInit(ctx) {
         case 'layer-imported':
           layerImportedNotification();
           break;
+        case 'layer-deleted':
+          layerDeletedNotification();
         default:
           break;
       }
     }
   })();
 
+  /* Layer deleted notification */
+  function layerDeletedNotification(){
+    var layer = $.cookie("layer-deleted");
+
+    if (!layer)
+      return;
+
+    var message = "You have deleted <strong>1</strong> layer from your ";
+    message += "project: <strong>" + layer + "</strong>";
+
+    libtoaster.showChangeNotification(message);
+
+    $.removeCookie("layer-deleted", { path: "/"});
+  }
+
+
   /* Layer imported notification */
   function layerImportedNotification(){
     var imported = $.cookie("layer-imported-alert");

bitbake/lib/toaster/toastergui/templates/basebuildpage.html

@@ -33,7 +33,7 @@
 
         $("#build-menu li a").each(function(){
           /* Set the page active state in the Build menu */
-          var currentUrl = window.location.href.split('?')[0]; 
+          var currentUrl = window.location.href.split('?')[0];
           if (currentUrl === $(this).prop("href")){
             $(this).parent().addClass("active");
           } else {
@@ -62,11 +62,13 @@
           {% endif %} >
           <a  href="{% url 'builddashboard' build.pk %}">Build summary</a>
         </li>
-        {% if build.target_set.all.0.is_image and build.outcome == 0 %}
-          <li class="nav-header">Images</li>
+        {% if build.has_images and build.outcome == build.SUCCEEDED %}
+          <li class="nav-header" data-menu-heading="images">Images</li>
           {% block nav-target %}
             {% for t in build.get_sorted_target_list %}
-            <li id="menu-{{t.target}}"><a href="{% url 'target' build.pk t.pk %}">{{t.target}}</a><li>
+              {% if t.has_images %}
+                <li id="menu-{{t.target}}"><a href="{% url 'target' build.pk t.pk %}">{{t.target}}</a><li>
+              {% endif %}
             {% endfor %}
           {% endblock %}
         {% endif %}
@@ -115,30 +117,32 @@
         {% endwith %}
 
           <!-- new custom image from image recipe in this build -->
-          <button class="btn btn-default btn-block navbar-btn" data-role="new-custom-image-trigger">New custom image</button>
-          {% include 'newcustomimage_modal.html' %}
-          <script>
-            // imageRecipes includes both custom image recipes and built-in
-            // image recipes, any of which can be used as the basis for a
-            // new custom image
-            var imageRecipes = {{ build.get_image_recipes | objects_to_dictionaries:"id,name" | json }};
+          {% if build.has_image_recipes %}
+            <button class="btn btn-default btn-block navbar-btn" data-role="new-custom-image-trigger">New custom image</button>
+            {% include 'newcustomimage_modal.html' %}
+            <script>
+              // imageRecipes includes both custom image recipes and built-in
+              // image recipes, any of which can be used as the basis for a
+              // new custom image
+              var imageRecipes = {{ build.get_image_recipes | objects_to_dictionaries:"id,name" | json }};
 
-            $(document).ready(function () {
-              var newCustomImageModal = $('#new-custom-image-modal');
-              var newCustomImageTrigger = $('[data-role="new-custom-image-trigger"]');
+              $(document).ready(function () {
+                var newCustomImageModal = $('#new-custom-image-modal');
+                var newCustomImageTrigger = $('[data-role="new-custom-image-trigger"]');
 
-              // show create new custom image modal to select an image built
-              // during this build as the basis for the custom recipe
-              newCustomImageTrigger.click(function () {
-                if (!imageRecipes.length) {
-                  return;
-                }
+                // show create new custom image modal to select an image built
+                // during this build as the basis for the custom recipe
+                newCustomImageTrigger.click(function () {
+                  if (!imageRecipes.length) {
+                    return;
+                  }
 
-                newCustomImageModalSetRecipes(imageRecipes);
-                newCustomImageModal.modal('show');
+                  newCustomImageModalSetRecipes(imageRecipes);
+                  newCustomImageModal.modal('show');
+                });
               });
-            });
-          </script>
+            </script>
+          {% endif %}
       </ul>
 
     </div>

bitbake/lib/toaster/toastergui/templates/builddashboard.html

@@ -1,8 +1,9 @@
 {% extends "basebuildpage.html" %}
 {% load humanize %}
 {% load projecttags %}
+{% load field_values_filter %}
 
-{% block title %} {{build.target_set.all|dictsort:"target"|join:", "}} {{build.machine}} - {{build.project.name}} - Toaster {% endblock %}
+{% block title %} {{build.get_sorted_target_list|field_values:"target"|join:", "}} {{build.machine}} - {{build.project.name}} - Toaster {% endblock %}
 {% block parentbreadcrumb %}
 {% if build.get_sorted_target_list.count > 0 %}
   {{build.get_sorted_target_list.0.target}}
@@ -15,7 +16,7 @@
 <!-- page title -->
 <div class="col-md-10">
  <div class="page-header build-data">
-     <h1>{{build.target_set.all|dictsort:"target"|join:", "}} {{build.machine}}</h1>
+     <h1>{{build.get_sorted_target_list|field_values:"target"|join:", "}} {{build.machine}}</h1>
  </div>
 
 <!-- build result bar -->
@@ -68,52 +69,58 @@
 
 {%if build.outcome == build.SUCCEEDED%}
 <!-- built images -->
-{% if hasImages %}
-    <h2>Images</h2>
+  {% if hasArtifacts %}
+    <h2 data-heading="build-artifacts">Build artifacts</h2>
     {% for target in targets %}
         {% if target.target.is_image %}
-    <div class="well well-transparent dashboard-section">
-        <h3><a href="{% url 'target' build.pk target.target.pk %}">{{target.target}}</a></h3>
-        <dl class="dl-horizontal">
-            <dt>Packages included</dt>
-            <dd><a href="{% url 'target' build.pk target.target.pk %}">{{target.npkg}}</a></dd>
-            <dt>Total package size</dt>
-            <dd>{{target.pkgsz|filtered_filesizeformat}}</dd>
-                        {% if target.targetHasNoImages %}
-                </dl>
-                <div class="row">
-                  <div class="col-md-7">
-                    <div class="alert alert-info">
-                      <p>
-                      <strong>This build did not create any image files</strong>
-                      </p>
-                      <p>
-                      This is probably because valid image and license manifest
-                      files from a previous build already exist in your
-                      <code>build/tmp/deploy</code>
-                      directory. You can
-                      also <a href="{% url 'target' build.pk target.target.pk %}">view the
-                        license manifest information</a> in Toaster.
-                      </p>
-                    </div>
-                  </div>
-                </div>
+    <div class="well well-transparent dashboard-section" data-artifacts-for-target="{{target.target.pk}}">
+        {% if target.npkg > 0 %}
+            <h3>
+                <a href="{% url 'target' build.pk target.target.pk %}" data-link="target-packages">
+                  {{target.target.target}}
+                </a>
+            </h3>
+            <dl class="dl-horizontal">
+                <dt>Packages included</dt>
+                <dd>
+                    <a href="{% url 'target' build.pk target.target.pk %}">
+                        <span data-value="target-package-count">{{target.npkg}}</span>
+                    </a>
+                </dd>
+                <dt>Total package size</dt>
+                <dd>
+                    <span data-value="target-package-size">{{target.pkgsz|filtered_filesizeformat}}</span>
+                </dd>
+            </dl>
         {% else %}
-            <dt>
-                <span class="glyphicon glyphicon-question-sign get-help" title="The location in disk of the license manifest, a document listing all packages installed in your image and their licenses"></span>
+            <h3>{{target.target.target}}</h3>
+        {% endif %}
 
-                License manifest
+        {% if target.targetHasImages %}
+          <dl class="dl-horizontal">
+            <dt>
+              Manifests
             </dt>
+
             <dd>
-                <a href="{% url 'target' build.pk target.target.pk %}">View in Toaster</a> |
-                <a href="{% url 'build_artifact' build.pk 'licensemanifest' target.target.pk %}">Download</a></dd>
+              <a data-link="license-manifest" href="{% url 'build_artifact' build.pk 'licensemanifest' target.target.pk %}">License manifest</a>
+            </dd>
+
+            {% if target.target.package_manifest_path %}
+              <dd>
+                <a data-link="package-manifest" href="{% url 'build_artifact' build.pk 'packagemanifest' target.target.pk %}">Package manifest</a>
+              </dd>
+            {% endif %}
+          </dl>
+
+          <dl class="dl-horizontal">
             <dt>
                 <span class="glyphicon glyphicon-question-sign get-help" title="Image files are stored in <code>build/tmp/deploy/images/</code>"></span>
                 Image files
             </dt>
             <dd>
-                <ul class="list-unstyled">
-                    {% for i in target.imageFiles %}
+                <ul class="list-unstyled" data-links="image-artifacts">
+                    {% for i in target.imageFiles|dictsort:"suffix" %}
                         <li>
                             <a href="{% url 'build_artifact' build.pk 'imagefile' i.id %}">
                                 {{i.suffix}}
@@ -123,12 +130,42 @@
                     {% endfor %}
                 </ul>
             </dd>
-        </dl>
+            <dt>
+                Kernel artifacts
+            </dt>
+            <dd>
+                <ul class="list-unstyled" data-links="kernel-artifacts">
+                    {% for artifact in target.target_kernel_artifacts|dictsort:"basename" %}
+                        <li>
+                            <a href="{% url 'build_artifact' build.id 'targetkernelartifact' artifact.id %}">{{artifact.basename}}</a>
+                            ({{artifact.file_size|filtered_filesizeformat}})
+                        </li>
+                    {% endfor %}
+                  </ul>
+            </dd>
+          </dl>
+        {% endif %}
+        {% if target.target_sdk_artifacts_count > 0 %}
+          <dl class="dl-horizontal">
+            <dt>
+              SDK artifacts
+            </dt>
+            <dd>
+              <ul class="list-unstyled" data-links="sdk-artifacts">
+                {% for artifact in target.target_sdk_artifacts|dictsort:"basename" %}
+                <li>
+                  <a href="{% url 'build_artifact' build.id 'targetsdkartifact' artifact.id %}">{{artifact.basename}}</a>
+                  ({{artifact.file_size|filtered_filesizeformat}})
+                </li>
+                {% endfor %}
+              </ul>
+            </dd>
+          </dl>
         {% endif %}
     </div>
         {% endif %}
     {% endfor %}
-{% endif %}
+  {% endif %}
 
 {%else%}
 <!-- error dump -->
@@ -261,7 +298,7 @@
         if (location.href.search('#warnings') > -1) {
             $('#warning-info').addClass('in');
         }
-	
+
 	//show warnings section when requested from the build outcome
 	$(".show-warnings").click(function() {
 	    $('#warning-info').addClass('in');

bitbake/lib/toaster/toastergui/templates/layerdetails.html

@@ -6,6 +6,20 @@
 {% block title %} {{layerversion.layer.name}} - {{project.name}} - Toaster {% endblock %}
 {% block pagecontent %}
 
+<div id="delete-layer-modal" class="modal fade" tabindex="-1" role="dialog">
+  <div class="modal-dialog" role="document">
+    <div class="modal-content">
+      <div class="modal-body">
+		  Are you sure you want to delete the <strong>{{layerversion.layer.name}}</strong> layer?
+      </div>
+      <div class="modal-footer">
+        <button type="button" id="layer-delete-confirmed" class="btn btn-primary">Delete</button>
+        <button type="button" class="btn btn-default btn-link" data-dismiss="modal">Cancel</button>
+      </div>
+    </div>
+  </div>
+</div>
+
 <div class="row">
   <div class="col-md-12">
     <ul class="breadcrumb">
@@ -44,7 +58,7 @@
 
     $(document).ready(function(){
       var ctx = {
-        xhrUpdateLayerUrl : "{% url 'xhr_updatelayer' %}",
+        xhrUpdateLayerUrl : "{% url 'xhr_layer' project.id layerversion.pk %}",
         layerVersion : {
           name : "{{layerversion.layer.name}}",
           id : {{layerversion.id}},
@@ -271,6 +285,10 @@
             </dd>
             {% endif %}
           </dl>
+          {% if layerversion.layer_source_id and layerversion.layer_source.sourcetype == layerversion.layer_source.TYPE_IMPORTED %}
+          <i class="icon-trash text-danger"></i>
+          <a href="#delete-layer-modal"  role="button" class="text-danger" data-toggle="modal" data-target="#delete-layer-modal">Delete {{layerversion.layer.name}}</a>
+          {% endif %}
         </div>
       </div>
     </div>

bitbake/lib/toaster/toastergui/templates/mrb_section.html

@@ -3,6 +3,7 @@
 {% load project_url_tag %}
 {% load humanize %}
 <script src="{% static 'js/mrbsection.js' %}"></script>
+
 <script>
   $(document).ready(function () {
     var ctx = {
@@ -18,165 +19,163 @@
   });
 </script>
 
-{%if mru %}
+{% if mru %}
 
-  {%if mrb_type == 'project' %}
-  <h2>
-    Latest project builds
+  {% if mrb_type == 'project' %}
+    <h2>
+      Latest project builds
 
-    {% if project.is_default %}
-    <span class="glyphicon glyphicon-question-sign get-help heading-help" title="" data-original-title="Builds in this project cannot be started from Toaster: they are started from the command line"></span>
-    {% endif %}
-  </h2>
+      {% if project.is_default %}
+        <span class="glyphicon glyphicon-question-sign get-help heading-help" data-original-title="Builds in this project cannot be started from Toaster: they are started from the command line"></span>
+        {% endif %}
+    </h2>
   {% else %}
     <div class="page-header">
-      <h1>
-      Latest builds
-      </h1>
+      <h1>Latest builds</h1>
     </div>
   {% endif %}
+
   <div id="latest-builds">
-  {% for build in mru %}
-  <div data-latest-build-result="{{ build.id }}" class="alert build-result {%if build.outcome == build.SUCCEEDED%}alert-success{%elif build.outcome == build.FAILED%}alert-danger{%else%}alert-info{%endif%}">
+    {% for build in mru %}
+      <div data-latest-build-result="{{build.id}}" class="alert build-result {% if build.outcome == build.SUCCEEDED %}alert-success{% elif build.outcome == build.FAILED %}alert-danger{% else %}alert-info{% endif %}">
         {% if mrb_type != 'project' %}
-        <div class="row project-name">
-          <div class="col-md-12">
-            <small>
-              <a class="alert-link text-uppercase" href={% project_url build.project %}>{{build.project.name}}</a>
-            </small>
+          <div class="row project-name">
+            <div class="col-md-12">
+              <small>
+                <a class="alert-link text-uppercase" href={% project_url build.project %}>{{build.project.name}}</a>
+              </small>
+            </div>
           </div>
-        </div>
         {% endif %}
+
         <div class="row">
           <div class="col-md-3">
-    {%if build.outcome == build.SUCCEEDED or build.outcome == build.FAILED %}
-              <a href="{%url 'builddashboard' build.pk%}" class="alert-link">
-    {% endif %}
+            {% if build.outcome == build.SUCCEEDED or build.outcome == build.FAILED %}
+              <a href="{% url 'builddashboard' build.pk %}" class="alert-link">
+            {% endif %}
+
             {% if build.target_set.all.count > 0 %}
-                <span data-toggle="tooltip"
-                  {% if build.target_set.all.count > 1 %}
-                    {{build.get_sorted_target_list.0.target}}
-                    title="Recipes:
-                    {% for target in build.get_sorted_target_list %}
-                        {% if target.task %}
-                            {{target.target}}:{{target.task}}
-                        {% else %}
-                            {{target.target}}
-                        {% endif %}
-                    {% endfor %}"
-                  {% endif %}
-                >
-                {% if build.target_set.all.0.task %}
-                    {{build.get_sorted_target_list.0.target}}:{{build.target_set.all.0.task}}
-                {% else %}
-                    {{build.get_sorted_target_list.0.target}}
-                {% endif %}
+              <span data-toggle="tooltip"
                 {% if build.target_set.all.count > 1 %}
-                    (+{{build.target_set.all.count|add:"-1"}})
+                  {{build.get_sorted_target_list.0.target}}
+                  title="Recipes:
+                  {% for target in build.get_sorted_target_list %}
+                    {% if target.task %}
+                      {{target.target}}:{{target.task}}
+                    {% else %}
+                      {{target.target}}
+                    {% endif %}
+                  {% endfor %}"
                 {% endif %}
-                </span>
-             {% endif %}
-    {%if build.outcome == build.SUCCEEDED or build.outcome == build.FAILED %}
-                </a>
-    {% endif %}
-            </div>
-    {% if build.outcome == build.SUCCEEDED or build.outcome == build.FAILED %}
-            <div class="col-md-2">
-                {% if build.completed_on|format_build_date  %}
-                    {{ build.completed_on|date:'d/m/y H:i' }}
+              >
+                {% if build.target_set.all.0.task %}
+                  {{build.get_sorted_target_list.0.target}}:{{build.target_set.all.0.task}}
                 {% else %}
-                    {{ build.completed_on|date:'H:i' }}
+                  {{build.get_sorted_target_list.0.target}}
                 {% endif %}
-            </div>
-    {% endif %}
-    {%if build.outcome == build.SUCCEEDED or build.outcome == build.FAILED %}
+
+                {% if build.target_set.all.count > 1 %}
+                  (+{{build.target_set.all.count|add:"-1"}})
+                {% endif %}
+              </span>
+            {% endif %}
+            {% if build.outcome == build.SUCCEEDED or build.outcome == build.FAILED %}
+              </a>
+            {% endif %}
+          </div>
+
+          {% if build.outcome == build.SUCCEEDED or build.outcome == build.FAILED %}
             <div class="col-md-2">
-      {% if  build.errors.count %}
+              {% if build.completed_on|format_build_date  %}
+                {{build.completed_on|date:'d/m/y H:i'}}
+              {% else %}
+                {{ build.completed_on|date:'H:i' }}
+              {% endif %}
+            </div>
+          {% endif %}
+
+          {% if build.outcome == build.SUCCEEDED or build.outcome == build.FAILED %}
+            <div class="col-md-2">
+              {% if build.errors.count %}
                 <span class="glyphicon glyphicon-minus-sign"></span>
-                <a href="{%url 'builddashboard' build.pk%}#errors" class="alert-link">{{build.errors.count}} error{{build.errors.count|pluralize}}</a>
-      {% endif %}
+                <a href="{%url 'builddashboard' build.pk%}#errors" class="alert-link">
+                  {{build.errors.count}} error{{build.errors.count|pluralize}}
+                </a>
+              {% endif %}
             </div>
+
             <div class="col-md-2">
-      {% if  build.warnings.count %}
+              {% if build.warnings.count %}
                 <span class="glyphicon glyphicon-warning-sign build-warnings"></span>
-                <a href="{%url 'builddashboard' build.pk%}#warnings" class="alert-link build-warnings">{{build.warnings.count}} warning{{build.warnings.count|pluralize}}</a>
-      {% endif %}
+                <a href="{%url 'builddashboard' build.pk%}#warnings" class="alert-link build-warnings">
+                  {{build.warnings.count}} warning{{build.warnings.count|pluralize}}
+                </a>
+              {% endif %}
             </div>
+
             <div class="col-md-3">
               Build time: <a class="alert-link" href="{% url 'buildtime' build.pk %}">{{ build.timespent_seconds|sectohms }}
               </a>
+
               {% if build.project.is_default %}
-                  <span class="pull-right glyphicon glyphicon-question-sign get-help
-                  {% if build.outcome == build.SUCCEEDED %}
-                      get-help-green
-                  {% elif build.outcome == build.FAILED %}
-                      get-help-red
-                  {% else %}
-                      get-help-blue
-                  {% endif %}
-                  " title="Builds in this project cannot be started from Toaster: they are started from the command line">
-                  </span>
+                <span class="pull-right glyphicon glyphicon-question-sign get-help {% if build.outcome == build.SUCCEEDED %}get-help-green{% elif build.outcome == build.FAILED %}get-help-red{% else %}get-help-blue{% endif %}"
+                title="Builds in this project cannot be started from Toaster: they are started from the command line">
+                </span>
               {% else %}
-                  <a href="#" class="run-again-btn alert-link
-                      {% if build.outcome == build.SUCCEEDED %}
-                          success
-                      {% elif build.outcome == build.FAILED %}
-                          danger
-                      {% else %}
-                          info
-                      {%endif%}
-                      pull-right"
-                      data-request-url="{% url 'xhr_buildrequest' build.project.pk %}"
-                      data-target='{{build.target_set.all|get_tasks|json}}'>
-                      <span class="glyphicon glyphicon-repeat"></span>
-                      Rebuild
-                  </a>
+                <a href="#" class="run-again-btn alert-link {% if build.outcome == build.SUCCEEDED %}success{% elif build.outcome == build.FAILED %}danger{% else %}info{% endif %} pull-right"
+                data-request-url="{% url 'xhr_buildrequest' build.project.pk %}"
+                data-target='{{build.target_set.all|get_tasks|json}}'>
+                  <span class="glyphicon glyphicon-repeat"></span>
+                  Rebuild
+                </a>
               {% endif %}
             </div>
-    {%endif%}
+          {% endif %}
 
-    {%if build.outcome == build.IN_PROGRESS %}
-    <div class="col-md-4" style="display:none" id="cancelling-msg-{{build.buildrequest.pk}}">
-      Cancelling the build ...
-    </div>
-    <div class="col-md-4 col-md-offset-1 progress-info">
-      <div class="progress" id="build-pc-done-title-{{build.pk}}">
-        <div id="build-pc-done-bar-{{build.pk}}" style="width: {{build.completeper}}%;" class="progress-bar"></div>
+          {% if build.outcome == build.IN_PROGRESS %}
+            <div class="col-md-4" style="display:none" id="cancelling-msg-{{build.buildrequest.pk}}">
+              Cancelling the build ...
+            </div>
+
+            <div class="col-md-4 col-md-offset-1 progress-info">
+              <div class="progress" id="build-pc-done-title-{{build.pk}}">
+                <div id="build-pc-done-bar-{{build.pk}}" style="width: {{build.completeper}}%;" class="progress-bar">
+                </div>
+              </div>
+            </div>
+
+            <div class="col-md-4 progress-info">
+              <span id="build-pc-done-{{build.pk}}">{{build.completeper}}</span>% of tasks complete
+              {# No build cancel for command line builds project #}
+              {% if build.project.is_default %}
+                <span class="glyphicon glyphicon-question-sign get-help get-help-blue pull-right" title="Builds in this project cannot be cancelled from Toaster: they can only be cancelled from the command line"></span>
+              {% else %}
+                <a href="#" class="cancel-build-btn pull-right alert-link"
+                data-buildrequest-id={{build.buildrequest.pk}}
+                data-request-url="{% url 'xhr_buildrequest' build.project.pk %}">
+                  <span class="glyphicon glyphicon-remove-circle"></span>
+                  Cancel
+                </a>
+              {% endif %}
+            </div>
+          {% endif %} {# end if in progress #}
+
+          {% if build.outcome == build.CANCELLED %}
+            <div class="col-md-6">
+              Build cancelled
+            </div>
+
+            <div class="col-md-3">
+              <a href="#" class="info pull-right run-again-btn alert-link"
+              data-request-url="{% url 'xhr_buildrequest' build.project.pk %}"
+              data-target='{{build.target_set.all|get_tasks|json}}'>
+                <span class="glyphicon glyphicon-repeat"></span>
+                Rebuild
+              </a>
+            </div>
+          {% endif %}
+        </div>
       </div>
-    </div>
-    <div class="col-md-4 progress-info"><span id="build-pc-done-{{build.pk}}">{{build.completeper}}</span>% of tasks complete
-    {# No build cancel for command line builds project #}
-    {% if build.project.is_default %}
-    <span class="glyphicon glyphicon-question-sign get-help get-help-blue pull-right" title="Builds in this project cannot be cancelled from Toaster: they can only be cancelled from the command line"></span>
-    {% else %}
-      <a href="#" class="cancel-build-btn pull-right alert-link"
-          data-buildrequest-id={{build.buildrequest.pk}}
-          data-request-url="{% url 'xhr_buildrequest' build.project.pk %}" >
-        <span class="glyphicon glyphicon-remove-circle"></span>
-        Cancel
-      </a>
-    </div>
-    {% endif %}
-
-    {%endif%} {# end if in progress #}
-
-    {% if build.outcome == build.CANCELLED %}
-    <div class="col-md-6">
-      Build cancelled
-    </div>
-    <div class="col-md-3">
-      <a href="#" class="info pull-right run-again-btn alert-link"
-        data-request-url="{% url 'xhr_buildrequest' build.project.pk %}"
-        data-target='{{build.target_set.all|get_tasks|json}}'>
-        <span class="glyphicon glyphicon-repeat"></span>
-        Rebuild
-      </a>
-    </div>
-    {% endif %}
+    {% endfor %}
   </div>
-</div>
-  {% endfor %}
-  </div>
-
-{%endif%}
-
+{% endif %}

bitbake/lib/toaster/toastergui/templates/projectconf.html

@@ -41,7 +41,8 @@
       </div>
       <button id="apply-change-dl_dir" class="btn btn-default" type="button">Save</button>
       <button id="cancel-change-dl_dir" type="button" class="btn btn-link">Cancel</button>
-      <p class="help-block" id="hintError-dl_dir">A valid directory cannot include spaces or any of these characters: . \ ? % * : | " " < ></p>
+      <p class="help-block" id="hintError-dl_dir" style="display:none;">The directory path cannot include spaces or any of these characters: . \ ? % * : | " " &lt; &gt;</p>
+      <p class="help-block" id="hintError-initialChar-dl_dir" style="display:none;">The directory path should either start with a /, e.g. /home/toaster/downloads; or with a variable, e.g. ${TOPDIR}/downloads.</p>
     </form>
     </dd>
     {% endif %}
@@ -88,7 +89,7 @@
         <input type="text" class="form-control" id="new-image_install" placeholder="Type one or more package names">
       </div>
       <button id="apply-change-image_install" class="btn btn-default" type="button">Save</button>
-      <button id="cancel-change-image_install" type="button" class="btn btn-link">Cancel</button>  
+      <button id="cancel-change-image_install" type="button" class="btn btn-link">Cancel</button>
     </form>
     </dd>
     {% endif %}
@@ -149,7 +150,8 @@
       </div>
       <button id="apply-change-sstate_dir" class="btn btn-default" type="button">Save</button>
       <button id="cancel-change-sstate_dir" type="button" class="btn btn-link">Cancel</button>
-      <p class="help-block" id="hintError-sstate_dir">A valid directory cannot include spaces or any of these characters: . \ ? % * : | " " < ></p>
+      <p class="help-block" id="hintError-sstate_dir" style="display:none;">The directory path cannot include spaces or any of these characters: . \ ? % * : | " " &lt; &gt;</p>
+      <p class="help-block" id="hintError-initialChar-sstate_dir" style="display:none;">The directory path should either start with a /, e.g. /home/toaster/sstate-cache; or with a variable, e.g. ${TOPDIR}/sstate-cache.</p>
     </form>
     </dd>
     {% endif %}
@@ -203,7 +205,6 @@
           <code>CVS_PROXY_HOST</code>
           <code>CVS_PROXY_PORT</code>
           <code><a href="http://www.yoctoproject.org/docs/1.6.1/ref-manual/ref-manual.html#var-PARALLEL_MAKE" target="_blank">PARALLEL_MAKE</a></code>
-          <code><a href="http://www.yoctoproject.org/docs/1.6.1/ref-manual/ref-manual.html#var-SSTATE_MIRRORS" target="_blank">SSTATE_MIRRORS</a></code>
           <code><a href="http://www.yoctoproject.org/docs/1.6.1/ref-manual/ref-manual.html#var-TMPDIR" target="_blank">TMPDIR</a></code></p>
           <p>Plus the following standard shell environment variables:</p>
           <p><code>http_proxy</code> <code>ftp_proxy</code> <code>https_proxy</code> <code>all_proxy</code></p>
@@ -369,8 +370,8 @@ function setEventHandlersForDynamicElements() {
     $("#new-config_var_"+pk).val(current_val);
     if ( $("#new-config_var_"+pk).val().length ) {
       $("#apply-change-config_var_"+pk).removeAttr("disabled");
-    } 
-    else { 
+    }
+    else {
       $("#apply-change-config_var_"+pk).attr("disabled");
     }
   });
@@ -410,7 +411,7 @@ function setEventHandlersForDynamicElements() {
     var pk = $(this).attr("x-data");
 
     // fade out the variable+value div, then refresh the variable list
-    $(this).fadeOut(); 
+    $(this).fadeOut();
     $(this).tooltip("hide");
     $("config_var_entry_"+pk).fadeOut();
     $('#config_var_value_'+pk).parent("dd").fadeOut();
@@ -553,7 +554,6 @@ $(document).ready(function() {
 
   // change DL_DIR variable
   $('#change-dl_dir-icon').click(function() {
-    $('#hintError-dl_dir').hide();
     $('#change-dl_dir-form').removeClass('has-error');
     // preset the edit value
     var current_val = $("#dl_dir").text().trim();
@@ -575,6 +575,8 @@ $(document).ready(function() {
   });
 
   $('#cancel-change-dl_dir').click(function(){
+    $("#hintError-dl_dir").hide();
+    $("#hintError-initialChar-dl_dir").hide();
     $("#change-dl_dir-form").slideUp(function() {
       $('#dl_dir, #change-dl_dir-icon').show();
     });
@@ -583,20 +585,31 @@ $(document).ready(function() {
   $("#new-dl_dir").on('input', function(){
     if ($(this).val().trim().length == 0) {
       $("#apply-change-dl_dir").attr("disabled","disabled");
+      $('#change-dl_dir-form').addClass('has-error');
+      $('#hintError-dl_dir').hide();
+      $('#hintError-initialChar-dl_dir').hide();
     }
     else {
       var input = $(this);
-      var re = /^\/([^ <>\\|":\.%\?\*]+)$/;
+      var reBeginWithSlash = /^\//;
+      var reCheckVariable = /^\$/;
+      var re = /([ <>\\|":\.%\?\*]+)/;
       var invalidDir = re.test(input.val());
-      console.log(invalidDir);
-      if ( invalidDir ) {
-        $('#change-dl_dir-form').removeClass('has-error');
-        $("#apply-change-dl_dir").removeAttr("disabled");
-        $('#hintError-dl_dir').hide();
-      } else {
+      var invalidSlash = reBeginWithSlash.test(input.val());
+      var invalidVar = reCheckVariable.test(input.val());
+      if (!invalidSlash && !invalidVar) {
+        $('#change-dl_dir-form').addClass('has-error');
+        $("#apply-change-dl_dir").attr("disabled","disabled");
+        $('#hintError-initialChar-dl_dir').show();
+      } else if (invalidDir) {
         $('#change-dl_dir-form').addClass('has-error');
         $("#apply-change-dl_dir").attr("disabled","disabled");
         $('#hintError-dl_dir').show();
+      } else {
+        $('#change-dl_dir-form').removeClass('has-error');
+        $("#apply-change-dl_dir").removeAttr("disabled");
+        $('#hintError-dl_dir').hide();
+        $('#hintError-initialChar-dl_dir').hide();
       }
     }
   });
@@ -905,7 +918,7 @@ $(document).ready(function() {
 
   // change SSTATE_DIR variable
   $('#change-sstate_dir-icon').click(function() {
-    $('#hintError-sstate_dir').hide();
+    $('#change-sstate_dir-form').removeClass('has-error');
     // preset the edit value
     var current_val = $("span#sstate_dir").text().trim();
     if (current_val == "Not set") {
@@ -927,6 +940,8 @@ $(document).ready(function() {
   });
 
   $('#cancel-change-sstate_dir').click(function(){
+    $("#hintError-sstate_dir").hide();
+    $("#hintError-initialChar-sstate_dir").hide();
     $("#change-sstate_dir-form").slideUp(function() {
       $('#sstate_dir, #change-sstate_dir-icon').show();
     });
@@ -935,20 +950,31 @@ $(document).ready(function() {
   $("#new-sstate_dir").on('input', function(){
     if ($(this).val().trim().length == 0) {
       $("#apply-change-sstate_dir").attr("disabled","disabled");
+      $('#change-sstate_dir-form').addClass('has-error');
+      $('#hintError-sstate_dir').hide();
+      $('#hintError-initialChar-sstate_dir').hide();
     }
     else {
       var input = $(this);
-      var re = /^\/([^ <>\\|":\.%\?\*]+)$/;
+      var reBeginWithSlash = /^\//;
+      var reCheckVariable = /^\$/;
+      var re = /([ <>\\|":\.%\?\*]+)/;
       var invalidDir = re.test(input.val());
-      console.log(invalidDir);
-      if ( invalidDir ) {
+      var invalidSlash = reBeginWithSlash.test(input.val());
+      var invalidVar = reCheckVariable.test(input.val());
+      if (!invalidSlash && !invalidVar) {
+        $('#change-sstate_dir-form').addClass('has-error');
+        $("#apply-change-sstate_dir").attr("disabled","disabled");
+        $('#hintError-initialChar-sstate_dir').show();
+      } else if (invalidDir) {
+          $('#change-sstate_dir-form').addClass('has-error');
+          $("#apply-change-sstate_dir").attr("disabled","disabled");
+          $('#hintError-sstate_dir').show();
+      } else {
         $('#change-sstate_dir-form').removeClass('has-error');
         $("#apply-change-sstate_dir").removeAttr("disabled");
         $('#hintError-sstate_dir').hide();
-      } else {
-        $('#change-sstate_dir-form').addClass('has-error');
-        $("#apply-change-sstate_dir").attr("disabled","disabled");
-        $('#hintError-sstate_dir').show();
+        $('#hintError-initialChar-sstate_dir').hide();
       }
     }
   });

bitbake/lib/toaster/toastergui/templatetags/field_values_filter.py

@@ -0,0 +1,18 @@
+from django import template
+
+register = template.Library()
+
+def field_values(iterable, field):
+    """
+    Convert an iterable of models into a list of strings, one for each model,
+    where the string for each model is the value of the field "field".
+    """
+    objects = []
+
+    if field:
+        for item in iterable:
+            objects.append(getattr(item, field))
+
+    return objects
+
+register.filter('field_values', field_values)

bitbake/lib/toaster/toastergui/urls.py

@@ -190,7 +190,10 @@ urlpatterns = patterns('toastergui.views',
             name='xhr_configvaredit'),
 
         url(r'^xhr_importlayer/$', 'xhr_importlayer', name='xhr_importlayer'),
-        url(r'^xhr_updatelayer/$', 'xhr_updatelayer', name='xhr_updatelayer'),
+
+        url(r'^xhr_layer/(?P<pid>\d+)/(?P<layerversion_id>\d+)$',
+            api.XhrLayer.as_view(),
+            name='xhr_layer'),
 
         # JS Unit tests
         url(r'^js-unit-tests/$', 'jsunittests', name='js-unit-tests'),

bitbake/lib/toaster/toastergui/views.py

@@ -30,7 +30,8 @@ from django.db import IntegrityError, Error
 from django.shortcuts import render, redirect, get_object_or_404
 from orm.models import Build, Target, Task, Layer, Layer_Version, Recipe, LogMessage, Variable
 from orm.models import Task_Dependency, Recipe_Dependency, Package, Package_File, Package_Dependency
-from orm.models import Target_Installed_Package, Target_File, Target_Image_File, BuildArtifact, CustomImagePackage
+from orm.models import Target_Installed_Package, Target_File, Target_Image_File, CustomImagePackage
+from orm.models import TargetKernelFile, TargetSDKFile
 from orm.models import BitbakeVersion, CustomImageRecipe
 from bldcontrol import bbcontroller
 from django.views.decorators.cache import cache_control
@@ -470,46 +471,58 @@ def builddashboard( request, build_id ):
     recipeCount = Recipe.objects.filter( layer_version__id__in = layerVersionId ).count( );
     tgts = Target.objects.filter( build_id = build_id ).order_by( 'target' );
 
-    ##
     # set up custom target list with computed package and image data
-    #
-
-    targets = [ ]
+    targets = []
     ntargets = 0
-    hasImages = False
-    targetHasNoImages = False
+
+    # True if at least one target for this build has an SDK artifact
+    # or image file
+    has_artifacts = False
+
     for t in tgts:
-        elem = { }
-        elem[ 'target' ] = t
-        if t.is_image:
-            hasImages = True
+        elem = {}
+        elem['target'] = t
+
+        target_has_images = False
+        image_files = []
+
         npkg = 0
         pkgsz = 0
         package = None
         for package in Package.objects.filter(id__in = [x.package_id for x in t.target_installed_package_set.all()]):
             pkgsz = pkgsz + package.size
-            if ( package.installed_name ):
+            if package.installed_name:
                 npkg = npkg + 1
-        elem[ 'npkg' ] = npkg
-        elem[ 'pkgsz' ] = pkgsz
-        ti = Target_Image_File.objects.filter( target_id = t.id )
-        imageFiles = [ ]
+        elem['npkg'] = npkg
+        elem['pkgsz'] = pkgsz
+        ti = Target_Image_File.objects.filter(target_id = t.id)
         for i in ti:
-            ndx = i.file_name.rfind( '/' )
-            if ( ndx < 0 ):
+            ndx = i.file_name.rfind('/')
+            if ndx < 0:
                 ndx = 0;
-            f = i.file_name[ ndx + 1: ]
-            imageFiles.append({
+            f = i.file_name[ndx + 1:]
+            image_files.append({
                 'id': i.id,
                 'path': f,
                 'size': i.file_size,
                 'suffix': i.suffix
             })
-        if t.is_image and (len(imageFiles) <= 0 or len(t.license_manifest_path) <= 0):
-            targetHasNoImages = True
-        elem[ 'imageFiles' ] = imageFiles
-        elem[ 'targetHasNoImages' ] = targetHasNoImages
-        targets.append( elem )
+        if len(image_files) > 0:
+            target_has_images = True
+        elem['targetHasImages'] = target_has_images
+
+        elem['imageFiles'] = image_files
+        elem['target_kernel_artifacts'] = t.targetkernelfile_set.all()
+
+        target_sdk_files = t.targetsdkfile_set.all()
+        target_sdk_artifacts_count = target_sdk_files.count()
+        elem['target_sdk_artifacts_count'] = target_sdk_artifacts_count
+        elem['target_sdk_artifacts'] = target_sdk_files
+
+        if target_has_images or target_sdk_artifacts_count > 0:
+            has_artifacts = True
+
+        targets.append(elem)
 
     ##
     # how many packages in this build - ignore anonymous ones
@@ -526,7 +539,7 @@ def builddashboard( request, build_id ):
     context = {
             'build'           : build,
             'project'         : build.project,
-            'hasImages'       : hasImages,
+            'hasArtifacts'    : has_artifacts,
             'ntargets'        : ntargets,
             'targets'         : targets,
             'recipecount'     : recipeCount,
@@ -1404,15 +1417,37 @@ if True:
         pid = prj.id
 
         from collections import Counter
-        freqtargets = []
-        try:
-            btargets = sum(build.target_set.all() for build in Build.objects.filter(project=prj, outcome__lt=Build.IN_PROGRESS))
-            brtargets = sum(br.brtarget_set.all() for br in BuildRequest.objects.filter(project = prj, state = BuildRequest.REQ_FAILED))
-            freqtargets = [x.target for x in btargets] + [x.target for x in brtargets]
-        except TypeError:
-            pass
-        freqtargets = Counter(freqtargets)
-        freqtargets = sorted(freqtargets, key = lambda x: freqtargets[x], reverse=True)
+
+        freqtargets = Counter(Target.objects.filter(
+            Q(build__project=prj),
+            ~Q(build__outcome=Build.IN_PROGRESS)
+        ).order_by("target").values_list("target", flat=True))
+
+        freqtargets = freqtargets.most_common(5)
+
+        # We now have the targets in order of frequency but if there are two
+        # with the same frequency then we need to make sure those are in
+        # alphabetical order without losing the frequency ordering
+
+        tmp = []
+        switch = None
+        for i, freqtartget in enumerate(freqtargets):
+            target, count = freqtartget
+            try:
+                target_next, count_next = freqtargets[i+1]
+                if count == count_next and target > target_next:
+                    switch = target
+                    continue
+            except IndexError:
+                pass
+
+            tmp.append(target)
+
+            if switch:
+                tmp.append(switch)
+                switch = None
+
+        freqtargets = tmp
 
         layers = [{"id": x.layercommit.pk, "orderid": x.pk, "name" : x.layercommit.layer.name,
                    "vcs_url": x.layercommit.layer.vcs_url, "vcs_reference" : x.layercommit.get_vcs_reference(),
@@ -1432,7 +1467,7 @@ if True:
             "layers" : layers,
             "targets" : [{"target" : x.target, "task" : x.task, "pk": x.pk} for x in prj.projecttarget_set.all()],
             "variables": [(x.name, x.value) for x in prj.projectvariable_set.all()],
-            "freqtargets": freqtargets[:5],
+            "freqtargets": freqtargets,
             "releases": [{"id": x.pk, "name": x.name, "description":x.description} for x in Release.objects.all()],
             "project_html": 1,
             "recipesTypeAheadUrl": reverse('xhr_recipestypeahead', args=(prj.pk,)),
@@ -1533,11 +1568,16 @@ if True:
     def xhr_configvaredit(request, pid):
         try:
             prj = Project.objects.get(id = pid)
+            # There are cases where user can add variables which hold values
+            # like http://, file:/// etc. In such case a simple split(":")
+            # would fail. One example is SSTATE_MIRRORS variable. So we use
+            # max_split var to handle them.
+            max_split = 1
             # add conf variables
             if 'configvarAdd' in request.POST:
                 t=request.POST['configvarAdd'].strip()
                 if ":" in t:
-                    variable, value = t.split(":")
+                    variable, value = t.split(":", max_split)
                 else:
                     variable = t
                     value = ""
@@ -1547,7 +1587,7 @@ if True:
             if 'configvarChange' in request.POST:
                 t=request.POST['configvarChange'].strip()
                 if ":" in t:
-                    variable, value = t.split(":")
+                    variable, value = t.split(":", max_split)
                 else:
                     variable = t
                     value = ""
@@ -1712,49 +1752,6 @@ if True:
 
         return HttpResponse(jsonfilter(json_response), content_type = "application/json")
 
-    def xhr_updatelayer(request):
-
-        def error_response(error):
-            return HttpResponse(jsonfilter({"error": error}), content_type = "application/json")
-
-        if "layer_version_id" not in request.POST:
-            return error_response("Please specify a layer version id")
-        try:
-            layer_version_id = request.POST["layer_version_id"]
-            layer_version = Layer_Version.objects.get(id=layer_version_id)
-        except Layer_Version.DoesNotExist:
-            return error_response("Cannot find layer to update")
-
-
-        if "vcs_url" in request.POST:
-            layer_version.layer.vcs_url = request.POST["vcs_url"]
-        if "dirpath" in request.POST:
-            layer_version.dirpath = request.POST["dirpath"]
-        if "commit" in request.POST:
-            layer_version.commit = request.POST["commit"]
-        if "up_branch" in request.POST:
-            layer_version.up_branch_id = int(request.POST["up_branch"])
-
-        if "add_dep" in request.POST:
-            lvd = LayerVersionDependency(layer_version=layer_version, depends_on_id=request.POST["add_dep"])
-            lvd.save()
-
-        if "rm_dep" in request.POST:
-            rm_dep = LayerVersionDependency.objects.get(layer_version=layer_version, depends_on_id=request.POST["rm_dep"])
-            rm_dep.delete()
-
-        if "summary" in request.POST:
-            layer_version.layer.summary = request.POST["summary"]
-        if "description" in request.POST:
-            layer_version.layer.description = request.POST["description"]
-
-        try:
-            layer_version.layer.save()
-            layer_version.save()
-        except Exception as e:
-            return error_response("Could not update layer version entry: %s" % e)
-
-        return HttpResponse(jsonfilter({"error": "ok",}), content_type = "application/json")
 
     @xhr_response
     def xhr_customrecipe(request):
@@ -2210,7 +2207,7 @@ if True:
         vars_blacklist  = {
             'PARALLEL_MAKE','BB_NUMBER_THREADS',
             'BB_DISKMON_DIRS','BB_NUMBER_THREADS','CVS_PROXY_HOST','CVS_PROXY_PORT',
-            'PARALLEL_MAKE','SSTATE_MIRRORS','TMPDIR',
+            'PARALLEL_MAKE','TMPDIR',
             'all_proxy','ftp_proxy','http_proxy ','https_proxy'
             }
 
@@ -2307,12 +2304,20 @@ if True:
         elif artifact_type == "imagefile":
             file_name = Target_Image_File.objects.get(target__build = build, pk = artifact_id).file_name
 
-        elif artifact_type == "buildartifact":
-            file_name = BuildArtifact.objects.get(build = build, pk = artifact_id).file_name
+        elif artifact_type == "targetkernelartifact":
+            target = TargetKernelFile.objects.get(pk=artifact_id)
+            file_name = target.file_name
+
+        elif artifact_type == "targetsdkartifact":
+            target = TargetSDKFile.objects.get(pk=artifact_id)
+            file_name = target.file_name
 
         elif artifact_type == "licensemanifest":
             file_name = Target.objects.get(build = build, pk = artifact_id).license_manifest_path
 
+        elif artifact_type == "packagemanifest":
+            file_name = Target.objects.get(build = build, pk = artifact_id).package_manifest_path
+
         elif artifact_type == "tasklogfile":
             file_name = Task.objects.get(build = build, pk = artifact_id).logfile
 

documentation/dev-manual/dev-manual-common-tasks.xml

@@ -7611,13 +7611,18 @@
                 </para>
 
                 <para>
-                    As described below in the
-                    "<link linkend='runtime-package-management-target-ipk'>Using IPK</link>"
-                    section, if you are using IPK as your package format, you
-                    can make use of the
-                    <filename>distro-feed-configs</filename> recipe provided
-                    by <filename>meta-oe</filename> in order to configure your
-                    target to use your IPK databases.
+                    You can use the
+                    <ulink url='&YOCTO_DOCS_REF_URL;#var-PACKAGE_FEED_ARCHS'><filename>PACKAGE_FEED_ARCHS</filename></ulink>,
+                    <ulink url='&YOCTO_DOCS_REF_URL;#var-PACKAGE_FEED_BASE_PATHS'><filename>PACKAGE_FEED_BASE_PATHS</filename></ulink>,
+                    and
+                    <ulink url='&YOCTO_DOCS_REF_URL;#var-PACKAGE_FEED_URIS'><filename>PACKAGE_FEED_URIS</filename></ulink>
+                    variables to pre-configure target images to use a package
+                    feed.
+                    If you do not define these variables, then manual steps
+                    as described in the subsequent sections are necessary to
+                    configure the target.
+                    You should set these variables before building the image
+                    in order to produce a correctly configured image.
                 </para>
 
                 <para>
@@ -7686,7 +7691,13 @@
                         The <filename>smart</filename> application performs
                         runtime package management of RPM packages.
                         You must perform an initial setup for
-                        <filename>smart</filename> on the target machine.
+                        <filename>smart</filename> on the target machine
+                        if the
+                        <ulink url='&YOCTO_DOCS_REF_URL;#var-PACKAGE_FEED_ARCHS'><filename>PACKAGE_FEED_ARCHS</filename></ulink>,
+                        <ulink url='&YOCTO_DOCS_REF_URL;#var-PACKAGE_FEED_BASE_PATHS'><filename>PACKAGE_FEED_BASE_PATHS</filename></ulink>, and
+                        <ulink url='&YOCTO_DOCS_REF_URL;#var-PACKAGE_FEED_URIS'><filename>PACKAGE_FEED_URIS</filename></ulink>
+                        variables have not been set or the target image was
+                        built before the variables were set.
                     </para>
 
                     <para>
@@ -7720,7 +7731,13 @@
                         The <filename>opkg</filename> application performs
                         runtime package management of IPK packages.
                         You must perform an initial setup for
-                        <filename>opkg</filename> on the target machine.
+                        <filename>opkg</filename> on the target machine
+                        if the
+                        <ulink url='&YOCTO_DOCS_REF_URL;#var-PACKAGE_FEED_ARCHS'><filename>PACKAGE_FEED_ARCHS</filename></ulink>,
+                        <ulink url='&YOCTO_DOCS_REF_URL;#var-PACKAGE_FEED_BASE_PATHS'><filename>PACKAGE_FEED_BASE_PATHS</filename></ulink>, and
+                        <ulink url='&YOCTO_DOCS_REF_URL;#var-PACKAGE_FEED_URIS'><filename>PACKAGE_FEED_URIS</filename></ulink>
+                        variables have not been set or the target image was
+                        built before the variables were set.
                     </para>
 
                     <para>
@@ -7769,7 +7786,13 @@
                         This application uses a source list file to find
                         available package databases.
                         You must perform an initial setup for
-                        <filename>apt</filename> on the target machine.
+                        <filename>apt</filename> on the target machine
+                        if the
+                        <ulink url='&YOCTO_DOCS_REF_URL;#var-PACKAGE_FEED_ARCHS'><filename>PACKAGE_FEED_ARCHS</filename></ulink>,
+                        <ulink url='&YOCTO_DOCS_REF_URL;#var-PACKAGE_FEED_BASE_PATHS'><filename>PACKAGE_FEED_BASE_PATHS</filename></ulink>, and
+                        <ulink url='&YOCTO_DOCS_REF_URL;#var-PACKAGE_FEED_URIS'><filename>PACKAGE_FEED_URIS</filename></ulink>
+                        variables have not been set or the target image was
+                        built before the variables were set.
                     </para>
 
                     <para>
@@ -9378,7 +9401,7 @@
                 A JSON file is used to define the packages needed by a test.
                 This file must be in the same path as the file used to define
                 the tests.
-                Furthermore, the filename must map directory to the test
+                Furthermore, the filename must map directly to the test
                 module name with a <filename>.json</filename> extension.
             </para>
 
@@ -9411,7 +9434,7 @@
                 DUT.
                 <literallayout class='monospaced'>
      {
-         "": {
+         "foo": {
              "pkg": "bar"
          },
          "foobar": [

documentation/ref-manual/closer-look.xml

@@ -1209,6 +1209,131 @@
                 which includes the environment setup script.
             </para>
         </section>
+
+        <section id='setscene-tasks-and-shared-state'>
+            <title>Setscene Tasks and Shared State</title>
+
+            <para>
+                The description of tasks so far assumes that BitBake needs to
+                build everything and there are no prebuilt objects available.
+                BitBake does support skipping tasks if prebuilt objects are
+                available.
+                These objects are usually made available in the form of a
+                shared state (sstate) cache.
+                <note>
+                    For information on variables affecting sstate, see the
+                    <link linkend='var-SSTATE_DIR'><filename>SSTATE_DIR</filename></link>
+                    and
+                    <link linkend='var-SSTATE_MIRRORS'><filename>SSTATE_MIRRORS</filename></link>
+                    variables.
+                </note>
+            </para>
+
+            <para>
+                The idea of a setscene task (i.e
+                <filename>do_</filename><replaceable>taskname</replaceable><filename>_setscene</filename>)
+                is a version of the task where
+                instead of building something, BitBake can skip to the end
+                result and simply place a set of files into specific locations
+                as needed.
+                In some cases, it makes sense to have a setscene task variant
+                (e.g. generating package files in the
+                <filename>do_package_write_*</filename> task).
+                In other cases, it does not make sense, (e.g. a
+                <link linkend='ref-tasks-patch'><filename>do_patch</filename></link>
+                task or
+                <link linkend='ref-tasks-unpack'><filename>do_unpack</filename></link>
+                task) since the work involved would be equal to or greater than
+                the underlying task.
+            </para>
+
+            <para>
+                In the OpenEmbedded build system, the common tasks that have
+                setscene variants are <link linkend='ref-tasks-package'><filename>do_package</filename></link>,
+                <filename>do_package_write_*</filename>,
+                <link linkend='ref-tasks-deploy'><filename>do_deploy</filename></link>,
+                <link linkend='ref-tasks-packagedata'><filename>do_packagedata</filename></link>,
+                and
+                <link linkend='ref-tasks-populate_sysroot'><filename>do_populate_sysroot</filename></link>.
+                Notice that these are most of the tasks whose output is an
+                end result.
+            </para>
+
+            <para>
+                The OpenEmbedded build system has knowledge of the relationship
+                between these tasks and other tasks that precede them.
+                For example, if BitBake runs
+                <filename>do_populate_sysroot_setscene</filename> for
+                something, there is little point in running any of the
+                <filename>do_fetch</filename>, <filename>do_unpack</filename>,
+                <filename>do_patch</filename>,
+                <filename>do_configure</filename>,
+                <filename>do_compile</filename>, and
+                <filename>do_install</filename> tasks.
+                However, if <filename>do_package</filename> needs to be run,
+                BitBake would need to run those other tasks.
+            </para>
+
+            <para>
+                It becomes more complicated if everything can come from an
+                sstate cache because some objects are simply not required at
+                all.
+                For example, you do not need a compiler or native tools, such
+                as quilt, if there is nothing to compile or patch.
+                If the <filename>do_package_write_*</filename> packages are
+                available from sstate, BitBake does not need the
+                <filename>do_package</filename> task data.
+            </para>
+
+            <para>
+                To handle all these complexities, BitBake runs in two phases.
+                The first is the "setscene" stage.
+                During this stage, BitBake first checks the sstate cache for
+                any targets it is planning to build.
+                BitBake does a fast check to see if the object exists rather
+                than a complete download.
+                If nothing exists, the second phase, which is the setscene
+                stage, completes and the main build proceeds.
+            </para>
+
+            <para>
+                If objects are found in the sstate cache, the OpenEmbedded
+                build system works backwards from the end targets specified
+                by the user.
+                For example, if an image is being built, the OpenEmbedded build
+                system first looks for the packages needed for that image and
+                the tools needed to construct an image.
+                If those are available, the compiler is not needed.
+                Thus, the compiler is not even downloaded.
+                If something was found to be unavailable, or the download or
+                setscene task fails, the OpenEmbedded build system then tries
+                to install dependencies, such as the compiler, from the cache.
+            </para>
+
+            <para>
+                The availability of objects in the sstate cache is handled by
+                the function specified by the
+                <ulink url='&YOCTO_DOCS_BB_URL;#var-BB_HASHCHECK_FUNCTION'><filename>BB_HASHCHECK_FUNCTION</filename></ulink>
+                variable and returns a list of the objects that are available.
+                The function specified by the
+                <ulink url='&YOCTO_DOCS_BB_URL;#var-BB_SETSCENE_DEPVALID'><filename>BB_SETSCENE_DEPVALID</filename></ulink>
+                variable is the function that determines whether a given
+                dependency needs to be followed, and whether for any given
+                relationship the function needs to be passed.
+                The function returns a True or False value.
+            </para>
+
+            <para>
+                Once the setscene process completes, the OpenEmbedded build
+                system has a list of tasks that it believes it can "accelerate"
+                and therefore does not need to run.
+                There is a final function call to the function specified by the
+                <ulink url='&YOCTO_DOCS_BB_URL;#var-BB_SETSCENE_VERIFY_FUNCTION'><filename>BB_SETSCENE_VERIFY_FUNCTION</filename></ulink>
+                variable that is able to require the tasks to be run that
+                that the OpenEmbedded build system initially was going to
+                skip.
+            </para>
+        </section>
     </section>
 
     <section id='images-dev-environment'>

documentation/ref-manual/ref-features.xml

@@ -146,7 +146,25 @@
                     bluetooth support (integrated BT only).</para></listitem>
                 <listitem><para><emphasis>bluez5:</emphasis> Include
                     BlueZ Version 5, which provides core Bluetooth layers and
-                    protocols support.</para></listitem>
+                    protocols support.
+                    <note>
+                        The default value for the
+                        <link linkend='var-DISTRO_FEATURES'><filename>DISTRO FEATURES</filename></link>
+                        variable includes "bluetooth", which causes bluez5
+                        to be backfilled in for bluetooth support.
+                        If you do not want bluez5 backfilled and would rather
+                        use bluez4, you need to use the
+                        <link linkend='var-DISTRO_FEATURES_BACKFILL_CONSIDERED'><filename>DISTRO_FEATURES_BACKFILL_CONSIDERED</filename></link>
+                        variable as follows:
+                        <literallayout class='monospaced'>
+     DISTRO_FEATURES_BACKFILL_CONSIDERED = "bluez5"
+                        </literallayout>
+                        Setting this variable tells the OpenEmbedded build
+                        system that you have considered but ruled
+                        out using the bluez5 feature and that bluez4 will be
+                        used.
+                    </note>
+                    </para></listitem>
                 <listitem><para><emphasis>cramfs:</emphasis> Include CramFS
                     support.</para></listitem>
                 <listitem><para><emphasis>directfb:</emphasis>

documentation/ref-manual/ref-tasks.xml

@@ -19,6 +19,10 @@
     <para>
         The following sections describe normal tasks associated with building
         a recipe.
+        For more information on tasks and dependencies, see the
+        "<ulink url='&YOCTO_DOCS_BB_URL;#tasks'>Tasks</ulink>" and
+        "<ulink url='&YOCTO_DOCS_BB_URL;#dependencies'>Dependencies</ulink>"
+        sections in the BitBake User Manual.
     </para>
 
     <section id='ref-tasks-build'>

documentation/ref-manual/ref-variables.xml

@@ -995,6 +995,29 @@
      BBCLASSEXTEND =+ "native nativesdk"
      BBCLASSEXTEND =+ "multilib:<replaceable>multilib_name</replaceable>"
                     </literallayout>
+                    <note>
+                        <para>
+                        Internally, the <filename>BBCLASSEXTEND</filename>
+                        mechanism generates recipe variants by rewriting
+                        variable values and applying overrides such as
+                        <filename>_class-native</filename>.
+                        For example, to generate a native version of a recipe,
+                        a
+                        <link linkend='var-DEPENDS'><filename>DEPENDS</filename></link>
+                        on "foo" is rewritten to a <filename>DEPENDS</filename>
+                        on "foo-native".
+                        </para>
+
+                        <para>
+                        Even when using <filename>BBCLASSEXTEND</filename>, the
+                        recipe is only parsed once.
+                        Parsing once adds some limitations.
+                        For example, it is not possible to
+                        include a different file depending on the variant,
+                        since <filename>include</filename> statements are
+                        processed when the recipe is parsed.
+                        </para>
+                    </note>
                 </para>
              </glossdef>
         </glossentry>
@@ -2770,12 +2793,56 @@
                     task of recipe "b".
                     This means anything that recipe "b" puts into sysroot
                     is available when recipe "a" is configuring itself.
+                    <note>
+                        <title>Notes</title>
+                        <itemizedlist>
+                            <listitem><para>
+                                <filename>DEPENDS</filename> is a list of
+                                recipe names.
+                                Or, to be more precise, it is a list of
+                                <link linkend='var-PROVIDES'><filename>PROVIDES</filename></link>
+                                names, which usually match recipe names.
+                                Putting a package name such as "foo-dev" in
+                                <filename>DEPENDS</filename> does not make
+                                sense.
+                                Use "foo" instead, as this will put files
+                                from all the packages that make up
+                                <filename>foo</filename>, which includes
+                                those from <filename>foo-dev</filename>, into
+                                the sysroot.
+                                </para></listitem>
+                            <listitem><para>
+                                Counterintuitively,
+                                <filename>DEPENDS</filename> is often necessary
+                                even for recipes that install precompiled
+                                components.
+                                For example, if <filename>libfoo</filename>
+                                is a precompiled library that links against
+                                <filename>libbar</filename>, then
+                                linking against <filename>libfoo</filename>
+                                requires both <filename>libfoo</filename>
+                                and <filename>libbar</filename> to be available
+                                in the sysroot.
+                                Without a <filename>DEPENDS</filename> from the
+                                recipe that installs <filename>libfoo</filename>
+                                to the recipe that installs
+                                <filename>libbar</filename>, other recipes might
+                                fail to link against
+                                <filename>libfoo</filename>.
+                                </para></listitem>
+                        </itemizedlist>
+                    </note>
                 </para>
 
                 <para>
                     For information on runtime dependencies, see the
                     <link linkend='var-RDEPENDS'><filename>RDEPENDS</filename></link>
                     variable.
+                    You can also see the
+                    "<ulink url='&YOCTO_DOCS_BB_URL;#tasks'>Tasks</ulink>" and
+                    "<ulink url='&YOCTO_DOCS_BB_URL;#dependencies'>Dependencies</ulink>"
+                    sections in the BitBake User Manual for additional
+                    information on tasks and dependencies.
                 </para>
             </glossdef>
         </glossentry>
@@ -9571,8 +9638,48 @@ recipes-graphics/xorg-font/font-alias_1.0.3.bb:PR = "${INC_PR}.3"
             <glossdef>
                 <para role="glossdeffirst">
 <!--                <para role="glossdeffirst"><imagedata fileref="figures/define-generic.png" /> -->
-                    The revision of the recipe.
-                    The default value for this variable is "r0".
+                    The revision of the recipe. The default value for this
+                    variable is "r0".
+                    Subsequent revisions of the recipe conventionally have the
+                    values "r1", "r2", and so forth.
+                    When
+                    <link linkend='var-PV'><filename>PV</filename></link>
+                    increases, <filename>PR</filename> is conventionally reset
+                    to "r0".
+                    <note>
+                        The OpenEmbedded build system does not need the aid of
+                        <filename>PR</filename> to know when to rebuild a
+                        recipe.
+                        The build system uses
+                        <link linkend='var-STAMP'><filename>STAMP</filename></link>
+                        and the
+                        <link linkend='shared-state-cache'>shared state cache</link>
+                        mechanisms.
+                    </note>
+                    The <filename>PR</filename> variable primarily becomes
+                    significant when a package manager dynamically installs
+                    packages on an already built image.
+                    In this case, <filename>PR</filename>, which is the default
+                    value of
+                    <link linkend='var-PKGR'><filename>PKGR</filename></link>,
+                    helps the package manager distinguish which package is the
+                    most recent one in cases where many packages have the same
+                    <filename>PV</filename> (i.e. <filename>PKGV</filename>).
+                    A component having many packages with the same
+                    <filename>PV</filename> usually means that the packages all
+                    install the same upstream version, but with later
+                    (<filename>PR</filename>) version packages including
+                    packaging fixes.
+                    <note>
+                        <filename>PR</filename> does not need to be increased
+                        for changes that do not change the package contents or
+                        metadata.
+                    </note>
+                    Because manually managing <filename>PR</filename> can be
+                    cumbersome and error-prone, an automated solution exists.
+                    See the
+                    "<ulink url='&YOCTO_DOCS_DEV_URL;#working-with-a-pr-service'>Working With a PR Service</ulink>"
+                    section for more information.
                 </para>
             </glossdef>
         </glossentry>
@@ -10088,14 +10195,20 @@ recipes-graphics/xorg-font/font-alias_1.0.3.bb:PR = "${INC_PR}.3"
                     <filename>RDEPENDS</filename> since some automatic
                     handling occurs:
                     <itemizedlist>
-                        <listitem><para><emphasis><filename>shlibdeps</filename></emphasis>:  If
-                            a runtime package contains a shared library
-                            (<filename>.so</filename>), the build
-                            processes the library in order to determine other
-                            libraries to which it is dynamically linked.
-                            The build process adds these libraries to
-                            <filename>RDEPENDS</filename> when creating the runtime
-                            package.</para></listitem>
+                        <listitem><para><emphasis><filename>shlibdeps</filename></emphasis>:
+                            If a runtime package contains a compiled binary,
+                            the build processes the binary in order to
+                            determine any shared libraries
+                            (<filename>.so</filename> files) to which it is
+                            dynamically linked.
+                            The build process adds the packages containing
+                            these libraries to <filename>RDEPENDS</filename>
+                            when creating the runtime package.
+                            In addition, if the shared library is versioned,
+                            the dependency will have the version appended to it
+                            in order to force an upgrade to the appropriate
+                            version if needed.
+                            </para></listitem>
                         <listitem><para><emphasis><filename>pcdeps</filename></emphasis>:  If
                             the package ships a <filename>pkg-config</filename>
                             information file, the build process uses this file
@@ -10135,6 +10248,35 @@ recipes-graphics/xorg-font/font-alias_1.0.3.bb:PR = "${INC_PR}.3"
                     For information on build-time dependencies, see the
                     <link linkend='var-DEPENDS'><filename>DEPENDS</filename></link>
                     variable.
+                    You can also see the
+                    "<ulink url='&YOCTO_DOCS_BB_URL;#tasks'>Tasks</ulink>" and
+                    "<ulink url='&YOCTO_DOCS_BB_URL;#dependencies'>Dependencies</ulink>"
+                    sections in the BitBake User Manual for additional
+                    information on tasks and dependencies.
+                </para>
+            </glossdef>
+        </glossentry>
+
+        <glossentry id='var-REMOVE_PKG_SUPPORT'><glossterm>REMOVE_PKG_SUPPORT</glossterm>
+            <info>
+                REMOVE_PKG_SUPPORT[doc] = "Allows for removal of supporting packages for images that do not contain a package manager."
+            </info>
+            <glossdef>
+                <para role="glossdeffirst">
+<!--                <para role="glossdeffirst"><imagedata fileref="figures/define-generic.png" /> -->
+                    For images not containing a package manager,
+                    <filename>REMOVE_PKG_SUPPORT</filename> allows the removal
+                    of supporting packages.
+                    Packages that will be uninstalled include:
+                    <filename>update-rc.d</filename>,
+                    <filename>base-passwd</filename>,
+                    <filename>shadow</filename>,
+                    <filename>run-postinsts</filename>, and
+                    <filename>update-alternatives</filename>.
+                </para>
+
+                <para>
+                    Set the variable to "1" to remove these packages.
                 </para>
             </glossdef>
         </glossentry>

documentation/ref-manual/technical-details.xml

@@ -690,6 +690,123 @@
      addtask do_deploy_setscene
      do_deploy[dirs] = "${DEPLOYDIR} ${B}"
             </literallayout>
+            The following list explains the previous example:
+            <itemizedlist>
+                <listitem><para>
+                    Adding "do_deploy" to <filename>SSTATETASKS</filename>
+                    adds some required sstate-related processing, which is
+                    implemented in the
+                    <link linkend='ref-classes-sstate'><filename>sstate</filename></link>
+                    class, to before and after the
+                    <link linkend='ref-tasks-deploy'><filename>do_deploy</filename></link>
+                    task.
+                    </para></listitem>
+                <listitem><para>
+                    The
+                    <filename>do_deploy[sstate-inputdirs] = "${DEPLOYDIR}"</filename>
+                    declares that <filename>do_deploy</filename> places its
+                    output in <filename>${DEPLOYDIR}</filename> when run
+                    normally (i.e. when not using the sstate cache).
+                    This output becomes the input to the shared state cache.
+                    </para></listitem>
+                <listitem><para>
+                    The
+                    <filename>do_deploy[sstate-outputdirs] = "${DEPLOY_DIR_IMAGE}"</filename>
+                    line causes the contents of the shared state cache to be
+                    copied to <filename>${DEPLOY_DIR_IMAGE}</filename>.
+                    <note>
+                        If <filename>do_deploy</filename> is not already in
+                        the shared state cache or if its input checksum
+                        (signature) has changed from when the output was
+                        cached, the task will be run to populate the shared
+                        state cache, after which the contents of the shared
+                        state cache is copied to
+                        <filename>${DEPLOY_DIR_IMAGE}</filename>.
+                        If <filename>do_deploy</filename> is in the shared
+                        state cache and its signature indicates that the
+                        cached output is still valid (i.e. if no
+                        relevant task inputs have changed), then the contents
+                        of the shared state cache will be copied directly to
+                        <filename>${DEPLOY_DIR_IMAGE}</filename> by the
+                        <filename>do_deploy_setscene</filename> task instead,
+                        skipping the <filename>do_deploy</filename> task.
+                    </note>
+                    </para></listitem>
+                <listitem><para>
+                    The following task definition is glue logic needed to make
+                    the previous settings effective:
+                    <literallayout class='monospaced'>
+     python do_deploy_setscene () {
+         sstate_setscene(d)
+     }
+     addtask do_deploy_setscene
+                    </literallayout>
+                    <filename>sstate_setscene()</filename> takes the flags
+                    above as input and accelerates the
+                    <filename>do_deploy</filename> task through the
+                    shared state cache if possible.
+                    If the task was accelerated,
+                    <filename>sstate_setscene()</filename> returns True.
+                    Otherwise, it returns False, and the normal
+                    <filename>do_deploy</filename> task runs.
+                    For more information, see the
+                    "<ulink url='&YOCTO_DOCS_BB_URL;#setscene'>setscene</ulink>"
+                    section in the BitBake User Manual.
+                    </para></listitem>
+                <listitem><para>
+                    The
+                    <filename>do_deploy[dirs] = "${DEPLOYDIR} ${B}"</filename>
+                    line creates <filename>${DEPLOYDIR}</filename> and
+                    <filename>${B}</filename> before the
+                    <filename>do_deploy</filename> task runs.
+                    For more information, see the
+                    "<ulink url='&YOCTO_DOCS_BB_URL;#variable-flags'>Variable Flags</ulink>"
+                    section in the BitBake User Manual.
+                    <note>
+                        In cases where
+                        <filename>sstate-inputdirs</filename> and
+                        <filename>sstate-outputdirs</filename> would be the
+                        same, you can use
+                        <filename>sstate-plaindirs</filename>.
+                        For example, to preserve the
+                        <filename>${PKGD}</filename> and
+                        <filename>${PKGDEST}</filename> output from the
+                        <link linkend='ref-tasks-package'><filename>do_package</filename></link>
+                        task, use the following:
+                        <literallayout class='monospaced'>
+     do_package[sstate-plaindirs] = "${PKGD} ${PKGDEST}"
+                        </literallayout>
+                    </note>
+                    </para></listitem>
+                <listitem><para>
+                     <filename>sstate-inputdirs</filename> and
+                     <filename>sstate-outputdirs</filename> can also be used
+                     with multiple directories.
+                     For example, the following declares
+                     <filename>PKGDESTWORK</filename> and
+                     <filename>SHLIBWORK</filename> as shared state
+                     input directories, which populates the shared state
+                     cache, and <filename>PKGDATA_DIR</filename> and
+                     <filename>SHLIBSDIR</filename> as the corresponding
+                     shared state output directories:
+                     <literallayout class='monospaced'>
+     do_package[sstate-inputdirs] = "${PKGDESTWORK} ${SHLIBSWORKDIR}"
+     do_package[sstate-outputdirs] = "${PKGDATA_DIR} ${SHLIBSDIR}"
+                     </literallayout>
+                     </para></listitem>
+                 <listitem><para>
+                     These methods also include the ability to take a lockfile
+                     when manipulating shared state directory structures,
+                     for cases where file additions or removals are sensitive:
+                     <literallayout class='monospaced'>
+     do_package[sstate-lockfile] = "${PACKAGELOCK}"
+                     </literallayout>
+                     </para></listitem>
+            </itemizedlist>
+        </para>
+
+<!--
+        <para>
             In this example, we add some extra flags to the task, a name field ("deploy"), an
             input directory where the task sends data, and the output
             directory where the data from the task should eventually be copied.
@@ -713,6 +830,7 @@
             shared state directory structures since some cases are sensitive to file
             additions or removals.
         </para>
+-->
 
         <para>
             Behind the scenes, the shared state code works by looking in

documentation/ref-manual/usingpoky.xml

@@ -356,23 +356,63 @@
     <section id='recipe-logging-mechanisms'>
         <title>Recipe Logging Mechanisms</title>
         <para>
-            Best practices exist while writing recipes that both log build progress and
-            act on build conditions such as warnings and errors.
-            Both Python and Bash language bindings exist for the logging mechanism:
+            Best practices exist while writing recipes that both log build
+            progress and act on build conditions such as warnings and errors.
+            For Python functions, the following logging functions exist.
+            All of these functions log to
+            <filename>${T}/log.do_</filename><replaceable>task</replaceable>,
+            and can also log to standard output (stdout) with the right
+            settings:
             <itemizedlist>
-                <listitem><para><emphasis>Python:</emphasis> For Python functions, BitBake
-                    supports several loglevels: <filename>bb.fatal</filename>,
-                    <filename>bb.error</filename>, <filename>bb.warn</filename>,
-                    <filename>bb.note</filename>, <filename>bb.plain</filename>,
-                    and <filename>bb.debug</filename>.</para></listitem>
-                <listitem><para><emphasis>Bash:</emphasis> For Bash functions, the same set
-                    of loglevels exist and are accessed with a similar syntax:
-                    <filename>bbfatal</filename>, <filename>bberror</filename>,
-                    <filename>bbwarn</filename>, <filename>bbnote</filename>,
-                    <filename>bbplain</filename>, and <filename>bbdebug</filename>.</para></listitem>
+                <listitem><para>
+                    <filename>bb.plain(</filename><replaceable>msg</replaceable><filename>)</filename>:
+                    Writes <replaceable>msg</replaceable> as is to the log while
+                    also logging to stdout.
+                    </para></listitem>
+                <listitem><para>
+                    <filename>bb.note(</filename><replaceable>msg</replaceable><filename>)</filename>:
+                    Writes "NOTE: <replaceable>msg</replaceable>" to the log.
+                    Also logs to stdout if BitBake is called with "-v".
+                    </para></listitem>
+                <listitem><para>
+                    <filename>bb.debug(</filename><replaceable>level</replaceable><filename>,&nbsp;</filename><replaceable>msg</replaceable><filename>)</filename>:
+                    Writes "DEBUG: <replaceable>msg</replaceable>" to the log.
+                    Also logs to stdout if the log level is greater than or
+                    equal to <replaceable>level</replaceable>.
+                    See the
+                    "<ulink url='&YOCTO_DOCS_BB_URL;#usage-and-syntax'>-D</ulink>"
+                    option in the BitBake User Manual for more information.
+                    </para></listitem>
+                <listitem><para>
+                    <filename>bb.warn(</filename><replaceable>msg</replaceable><filename>)</filename>:
+                    Writes "WARNING: <replaceable>msg</replaceable>" to the log
+                    while also logging to stdout.
+                    </para></listitem>
+                <listitem><para>
+                    <filename>bb.error(</filename><replaceable>msg</replaceable><filename>)</filename>:
+                    Writes "ERROR: <replaceable>msg</replaceable>" to the log
+                    while also logging to stdout.
+                    <note>
+                        Calling this function does not cause the task to fail.
+                    </note>
+                    </para></listitem>
+                <listitem><para>
+                    <filename>bb.fatal(</filename><replaceable>msg</replaceable><filename>)</filename>:
+                    This logging function is similar to
+                    <filename>bb.error(</filename><replaceable>msg</replaceable><filename>)</filename>
+                    but also causes the calling task to fail.
+                    </para></listitem>
             </itemizedlist>
         </para>
 
+        <para>
+            The same logging functions are also available in shell functions,
+            under the names
+            <filename>bbplain</filename>, <filename>bbnote</filename>,
+            <filename>bbdebug</filename>, <filename>bbwarn</filename>,
+            <filename>bberror</filename>, and <filename>bbfatal</filename>.
+        </para>
+
         <para>
             For guidance on how logging is handled in both Python and Bash recipes, see the
             <filename>logging.bbclass</filename> file in the

documentation/toaster-manual/toaster-manual-setup-and-use.xml

@@ -31,7 +31,7 @@
             <filename>poky/build</filename>), start Toaster using this
             command:
             <literallayout class='monospaced'>
-     $ source ../bitbake/bin/toaster
+     $ source toaster start
             </literallayout>
             You can now run your builds from the command line, or with
             Toaster as explained in section
@@ -56,7 +56,23 @@
             set a different port.
             For example, the following command sets the port to "8400":
             <literallayout class='monospaced'>
-     $ source ../bitbake/bin/toaster webport=8400
+     $ source toaster start webport=8400
+            </literallayout>
+        </para>
+    </section>
+
+    <section id='setting-a-different-address'>
+        <title>Setting a Different Address</title>
+
+        <para>
+            By default, Toaster binds to the loop back address
+            (i.e. localhost).
+            You can use the <filename>WEBPORT</filename> parameter to
+            set a different host.
+            For example, the following command sets the host and port
+            to "0.0.0.0:8400":
+            <literallayout class='monospaced'>
+     $ source toaster start webport=0.0.0.0:8400
             </literallayout>
         </para>
     </section>
@@ -116,12 +132,14 @@
             create a superuser by following these steps:
             <orderedlist>
               <listitem><para>
-                  If you used <filename>virtualenv</filename>, which is
+                  If you used <filename>pip3</filename>, which is
                   recommended, to set up the Toaster system dependencies,
-                  you need be sure the virtual environment is activated.
-                  To activate this environment, use the following command:
+                  you need be sure the local user path is in your
+                  <filename>PATH</filename> list.
+                  To append the pip3 local user path, use the following
+                  command:
                   <literallayout class='monospaced'>
-   $ source venv/bin/activate
+   $ export PATH=$PATH:$HOME/.local/bin
                   </literallayout>
                   </para></listitem>
               <listitem><para>
@@ -211,16 +229,23 @@
                       Use the Mysql database server.
                       </para></listitem>
                   <listitem><para>
-                      If you are using Ubuntu 14.04.3, run the following:
+                      If you are using Ubuntu 16.04, run the following:
                       <literallayout class='monospaced'>
-   $ sudo apt-get install apache2 libapache2-mod-wsgi mysql-server virtualenv libmysqlclient-dev
+   $ sudo apt-get install apache2 libapache2-mod-wsgi-py3 mysql-server python3-pip libmysqlclient-dev
                       </literallayout>
                       </para></listitem>
                   <listitem><para>
-                      If you are using Fedora 22 or a RedHat distribution, run
+                      If you are using Fedora 24 or a RedHat distribution, run
                       the following:
                       <literallayout class='monospaced'>
-   $ sudo dnf install httpd mod_wsgi python-virtualenv gcc mysql-devel
+   $ sudo dnf install httpd python3-mod_wsgi python3-pip mariadb-server mariadb-devel python3-devel
+                      </literallayout>
+                      </para></listitem>
+                  <listitem><para>
+                      If you are using openSUSE Leap 42.1, run
+                      the following:
+                      <literallayout class='monospaced'>
+   $ sudo zypper install apache2 apache2-mod_wsgi-python3 python3-pip mariadb mariadb-client python3-devel
                       </literallayout>
                       </para></listitem>
               </itemizedlist>
@@ -233,29 +258,32 @@
           <para>
               Perform the following steps to install Toaster:
               <orderedlist>
+                  <listitem><para>
+                      Create toaster user and set its home directory to
+                      <filename>/var/www/toaster</filename>:
+                      <literallayout class='monospaced'>
+    $ sudo /usr/sbin/useradd toaster -md /var/www/toaster -s /bin/false
+    $ sudo su - toaster -s /bin/bash
+                      </literallayout>
+                      </para></listitem>
                   <listitem><para>
                       Checkout a copy of <filename>poky</filename>
                       into the web server directory.
                       You will be using <filename>/var/www/toaster</filename>:
                       <literallayout class='monospaced'>
-   $ mkdir -p /var/www/toaster
-   $ cd /var/www/toaster/
    $ git clone git://git.yoctoproject.org/poky
    $ git checkout &DISTRO_NAME_NO_CAP;
                       </literallayout>
                       </para></listitem>
                   <listitem><para>
-                      Initialize a virtual environment and install Toaster
-                      dependencies.
-                      Using a virtual environment keeps the Python packages
+                      Install Toaster
+                      dependencies using the --user flag which
+                      keeps the Python packages
                       isolated from your system-provided packages:
                       <literallayout class='monospaced'>
    $ cd /var/www/toaster/
-   $ virtualenv venv
-   $ source ./venv/bin/activate
-   $ pip install -r ./poky/bitbake/toaster-requirements.txt
-   $ pip install mysql
-   $ pip install MySQL-python
+   $ pip3 install --user -r ./poky/bitbake/toaster-requirements.txt
+   $ pip3 install --user mysqlclient
                       </literallayout>
                       <note>
                           Isolating these packages is not required but is
@@ -270,7 +298,9 @@
                       as follows:
                       <itemizedlist>
                           <listitem><para>
-                              Edit the <filename>DATABASE</filename> settings:
+                              Edit the
+                              <ulink url='http://docs.djangoproject.com/en/1.8/ref/settings/#std:setting-SECRET_KEY'>DATABASE</ulink>
+                              settings:
                               <literallayout class='monospaced'>
    DATABASES = {
        'default': {
@@ -285,13 +315,15 @@
                               </literallayout>
                               </para></listitem>
                           <listitem><para>
-                              Edit the <filename>SECRET_KEY</filename>:
+                              Edit the
+                              <ulink url='http://docs.djangoproject.com/en/1.8/ref/settings/#std:setting-SECRET_KEY'>SECRET_KEY</ulink>:
                               <literallayout class='monospaced'>
    SECRET_KEY = '<replaceable>your_secret_key</replaceable>'
                               </literallayout>
                               </para></listitem>
                           <listitem><para>
-                              Edit the <filename>STATIC_ROOT</filename>:
+                              Edit the
+                              <ulink url='http://docs.djangoproject.com/en/1.8/ref/settings/#std:setting-SECRET_KEY'>STATIC_ROOT</ulink>:
                               <literallayout class='monospaced'>
    STATIC_ROOT = '/var/www/toaster/static_files/'
                               </literallayout>
@@ -314,9 +346,9 @@
                       default data, and gather the statically-served files:
                       <literallayout class='monospaced'>
    $ cd  /var/www/toaster/poky/
-   $ ./bitbake/lib/toaster/manage.py syncdb
    $ ./bitbake/lib/toaster/manage.py migrate
-   $ TOASTER_DIR=`pwd` TOASTER_CONF=./meta-poky/conf/toasterconf.json ./bitbake/lib/toaster/manage.py checksettings
+   $ TOASTER_DIR=`pwd` TOASTER_CONF=./meta-poky/conf/toasterconf.json \
+     ./bitbake/lib/toaster/manage.py checksettings
    $ ./bitbake/lib/toaster/manage.py collectstatic
                       </literallayout>
                       </para>
@@ -324,8 +356,8 @@
                       <para>
                           For the above set of commands, after moving to the
                           <filename>poky</filename> directory,
-                          the <filename>syncdb</filename> and <filename>migrate</filename>
-                          commands ensure the database
+                          the <filename>migrate</filename>
+                          command ensures the database
                           schema has had changes propagated correctly (i.e.
                           migrations).
                       </para>
@@ -365,7 +397,8 @@
                           Finally, the <filename>collectstatic</filename> command
                           is a Django framework command that collects all the
                           statically served files into a designated directory to
-                          be served up by the Apache web server.
+                          be served up by the Apache web server as defined by
+                          <filename>STATIC_ROOT</filename>.
                       </para></listitem>
                   <listitem><para>
                       Add an Apache configuration file for Toaster to your Apache web
@@ -377,22 +410,36 @@
                       If you are using Fedora or RedHat, put it here:
                       <literallayout class='monospaced'>
    /etc/httpd/conf.d/toaster.conf
+                      </literallayout>
+                      If you are using OpenSUSE, put it here:
+                      <literallayout class='monospaced'>
+   /etc/apache2/conf.d/toaster.conf
                       </literallayout>
                       Following is a sample Apache configuration for Toaster
                       you can follow:
                       <literallayout class='monospaced'>
    Alias /static /var/www/toaster/static_files
    &lt;Directory /var/www/toaster/static_files&gt;
-           Order allow,deny
-           Allow from all
-           Require all granted
+           &lt;IfModule mod_access_compat.c&gt;
+                   Order allow,deny
+                   Allow from all
+           &lt;/IfModule&gt;
+           &lt;IfModule !mod_access_compat.c&gt;
+                   Require all granted
+           &lt;/IfModule&gt;
    &lt;/Directory&gt;
 
-   WSGIDaemonProcess toaster_wsgi python-path=/var/www/toaster/poky/bitbake/lib/toaster:/var/www/toaster/venv/lib/python2.7/site-packages
+   &lt;Directory /var/www/toaster/poky/bitbake/lib/toaster/toastermain&gt;
+           &lt;Files "wsgi.py"&gt;
+              Require all granted
+           &lt;/Files&gt;
+   &lt;/Directory&gt;
+
+   WSGIDaemonProcess toaster_wsgi python-path=/var/www/toaster/poky/bitbake/lib/toaster:/var/www/toaster/.local/lib/python3.4/site-packages
 
    WSGIScriptAlias / "/var/www/toaster/poky/bitbake/lib/toaster/toastermain/wsgi.py"
    &lt;Location /&gt;
-       WSGIProcessGroup toastern_wsgi
+       WSGIProcessGroup toaster_wsgi
    &lt;/Location&gt;
                       </literallayout>
                       If you are using Ubuntu or Debian,
@@ -404,7 +451,7 @@
                       </literallayout>
                       Finally, restart Apache to make sure all new configuration
                       is loaded.
-                      For Ubuntu and Debian use:
+                      For Ubuntu, Debian, and OpenSUSE use:
                       <literallayout class='monospaced'>
    $ sudo service apache2 restart
                       </literallayout>
@@ -414,22 +461,53 @@
                       </literallayout>
                       </para></listitem>
                   <listitem><para>
-                      Install the build runner service.
-                      This service needs to be running in order to dispatch
-                      builds.
-                      Use this command:
+                      Prepare the systemd service to run Toaster builds.
+                      Here is a sample configuration file for the service:
                       <literallayout class='monospaced'>
-   /var/www/toaster/poky/bitbake/lib/toaster/manage.py runbuilds
+   [Unit]
+   Description=Toaster runbuilds
+
+   [Service]
+   Type=forking
+   User=toaster
+   ExecStart=/usr/bin/screen -d -m -S runbuilds /var/www/toaster/poky/bitbake/lib/toaster/runbuilds-service.sh start
+   ExecStop=/usr/bin/screen -S runbuilds -X quit
+   WorkingDirectory=/var/www/toaster/poky
+
+   [Install]
+   WantedBy=multi-user.target
                       </literallayout>
-                      Here is an example:
+                      Prepare the <filename>runbuilds-service.sh</filename>
+                      script that you need to place in the
+                      <filename>/var/www/toaster/poky/bitbake/lib/toaster/</filename>
+                      directory by setting up executable permissions:
                       <literallayout class='monospaced'>
-   #!/bin/sh
-   # toaster run builds dispatcher
-   cd /var/www/toaster/
-   source ./venv/bin/activate
-   ./bitbake/lib/toaster/manage.py runbuilds
+   #!/bin/bash
+
+   #export http_proxy=http://proxy.host.com:8080
+   #export https_proxy=http://proxy.host.com:8080
+   #export GIT_PROXY_COMMAND=$HOME/bin/gitproxy
+
+   cd ~/poky/
+   source ./oe-init-build-env build
+   source ../bitbake/bin/toaster $1 noweb
+   [ "$1" == 'start' ] &amp;&amp; /bin/bash
                       </literallayout>
                       </para></listitem>
+                  <listitem><para>
+                      Run the service:
+                      <literallayout class='monospaced'>
+   # service runbuilds start
+                      </literallayout>
+                      Since the service is running in a detached screen
+                      session, you can attach to it using this command:
+                      <literallayout class='monospaced'>
+   $ sudo su - toaster
+   $ screen -rS runbuilds
+                      </literallayout>
+                      You can detach from the service again using "Ctrl-a"
+                      followed by "d" key combination.
+                      </para></listitem>
               </orderedlist>
               You can now open up a browser and start using Toaster.
           </para>

documentation/toaster-manual/toaster-manual-start.xml

@@ -22,6 +22,11 @@
             and
             "<ulink url='&YOCTO_DOCS_QS_URL;#releases'>Yocto Project Release</ulink>"
             sections in the Yocto Project Quick Start.
+ 	    For Ubuntu/Debian, you might also need to do an additional install
+            of pip3.
+                <literallayout class='monospaced'>
+     $ sudo apt-get install python3-pip
+                </literallayout>
         </para>
     </section>
 
@@ -42,56 +47,6 @@
             install-compatible format.
         </para>
 
-        <section id='toaster-virtual-environment'>
-            <title>Set Up a Python Virtual Environment</title>
-
-            <para>
-                Set up a Python virtual environment that allows you
-                to maintain a dedicated Python executable and its own
-                set of installed modules.
-                Doing so separates the executable from Python and the
-                modules provided by the operating system.
-                This separation avoids any version conflicts.
-                <note>
-                    Creating a virtual environment is not absolutely
-                    necessary.
-                    However, doing so is highly recommended.
-                </note>
-            </para>
-
-            <para>
-                Follow these steps to set up your virtual environment.
-                These steps assume a Ubuntu distribution:
-                <orderedlist>
-                    <listitem><para><emphasis>Install <filename>virtualenv</filename>:</emphasis>
-                        Install the supported
-                        <filename>python-virtualenv</filename> package from your
-                        distribution rather than using <filename>pip</filename>.
-                        <literallayout class='monospaced'>
-     $ sudo apt-get install python-virtualenv
-                        </literallayout>
-                        </para></listitem>
-                    <listitem><para><emphasis>Create and Activate a Virtual Environment:</emphasis>
-                        <literallayout class='monospaced'>
-     $ virtualenv venv
-     $ source venv/bin/activate
-                        </literallayout>
-                        </para></listitem>
-                </orderedlist>
-                <note>
-                    After setting up a virtual environment in
-                    which to run Toaster, you must initialize that
-                    virtual environment each time you want to start
-                    Toaster.
-                    Use the following to initialize the environment just
-                    before you start Toaster:
-                    <literallayout class='monospaced'>
-     $ source venv/bin/activate
-                    </literallayout>
-                </note>
-            </para>
-        </section>
-
         <section id='toaster-load-packages'>
             <title>Install Toaster Packages</title>
 
@@ -99,7 +54,21 @@
                 You need to install the packages that Toaster requires.
                 Use this command:
                 <literallayout class='monospaced'>
-     $ pip install -r bitbake/toaster-requirements.txt
+     $ $ pip3 install --user -r bitbake/toaster-requirements.txt
+                </literallayout>
+                The previous command installs the necessary Toaster modules
+                into a local python 3 cache in your
+                <filename>$HOME</filename> directory.
+                The caches is actually located in
+                <filename>$HOME/.local</filename>.
+                To see what packages have been installed into your
+                <filename>$HOME</filename> directory, do the following:
+                <literallayout class='monospaced'>
+     $ pip3 list installed --local
+                </literallayout>
+                If you need to remove something, the following works:
+                <literallayout class='monospaced'>
+     $ pip3 uninstall PackageNameToUninstall
                 </literallayout>
             </para>
         </section>

meta-poky/conf/distro/include/maintainers.inc

@@ -367,7 +367,6 @@ RECIPE_MAINTAINER_pn-libidn = "Alexander Kanavin <alexander.kanavin@intel.com>"
 RECIPE_MAINTAINER_pn-libinput = "Jussi Kukkonen <jussi.kukkonen@intel.com>"
 RECIPE_MAINTAINER_pn-libjson = "Alexander Kanavin <alexander.kanavin@intel.com>"
 RECIPE_MAINTAINER_pn-libksba = "Alexander Kanavin <alexander.kanavin@intel.com>"
-RECIPE_MAINTAINER_pn-libmad = "Tanu Kaskinen <tanuk@iki.fi>"
 RECIPE_MAINTAINER_pn-libmatchbox = "Jussi Kukkonen <jussi.kukkonen@intel.com>"
 RECIPE_MAINTAINER_pn-libmpc = "Richard Purdie <richard.purdie@linuxfoundation.org>"
 RECIPE_MAINTAINER_pn-libnewt = "Hongxu Jia <hongxu.jia@windriver.com>"
@@ -520,6 +519,7 @@ RECIPE_MAINTAINER_pn-mobile-broadband-provider-info = "Alexander Kanavin <alexan
 RECIPE_MAINTAINER_pn-modutils-initscripts = "Robert Yang <liezhi.yang@windriver.com>"
 RECIPE_MAINTAINER_pn-mpeg2dec = "Alexander Kanavin <alexander.kanavin@intel.com>"
 RECIPE_MAINTAINER_pn-mpfr = "Richard Purdie <richard.purdie@linuxfoundation.org>"
+RECIPE_MAINTAINER_pn-mpg123 = "Alexander Kanavin <alexander.kanavin@intel.com>"
 RECIPE_MAINTAINER_pn-msmtp = "Alexander Kanavin <alexander.kanavin@intel.com>"
 RECIPE_MAINTAINER_pn-mtd-utils = "Alexander Kanavin <alexander.kanavin@intel.com>"
 RECIPE_MAINTAINER_pn-mtdev = "Jussi Kukkonen <jussi.kukkonen@intel.com>"
@@ -759,7 +759,7 @@ RECIPE_MAINTAINER_pn-usbutils = "Alexander Kanavin <alexander.kanavin@intel.com>
 RECIPE_MAINTAINER_pn-util-linux = "Chen Qi <Qi.Chen@windriver.com>"
 RECIPE_MAINTAINER_pn-util-macros = "Jussi Kukkonen <jussi.kukkonen@intel.com>"
 RECIPE_MAINTAINER_pn-v86d = "Alexander Kanavin <alexander.kanavin@intel.com>"
-RECIPE_MAINTAINER_pn-vala = "Jussi Kukkonen <jussi.kukkonen@intel.com>"
+RECIPE_MAINTAINER_pn-vala = "Alexander Kanavin <alexander.kanavin@intel.com>"
 RECIPE_MAINTAINER_pn-valgrind = "Alexander Kanavin <alexander.kanavin@intel.com>"
 RECIPE_MAINTAINER_pn-videoproto = "Jussi Kukkonen <jussi.kukkonen@intel.com>"
 RECIPE_MAINTAINER_pn-volatile-binds = "Chen Qi <Qi.Chen@windriver.com>"

meta-poky/conf/distro/poky.conf

@@ -78,12 +78,9 @@ SANITY_TESTED_DISTROS ?= " \
             Ubuntu-15.04 \n \
             Ubuntu-15.10 \n \
             Ubuntu-16.04 \n \
-            Fedora-21 \n \
             Fedora-22 \n \
             Fedora-23 \n \
-            CentOS-6.* \n \
             CentOS-7.* \n \
-            Debian-7.* \n \
             Debian-8.* \n \
             openSUSE-project-13.2 \n \
             "

meta-poky/conf/local.conf.sample.extended

@@ -371,16 +371,6 @@
 #IPK_HIERARCHICAL_FEED = "1"
 #
 
-# Using RPM4
-#
-# Currently the rootfs_rpm code has a hard depends on rpmresolve:do_populate_sysroot,
-# when using rpm4 the rpmresolve code will not compile due to a missing header file.
-# That dependency needs to be removed when using RPM4, also the PREFERRED_VERSION needs
-# to be set. This example shows how to enable rpm4
-# PREFERRED_VERSION_rpm = "4.11.2"
-# PREFERRED_VERSION_rpm-native = "4.11.2"
-# RPMROOTFSDEPENDS_remove = "rpmresolve-native:do_populate_sysroot"
-
 #
 # Use busybox/mdev for system initialization
 #

meta-poky/conf/toasterconf.json

@@ -40,13 +40,13 @@
             "name": "OpenEmbedded",
             "sourcetype": "layerindex",
             "apiurl": "http://layers.openembedded.org/layerindex/api/",
-            "branches": ["master", "jethro", "krogoth"]
+            "branches": ["master"]
         },
         {
             "name": "Imported layers",
             "sourcetype": "imported",
             "apiurl": "",
-            "branches": ["master", "jethro", "krogoth", "HEAD"]
+            "branches": ["master", "HEAD"]
 
         }
     ],
@@ -57,18 +57,6 @@
             "branch": "master",
             "dirpath": "bitbake"
         },
-        {
-            "name": "jethro",
-            "giturl": "remote:origin",
-            "branch": "jethro",
-            "dirpath": "bitbake"
-        },
-        {
-            "name": "krogoth",
-            "giturl": "remote:origin",
-            "branch": "krogoth",
-            "dirpath": "bitbake"
-        },
         {
             "name": "HEAD",
             "giturl": "remote:origin",
@@ -89,24 +77,6 @@
             "layersourcepriority": { "Imported layers": 99, "Local Yocto Project" : 10, "OpenEmbedded" :  0 },
             "helptext": "Toaster will run your builds using the tip of the <a href=\"http://git.yoctoproject.org/cgit/cgit.cgi/poky/log/\">Yocto Project master branch</a>, where active development takes place. This is not a stable branch, so your builds might not work as expected."
         },
-        {
-            "name": "jethro",
-            "description": "Yocto Project 2.0 Jethro",
-            "bitbake": "jethro",
-            "branch": "jethro",
-            "defaultlayers": [ "openembedded-core", "meta-yocto", "meta-yocto-bsp"],
-            "layersourcepriority": { "Imported layers": 99, "Local Yocto Project" : 10, "OpenEmbedded" :  0 },
-            "helptext": "Toaster will run your builds with the tip of the <a href=\"http://git.yoctoproject.org/cgit/cgit.cgi/poky/log/?h=jethro\">Yocto Project 2.0 \"Jethro\"</a> branch."
-        },
-        {
-            "name": "krogoth",
-            "description": "Yocto Project 2.1 Krogoth",
-            "bitbake": "krogoth",
-            "branch": "krogoth",
-            "defaultlayers": [ "openembedded-core", "meta-poky", "meta-yocto-bsp"],
-            "layersourcepriority": { "Imported layers": 99, "Local Yocto Project" : 10, "OpenEmbedded" :  0 },
-            "helptext": "Toaster will run your builds with the tip of the <a href=\"http://git.yoctoproject.org/cgit/cgit.cgi/poky/log/?h=krogoth\">Yocto Project 2.1 \"Krogoth\"</a> branch."
-        },
         {
             "name": "local",
             "description": "Local Yocto Project",

meta-skeleton/recipes-skeleton/service/service_0.1.bb

@@ -9,7 +9,7 @@ SRC_URI = "file://skeleton \
 	   "
 
 do_compile () {
-	${CC} ${WORKDIR}/skeleton_test.c -o ${WORKDIR}/skeleton-test
+	${CC} ${CFLAGS} ${LDFLAGS} ${WORKDIR}/skeleton_test.c -o ${WORKDIR}/skeleton-test
 }
 
 do_install () {

meta-yocto-bsp/conf/machine/mpc8315e-rdb.conf

@@ -22,7 +22,6 @@ XSERVER ?= "xserver-xorg \
             xf86-input-evdev \
             xf86-video-fbdev"
 
-PREFERRED_VERSION_u-boot ?= "v2013.07%"
 UBOOT_ENTRYPOINT = "0x00000000"
 
 KERNEL_DEVICETREE = "mpc8315erdb.dtb"

meta-yocto-bsp/recipes-kernel/linux/linux-yocto_4.1.bbappend

@@ -7,8 +7,8 @@ KBRANCH_mpc8315e-rdb = "standard/fsl-mpc8315e-rdb"
 KMACHINE_genericx86 ?= "common-pc"
 KMACHINE_genericx86-64 ?= "common-pc-64"
 
-SRCREV_machine_genericx86    ?= "9ba8c36e9ea7419d06accab5311e7fb0d56513ff"
-SRCREV_machine_genericx86-64 ?= "9ba8c36e9ea7419d06accab5311e7fb0d56513ff"
+SRCREV_machine_genericx86    ?= "75d56a13f86fc48002e4a3f9ed60546db30432b7"
+SRCREV_machine_genericx86-64 ?= "75d56a13f86fc48002e4a3f9ed60546db30432b7"
 SRCREV_machine_edgerouter ?= "403eda4633e9037fb715d0d1e8ae847b2bd0651a"
 SRCREV_machine_beaglebone ?= "859aecf685fcd9d30490a6da459fb76b48947075"
 SRCREV_machine_mpc8315e-rdb ?= "403eda4633e9037fb715d0d1e8ae847b2bd0651a"

meta-yocto-bsp/recipes-kernel/linux/linux-yocto_4.4.bbappend

@@ -7,8 +7,8 @@ KBRANCH_edgerouter = "standard/edgerouter"
 KBRANCH_beaglebone = "standard/beaglebone"
 KBRANCH_mpc8315e-rdb = "standard/fsl-mpc8315e-rdb"
 
-SRCREV_machine_genericx86    ?= "bc64c8124504681545cb97a22b69a4e4bfeb55e2"
-SRCREV_machine_genericx86-64 ?= "bc64c8124504681545cb97a22b69a4e4bfeb55e2"
+SRCREV_machine_genericx86    ?= "0e30a74f2cbbab0c7014561fe4eab1c9d8bfe560"
+SRCREV_machine_genericx86-64 ?= "0e30a74f2cbbab0c7014561fe4eab1c9d8bfe560"
 SRCREV_machine_edgerouter ?= "628bf627561c6285d99fb978e11d4c15fc29324b"
 SRCREV_machine_beaglebone ?= "628bf627561c6285d99fb978e11d4c15fc29324b"
 SRCREV_machine_mpc8315e-rdb ?= "94ac8da44990afd2d43c0ccd713420fb1cfa0792"

meta/classes/autotools.bbclass

@@ -312,6 +312,10 @@ autotools_do_configure() {
 	fi
 }
 
+autotools_do_compile() {
+    oe_runmake
+}
+
 autotools_do_install() {
 	oe_runmake 'DESTDIR=${D}' install
 	# Info dir listing isn't interesting at this point so remove it if it exists.
@@ -322,6 +326,6 @@ autotools_do_install() {
 
 inherit siteconfig
 
-EXPORT_FUNCTIONS do_configure do_install
+EXPORT_FUNCTIONS do_configure do_compile do_install
 
 B = "${WORKDIR}/build"

meta/classes/buildstats-summary.bbclass

@@ -30,7 +30,11 @@ python buildstats_summary () {
                 header_printed = True
                 bb.note("Build completion summary:")
 
-            bb.note("  {0}: {1}% sstate reuse ({2} setscene, {3} scratch)".format(t, 100*len(sstate)/(len(sstate)+len(no_sstate)), len(sstate), len(no_sstate)))
+            sstate_count = len(sstate)
+            no_sstate_count = len(no_sstate)
+            total_count = sstate_count + no_sstate_count
+            bb.note("  {0}: {1:.1f}% sstate reuse({2} setscene, {3} scratch)".format(
+                t, round(100 * sstate_count / total_count, 1), sstate_count, no_sstate_count))
 }
 addhandler buildstats_summary
 buildstats_summary[eventmask] = "bb.event.BuildCompleted"

meta/classes/cmake.bbclass

@@ -124,6 +124,7 @@ cmake_do_configure() {
 	  -Wno-dev
 }
 
+do_compile[progress] = "percent"
 cmake_do_compile()  {
 	cd ${B}
 	base_do_compile VERBOSE=1

meta/classes/devshell.bbclass

@@ -65,9 +65,6 @@ def devpyshell(d):
         os.dup2(m, sys.stdout.fileno())
         os.dup2(m, sys.stderr.fileno())
 
-        sys.stdout = os.fdopen(sys.stdout.fileno(), 'w', 0)
-        sys.stdin = os.fdopen(sys.stdin.fileno(), 'r', 0)
-
         bb.utils.nonblockingfd(sys.stdout)
         bb.utils.nonblockingfd(sys.stderr)
         bb.utils.nonblockingfd(sys.stdin)
@@ -93,6 +90,7 @@ def devpyshell(d):
             else:
                 prompt = ps1
             sys.stdout.write(prompt)
+            sys.stdout.flush()
 
         # Restore Ctrl+C since bitbake masks this
         def signal_handler(signal, frame):
@@ -114,6 +112,7 @@ def devpyshell(d):
                         continue
                 except EOFError as e:
                     sys.stdout.write("\n")
+                    sys.stdout.flush()
                 except (OSError, IOError) as e:
                     if e.errno == 11:
                         continue

meta/classes/gobject-introspection.bbclass

@@ -24,11 +24,6 @@ DEPENDS_append_class-target = " gobject-introspection gobject-introspection-nati
 DEPENDS_append_class-native = " gobject-introspection-native"
 DEPENDS_append_class-nativesdk = " gobject-introspection-native"
 
-# This is necessary for python scripts to succeed - distutils fails if these
-# are not set
-export STAGING_INCDIR
-export STAGING_LIBDIR
-
 # This is used by introspection tools to find .gir includes
 export XDG_DATA_DIRS = "${STAGING_DATADIR}"
 

meta/classes/image.bbclass

@@ -198,6 +198,17 @@ fakeroot python do_rootfs () {
     from oe.rootfs import create_rootfs
     from oe.manifest import create_manifest
 
+    # NOTE: if you add, remove or significantly refactor the stages of this
+    # process then you should recalculate the weightings here. This is quite
+    # easy to do - just change the MultiStageProgressReporter line temporarily
+    # to pass debug=True as the last parameter and you'll get a printout of
+    # the weightings as well as a map to the lines where next_stage() was
+    # called. Of course this isn't critical, but it helps to keep the progress
+    # reporting accurate.
+    stage_weights = [1, 203, 354, 186, 65, 4228, 1, 353, 49, 330, 382, 23, 1]
+    progress_reporter = bb.progress.MultiStageProgressReporter(d, stage_weights)
+    progress_reporter.next_stage()
+
     # Handle package exclusions
     excl_pkgs = d.getVar("PACKAGE_EXCLUDE", True).split()
     inst_pkgs = d.getVar("PACKAGE_INSTALL", True).split()
@@ -230,8 +241,12 @@ fakeroot python do_rootfs () {
     # Generate the initial manifest
     create_manifest(d)
 
-    # Generate rootfs
-    create_rootfs(d)
+    progress_reporter.next_stage()
+
+    # generate rootfs
+    create_rootfs(d, progress_reporter=progress_reporter)
+
+    progress_reporter.finish()
 }
 do_rootfs[dirs] = "${TOPDIR}"
 do_rootfs[cleandirs] += "${S}"
@@ -260,6 +275,36 @@ do_image_complete[dirs] = "${TOPDIR}"
 do_image_complete[umask] = "022"
 addtask do_image_complete after do_image before do_build
 
+# Add image-level QA/sanity checks to IMAGE_QA_COMMANDS
+#
+# IMAGE_QA_COMMANDS += " \
+#     image_check_everything_ok \
+# "
+# This task runs all functions in IMAGE_QA_COMMANDS after the image
+# construction has completed in order to validate the resulting image.
+fakeroot python do_image_qa () {
+    from oe.utils import ImageQAFailed
+
+    qa_cmds = (d.getVar('IMAGE_QA_COMMANDS', True) or '').split()
+    qamsg = ""
+
+    for cmd in qa_cmds:
+        try:
+            bb.build.exec_func(cmd, d)
+        except oe.utils.ImageQAFailed as e:
+            qamsg = qamsg + '\tImage QA function %s failed: %s\n' % (e.name, e.description)
+        except bb.build.FuncFailed as e:
+            qamsg = qamsg + '\tImage QA function %s failed' % e.name
+            if e.logfile:
+                qamsg = qamsg + ' (log file is located at %s)' % e.logfile
+            qamsg = qamsg + '\n'
+
+    if qamsg:
+        imgname = d.getVar('IMAGE_NAME', True)
+        bb.fatal("QA errors found whilst validating image: %s\n%s" % (imgname, qamsg))
+}
+addtask do_image_qa after do_image_complete before do_build
+
 #
 # Write environment variables used by wic
 # to tmp/sysroots/<machine>/imgdata/<image>.env

meta/classes/image_types.bbclass

@@ -215,7 +215,54 @@ IMAGE_CMD_wic[vardepsexclude] = "WKS_FULL_PATH WKS_FILES"
 
 # Rebuild when the wks file or vars in WICVARS change
 USING_WIC = "${@bb.utils.contains_any('IMAGE_FSTYPES', 'wic ' + ' '.join('wic.%s' % c for c in '${COMPRESSIONTYPES}'.split()), '1', '', d)}"
-do_image_wic[file-checksums] += "${@'${WKS_FULL_PATH}:%s' % os.path.exists('${WKS_FULL_PATH}') if '${USING_WIC}' else ''}"
+WKS_FILE_CHECKSUM = "${@'${WKS_FULL_PATH}:%s' % os.path.exists('${WKS_FULL_PATH}') if '${USING_WIC}' else ''}"
+do_image_wic[file-checksums] += "${WKS_FILE_CHECKSUM}"
+
+python do_write_wks_template () {
+    """Write out expanded template contents to WKS_FULL_PATH."""
+    import re
+
+    template_body = d.getVar('_WKS_TEMPLATE', True)
+
+    # Remove any remnant variable references left behind by the expansion
+    # due to undefined variables
+    expand_var_regexp = re.compile(r"\${[^{}@\n\t :]+}")
+    while True:
+        new_body = re.sub(expand_var_regexp, '', template_body)
+        if new_body == template_body:
+            break
+        else:
+            template_body = new_body
+
+    wks_file = d.getVar('WKS_FULL_PATH', True)
+    with open(wks_file, 'w') as f:
+        f.write(template_body)
+}
+
+python () {
+    if d.getVar('USING_WIC', True):
+        wks_file_u = d.getVar('WKS_FULL_PATH', False)
+        wks_file = d.expand(wks_file_u)
+        base, ext = os.path.splitext(wks_file)
+        if ext == '.in' and os.path.exists(wks_file):
+            wks_out_file = os.path.join(d.getVar('WORKDIR', True), os.path.basename(base))
+            d.setVar('WKS_FULL_PATH', wks_out_file)
+            d.setVar('WKS_TEMPLATE_PATH', wks_file_u)
+            d.setVar('WKS_FILE_CHECKSUM', '${WKS_TEMPLATE_PATH}:True')
+
+            try:
+                with open(wks_file, 'r') as f:
+                    body = f.read()
+            except (IOError, OSError) as exc:
+                pass
+            else:
+                # Previously, I used expandWithRefs to get the dependency list
+                # and add it to WICVARS, but there's no point re-parsing the
+                # file in process_wks_template as well, so just put it in
+                # a variable and let the metadata deal with the deps.
+                d.setVar('_WKS_TEMPLATE', body)
+                bb.build.addtask('do_write_wks_template', 'do_image_wic', None, d)
+}
 
 EXTRA_IMAGECMD = ""
 

meta/classes/kernel-module-split.bbclass

@@ -30,6 +30,8 @@ PACKAGESPLITFUNCS_prepend = "split_kernel_module_packages "
 
 KERNEL_MODULES_META_PACKAGE ?= "kernel-modules"
 
+KERNEL_MODULE_PACKAGE_PREFIX ?= ""
+
 python split_kernel_module_packages () {
     import re
 
@@ -181,7 +183,9 @@ python split_kernel_module_packages () {
 
     module_deps = parse_depmod()
     module_regex = '^(.*)\.k?o$'
-    module_pattern = 'kernel-module-%s'
+
+    module_pattern_prefix = d.getVar('KERNEL_MODULE_PACKAGE_PREFIX', True)
+    module_pattern = module_pattern_prefix + 'kernel-module-%s'
 
     postinst = d.getVar('pkg_postinst_modules', True)
     postrm = d.getVar('pkg_postrm_modules', True)

meta/classes/kernel.bbclass

@@ -232,8 +232,7 @@ do_bundle_initramfs () {
 			if [ -n "$realpath" ]; then
 				mv -f $realpath $realpath.initramfs
 				mv -f $realpath.bak $realpath
-				cd ${B}/${KERNEL_OUTPUT_DIR}
-				ln -sf $linkpath.initramfs
+				ln -sf $linkpath.initramfs ${B}/${KERNEL_OUTPUT_DIR}/$type.initramfs
 			else
 				mv -f ${KERNEL_OUTPUT_DIR}/$type ${KERNEL_OUTPUT_DIR}/$type.initramfs
 				mv -f ${KERNEL_OUTPUT_DIR}/$type.bak ${KERNEL_OUTPUT_DIR}/$type
@@ -586,8 +585,7 @@ kernel_do_deploy() {
 			initramfs_base_name=${type}-${INITRAMFS_BASE_NAME}
 			initramfs_symlink_name=${type}-initramfs-${MACHINE}
 			install -m 0644 ${KERNEL_OUTPUT_DIR}/${type}.initramfs ${DEPLOYDIR}/${initramfs_base_name}.bin
-			cd ${DEPLOYDIR}
-			ln -sf ${initramfs_base_name}.bin ${initramfs_symlink_name}.bin
+			ln -sf ${initramfs_base_name}.bin ${DEPLOYDIR}/${initramfs_symlink_name}.bin
 		fi
 	done
 }

meta/classes/libc-package.bbclass

@@ -47,15 +47,6 @@ python __anonymous () {
 
 OVERRIDES_append = ":${TARGET_ARCH}-${TARGET_OS}"
 
-do_configure_prepend() {
-        if [ -e ${S}/elf/ldd.bash.in ]; then
-                sed -e "s#@BASH@#/bin/sh#" -i ${S}/elf/ldd.bash.in
-        fi
-}
-
-
-
-# indentation removed on purpose
 locale_base_postinst() {
 #!/bin/sh
 
@@ -63,33 +54,14 @@ if [ "x$D" != "x" ]; then
 	exit 1
 fi
 
-rm -rf ${TMP_LOCALE}
-mkdir -p ${TMP_LOCALE}
-if [ -f ${localedir}/locale-archive ]; then
-        cp ${localedir}/locale-archive ${TMP_LOCALE}/
-fi
-localedef --inputfile=${datadir}/i18n/locales/%s --charmap=%s --prefix=/tmp/locale %s
-mkdir -p ${localedir}/
-mv ${TMP_LOCALE}/locale-archive ${localedir}/
-rm -rf ${TMP_LOCALE}
+localedef --inputfile=${datadir}/i18n/locales/%s --charmap=%s %s
 }
 
-# indentation removed on purpose
 locale_base_postrm() {
 #!/bin/sh
-
-rm -rf ${TMP_LOCALE}
-mkdir -p ${TMP_LOCALE}
-if [ -f ${localedir}/locale-archive ]; then
-	cp ${localedir}/locale-archive ${TMP_LOCALE}/
-fi
-localedef --delete-from-archive --inputfile=${datadir}/locales/%s --charmap=%s --prefix=/tmp/locale %s
-mv ${TMP_LOCALE}/locale-archive ${localedir}/
-rm -rf ${TMP_LOCALE}
+localedef --delete-from-archive --inputfile=${datadir}/locales/%s --charmap=%s %s
 }
 
-
-TMP_LOCALE="/tmp/locale${localedir}"
 LOCALETREESRC ?= "${PKGD}"
 
 do_prep_locale_tree() {
@@ -290,7 +262,7 @@ python package_do_split_gconvs () {
                 bb.error("locale_arch_options not found for target_arch=" + target_arch)
                 raise bb.build.FuncFailed("unknown arch:" + target_arch + " for locale_arch_options")
 
-            localedef_opts += " --force --old-style --no-archive --prefix=%s \
+            localedef_opts += " --force  --no-archive --prefix=%s \
                 --inputfile=%s/%s/i18n/locales/%s --charmap=%s %s/%s" \
                 % (treedir, treedir, datadir, locale, encoding, outputpath, name)
 
@@ -298,7 +270,7 @@ python package_do_split_gconvs () {
                 (path, i18npath, gconvpath, localedef_opts)
         else: # earlier slower qemu way 
             qemu = qemu_target_binary(d) 
-            localedef_opts = "--force --old-style --no-archive --prefix=%s \
+            localedef_opts = "--force --no-archive --prefix=%s \
                 --inputfile=%s/i18n/locales/%s --charmap=%s %s" \
                 % (treedir, datadir, locale, encoding, name)
 

meta/classes/license.bbclass

@@ -668,8 +668,6 @@ do_rootfs[recrdeptask] += "do_populate_lic"
 IMAGE_POSTPROCESS_COMMAND_prepend = "write_deploy_manifest; "
 do_image[recrdeptask] += "do_populate_lic"
 
-do_populate_lic_setscene[dirs] = "${LICSSTATEDIR}/${PN}"
-do_populate_lic_setscene[cleandirs] = "${LICSSTATEDIR}"
 python do_populate_lic_setscene () {
     sstate_setscene(d)
 }

meta/classes/populate_sdk_ext.bbclass

@@ -39,7 +39,7 @@ SDK_UPDATE_URL ?= ""
 
 SDK_TARGETS ?= "${PN}"
 
-def get_sdk_install_targets(d):
+def get_sdk_install_targets(d, images_only=False):
     sdk_install_targets = ''
     if d.getVar('SDK_EXT_TYPE', True) != 'minimal':
         sdk_install_targets = d.getVar('SDK_TARGETS', True)
@@ -50,8 +50,9 @@ def get_sdk_install_targets(d):
                 if v[0] not in sdk_install_targets:
                     sdk_install_targets += ' {}'.format(v[0])
 
-    if d.getVar('SDK_INCLUDE_PKGDATA', True) == '1':
-        sdk_install_targets += ' meta-world-pkgdata:do_allpackagedata'
+    if not images_only:
+        if d.getVar('SDK_INCLUDE_PKGDATA', True) == '1':
+            sdk_install_targets += ' meta-world-pkgdata:do_allpackagedata'
 
     return sdk_install_targets
 
@@ -223,10 +224,13 @@ python copy_buildsystem () {
             # warning.
             f.write('SIGGEN_LOCKEDSIGS_SSTATE_EXISTS_CHECK = "none"\n\n')
 
-            # Error if the sigs in the locked-signature file don't match
+            # Warn if the sigs in the locked-signature file don't match
             # the sig computed from the metadata.
             f.write('SIGGEN_LOCKEDSIGS_TASKSIG_CHECK = "warn"\n\n')
 
+            # Set up whitelist for run on install
+            f.write('BB_SETSCENE_ENFORCE_WHITELIST = "%:* *:do_shared_workdir *:do_rm_work"\n\n')
+
             # Hide the config information from bitbake output (since it's fixed within the SDK)
             f.write('BUILDCFG_HEADER = ""\n')
 
@@ -280,7 +284,7 @@ python copy_buildsystem () {
             f.write('\n')
 
     # Filter the locked signatures file to just the sstate tasks we are interested in
-    excluded_targets = d.getVar('SDK_TARGETS', True)
+    excluded_targets = get_sdk_install_targets(d, images_only=True)
     sigfile = d.getVar('WORKDIR', True) + '/locked-sigs.inc'
     lockedsigs_pruned = baseoutpath + '/conf/locked-sigs.inc'
     oe.copy_buildsystem.prune_lockedsigs([],
@@ -424,7 +428,7 @@ sdk_ext_postinst() {
 		# current working directory when first ran, nor will it set $1 when
 		# sourcing a script. That is why this has to look so ugly.
 		LOGFILE="$target_sdk_dir/preparing_build_system.log"
-		sh -c ". buildtools/environment-setup* > $LOGFILE && cd $target_sdk_dir/`dirname ${oe_init_build_env_path}` && set $target_sdk_dir && . $target_sdk_dir/${oe_init_build_env_path} $target_sdk_dir >> $LOGFILE && python $target_sdk_dir/ext-sdk-prepare.py '${SDK_INSTALL_TARGETS}' >> $LOGFILE 2>&1" || { echo "ERROR: SDK preparation failed: see $LOGFILE"; cat "$LOGFILE"; echo "printf 'ERROR: this SDK was not fully installed and needs reinstalling\n'" >> $env_setup_script ; exit 1 ; }
+		sh -c ". buildtools/environment-setup* > $LOGFILE && cd $target_sdk_dir/`dirname ${oe_init_build_env_path}` && set $target_sdk_dir && . $target_sdk_dir/${oe_init_build_env_path} $target_sdk_dir >> $LOGFILE && python $target_sdk_dir/ext-sdk-prepare.py '${SDK_INSTALL_TARGETS}' >> $LOGFILE 2>&1" || { echo "ERROR: SDK preparation failed: see $LOGFILE for a slightly more detailed log"; echo "printf 'ERROR: this SDK was not fully installed and needs reinstalling\n'" >> $env_setup_script ; exit 1 ; }
 		rm $target_sdk_dir/ext-sdk-prepare.py
 	fi
 	echo done
@@ -447,7 +451,13 @@ fakeroot python do_populate_sdk_ext() {
 }
 
 def get_ext_sdk_depends(d):
-    return d.getVarFlag('do_rootfs', 'depends', True) + ' ' + d.getVarFlag('do_build', 'depends', True)
+    # Note: the deps varflag is a list not a string, so we need to specify expand=False
+    deps = d.getVarFlag('do_image_complete', 'deps', False)
+    pn = d.getVar('PN', True)
+    deplist = ['%s:%s' % (pn, dep) for dep in deps]
+    for task in ['do_image_complete', 'do_rootfs', 'do_build']:
+        deplist.extend((d.getVarFlag(task, 'depends', True) or '').split())
+    return ' '.join(deplist)
 
 python do_sdk_depends() {
     # We have to do this separately in its own task so we avoid recursing into

meta/classes/python3native.bbclass

@@ -2,4 +2,12 @@ inherit python3-dir
 
 PYTHON="${STAGING_BINDIR_NATIVE}/${PYTHON_PN}-native/${PYTHON_PN}"
 EXTRANATIVEPATH += "${PYTHON_PN}-native"
-DEPENDS += " ${PYTHON_PN}-native "
+DEPENDS_append = " ${PYTHON_PN}-native "
+
+# python-config and other scripts are using distutils modules
+# which we patch to access these variables
+export STAGING_INCDIR
+export STAGING_LIBDIR
+
+# autoconf macros will use their internal default preference otherwise
+export PYTHON

meta/classes/pythonnative.bbclass

@@ -5,4 +5,12 @@ PYTHON="${STAGING_BINDIR_NATIVE}/${PYTHON_PN}-native/${PYTHON_PN}"
 # PYTHON_EXECUTABLE is used by cmake
 PYTHON_EXECUTABLE="${PYTHON}"
 EXTRANATIVEPATH += "${PYTHON_PN}-native"
-DEPENDS += " ${PYTHON_PN}-native "
+DEPENDS_append = " ${PYTHON_PN}-native "
+
+# python-config and other scripts are using distutils modules
+# which we patch to access these variables
+export STAGING_INCDIR
+export STAGING_LIBDIR
+
+# autoconf macros will use their internal default preference otherwise
+export PYTHON

meta/classes/report-error.bbclass

@@ -59,9 +59,11 @@ python errorreport_handler () {
                     logFile = codecs.open(log, 'r', 'utf-8')
                     logdata = logFile.read()
 
-                    topdir = e.data.getVar('TOPDIR', True)
-                    if topdir:
-                        logdata = logdata.replace(topdir, ' ')
+                    # Replace host-specific paths so the logs are cleaner
+                    for d in ("TOPDIR", "TMPDIR"):
+                        s = e.data.getVar(d, True)
+                        if s:
+                            logdata = logdata.replace(s, d)
 
                     logFile.close()
                 except:

meta/classes/rootfs-postcommands.bbclass

@@ -45,6 +45,15 @@ systemd_create_users () {
 			[ "$id" != "-" ] && groupadd_params="$groupadd_params --gid $id"
 			groupadd_params="$groupadd_params --system $name"
 			eval groupadd --root ${IMAGE_ROOTFS} $groupadd_params || true
+		elif [ "$type" = "m" ]; then
+			group=$id
+			if [ ! `grep -q "^${group}:" ${IMAGE_ROOTFS}${sysconfdir}/group` ]; then
+				eval groupadd --root ${IMAGE_ROOTFS} --system $group
+			fi
+			if [ ! `grep -q "^${name}:" ${IMAGE_ROOTFS}${sysconfdir}/passwd` ]; then
+				eval useradd --root ${IMAGE_ROOTFS} --shell /sbin/nologin --system $name
+			fi
+			eval usermod --root ${IMAGE_ROOTFS} -a -G $group $name
 		fi
 		done
 	done

meta/classes/sstate.bbclass

@@ -17,6 +17,9 @@ SSTATE_EXTRAPATH   = ""
 SSTATE_EXTRAPATHWILDCARD = ""
 SSTATE_PATHSPEC   = "${SSTATE_DIR}/${SSTATE_EXTRAPATHWILDCARD}*/${SSTATE_PKGSPEC}"
 
+# explicitly make PV to depend on evaluated value of PV variable
+PV[vardepvalue] = "${PV}"
+
 # We don't want the sstate to depend on things like the distro string
 # of the system, we let the sstate paths take care of this.
 SSTATE_EXTRAPATH[vardepvalue] = ""
@@ -719,6 +722,7 @@ def sstate_checkhashes(sq_fn, sq_task, sq_hash, sq_hashfn, d, siginfo=False):
 
     ret = []
     missed = []
+    missing = []
     extension = ".tgz"
     if siginfo:
         extension = extension + ".siginfo"
@@ -740,6 +744,18 @@ def sstate_checkhashes(sq_fn, sq_task, sq_hash, sq_hashfn, d, siginfo=False):
 
         return spec, extrapath, tname
 
+    def sstate_pkg_to_pn(pkg, d):
+        """
+        Translate an sstate filename to a PN value by way of SSTATE_PKGSPEC. This is slightly hacky but
+        we don't have access to everything in this context.
+        """
+        pkgspec = d.getVar('SSTATE_PKGSPEC', False)
+        try:
+            idx = pkgspec.split(':').index('${PN}')
+        except ValueError:
+            bb.fatal('Unable to find ${PN} in SSTATE_PKGSPEC')
+        return pkg.split(':')[idx]
+
 
     for task in range(len(sq_fn)):
 
@@ -774,6 +790,8 @@ def sstate_checkhashes(sq_fn, sq_task, sq_hash, sq_hashfn, d, siginfo=False):
         if localdata.getVar('BB_NO_NETWORK', True) == "1" and localdata.getVar('SSTATE_MIRROR_ALLOW_NETWORK', True) == "1":
             localdata.delVar('BB_NO_NETWORK')
 
+        whitelist = bb.runqueue.get_setscene_enforce_whitelist(d)
+
         from bb.fetch2 import FetchConnectionCache
         def checkstatus_init(thread_worker):
             thread_worker.connection_cache = FetchConnectionCache()
@@ -800,7 +818,14 @@ def sstate_checkhashes(sq_fn, sq_task, sq_hash, sq_hashfn, d, siginfo=False):
             except:
                 missed.append(task)
                 bb.debug(2, "SState: Unsuccessful fetch test for %s" % srcuri)
-                pass     
+                if whitelist:
+                    pn = sstate_pkg_to_pn(sstatefile, d)
+                    taskname = sq_task[task]
+                    if not bb.runqueue.check_setscene_enforce_whitelist(pn, taskname, whitelist):
+                        missing.append(task)
+                        bb.error('Sstate artifact unavailable for %s.%s' % (pn, taskname))
+                pass
+            bb.event.fire(bb.event.ProcessProgress("Checking sstate mirror object availability", len(tasklist) - thread_worker.tasks.qsize()), d)
 
         tasklist = []
         for task in range(len(sq_fn)):
@@ -811,7 +836,7 @@ def sstate_checkhashes(sq_fn, sq_task, sq_hash, sq_hashfn, d, siginfo=False):
             tasklist.append((task, sstatefile))
 
         if tasklist:
-            bb.note("Checking sstate mirror object availability (for %s objects)" % len(tasklist))
+            bb.event.fire(bb.event.ProcessStarted("Checking sstate mirror object availability", len(tasklist)), d)
             import multiprocessing
             nproc = min(multiprocessing.cpu_count(), len(tasklist))
 
@@ -821,6 +846,9 @@ def sstate_checkhashes(sq_fn, sq_task, sq_hash, sq_hashfn, d, siginfo=False):
                 pool.add_task(checkstatus, t)
             pool.start()
             pool.wait_completion()
+            bb.event.fire(bb.event.ProcessFinished("Checking sstate mirror object availability"), d)
+            if whitelist and missing:
+                bb.fatal('Required artifacts were unavailable - exiting')
 
     inheritlist = d.getVar("INHERIT", True)
     if "toaster" in inheritlist:

meta/classes/testimage.bbclass

@@ -8,7 +8,7 @@
 # To use it add testimage to global inherit and call your target image with -c testimage
 # You can try it out like this:
 # - first build a qemu core-image-sato
-# - add INHERIT += "testimage" in local.conf
+# - add IMAGE_CLASSES += "testimage" in local.conf
 # - then bitbake core-image-sato -c testimage. That will run a standard suite of tests.
 
 # You can set (or append to) TEST_SUITES in local.conf to select the tests
@@ -51,16 +51,13 @@ DEFAULT_TEST_SUITES_pn-core-image-sato = "${NETTESTSUITE} connman xorg parselogs
 DEFAULT_TEST_SUITES_pn-core-image-sato-sdk = "${NETTESTSUITE} connman xorg perl python \
     ${DEVTESTSUITE} parselogs ${RPMTESTSUITE}"
 DEFAULT_TEST_SUITES_pn-core-image-lsb-dev = "${NETTESTSUITE} pam perl python parselogs ${RPMTESTSUITE}"
-DEFAULT_TEST_SUITES_pn-core-image-lsb-sdk = "${NETTESTSUITE} buildcvs buildiptables buildsudoku \
+DEFAULT_TEST_SUITES_pn-core-image-lsb-sdk = "${NETTESTSUITE} buildcvs buildiptables buildgalculator \
     connman ${DEVTESTSUITE} pam perl python parselogs ${RPMTESTSUITE}"
 DEFAULT_TEST_SUITES_pn-meta-toolchain = "auto"
 
 # aarch64 has no graphics
 DEFAULT_TEST_SUITES_remove_aarch64 = "xorg"
 
-#qemumips is too slow for buildsudoku
-DEFAULT_TEST_SUITES_remove_qemumips = "buildsudoku"
-
 TEST_SUITES ?= "${DEFAULT_TEST_SUITES}"
 
 TEST_QEMUBOOT_TIMEOUT ?= "1000"

meta/classes/testsdk.bbclass

@@ -65,7 +65,7 @@ def testsdk_main(d):
     try:
         subprocess.check_output("cd %s; %s <<EOF\n./tc\nY\nEOF" % (sdktestdir, tcname), shell=True)
     except subprocess.CalledProcessError as e:
-        bb.fatal("Couldn't install the SDK:\n%s" % e.output)
+        bb.fatal("Couldn't install the SDK:\n%s" % e.output.decode("utf-8"))
 
     try:
         run_test_context(SDKTestContext, d, sdktestdir, tcname, pn)
@@ -113,10 +113,18 @@ def testsdkext_main(d):
     testdir = d.expand("${WORKDIR}/testsdkext/")
     bb.utils.remove(testdir, True)
     bb.utils.mkdirhier(testdir)
+    sdkdir = os.path.join(testdir, 'tc')
     try:
-        subprocess.check_output("%s -y -d %s/tc" % (tcname, testdir), shell=True)
+        subprocess.check_output("%s -y -d %s" % (tcname, sdkdir), shell=True)
     except subprocess.CalledProcessError as e:
-        bb.fatal("Couldn't install the SDK EXT:\n%s" % e.output)
+        msg = "Couldn't install the extensible SDK:\n%s" % e.output.decode("utf-8")
+        logfn = os.path.join(sdkdir, 'preparing_build_system.log')
+        if os.path.exists(logfn):
+            msg += '\n\nContents of preparing_build_system.log:\n'
+            with open(logfn, 'r') as f:
+                for line in f:
+                    msg += line
+        bb.fatal(msg)
 
     try:
         bb.plain("Running SDK Compatibility tests ...")

meta/classes/toaster.bbclass

@@ -136,60 +136,16 @@ python toaster_package_dumpdata() {
 
 # 2. Dump output image files information
 
-python toaster_image_dumpdata() {
-    """
-    Image filename for output images is not standardized.
-    image_types.bbclass will spell out IMAGE_CMD_xxx variables that actually
-    have hardcoded ways to create image file names in them.
-    So we look for files starting with the set name.
-
-    We also look for other files in the images/ directory which don't
-    match IMAGE_NAME, such as the kernel bzImage, modules tarball etc.
-    """
-
-    dir_to_walk = d.getVar('DEPLOY_DIR_IMAGE', True);
-    image_name = d.getVar('IMAGE_NAME', True);
-    image_info_data = {}
-    artifact_info_data = {}
-
-    # collect all images and artifacts in the images directory
-    for dirpath, dirnames, filenames in os.walk(dir_to_walk):
-        for filename in filenames:
-            full_path = os.path.join(dirpath, filename)
-            try:
-                if filename.startswith(image_name):
-                    # image
-                    image_info_data[full_path] = os.stat(full_path).st_size
-                else:
-                    # other non-image artifact
-                    if not os.path.islink(full_path):
-                        artifact_info_data[full_path] = os.stat(full_path).st_size
-            except OSError as e:
-                bb.event.fire(bb.event.MetadataEvent("OSErrorException", e), d)
-
-    bb.event.fire(bb.event.MetadataEvent("ImageFileSize", image_info_data), d)
-    bb.event.fire(bb.event.MetadataEvent("ArtifactFileSize", artifact_info_data), d)
-}
-
 python toaster_artifact_dumpdata() {
     """
-    Dump data about artifacts in the SDK_DEPLOY directory
+    Dump data about SDK variables
     """
 
-    dir_to_walk = d.getVar("SDK_DEPLOY", True)
-    artifact_info_data = {}
+    event_data = {
+      "TOOLCHAIN_OUTPUTNAME": d.getVar("TOOLCHAIN_OUTPUTNAME", True)
+    }
 
-    # collect all artifacts in the sdk directory
-    for dirpath, dirnames, filenames in os.walk(dir_to_walk):
-        for filename in filenames:
-            full_path = os.path.join(dirpath, filename)
-            try:
-                if not os.path.islink(full_path):
-                    artifact_info_data[full_path] = os.stat(full_path).st_size
-            except OSError as e:
-                bb.event.fire(bb.event.MetadataEvent("OSErrorException", e), d)
-
-    bb.event.fire(bb.event.MetadataEvent("ArtifactFileSize", artifact_info_data), d)
+    bb.event.fire(bb.event.MetadataEvent("SDKArtifactInfo", event_data), d)
 }
 
 # collect list of buildstats files based on fired events; when the build completes, collect all stats and fire an event with collected data
@@ -332,15 +288,22 @@ python toaster_buildhistory_dump() {
                             images[target][dependsname] = {'size': 0, 'depends' : []}
                         images[target][pname]['depends'].append((dependsname, deptype))
 
-            with open("%s/files-in-image.txt" % installed_img_path, "r") as fin:
-                for line in fin:
-                    lc = [ x for x in line.strip().split(" ") if len(x) > 0 ]
-                    if lc[0].startswith("l"):
-                        files[target]['syms'].append(lc)
-                    elif lc[0].startswith("d"):
-                        files[target]['dirs'].append(lc)
-                    else:
-                        files[target]['files'].append(lc)
+            # files-in-image.txt is only generated if an image file is created,
+            # so the file entries ('syms', 'dirs', 'files') for a target will be
+            # empty for rootfs builds and other "image" tasks which don't
+            # produce image files
+            # (e.g. "bitbake core-image-minimal -c populate_sdk")
+            files_in_image_path = "%s/files-in-image.txt" % installed_img_path
+            if os.path.exists(files_in_image_path):
+                with open(files_in_image_path, "r") as fin:
+                    for line in fin:
+                        lc = [ x for x in line.strip().split(" ") if len(x) > 0 ]
+                        if lc[0].startswith("l"):
+                            files[target]['syms'].append(lc)
+                        elif lc[0].startswith("d"):
+                            files[target]['dirs'].append(lc)
+                        else:
+                            files[target]['files'].append(lc)
 
             for pname in images[target]:
                 if not pname in allpkgs:
@@ -361,17 +324,6 @@ python toaster_buildhistory_dump() {
 
 }
 
-# dump information related to license manifest path
-
-python toaster_licensemanifest_dump() {
-    deploy_dir = d.getVar('DEPLOY_DIR', True);
-    image_name = d.getVar('IMAGE_NAME', True);
-
-    data = { 'deploy_dir' : deploy_dir, 'image_name' : image_name }
-
-    bb.event.fire(bb.event.MetadataEvent("LicenseManifestPath", data), d)
-}
-
 # set event handlers
 addhandler toaster_layerinfo_dumpdata
 toaster_layerinfo_dumpdata[eventmask] = "bb.event.TreeDataPreparationCompleted"
@@ -388,11 +340,8 @@ do_packagedata_setscene[vardepsexclude] += "toaster_package_dumpdata "
 do_package[postfuncs] += "toaster_package_dumpdata "
 do_package[vardepsexclude] += "toaster_package_dumpdata "
 
-do_image_complete[postfuncs] += "toaster_image_dumpdata "
-do_image_complete[vardepsexclude] += "toaster_image_dumpdata "
-
-do_rootfs[postfuncs] += "toaster_licensemanifest_dump "
-do_rootfs[vardepsexclude] += "toaster_licensemanifest_dump "
-
 do_populate_sdk[postfuncs] += "toaster_artifact_dumpdata "
 do_populate_sdk[vardepsexclude] += "toaster_artifact_dumpdata "
+
+do_populate_sdk_ext[postfuncs] += "toaster_artifact_dumpdata "
+do_populate_sdk_ext[vardepsexclude] += "toaster_artifact_dumpdata "

meta/classes/toolchain-scripts.bbclass

@@ -8,7 +8,8 @@ TARGET_CC_ARCH_append_libc-musl = " -mmusl"
 
 # This function creates an environment-setup-script for use in a deployable SDK
 toolchain_create_sdk_env_script () {
-	# Create environment setup script
+	# Create environment setup script.  Remember that $SDKTARGETSYSROOT should
+	# only be expanded on the target at runtime.
 	base_sbindir=${10:-${base_sbindir_nativesdk}}
 	base_bindir=${9:-${base_bindir_nativesdk}}
 	sbindir=${8:-${sbindir_nativesdk}}
@@ -29,7 +30,7 @@ toolchain_create_sdk_env_script () {
 	echo "export PATH=$sdkpathnative$bindir:$sdkpathnative$sbindir:$sdkpathnative$base_bindir:$sdkpathnative$base_sbindir:$sdkpathnative$bindir/../${HOST_SYS}/bin:$sdkpathnative$bindir/${TARGET_SYS}"$EXTRAPATH':$PATH' >> $script
 	echo "export CCACHE_PATH=$sdkpathnative$bindir:$sdkpathnative$bindir/../${HOST_SYS}/bin:$sdkpathnative$bindir/${TARGET_SYS}"$EXTRAPATH':$CCACHE_PATH' >> $script
 	echo 'export PKG_CONFIG_SYSROOT_DIR=$SDKTARGETSYSROOT' >> $script
-	echo 'export PKG_CONFIG_PATH=$SDKTARGETSYSROOT'"$libdir"'/pkgconfig' >> $script
+	echo 'export PKG_CONFIG_PATH=$SDKTARGETSYSROOT'"$libdir"'/pkgconfig:$SDKTARGETSYSROOT'"$prefix"'/share/pkgconfig' >> $script
 	echo 'export CONFIG_SITE=${SDKPATH}/site-config-'"${multimach_target_sys}" >> $script
 	echo "export OECORE_NATIVE_SYSROOT=\"$sdkpathnative\"" >> $script
 	echo 'export OECORE_TARGET_SYSROOT="$SDKTARGETSYSROOT"' >> $script

meta/classes/uboot-config.bbclass

@@ -3,7 +3,7 @@
 # The format to specify it, in the machine, is:
 #
 # UBOOT_CONFIG ??= <default>
-# UBOOT_CONFIG[foo] = "config,images"
+# UBOOT_CONFIG[foo] = "config,images,binary"
 #
 # or
 #
@@ -11,9 +11,13 @@
 #
 # Copyright 2013, 2014 (C) O.S. Systems Software LTDA.
 
+UBOOT_BINARY ?= "u-boot.${UBOOT_SUFFIX}"
+
 python () {
     ubootmachine = d.getVar("UBOOT_MACHINE", True)
     ubootconfigflags = d.getVarFlags('UBOOT_CONFIG')
+    ubootbinary = d.getVar('UBOOT_BINARY', True)
+    ubootbinaries = d.getVar('UBOOT_BINARIES', True)
     # The "doc" varflag is special, we don't want to see it here
     ubootconfigflags.pop('doc', None)
 
@@ -27,6 +31,9 @@ python () {
     if ubootmachine and ubootconfigflags:
         raise bb.parse.SkipPackage("You cannot use UBOOT_MACHINE and UBOOT_CONFIG at the same time.")
 
+    if ubootconfigflags and ubootbinaries:
+        raise bb.parse.SkipPackage("You cannot use UBOOT_BINARIES as it is internal to uboot_config.bbclass.")
+
     if not ubootconfigflags:
         return
 
@@ -36,13 +43,19 @@ python () {
             for f, v in ubootconfigflags.items():
                 if config == f: 
                     items = v.split(',')
-                    if items[0] and len(items) > 2:
-                        raise bb.parse.SkipPackage('Only config,images can be specified!')
+                    if items[0] and len(items) > 3:
+                        raise bb.parse.SkipPackage('Only config,images,binary can be specified!')
                     d.appendVar('UBOOT_MACHINE', ' ' + items[0])
                     # IMAGE_FSTYPES appending
                     if len(items) > 1 and items[1]:
                         bb.debug(1, "Appending '%s' to IMAGE_FSTYPES." % items[1])
                         d.appendVar('IMAGE_FSTYPES', ' ' + items[1])
+                    if len(items) > 2 and items[2]:
+                        bb.debug(1, "Appending '%s' to UBOOT_BINARIES." % items[2])
+                        d.appendVar('UBOOT_BINARIES', ' ' + items[2])
+                    else:
+                        bb.debug(1, "Appending '%s' to UBOOT_BINARIES." % ubootbinary)
+                        d.appendVar('UBOOT_BINARIES', ' ' + ubootbinary)
                     break
     elif len(ubootconfig) == 0:
        raise bb.parse.SkipPackage('You must set a default in UBOOT_CONFIG.')

meta/classes/useradd-staticids.bbclass

@@ -43,7 +43,7 @@ def update_useradd_static_config(d):
                         if fields[0] not in id_table:
                             id_table[fields[0]] = fields
                         else:
-                            id_table[fields[0]] = list(itertools.imap(lambda x, y: x or y, fields, id_table[fields[0]]))
+                            id_table[fields[0]] = list(map(lambda x, y: x or y, fields, id_table[fields[0]]))
             except IOError as e:
                 if e.errno == errno.ENOENT:
                     pass

meta/conf/bitbake.conf

@@ -15,7 +15,7 @@ BASELIB_powerpc64 = "lib64"
 # Path prefixes
 export base_prefix = ""
 export prefix = "/usr"
-export exec_prefix = "/usr"
+export exec_prefix = "${prefix}"
 
 # Base paths
 export base_bindir = "${base_prefix}/bin"
@@ -450,10 +450,10 @@ CCACHE ??= ""
 # Disable ccache explicitly if CCACHE is null since gcc may be a symlink
 # of ccache some distributions (e.g., Fedora 17).
 export CCACHE_DISABLE ??= "${@[0,1][d.getVar('CCACHE', True) == '']}"
-# Assign CCACHE_DIR a default value to fix a bug of ccache 3.1.7,
-# since it would always create CCACHE_DIR/.ccache even if
-# CCACHE_DISABLE = 1.
-export CCACHE_DIR ??= "${@os.getenv('HOME')}"
+# ccache < 3.1.10 will create CCACHE_DIR on startup even if disabled, and
+# autogen sets HOME=/dev/null so in certain situations builds can fail.
+# Explicitly export CCACHE_DIR until we can assume ccache >3.1.10 on the host.
+export CCACHE_DIR ??= "${@os.getenv('HOME')}/.ccache"
 
 TOOLCHAIN_OPTIONS = " --sysroot=${STAGING_DIR_TARGET}"
 
@@ -522,12 +522,7 @@ export BUILD_LDFLAGS = "-L${STAGING_LIBDIR_NATIVE} \
                         -Wl,-rpath,${STAGING_BASE_LIBDIR_NATIVE} \
                         -Wl,-O1"
 
-BUILDSDK_LDFLAGS = "-L${STAGING_LIBDIR} \
-                    -Wl,-rpath-link,${STAGING_LIBDIR} \
-                    -Wl,-rpath,${libdir} -Wl,-O1 \
-                    -L${STAGING_DIR_HOST}${base_libdir} \
-                    -Wl,-rpath-link,${STAGING_DIR_HOST}${base_libdir} \
-                    -Wl,-rpath,${base_libdir} -Wl,-O1"
+BUILDSDK_LDFLAGS = "-Wl,-O1"
 
 LINKER_HASH_STYLE ??= "gnu"
 # mips does not support GNU hash style therefore we override
@@ -817,12 +812,14 @@ BB_HASHCONFIG_WHITELIST ?= "${BB_HASHBASE_WHITELIST} DATE TIME SSH_AGENT_PID \
     SSH_AUTH_SOCK PSEUDO_BUILD BB_ENV_EXTRAWHITE DISABLE_SANITY_CHECKS \
     PARALLEL_MAKE BB_NUMBER_THREADS BB_ORIGENV BB_INVALIDCONF BBINCLUDED \
     GIT_PROXY_COMMAND ALL_PROXY all_proxy NO_PROXY no_proxy FTP_PROXY ftp_proxy \
-    HTTP_PROXY http_proxy HTTPS_PROXY https_proxy SOCKS5_USER SOCKS5_PASSWD"
+    HTTP_PROXY http_proxy HTTPS_PROXY https_proxy SOCKS5_USER SOCKS5_PASSWD \
+    BB_SETSCENE_ENFORCE"
 BB_SIGNATURE_EXCLUDE_FLAGS ?= "doc deps depends \
     lockfiles type vardepsexclude vardeps vardepvalue vardepvalueexclude \
     file-checksums python func task export unexport noexec nostamp dirs cleandirs \
     sstate-lockfile-shared prefuncs postfuncs export_func deptask rdeptask \
-    recrdeptask nodeprrecs stamp-extra-info sstate-outputdirs filename lineno"
+    recrdeptask nodeprrecs stamp-extra-info sstate-outputdirs filename lineno \
+    progress"
 
 MLPREFIX ??= ""
 MULTILIB_VARIANTS ??= ""

meta/conf/distro/include/distro_alias.inc

@@ -46,7 +46,6 @@ DISTRO_PN_ALIAS_pn-connman-gnome = "Intel"
 DISTRO_PN_ALIAS_pn-console-tools = "Debian=console-tools Ubuntu=console-tools"
 DISTRO_PN_ALIAS_pn-core-image-base = "OE-Core"
 DISTRO_PN_ALIAS_pn-core-image-clutter = "OE-Core"
-DISTRO_PN_ALIAS_pn-core-image-directfb = "OE-Core"
 DISTRO_PN_ALIAS_pn-core-image-full-cmdline = "OE-Core"
 DISTRO_PN_ALIAS_pn-core-image-kernel-dev = "OE-Core"
 DISTRO_PN_ALIAS_pn-core-image-lsb = "OE-Core"
@@ -75,7 +74,6 @@ DISTRO_PN_ALIAS_pn-db = "Debian=db5.1 Ubuntu=db5.1"
 DISTRO_PN_ALIAS_pn-dbus-test = "Fedora=dbus Ubuntu=dbus"
 DISTRO_PN_ALIAS_pn-dbus-wait = "OpenedHand"
 DISTRO_PN_ALIAS_pn-depmodwrapper-cross = "OE-Core"
-DISTRO_PN_ALIAS_pn-directfb-examples = "Debian=directfb Fedora=directfb"
 DISTRO_PN_ALIAS_pn-distcc = "Debian=distcc Fedora=distcc"
 DISTRO_PN_ALIAS_pn-distcc-config = "OpenedHand"
 DISTRO_PN_ALIAS_pn-dmxproto = "Meego=xorg-x11-proto-dmxproto Ubuntu=x11proto-dmx Debian=x11proto-dmx"
@@ -216,7 +214,6 @@ DISTRO_PN_ALIAS_pn-libxft = "Mandriva=libxft Debian=libxft2 Ubuntu=libxft2"
 DISTRO_PN_ALIAS_pn-libxi = "Ubuntu=libxi Fedora=libXi"
 DISTRO_PN_ALIAS_pn-libxkbcommon = "Fedora=libxkbcommon Debian=libxkbcommon"
 DISTRO_PN_ALIAS_pn-libxscrnsaver = "Fedora=libXScrnSaver Ubuntu=libxss1 Mandriva=libxscrnsaver"
-DISTRO_PN_ALIAS_pn-libxsettings-client = "Debian=libxsettings-client0 Ubuntu=libxsettings-client0 Mandriva=libXsettings-client0"
 DISTRO_PN_ALIAS_pn-libxxf86misc = "Mandriva=libxxf86misc Fedora=libXxf86misc"
 DISTRO_PN_ALIAS_pn-linux-dummy = "Intel"
 DISTRO_PN_ALIAS_pn-linux-firmware = "Fedora=linux-firmware Ubuntu=linux-firmware"
@@ -288,7 +285,6 @@ DISTRO_PN_ALIAS_pn-packagegroup-core-boot = "OE-Core"
 DISTRO_PN_ALIAS_pn-packagegroup-core-buildessential = "OE-Core"
 DISTRO_PN_ALIAS_pn-packagegroup-core-clutter = "OE-Core"
 DISTRO_PN_ALIAS_pn-packagegroup-core-device-devel = "OE-Core"
-DISTRO_PN_ALIAS_pn-packagegroup-core-directfb = "OE-Core"
 DISTRO_PN_ALIAS_pn-packagegroup-core-eclipse-debug = "OE-Core"
 DISTRO_PN_ALIAS_pn-packagegroup-core-full-cmdline = "OE-Core"
 DISTRO_PN_ALIAS_pn-packagegroup-core-lsb = "OE-Core"

meta/conf/distro/include/security_flags.inc

@@ -43,7 +43,7 @@ SECURITY_CFLAGS_pn-flac = "${SECURITY_NO_PIE_CFLAGS}"
 SECURITY_CFLAGS_pn-flex = "${SECURITY_NO_PIE_CFLAGS}"
 SECURITY_CFLAGS_pn-gcc = "${SECURITY_NO_PIE_CFLAGS}"
 SECURITY_CFLAGS_pn-gcc-runtime = ""
-SECURITY_CFLAGS_pn-gcc-sanitizers = ""
+SECURITY_CFLAGS_pn-gcc-sanitizers = "${SECURITY_NO_PIE_CFLAGS}"
 SECURITY_CFLAGS_pn-gdb = "${SECURITY_NO_PIE_CFLAGS}"
 SECURITY_CFLAGS_pn-gmp = "${SECURITY_NO_PIE_CFLAGS}"
 SECURITY_CFLAGS_pn-gnutls = "${SECURITY_NO_PIE_CFLAGS}"
@@ -62,9 +62,10 @@ SECURITY_CFLAGS_pn-kexec-tools = "${SECURITY_NO_PIE_CFLAGS}"
 SECURITY_CFLAGS_pn-iptables = "${SECURITY_NO_PIE_CFLAGS}"
 SECURITY_CFLAGS_pn-libaio = "${SECURITY_NO_PIE_CFLAGS}"
 SECURITY_CFLAGS_pn-libcap = "${SECURITY_NO_PIE_CFLAGS}"
-SECURITY_CFLAGS_pn-libgcc = ""
+SECURITY_CFLAGS_pn-libgcc = "${SECURITY_NO_PIE_CFLAGS}"
 SECURITY_CFLAGS_pn-libid3tag = "${SECURITY_NO_PIE_CFLAGS}"
 SECURITY_CFLAGS_pn-libnewt = "${SECURITY_NO_PIE_CFLAGS}"
+SECURITY_CFLAGS_pn-libnewt-python = "${SECURITY_NO_PIE_CFLAGS}"
 SECURITY_CFLAGS_pn-libglu = "${SECURITY_NO_PIE_CFLAGS}"
 SECURITY_CFLAGS_pn-libpcap = "${SECURITY_NO_PIE_CFLAGS}"
 SECURITY_CFLAGS_pn-libpcre = "${SECURITY_NO_PIE_CFLAGS}"
@@ -80,6 +81,7 @@ SECURITY_CFLAGS_pn-python-pycurl = "${SECURITY_NO_PIE_CFLAGS}"
 SECURITY_CFLAGS_pn-python-smartpm = "${SECURITY_NO_PIE_CFLAGS}"
 SECURITY_CFLAGS_pn-python-numpy = "${SECURITY_NO_PIE_CFLAGS}"
 SECURITY_CFLAGS_pn-python3-numpy = "${SECURITY_NO_PIE_CFLAGS}"
+SECURITY_CFLAGS_pn-python3-pycairo = "${SECURITY_NO_PIE_CFLAGS}"
 SECURITY_CFLAGS_pn-python3 = "${SECURITY_NO_PIE_CFLAGS}"
 # Revert RPM to using internally supported values
 SECURITY_CFLAGS_pn-rpm = "${lcl_maybe_fortify} -fstack-protector"
@@ -109,10 +111,8 @@ TARGET_CFLAGS_append_class-target = " ${SECURITY_CFLAGS}"
 TARGET_LDFLAGS_append_class-target = " ${SECURITY_LDFLAGS}"
 
 SECURITY_LDFLAGS_remove_pn-gcc-runtime = "-fstack-protector-strong"
-SECURITY_LDFLAGS_remove_pn-gcc-sanitizers = "-fstack-protector-strong"
 SECURITY_LDFLAGS_remove_pn-glibc = "-fstack-protector-strong"
 SECURITY_LDFLAGS_remove_pn-glibc-initial = "-fstack-protector-strong"
-SECURITY_LDFLAGS_remove_pn-libgcc = "-fstack-protector-strong"
 SECURITY_LDFLAGS_remove_pn-uclibc = "-fstack-protector-strong"
 SECURITY_LDFLAGS_remove_pn-uclibc-initial = "-fstack-protector-strong"
 SECURITY_LDFLAGS_pn-xf86-video-fbdev = "${SECURITY_X_LDFLAGS}"

meta/conf/machine/include/tune-ppce500mc.inc

@@ -12,3 +12,6 @@ PACKAGE_EXTRA_ARCHS_tune-ppce500mc = "${PACKAGE_EXTRA_ARCHS_tune-powerpc} ppce50
 
 # glibc configure options to get e500mc specific library (for sqrt)
 GLIBC_EXTRA_OECONF += "${@bb.utils.contains('TUNE_FEATURES', 'ppce500mc', '-with-cpu=e500mc', '', d)}"
+
+# pass -mcpu=e500mc for ppce500mc kernel cross compile
+TARGET_CC_KERNEL_ARCH = "-mcpu=e500mc"

meta/files/ext-sdk-prepare.py

@@ -27,22 +27,6 @@ def exec_watch(cmd, **options):
 
     return process.returncode, buf
 
-def check_unexpected(lines, recipes):
-    """Check for unexpected output lines from dry run"""
-    unexpected = []
-    for line in lines.splitlines():
-        if 'Running task' in line:
-            for recipe in recipes:
-                if recipe in line:
-                    break
-            else:
-                line = line.split('Running', 1)[-1]
-                if 'do_rm_work' not in line:
-                    unexpected.append(line.rstrip())
-        elif 'Running setscene' in line:
-            unexpected.append(line.rstrip())
-    return unexpected
-
 def main():
     if len(sys.argv) < 2:
         sdk_targets = []
@@ -55,44 +39,10 @@ def main():
 
     print('Preparing SDK for %s...' % ', '.join(sdk_targets))
 
-    ret, out = exec_watch('bitbake %s --setscene-only' % ' '.join(sdk_targets))
+    ret, out = exec_watch('BB_SETSCENE_ENFORCE=1 bitbake %s' % ' '.join(sdk_targets))
     if ret:
         return ret
 
-    targetlist = []
-    for target in sdk_targets:
-        if ':' in target:
-            target = target.split(':')[0]
-        if not target in targetlist:
-            targetlist.append(target)
-
-    recipes = []
-    for target in targetlist:
-        try:
-            out = subprocess.check_output(('bitbake -e %s' % target).split(), stderr=subprocess.STDOUT)
-            for line in out.splitlines():
-                if line.startswith('FILE='):
-                    splitval = line.rstrip().split('=')
-                    if len(splitval) > 1:
-                        recipes.append(splitval[1].strip('"'))
-                    break
-        except subprocess.CalledProcessError as e:
-            print('ERROR: Failed to get recipe for target %s:\n%s' % (target, e.output))
-            return 1
-
-    try:
-        out = subprocess.check_output('bitbake %s -n' % ' '.join(sdk_targets), stderr=subprocess.STDOUT, shell=True)
-        unexpected = check_unexpected(out, recipes)
-    except subprocess.CalledProcessError as e:
-        print('ERROR: Failed to execute dry-run:\n%s' % e.output)
-        return 1
-
-    if unexpected:
-        print('ERROR: Unexpected tasks or setscene left over to be executed:')
-        for line in unexpected:
-            print('  ' + line)
-        return 1
-
 if __name__ == "__main__":
     try:
         ret = main()

meta/files/toolchain-shar-extract.sh

@@ -2,6 +2,7 @@
 
 [ -z "$ENVCLEANED" ] && exec /usr/bin/env -i ENVCLEANED=1 HOME="$HOME" \
 	LC_ALL=en_US.UTF-8 \
+	TERM=$TERM \
 	http_proxy="$http_proxy" https_proxy="$https_proxy" ftp_proxy="$ftp_proxy" \
 	no_proxy="$no_proxy" all_proxy="$all_proxy" GIT_PROXY_COMMAND="$GIT_PROXY_COMMAND" "$0" "$@"
 [ -f /etc/environment ] && . /etc/environment

meta/lib/oe/buildhistory_analysis.py

@@ -359,6 +359,24 @@ def compare_dict_blobs(path, ablob, bblob, report_all, report_ver):
                 if ' '.join(alist) == ' '.join(blist):
                     continue
 
+            if key == 'PKGR' and not report_all:
+                vers = []
+                # strip leading 'r' and dots
+                for ver in (astr.split()[0], bstr.split()[0]):
+                    if ver.startswith('r'):
+                        ver = ver[1:]
+                    vers.append(ver.replace('.', ''))
+                maxlen = max(len(vers[0]), len(vers[1]))
+                try:
+                    # pad with '0' and convert to int
+                    vers = [int(ver.ljust(maxlen, '0')) for ver in vers]
+                except ValueError:
+                    pass
+                else:
+                     # skip decrements and increments
+                    if abs(vers[0] - vers[1]) == 1:
+                        continue
+
             chg = ChangeRecord(path, key, astr, bstr, monitored)
             changes.append(chg)
     return changes

meta/lib/oe/package.py

@@ -114,7 +114,12 @@ def read_shlib_providers(d):
             m = list_re.match(file)
             if m:
                 dep_pkg = m.group(1)
-                fd = open(os.path.join(dir, file))
+                try:
+                    fd = open(os.path.join(dir, file))
+                except IOError:
+                    # During a build unrelated shlib files may be deleted, so
+                    # handle files disappearing between the listdirs and open.
+                    continue
                 lines = fd.readlines()
                 fd.close()
                 for l in lines:

meta/lib/oe/package_manager.py

@@ -700,18 +700,19 @@ class RpmPM(PackageManager):
             arch_list = self.feed_archs.split()
         else:
             # List must be prefered to least preferred order
-            default_platform_extra = set()
-            platform_extra = set()
+            default_platform_extra = list()
+            platform_extra = list()
             bbextendvariant = self.d.getVar('BBEXTENDVARIANT', True) or ""
             for mlib in self.ml_os_list:
                 for arch in self.ml_prefix_list[mlib]:
                     plt = arch.replace('-', '_') + '-.*-' + self.ml_os_list[mlib]
                     if mlib == bbextendvariant:
-                            default_platform_extra.add(plt)
+                        if plt not in default_platform_extra:
+                            default_platform_extra.append(plt)
                     else:
-                            platform_extra.add(plt)
-
-            platform_extra = platform_extra.union(default_platform_extra)
+                        if plt not in platform_extra:
+                            platform_extra.append(plt)
+            platform_extra = default_platform_extra + platform_extra
 
             for canonical_arch in platform_extra:
                 arch = canonical_arch.split('-')[0]
@@ -1185,6 +1186,9 @@ class RpmPM(PackageManager):
                     new_depends[new_depend] = deps[depend]
                 pkgs = bb.utils.join_deps(new_depends, commasep=True).split(', ')
         pkgs = self._pkg_translate_oe_to_smart(pkgs, attempt_only)
+        if not pkgs:
+            bb.note("There are no packages to install")
+            return
         if not attempt_only:
             bb.note('to be installed: %s' % ' '.join(pkgs))
             cmd = "%s %s install -y %s" % \
@@ -1563,7 +1567,7 @@ class OpkgPM(OpkgDpkgPM):
         self.deploy_dir = self.d.getVar("DEPLOY_DIR_IPK", True)
         self.deploy_lock_file = os.path.join(self.deploy_dir, "deploy.lock")
         self.opkg_cmd = bb.utils.which(os.getenv('PATH'), "opkg")
-        self.opkg_args = "--volatile-cache -f %s -o %s " % (self.config_file, target_rootfs)
+        self.opkg_args = "--volatile-cache -f %s -t %s -o %s " % (self.config_file, self.d.expand('${T}/ipktemp/') ,target_rootfs)
         self.opkg_args += self.d.getVar("OPKG_ARGS", True)
 
         opkg_lib_dir = self.d.getVar('OPKGLIBDIR', True)

meta/lib/oe/patch.py

@@ -117,43 +117,50 @@ class PatchSet(object):
                 return None
             return os.sep.join(filesplit[striplevel:])
 
-        copiedmode = False
-        filelist = []
-        with open(patchfile) as f:
-            for line in f:
-                if line.startswith('--- '):
-                    patchpth = patchedpath(line)
-                    if not patchpth:
-                        break
-                    if copiedmode:
-                        addedfile = patchpth
-                    else:
-                        removedfile = patchpth
-                elif line.startswith('+++ '):
-                    addedfile = patchedpath(line)
-                    if not addedfile:
-                        break
-                elif line.startswith('*** '):
-                    copiedmode = True
-                    removedfile = patchedpath(line)
-                    if not removedfile:
-                        break
-                else:
-                    removedfile = None
-                    addedfile = None
+        for encoding in ['utf-8', 'latin-1']:
+            try:
+                copiedmode = False
+                filelist = []
+                with open(patchfile) as f:
+                    for line in f:
+                        if line.startswith('--- '):
+                            patchpth = patchedpath(line)
+                            if not patchpth:
+                                break
+                            if copiedmode:
+                                addedfile = patchpth
+                            else:
+                                removedfile = patchpth
+                        elif line.startswith('+++ '):
+                            addedfile = patchedpath(line)
+                            if not addedfile:
+                                break
+                        elif line.startswith('*** '):
+                            copiedmode = True
+                            removedfile = patchedpath(line)
+                            if not removedfile:
+                                break
+                        else:
+                            removedfile = None
+                            addedfile = None
 
-                if addedfile and removedfile:
-                    if removedfile == '/dev/null':
-                        mode = 'A'
-                    elif addedfile == '/dev/null':
-                        mode = 'D'
-                    else:
-                        mode = 'M'
-                    if srcdir:
-                        fullpath = os.path.abspath(os.path.join(srcdir, addedfile))
-                    else:
-                        fullpath = addedfile
-                    filelist.append((fullpath, mode))
+                        if addedfile and removedfile:
+                            if removedfile == '/dev/null':
+                                mode = 'A'
+                            elif addedfile == '/dev/null':
+                                mode = 'D'
+                            else:
+                                mode = 'M'
+                            if srcdir:
+                                fullpath = os.path.abspath(os.path.join(srcdir, addedfile))
+                            else:
+                                fullpath = addedfile
+                            filelist.append((fullpath, mode))
+            except UnicodeDecodeError:
+                continue
+            break
+        else:
+            raise PatchError('Unable to decode %s' % patchfile)
 
         return filelist
 
@@ -280,12 +287,19 @@ class GitApplyTree(PatchTree):
         """
         Extract just the header lines from the top of a patch file
         """
-        lines = []
-        with open(patchfile, 'r') as f:
-            for line in f.readlines():
-                if line.startswith('Index: ') or line.startswith('diff -') or line.startswith('---'):
-                    break
-                lines.append(line)
+        for encoding in ['utf-8', 'latin-1']:
+            lines = []
+            try:
+                with open(patchfile, 'r', encoding=encoding) as f:
+                    for line in f:
+                        if line.startswith('Index: ') or line.startswith('diff -') or line.startswith('---'):
+                            break
+                        lines.append(line)
+            except UnicodeDecodeError:
+                continue
+            break
+        else:
+            raise PatchError('Unable to find a character encoding to decode %s' % patchfile)
         return lines
 
     @staticmethod

meta/lib/oe/path.py

@@ -66,21 +66,21 @@ def copytree(src, dst):
 
     bb.utils.mkdirhier(dst)
     cmd = 'tar -cf - -C %s -p . | tar -xf - -C %s' % (src, dst)
-    check_output(cmd, shell=True, stderr=subprocess.STDOUT)
+    subprocess.check_output(cmd, shell=True, stderr=subprocess.STDOUT)
 
 def copyhardlinktree(src, dst):
     """ Make the hard link when possible, otherwise copy. """
     bb.utils.mkdirhier(dst)
     if os.path.isdir(src) and not len(os.listdir(src)):
-        return	
+        return
 
     if (os.stat(src).st_dev ==  os.stat(dst).st_dev):
         # Need to copy directories only with tar first since cp will error if two 
         # writers try and create a directory at the same time
         cmd = 'cd %s; find . -type d -print | tar -cf - -C %s -p --files-from - --no-recursion | tar -xf - -C %s' % (src, src, dst)
-        check_output(cmd, shell=True, stderr=subprocess.STDOUT)
+        subprocess.check_output(cmd, shell=True, stderr=subprocess.STDOUT)
         cmd = 'cd %s; find . -print0 | cpio --null -pdlu %s' % (src, dst)
-        check_output(cmd, shell=True, stderr=subprocess.STDOUT)
+        subprocess.check_output(cmd, shell=True, stderr=subprocess.STDOUT)
     else:
         copytree(src, dst)
 
@@ -105,47 +105,6 @@ def symlink(source, destination, force=False):
         if e.errno != errno.EEXIST or os.readlink(destination) != source:
             raise
 
-class CalledProcessError(Exception):
-    def __init__(self, retcode, cmd, output = None):
-        self.retcode = retcode
-        self.cmd = cmd
-        self.output = output
-    def __str__(self):
-        return "Command '%s' returned non-zero exit status %d with output %s" % (self.cmd, self.retcode, self.output)
-
-# Not needed when we move to python 2.7
-def check_output(*popenargs, **kwargs):
-    r"""Run command with arguments and return its output as a byte string.
-
-    If the exit code was non-zero it raises a CalledProcessError.  The
-    CalledProcessError object will have the return code in the returncode
-    attribute and output in the output attribute.
-
-    The arguments are the same as for the Popen constructor.  Example:
-
-    >>> check_output(["ls", "-l", "/dev/null"])
-    'crw-rw-rw- 1 root root 1, 3 Oct 18  2007 /dev/null\n'
-
-    The stdout argument is not allowed as it is used internally.
-    To capture standard error in the result, use stderr=STDOUT.
-
-    >>> check_output(["/bin/sh", "-c",
-    ...               "ls -l non_existent_file ; exit 0"],
-    ...              stderr=STDOUT)
-    'ls: non_existent_file: No such file or directory\n'
-    """
-    if 'stdout' in kwargs:
-        raise ValueError('stdout argument not allowed, it will be overridden.')
-    process = subprocess.Popen(stdout=subprocess.PIPE, *popenargs, **kwargs)
-    output, unused_err = process.communicate()
-    retcode = process.poll()
-    if retcode:
-        cmd = kwargs.get("args")
-        if cmd is None:
-            cmd = popenargs[0]
-        raise CalledProcessError(retcode, cmd, output=output)
-    return output
-
 def find(dir, **walkoptions):
     """ Given a directory, recurses into that directory,
     returning all files as absolute paths. """

meta/lib/oe/recipeutils.py

@@ -2,7 +2,7 @@
 #
 # Some code borrowed from the OE layer index
 #
-# Copyright (C) 2013-2015 Intel Corporation
+# Copyright (C) 2013-2016 Intel Corporation
 #
 
 import sys
@@ -15,6 +15,7 @@ from . import utils
 import shutil
 import re
 import fnmatch
+import glob
 from collections import OrderedDict, defaultdict
 
 
@@ -365,6 +366,7 @@ def copy_recipe_files(d, tgt_dir, whole_dir=False, download=True):
     # Copy local files to target directory and gather any remote files
     bb_dir = os.path.dirname(d.getVar('FILE', True)) + os.sep
     remotes = []
+    copied = []
     includes = [path for path in d.getVar('BBINCLUDED', True).split() if
                 path.startswith(bb_dir) and os.path.exists(path)]
     for path in fetch.localpaths() + includes:
@@ -376,13 +378,14 @@ def copy_recipe_files(d, tgt_dir, whole_dir=False, download=True):
                 if not os.path.exists(subdir):
                     os.makedirs(subdir)
                 shutil.copy2(path, os.path.join(tgt_dir, relpath))
+                copied.append(relpath)
         else:
             remotes.append(path)
     # Simply copy whole meta dir, if requested
     if whole_dir:
         shutil.copytree(bb_dir, tgt_dir)
 
-    return remotes
+    return copied, remotes
 
 
 def get_recipe_local_files(d, patches=False):
@@ -448,6 +451,60 @@ def validate_pn(pn):
     return ''
 
 
+def get_bbfile_path(d, destdir, extrapathhint=None):
+    """
+    Determine the correct path for a recipe within a layer
+    Parameters:
+        d: Recipe-specific datastore
+        destdir: destination directory. Can be the path to the base of the layer or a
+            partial path somewhere within the layer.
+        extrapathhint: a path relative to the base of the layer to try
+    """
+    import bb.cookerdata
+
+    destdir = os.path.abspath(destdir)
+    destlayerdir = find_layerdir(destdir)
+
+    # Parse the specified layer's layer.conf file directly, in case the layer isn't in bblayers.conf
+    confdata = d.createCopy()
+    confdata.setVar('BBFILES', '')
+    confdata.setVar('LAYERDIR', destlayerdir)
+    destlayerconf = os.path.join(destlayerdir, "conf", "layer.conf")
+    confdata = bb.cookerdata.parse_config_file(destlayerconf, confdata)
+    pn = d.getVar('PN', True)
+
+    bbfilespecs = (confdata.getVar('BBFILES', True) or '').split()
+    if destdir == destlayerdir:
+        for bbfilespec in bbfilespecs:
+            if not bbfilespec.endswith('.bbappend'):
+                for match in glob.glob(bbfilespec):
+                    splitext = os.path.splitext(os.path.basename(match))
+                    if splitext[1] == '.bb':
+                        mpn = splitext[0].split('_')[0]
+                        if mpn == pn:
+                            return os.path.dirname(match)
+
+    # Try to make up a path that matches BBFILES
+    # this is a little crude, but better than nothing
+    bpn = d.getVar('BPN', True)
+    recipefn = os.path.basename(d.getVar('FILE', True))
+    pathoptions = [destdir]
+    if extrapathhint:
+        pathoptions.append(os.path.join(destdir, extrapathhint))
+    if destdir == destlayerdir:
+        pathoptions.append(os.path.join(destdir, 'recipes-%s' % bpn, bpn))
+        pathoptions.append(os.path.join(destdir, 'recipes', bpn))
+        pathoptions.append(os.path.join(destdir, bpn))
+    elif not destdir.endswith(('/' + pn, '/' + bpn)):
+        pathoptions.append(os.path.join(destdir, bpn))
+    closepath = ''
+    for pathoption in pathoptions:
+        bbfilepath = os.path.join(pathoption, 'test.bb')
+        for bbfilespec in bbfilespecs:
+            if fnmatch.fnmatchcase(bbfilepath, bbfilespec):
+                return pathoption
+    return None
+
 def get_bbappend_path(d, destlayerdir, wildcardver=False):
     """Determine how a bbappend for a recipe should be named and located within another layer"""
 
@@ -728,14 +785,16 @@ def bbappend_recipe(rd, destlayerdir, srcfiles, install=None, wildcardver=False,
 
 
 def find_layerdir(fn):
-    """ Figure out relative path to base of layer for a file (e.g. a recipe)"""
-    pth = os.path.dirname(fn)
+    """ Figure out the path to the base of the layer containing a file (e.g. a recipe)"""
+    pth = fn
     layerdir = ''
     while pth:
         if os.path.exists(os.path.join(pth, 'conf', 'layer.conf')):
             layerdir = pth
             break
         pth = os.path.dirname(pth)
+        if pth == '/':
+            return None
     return layerdir