Allow for simultaneous do_rootfs tasks with rpm

Give each rootfs its own RPM channel to use.  This puts the RPM metadata
in a private subdirectory of $WORKDIR, rather than living in DEPLOY_DIR
where other tasks may race with it.

This allows us to reduce the time that the rpm.lock is held to only the
time needed to hardlink the RPMs, allowing the majority of the rootfs
operation to run in parallel.

Also, this fixes the smart tests by generating an index for all packages
at the time of the test, rather than using the one provided by the
rootfs process.

Original credit for the enhancement should go to Steven Walter
stevenrwalter@gmail.com.

(From OE-Core rev: a92c196449c516fe51786d429078bbb1213bb029)

Signed-off-by: Stephano Cetola <stephano.cetola@linux.intel.com>
Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>

bitbake/LICENSE

@@ -10,4 +10,6 @@ Foundation and individual contributors.
 
 * Twitter typeahead.js redistributed under the MIT license. Note that the JS source has one small modification, so the full unminified file is currently included to make it obvious where this is.
 
+* jsrender is redistributed under the MIT license.
+
 * QUnit is redistributed under the MIT license.

bitbake/bin/bitbake-worker

@@ -436,8 +436,7 @@ class BitbakeWorker(object):
             self.build_pipes[pipe].read()
 
 try:
-    sys.stdin = sys.stdin.detach()
-    worker = BitbakeWorker(sys.stdin)
+    worker = BitbakeWorker(os.fdopen(sys.stdin.fileno(), 'rb'))
     if not profiling:
         worker.serve()
     else:

bitbake/bin/toaster

@@ -59,7 +59,8 @@ webserverStartAll()
         echo "Failed migrations, aborting system start" 1>&2
         return $retval
     fi
-
+    # Make sure that checksettings can pick up any value for TEMPLATECONF
+    export TEMPLATECONF
     $MANAGE checksettings --traceback || retval=1
 
     if [ $retval -eq 1 ]; then
@@ -100,7 +101,6 @@ stop_system()
     fi
     webserverKillAll
     # unset exported variables
-    unset TOASTER_CONF
     unset TOASTER_DIR
     unset BITBAKE_UI
     unset BBBASEDIR
@@ -153,23 +153,9 @@ if [ -n "$TEMPLATECONF" ]; then
         if [ -d "$OEROOT/$TEMPLATECONF" ]; then
             TEMPLATECONF="$OEROOT/$TEMPLATECONF"
         fi
-        if [ ! -d "$TEMPLATECONF" ]; then
-            echo >&2 "Error: '$TEMPLATECONF' must be a directory containing toasterconf.json"
-            return 1
-        fi
     fi
 fi
 
-if [ "$TOASTER_CONF" = "" ]; then
-    TOASTER_CONF="$TEMPLATECONF/toasterconf.json"
-    export TOASTER_CONF=$(python3 -c "import os; print(os.path.realpath('$TOASTER_CONF'))")
-fi
-
-if [ ! -f $TOASTER_CONF ]; then
-    echo "$TOASTER_CONF configuration file not found. Set TOASTER_CONF to specify file or fix .templateconf"
-    return 1
-fi
-
 # this defines the dir toaster will use for
 # 1) clones of layers (in _toaster_clones )
 # 2) the build dir (in build)
@@ -203,6 +189,10 @@ for param in $*; do
                 ADDR_PORT="localhost:$PORT"
             fi
     ;;
+    --help)
+            echo "$HELP"
+            return 0
+    ;;
     *)
             echo "$HELP"
             return 1

bitbake/bin/toaster-eventreplay

@@ -21,154 +21,106 @@
 # with this program; if not, write to the Free Software Foundation, Inc.,
 # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
 
+"""
+This command takes a filename as a single parameter. The filename is read
+as a build eventlog, and the ToasterUI is used to process events in the file
+and log data in the database
+"""
 
-# This command takes a filename as a single parameter. The filename is read
-# as a build eventlog, and the ToasterUI is used to process events in the file
-# and log data in the database
-
-from __future__ import print_function
 import os
-import sys, logging
+import sys
+import json
+import pickle
+import codecs
+
+from collections import namedtuple
 
 # mangle syspath to allow easy import of modules
-sys.path.insert(0, os.path.join(os.path.dirname(os.path.dirname(os.path.abspath(__file__))),
-                                'lib'))
-
+from os.path import join, dirname, abspath
+sys.path.insert(0, join(dirname(dirname(abspath(__file__))), 'lib'))
 
 import bb.cooker
 from bb.ui import toasterui
-import sys
-import logging
 
-import json, pickle
+class EventPlayer:
+    """Emulate a connection to a bitbake server."""
 
+    def __init__(self, eventfile, variables):
+        self.eventfile = eventfile
+        self.variables = variables
+        self.eventmask = []
 
-class FileReadEventsServerConnection():
-    """  Emulates a connection to a bitbake server that feeds
-        events coming actually read from a saved log file.
-    """
+    def waitEvent(self, _timeout):
+        """Read event from the file."""
+        line = self.eventfile.readline().strip()
+        if not line:
+            return
+        try:
+            event_str = json.loads(line)['vars'].encode('utf-8')
+            event = pickle.loads(codecs.decode(event_str, 'base64'))
+            event_name = "%s.%s" % (event.__module__, event.__class__.__name__)
+            if event_name not in self.eventmask:
+                return
+            return event
+        except ValueError as err:
+            print("Failed loading ", line)
+            raise err
 
-    class MockConnection():
-        """ fill-in for the proxy to the server. we just return generic data
+    def runCommand(self, command_line):
+        """Emulate running a command on the server."""
+        name = command_line[0]
+
+        if name == "getVariable":
+            var_name = command_line[1]
+            variable = self.variables.get(var_name)
+            if variable:
+                return variable['v'], None
+            return None, "Missing variable %s" % var_name
+
+        elif name == "getAllKeysWithFlags":
+            dump = {}
+            flaglist = command_line[1]
+            for key, val in self.variables.items():
+                try:
+                    if not key.startswith("__"):
+                        dump[key] = {
+                            'v': val['v'],
+                            'history' : val['history'],
+                        }
+                        for flag in flaglist:
+                            dump[key][flag] = val[flag]
+                except Exception as err:
+                    print(err)
+            return (dump, None)
+
+        elif name == 'setEventMask':
+            self.eventmask = command_line[-1]
+            return True, None
+
+        else:
+            raise Exception("Command %s not implemented" % command_line[0])
+
+    def getEventHandle(self):
         """
-        def __init__(self, sc):
-            self._sc = sc
+        This method is called by toasterui.
+        The return value is passed to self.runCommand but not used there.
+        """
+        pass
 
-        def runCommand(self, commandArray):
-            """ emulates running a command on the server; only read-only commands are accepted """
-            command_name = commandArray[0]
+def main(argv):
+    with open(argv[-1]) as eventfile:
+        # load variables from the first line
+        variables = json.loads(eventfile.readline().strip())['allvariables']
 
-            if command_name == "getVariable":
-                if commandArray[1] in self._sc._variables:
-                    return (self._sc._variables[commandArray[1]]['v'], None)
-                return (None, "Missing variable")
-
-            elif command_name == "getAllKeysWithFlags":
-                dump = {}
-                flaglist = commandArray[1]
-                for k in self._sc._variables.keys():
-                    try:
-                        if not k.startswith("__"):
-                            v = self._sc._variables[k]['v']
-                            dump[k] = {
-                                'v' : v ,
-                                'history' : self._sc._variables[k]['history'],
-                            }
-                            for d in flaglist:
-                                dump[k][d] = self._sc._variables[k][d]
-                    except Exception as e:
-                        print(e)
-                return (dump, None)
-            else:
-                raise Exception("Command %s not implemented" % commandArray[0])
-
-        def terminateServer(self):
-            """ do not do anything """
-            pass
-
-
-
-    class EventReader():
-        def __init__(self, sc):
-            self._sc = sc
-            self.firstraise = 0
-
-        def _create_event(self, line):
-            def _import_class(name):
-                assert len(name) > 0
-                assert "." in name, name
-
-                components = name.strip().split(".")
-                modulename = ".".join(components[:-1])
-                moduleklass = components[-1]
-
-                module = __import__(modulename, fromlist=[str(moduleklass)])
-                return getattr(module, moduleklass)
-
-            # we build a toaster event out of current event log line
-            try:
-                event_data = json.loads(line.strip())
-                event_class = _import_class(event_data['class'])
-                event_object = pickle.loads(json.loads(event_data['vars']))
-            except ValueError as e:
-                print("Failed loading ", line)
-                raise e
-
-            if not isinstance(event_object, event_class):
-                raise Exception("Error loading objects %s class %s ", event_object, event_class)
-
-            return event_object
-
-        def waitEvent(self, timeout):
-
-            nextline = self._sc._eventfile.readline()
-            if len(nextline) == 0:
-                # the build data ended, while toasterui still waits for events.
-                # this happens when the server was abruptly stopped, so we simulate this
-                self.firstraise += 1
-                if self.firstraise == 1:
-                    raise KeyboardInterrupt()
-                else:
-                    return None
-            else:
-                self._sc.lineno += 1
-            return self._create_event(nextline)
-
-
-    def _readVariables(self, variableline):
-        self._variables = json.loads(variableline.strip())['allvariables']
-
-
-    def __init__(self, file_name):
-        self.connection = FileReadEventsServerConnection.MockConnection(self)
-        self._eventfile = open(file_name, "r")
-
-        # we expect to have the variable dump at the start of the file
-        self.lineno = 1
-        self._readVariables(self._eventfile.readline())
-
-        self.events = FileReadEventsServerConnection.EventReader(self)
-
-
-
-
-
-class MockConfigParameters():
-    """ stand-in for cookerdata.ConfigParameters; as we don't really config a cooker, this
-        serves just to supply needed interfaces for the toaster ui to work """
-    def __init__(self):
-        self.observe_only = True            # we can only read files
+        params = namedtuple('ConfigParams', ['observe_only'])(True)
+        player = EventPlayer(eventfile, variables)
 
+        return toasterui.main(player, player, params)
 
 # run toaster ui on our mock bitbake class
 if __name__ == "__main__":
-    if len(sys.argv) < 2:
-        print("Usage: %s event.log " % sys.argv[0])
+    if len(sys.argv) != 2:
+        print("Usage: %s <event file>" % os.path.basename(sys.argv[0]))
         sys.exit(1)
 
-    file_name = sys.argv[-1]
-    mock_connection = FileReadEventsServerConnection(file_name)
-    configParams = MockConfigParameters()
-
-    # run the main program and set exit code to the returned value
-    sys.exit(toasterui.main(mock_connection.connection, mock_connection.events, configParams))
+    sys.exit(main(sys.argv))

bitbake/doc/bitbake-user-manual/bitbake-user-manual-fetching.xml

@@ -670,8 +670,8 @@
                             The <filename>module</filename> and <filename>vob</filename>
                             options are combined to create the <filename>load</filename> rule in
                             the view config spec.
-                            As an example, consider the <filename>vob</filename> and 
-                            <filename>module</filename> values from the 
+                            As an example, consider the <filename>vob</filename> and
+                            <filename>module</filename> values from the
                             <filename>SRC_URI</filename> statement at the start of this section.
                             Combining those values results in the following:
                             <literallayout class='monospaced'>
@@ -716,6 +716,68 @@
             </para>
         </section>
 
+        <section id='perforce-fetcher'>
+            <title>Perforce Fetcher (<filename>p4://</filename>)</title>
+
+            <para>
+                This fetcher submodule fetches code from the
+                <ulink url='https://www.perforce.com/'>Perforce</ulink>
+                source control system.
+                The executable used is specified by
+                <filename>FETCHCMD_p4</filename>, which defaults
+                to "p4".
+                The fetcher's temporary working directory is set by
+                <link linkend='var-P4DIR'><filename>P4DIR</filename></link>,
+                which defaults to "DL_DIR/p4".
+            </para>
+
+            <para>
+                To use this fetcher, make sure your recipe has proper
+                <link linkend='var-SRC_URI'><filename>SRC_URI</filename></link>,
+                <link linkend='var-SRCREV'><filename>SRCREV</filename></link>, and
+                <link linkend='var-PV'><filename>PV</filename></link> values.
+                The p4 executable is able to use the config file defined by your
+                system's <filename>P4CONFIG</filename> environment variable in
+                order to define the Perforce server URL and port, username, and
+                password if you do not wish to keep those values in a recipe
+                itself.
+                If you choose not to use <filename>P4CONFIG</filename>,
+                or to explicitly set variables that <filename>P4CONFIG</filename>
+                can contain, you can specify the <filename>P4PORT</filename> value,
+                which is the server's URL and port number, and you can
+                specify a username and password directly in your recipe within
+                <filename>SRC_URI</filename>.
+            </para>
+
+            <para>
+                Here is an example that relies on <filename>P4CONFIG</filename>
+                to specify the server URL and port, username, and password, and
+                fetches the Head Revision:
+                <literallayout class='monospaced'>
+    SRC_URI = "p4://example-depot/main/source/..."
+    SRCREV = "${AUTOREV}"
+    PV = "p4-${SRCPV}"
+    S = "${WORKDIR}/p4"
+                </literallayout>
+            </para>
+
+            <para>
+                Here is an example that specifies the server URL and port,
+                username, and password, and fetches a Revision based on a Label:
+                <literallayout class='monospaced'>
+    P4PORT = "tcp:p4server.example.net:1666"
+    SRC_URI = "p4://user:passwd@example-depot/main/source/..."
+    SRCREV = "release-1.0"
+    PV = "p4-${SRCPV}"
+    S = "${WORKDIR}/p4"
+                </literallayout>
+                <note>
+                    You should always set <filename>S</filename>
+                    to <filename>"${WORKDIR}/p4"</filename> in your recipe.
+                </note>
+            </para>
+        </section>
+
         <section id='other-fetchers'>
             <title>Other Fetchers</title>
 
@@ -725,9 +787,6 @@
                     <listitem><para>
                         Bazaar (<filename>bzr://</filename>)
                         </para></listitem>
-                    <listitem><para>
-                        Perforce (<filename>p4://</filename>)
-                        </para></listitem>
                     <listitem><para>
                         Trees using Git Annex (<filename>gitannex://</filename>)
                         </para></listitem>

bitbake/doc/bitbake-user-manual/bitbake-user-manual-intro.xml

@@ -634,6 +634,25 @@
                 </para>
             </section>
 
+            <section id='executing-a-list-of-task-and-recipe-combinations'>
+                <title>Executing a List of Task and Recipe Combinations</title>
+
+                <para>
+                    The BitBake command line supports specifying different
+                    tasks for individual targets when you specify multiple
+                    targets.
+                    For example, suppose you had two targets (or recipes)
+                    <filename>myfirstrecipe</filename> and
+                    <filename>mysecondrecipe</filename> and you needed
+                    BitBake to run <filename>taskA</filename> for the first
+                    recipe and <filename>taskB</filename> for the second
+                    recipe:
+                    <literallayout class='monospaced'>
+     $ bitbake myfirstrecipe:do_taskA mysecondrecipe:do_taskB
+                    </literallayout>
+                </para>
+            </section>
+
             <section id='generating-dependency-graphs'>
                 <title>Generating Dependency Graphs</title>
 

bitbake/doc/bitbake-user-manual/bitbake-user-manual-metadata.xml

@@ -50,22 +50,52 @@
             <title>Variable Expansion</title>
 
             <para>
-                BitBake supports variables referencing one another's
-                contents using a syntax that is similar to shell scripting.
-                Following is an example that results in <filename>A</filename>
-                containing "aval" and <filename>B</filename> evaluating to
-                "preavalpost" based on that current value of
-                <filename>A</filename>.
+                Variables can reference the contents of other variables
+                using a syntax that is similar to variable expansion in
+                Bourne shells.
+                The following assignments
+                result in A containing "aval" and B evaluating to "preavalpost".
                 <literallayout class='monospaced'>
      A = "aval"
      B = "pre${A}post"
                 </literallayout>
-                You should realize that whenever <filename>B</filename> is
-                referenced, its evaluation will depend on the state of
-                <filename>A</filename> at that time.
-                Thus, later evaluations of <filename>B</filename> in the
-                previous example could result in different values
-                depending on the value of <filename>A</filename>.
+                <note>
+                    Unlike in Bourne shells, the curly braces are mandatory:
+                    Only <filename>${FOO}</filename> and not
+                    <filename>$FOO</filename> is recognized as an expansion of
+                    <filename>FOO</filename>.
+                </note>
+                The "=" operator does not immediately expand variable
+                references in the right-hand side.
+                Instead, expansion is deferred until the variable assigned to
+                is actually used.
+                The result depends on the current values of the referenced
+                variables.
+                The following example should clarify this behavior:
+                <literallayout class='monospaced'>
+     A = "${B} baz"
+     B = "${C} bar"
+     C = "foo"
+     *At this point, ${A} equals "foo bar baz"*
+     C = "qux"
+     *At this point, ${A} equals "qux bar baz"*
+     B = "norf"
+     *At this point, ${A} equals "norf baz"*
+                </literallayout>
+                Contrast this behavior with the
+                <link linkend='immediate-variable-expansion'>immediate variable expansion</link>
+                operator (i.e. ":=").
+            </para>
+
+            <para>
+                If the variable expansion syntax is used on a variable that
+                does not exist, the string is kept as is.
+                For example, given the following assignment,
+                <filename>BAR</filename> expands to the literal string
+                "${FOO}" as long as <filename>FOO</filename> does not exist.
+                <literallayout class='monospaced'>
+     BAR = "${FOO}"
+                </literallayout>
             </para>
         </section>
 
@@ -259,6 +289,60 @@
                 "789 123456" and <filename>FOO2</filename> becomes
                 "ghi abcdef".
             </para>
+
+            <para>
+                Like "_append" and "_prepend", "_remove"
+                is deferred until after parsing completes.
+            </para>
+        </section>
+
+        <section id='override-style-operation-advantages'>
+            <title>Override Style Operation Advantages</title>
+
+            <para>
+                An advantage of the override style operations
+                "_append", "_prepend", and "_remove" as compared to the
+                "+=" and "=+" operators is that the override style
+                operators provide guaranteed operations.
+                For example, consider a class <filename>foo.bbclass</filename>
+                that needs to add the value "val" to the variable
+                <filename>FOO</filename>, and a recipe that uses
+                <filename>foo.bbclass</filename> as follows:
+                <literallayout class='monospaced'>
+     inherit foo
+
+     FOO = "initial"
+                </literallayout>
+                If <filename>foo.bbclass</filename> uses the "+=" operator,
+                as follows, then the final value of <filename>FOO</filename>
+                will be "initial", which is not what is desired:
+                <literallayout class='monospaced'>
+     FOO += "val"
+                </literallayout>
+                If, on the other hand, <filename>foo.bbclass</filename>
+                uses the "_append" operator, then the final value of
+                <filename>FOO</filename> will be "initial val", as intended:
+                <literallayout class='monospaced'>
+     FOO_append = " val"
+                </literallayout>
+                <note>
+                    It is never necessary to use "+=" together with "_append".
+                    The following sequence of assignments appends "barbaz" to
+                    <filename>FOO</filename>:
+                    <literallayout class='monospaced'>
+     FOO_append = "bar"
+     FOO_append = "baz"
+                    </literallayout>
+                    The only effect of changing the second assignment in the
+                    previous example to use "+=" would be to add a space before
+                    "baz" in the appended value (due to how the "+=" operator
+                    works).
+                </note>
+                Another advantage of the override style operations is that
+                you can combine them with other overrides as described in the
+                "<link linkend='conditional-syntax-overrides'>Conditional Syntax (Overrides)</link>"
+                section.
+            </para>
         </section>
 
         <section id='variable-flag-syntax'>
@@ -277,8 +361,7 @@
                 You can define, append, and prepend values to variable flags.
                 All the standard syntax operations previously mentioned work
                 for variable flags except for override style syntax
-                (i.e. <filename>_prepend</filename>, <filename>_append</filename>,
-                and <filename>_remove</filename>).
+                (i.e. "_prepend", "_append", and "_remove").
             </para>
 
             <para>
@@ -330,6 +413,21 @@
      PN = "${@bb.parse.BBHandler.vars_from_file(d.getVar('FILE', False),d)[0] or 'defaultpkgname'}"
      PV = "${@bb.parse.BBHandler.vars_from_file(d.getVar('FILE', False),d)[1] or '1.0'}"
                 </literallayout>
+                <note>
+                    Inline Python expressions work just like variable expansions
+                    insofar as the "=" and ":=" operators are concerned.
+                    Given the following assignment, <filename>foo()</filename>
+                    is called each time <filename>FOO</filename> is expanded:
+                    <literallayout class='monospaced'>
+     FOO = "${@foo()}"
+                    </literallayout>
+                    Contrast this with the following immediate assignment, where
+                    <filename>foo()</filename> is only called once, while the
+                    assignment is parsed:
+                    <literallayout class='monospaced'>
+     FOO := "${@foo()}"
+                    </literallayout>
+                </note>
             </para>
         </section>
 
@@ -545,15 +643,15 @@
      OVERRIDES = "foo"
      A = "Y"
      A_foo_append = "Z"
-     A_foo_append += "X"
+     A_foo_append = "X"
                 </literallayout>
                 For this case, before any overrides are resolved,
                 <filename>A</filename> is set to "Y" using an immediate assignment.
                 After this immediate assignment, <filename>A_foo</filename> is set
                 to "Z", and then further appended with
-                "X" leaving the variable set to "Z X".
+                "X" leaving the variable set to "ZX".
                 Finally, applying the override for "foo" results in the conditional
-                variable <filename>A</filename> becoming "Z X" (i.e.
+                variable <filename>A</filename> becoming "ZX" (i.e.
                 <filename>A</filename> is replaced with <filename>A_foo</filename>).
             </para>
 
@@ -572,7 +670,7 @@
                 Initially, <filename>A</filename> is set to "1 45" because
                 of the three statements that use immediate operators.
                 After these assignments are made, BitBake applies the
-                <filename>_append</filename> operations.
+                "_append" operations.
                 Those operations result in <filename>A</filename> becoming "1 4523".
             </para>
         </section>
@@ -1624,15 +1722,32 @@
         <title>Dependencies</title>
 
         <para>
-            To allow for efficient operation given multiple processes
-            executing in parallel, BitBake handles dependencies at
-            the task level.
-            BitBake supports a robust method to handle these dependencies.
-        </para>
+            To allow for efficient parallel processing, BitBake handles
+            dependencies at the task level.
+            Dependencies can exist both between tasks within a single recipe
+            and between tasks in different recipes.
+            Following are examples of each:
+            <itemizedlist>
+                <listitem><para>For tasks within a single recipe, a
+                    recipe's <filename>do_configure</filename>
+                    task might need to complete before its
+                    <filename>do_compile</filename> task can run.
+                    </para></listitem>
+                <listitem><para>For tasks in different recipes, one
+                    recipe's <filename>do_configure</filename>
+                    task might require another recipe's
+                    <filename>do_populate_sysroot</filename>
+                    task to finish first such that the libraries and headers
+                    provided by the other recipe are available.
+                    </para></listitem>
+             </itemizedlist>
+         </para>
 
-        <para>
-            This section describes several types of dependency mechanisms.
-        </para>
+         <para>
+             This section describes several ways to declare dependencies.
+             Remember, even though dependencies are declared in different ways, they
+             are all simply dependencies between tasks.
+         </para>
 
         <section id='dependencies-internal-to-the-bb-file'>
             <title>Dependencies Internal to the <filename>.bb</filename> File</title>
@@ -1653,6 +1768,40 @@
                 task.
                 And, the <filename>do_build</filename> depends on the completion
                 of the <filename>printdate</filename> task.
+                <note>
+                    Recipes are built by having their
+                    <filename>do_build</filename> (not to be confused with
+                    <filename>do_compile</filename>) tasks executed.
+                    For a task to run when a recipe is built, the task must
+                    therefore be a direct or indirect dependency of
+                    <filename>do_build</filename>.
+                    For illustration, here are some examples:
+                    <itemizedlist>
+                        <listitem><para>
+                            The directive
+                            <filename>addtask mytask before do_build</filename>
+                            causes <filename>mytask</filename> to run when the
+                            recipe is built.
+                            In this example, <filename>mytask</filename> is run
+                            at an unspecified time relative to other tasks within
+                            the recipe, since <filename>after</filename> is not used.
+                            </para></listitem>
+                        <listitem><para>
+                            The directive
+                            <filename>addtask mytask after do_configure</filename>
+                            by itself does not cause <filename>mytask</filename>
+                            to run when the recipe is built.
+                            The task can still be run manually using the following:
+                            <literallayout class='monospaced'>
+     $ bitbake <replaceable>recipe</replaceable> -c mytask
+                            </literallayout>
+                            <filename>mytask</filename> could also be declared as
+                            a dependency of some other task.
+                            Regardless, the task is run after
+                            <filename>do_configure</filename>.
+                            </para></listitem>
+                    </itemizedlist>
+                </note>
             </para>
         </section>
 
@@ -1721,6 +1870,20 @@
                 and added.
             </para>
 
+            <para>
+                The "recrdeptask" flag is most commonly used in high-level
+                recipes that need to wait for some task to finish "globally".
+                For example, <filename>image.bbclass</filename> has the following:
+                <literallayout class='monospaced'>
+     do_rootfs[recrdeptask] += "do_packagedata"
+                </literallayout>
+                This statement says that the <filename>do_packagedata</filename>
+                task of the current recipe and all recipes reachable
+                (by way of dependencies) from the
+                image recipe must run before the <filename>do_rootfs</filename>
+                task can run.
+            </para>
+
             <para>
                 You might want to not only have BitBake look for
                 dependencies of those tasks, but also have BitBake look
@@ -1785,9 +1948,9 @@
                     </thead>
                     <tbody>
                         <row>
-                            <entry align="left"><filename>d.getVar("X", expand=False)</filename></entry>
+                            <entry align="left"><filename>d.getVar("X", expand)</filename></entry>
                             <entry align="left">Returns the value of variable "X".
-                                Using "expand=True" expands the value.</entry>
+                            Using "expand=True" expands the value.</entry>
                         </row>
                         <row>
                             <entry align="left"><filename>d.setVar("X", "value")</filename></entry>
@@ -1810,9 +1973,9 @@
                             <entry align="left">Renames the variable "X" to "Y".</entry>
                         </row>
                         <row>
-                            <entry align="left"><filename>d.getVarFlag("X", flag, expand=False)</filename></entry>
-                            <entry align="left">Gets then named flag from the variable "X".
-                                Using "expand=True" expands the named flag.</entry>
+                            <entry align="left"><filename>d.getVarFlag("X", flag, expand)</filename></entry>
+                            <entry align="left">Returns the value of variable "X".
+                                Using "expand=True" expands the value.</entry>
                         </row>
                         <row>
                             <entry align="left"><filename>d.setVarFlag("X", flag, "value")</filename></entry>

bitbake/doc/bitbake-user-manual/bitbake-user-manual-ref-variables.xml

@@ -52,7 +52,7 @@
        <link linkend='var-MIRRORS'>M</link>
 <!--               <link linkend='var-glossary-n'>N</link> -->
        <link linkend='var-OVERRIDES'>O</link>
-       <link linkend='var-PACKAGES'>P</link>
+       <link linkend='var-P4DIR'>P</link>
 <!--       <link linkend='var-QMAKE_PROFILES'>Q</link> -->
        <link linkend='var-RDEPENDS'>R</link>
        <link linkend='var-SECTION'>S</link>
@@ -986,6 +986,29 @@
      BBCLASSEXTEND =+ "native nativesdk"
      BBCLASSEXTEND =+ "multilib:<replaceable>multilib_name</replaceable>"
                     </literallayout>
+                    <note>
+                        <para>
+                        Internally, the <filename>BBCLASSEXTEND</filename>
+                        mechanism generates recipe variants by rewriting
+                        variable values and applying overrides such as
+                        <filename>_class-native</filename>.
+                        For example, to generate a native version of a recipe,
+                        a
+                        <link linkend='var-DEPENDS'><filename>DEPENDS</filename></link>
+                        on "foo" is rewritten to a <filename>DEPENDS</filename>
+                        on "foo-native".
+                        </para>
+
+                        <para>
+                        Even when using <filename>BBCLASSEXTEND</filename>, the
+                        recipe is only parsed once.
+                        Parsing once adds some limitations.
+                        For example, it is not possible to
+                        include a different file depending on the variant,
+                        since <filename>include</filename> statements are
+                        processed when the recipe is parsed.
+                        </para>
+                    </note>
                 </para>
              </glossdef>
         </glossentry>
@@ -994,7 +1017,7 @@
             <glossdef>
                 <para>
                     Sets the BitBake debug output level to a specific value
-                    as incremented by the <filename>-d</filename> command line
+                    as incremented by the <filename>-D</filename> command line
                     option.
                     <note>
                         You must set this variable in the external environment
@@ -1748,6 +1771,15 @@
 
     <glossdiv id='var-glossary-p'><title>P</title>
 
+        <glossentry id='var-P4DIR'><glossterm>P4DIR</glossterm>
+            <glossdef>
+                <para>
+                    The directory in which a local copy of a Perforce depot
+                    is stored when it is fetched.
+                </para>
+            </glossdef>
+        </glossentry>
+
         <glossentry id='var-PACKAGES'><glossterm>PACKAGES</glossterm>
             <glossdef>
                 <para>The list of packages the recipe creates.
@@ -1944,6 +1976,27 @@
                     The <filename>PROVIDES</filename> statement results in
                     the "libav" recipe also being known as "libpostproc".
                 </para>
+
+                <para>
+                    In addition to providing recipes under alternate names,
+                    the <filename>PROVIDES</filename> mechanism is also used
+                    to implement virtual targets.
+                    A virtual target is a name that corresponds to some
+                    particular functionality (e.g. a Linux kernel).
+                    Recipes that provide the functionality in question list the
+                    virtual target in <filename>PROVIDES</filename>.
+                    Recipes that depend on the functionality in question can
+                    include the virtual target in
+                    <link linkend='var-DEPENDS'><filename>DEPENDS</filename></link>
+                    to leave the choice of provider open.
+                </para>
+
+                <para>
+                    Conventionally, virtual targets have names on the form
+                    "virtual/function" (e.g. "virtual/kernel").
+                    The slash is simply part of the name and has no
+                    syntactical significance.
+                </para>
             </glossdef>
         </glossentry>
 

bitbake/lib/bb/build.py

@@ -35,6 +35,7 @@ import stat
 import bb
 import bb.msg
 import bb.process
+import bb.progress
 from bb import data, event, utils
 
 bblogger = logging.getLogger('BitBake')
@@ -137,6 +138,25 @@ class TaskInvalid(TaskBase):
         super(TaskInvalid, self).__init__(task, None, metadata)
         self._message = "No such task '%s'" % task
 
+class TaskProgress(event.Event):
+    """
+    Task made some progress that could be reported to the user, usually in
+    the form of a progress bar or similar.
+    NOTE: this class does not inherit from TaskBase since it doesn't need
+    to - it's fired within the task context itself, so we don't have any of
+    the context information that you do in the case of the other events.
+    The event PID can be used to determine which task it came from.
+    The progress value is normally 0-100, but can also be negative
+    indicating that progress has been made but we aren't able to determine
+    how much.
+    The rate is optional, this is simply an extra string to display to the
+    user if specified.
+    """
+    def __init__(self, progress, rate=None):
+        self.progress = progress
+        self.rate = rate
+        event.Event.__init__(self)
+
 
 class LogTee(object):
     def __init__(self, logger, outfile):
@@ -168,6 +188,11 @@ class LogTee(object):
 def exec_func(func, d, dirs = None, pythonexception=False):
     """Execute a BB 'function'"""
 
+    try:
+        oldcwd = os.getcwd()
+    except:
+        oldcwd = None
+
     body = d.getVar(func, False)
     if not body:
         if body is None:
@@ -191,9 +216,7 @@ def exec_func(func, d, dirs = None, pythonexception=False):
             bb.utils.mkdirhier(adir)
         adir = dirs[-1]
     else:
-        adir = d.getVar('B', True)
-        bb.utils.mkdirhier(adir)
-
+        adir = None
     ispython = flags.get('python')
 
     lockflag = flags.get('lockfiles')
@@ -237,6 +260,13 @@ def exec_func(func, d, dirs = None, pythonexception=False):
         else:
             exec_func_shell(func, d, runfile, cwd=adir)
 
+    if oldcwd and os.getcwd() != oldcwd:
+        try:
+            bb.warn("Task %s changed cwd to %s" % (func, os.getcwd()))
+            os.chdir(oldcwd)
+        except:
+            pass
+
 _functionfmt = """
 {function}(d)
 """
@@ -252,7 +282,8 @@ def exec_func_python(func, d, runfile, cwd=None, pythonexception=False):
     if cwd:
         try:
             olddir = os.getcwd()
-        except OSError:
+        except OSError as e:
+            bb.warn("%s: Cannot get cwd: %s" % (func, e))
             olddir = None
         os.chdir(cwd)
 
@@ -278,8 +309,8 @@ def exec_func_python(func, d, runfile, cwd=None, pythonexception=False):
         if cwd and olddir:
             try:
                 os.chdir(olddir)
-            except OSError:
-                pass
+            except OSError as e:
+                bb.warn("%s: Cannot restore cwd %s: %s" % (func, olddir, e))
 
 def shell_trap_code():
     return '''#!/bin/sh\n
@@ -340,34 +371,57 @@ exit $ret
     else:
         logfile = sys.stdout
 
+    progress = d.getVarFlag(func, 'progress', True)
+    if progress:
+        if progress == 'percent':
+            # Use default regex
+            logfile = bb.progress.BasicProgressHandler(d, outfile=logfile)
+        elif progress.startswith('percent:'):
+            # Use specified regex
+            logfile = bb.progress.BasicProgressHandler(d, regex=progress.split(':', 1)[1], outfile=logfile)
+        elif progress.startswith('outof:'):
+            # Use specified regex
+            logfile = bb.progress.OutOfProgressHandler(d, regex=progress.split(':', 1)[1], outfile=logfile)
+        else:
+            bb.warn('%s: invalid task progress varflag value "%s", ignoring' % (func, progress))
+
+    fifobuffer = bytearray()
     def readfifo(data):
-        lines = data.split(b'\0')
-        for line in lines:
-            splitval = line.split(b' ', 1)
-            cmd = splitval[0]
-            if len(splitval) > 1:
-                value = splitval[1].decode("utf-8")
+        nonlocal fifobuffer
+        fifobuffer.extend(data)
+        while fifobuffer:
+            message, token, nextmsg = fifobuffer.partition(b"\00")
+            if token:
+                splitval = message.split(b' ', 1)
+                cmd = splitval[0].decode("utf-8")
+                if len(splitval) > 1:
+                    value = splitval[1].decode("utf-8")
+                else:
+                    value = ''
+                if cmd == 'bbplain':
+                    bb.plain(value)
+                elif cmd == 'bbnote':
+                    bb.note(value)
+                elif cmd == 'bbwarn':
+                    bb.warn(value)
+                elif cmd == 'bberror':
+                    bb.error(value)
+                elif cmd == 'bbfatal':
+                    # The caller will call exit themselves, so bb.error() is
+                    # what we want here rather than bb.fatal()
+                    bb.error(value)
+                elif cmd == 'bbfatal_log':
+                    bb.error(value, forcelog=True)
+                elif cmd == 'bbdebug':
+                    splitval = value.split(' ', 1)
+                    level = int(splitval[0])
+                    value = splitval[1]
+                    bb.debug(level, value)
+                else:
+                    bb.warn("Unrecognised command '%s' on FIFO" % cmd)
+                fifobuffer = nextmsg
             else:
-                value = ''
-            if cmd == 'bbplain':
-                bb.plain(value)
-            elif cmd == 'bbnote':
-                bb.note(value)
-            elif cmd == 'bbwarn':
-                bb.warn(value)
-            elif cmd == 'bberror':
-                bb.error(value)
-            elif cmd == 'bbfatal':
-                # The caller will call exit themselves, so bb.error() is
-                # what we want here rather than bb.fatal()
-                bb.error(value)
-            elif cmd == 'bbfatal_log':
-                bb.error(value, forcelog=True)
-            elif cmd == 'bbdebug':
-                splitval = value.split(' ', 1)
-                level = int(splitval[0])
-                value = splitval[1]
-                bb.debug(level, value)
+                break
 
     tempdir = d.getVar('T', True)
     fifopath = os.path.join(tempdir, 'fifo.%s' % os.getpid())
@@ -778,6 +832,7 @@ def deltask(task, d):
     bbtasks = d.getVar('__BBTASKS', False) or []
     if task in bbtasks:
         bbtasks.remove(task)
+        d.delVarFlag(task, 'task')
         d.setVar('__BBTASKS', bbtasks)
 
     d.delVarFlag(task, 'deps')

bitbake/lib/bb/cache.py

@@ -280,72 +280,74 @@ class Cache(object):
         cache_ok = True
         if self.caches_array:
             for cache_class in self.caches_array:
-                if type(cache_class) is type and issubclass(cache_class, RecipeInfoCommon):
-                    cachefile = getCacheFile(self.cachedir, cache_class.cachefile, self.data_hash)
-                    cache_ok = cache_ok and os.path.exists(cachefile)
-                    cache_class.init_cacheData(self)
+                cachefile = getCacheFile(self.cachedir, cache_class.cachefile, self.data_hash)
+                cache_ok = cache_ok and os.path.exists(cachefile)
+                cache_class.init_cacheData(self)
         if cache_ok:
             self.load_cachefile()
         elif os.path.isfile(self.cachefile):
             logger.info("Out of date cache found, rebuilding...")
 
     def load_cachefile(self):
-        # Firstly, using core cache file information for
-        # valid checking
-        with open(self.cachefile, "rb") as cachefile:
-            pickled = pickle.Unpickler(cachefile)
-            try:
-                cache_ver = pickled.load()
-                bitbake_ver = pickled.load()
-            except Exception:
-                logger.info('Invalid cache, rebuilding...')
-                return
-
-            if cache_ver != __cache_version__:
-                logger.info('Cache version mismatch, rebuilding...')
-                return
-            elif bitbake_ver != bb.__version__:
-                logger.info('Bitbake version mismatch, rebuilding...')
-                return
-
-
         cachesize = 0
         previous_progress = 0
         previous_percent = 0
 
         # Calculate the correct cachesize of all those cache files
         for cache_class in self.caches_array:
-            if type(cache_class) is type and issubclass(cache_class, RecipeInfoCommon):
-                cachefile = getCacheFile(self.cachedir, cache_class.cachefile, self.data_hash)
-                with open(cachefile, "rb") as cachefile:
-                    cachesize += os.fstat(cachefile.fileno()).st_size
+            cachefile = getCacheFile(self.cachedir, cache_class.cachefile, self.data_hash)
+            with open(cachefile, "rb") as cachefile:
+                cachesize += os.fstat(cachefile.fileno()).st_size
 
         bb.event.fire(bb.event.CacheLoadStarted(cachesize), self.data)
         
         for cache_class in self.caches_array:
-            if type(cache_class) is type and issubclass(cache_class, RecipeInfoCommon):
-                cachefile = getCacheFile(self.cachedir, cache_class.cachefile, self.data_hash)
-                with open(cachefile, "rb") as cachefile:
-                    pickled = pickle.Unpickler(cachefile)                    
-                    while cachefile:
-                        try:
-                            key = pickled.load()
-                            value = pickled.load()
-                        except Exception:
-                            break
-                        if key in self.depends_cache:
-                            self.depends_cache[key].append(value)
-                        else:
-                            self.depends_cache[key] = [value]
-                        # only fire events on even percentage boundaries
-                        current_progress = cachefile.tell() + previous_progress
-                        current_percent = 100 * current_progress / cachesize
-                        if current_percent > previous_percent:
-                            previous_percent = current_percent
-                            bb.event.fire(bb.event.CacheLoadProgress(current_progress, cachesize),
-                                          self.data)
+            cachefile = getCacheFile(self.cachedir, cache_class.cachefile, self.data_hash)
+            with open(cachefile, "rb") as cachefile:
+                pickled = pickle.Unpickler(cachefile)
+                # Check cache version information
+                try:
+                    cache_ver = pickled.load()
+                    bitbake_ver = pickled.load()
+                except Exception:
+                    logger.info('Invalid cache, rebuilding...')
+                    return
 
-                    previous_progress += current_progress
+                if cache_ver != __cache_version__:
+                    logger.info('Cache version mismatch, rebuilding...')
+                    return
+                elif bitbake_ver != bb.__version__:
+                     logger.info('Bitbake version mismatch, rebuilding...')
+                     return
+
+                # Load the rest of the cache file
+                current_progress = 0
+                while cachefile:
+                    try:
+                        key = pickled.load()
+                        value = pickled.load()
+                    except Exception:
+                        break
+                    if not isinstance(key, str):
+                        bb.warn("%s from extras cache is not a string?" % key)
+                        break
+                    if not isinstance(value, RecipeInfoCommon):
+                        bb.warn("%s from extras cache is not a RecipeInfoCommon class?" % value)
+                        break
+
+                    if key in self.depends_cache:
+                        self.depends_cache[key].append(value)
+                    else:
+                        self.depends_cache[key] = [value]
+                    # only fire events on even percentage boundaries
+                    current_progress = cachefile.tell() + previous_progress
+                    current_percent = 100 * current_progress / cachesize
+                    if current_percent > previous_percent:
+                        previous_percent = current_percent
+                        bb.event.fire(bb.event.CacheLoadProgress(current_progress, cachesize),
+                                      self.data)
+
+                previous_progress += current_progress
 
         # Note: depends cache number is corresponding to the parsing file numbers.
         # The same file has several caches, still regarded as one item in the cache
@@ -395,6 +397,7 @@ class Cache(object):
     @classmethod
     def parse(cls, filename, appends, configdata, caches_array):
         """Parse the specified filename, returning the recipe information"""
+        logger.debug(1, "Parsing %s", filename)
         infos = []
         datastores = cls.load_bbfile(filename, appends, configdata)
         depends = []
@@ -408,9 +411,8 @@ class Cache(object):
 
             info_array = []
             for cache_class in caches_array:
-                if type(cache_class) is type and issubclass(cache_class, RecipeInfoCommon):
-                    info = cache_class(filename, data)
-                    info_array.append(info)
+                info = cache_class(filename, data)
+                info_array.append(info)
             infos.append((virtualfn, info_array))
 
         return infos
@@ -432,7 +434,6 @@ class Cache(object):
                 virtualfn = self.realfn2virtual(filename, variant)
                 infos.append((virtualfn, self.depends_cache[virtualfn]))
         else:
-            logger.debug(1, "Parsing %s", filename)
             return self.parse(filename, appends, configdata, self.caches_array)
 
         return cached, infos
@@ -556,6 +557,9 @@ class Cache(object):
             if virtualfn not in self.depends_cache:
                 logger.debug(2, "Cache: %s is not cached", virtualfn)
                 invalid = True
+            elif len(self.depends_cache[virtualfn]) != len(self.caches_array):
+                logger.debug(2, "Cache: Extra caches missing for %s?" % virtualfn)
+                invalid = True
 
         # If any one of the variants is not present, mark as invalid for all
         if invalid:
@@ -597,30 +601,19 @@ class Cache(object):
             logger.debug(2, "Cache is clean, not saving.")
             return
 
-        file_dict = {}
-        pickler_dict = {}
         for cache_class in self.caches_array:
-            if type(cache_class) is type and issubclass(cache_class, RecipeInfoCommon):
-                cache_class_name = cache_class.__name__
-                cachefile = getCacheFile(self.cachedir, cache_class.cachefile, self.data_hash)
-                file_dict[cache_class_name] = open(cachefile, "wb")
-                pickler_dict[cache_class_name] =  pickle.Pickler(file_dict[cache_class_name], pickle.HIGHEST_PROTOCOL)
-                   
-        pickler_dict['CoreRecipeInfo'].dump(__cache_version__)
-        pickler_dict['CoreRecipeInfo'].dump(bb.__version__)
+            cache_class_name = cache_class.__name__
+            cachefile = getCacheFile(self.cachedir, cache_class.cachefile, self.data_hash)
+            with open(cachefile, "wb") as f:
+                p = pickle.Pickler(f, pickle.HIGHEST_PROTOCOL)
+                p.dump(__cache_version__)
+                p.dump(bb.__version__)
 
-        try:
-            for key, info_array in self.depends_cache.items():
-                for info in info_array:
-                    if isinstance(info, RecipeInfoCommon):
-                        cache_class_name = info.__class__.__name__
-                        pickler_dict[cache_class_name].dump(key)
-                        pickler_dict[cache_class_name].dump(info)
-        finally:
-            for cache_class in self.caches_array:
-                if type(cache_class) is type and issubclass(cache_class, RecipeInfoCommon):
-                    cache_class_name = cache_class.__name__
-                    file_dict[cache_class_name].close()
+                for key, info_array in self.depends_cache.items():
+                    for info in info_array:
+                        if isinstance(info, RecipeInfoCommon) and info.__class__.__name__ == cache_class_name:
+                            p.dump(key)
+                            p.dump(info)
 
         del self.depends_cache
 
@@ -652,8 +645,7 @@ class Cache(object):
 
         info_array = []
         for cache_class in self.caches_array:
-            if type(cache_class) is type and issubclass(cache_class, RecipeInfoCommon):
-                info_array.append(cache_class(realfn, data))
+            info_array.append(cache_class(realfn, data))
         self.add_info(file_name, info_array, cacheData, parsed)
 
     @staticmethod
@@ -721,8 +713,9 @@ class CacheData(object):
     def __init__(self, caches_array):
         self.caches_array = caches_array
         for cache_class in self.caches_array:
-            if type(cache_class) is type and issubclass(cache_class, RecipeInfoCommon):
-                cache_class.init_cacheData(self)        
+            if not issubclass(cache_class, RecipeInfoCommon):
+                bb.error("Extra cache data class %s should subclass RecipeInfoCommon class" % cache_class)
+            cache_class.init_cacheData(self)
 
         # Direct cache variables
         self.task_queues = {}

bitbake/lib/bb/checksum.py

@@ -120,13 +120,15 @@ class FileChecksumCache(MultiProcessCache):
                             checksums.extend(checksum_dir(f))
                     else:
                         checksum = checksum_file(f)
-                        checksums.append((f, checksum))
+                        if checksum:
+                            checksums.append((f, checksum))
             elif os.path.isdir(pth):
                 if not os.path.islink(pth):
                     checksums.extend(checksum_dir(pth))
             else:
                 checksum = checksum_file(pth)
-                checksums.append((pth, checksum))
+                if checksum:
+                    checksums.append((pth, checksum))
 
         checksums.sort(key=operator.itemgetter(1))
         return checksums

bitbake/lib/bb/cooker.py

@@ -744,7 +744,7 @@ class BBCooker:
             for dep in rq.rqdata.runtaskentries[tid].depends:
                 depfn = bb.runqueue.fn_from_tid(dep)
                 deppn = self.recipecache.pkg_fn[depfn]
-                dotname = "%s.%s" % (pn, bb.runqueue.taskname_from_tid(dep))
+                dotname = "%s.%s" % (pn, bb.runqueue.taskname_from_tid(tid))
                 if not dotname in depend_tree["tdepends"]:
                     depend_tree["tdepends"][dotname] = []
                 depend_tree["tdepends"][dotname].append("%s.%s" % (deppn, bb.runqueue.taskname_from_tid(dep)))
@@ -1235,6 +1235,7 @@ class BBCooker:
         """
         Build the file matching regexp buildfile
         """
+        bb.event.fire(bb.event.BuildInit(), self.expanded_data)
 
         # Too many people use -b because they think it's how you normally
         # specify a target to be built, so show a warning
@@ -1377,6 +1378,9 @@ class BBCooker:
         if not task.startswith("do_"):
             task = "do_%s" % task
 
+        packages = ["%s:%s" % (target, task) for target in targets]
+        bb.event.fire(bb.event.BuildInit(packages), self.expanded_data)
+
         taskdata, runlist, fulltargetlist = self.buildTaskData(targets, task, self.configuration.abort)
 
         buildname = self.data.getVar("BUILDNAME", False)

bitbake/lib/bb/event.py

@@ -382,7 +382,11 @@ class BuildBase(Event):
 
 
 
-
+class BuildInit(BuildBase):
+    """buildFile or buildTargets was invoked"""
+    def __init__(self, p=[]):
+        name = None
+        BuildBase.__init__(self, name, p)
 
 class BuildStarted(BuildBase, OperationStarted):
     """bbmake build run started"""
@@ -647,6 +651,33 @@ class MetadataEvent(Event):
         self.type = eventtype
         self._localdata = eventdata
 
+class ProcessStarted(Event):
+    """
+    Generic process started event (usually part of the initial startup)
+    where further progress events will be delivered
+    """
+    def __init__(self, processname, total):
+        Event.__init__(self)
+        self.processname = processname
+        self.total = total
+
+class ProcessProgress(Event):
+    """
+    Generic process progress event (usually part of the initial startup)
+    """
+    def __init__(self, processname, progress):
+        Event.__init__(self)
+        self.processname = processname
+        self.progress = progress
+
+class ProcessFinished(Event):
+    """
+    Generic process finished event (usually part of the initial startup)
+    """
+    def __init__(self, processname):
+        Event.__init__(self)
+        self.processname = processname
+
 class SanityCheck(Event):
     """
     Event to run sanity checks, either raise errors or generate events as return status.

bitbake/lib/bb/fetch2/__init__.py

@@ -664,7 +664,7 @@ def verify_donestamp(ud, d, origud=None):
         # as an upgrade path from the previous done stamp file format.
         if checksums != precomputed_checksums:
             with open(ud.donestamp, "wb") as cachefile:
-                p = pickle.Pickler(cachefile, pickle.HIGHEST_PROTOCOL)
+                p = pickle.Pickler(cachefile, 2)
                 p.dump(checksums)
         return True
     except ChecksumError as e:
@@ -698,7 +698,7 @@ def update_stamp(ud, d):
             checksums = verify_checksum(ud, d)
             # Store the checksums for later re-verification against the recipe
             with open(ud.donestamp, "wb") as cachefile:
-                p = pickle.Pickler(cachefile, pickle.HIGHEST_PROTOCOL)
+                p = pickle.Pickler(cachefile, 2)
                 p.dump(checksums)
         except ChecksumError as e:
             # Checksums failed to verify, trigger re-download and remove the
@@ -779,7 +779,7 @@ def localpath(url, d):
     fetcher = bb.fetch2.Fetch([url], d)
     return fetcher.localpath(url)
 
-def runfetchcmd(cmd, d, quiet=False, cleanup=None):
+def runfetchcmd(cmd, d, quiet=False, cleanup=None, log=None):
     """
     Run cmd returning the command output
     Raise an error if interrupted or cmd fails
@@ -821,7 +821,7 @@ def runfetchcmd(cmd, d, quiet=False, cleanup=None):
     error_message = ""
 
     try:
-        (output, errors) = bb.process.run(cmd, shell=True, stderr=subprocess.PIPE)
+        (output, errors) = bb.process.run(cmd, log=log, shell=True, stderr=subprocess.PIPE)
         success = True
     except bb.process.NotFoundError as e:
         error_message = "Fetch command %s" % (e.command)
@@ -832,7 +832,7 @@ def runfetchcmd(cmd, d, quiet=False, cleanup=None):
             output = "output:\n%s" % e.stderr
         else:
             output = "no output"
-        error_message = "Fetch command failed with exit code %s, %s" % (e.exitcode, output)
+        error_message = "Fetch command %s failed with exit code %s, %s" % (e.command, e.exitcode, output)
     except bb.process.CmdError as e:
         error_message = "Fetch command %s could not be run:\n%s" % (e.command, e.msg)
     if not success:
@@ -1431,7 +1431,7 @@ class FetchMethod(object):
                     if urlpath.find("/") != -1:
                         destdir = urlpath.rsplit("/", 1)[0] + '/'
                         bb.utils.mkdirhier("%s/%s" % (unpackdir, destdir))
-                cmd = 'cp -fpPR %s %s' % (file, destdir)
+                cmd = 'cp -fpPRH %s %s' % (file, destdir)
 
         if not cmd:
             return

bitbake/lib/bb/fetch2/bzr.py

@@ -106,7 +106,7 @@ class Bzr(FetchMethod):
         if scmdata == "keep":
             tar_flags = ""
         else:
-            tar_flags = "--exclude '.bzr' --exclude '.bzrtags'"
+            tar_flags = "--exclude='.bzr' --exclude='.bzrtags'"
 
         # tar them up to a defined filename
         runfetchcmd("tar %s -czf %s %s" % (tar_flags, ud.localpath, os.path.basename(ud.pkgdir)), d, cleanup = [ud.localpath])

bitbake/lib/bb/fetch2/cvs.py

@@ -147,7 +147,7 @@ class Cvs(FetchMethod):
         if scmdata == "keep":
             tar_flags = ""
         else:
-            tar_flags = "--exclude 'CVS'"
+            tar_flags = "--exclude='CVS'"
 
         # tar them up to a defined filename
         if 'fullpath' in ud.parm:

bitbake/lib/bb/fetch2/git.py

@@ -71,11 +71,53 @@ import os
 import re
 import bb
 import errno
+import bb.progress
 from   bb    import data
 from   bb.fetch2 import FetchMethod
 from   bb.fetch2 import runfetchcmd
 from   bb.fetch2 import logger
 
+
+class GitProgressHandler(bb.progress.LineFilterProgressHandler):
+    """Extract progress information from git output"""
+    def __init__(self, d):
+        self._buffer = ''
+        self._count = 0
+        super(GitProgressHandler, self).__init__(d)
+        # Send an initial progress event so the bar gets shown
+        self._fire_progress(-1)
+
+    def write(self, string):
+        self._buffer += string
+        stages = ['Counting objects', 'Compressing objects', 'Receiving objects', 'Resolving deltas']
+        stage_weights = [0.2, 0.05, 0.5, 0.25]
+        stagenum = 0
+        for i, stage in reversed(list(enumerate(stages))):
+            if stage in self._buffer:
+                stagenum = i
+                self._buffer = ''
+                break
+        self._status = stages[stagenum]
+        percs = re.findall(r'(\d+)%', string)
+        if percs:
+            progress = int(round((int(percs[-1]) * stage_weights[stagenum]) + (sum(stage_weights[:stagenum]) * 100)))
+            rates = re.findall(r'([\d.]+ [a-zA-Z]*/s+)', string)
+            if rates:
+                rate = rates[-1]
+            else:
+                rate = None
+            self.update(progress, rate)
+        else:
+            if stagenum == 0:
+                percs = re.findall(r': (\d+)', string)
+                if percs:
+                    count = int(percs[-1])
+                    if count > self._count:
+                        self._count = count
+                        self._fire_progress(-count)
+        super(GitProgressHandler, self).write(string)
+
+
 class Git(FetchMethod):
     """Class to fetch a module or modules from git repositories"""
     def init(self, d):
@@ -196,10 +238,11 @@ class Git(FetchMethod):
             # We do this since git will use a "-l" option automatically for local urls where possible
             if repourl.startswith("file://"):
                 repourl = repourl[7:]
-            clone_cmd = "%s clone --bare --mirror %s %s" % (ud.basecmd, repourl, ud.clonedir)
+            clone_cmd = "LANG=C %s clone --bare --mirror %s %s --progress" % (ud.basecmd, repourl, ud.clonedir)
             if ud.proto.lower() != 'file':
                 bb.fetch2.check_network_access(d, clone_cmd)
-            runfetchcmd(clone_cmd, d)
+            progresshandler = GitProgressHandler(d)
+            runfetchcmd(clone_cmd, d, log=progresshandler)
 
         os.chdir(ud.clonedir)
         # Update the checkout if needed
@@ -214,10 +257,11 @@ class Git(FetchMethod):
                 logger.debug(1, "No Origin")
 
             runfetchcmd("%s remote add --mirror=fetch origin %s" % (ud.basecmd, repourl), d)
-            fetch_cmd = "%s fetch -f --prune %s refs/*:refs/*" % (ud.basecmd, repourl)
+            fetch_cmd = "LANG=C %s fetch -f --prune --progress %s refs/*:refs/*" % (ud.basecmd, repourl)
             if ud.proto.lower() != 'file':
                 bb.fetch2.check_network_access(d, fetch_cmd, ud.url)
-            runfetchcmd(fetch_cmd, d)
+            progresshandler = GitProgressHandler(d)
+            runfetchcmd(fetch_cmd, d, log=progresshandler)
             runfetchcmd("%s prune-packed" % ud.basecmd, d)
             runfetchcmd("%s pack-redundant --all | xargs -r rm" % ud.basecmd, d)
             try:

bitbake/lib/bb/fetch2/gitannex.py

@@ -66,7 +66,7 @@ class GitANNEX(Git):
 
         os.chdir(ud.destdir)
         try:
-            runfetchcmd("%s annex sync" % (ud.basecmd), d)
+            runfetchcmd("%s annex init" % (ud.basecmd), d)
         except bb.fetch.FetchError:
             pass
 

bitbake/lib/bb/fetch2/repo.py

@@ -83,7 +83,7 @@ class Repo(FetchMethod):
         if scmdata == "keep":
             tar_flags = ""
         else:
-            tar_flags = "--exclude '.repo' --exclude '.git'"
+            tar_flags = "--exclude='.repo' --exclude='.git'"
 
         # Create a cache
         runfetchcmd("tar %s -czf %s %s" % (tar_flags, ud.localpath, os.path.join(".", "*") ), d)

bitbake/lib/bb/fetch2/svn.py

@@ -150,7 +150,7 @@ class Svn(FetchMethod):
         if scmdata == "keep":
             tar_flags = ""
         else:
-            tar_flags = "--exclude '.svn'"
+            tar_flags = "--exclude='.svn'"
 
         os.chdir(ud.pkgdir)
         # tar them up to a defined filename

bitbake/lib/bb/fetch2/wget.py

@@ -31,6 +31,7 @@ import subprocess
 import os
 import logging
 import bb
+import bb.progress
 import urllib.request, urllib.parse, urllib.error
 from   bb import data
 from   bb.fetch2 import FetchMethod
@@ -41,6 +42,27 @@ from   bb.utils import export_proxies
 from   bs4 import BeautifulSoup
 from   bs4 import SoupStrainer
 
+class WgetProgressHandler(bb.progress.LineFilterProgressHandler):
+    """
+    Extract progress information from wget output.
+    Note: relies on --progress=dot (with -v or without -q/-nv) being
+    specified on the wget command line.
+    """
+    def __init__(self, d):
+        super(WgetProgressHandler, self).__init__(d)
+        # Send an initial progress event so the bar gets shown
+        self._fire_progress(0)
+
+    def writeline(self, line):
+        percs = re.findall(r'(\d+)%\s+([\d.]+[A-Z])', line)
+        if percs:
+            progress = int(percs[-1][0])
+            rate = percs[-1][1] + '/s'
+            self.update(progress, rate)
+            return False
+        return True
+
+
 class Wget(FetchMethod):
     """Class to fetch urls via 'wget'"""
     def supports(self, ud, d):
@@ -66,13 +88,15 @@ class Wget(FetchMethod):
         if not ud.localfile:
             ud.localfile = data.expand(urllib.parse.unquote(ud.host + ud.path).replace("/", "."), d)
 
-        self.basecmd = d.getVar("FETCHCMD_wget", True) or "/usr/bin/env wget -t 2 -T 30 -nv --passive-ftp --no-check-certificate"
+        self.basecmd = d.getVar("FETCHCMD_wget", True) or "/usr/bin/env wget -t 2 -T 30 --passive-ftp --no-check-certificate"
 
     def _runwget(self, ud, d, command, quiet):
 
+        progresshandler = WgetProgressHandler(d)
+
         logger.debug(2, "Fetching %s using command '%s'" % (ud.url, command))
         bb.fetch2.check_network_access(d, command)
-        runfetchcmd(command, d, quiet)
+        runfetchcmd(command + ' --progress=dot -v', d, quiet, log=progresshandler)
 
     def download(self, ud, d):
         """Fetch urls"""
@@ -104,7 +128,7 @@ class Wget(FetchMethod):
 
         return True
 
-    def checkstatus(self, fetch, ud, d):
+    def checkstatus(self, fetch, ud, d, try_again=True):
         import urllib.request, urllib.error, urllib.parse, socket, http.client
         from urllib.response import addinfourl
         from bb.fetch2 import FetchConnectionCache
@@ -278,9 +302,13 @@ class Wget(FetchMethod):
             r.get_method = lambda: "HEAD"
             opener.open(r)
         except urllib.error.URLError as e:
-            # debug for now to avoid spamming the logs in e.g. remote sstate searches
-            logger.debug(2, "checkstatus() urlopen failed: %s" % e)
-            return False
+            if try_again:
+                logger.debug(2, "checkstatus: trying again")
+                return self.checkstatus(fetch, ud, d, False)
+            else:
+                # debug for now to avoid spamming the logs in e.g. remote sstate searches
+                logger.debug(2, "checkstatus() urlopen failed: %s" % e)
+                return False
         return True
 
     def _parse_path(self, regex, s):

bitbake/lib/bb/main.py

@@ -179,6 +179,9 @@ class BitBakeConfigParameters(cookerdata.ConfigParameters):
         parser.add_option("-D", "--debug", action="count", dest="debug", default=0,
                           help="Increase the debug level. You can specify this more than once.")
 
+        parser.add_option("-q", "--quiet", action="store_true", dest="quiet", default=False,
+                          help="Output less log message data to the terminal.")
+
         parser.add_option("-n", "--dry-run", action="store_true", dest="dry_run", default=False,
                           help="Don't execute, just go through the motions.")
 
@@ -244,9 +247,16 @@ class BitBakeConfigParameters(cookerdata.ConfigParameters):
                           help="Run bitbake without a UI, only starting a server "
                                "(cooker) process.")
 
+        parser.add_option("", "--foreground", action="store_true",
+                          help="Run bitbake server in foreground.")
+
         parser.add_option("-B", "--bind", action="store", dest="bind", default=False,
                           help="The name/address for the bitbake server to bind to.")
 
+        parser.add_option("-T", "--idle-timeout", type=int,
+                          default=int(os.environ.get("BBTIMEOUT", "0")),
+                          help="Set timeout to unload bitbake server due to inactivity")
+
         parser.add_option("", "--no-setscene", action="store_true",
                           dest="nosetscene", default=False,
                           help="Do not run any setscene tasks. sstate will be ignored and "
@@ -279,6 +289,12 @@ class BitBakeConfigParameters(cookerdata.ConfigParameters):
 
         options, targets = parser.parse_args(argv)
 
+        if options.quiet and options.verbose:
+            parser.error("options --quiet and --verbose are mutually exclusive")
+
+        if options.quiet and options.debug:
+            parser.error("options --quiet and --debug are mutually exclusive")
+
         # use configuration files from environment variables
         if "BBPRECONF" in os.environ:
             options.prefile.append(os.environ["BBPRECONF"])
@@ -294,8 +310,10 @@ class BitBakeConfigParameters(cookerdata.ConfigParameters):
 
         # if BBSERVER says to autodetect, let's do that
         if options.remote_server:
-            [host, port] = options.remote_server.split(":", 2)
-            port = int(port)
+            port = -1
+            if options.remote_server != 'autostart':
+                host, port = options.remote_server.split(":", 2)
+                port = int(port)
             # use automatic port if port set to -1, means read it from
             # the bitbake.lock file; this is a bit tricky, but we always expect
             # to be in the base of the build directory if we need to have a
@@ -312,17 +330,19 @@ class BitBakeConfigParameters(cookerdata.ConfigParameters):
                     lf.close()
                     options.remote_server = remotedef
                 except Exception as e:
-                    raise BBMainException("Failed to read bitbake.lock (%s), invalid port" % str(e))
+                    if options.remote_server != 'autostart':
+                        raise BBMainException("Failed to read bitbake.lock (%s), invalid port" % str(e))
 
         return options, targets[1:]
 
 
 def start_server(servermodule, configParams, configuration, features):
     server = servermodule.BitBakeServer()
-    single_use = not configParams.server_only
+    single_use = not configParams.server_only and os.getenv('BBSERVER') != 'autostart'
     if configParams.bind:
         (host, port) = configParams.bind.split(':')
-        server.initServer((host, int(port)), single_use)
+        server.initServer((host, int(port)), single_use=single_use,
+                          idle_timeout=configParams.idle_timeout)
         configuration.interface = [server.serverImpl.host, server.serverImpl.port]
     else:
         server.initServer(single_use=single_use)
@@ -345,7 +365,8 @@ def start_server(servermodule, configParams, configuration, features):
             if isinstance(event, logging.LogRecord):
                 logger.handle(event)
         raise
-    server.detach()
+    if not configParams.foreground:
+        server.detach()
     cooker.lock.close()
     return server
 
@@ -386,6 +407,10 @@ def bitbake_main(configParams, configuration):
                                   ("the BBSERVER environment variable" if "BBSERVER" in os.environ \
                                    else "the '--remote-server' option"))
 
+    elif configParams.foreground:
+        raise BBMainException("FATAL: The '--foreground' option can only be used "
+                              "with --server-only.\n")
+
     if configParams.bind and configParams.servertype != "xmlrpc":
         raise BBMainException("FATAL: If '-B' or '--bind' is defined, we must "
                               "set the servertype as 'xmlrpc'.\n")
@@ -436,6 +461,14 @@ def bitbake_main(configParams, configuration):
         server = start_server(servermodule, configParams, configuration, featureset)
         bb.event.ui_queue = []
     else:
+        if os.getenv('BBSERVER') == 'autostart':
+            if configParams.remote_server == 'autostart' or \
+               not servermodule.check_connection(configParams.remote_server, timeout=2):
+                configParams.bind = 'localhost:0'
+                srv = start_server(servermodule, configParams, configuration, featureset)
+                configParams.remote_server = '%s:%d' % tuple(configuration.interface)
+                bb.event.ui_queue = []
+
         # we start a stub server that is actually a XMLRPClient that connects to a real server
         server = servermodule.BitBakeXMLRPCClient(configParams.observe_only,
                                                   configParams.xmlrpctoken)
@@ -475,6 +508,8 @@ def bitbake_main(configParams, configuration):
     else:
         print("Bitbake server address: %s, server port: %s" % (server.serverImpl.host,
                                                                server.serverImpl.port))
+        if configParams.foreground:
+            server.serverImpl.serve_forever()
         return 0
 
     return 1

bitbake/lib/bb/msg.py

@@ -182,9 +182,12 @@ def constructLogOptions():
         debug_domains["BitBake.%s" % domainarg] = logging.DEBUG - dlevel + 1
     return level, debug_domains
 
-def addDefaultlogFilter(handler, cls = BBLogFilter):
+def addDefaultlogFilter(handler, cls = BBLogFilter, forcelevel=None):
     level, debug_domains = constructLogOptions()
 
+    if forcelevel is not None:
+        level = forcelevel
+
     cls(handler, level, debug_domains)
 
 #

bitbake/lib/bb/persist_data.py

@@ -131,14 +131,14 @@ class SQLTable(collections.MutableMapping):
         return [row[1] for row in data]
 
     def values(self):
-        return list(self.values())
+        return list(self.itervalues())
 
     def itervalues(self):
         data = self._execute("SELECT value FROM %s;" % self.table)
         return (row[0] for row in data)
 
     def items(self):
-        return list(self.items())
+        return list(self.iteritems())
 
     def iteritems(self):
         return self._execute("SELECT * FROM %s;" % self.table)

bitbake/lib/bb/progress.py

@@ -0,0 +1,276 @@
+"""
+BitBake progress handling code
+"""
+
+# Copyright (C) 2016 Intel Corporation
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License version 2 as
+# published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License along
+# with this program; if not, write to the Free Software Foundation, Inc.,
+# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+import sys
+import re
+import time
+import inspect
+import bb.event
+import bb.build
+
+class ProgressHandler(object):
+    """
+    Base class that can pretend to be a file object well enough to be
+    used to build objects to intercept console output and determine the
+    progress of some operation.
+    """
+    def __init__(self, d, outfile=None):
+        self._progress = 0
+        self._data = d
+        self._lastevent = 0
+        if outfile:
+            self._outfile = outfile
+        else:
+            self._outfile = sys.stdout
+
+    def _fire_progress(self, taskprogress, rate=None):
+        """Internal function to fire the progress event"""
+        bb.event.fire(bb.build.TaskProgress(taskprogress, rate), self._data)
+
+    def write(self, string):
+        self._outfile.write(string)
+
+    def flush(self):
+        self._outfile.flush()
+
+    def update(self, progress, rate=None):
+        ts = time.time()
+        if progress > 100:
+            progress = 100
+        if progress != self._progress or self._lastevent + 1 < ts:
+            self._fire_progress(progress, rate)
+            self._lastevent = ts
+            self._progress = progress
+
+class LineFilterProgressHandler(ProgressHandler):
+    """
+    A ProgressHandler variant that provides the ability to filter out
+    the lines if they contain progress information. Additionally, it
+    filters out anything before the last line feed on a line. This can
+    be used to keep the logs clean of output that we've only enabled for
+    getting progress, assuming that that can be done on a per-line
+    basis.
+    """
+    def __init__(self, d, outfile=None):
+        self._linebuffer = ''
+        super(LineFilterProgressHandler, self).__init__(d, outfile)
+
+    def write(self, string):
+        self._linebuffer += string
+        while True:
+            breakpos = self._linebuffer.find('\n') + 1
+            if breakpos == 0:
+                break
+            line = self._linebuffer[:breakpos]
+            self._linebuffer = self._linebuffer[breakpos:]
+            # Drop any line feeds and anything that precedes them
+            lbreakpos = line.rfind('\r') + 1
+            if lbreakpos:
+                line = line[lbreakpos:]
+            if self.writeline(line):
+                super(LineFilterProgressHandler, self).write(line)
+
+    def writeline(self, line):
+        return True
+
+class BasicProgressHandler(ProgressHandler):
+    def __init__(self, d, regex=r'(\d+)%', outfile=None):
+        super(BasicProgressHandler, self).__init__(d, outfile)
+        self._regex = re.compile(regex)
+        # Send an initial progress event so the bar gets shown
+        self._fire_progress(0)
+
+    def write(self, string):
+        percs = self._regex.findall(string)
+        if percs:
+            progress = int(percs[-1])
+            self.update(progress)
+        super(BasicProgressHandler, self).write(string)
+
+class OutOfProgressHandler(ProgressHandler):
+    def __init__(self, d, regex, outfile=None):
+        super(OutOfProgressHandler, self).__init__(d, outfile)
+        self._regex = re.compile(regex)
+        # Send an initial progress event so the bar gets shown
+        self._fire_progress(0)
+
+    def write(self, string):
+        nums = self._regex.findall(string)
+        if nums:
+            progress = (float(nums[-1][0]) / float(nums[-1][1])) * 100
+            self.update(progress)
+        super(OutOfProgressHandler, self).write(string)
+
+class MultiStageProgressReporter(object):
+    """
+    Class which allows reporting progress without the caller
+    having to know where they are in the overall sequence. Useful
+    for tasks made up of python code spread across multiple
+    classes / functions - the progress reporter object can
+    be passed around or stored at the object level and calls
+    to next_stage() and update() made whereever needed.
+    """
+    def __init__(self, d, stage_weights, debug=False):
+        """
+        Initialise the progress reporter.
+
+        Parameters:
+        * d: the datastore (needed for firing the events)
+        * stage_weights: a list of weight values, one for each stage.
+          The value is scaled internally so you only need to specify
+          values relative to other values in the list, so if there
+          are two stages and the first takes 2s and the second takes
+          10s you would specify [2, 10] (or [1, 5], it doesn't matter).
+        * debug: specify True (and ensure you call finish() at the end)
+          in order to show a printout of the calculated stage weights
+          based on timing each stage. Use this to determine what the
+          weights should be when you're not sure.
+        """
+        self._data = d
+        total = sum(stage_weights)
+        self._stage_weights = [float(x)/total for x in stage_weights]
+        self._stage = -1
+        self._base_progress = 0
+        # Send an initial progress event so the bar gets shown
+        self._fire_progress(0)
+        self._debug = debug
+        self._finished = False
+        if self._debug:
+            self._last_time = time.time()
+            self._stage_times = []
+            self._stage_total = None
+            self._callers = []
+
+    def _fire_progress(self, taskprogress):
+        bb.event.fire(bb.build.TaskProgress(taskprogress), self._data)
+
+    def next_stage(self, stage_total=None):
+        """
+        Move to the next stage.
+        Parameters:
+        * stage_total: optional total for progress within the stage,
+          see update() for details
+        NOTE: you need to call this before the first stage.
+        """
+        self._stage += 1
+        self._stage_total = stage_total
+        if self._stage == 0:
+            # First stage
+            if self._debug:
+                self._last_time = time.time()
+        else:
+            if self._stage < len(self._stage_weights):
+                self._base_progress = sum(self._stage_weights[:self._stage]) * 100
+                if self._debug:
+                    currtime = time.time()
+                    self._stage_times.append(currtime - self._last_time)
+                    self._last_time = currtime
+                    self._callers.append(inspect.getouterframes(inspect.currentframe())[1])
+            elif not self._debug:
+                bb.warn('ProgressReporter: current stage beyond declared number of stages')
+                self._base_progress = 100
+            self._fire_progress(self._base_progress)
+
+    def update(self, stage_progress):
+        """
+        Update progress within the current stage.
+        Parameters:
+        * stage_progress: progress value within the stage. If stage_total
+          was specified when next_stage() was last called, then this
+          value is considered to be out of stage_total, otherwise it should
+          be a percentage value from 0 to 100.
+        """
+        if self._stage_total:
+            stage_progress = (float(stage_progress) / self._stage_total) * 100
+        if self._stage < 0:
+            bb.warn('ProgressReporter: update called before first call to next_stage()')
+        elif self._stage < len(self._stage_weights):
+            progress = self._base_progress + (stage_progress * self._stage_weights[self._stage])
+        else:
+            progress = self._base_progress
+        if progress > 100:
+            progress = 100
+        self._fire_progress(progress)
+
+    def finish(self):
+        if self._finished:
+            return
+        self._finished = True
+        if self._debug:
+            import math
+            self._stage_times.append(time.time() - self._last_time)
+            mintime = max(min(self._stage_times), 0.01)
+            self._callers.append(None)
+            stage_weights = [int(math.ceil(x / mintime)) for x in self._stage_times]
+            bb.warn('Stage weights: %s' % stage_weights)
+            out = []
+            for stage_weight, caller in zip(stage_weights, self._callers):
+                if caller:
+                    out.append('Up to %s:%d: %d' % (caller[1], caller[2], stage_weight))
+                else:
+                    out.append('Up to finish: %d' % stage_weight)
+            bb.warn('Stage times:\n  %s' % '\n  '.join(out))
+
+class MultiStageProcessProgressReporter(MultiStageProgressReporter):
+    """
+    Version of MultiStageProgressReporter intended for use with
+    standalone processes (such as preparing the runqueue)
+    """
+    def __init__(self, d, processname, stage_weights, debug=False):
+        self._processname = processname
+        self._started = False
+        MultiStageProgressReporter.__init__(self, d, stage_weights, debug)
+
+    def start(self):
+        if not self._started:
+            bb.event.fire(bb.event.ProcessStarted(self._processname, 100), self._data)
+            self._started = True
+
+    def _fire_progress(self, taskprogress):
+        if taskprogress == 0:
+            self.start()
+            return
+        bb.event.fire(bb.event.ProcessProgress(self._processname, taskprogress), self._data)
+
+    def finish(self):
+        MultiStageProgressReporter.finish(self)
+        bb.event.fire(bb.event.ProcessFinished(self._processname), self._data)
+
+class DummyMultiStageProcessProgressReporter(MultiStageProgressReporter):
+    """
+    MultiStageProcessProgressReporter that takes the calls and does nothing
+    with them (to avoid a bunch of "if progress_reporter:" checks)
+    """
+    def __init__(self):
+        MultiStageProcessProgressReporter.__init__(self, "", None, [])
+
+    def _fire_progress(self, taskprogress, rate=None):
+        pass
+
+    def start(self):
+        pass
+
+    def next_stage(self, stage_total=None):
+        pass
+
+    def update(self, stage_progress):
+        pass
+
+    def finish(self):
+        pass

bitbake/lib/bb/runqueue.py

@@ -240,6 +240,8 @@ class RunQueueData:
 
         self.stampwhitelist = cfgData.getVar("BB_STAMP_WHITELIST", True) or ""
         self.multi_provider_whitelist = (cfgData.getVar("MULTI_PROVIDER_WHITELIST", True) or "").split()
+        self.setscenewhitelist = get_setscene_enforce_whitelist(cfgData)
+        self.init_progress_reporter = bb.progress.DummyMultiStageProcessProgressReporter()
 
         self.reset()
 
@@ -431,7 +433,8 @@ class RunQueueData:
             # Nothing to do
             return 0
 
-        logger.info("Preparing RunQueue")
+        self.init_progress_reporter.start()
+        self.init_progress_reporter.next_stage()
 
         # Step A - Work out a list of tasks to run
         #
@@ -561,8 +564,9 @@ class RunQueueData:
         # e.g. do_sometask[recrdeptask] = "do_someothertask"
         # (makes sure sometask runs after someothertask of all DEPENDS, RDEPENDS and intertask dependencies, recursively)
         # We need to do this separately since we need all of runtaskentries[*].depends to be complete before this is processed
+        self.init_progress_reporter.next_stage(len(recursivetasks))
         extradeps = {}
-        for tid in recursivetasks:
+        for taskcounter, tid in enumerate(recursivetasks):
             extradeps[tid] = set(self.runtaskentries[tid].depends)
 
             tasknames = recursivetasks[tid]
@@ -584,6 +588,7 @@ class RunQueueData:
             if tid in recursiveitasks:
                 for dep in recursiveitasks[tid]:
                     generate_recdeps(dep)
+            self.init_progress_reporter.update(taskcounter)
 
         # Remove circular references so that do_a[recrdeptask] = "do_a do_b" can work
         for tid in recursivetasks:
@@ -599,6 +604,8 @@ class RunQueueData:
                 logger.debug(2, "Task %s contains self reference!", tid)
                 self.runtaskentries[tid].depends.remove(tid)
 
+        self.init_progress_reporter.next_stage()
+
         # Step B - Mark all active tasks
         #
         # Start with the tasks we were asked to run and mark all dependencies
@@ -663,6 +670,8 @@ class RunQueueData:
             else:
                 mark_active(tid, 1)
 
+        self.init_progress_reporter.next_stage()
+
         # Step C - Prune all inactive tasks
         #
         # Once all active tasks are marked, prune the ones we don't need.
@@ -673,6 +682,8 @@ class RunQueueData:
                 del self.runtaskentries[tid]
                 delcount += 1
 
+        self.init_progress_reporter.next_stage()
+
         #
         # Step D - Sanity checks and computation
         #
@@ -688,11 +699,15 @@ class RunQueueData:
 
         logger.verbose("Assign Weightings")
 
+        self.init_progress_reporter.next_stage()
+
         # Generate a list of reverse dependencies to ease future calculations
         for tid in self.runtaskentries:
             for dep in self.runtaskentries[tid].depends:
                 self.runtaskentries[dep].revdeps.add(tid)
 
+        self.init_progress_reporter.next_stage()
+
         # Identify tasks at the end of dependency chains
         # Error on circular dependency loops (length two)
         endpoints = []
@@ -708,10 +723,14 @@ class RunQueueData:
 
         logger.verbose("Compute totals (have %s endpoint(s))", len(endpoints))
 
+        self.init_progress_reporter.next_stage()
+
         # Calculate task weights
         # Check of higher length circular dependencies
         self.runq_weight = self.calculate_task_weights(endpoints)
 
+        self.init_progress_reporter.next_stage()
+
         # Sanity Check - Check for multiple tasks building the same provider
         prov_list = {}
         seen_fn = []
@@ -803,6 +822,8 @@ class RunQueueData:
                 else:
                     logger.error(msg)
 
+        self.init_progress_reporter.next_stage()
+
         # Create a whitelist usable by the stamp checks
         stampfnwhitelist = []
         for entry in self.stampwhitelist.split():
@@ -812,6 +833,8 @@ class RunQueueData:
             stampfnwhitelist.append(fn)
         self.stampfnwhitelist = stampfnwhitelist
 
+        self.init_progress_reporter.next_stage()
+
         # Iterate over the task list looking for tasks with a 'setscene' function
         self.runq_setscene_tids = []
         if not self.cooker.configuration.nosetscene:
@@ -836,6 +859,8 @@ class RunQueueData:
                 logger.verbose("Invalidate task %s, %s", taskname, fn)
                 bb.parse.siggen.invalidate_task(taskname, self.dataCache, fn)
 
+        self.init_progress_reporter.next_stage()
+
         # Invalidate task if force mode active
         if self.cooker.configuration.force:
             for (fn, target) in self.target_pairs:
@@ -849,6 +874,8 @@ class RunQueueData:
                         st = "do_%s" % st
                     invalidate_task(fn, st, True)
 
+        self.init_progress_reporter.next_stage()
+
         # Create and print to the logs a virtual/xxxx -> PN (fn) table
         virtmap = taskData.get_providermap(prefix="virtual/")
         virtpnmap = {}
@@ -858,6 +885,8 @@ class RunQueueData:
         if hasattr(bb.parse.siggen, "tasks_resolved"):
             bb.parse.siggen.tasks_resolved(virtmap, virtpnmap, self.dataCache)
 
+        self.init_progress_reporter.next_stage()
+
         # Iterate over the task list and call into the siggen code
         dealtwith = set()
         todeal = set(self.runtaskentries)
@@ -1095,14 +1124,25 @@ class RunQueue:
 
         if self.state is runQueuePrepare:
             self.rqexe = RunQueueExecuteDummy(self)
+            # NOTE: if you add, remove or significantly refactor the stages of this
+            # process then you should recalculate the weightings here. This is quite
+            # easy to do - just change the next line temporarily to pass debug=True as
+            # the last parameter and you'll get a printout of the weightings as well
+            # as a map to the lines where next_stage() was called. Of course this isn't
+            # critical, but it helps to keep the progress reporting accurate.
+            self.rqdata.init_progress_reporter = bb.progress.MultiStageProcessProgressReporter(self.cooker.data,
+                                                            "Initialising tasks",
+                                                            [43, 967, 4, 3, 1, 5, 3, 7, 13, 1, 2, 1, 1, 246, 35, 1, 38, 1, 35, 2, 338, 204, 142, 3, 3, 37, 244])
             if self.rqdata.prepare() == 0:
                 self.state = runQueueComplete
             else:
                 self.state = runQueueSceneInit
+                self.rqdata.init_progress_reporter.next_stage()
 
                 # we are ready to run,  emit dependency info to any UI or class which
                 # needs it
                 depgraph = self.cooker.buildDependTree(self, self.rqdata.taskData)
+                self.rqdata.init_progress_reporter.next_stage()
                 bb.event.fire(bb.event.DepTreeGenerated(depgraph), self.cooker.data)
 
         if self.state is runQueueSceneInit:
@@ -1115,7 +1155,9 @@ class RunQueue:
                     self.write_diffscenetasks(invalidtasks)
                 self.state = runQueueComplete
             else:
+                self.rqdata.init_progress_reporter.next_stage()
                 self.start_worker()
+                self.rqdata.init_progress_reporter.next_stage()
                 self.rqexe = RunQueueExecuteScenequeue(self)
 
         if self.state in [runQueueSceneRun, runQueueRunning, runQueueCleanUp]:
@@ -1128,6 +1170,8 @@ class RunQueue:
             if self.cooker.configuration.setsceneonly:
                 self.state = runQueueComplete
             else:
+                # Just in case we didn't setscene
+                self.rqdata.init_progress_reporter.finish()
                 logger.info("Executing RunQueue Tasks")
                 self.rqexe = RunQueueExecuteTasks(self)
                 self.state = runQueueRunning
@@ -1175,8 +1219,8 @@ class RunQueue:
                 pass
             self.state = runQueueComplete
             raise
-        except:
-            logger.error("An uncaught exception occured in runqueue, please see the failure below:")
+        except Exception as err:
+            logger.exception("An uncaught exception occurred in runqueue")
             try:
                 self.teardown_workers()
             except:
@@ -1596,8 +1640,51 @@ class RunQueueExecuteTasks(RunQueueExecute):
         Run the tasks in a queue prepared by rqdata.prepare()
         """
 
-        self.rq.read_workers()
+        if self.rqdata.setscenewhitelist:
+            # Check tasks that are going to run against the whitelist
+            def check_norun_task(tid, showerror=False):
+                fn = fn_from_tid(tid)
+                taskname = taskname_from_tid(tid)
+                # Ignore covered tasks
+                if tid in self.rq.scenequeue_covered:
+                    return False
+                # Ignore stamped tasks
+                if self.rq.check_stamp_task(tid, taskname, cache=self.stampcache):
+                    return False
+                # Ignore noexec tasks
+                taskdep = self.rqdata.dataCache.task_deps[fn]
+                if 'noexec' in taskdep and taskname in taskdep['noexec']:
+                    return False
 
+                pn = self.rqdata.dataCache.pkg_fn[fn]
+                if not check_setscene_enforce_whitelist(pn, taskname, self.rqdata.setscenewhitelist):
+                    if showerror:
+                        if tid in self.rqdata.runq_setscene_tids:
+                            logger.error('Task %s.%s attempted to execute unexpectedly and should have been setscened' % (pn, taskname))
+                        else:
+                            logger.error('Task %s.%s attempted to execute unexpectedly' % (pn, taskname))
+                    return True
+                return False
+            # Look to see if any tasks that we think shouldn't run are going to
+            unexpected = False
+            for tid in self.rqdata.runtaskentries:
+                if check_norun_task(tid):
+                    unexpected = True
+                    break
+            if unexpected:
+                # Run through the tasks in the rough order they'd have executed and print errors
+                # (since the order can be useful - usually missing sstate for the last few tasks
+                # is the cause of the problem)
+                task = self.sched.next()
+                while task is not None:
+                    check_norun_task(task, showerror=True)
+                    self.task_skip(task, 'Setscene enforcement check')
+                    task = self.sched.next()
+
+                self.rq.state = runQueueCleanUp
+                return True
+
+        self.rq.read_workers()
 
         if self.stats.total == 0:
             # nothing to do
@@ -1725,6 +1812,8 @@ class RunQueueExecuteScenequeue(RunQueueExecute):
         # therefore aims to collapse the huge runqueue dependency tree into a smaller one
         # only containing the setscene functions.
 
+        self.rqdata.init_progress_reporter.next_stage()
+
         # First process the chains up to the first setscene task.
         endpoints = {}
         for tid in self.rqdata.runtaskentries:
@@ -1734,6 +1823,8 @@ class RunQueueExecuteScenequeue(RunQueueExecute):
                 #bb.warn("Added endpoint %s" % (tid))
                 endpoints[tid] = set()
 
+        self.rqdata.init_progress_reporter.next_stage()
+
         # Secondly process the chains between setscene tasks.
         for tid in self.rqdata.runq_setscene_tids:
             #bb.warn("Added endpoint 2 %s" % (tid))
@@ -1743,6 +1834,8 @@ class RunQueueExecuteScenequeue(RunQueueExecute):
                     #bb.warn("  Added endpoint 3 %s" % (dep))
                     endpoints[dep].add(tid)
 
+        self.rqdata.init_progress_reporter.next_stage()
+
         def process_endpoints(endpoints):
             newendpoints = {}
             for point, task in endpoints.items():
@@ -1767,6 +1860,8 @@ class RunQueueExecuteScenequeue(RunQueueExecute):
 
         process_endpoints(endpoints)
 
+        self.rqdata.init_progress_reporter.next_stage()
+
         # Build a list of setscene tasks which are "unskippable"
         # These are direct endpoints referenced by the build
         endpoints2 = {}
@@ -1803,7 +1898,9 @@ class RunQueueExecuteScenequeue(RunQueueExecute):
             if sq_revdeps_new2[tid]:
                 self.unskippable.append(tid)
 
-        for tid in self.rqdata.runtaskentries:
+        self.rqdata.init_progress_reporter.next_stage(len(self.rqdata.runtaskentries))
+
+        for taskcounter, tid in enumerate(self.rqdata.runtaskentries):
             if tid in self.rqdata.runq_setscene_tids:
                 deps = set()
                 for dep in sq_revdeps_new[tid]:
@@ -1811,6 +1908,9 @@ class RunQueueExecuteScenequeue(RunQueueExecute):
                 sq_revdeps_squash[tid] = deps
             elif len(sq_revdeps_new[tid]) != 0:
                 bb.msg.fatal("RunQueue", "Something went badly wrong during scenequeue generation, aborting. Please report this problem.")
+            self.rqdata.init_progress_reporter.update(taskcounter)
+
+        self.rqdata.init_progress_reporter.next_stage()
 
         # Resolve setscene inter-task dependencies
         # e.g. do_sometask_setscene[depends] = "targetname:do_someothertask_setscene"
@@ -1838,10 +1938,14 @@ class RunQueueExecuteScenequeue(RunQueueExecute):
                     # Have to zero this to avoid circular dependencies
                     sq_revdeps_squash[deptid] = set()
 
+        self.rqdata.init_progress_reporter.next_stage()
+
         for task in self.sq_harddeps:
              for dep in self.sq_harddeps[task]:
                  sq_revdeps_squash[dep].add(task)
 
+        self.rqdata.init_progress_reporter.next_stage()
+
         #for tid in sq_revdeps_squash:
         #    for dep in sq_revdeps_squash[tid]:
         #        data = data + "\n   %s" % dep
@@ -1857,6 +1961,8 @@ class RunQueueExecuteScenequeue(RunQueueExecute):
             for dep in self.sq_revdeps[tid]:
                 self.sq_deps[dep].add(tid)
 
+        self.rqdata.init_progress_reporter.next_stage()
+
         for tid in self.sq_revdeps:
             if len(self.sq_revdeps[tid]) == 0:
                 self.runq_buildable.add(tid)
@@ -1912,6 +2018,8 @@ class RunQueueExecuteScenequeue(RunQueueExecute):
                     logger.debug(2, 'No package found, so skipping setscene task %s', tid)
                     self.outrightfail.append(tid)
 
+        self.rqdata.init_progress_reporter.finish()
+
         logger.info('Executing SetScene Tasks')
 
         self.rq.state = runQueueSceneRun
@@ -1940,6 +2048,16 @@ class RunQueueExecuteScenequeue(RunQueueExecute):
         self.scenequeue_covered.add(task)
         self.scenequeue_updatecounters(task)
 
+    def check_taskfail(self, task):
+        if self.rqdata.setscenewhitelist:
+            realtask = task.split('_setscene')[0]
+            fn = fn_from_tid(realtask)
+            taskname = taskname_from_tid(realtask)
+            pn = self.rqdata.dataCache.pkg_fn[fn]
+            if not check_setscene_enforce_whitelist(pn, taskname, self.rqdata.setscenewhitelist):
+                logger.error('Task %s.%s failed' % (pn, taskname + "_setscene"))
+                self.rq.state = runQueueCleanUp
+
     def task_complete(self, task):
         self.stats.taskCompleted()
         bb.event.fire(sceneQueueTaskCompleted(task, self.stats, self.rq), self.cfgData)
@@ -1950,6 +2068,7 @@ class RunQueueExecuteScenequeue(RunQueueExecute):
         bb.event.fire(sceneQueueTaskFailed(task, self.stats, result, self), self.cfgData)
         self.scenequeue_notcovered.add(task)
         self.scenequeue_updatecounters(task, True)
+        self.check_taskfail(task)
 
     def task_failoutright(self, task):
         self.runq_running.add(task)
@@ -2231,3 +2350,27 @@ class runQueuePipe():
         if len(self.queue) > 0:
             print("Warning, worker left partial message: %s" % self.queue)
         self.input.close()
+
+def get_setscene_enforce_whitelist(d):
+    if d.getVar('BB_SETSCENE_ENFORCE', True) != '1':
+        return None
+    whitelist = (d.getVar("BB_SETSCENE_ENFORCE_WHITELIST", True) or "").split()
+    outlist = []
+    for item in whitelist[:]:
+        if item.startswith('%:'):
+            for target in sys.argv[1:]:
+                if not target.startswith('-'):
+                    outlist.append(target.split(':')[0] + ':' + item.split(':')[1])
+        else:
+            outlist.append(item)
+    return outlist
+
+def check_setscene_enforce_whitelist(pn, taskname, whitelist):
+    import fnmatch
+    if whitelist:
+        item = '%s:%s' % (pn, taskname)
+        for whitelist_item in whitelist:
+            if fnmatch.fnmatch(item, whitelist_item):
+                return True
+        return False
+    return True

bitbake/lib/bb/server/xmlrpc.py

@@ -85,6 +85,23 @@ def _create_server(host, port, timeout = 60):
     s = xmlrpc.client.ServerProxy("http://%s:%d/" % (host, port), transport=t, allow_none=True, use_builtin_types=True)
     return s, t
 
+def check_connection(remote, timeout):
+    try:
+        host, port = remote.split(":")
+        port = int(port)
+    except Exception as e:
+        bb.warn("Failed to read remote definition (%s)" % str(e))
+        raise e
+
+    server, _transport = _create_server(host, port, timeout)
+    try:
+        ret, err =  server.runCommand(['getVariable', 'TOPDIR'])
+        if err or not ret:
+            return False
+    except ConnectionError:
+        return False
+    return True
+
 class BitBakeServerCommands():
 
     def __init__(self, server):
@@ -188,7 +205,7 @@ class XMLRPCServer(SimpleXMLRPCServer, BaseImplServer):
     # remove this when you're done with debugging
     # allow_reuse_address = True
 
-    def __init__(self, interface, single_use=False):
+    def __init__(self, interface, single_use=False, idle_timeout=0):
         """
         Constructor
         """
@@ -206,6 +223,10 @@ class XMLRPCServer(SimpleXMLRPCServer, BaseImplServer):
         self.commands = BitBakeServerCommands(self)
         self.autoregister_all_functions(self.commands, "")
         self.interface = interface
+        self.time = time.time()
+        self.idle_timeout = idle_timeout
+        if idle_timeout:
+            self.register_idle_function(self.handle_idle_timeout, self)
 
     def addcooker(self, cooker):
         BaseImplServer.addcooker(self, cooker)
@@ -221,6 +242,12 @@ class XMLRPCServer(SimpleXMLRPCServer, BaseImplServer):
             if name.startswith(prefix):
                 self.register_function(method, name[len(prefix):])
 
+    def handle_idle_timeout(self, server, data, abort):
+        if not abort:
+            if time.time() - server.time > server.idle_timeout:
+                server.quit = True
+                print("Server idle timeout expired")
+        return []
 
     def serve_forever(self):
         # Start the actual XMLRPC server
@@ -263,6 +290,8 @@ class XMLRPCServer(SimpleXMLRPCServer, BaseImplServer):
             try:
                 fd_sets = select.select(fds, [], [], socktimeout)
                 if fd_sets[0] and self in fd_sets[0]:
+                    if self.idle_timeout:
+                        self.time = time.time()
                     self._handle_request_noblock()
             except IOError:
                 # we ignore interrupted calls
@@ -334,9 +363,10 @@ class BitBakeXMLRPCServerConnection(BitBakeBaseServerConnection):
             pass
 
 class BitBakeServer(BitBakeBaseServer):
-    def initServer(self, interface = ("localhost", 0), single_use = False):
+    def initServer(self, interface = ("localhost", 0),
+                   single_use = False, idle_timeout=0):
         self.interface = interface
-        self.serverImpl = XMLRPCServer(interface, single_use)
+        self.serverImpl = XMLRPCServer(interface, single_use, idle_timeout)
 
     def detach(self):
         daemonize.createDaemon(self.serverImpl.serve_forever, "bitbake-cookerdaemon.log")

bitbake/lib/bb/siggen.py

@@ -293,11 +293,11 @@ class SignatureGeneratorBasic(SignatureGenerator):
 
         computed_basehash = calc_basehash(data)
         if computed_basehash != self.basehash[k]:
-            bb.error("Basehash mismatch %s verses %s for %s" % (computed_basehash, self.basehash[k], k))
+            bb.error("Basehash mismatch %s versus %s for %s" % (computed_basehash, self.basehash[k], k))
         if runtime and k in self.taskhash:
             computed_taskhash = calc_taskhash(data)
             if computed_taskhash != self.taskhash[k]:
-                bb.error("Taskhash mismatch %s verses %s for %s" % (computed_taskhash, self.taskhash[k], k))
+                bb.error("Taskhash mismatch %s versus %s for %s" % (computed_taskhash, self.taskhash[k], k))
 
 
     def dump_sigs(self, dataCache, options):

bitbake/lib/bb/ui/buildinfohelper.py

@@ -37,7 +37,7 @@ os.environ["DJANGO_SETTINGS_MODULE"] =\
 django.setup()
 
 from orm.models import Build, Task, Recipe, Layer_Version, Layer, Target, LogMessage, HelpText
-from orm.models import Target_Image_File, BuildArtifact
+from orm.models import Target_Image_File, TargetKernelFile, TargetSDKFile
 from orm.models import Variable, VariableHistory
 from orm.models import Package, Package_File, Target_Installed_Package, Target_File
 from orm.models import Task_Dependency, Package_Dependency
@@ -121,11 +121,27 @@ class ORMWrapper(object):
 
         return vars(self)[dictname][key]
 
+    def get_similar_target_with_image_files(self, target):
+        """
+        Get a Target object "similar" to target; i.e. with the same target
+        name ('core-image-minimal' etc.) and machine.
+        """
+        return target.get_similar_target_with_image_files()
+
+    def get_similar_target_with_sdk_files(self, target):
+        return target.get_similar_target_with_sdk_files()
+
+    def clone_image_artifacts(self, target_from, target_to):
+        target_to.clone_image_artifacts_from(target_from)
+
+    def clone_sdk_artifacts(self, target_from, target_to):
+        target_to.clone_sdk_artifacts_from(target_from)
+
     def _timestamp_to_datetime(self, secs):
         """
         Convert timestamp in seconds to Python datetime
         """
-        return datetime(1970, 1, 1) + timedelta(seconds=secs)
+        return timezone.make_aware(datetime(1970, 1, 1) + timedelta(seconds=secs))
 
     # pylint: disable=no-self-use
     # we disable detection of no self use in functions because the methods actually work on the object
@@ -134,55 +150,33 @@ class ORMWrapper(object):
     # pylint: disable=bad-continuation
     # we do not follow the python conventions for continuation indentation due to long lines here
 
-    def create_build_object(self, build_info, brbe, project_id):
-        assert 'machine' in build_info
-        assert 'distro' in build_info
-        assert 'distro_version' in build_info
-        assert 'started_on' in build_info
-        assert 'cooker_log_path' in build_info
-        assert 'build_name' in build_info
-        assert 'bitbake_version' in build_info
-
+    def get_or_create_build_object(self, brbe):
         prj = None
         buildrequest = None
-        if brbe is not None:            # this build was triggered by a request from a user
+        if brbe is not None:
+            # Toaster-triggered build
             logger.debug(1, "buildinfohelper: brbe is %s" % brbe)
             br, _ = brbe.split(":")
-            buildrequest = BuildRequest.objects.get(pk = br)
+            buildrequest = BuildRequest.objects.get(pk=br)
             prj = buildrequest.project
-
-        elif project_id is not None:    # this build was triggered by an external system for a specific project
-            logger.debug(1, "buildinfohelper: project is %s" % prj)
-            prj = Project.objects.get(pk = project_id)
-
-        else:                           # this build was triggered by a legacy system, or command line interactive mode
+        else:
+            # CLI build
             prj = Project.objects.get_or_create_default_project()
             logger.debug(1, "buildinfohelper: project is not specified, defaulting to %s" % prj)
 
-
         if buildrequest is not None:
+            # reuse existing Build object
             build = buildrequest.build
-            logger.info("Updating existing build, with %s", build_info)
             build.project = prj
-            build.machine=build_info['machine']
-            build.distro=build_info['distro']
-            build.distro_version=build_info['distro_version']
-            build.cooker_log_path=build_info['cooker_log_path']
-            build.build_name=build_info['build_name']
-            build.bitbake_version=build_info['bitbake_version']
             build.save()
-
         else:
+            # create new Build object
+            now = timezone.now()
             build = Build.objects.create(
-                                    project = prj,
-                                    machine=build_info['machine'],
-                                    distro=build_info['distro'],
-                                    distro_version=build_info['distro_version'],
-                                    started_on=build_info['started_on'],
-                                    completed_on=build_info['started_on'],
-                                    cooker_log_path=build_info['cooker_log_path'],
-                                    build_name=build_info['build_name'],
-                                    bitbake_version=build_info['bitbake_version'])
+                project=prj,
+                started_on=now,
+                completed_on=now,
+                build_name='')
 
         logger.debug(1, "buildinfohelper: build is created %s" % build)
 
@@ -192,8 +186,17 @@ class ORMWrapper(object):
 
         return build
 
+    def update_build(self, build, data_dict):
+        for key in data_dict:
+            setattr(build, key, data_dict[key])
+        build.save()
+
     @staticmethod
     def get_or_create_targets(target_info):
+        """
+        NB get_or_create() is used here because for Toaster-triggered builds,
+        we already created the targets when the build was triggered.
+        """
         result = []
         for target in target_info['targets']:
             task = ''
@@ -203,17 +206,14 @@ class ORMWrapper(object):
                 task = task[3:]
             if task == 'build':
                 task = ''
-            obj, created = Target.objects.get_or_create(build=target_info['build'],
-                                                        target=target)
-            if created:
-                obj.is_image = False
-                if task:
-                    obj.task = task
-                obj.save()
+
+            obj, _ = Target.objects.get_or_create(build=target_info['build'],
+                                                  target=target,
+                                                  task=task)
             result.append(obj)
         return result
 
-    def update_build_object(self, build, errors, warnings, taskfailures):
+    def update_build_stats_and_outcome(self, build, errors, warnings, taskfailures):
         assert isinstance(build,Build)
         assert isinstance(errors, int)
         assert isinstance(warnings, int)
@@ -239,6 +239,10 @@ class ORMWrapper(object):
         target.license_manifest_path = license_manifest_path
         target.save()
 
+    def update_target_set_package_manifest(self, target, package_manifest_path):
+        target.package_manifest_path = package_manifest_path
+        target.save()
+
     def update_task_object(self, build, task_name, recipe_name, task_stats):
         """
         Find the task for build which matches the recipe and task name
@@ -372,7 +376,7 @@ class ORMWrapper(object):
             layer_copy, c = Layer_Version.objects.get_or_create(
                 build=build_obj,
                 layer=layer_obj.layer,
-                up_branch=layer_obj.up_branch,
+                release=layer_obj.release,
                 branch=layer_version_information['branch'],
                 commit=layer_version_information['commit'],
                 local_path=layer_version_information['local_path'],
@@ -415,13 +419,24 @@ class ORMWrapper(object):
         assert 'name' in layer_information
         assert 'layer_index_url' in layer_information
 
+        # From command line builds we have no brbe as the request is directly
+        # from bitbake
         if brbe is None:
-            layer_object, _ = Layer.objects.get_or_create(
-                                name=layer_information['name'],
-                                layer_index_url=layer_information['layer_index_url'])
+            # If we don't have git commit sha then we're using a non-git
+            # layer so set the layer_source_dir to identify it as such
+            if not layer_information['version']['commit']:
+                local_source_dir = layer_information["local_path"]
+            else:
+                local_source_dir = None
+
+            layer_object, _ = \
+                Layer.objects.get_or_create(
+                    name=layer_information['name'],
+                    local_source_dir=local_source_dir,
+                    layer_index_url=layer_information['layer_index_url'])
+
             return layer_object
         else:
-            # we are under managed mode; we must match the layer used in the Project Layer
             br_id, be_id = brbe.split(":")
 
             # find layer by checkout path;
@@ -446,6 +461,11 @@ class ORMWrapper(object):
                     if brl.layer_version:
                         return brl.layer_version
 
+                # This might be a local layer (i.e. no git info) so try
+                # matching local_source_dir
+                if brl.local_source_dir and brl.local_source_dir == layer_information["local_path"]:
+                    return brl.layer_version
+
                     # we matched the BRLayer, but we need the layer_version that generated this BR; reverse of the Project.schedule_build()
                     #logger.debug(1, "Matched %s to BRlayer %s" % (pformat(layer_information["local_path"]), localdirname))
 
@@ -604,8 +624,8 @@ class ORMWrapper(object):
                         Recipe,
                         name=built_recipe.name,
                         layer_version__build=None,
-                        layer_version__up_branch=
-                        built_recipe.layer_version.up_branch,
+                        layer_version__release=
+                        built_recipe.layer_version.release,
                         file_path=built_recipe.file_path,
                         version=built_recipe.version
                     )
@@ -674,20 +694,23 @@ class ORMWrapper(object):
             logger.warning("buildinfohelper: target_package_info could not identify recipes: \n%s", errormsg)
 
     def save_target_image_file_information(self, target_obj, file_name, file_size):
-        Target_Image_File.objects.create( target = target_obj,
-                            file_name = file_name,
-                            file_size = file_size)
+        Target_Image_File.objects.create(target=target_obj,
+            file_name=file_name, file_size=file_size)
 
-    def save_artifact_information(self, build_obj, file_name, file_size):
-        # we skip the image files from other builds
-        if Target_Image_File.objects.filter(file_name = file_name).count() > 0:
-            return
+    def save_target_kernel_file(self, target_obj, file_name, file_size):
+        """
+        Save kernel file (bzImage, modules*) information for a Target target_obj.
+        """
+        TargetKernelFile.objects.create(target=target_obj,
+            file_name=file_name, file_size=file_size)
 
-        # do not update artifacts found in other builds
-        if BuildArtifact.objects.filter(file_name = file_name).count() > 0:
-            return
-
-        BuildArtifact.objects.create(build = build_obj, file_name = file_name, file_size = file_size)
+    def save_target_sdk_file(self, target_obj, file_name, file_size):
+        """
+        Save SDK artifacts to the database, associating them with a
+        Target object.
+        """
+        TargetSDKFile.objects.create(target=target_obj, file_name=file_name,
+            file_size=file_size)
 
     def create_logmessage(self, log_information):
         assert 'build' in log_information
@@ -855,6 +878,11 @@ class BuildInfoHelper(object):
         Keeps in memory all data that needs matching before writing it to the database
     """
 
+    # tasks which produce image files; note we include '', as we set
+    # the task for a target to '' (i.e. 'build') if no target is
+    # explicitly defined
+    IMAGE_GENERATING_TASKS = ['', 'build', 'image', 'populate_sdk_ext']
+
     # pylint: disable=protected-access
     # the code will look into the protected variables of the event; no easy way around this
     # pylint: disable=bad-continuation
@@ -886,22 +914,55 @@ class BuildInfoHelper(object):
     ###################
     ## methods to convert event/external info into objects that the ORM layer uses
 
+    def _ensure_build(self):
+        """
+        Ensure the current build object exists and is up to date with
+        data on the bitbake server
+        """
+        if not 'build' in self.internal_state or not self.internal_state['build']:
+            # create the Build object
+            self.internal_state['build'] = \
+                self.orm_wrapper.get_or_create_build_object(self.brbe)
 
-    def _get_build_information(self, build_log_path):
+        build = self.internal_state['build']
+
+        # update missing fields on the Build object with found data
         build_info = {}
-        build_info['machine'] = self.server.runCommand(["getVariable", "MACHINE"])[0]
-        build_info['distro'] = self.server.runCommand(["getVariable", "DISTRO"])[0]
-        build_info['distro_version'] = self.server.runCommand(["getVariable", "DISTRO_VERSION"])[0]
-        build_info['started_on'] = timezone.now()
-        build_info['completed_on'] = timezone.now()
-        build_info['cooker_log_path'] = build_log_path
-        build_info['build_name'] = self.server.runCommand(["getVariable", "BUILDNAME"])[0]
-        build_info['bitbake_version'] = self.server.runCommand(["getVariable", "BB_VERSION"])[0]
-        build_info['project'] = self.project = self.server.runCommand(["getVariable", "TOASTER_PROJECT"])[0]
-        return build_info
+
+        # set to True if at least one field is going to be set
+        changed = False
+
+        if not build.build_name:
+            build_name = self.server.runCommand(["getVariable", "BUILDNAME"])[0]
+
+            # only reset the build name if the one on the server is actually
+            # a valid value for the build_name field
+            if build_name != None:
+                build_info['build_name'] = build_name
+                changed = True
+
+        if not build.machine:
+            build_info['machine'] = self.server.runCommand(["getVariable", "MACHINE"])[0]
+            changed = True
+
+        if not build.distro:
+            build_info['distro'] = self.server.runCommand(["getVariable", "DISTRO"])[0]
+            changed = True
+
+        if not build.distro_version:
+            build_info['distro_version'] = self.server.runCommand(["getVariable", "DISTRO_VERSION"])[0]
+            changed = True
+
+        if not build.bitbake_version:
+            build_info['bitbake_version'] = self.server.runCommand(["getVariable", "BB_VERSION"])[0]
+            changed = True
+
+        if changed:
+            self.orm_wrapper.update_build(self.internal_state['build'], build_info)
 
     def _get_task_information(self, event, recipe):
         assert 'taskname' in vars(event)
+        self._ensure_build()
 
         task_information = {}
         task_information['build'] = self.internal_state['build']
@@ -916,8 +977,9 @@ class BuildInfoHelper(object):
         return task_information
 
     def _get_layer_version_for_path(self, path):
+        self._ensure_build()
+
         assert path.startswith("/")
-        assert 'build' in self.internal_state
 
         def _slkey_interactive(layer_version):
             assert isinstance(layer_version, Layer_Version)
@@ -928,6 +990,9 @@ class BuildInfoHelper(object):
             # we can match to the recipe file path
             if path.startswith(lvo.local_path):
                 return lvo
+            if lvo.layer.local_source_dir and \
+               path.startswith(lvo.layer.local_source_dir):
+                return lvo
 
         #if we get here, we didn't read layers correctly; dump whatever information we have on the error log
         logger.warning("Could not match layer version for recipe path %s : %s", path, self.orm_wrapper.layer_version_objects)
@@ -961,6 +1026,8 @@ class BuildInfoHelper(object):
         return recipe_info
 
     def _get_path_information(self, task_object):
+        self._ensure_build()
+
         assert isinstance(task_object, Task)
         build_stats_format = "{tmpdir}/buildstats/{buildname}/{package}/"
         build_stats_path = []
@@ -1003,17 +1070,31 @@ class BuildInfoHelper(object):
             except NotExisting as nee:
                 logger.warning("buildinfohelper: cannot identify layer exception:%s ", nee)
 
+    def store_started_build(self):
+        self._ensure_build()
 
-    def store_started_build(self, event, build_log_path):
+    def save_build_log_file_path(self, build_log_path):
+        self._ensure_build()
+
+        if not self.internal_state['build'].cooker_log_path:
+            data_dict = {'cooker_log_path': build_log_path}
+            self.orm_wrapper.update_build(self.internal_state['build'], data_dict)
+
+    def save_build_targets(self, event):
+        self._ensure_build()
+
+        # create target information
         assert '_pkgs' in vars(event)
-        build_information = self._get_build_information(build_log_path)
+        target_information = {}
+        target_information['targets'] = event._pkgs
+        target_information['build'] = self.internal_state['build']
 
-        # Update brbe and project as they can be changed for every build
-        self.project = build_information['project']
+        self.internal_state['targets'] = self.orm_wrapper.get_or_create_targets(target_information)
 
-        build_obj = self.orm_wrapper.create_build_object(build_information, self.brbe, self.project)
+    def save_build_layers_and_variables(self):
+        self._ensure_build()
 
-        self.internal_state['build'] = build_obj
+        build_obj = self.internal_state['build']
 
         # save layer version information for this build
         if not 'lvs' in self.internal_state:
@@ -1024,13 +1105,6 @@ class BuildInfoHelper(object):
 
             del self.internal_state['lvs']
 
-        # create target information
-        target_information = {}
-        target_information['targets'] = event._pkgs
-        target_information['build'] = build_obj
-
-        self.internal_state['targets'] = self.orm_wrapper.get_or_create_targets(target_information)
-
         # Save build configuration
         data = self.server.runCommand(["getAllKeysWithFlags", ["doc", "func"]])[0]
 
@@ -1061,6 +1135,25 @@ class BuildInfoHelper(object):
 
         return self.brbe
 
+    def set_recipes_to_parse(self, num_recipes):
+        """
+        Set the number of recipes which need to be parsed for this build.
+        This is set the first time ParseStarted is received by toasterui.
+        """
+        self._ensure_build()
+        self.internal_state['build'].recipes_to_parse = num_recipes
+        self.internal_state['build'].save()
+
+    def set_recipes_parsed(self, num_recipes):
+        """
+        Set the number of recipes parsed so far for this build; this is updated
+        each time a ParseProgress or ParseCompleted event is received by
+        toasterui.
+        """
+        self._ensure_build()
+        if num_recipes <= self.internal_state['build'].recipes_to_parse:
+            self.internal_state['build'].recipes_parsed = num_recipes
+            self.internal_state['build'].save()
 
     def update_target_image_file(self, event):
         evdata = BuildInfoHelper._get_data_from_event(event)
@@ -1073,23 +1166,17 @@ class BuildInfoHelper(object):
                         self.orm_wrapper.save_target_image_file_information(t, output, evdata[output])
 
     def update_artifact_image_file(self, event):
+        self._ensure_build()
         evdata = BuildInfoHelper._get_data_from_event(event)
         for artifact_path in evdata.keys():
-            self.orm_wrapper.save_artifact_information(self.internal_state['build'], artifact_path, evdata[artifact_path])
+            self.orm_wrapper.save_artifact_information(
+                self.internal_state['build'], artifact_path,
+                evdata[artifact_path])
 
     def update_build_information(self, event, errors, warnings, taskfailures):
-        if 'build' in self.internal_state:
-            self.orm_wrapper.update_build_object(self.internal_state['build'], errors, warnings, taskfailures)
-
-
-    def store_license_manifest_path(self, event):
-        deploy_dir = BuildInfoHelper._get_data_from_event(event)['deploy_dir']
-        image_name = BuildInfoHelper._get_data_from_event(event)['image_name']
-        path = deploy_dir + "/licenses/" + image_name + "/license.manifest"
-        for target in self.internal_state['targets']:
-            if target.target in image_name:
-                self.orm_wrapper.update_target_set_license_manifest(target, path)
-
+        self._ensure_build()
+        self.orm_wrapper.update_build_stats_and_outcome(
+            self.internal_state['build'], errors, warnings, taskfailures)
 
     def store_started_task(self, event):
         assert isinstance(event, (bb.runqueue.sceneQueueTaskStarted, bb.runqueue.runQueueTaskStarted, bb.runqueue.runQueueTaskSkipped))
@@ -1132,6 +1219,7 @@ class BuildInfoHelper(object):
 
 
     def store_tasks_stats(self, event):
+        self._ensure_build()
         task_data = BuildInfoHelper._get_data_from_event(event)
 
         for (task_file, task_name, task_stats, recipe_name) in task_data:
@@ -1227,6 +1315,8 @@ class BuildInfoHelper(object):
 
 
     def store_target_package_data(self, event):
+        self._ensure_build()
+
         # for all image targets
         for target in self.internal_state['targets']:
             if target.is_image:
@@ -1241,11 +1331,16 @@ class BuildInfoHelper(object):
                     logger.warning("KeyError in save_target_package_information"
                                    "%s ", e)
 
-                try:
-                    self.orm_wrapper.save_target_file_information(self.internal_state['build'], target, filedata)
-                except KeyError as e:
-                    logger.warning("KeyError in save_target_file_information"
-                                   "%s ", e)
+                # only try to find files in the image if the task for this
+                # target is one which produces image files; otherwise, the old
+                # list of files in the files-in-image.txt file will be
+                # appended to the target even if it didn't produce any images
+                if target.task in BuildInfoHelper.IMAGE_GENERATING_TASKS:
+                    try:
+                        self.orm_wrapper.save_target_file_information(self.internal_state['build'], target, filedata)
+                    except KeyError as e:
+                        logger.warning("KeyError in save_target_file_information"
+                                       "%s ", e)
 
 
 
@@ -1255,10 +1350,9 @@ class BuildInfoHelper(object):
         note that this only gets called for command line builds which are
         interrupted, so it doesn't touch any BuildRequest objects
         """
-        build = self.internal_state['build']
-        if build:
-            build.outcome = Build.CANCELLED
-            build.save()
+        self._ensure_build()
+        self.internal_state['build'].outcome = Build.CANCELLED
+        self.internal_state['build'].save()
 
     def store_dependency_information(self, event):
         assert '_depgraph' in vars(event)
@@ -1404,6 +1498,8 @@ class BuildInfoHelper(object):
 
 
     def store_build_package_information(self, event):
+        self._ensure_build()
+
         package_info = BuildInfoHelper._get_data_from_event(event)
         self.orm_wrapper.save_build_package_information(
             self.internal_state['build'],
@@ -1419,6 +1515,10 @@ class BuildInfoHelper(object):
 
     def _store_build_done(self, errorcode):
         logger.info("Build exited with errorcode %d", errorcode)
+
+        if not self.brbe:
+            return
+
         br_id, be_id = self.brbe.split(":")
         be = BuildEnvironment.objects.get(pk = be_id)
         be.lock = BuildEnvironment.LOCK_LOCK
@@ -1440,7 +1540,6 @@ class BuildInfoHelper(object):
             br.state = BuildRequest.REQ_FAILED
         br.save()
 
-
     def store_log_error(self, text):
         mockevent = MockEvent()
         mockevent.levelno = formatter.ERROR
@@ -1459,24 +1558,22 @@ class BuildInfoHelper(object):
 
 
     def store_log_event(self, event):
+        self._ensure_build()
+
         if event.levelno < formatter.WARNING:
             return
 
         if 'args' in vars(event):
             event.msg = event.msg % event.args
 
-        if not 'build' in self.internal_state:
-            if self.brbe is None:
-                if not 'backlog' in self.internal_state:
-                    self.internal_state['backlog'] = []
-                self.internal_state['backlog'].append(event)
-                return
-            else:   # we're under Toaster control, the build is already created
-                br, _ = self.brbe.split(":")
-                buildrequest = BuildRequest.objects.get(pk = br)
-                self.internal_state['build'] = buildrequest.build
+        # early return for CLI builds
+        if self.brbe is None:
+            if not 'backlog' in self.internal_state:
+                self.internal_state['backlog'] = []
+            self.internal_state['backlog'].append(event)
+            return
 
-        if 'build' in self.internal_state and 'backlog' in self.internal_state:
+        if 'backlog' in self.internal_state:
             # if we have a backlog of events, do our best to save them here
             if len(self.internal_state['backlog']):
                 tempevent = self.internal_state['backlog'].pop()
@@ -1506,19 +1603,311 @@ class BuildInfoHelper(object):
 
         self.orm_wrapper.create_logmessage(log_information)
 
+    def _get_filenames_from_image_license(self, image_license_manifest_path):
+        """
+        Find the FILES line in the image_license.manifest file,
+        which has the basenames of the bzImage and modules files
+        in this format:
+        FILES: bzImage--4.4.11+git0+3a5f494784_53e84104c5-r0-qemux86-20160603165040.bin modules--4.4.11+git0+3a5f494784_53e84104c5-r0-qemux86-20160603165040.tgz
+        """
+        files = []
+        with open(image_license_manifest_path) as image_license:
+            for line in image_license:
+                if line.startswith('FILES'):
+                    files_str = line.split(':')[1].strip()
+                    files_str = re.sub(r' {2,}', ' ', files_str)
+                    files = files_str.split(' ')
+        return files
+
+    def _endswith(self, str_to_test, endings):
+        """
+        Returns True if str ends with one of the strings in the list
+        endings, False otherwise
+        """
+        endswith = False
+        for ending in endings:
+            if str_to_test.endswith(ending):
+                endswith = True
+                break
+        return endswith
+
+    def _get_image_files(self, deploy_dir_image, image_name, image_file_extensions):
+        """
+        Find files in deploy_dir_image whose basename starts with the
+        string image_name and ends with one of the strings in
+        image_file_extensions.
+
+        Returns a list of file dictionaries like
+
+        [
+            {
+                'path': '/path/to/image/file',
+                'size': <file size in bytes>
+            }
+        ]
+        """
+        image_files = []
+
+        for dirpath, _, filenames in os.walk(deploy_dir_image):
+            for filename in filenames:
+                if filename.startswith(image_name) and \
+                self._endswith(filename, image_file_extensions):
+                    image_file_path = os.path.join(dirpath, filename)
+                    image_file_size = os.stat(image_file_path).st_size
+
+                    image_files.append({
+                        'path': image_file_path,
+                        'size': image_file_size
+                    })
+
+        return image_files
+
+    def scan_image_artifacts(self):
+        """
+        Scan for built image artifacts in DEPLOY_DIR_IMAGE and associate them
+        with a Target object in self.internal_state['targets'].
+
+        We have two situations to handle:
+
+        1. This is the first time a target + machine has been built, so
+        add files from the DEPLOY_DIR_IMAGE to the target.
+
+        OR
+
+        2. There are no new files for the target (they were already produced by
+        a previous build), so copy them from the most recent previous build with
+        the same target, task and machine.
+        """
+        deploy_dir_image = \
+            self.server.runCommand(['getVariable', 'DEPLOY_DIR_IMAGE'])[0]
+
+        # if there's no DEPLOY_DIR_IMAGE, there aren't going to be
+        # any image artifacts, so we can return immediately
+        if not deploy_dir_image:
+            return
+
+        buildname = self.server.runCommand(['getVariable', 'BUILDNAME'])[0]
+        machine = self.server.runCommand(['getVariable', 'MACHINE'])[0]
+        image_name = self.server.runCommand(['getVariable', 'IMAGE_NAME'])[0]
+
+        # location of the manifest files for this build;
+        # note that this file is only produced if an image is produced
+        license_directory = \
+            self.server.runCommand(['getVariable', 'LICENSE_DIRECTORY'])[0]
+
+        # file name extensions for image files
+        image_file_extensions_unique = {}
+        image_fstypes = self.server.runCommand(
+            ['getVariable', 'IMAGE_FSTYPES'])[0]
+        if image_fstypes != None:
+            image_types_str = image_fstypes.strip()
+            image_file_extensions = re.sub(r' {2,}', ' ', image_types_str)
+            image_file_extensions_unique = set(image_file_extensions.split(' '))
+
+        targets = self.internal_state['targets']
+
+        # filter out anything which isn't an image target
+        image_targets = [target for target in targets if target.is_image]
+
+        for image_target in image_targets:
+            # this is set to True if we find at least one file relating to
+            # this target; if this remains False after the scan, we copy the
+            # files from the most-recent Target with the same target + machine
+            # onto this Target instead
+            has_files = False
+
+            # we construct this because by the time we reach
+            # BuildCompleted, this has reset to
+            # 'defaultpkgname-<MACHINE>-<BUILDNAME>';
+            # we need to change it to
+            # <TARGET>-<MACHINE>-<BUILDNAME>
+            real_image_name = re.sub(r'^defaultpkgname', image_target.target,
+                image_name)
+
+            image_license_manifest_path = os.path.join(
+                license_directory,
+                real_image_name,
+                'image_license.manifest')
+
+            image_package_manifest_path = os.path.join(
+                license_directory,
+                real_image_name,
+                'image_license.manifest')
+
+            # if image_license.manifest exists, we can read the names of bzImage
+            # and modules files for this build from it, then look for them
+            # in the DEPLOY_DIR_IMAGE; note that this file is only produced
+            # if an image file was produced
+            if os.path.isfile(image_license_manifest_path):
+                has_files = True
+
+                basenames = self._get_filenames_from_image_license(
+                    image_license_manifest_path)
+
+                for basename in basenames:
+                    artifact_path = os.path.join(deploy_dir_image, basename)
+                    artifact_size = os.stat(artifact_path).st_size
+
+                    # note that the artifact will only be saved against this
+                    # build if it hasn't been already
+                    self.orm_wrapper.save_target_kernel_file(image_target,
+                        artifact_path, artifact_size)
+
+                # store the license manifest path on the target
+                # (this file is also created any time an image file is created)
+                license_manifest_path = os.path.join(license_directory,
+                    real_image_name, 'license.manifest')
+
+                self.orm_wrapper.update_target_set_license_manifest(
+                    image_target, license_manifest_path)
+
+                # store the package manifest path on the target (this file
+                # is created any time an image file is created)
+                package_manifest_path = os.path.join(deploy_dir_image,
+                    real_image_name + '.rootfs.manifest')
+
+                if os.path.exists(package_manifest_path):
+                    self.orm_wrapper.update_target_set_package_manifest(
+                        image_target, package_manifest_path)
+
+            # scan the directory for image files relating to this build
+            # (via real_image_name); note that we don't have to set
+            # has_files = True, as searching for the license manifest file
+            # will already have set it to true if at least one image file was
+            # produced; note that the real_image_name includes BUILDNAME, which
+            # in turn includes a timestamp; so if no files were produced for
+            # this timestamp (i.e. the build reused existing image files already
+            # in the directory), no files will be recorded against this target
+            image_files = self._get_image_files(deploy_dir_image,
+                real_image_name, image_file_extensions_unique)
+
+            for image_file in image_files:
+                self.orm_wrapper.save_target_image_file_information(
+                    image_target, image_file['path'], image_file['size'])
+
+            if not has_files:
+                # copy image files and build artifacts from the
+                # most-recently-built Target with the
+                # same target + machine as this Target; also copy the license
+                # manifest path, as that is not treated as an artifact and needs
+                # to be set separately
+                similar_target = \
+                    self.orm_wrapper.get_similar_target_with_image_files(
+                        image_target)
+
+                if similar_target:
+                    logger.info('image artifacts for target %s cloned from ' \
+                        'target %s' % (image_target.pk, similar_target.pk))
+                    self.orm_wrapper.clone_image_artifacts(similar_target,
+                        image_target)
+
+    def _get_sdk_targets(self):
+        """
+        Return targets which could generate SDK artifacts, i.e.
+        "do_populate_sdk" and "do_populate_sdk_ext".
+        """
+        return [target for target in self.internal_state['targets'] \
+            if target.task in ['populate_sdk', 'populate_sdk_ext']]
+
+    def scan_sdk_artifacts(self, event):
+        """
+        Note that we have to intercept an SDKArtifactInfo event from
+        toaster.bbclass (via toasterui) to get hold of the SDK variables we
+        need to be able to scan for files accurately: this is because
+        variables like TOOLCHAIN_OUTPUTNAME have reset to None by the time
+        BuildCompleted is fired by bitbake, so we have to get those values
+        while the build is still in progress.
+
+        For populate_sdk_ext, this runs twice, with two different
+        TOOLCHAIN_OUTPUTNAME settings, each of which will capture some of the
+        files in the SDK output directory.
+        """
+        sdk_vars = BuildInfoHelper._get_data_from_event(event)
+        toolchain_outputname = sdk_vars['TOOLCHAIN_OUTPUTNAME']
+
+        # targets which might have created SDK artifacts
+        sdk_targets = self._get_sdk_targets()
+
+        # location of SDK artifacts
+        tmpdir = self.server.runCommand(['getVariable', 'TMPDIR'])[0]
+        sdk_dir = os.path.join(tmpdir, 'deploy', 'sdk')
+
+        # all files in the SDK directory
+        artifacts = []
+        for dir_path, _, filenames in os.walk(sdk_dir):
+            for filename in filenames:
+                full_path = os.path.join(dir_path, filename)
+                if not os.path.islink(full_path):
+                    artifacts.append(full_path)
+
+        for sdk_target in sdk_targets:
+            # find files in the SDK directory which haven't already been
+            # recorded against a Target and whose basename matches
+            # TOOLCHAIN_OUTPUTNAME
+            for artifact_path in artifacts:
+                basename = os.path.basename(artifact_path)
+
+                toolchain_match = basename.startswith(toolchain_outputname)
+
+                # files which match the name of the target which produced them;
+                # for example,
+                # poky-glibc-x86_64-core-image-sato-i586-toolchain-ext-2.1+snapshot.sh
+                target_match = re.search(sdk_target.target, basename)
+
+                # targets which produce "*-nativesdk-*" files
+                is_ext_sdk_target = sdk_target.task in \
+                    ['do_populate_sdk_ext', 'populate_sdk_ext']
+
+                # SDK files which don't match the target name, i.e.
+                # x86_64-nativesdk-libc.*
+                # poky-glibc-x86_64-buildtools-tarball-i586-buildtools-nativesdk-standalone-2.1+snapshot*
+                is_ext_sdk_file = re.search('-nativesdk-', basename)
+
+                file_from_target = (toolchain_match and target_match) or \
+                    (is_ext_sdk_target and is_ext_sdk_file)
+
+                if file_from_target:
+                    # don't record the file if it's already been added to this
+                    # target
+                    matching_files = TargetSDKFile.objects.filter(
+                        target=sdk_target, file_name=artifact_path)
+
+                    if matching_files.count() == 0:
+                        artifact_size = os.stat(artifact_path).st_size
+
+                        self.orm_wrapper.save_target_sdk_file(
+                            sdk_target, artifact_path, artifact_size)
+
+    def clone_required_sdk_artifacts(self):
+        """
+        If an SDK target doesn't have any SDK artifacts, this means that
+        the postfuncs of populate_sdk or populate_sdk_ext didn't fire, which
+        in turn means that the targets of this build didn't generate any new
+        artifacts.
+
+        In this case, clone SDK artifacts for targets in the current build
+        from existing targets for this build.
+        """
+        sdk_targets = self._get_sdk_targets()
+        for sdk_target in sdk_targets:
+            # only clone for SDK targets which have no TargetSDKFiles yet
+            if sdk_target.targetsdkfile_set.all().count() == 0:
+                similar_target = \
+                    self.orm_wrapper.get_similar_target_with_sdk_files(
+                        sdk_target)
+                if similar_target:
+                    logger.info('SDK artifacts for target %s cloned from ' \
+                        'target %s' % (sdk_target.pk, similar_target.pk))
+                    self.orm_wrapper.clone_sdk_artifacts(similar_target,
+                        sdk_target)
+
     def close(self, errorcode):
-        if self.brbe is not None:
-            self._store_build_done(errorcode)
+        self._store_build_done(errorcode)
 
         if 'backlog' in self.internal_state:
-            if 'build' in self.internal_state:
-                # we save missed events in the database for the current build
-                tempevent = self.internal_state['backlog'].pop()
-                self.store_log_event(tempevent)
-            else:
-                # we have no build, and we still have events; something amazingly wrong happend
-                for event in self.internal_state['backlog']:
-                    logger.error("UNSAVED log: %s", event.msg)
+            # we save missed events in the database for the current build
+            tempevent = self.internal_state['backlog'].pop()
+            self.store_log_event(tempevent)
 
         if not connection.features.autocommits_when_autocommit_is_off:
             transaction.set_autocommit(True)
@@ -1527,3 +1916,7 @@ class BuildInfoHelper(object):
         # being incorrectly attached to the previous Toaster-triggered build;
         # see https://bugzilla.yoctoproject.org/show_bug.cgi?id=9021
         self.brbe = None
+
+        # unset the internal Build object to prevent it being reused for the
+        # next build
+        self.internal_state['build'] = None

bitbake/lib/bb/ui/knotty.py

@@ -40,10 +40,13 @@ logger = logging.getLogger("BitBake")
 interactive = sys.stdout.isatty()
 
 class BBProgress(progressbar.ProgressBar):
-    def __init__(self, msg, maxval):
+    def __init__(self, msg, maxval, widgets=None, extrapos=-1):
         self.msg = msg
-        widgets = [progressbar.Percentage(), ' ', progressbar.Bar(), ' ',
-           progressbar.ETA()]
+        self.extrapos = extrapos
+        if not widgets:
+            widgets = [progressbar.Percentage(), ' ', progressbar.Bar(), ' ',
+            progressbar.ETA()]
+            self.extrapos = 4
 
         try:
             self._resize_default = signal.getsignal(signal.SIGWINCH)
@@ -51,23 +54,45 @@ class BBProgress(progressbar.ProgressBar):
             self._resize_default = None
         progressbar.ProgressBar.__init__(self, maxval, [self.msg + ": "] + widgets, fd=sys.stdout)
 
-    def _handle_resize(self, signum, frame):
+    def _handle_resize(self, signum=None, frame=None):
         progressbar.ProgressBar._handle_resize(self, signum, frame)
         if self._resize_default:
             self._resize_default(signum, frame)
+
     def finish(self):
         progressbar.ProgressBar.finish(self)
         if self._resize_default:
             signal.signal(signal.SIGWINCH, self._resize_default)
 
+    def setmessage(self, msg):
+        self.msg = msg
+        self.widgets[0] = msg
+
+    def setextra(self, extra):
+        if self.extrapos > -1:
+            if extra:
+                extrastr = str(extra)
+                if extrastr[0] != ' ':
+                    extrastr = ' ' + extrastr
+                if extrastr[-1] != ' ':
+                    extrastr += ' '
+            else:
+                extrastr = ' '
+            self.widgets[self.extrapos] = extrastr
+
+    def _need_update(self):
+        # We always want the bar to print when update() is called
+        return True
+
 class NonInteractiveProgress(object):
     fobj = sys.stdout
 
     def __init__(self, msg, maxval):
         self.msg = msg
         self.maxval = maxval
+        self.finished = False
 
-    def start(self):
+    def start(self, update=True):
         self.fobj.write("%s..." % self.msg)
         self.fobj.flush()
         return self
@@ -76,8 +101,11 @@ class NonInteractiveProgress(object):
         pass
 
     def finish(self):
+        if self.finished:
+            return
         self.fobj.write("done.\n")
         self.fobj.flush()
+        self.finished = True
 
 def new_progress(msg, maxval):
     if interactive:
@@ -134,7 +162,7 @@ class TerminalFilter(object):
                 cr = (25, 80)
         return cr
 
-    def __init__(self, main, helper, console, errconsole, format):
+    def __init__(self, main, helper, console, errconsole, format, quiet):
         self.main = main
         self.helper = helper
         self.cuu = None
@@ -142,6 +170,7 @@ class TerminalFilter(object):
         self.interactive = sys.stdout.isatty()
         self.footer_present = False
         self.lastpids = []
+        self.quiet = quiet
 
         if not self.interactive:
             return
@@ -181,6 +210,8 @@ class TerminalFilter(object):
         console.addFilter(InteractConsoleLogFilter(self, format))
         errconsole.addFilter(InteractConsoleLogFilter(self, format))
 
+        self.main_progress = None
+
     def clearFooter(self):
         if self.footer_present:
             lines = self.footer_present
@@ -195,28 +226,69 @@ class TerminalFilter(object):
         activetasks = self.helper.running_tasks
         failedtasks = self.helper.failed_tasks
         runningpids = self.helper.running_pids
-        if self.footer_present and (self.lastcount == self.helper.tasknumber_current) and (self.lastpids == runningpids):
+        if self.footer_present and not self.helper.needUpdate:
             return
+        self.helper.needUpdate = False
         if self.footer_present:
             self.clearFooter()
         if (not self.helper.tasknumber_total or self.helper.tasknumber_current == self.helper.tasknumber_total) and not len(activetasks):
             return
         tasks = []
         for t in runningpids:
-            tasks.append("%s (pid %s)" % (activetasks[t]["title"], t))
+            progress = activetasks[t].get("progress", None)
+            if progress is not None:
+                pbar = activetasks[t].get("progressbar", None)
+                rate = activetasks[t].get("rate", None)
+                start_time = activetasks[t].get("starttime", None)
+                if not pbar or pbar.bouncing != (progress < 0):
+                    if progress < 0:
+                        pbar = BBProgress("0: %s (pid %s) " % (activetasks[t]["title"], t), 100, widgets=[progressbar.BouncingSlider(), ''], extrapos=2)
+                        pbar.bouncing = True
+                    else:
+                        pbar = BBProgress("0: %s (pid %s) " % (activetasks[t]["title"], t), 100, widgets=[progressbar.Percentage(), ' ', progressbar.Bar(), ''], extrapos=4)
+                        pbar.bouncing = False
+                    activetasks[t]["progressbar"] = pbar
+                tasks.append((pbar, progress, rate, start_time))
+            else:
+                tasks.append("%s (pid %s)" % (activetasks[t]["title"], t))
 
         if self.main.shutdown:
             content = "Waiting for %s running tasks to finish:" % len(activetasks)
-        elif not len(activetasks):
-            content = "No currently running tasks (%s of %s)" % (self.helper.tasknumber_current, self.helper.tasknumber_total)
-        else:
-            content = "Currently %s running tasks (%s of %s):" % (len(activetasks), self.helper.tasknumber_current, self.helper.tasknumber_total)
-        print(content)
-        lines = 1 + int(len(content) / (self.columns + 1))
-        for tasknum, task in enumerate(tasks[:(self.rows - 2)]):
-            content = "%s: %s" % (tasknum, task)
             print(content)
-            lines = lines + 1 + int(len(content) / (self.columns + 1))
+        else:
+            if not len(activetasks):
+                content = "No currently running tasks (%s of %s)" % (self.helper.tasknumber_current, self.helper.tasknumber_total)
+            else:
+                content = "Currently %2s running tasks (%s of %s)" % (len(activetasks), self.helper.tasknumber_current, self.helper.tasknumber_total)
+            maxtask = self.helper.tasknumber_total + 1
+            if not self.main_progress or self.main_progress.maxval != maxtask:
+                widgets = [' ', progressbar.Percentage(), ' ', progressbar.Bar()]
+                self.main_progress = BBProgress("Running tasks", maxtask, widgets=widgets)
+                self.main_progress.start(False)
+            self.main_progress.setmessage(content)
+            self.main_progress.update(self.helper.tasknumber_current)
+            print('')
+        lines = 1 + int(len(content) / (self.columns + 1))
+        if not self.quiet:
+            for tasknum, task in enumerate(tasks[:(self.rows - 2)]):
+                if isinstance(task, tuple):
+                    pbar, progress, rate, start_time = task
+                    if not pbar.start_time:
+                        pbar.start(False)
+                        if start_time:
+                            pbar.start_time = start_time
+                    pbar.setmessage('%s:%s' % (tasknum, pbar.msg.split(':', 1)[1]))
+                    if progress > -1:
+                        pbar.setextra(rate)
+                        output = pbar.update(progress)
+                    else:
+                        output = pbar.update(1)
+                    if not output or (len(output) <= pbar.term_width):
+                        print('')
+                else:
+                    content = "%s: %s" % (tasknum, task)
+                    print(content)
+                lines = lines + 1 + int(len(content) / (self.columns + 1))
         self.footer_present = lines
         self.lastpids = runningpids[:]
         self.lastcount = self.helper.tasknumber_current
@@ -249,7 +321,8 @@ _evt_list = [ "bb.runqueue.runQueueExitWait", "bb.event.LogExecTTY", "logging.Lo
               "bb.command.CommandExit", "bb.command.CommandCompleted",  "bb.cooker.CookerExit",
               "bb.event.MultipleProviders", "bb.event.NoProvider", "bb.runqueue.sceneQueueTaskStarted",
               "bb.runqueue.runQueueTaskStarted", "bb.runqueue.runQueueTaskFailed", "bb.runqueue.sceneQueueTaskFailed",
-              "bb.event.BuildBase", "bb.build.TaskStarted", "bb.build.TaskSucceeded", "bb.build.TaskFailedSilent"]
+              "bb.event.BuildBase", "bb.build.TaskStarted", "bb.build.TaskSucceeded", "bb.build.TaskFailedSilent",
+              "bb.build.TaskProgress", "bb.event.ProcessStarted", "bb.event.ProcessProgress", "bb.event.ProcessFinished"]
 
 def main(server, eventHandler, params, tf = TerminalFilter):
 
@@ -266,7 +339,10 @@ def main(server, eventHandler, params, tf = TerminalFilter):
     errconsole = logging.StreamHandler(sys.stderr)
     format_str = "%(levelname)s: %(message)s"
     format = bb.msg.BBLogFormatter(format_str)
-    bb.msg.addDefaultlogFilter(console, bb.msg.BBLogFilterStdOut)
+    if params.options.quiet:
+        bb.msg.addDefaultlogFilter(console, bb.msg.BBLogFilterStdOut, bb.msg.BBLogFormatter.WARNING)
+    else:
+        bb.msg.addDefaultlogFilter(console, bb.msg.BBLogFilterStdOut)
     bb.msg.addDefaultlogFilter(errconsole, bb.msg.BBLogFilterStdErr)
     console.setFormatter(format)
     errconsole.setFormatter(format)
@@ -287,6 +363,12 @@ def main(server, eventHandler, params, tf = TerminalFilter):
         bb.msg.addDefaultlogFilter(consolelog)
         consolelog.setFormatter(conlogformat)
         logger.addHandler(consolelog)
+        loglink = os.path.join(os.path.dirname(consolelogfile), 'console-latest.log')
+        bb.utils.remove(loglink)
+        try:
+           os.symlink(os.path.basename(consolelogfile), loglink)
+        except OSError:
+           pass
 
     llevel, debug_domains = bb.msg.constructLogOptions()
     server.runCommand(["setEventMask", server.getEventHandle(), llevel, debug_domains, _evt_list])
@@ -323,7 +405,7 @@ def main(server, eventHandler, params, tf = TerminalFilter):
     warnings = 0
     taskfailures = []
 
-    termfilter = tf(main, helper, console, errconsole, format)
+    termfilter = tf(main, helper, console, errconsole, format, params.options.quiet)
     atexit.register(termfilter.finish)
 
     while True:
@@ -415,15 +497,19 @@ def main(server, eventHandler, params, tf = TerminalFilter):
                 parseprogress = new_progress("Parsing recipes", event.total).start()
                 continue
             if isinstance(event, bb.event.ParseProgress):
-                parseprogress.update(event.current)
+                if parseprogress:
+                    parseprogress.update(event.current)
+                else:
+                    bb.warn("Got ParseProgress event for parsing that never started?")
                 continue
             if isinstance(event, bb.event.ParseCompleted):
                 if not parseprogress:
                     continue
-
                 parseprogress.finish()
-                print(("Parsing of %d .bb files complete (%d cached, %d parsed). %d targets, %d skipped, %d masked, %d errors."
-                    % ( event.total, event.cached, event.parsed, event.virtuals, event.skipped, event.masked, event.errors)))
+                pasreprogress = None
+                if not params.options.quiet:
+                    print(("Parsing of %d .bb files complete (%d cached, %d parsed). %d targets, %d skipped, %d masked, %d errors."
+                        % ( event.total, event.cached, event.parsed, event.virtuals, event.skipped, event.masked, event.errors)))
                 continue
 
             if isinstance(event, bb.event.CacheLoadStarted):
@@ -434,7 +520,8 @@ def main(server, eventHandler, params, tf = TerminalFilter):
                 continue
             if isinstance(event, bb.event.CacheLoadCompleted):
                 cacheprogress.finish()
-                print("Loaded %d entries from dependency cache." % event.num_entries)
+                if not params.options.quiet:
+                    print("Loaded %d entries from dependency cache." % event.num_entries)
                 continue
 
             if isinstance(event, bb.command.CommandFailed):
@@ -518,6 +605,22 @@ def main(server, eventHandler, params, tf = TerminalFilter):
             if isinstance(event, bb.event.DepTreeGenerated):
                 continue
 
+            if isinstance(event, bb.event.ProcessStarted):
+                parseprogress = new_progress(event.processname, event.total)
+                parseprogress.start(False)
+                continue
+            if isinstance(event, bb.event.ProcessProgress):
+                if parseprogress:
+                    parseprogress.update(event.progress)
+                else:
+                    bb.warn("Got ProcessProgress event for someting that never started?")
+                continue
+            if isinstance(event, bb.event.ProcessFinished):
+                if parseprogress:
+                    parseprogress.finish()
+                parseprogress = None
+                continue
+
             # ignore
             if isinstance(event, (bb.event.BuildBase,
                                   bb.event.MetadataEvent,
@@ -529,7 +632,8 @@ def main(server, eventHandler, params, tf = TerminalFilter):
                                   bb.event.OperationStarted,
                                   bb.event.OperationCompleted,
                                   bb.event.OperationProgress,
-                                  bb.event.DiskFull)):
+                                  bb.event.DiskFull,
+                                  bb.build.TaskProgress)):
                 continue
 
             logger.error("Unknown event: %s", event)
@@ -582,7 +686,7 @@ def main(server, eventHandler, params, tf = TerminalFilter):
         if return_value and errors:
             summary += pluralise("\nSummary: There was %s ERROR message shown, returning a non-zero exit code.",
                                  "\nSummary: There were %s ERROR messages shown, returning a non-zero exit code.", errors)
-        if summary:
+        if summary and not params.options.quiet:
             print(summary)
 
         if interrupted:

bitbake/lib/bb/ui/toasterui.py

@@ -102,6 +102,7 @@ _evt_list = [
     "bb.command.CommandExit",
     "bb.command.CommandFailed",
     "bb.cooker.CookerExit",
+    "bb.event.BuildInit",
     "bb.event.BuildCompleted",
     "bb.event.BuildStarted",
     "bb.event.CacheLoadCompleted",
@@ -115,6 +116,7 @@ _evt_list = [
     "bb.event.NoProvider",
     "bb.event.ParseCompleted",
     "bb.event.ParseProgress",
+    "bb.event.ParseStarted",
     "bb.event.RecipeParsed",
     "bb.event.SanityCheck",
     "bb.event.SanityCheckPassed",
@@ -231,19 +233,35 @@ def main(server, eventHandler, params):
             # pylint: disable=protected-access
             # the code will look into the protected variables of the event; no easy way around this
 
-            # we treat ParseStarted as the first event of toaster-triggered
-            # builds; that way we get the Build Configuration included in the log
-            # and any errors that occur before BuildStarted is fired
             if isinstance(event, bb.event.ParseStarted):
                 if not (build_log and build_log_file_path):
                     build_log, build_log_file_path = _open_build_log(log_dir)
+
+                buildinfohelper.store_started_build()
+                buildinfohelper.save_build_log_file_path(build_log_file_path)
+                buildinfohelper.set_recipes_to_parse(event.total)
                 continue
 
-            if isinstance(event, bb.event.BuildStarted):
+            # create a build object in buildinfohelper from either BuildInit
+            # (if available) or BuildStarted (for jethro and previous versions)
+            if isinstance(event, (bb.event.BuildStarted, bb.event.BuildInit)):
                 if not (build_log and build_log_file_path):
                     build_log, build_log_file_path = _open_build_log(log_dir)
 
-                buildinfohelper.store_started_build(event, build_log_file_path)
+                buildinfohelper.save_build_targets(event)
+                buildinfohelper.save_build_log_file_path(build_log_file_path)
+
+                # get additional data from BuildStarted
+                if isinstance(event, bb.event.BuildStarted):
+                    buildinfohelper.save_build_layers_and_variables()
+                continue
+
+            if isinstance(event, bb.event.ParseProgress):
+                buildinfohelper.set_recipes_parsed(event.current)
+                continue
+
+            if isinstance(event, bb.event.ParseCompleted):
+                buildinfohelper.set_recipes_parsed(event.total)
                 continue
 
             if isinstance(event, (bb.build.TaskStarted, bb.build.TaskSucceeded, bb.build.TaskFailedSilent)):
@@ -289,10 +307,6 @@ def main(server, eventHandler, params):
             # timing and error informations from the parsing phase in Toaster
             if isinstance(event, (bb.event.SanityCheckPassed, bb.event.SanityCheck)):
                 continue
-            if isinstance(event, bb.event.ParseProgress):
-                continue
-            if isinstance(event, bb.event.ParseCompleted):
-                continue
             if isinstance(event, bb.event.CacheLoadStarted):
                 continue
             if isinstance(event, bb.event.CacheLoadProgress):
@@ -363,6 +377,9 @@ def main(server, eventHandler, params):
                     errors += 1
                     errorcode = 1
                     logger.error("Command execution failed: %s", event.error)
+                elif isinstance(event, bb.event.BuildCompleted):
+                    buildinfohelper.scan_image_artifacts()
+                    buildinfohelper.clone_required_sdk_artifacts()
 
                 # turn off logging to the current build log
                 _close_build_log(build_log)
@@ -410,12 +427,8 @@ def main(server, eventHandler, params):
                     buildinfohelper.store_target_package_data(event)
                 elif event.type == "MissedSstate":
                     buildinfohelper.store_missed_state_tasks(event)
-                elif event.type == "ImageFileSize":
-                    buildinfohelper.update_target_image_file(event)
-                elif event.type == "ArtifactFileSize":
-                    buildinfohelper.update_artifact_image_file(event)
-                elif event.type == "LicenseManifestPath":
-                    buildinfohelper.store_license_manifest_path(event)
+                elif event.type == "SDKArtifactInfo":
+                    buildinfohelper.scan_sdk_artifacts(event)
                 elif event.type == "SetBRBE":
                     buildinfohelper.brbe = buildinfohelper._get_data_from_event(event)
                 elif event.type == "OSErrorException":

bitbake/lib/bb/ui/uihelper.py

@@ -18,6 +18,7 @@
 # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
 
 import bb.build
+import time
 
 class BBUIHelper:
     def __init__(self):
@@ -31,7 +32,7 @@ class BBUIHelper:
 
     def eventHandler(self, event):
         if isinstance(event, bb.build.TaskStarted):
-            self.running_tasks[event.pid] = { 'title' : "%s %s" % (event._package, event._task) }
+            self.running_tasks[event.pid] = { 'title' : "%s %s" % (event._package, event._task), 'starttime' : time.time() }
             self.running_pids.append(event.pid)
             self.needUpdate = True
         if isinstance(event, bb.build.TaskSucceeded):
@@ -52,6 +53,11 @@ class BBUIHelper:
             self.tasknumber_current = event.stats.completed + event.stats.active + event.stats.failed + 1
             self.tasknumber_total = event.stats.total
             self.needUpdate = True
+        if isinstance(event, bb.build.TaskProgress):
+            if event.pid > 0:
+                self.running_tasks[event.pid]['progress'] = event.progress
+                self.running_tasks[event.pid]['rate'] = event.rate
+                self.needUpdate = True
 
     def getTasks(self):
         self.needUpdate = False

bitbake/lib/bb/utils.py

@@ -250,6 +250,7 @@ def explode_dep_versions2(s):
         if not (i in r and r[i]):
             r[lastdep] = []
 
+    r = collections.OrderedDict(sorted(r.items(), key=lambda x: x[0]))
     return r
 
 def explode_dep_versions(s):
@@ -374,6 +375,12 @@ def _print_exception(t, value, tb, realfile, text, context):
             level = level + 1
 
         error.append("Exception: %s" % ''.join(exception))
+
+        # If the exception is from spwaning a task, let's be helpful and display
+        # the output (which hopefully includes stderr).
+        if isinstance(value, subprocess.CalledProcessError):
+            error.append("Subprocess output:")
+            error.append(value.output.decode("utf-8", errors="ignore"))
     finally:
         logger.error("\n".join(error))
 
@@ -574,6 +581,7 @@ def preserved_envvars_exported():
         'TERM',
         'USER',
         'LC_ALL',
+        'BBSERVER',
     ]
 
 def preserved_envvars():
@@ -598,7 +606,6 @@ def filter_environment(good_vars):
             continue
 
         removed_vars[key] = os.environ[key]
-        os.unsetenv(key)
         del os.environ[key]
 
     # If we spawn a python process, we need to have a UTF-8 locale, else python's file
@@ -1444,9 +1451,8 @@ def set_process_name(name):
     # This is nice to have for debugging, not essential
     try:
         libc = cdll.LoadLibrary('libc.so.6')
-        buff = create_string_buffer(len(name)+1)
-        buff.value = name
-        libc.prctl(15, byref(buff), 0, 0, 0)
+        buf = create_string_buffer(bytes(name, 'utf-8'))
+        libc.prctl(15, byref(buf), 0, 0, 0)
     except:
         pass
 

bitbake/lib/progressbar.py

@@ -1,384 +0,0 @@
-#!/usr/bin/env python
-# -*- coding: iso-8859-1 -*-
-#
-# progressbar  - Text progressbar library for python.
-# Copyright (c) 2005 Nilton Volpato
-#
-# This library is free software; you can redistribute it and/or
-# modify it under the terms of the GNU Lesser General Public
-# License as published by the Free Software Foundation; either
-# version 2.1 of the License, or (at your option) any later version.
-#
-# This library is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-# Lesser General Public License for more details.
-#
-# You should have received a copy of the GNU Lesser General Public
-# License along with this library; if not, write to the Free Software
-# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
-
-
-"""Text progressbar library for python.
-
-This library provides a text mode progressbar. This is typically used
-to display the progress of a long running operation, providing a
-visual clue that processing is underway.
-
-The ProgressBar class manages the progress, and the format of the line
-is given by a number of widgets. A widget is an object that may
-display diferently depending on the state of the progress. There are
-three types of widget:
-- a string, which always shows itself;
-- a ProgressBarWidget, which may return a diferent value every time
-it's update method is called; and
-- a ProgressBarWidgetHFill, which is like ProgressBarWidget, except it
-expands to fill the remaining width of the line.
-
-The progressbar module is very easy to use, yet very powerful. And
-automatically supports features like auto-resizing when available.
-"""
-
-from __future__ import division
-
-__author__ = "Nilton Volpato"
-__author_email__ = "first-name dot last-name @ gmail.com"
-__date__ = "2006-05-07"
-__version__ = "2.3-dev"
-
-import sys, time, os
-from array import array
-try:
-    from fcntl import ioctl
-    import termios
-except ImportError:
-    pass
-import signal
-try:
-    basestring
-except NameError:
-    basestring = (str,)
-
-class ProgressBarWidget(object):
-    """This is an element of ProgressBar formatting.
-
-    The ProgressBar object will call it's update value when an update
-    is needed. It's size may change between call, but the results will
-    not be good if the size changes drastically and repeatedly.
-    """
-    def update(self, pbar):
-        """Returns the string representing the widget.
-
-        The parameter pbar is a reference to the calling ProgressBar,
-        where one can access attributes of the class for knowing how
-        the update must be made.
-
-        At least this function must be overriden."""
-        pass
-
-class ProgressBarWidgetHFill(object):
-    """This is a variable width element of ProgressBar formatting.
-
-    The ProgressBar object will call it's update value, informing the
-    width this object must the made. This is like TeX \\hfill, it will
-    expand to fill the line. You can use more than one in the same
-    line, and they will all have the same width, and together will
-    fill the line.
-    """
-    def update(self, pbar, width):
-        """Returns the string representing the widget.
-
-        The parameter pbar is a reference to the calling ProgressBar,
-        where one can access attributes of the class for knowing how
-        the update must be made. The parameter width is the total
-        horizontal width the widget must have.
-
-        At least this function must be overriden."""
-        pass
-
-
-class ETA(ProgressBarWidget):
-    "Widget for the Estimated Time of Arrival"
-    def format_time(self, seconds):
-        return time.strftime('%H:%M:%S', time.gmtime(seconds))
-    def update(self, pbar):
-        if pbar.currval == 0:
-            return 'ETA:  --:--:--'
-        elif pbar.finished:
-            return 'Time: %s' % self.format_time(pbar.seconds_elapsed)
-        else:
-            elapsed = pbar.seconds_elapsed
-            eta = elapsed * pbar.maxval / pbar.currval - elapsed
-            return 'ETA:  %s' % self.format_time(eta)
-
-class FileTransferSpeed(ProgressBarWidget):
-    "Widget for showing the transfer speed (useful for file transfers)."
-    def __init__(self, unit='B'):
-        self.unit = unit
-        self.fmt = '%6.2f %s'
-        self.prefixes = ['', 'K', 'M', 'G', 'T', 'P']
-    def update(self, pbar):
-        if pbar.seconds_elapsed < 2e-6:#== 0:
-            bps = 0.0
-        else:
-            bps = pbar.currval / pbar.seconds_elapsed
-        spd = bps
-        for u in self.prefixes:
-            if spd < 1000:
-                break
-            spd /= 1000
-        return self.fmt % (spd, u + self.unit + '/s')
-
-class RotatingMarker(ProgressBarWidget):
-    "A rotating marker for filling the bar of progress."
-    def __init__(self, markers='|/-\\'):
-        self.markers = markers
-        self.curmark = -1
-    def update(self, pbar):
-        if pbar.finished:
-            return self.markers[0]
-        self.curmark = (self.curmark + 1) % len(self.markers)
-        return self.markers[self.curmark]
-
-class Percentage(ProgressBarWidget):
-    "Just the percentage done."
-    def update(self, pbar):
-        return '%3d%%' % pbar.percentage()
-
-class SimpleProgress(ProgressBarWidget):
-    "Returns what is already done and the total, e.g.: '5 of 47'"
-    def __init__(self, sep=' of '):
-        self.sep = sep
-    def update(self, pbar):
-        return '%d%s%d' % (pbar.currval, self.sep, pbar.maxval)
-
-class Bar(ProgressBarWidgetHFill):
-    "The bar of progress. It will stretch to fill the line."
-    def __init__(self, marker='#', left='|', right='|'):
-        self.marker = marker
-        self.left = left
-        self.right = right
-    def _format_marker(self, pbar):
-        if isinstance(self.marker, basestring):
-            return self.marker
-        else:
-            return self.marker.update(pbar)
-    def update(self, pbar, width):
-        percent = pbar.percentage()
-        cwidth = width - len(self.left) - len(self.right)
-        marked_width = int(percent * cwidth // 100)
-        m = self._format_marker(pbar)
-        bar = (self.left + (m * marked_width).ljust(cwidth) + self.right)
-        return bar
-
-class ReverseBar(Bar):
-    "The reverse bar of progress, or bar of regress. :)"
-    def update(self, pbar, width):
-        percent = pbar.percentage()
-        cwidth = width - len(self.left) - len(self.right)
-        marked_width = int(percent * cwidth // 100)
-        m = self._format_marker(pbar)
-        bar = (self.left + (m*marked_width).rjust(cwidth) + self.right)
-        return bar
-
-default_widgets = [Percentage(), ' ', Bar()]
-class ProgressBar(object):
-    """This is the ProgressBar class, it updates and prints the bar.
-
-    A common way of using it is like:
-    >>> pbar = ProgressBar().start()
-    >>> for i in xrange(100):
-    ...    # do something
-    ...    pbar.update(i+1)
-    ...
-    >>> pbar.finish()
-
-    You can also use a progressbar as an iterator:
-    >>> progress = ProgressBar()
-    >>> for i in progress(some_iterable):
-    ...    # do something
-    ...
-
-    But anything you want to do is possible (well, almost anything).
-    You can supply different widgets of any type in any order. And you
-    can even write your own widgets! There are many widgets already
-    shipped and you should experiment with them.
-
-    The term_width parameter must be an integer or None. In the latter case
-    it will try to guess it, if it fails it will default to 80 columns.
-
-    When implementing a widget update method you may access any
-    attribute or function of the ProgressBar object calling the
-    widget's update method. The most important attributes you would
-    like to access are:
-    - currval: current value of the progress, 0 <= currval <= maxval
-    - maxval: maximum (and final) value of the progress
-    - finished: True if the bar has finished (reached 100%), False o/w
-    - start_time: the time when start() method of ProgressBar was called
-    - seconds_elapsed: seconds elapsed since start_time
-    - percentage(): percentage of the progress [0..100]. This is a method.
-
-    The attributes above are unlikely to change between different versions,
-    the other ones may change or cease to exist without notice, so try to rely
-    only on the ones documented above if you are extending the progress bar.
-    """
-
-    __slots__ = ('currval', 'fd', 'finished', 'last_update_time', 'maxval',
-                 'next_update', 'num_intervals', 'seconds_elapsed',
-                 'signal_set', 'start_time', 'term_width', 'update_interval',
-                 'widgets', '_iterable')
-
-    _DEFAULT_MAXVAL = 100
-
-    def __init__(self, maxval=None, widgets=default_widgets, term_width=None,
-                 fd=sys.stderr):
-        self.maxval = maxval
-        self.widgets = widgets
-        self.fd = fd
-        self.signal_set = False
-        if term_width is not None:
-            self.term_width = term_width
-        else:
-            try:
-                self._handle_resize(None, None)
-                signal.signal(signal.SIGWINCH, self._handle_resize)
-                self.signal_set = True
-            except (SystemExit, KeyboardInterrupt):
-                raise
-            except:
-                self.term_width = int(os.environ.get('COLUMNS', 80)) - 1
-
-        self.currval = 0
-        self.finished = False
-        self.start_time = None
-        self.last_update_time = None
-        self.seconds_elapsed = 0
-        self._iterable = None
-
-    def __call__(self, iterable):
-        try:
-            self.maxval = len(iterable)
-        except TypeError:
-            # If the iterable has no length, then rely on the value provided
-            # by the user, otherwise fail.
-            if not (isinstance(self.maxval, (int, long)) and self.maxval > 0):
-                raise RuntimeError('Could not determine maxval from iterable. '
-                                   'You must explicitly provide a maxval.')
-        self._iterable = iter(iterable)
-        self.start()
-        return self
-
-    def __iter__(self):
-        return self
-
-    def next(self):
-        try:
-            next = self._iterable.next()
-            self.update(self.currval + 1)
-            return next
-        except StopIteration:
-            self.finish()
-            raise
-
-    def _handle_resize(self, signum, frame):
-        h, w = array('h', ioctl(self.fd, termios.TIOCGWINSZ, '\0' * 8))[:2]
-        self.term_width = w
-
-    def percentage(self):
-        "Returns the percentage of the progress."
-        return self.currval * 100.0 / self.maxval
-
-    def _format_widgets(self):
-        r = []
-        hfill_inds = []
-        num_hfill = 0
-        currwidth = 0
-        for i, w in enumerate(self.widgets):
-            if isinstance(w, ProgressBarWidgetHFill):
-                r.append(w)
-                hfill_inds.append(i)
-                num_hfill += 1
-            elif isinstance(w, basestring):
-                r.append(w)
-                currwidth += len(w)
-            else:
-                weval = w.update(self)
-                currwidth += len(weval)
-                r.append(weval)
-        for iw in hfill_inds:
-            widget_width = int((self.term_width - currwidth) // num_hfill)
-            r[iw] = r[iw].update(self, widget_width)
-        return r
-
-    def _format_line(self):
-        return ''.join(self._format_widgets()).ljust(self.term_width)
-
-    def _next_update(self):
-        return int((int(self.num_intervals *
-                        (self.currval / self.maxval)) + 1) *
-                   self.update_interval)
-
-    def _need_update(self):
-        """Returns true when the progressbar should print an updated line.
-
-        You can override this method if you want finer grained control over
-        updates.
-
-        The current implementation is optimized to be as fast as possible and
-        as economical as possible in the number of updates. However, depending
-        on your usage you may want to do more updates. For instance, if your
-        progressbar stays in the same percentage for a long time, and you want
-        to update other widgets, like ETA, then you could return True after
-        some time has passed with no updates.
-
-        Ideally you could call self._format_line() and see if it's different
-        from the previous _format_line() call, but calling _format_line() takes
-        around 20 times more time than calling this implementation of
-        _need_update().
-        """
-        return self.currval >= self.next_update
-
-    def update(self, value):
-        "Updates the progress bar to a new value."
-        assert 0 <= value <= self.maxval, '0 <= %d <= %d' % (value, self.maxval)
-        self.currval = value
-        if not self._need_update():
-            return
-        if self.start_time is None:
-            raise RuntimeError('You must call start() before calling update()')
-        now = time.time()
-        self.seconds_elapsed = now - self.start_time
-        self.next_update = self._next_update()
-        self.fd.write(self._format_line() + '\r')
-        self.last_update_time = now
-
-    def start(self):
-        """Starts measuring time, and prints the bar at 0%.
-
-        It returns self so you can use it like this:
-        >>> pbar = ProgressBar().start()
-        >>> for i in xrange(100):
-        ...    # do something
-        ...    pbar.update(i+1)
-        ...
-        >>> pbar.finish()
-        """
-        if self.maxval is None:
-            self.maxval = self._DEFAULT_MAXVAL
-        assert self.maxval > 0
-
-        self.num_intervals = max(100, self.term_width)
-        self.update_interval = self.maxval / self.num_intervals
-        self.next_update = 0
-
-        self.start_time = self.last_update_time = time.time()
-        self.update(0)
-        return self
-
-    def finish(self):
-        """Used to tell the progress is finished."""
-        self.finished = True
-        self.update(self.maxval)
-        self.fd.write('\n')
-        if self.signal_set:
-            signal.signal(signal.SIGWINCH, signal.SIG_DFL)

bitbake/lib/progressbar/LICENSE.txt

@@ -0,0 +1,52 @@
+You can redistribute and/or modify this library under the terms of the
+GNU LGPL license or BSD license (or both).
+
+---
+
+progressbar - Text progress bar library for python.
+Copyright (C) 2005 Nilton Volpato
+
+This library is free software; you can redistribute it and/or
+modify it under the terms of the GNU Lesser General Public
+License as published by the Free Software Foundation; either
+version 2.1 of the License, or (at your option) any later version.
+
+This library is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+Lesser General Public License for more details.
+
+You should have received a copy of the GNU Lesser General Public
+License along with this library; if not, write to the Free Software
+Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
+
+---
+
+progressbar - Text progress bar library for python
+Copyright (c) 2008 Nilton Volpato
+
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are met:
+
+ a. Redistributions of source code must retain the above copyright notice,
+    this list of conditions and the following disclaimer.
+ b. Redistributions in binary form must reproduce the above copyright
+    notice, this list of conditions and the following disclaimer in the
+    documentation and/or other materials provided with the distribution.
+ c. Neither the name of the author nor the names of its contributors
+    may be used to endorse or promote products derived from this software
+    without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR
+ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+DAMAGE.

bitbake/lib/progressbar/__init__.py

@@ -0,0 +1,49 @@
+#!/usr/bin/python
+# -*- coding: utf-8 -*-
+#
+# progressbar  - Text progress bar library for Python.
+# Copyright (c) 2005 Nilton Volpato
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation; either
+# version 2.1 of the License, or (at your option) any later version.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
+
+"""Text progress bar library for Python.
+
+A text progress bar is typically used to display the progress of a long
+running operation, providing a visual cue that processing is underway.
+
+The ProgressBar class manages the current progress, and the format of the line
+is given by a number of widgets. A widget is an object that may display
+differently depending on the state of the progress bar. There are three types
+of widgets:
+ - a string, which always shows itself
+
+ - a ProgressBarWidget, which may return a different value every time its
+   update method is called
+
+ - a ProgressBarWidgetHFill, which is like ProgressBarWidget, except it
+   expands to fill the remaining width of the line.
+
+The progressbar module is very easy to use, yet very powerful. It will also
+automatically enable features like auto-resizing when the system supports it.
+"""
+
+__author__ = 'Nilton Volpato'
+__author_email__ = 'first-name dot last-name @ gmail.com'
+__date__ = '2011-05-14'
+__version__ = '2.3'
+
+from .compat import *
+from .widgets import *
+from .progressbar import *

bitbake/lib/progressbar/compat.py

@@ -0,0 +1,44 @@
+# -*- coding: utf-8 -*-
+#
+# progressbar  - Text progress bar library for Python.
+# Copyright (c) 2005 Nilton Volpato
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation; either
+# version 2.1 of the License, or (at your option) any later version.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
+
+"""Compatibility methods and classes for the progressbar module."""
+
+
+# Python 3.x (and backports) use a modified iterator syntax
+# This will allow 2.x to behave with 3.x iterators
+try:
+  next
+except NameError:
+    def next(iter):
+        try:
+            # Try new style iterators
+            return iter.__next__()
+        except AttributeError:
+            # Fallback in case of a "native" iterator
+            return iter.next()
+
+
+# Python < 2.5 does not have "any"
+try:
+  any
+except NameError:
+   def any(iterator):
+      for item in iterator:
+         if item: return True
+      return False

bitbake/lib/progressbar/progressbar.py

@@ -0,0 +1,315 @@
+# -*- coding: utf-8 -*-
+#
+# progressbar  - Text progress bar library for Python.
+# Copyright (c) 2005 Nilton Volpato
+#
+# (With some small changes after importing into BitBake)
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation; either
+# version 2.1 of the License, or (at your option) any later version.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
+
+"""Main ProgressBar class."""
+
+from __future__ import division
+
+import math
+import os
+import signal
+import sys
+import time
+
+try:
+    from fcntl import ioctl
+    from array import array
+    import termios
+except ImportError:
+    pass
+
+from .compat import *  # for: any, next
+from . import widgets
+
+
+class UnknownLength: pass
+
+
+class ProgressBar(object):
+    """The ProgressBar class which updates and prints the bar.
+
+    A common way of using it is like:
+    >>> pbar = ProgressBar().start()
+    >>> for i in range(100):
+    ...    # do something
+    ...    pbar.update(i+1)
+    ...
+    >>> pbar.finish()
+
+    You can also use a ProgressBar as an iterator:
+    >>> progress = ProgressBar()
+    >>> for i in progress(some_iterable):
+    ...    # do something
+    ...
+
+    Since the progress bar is incredibly customizable you can specify
+    different widgets of any type in any order. You can even write your own
+    widgets! However, since there are already a good number of widgets you
+    should probably play around with them before moving on to create your own
+    widgets.
+
+    The term_width parameter represents the current terminal width. If the
+    parameter is set to an integer then the progress bar will use that,
+    otherwise it will attempt to determine the terminal width falling back to
+    80 columns if the width cannot be determined.
+
+    When implementing a widget's update method you are passed a reference to
+    the current progress bar. As a result, you have access to the
+    ProgressBar's methods and attributes. Although there is nothing preventing
+    you from changing the ProgressBar you should treat it as read only.
+
+    Useful methods and attributes include (Public API):
+     - currval: current progress (0 <= currval <= maxval)
+     - maxval: maximum (and final) value
+     - finished: True if the bar has finished (reached 100%)
+     - start_time: the time when start() method of ProgressBar was called
+     - seconds_elapsed: seconds elapsed since start_time and last call to
+                        update
+     - percentage(): progress in percent [0..100]
+    """
+
+    __slots__ = ('currval', 'fd', 'finished', 'last_update_time',
+                 'left_justify', 'maxval', 'next_update', 'num_intervals',
+                 'poll', 'seconds_elapsed', 'signal_set', 'start_time',
+                 'term_width', 'update_interval', 'widgets', '_time_sensitive',
+                 '__iterable')
+
+    _DEFAULT_MAXVAL = 100
+    _DEFAULT_TERMSIZE = 80
+    _DEFAULT_WIDGETS = [widgets.Percentage(), ' ', widgets.Bar()]
+
+    def __init__(self, maxval=None, widgets=None, term_width=None, poll=1,
+                 left_justify=True, fd=sys.stderr):
+        """Initializes a progress bar with sane defaults."""
+
+        # Don't share a reference with any other progress bars
+        if widgets is None:
+            widgets = list(self._DEFAULT_WIDGETS)
+
+        self.maxval = maxval
+        self.widgets = widgets
+        self.fd = fd
+        self.left_justify = left_justify
+
+        self.signal_set = False
+        if term_width is not None:
+            self.term_width = term_width
+        else:
+            try:
+                self._handle_resize(None, None)
+                signal.signal(signal.SIGWINCH, self._handle_resize)
+                self.signal_set = True
+            except (SystemExit, KeyboardInterrupt): raise
+            except Exception as e:
+                print("DEBUG 5 %s" % e)
+                self.term_width = self._env_size()
+
+        self.__iterable = None
+        self._update_widgets()
+        self.currval = 0
+        self.finished = False
+        self.last_update_time = None
+        self.poll = poll
+        self.seconds_elapsed = 0
+        self.start_time = None
+        self.update_interval = 1
+        self.next_update = 0
+
+
+    def __call__(self, iterable):
+        """Use a ProgressBar to iterate through an iterable."""
+
+        try:
+            self.maxval = len(iterable)
+        except:
+            if self.maxval is None:
+                self.maxval = UnknownLength
+
+        self.__iterable = iter(iterable)
+        return self
+
+
+    def __iter__(self):
+        return self
+
+
+    def __next__(self):
+        try:
+            value = next(self.__iterable)
+            if self.start_time is None:
+                self.start()
+            else:
+                self.update(self.currval + 1)
+            return value
+        except StopIteration:
+            if self.start_time is None:
+                self.start()
+            self.finish()
+            raise
+
+
+    # Create an alias so that Python 2.x won't complain about not being
+    # an iterator.
+    next = __next__
+
+
+    def _env_size(self):
+        """Tries to find the term_width from the environment."""
+
+        return int(os.environ.get('COLUMNS', self._DEFAULT_TERMSIZE)) - 1
+
+
+    def _handle_resize(self, signum=None, frame=None):
+        """Tries to catch resize signals sent from the terminal."""
+
+        h, w = array('h', ioctl(self.fd, termios.TIOCGWINSZ, '\0' * 8))[:2]
+        self.term_width = w
+
+
+    def percentage(self):
+        """Returns the progress as a percentage."""
+        if self.currval >= self.maxval:
+            return 100.0
+        return (self.currval * 100.0 / self.maxval) if self.maxval else 100.00
+
+    percent = property(percentage)
+
+
+    def _format_widgets(self):
+        result = []
+        expanding = []
+        width = self.term_width
+
+        for index, widget in enumerate(self.widgets):
+            if isinstance(widget, widgets.WidgetHFill):
+                result.append(widget)
+                expanding.insert(0, index)
+            else:
+                widget = widgets.format_updatable(widget, self)
+                result.append(widget)
+                width -= len(widget)
+
+        count = len(expanding)
+        while count:
+            portion = max(int(math.ceil(width * 1. / count)), 0)
+            index = expanding.pop()
+            count -= 1
+
+            widget = result[index].update(self, portion)
+            width -= len(widget)
+            result[index] = widget
+
+        return result
+
+
+    def _format_line(self):
+        """Joins the widgets and justifies the line."""
+
+        widgets = ''.join(self._format_widgets())
+
+        if self.left_justify: return widgets.ljust(self.term_width)
+        else: return widgets.rjust(self.term_width)
+
+
+    def _need_update(self):
+        """Returns whether the ProgressBar should redraw the line."""
+        if self.currval >= self.next_update or self.finished: return True
+
+        delta = time.time() - self.last_update_time
+        return self._time_sensitive and delta > self.poll
+
+
+    def _update_widgets(self):
+        """Checks all widgets for the time sensitive bit."""
+
+        self._time_sensitive = any(getattr(w, 'TIME_SENSITIVE', False)
+                                    for w in self.widgets)
+
+
+    def update(self, value=None):
+        """Updates the ProgressBar to a new value."""
+
+        if value is not None and value is not UnknownLength:
+            if (self.maxval is not UnknownLength
+                and not 0 <= value <= self.maxval):
+
+                raise ValueError('Value out of range')
+
+            self.currval = value
+
+
+        if not self._need_update(): return
+        if self.start_time is None:
+            raise RuntimeError('You must call "start" before calling "update"')
+
+        now = time.time()
+        self.seconds_elapsed = now - self.start_time
+        self.next_update = self.currval + self.update_interval
+        output = self._format_line()
+        self.fd.write(output + '\r')
+        self.fd.flush()
+        self.last_update_time = now
+        return output
+
+
+    def start(self, update=True):
+        """Starts measuring time, and prints the bar at 0%.
+
+        It returns self so you can use it like this:
+        >>> pbar = ProgressBar().start()
+        >>> for i in range(100):
+        ...    # do something
+        ...    pbar.update(i+1)
+        ...
+        >>> pbar.finish()
+        """
+
+        if self.maxval is None:
+            self.maxval = self._DEFAULT_MAXVAL
+
+        self.num_intervals = max(100, self.term_width)
+        self.next_update = 0
+
+        if self.maxval is not UnknownLength:
+            if self.maxval < 0: raise ValueError('Value out of range')
+            self.update_interval = self.maxval / self.num_intervals
+
+
+        self.start_time = time.time()
+        if update:
+            self.last_update_time = self.start_time
+            self.update(0)
+        else:
+            self.last_update_time = 0
+
+        return self
+
+
+    def finish(self):
+        """Puts the ProgressBar bar in the finished state."""
+
+        if self.finished:
+            return
+        self.finished = True
+        self.update(self.maxval)
+        self.fd.write('\n')
+        if self.signal_set:
+            signal.signal(signal.SIGWINCH, signal.SIG_DFL)

bitbake/lib/progressbar/widgets.py

@@ -0,0 +1,391 @@
+# -*- coding: utf-8 -*-
+#
+# progressbar  - Text progress bar library for Python.
+# Copyright (c) 2005 Nilton Volpato
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation; either
+# version 2.1 of the License, or (at your option) any later version.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
+
+"""Default ProgressBar widgets."""
+
+from __future__ import division
+
+import datetime
+import math
+
+try:
+    from abc import ABCMeta, abstractmethod
+except ImportError:
+    AbstractWidget = object
+    abstractmethod = lambda fn: fn
+else:
+    AbstractWidget = ABCMeta('AbstractWidget', (object,), {})
+
+
+def format_updatable(updatable, pbar):
+    if hasattr(updatable, 'update'): return updatable.update(pbar)
+    else: return updatable
+
+
+class Widget(AbstractWidget):
+    """The base class for all widgets.
+
+    The ProgressBar will call the widget's update value when the widget should
+    be updated. The widget's size may change between calls, but the widget may
+    display incorrectly if the size changes drastically and repeatedly.
+
+    The boolean TIME_SENSITIVE informs the ProgressBar that it should be
+    updated more often because it is time sensitive.
+    """
+
+    TIME_SENSITIVE = False
+    __slots__ = ()
+
+    @abstractmethod
+    def update(self, pbar):
+        """Updates the widget.
+
+        pbar - a reference to the calling ProgressBar
+        """
+
+
+class WidgetHFill(Widget):
+    """The base class for all variable width widgets.
+
+    This widget is much like the \\hfill command in TeX, it will expand to
+    fill the line. You can use more than one in the same line, and they will
+    all have the same width, and together will fill the line.
+    """
+
+    @abstractmethod
+    def update(self, pbar, width):
+        """Updates the widget providing the total width the widget must fill.
+
+        pbar - a reference to the calling ProgressBar
+        width - The total width the widget must fill
+        """
+
+
+class Timer(Widget):
+    """Widget which displays the elapsed seconds."""
+
+    __slots__ = ('format_string',)
+    TIME_SENSITIVE = True
+
+    def __init__(self, format='Elapsed Time: %s'):
+        self.format_string = format
+
+    @staticmethod
+    def format_time(seconds):
+        """Formats time as the string "HH:MM:SS"."""
+
+        return str(datetime.timedelta(seconds=int(seconds)))
+
+
+    def update(self, pbar):
+        """Updates the widget to show the elapsed time."""
+
+        return self.format_string % self.format_time(pbar.seconds_elapsed)
+
+
+class ETA(Timer):
+    """Widget which attempts to estimate the time of arrival."""
+
+    TIME_SENSITIVE = True
+
+    def update(self, pbar):
+        """Updates the widget to show the ETA or total time when finished."""
+
+        if pbar.currval == 0:
+            return 'ETA:  --:--:--'
+        elif pbar.finished:
+            return 'Time: %s' % self.format_time(pbar.seconds_elapsed)
+        else:
+            elapsed = pbar.seconds_elapsed
+            eta = elapsed * pbar.maxval / pbar.currval - elapsed
+            return 'ETA:  %s' % self.format_time(eta)
+
+
+class AdaptiveETA(Timer):
+    """Widget which attempts to estimate the time of arrival.
+
+    Uses a weighted average of two estimates:
+      1) ETA based on the total progress and time elapsed so far
+      2) ETA based on the progress as per the last 10 update reports
+
+    The weight depends on the current progress so that to begin with the
+    total progress is used and at the end only the most recent progress is
+    used.
+    """
+
+    TIME_SENSITIVE = True
+    NUM_SAMPLES = 10
+
+    def _update_samples(self, currval, elapsed):
+        sample = (currval, elapsed)
+        if not hasattr(self, 'samples'):
+            self.samples = [sample] * (self.NUM_SAMPLES + 1)
+        else:
+            self.samples.append(sample)
+        return self.samples.pop(0)
+
+    def _eta(self, maxval, currval, elapsed):
+        return elapsed * maxval / float(currval) - elapsed
+
+    def update(self, pbar):
+        """Updates the widget to show the ETA or total time when finished."""
+        if pbar.currval == 0:
+            return 'ETA:  --:--:--'
+        elif pbar.finished:
+            return 'Time: %s' % self.format_time(pbar.seconds_elapsed)
+        else:
+            elapsed = pbar.seconds_elapsed
+            currval1, elapsed1 = self._update_samples(pbar.currval, elapsed)
+            eta = self._eta(pbar.maxval, pbar.currval, elapsed)
+            if pbar.currval > currval1:
+                etasamp = self._eta(pbar.maxval - currval1,
+                                    pbar.currval - currval1,
+                                    elapsed - elapsed1)
+                weight = (pbar.currval / float(pbar.maxval)) ** 0.5
+                eta = (1 - weight) * eta + weight * etasamp
+            return 'ETA:  %s' % self.format_time(eta)
+
+
+class FileTransferSpeed(Widget):
+    """Widget for showing the transfer speed (useful for file transfers)."""
+
+    FORMAT = '%6.2f %s%s/s'
+    PREFIXES = ' kMGTPEZY'
+    __slots__ = ('unit',)
+
+    def __init__(self, unit='B'):
+        self.unit = unit
+
+    def update(self, pbar):
+        """Updates the widget with the current SI prefixed speed."""
+
+        if pbar.seconds_elapsed < 2e-6 or pbar.currval < 2e-6: # =~ 0
+            scaled = power = 0
+        else:
+            speed = pbar.currval / pbar.seconds_elapsed
+            power = int(math.log(speed, 1000))
+            scaled = speed / 1000.**power
+
+        return self.FORMAT % (scaled, self.PREFIXES[power], self.unit)
+
+
+class AnimatedMarker(Widget):
+    """An animated marker for the progress bar which defaults to appear as if
+    it were rotating.
+    """
+
+    __slots__ = ('markers', 'curmark')
+
+    def __init__(self, markers='|/-\\'):
+        self.markers = markers
+        self.curmark = -1
+
+    def update(self, pbar):
+        """Updates the widget to show the next marker or the first marker when
+        finished"""
+
+        if pbar.finished: return self.markers[0]
+
+        self.curmark = (self.curmark + 1) % len(self.markers)
+        return self.markers[self.curmark]
+
+# Alias for backwards compatibility
+RotatingMarker = AnimatedMarker
+
+
+class Counter(Widget):
+    """Displays the current count."""
+
+    __slots__ = ('format_string',)
+
+    def __init__(self, format='%d'):
+        self.format_string = format
+
+    def update(self, pbar):
+        return self.format_string % pbar.currval
+
+
+class Percentage(Widget):
+    """Displays the current percentage as a number with a percent sign."""
+
+    def update(self, pbar):
+        return '%3d%%' % pbar.percentage()
+
+
+class FormatLabel(Timer):
+    """Displays a formatted label."""
+
+    mapping = {
+        'elapsed': ('seconds_elapsed', Timer.format_time),
+        'finished': ('finished', None),
+        'last_update': ('last_update_time', None),
+        'max': ('maxval', None),
+        'seconds': ('seconds_elapsed', None),
+        'start': ('start_time', None),
+        'value': ('currval', None)
+    }
+
+    __slots__ = ('format_string',)
+    def __init__(self, format):
+        self.format_string = format
+
+    def update(self, pbar):
+        context = {}
+        for name, (key, transform) in self.mapping.items():
+            try:
+                value = getattr(pbar, key)
+
+                if transform is None:
+                   context[name] = value
+                else:
+                   context[name] = transform(value)
+            except: pass
+
+        return self.format_string % context
+
+
+class SimpleProgress(Widget):
+    """Returns progress as a count of the total (e.g.: "5 of 47")."""
+
+    __slots__ = ('sep',)
+
+    def __init__(self, sep=' of '):
+        self.sep = sep
+
+    def update(self, pbar):
+        return '%d%s%d' % (pbar.currval, self.sep, pbar.maxval)
+
+
+class Bar(WidgetHFill):
+    """A progress bar which stretches to fill the line."""
+
+    __slots__ = ('marker', 'left', 'right', 'fill', 'fill_left')
+
+    def __init__(self, marker='#', left='|', right='|', fill=' ',
+                 fill_left=True):
+        """Creates a customizable progress bar.
+
+        marker - string or updatable object to use as a marker
+        left - string or updatable object to use as a left border
+        right - string or updatable object to use as a right border
+        fill - character to use for the empty part of the progress bar
+        fill_left - whether to fill from the left or the right
+        """
+        self.marker = marker
+        self.left = left
+        self.right = right
+        self.fill = fill
+        self.fill_left = fill_left
+
+
+    def update(self, pbar, width):
+        """Updates the progress bar and its subcomponents."""
+
+        left, marked, right = (format_updatable(i, pbar) for i in
+                               (self.left, self.marker, self.right))
+
+        width -= len(left) + len(right)
+        # Marked must *always* have length of 1
+        if pbar.maxval:
+          marked *= int(pbar.currval / pbar.maxval * width)
+        else:
+          marked = ''
+
+        if self.fill_left:
+            return '%s%s%s' % (left, marked.ljust(width, self.fill), right)
+        else:
+            return '%s%s%s' % (left, marked.rjust(width, self.fill), right)
+
+
+class ReverseBar(Bar):
+    """A bar which has a marker which bounces from side to side."""
+
+    def __init__(self, marker='#', left='|', right='|', fill=' ',
+                 fill_left=False):
+        """Creates a customizable progress bar.
+
+        marker - string or updatable object to use as a marker
+        left - string or updatable object to use as a left border
+        right - string or updatable object to use as a right border
+        fill - character to use for the empty part of the progress bar
+        fill_left - whether to fill from the left or the right
+        """
+        self.marker = marker
+        self.left = left
+        self.right = right
+        self.fill = fill
+        self.fill_left = fill_left
+
+
+class BouncingBar(Bar):
+    def update(self, pbar, width):
+        """Updates the progress bar and its subcomponents."""
+
+        left, marker, right = (format_updatable(i, pbar) for i in
+                               (self.left, self.marker, self.right))
+
+        width -= len(left) + len(right)
+
+        if pbar.finished: return '%s%s%s' % (left, width * marker, right)
+
+        position = int(pbar.currval % (width * 2 - 1))
+        if position > width: position = width * 2 - position
+        lpad = self.fill * (position - 1)
+        rpad = self.fill * (width - len(marker) - len(lpad))
+
+        # Swap if we want to bounce the other way
+        if not self.fill_left: rpad, lpad = lpad, rpad
+
+        return '%s%s%s%s%s' % (left, lpad, marker, rpad, right)
+
+
+class BouncingSlider(Bar):
+    """
+    A slider that bounces back and forth in response to update() calls
+    without reference to the actual value. Based on a combination of
+    BouncingBar from a newer version of this module and RotatingMarker.
+    """
+    def __init__(self, marker='<=>'):
+        self.curmark = -1
+        self.forward = True
+        Bar.__init__(self, marker=marker)
+    def update(self, pbar, width):
+        left, marker, right = (format_updatable(i, pbar) for i in
+                               (self.left, self.marker, self.right))
+
+        width -= len(left) + len(right)
+        if width < 0:
+            return ''
+
+        if pbar.finished: return '%s%s%s' % (left, width * '=', right)
+
+        self.curmark = self.curmark + 1
+        position = int(self.curmark % (width * 2 - 1))
+        if position + len(marker) > width:
+            self.forward = not self.forward
+            self.curmark = 1
+            position = 1
+        lpad = ' ' * (position - 1)
+        rpad = ' ' * (width - len(marker) - len(lpad))
+
+        if not self.forward:
+            temp = lpad
+            lpad = rpad
+            rpad = temp
+        return '%s%s%s%s%s' % (left, lpad, marker, rpad, right)

bitbake/lib/toaster/bldcollector/admin.py

@@ -1,33 +1,33 @@
 from django.contrib import admin
-from django.contrib.admin.filters import RelatedFieldListFilter
-from orm.models import BitbakeVersion, Release, LayerSource, ToasterSetting
-from django.forms.widgets import Textarea
+from orm.models import BitbakeVersion, Release, ToasterSetting, Layer_Version
 from django import forms
 import django.db.models as models
 
-from django.contrib.admin import widgets, helpers
-
-class LayerSourceAdmin(admin.ModelAdmin):
-    pass
 
 class BitbakeVersionAdmin(admin.ModelAdmin):
 
-    # we override the formfield for db URLField because of broken URL validation
+    # we override the formfield for db URLField
+    # because of broken URL validation
 
     def formfield_for_dbfield(self, db_field, **kwargs):
         if isinstance(db_field, models.fields.URLField):
             return forms.fields.CharField()
-        return super(BitbakeVersionAdmin, self).formfield_for_dbfield(db_field, **kwargs)
-
+        return super(BitbakeVersionAdmin, self).formfield_for_dbfield(
+            db_field, **kwargs)
 
 
 class ReleaseAdmin(admin.ModelAdmin):
     pass
 
+
 class ToasterSettingAdmin(admin.ModelAdmin):
     pass
 
-admin.site.register(LayerSource, LayerSourceAdmin)
+
+class LayerVersionsAdmin(admin.ModelAdmin):
+    pass
+
+admin.site.register(Layer_Version, LayerVersionsAdmin)
 admin.site.register(BitbakeVersion, BitbakeVersionAdmin)
 admin.site.register(Release, ReleaseAdmin)
 admin.site.register(ToasterSetting, ToasterSettingAdmin)

bitbake/lib/toaster/bldcontrol/localhostbecontroller.py

@@ -89,6 +89,10 @@ class LocalhostBEController(BuildEnvironmentController):
         """ a word of attention: by convention, the first layer for any build will be poky! """
 
         assert self.be.sourcedir is not None
+
+        layerlist = []
+        nongitlayerlist = []
+
         # set layers in the layersource
 
         # 1. get a list of repos with branches, and map dirpaths for each layer
@@ -102,6 +106,13 @@ class LocalhostBEController(BuildEnvironmentController):
             # as it's generated by us layer on if needed
             if CustomImageRecipe.LAYER_NAME in layer.name:
                 continue
+
+            # If we have local layers then we don't need clone them
+            # For local layers giturl will be empty
+            if not layer.giturl:
+                nongitlayerlist.append(layer.layer_version.layer.local_source_dir)
+                continue
+
             if not (layer.giturl, layer.commit) in gitrepos:
                 gitrepos[(layer.giturl, layer.commit)] = []
             gitrepos[(layer.giturl, layer.commit)].append( (layer.name, layer.dirpath) )
@@ -131,7 +142,6 @@ class LocalhostBEController(BuildEnvironmentController):
 
         logger.info("Using pre-checked out source for layer %s", cached_layers)
 
-        layerlist = []
 
 
         # 3. checkout the repositories
@@ -245,6 +255,7 @@ class LocalhostBEController(BuildEnvironmentController):
             layerlist.append(layerpath)
 
         self.islayerset = True
+        layerlist.extend(nongitlayerlist)
         return layerlist
 
     def readServerLogFile(self):

bitbake/lib/toaster/bldcontrol/management/commands/checksettings.py

@@ -1,10 +1,15 @@
 from django.core.management.base import NoArgsCommand, CommandError
 from django.db import transaction
+
+from django.core.management import call_command
 from bldcontrol.bbcontroller import getBuildEnvironmentController, ShellCmdException
 from bldcontrol.models import BuildRequest, BuildEnvironment, BRError
-from orm.models import ToasterSetting, Build
+from orm.models import ToasterSetting, Build, Layer
+
 import os
 import traceback
+import warnings
+
 
 def DN(path):
     if path is None:
@@ -21,39 +26,6 @@ class Command(NoArgsCommand):
         super(Command, self).__init__(*args, **kwargs)
         self.guesspath = DN(DN(DN(DN(DN(DN(DN(__file__)))))))
 
-    def _find_first_path_for_file(self, startdirectory, filename, level=0):
-        if level < 0:
-            return None
-        dirs = []
-        for i in os.listdir(startdirectory):
-            j = os.path.join(startdirectory, i)
-            if os.path.isfile(j):
-                if i == filename:
-                    return startdirectory
-            elif os.path.isdir(j):
-                dirs.append(j)
-        for j in dirs:
-            ret = self._find_first_path_for_file(j, filename, level - 1)
-            if ret is not None:
-                return ret
-        return None
-
-    def _recursive_list_directories(self, startdirectory, level=0):
-        if level < 0:
-            return []
-        dirs = []
-        try:
-            for i in os.listdir(startdirectory):
-                j = os.path.join(startdirectory, i)
-                if os.path.isdir(j):
-                    dirs.append(j)
-        except OSError:
-            pass
-        for j in dirs:
-            dirs = dirs + self._recursive_list_directories(j, level - 1)
-        return dirs
-
-
     def _verify_build_environment(self):
         # provide a local build env. This will be extended later to include non local
         if BuildEnvironment.objects.count() == 0:
@@ -94,30 +66,58 @@ class Command(NoArgsCommand):
                     print("\n -- Validation: The build directory must to be set to an absolute path.")
                     is_changed = _update_builddir()
 
-
                 if is_changed:
                     print("\nBuild configuration saved")
                     be.save()
                     return True
 
-
                 if be.needs_import:
                     try:
-                        config_file = os.environ.get('TOASTER_CONF')
-                        print("\nImporting file: %s" % config_file)
-                        from .loadconf import Command as LoadConfigCommand
+                        print("Loading default settings")
+                        call_command("loaddata", "settings")
+                        template_conf = os.environ.get("TEMPLATECONF", "")
+
+                        if "poky" in template_conf:
+                            print("Loading poky configuration")
+                            call_command("loaddata", "poky")
+                        else:
+                            print("Loading OE-Core configuration")
+                            call_command("loaddata", "oe-core")
+                            if template_conf:
+                                oe_core_path = os.realpath(template_conf +
+                                                           "/../")
+                            else:
+                                print("TEMPLATECONF not found. You may have to"
+                                      " manually configure layer paths")
+                                oe_core_path = input("Please enter the path of"
+                                                     " your openembedded-core "
+                                                     "layer: ")
+                            # Update the layer instances of openemebedded-core
+                            for layer in Layer.objects.filter(
+                                    name="openembedded-core"):
+                                layer.local_source_dir = oe_core_path
+                                layer.save()
+
+                        # Import the custom fixture if it's present
+                        with warnings.catch_warnings():
+                            warnings.filterwarnings(
+                                action="ignore",
+                                message="^.*No fixture named.*$")
+                            print("Importing custom settings if present")
+                            call_command("loaddata", "custom")
 
-                        LoadConfigCommand()._import_layer_config(config_file)
                         # we run lsupdates after config update
-                        print("\nLayer configuration imported. Updating information from the layer sources, please wait.\nYou can re-update any time later by running bitbake/lib/toaster/manage.py lsupdates")
-                        from django.core.management import call_command
+                        print("\nFetching information from the layer index, "
+                              "please wait.\nYou can re-update any time later "
+                              "by running bitbake/lib/toaster/manage.py "
+                              "lsupdates\n")
                         call_command("lsupdates")
 
                         # we don't look for any other config files
                         return is_changed
                     except Exception as e:
-                        print("Failure while trying to import the toaster config file %s: %s" %\
-                            (config_file, e))
+                        print("Failure while trying to setup toaster: %s"
+                              % e)
                         traceback.print_exc()
 
                 return is_changed

bitbake/lib/toaster/bldcontrol/management/commands/loadconf.py

@@ -1,183 +0,0 @@
-from django.core.management.base import BaseCommand, CommandError
-from orm.models import LayerSource, ToasterSetting, Branch, Layer, Layer_Version
-from orm.models import BitbakeVersion, Release, ReleaseDefaultLayer, ReleaseLayerSourcePriority
-from django.db import IntegrityError
-import os
-
-from .checksettings import DN
-
-import logging
-logger = logging.getLogger("toaster")
-
-def _reduce_canon_path(path):
-    components = []
-    for c in path.split("/"):
-        if c == "..":
-            del components[-1]
-        elif c == ".":
-            pass
-        else:
-            components.append(c)
-    if len(components) < 2:
-        components.append('')
-    return "/".join(components)
-
-def _get_id_for_sourcetype(s):
-    for i in LayerSource.SOURCE_TYPE:
-        if s == i[1]:
-            return i[0]
-    raise Exception("Could not find definition for sourcetype '%s'. Valid source types are %s" % (str(s), ', '.join(map(lambda x: "'%s'" % x[1], LayerSource.SOURCE_TYPE ))))
-
-class Command(BaseCommand):
-    help = "Loads a toasterconf.json file in the database"
-    args = "filepath"
-
-
-
-    def _import_layer_config(self, filepath):
-        if not os.path.exists(filepath) or not os.path.isfile(filepath):
-            raise Exception("Failed to find toaster config file %s ." % filepath)
-
-        import json
-        data = json.loads(open(filepath, "r").read())
-
-        # verify config file validity before updating settings
-        for i in ['bitbake', 'releases', 'defaultrelease', 'config', 'layersources']:
-            assert i in data
-
-        def _read_git_url_from_local_repository(address):
-            url = None
-            # we detect the remote name at runtime
-            import subprocess
-            (remote, remote_name) = address.split(":", 1)
-            cmd = subprocess.Popen("git remote -v", shell=True, cwd = os.path.dirname(filepath), stdout=subprocess.PIPE, stderr = subprocess.PIPE)
-            (out,err) = cmd.communicate()
-            if cmd.returncode != 0:
-                logging.warning("Error while importing layer vcs_url: git error: %s" % err)
-            for line in out.decode('utf-8').split("\n"):
-                try:
-                    (name, path) = line.split("\t", 1)
-                    if name == remote_name:
-                        url = path.split(" ")[0]
-                        break
-                except ValueError:
-                    pass
-            if url == None:
-                logging.warning("Error while looking for remote \"%s\" in \"%s\"" % (remote_name, out))
-            return url
-
-
-        # import bitbake data
-        for bvi in data['bitbake']:
-            bvo, created = BitbakeVersion.objects.get_or_create(name=bvi['name'])
-            if bvi['giturl'].startswith("remote:"):
-                bvo.giturl = _read_git_url_from_local_repository(bvi['giturl'])
-                if bvo.giturl is None:
-                    logger.error("The toaster config file references the local git repo, but Toaster cannot detect it.\nYour local configuration for bitbake version %s is invalid. Make sure that the toasterconf.json file is correct." % bvi['name'])
-
-            if bvo.giturl is None:
-                bvo.giturl = bvi['giturl']
-            bvo.branch = bvi['branch']
-            bvo.dirpath = bvi['dirpath']
-            bvo.save()
-
-        # set the layer sources
-        for lsi in data['layersources']:
-            assert 'sourcetype' in lsi
-            assert 'apiurl' in lsi
-            assert 'name' in lsi
-            assert 'branches' in lsi
-
-
-            if _get_id_for_sourcetype(lsi['sourcetype']) == LayerSource.TYPE_LAYERINDEX or lsi['apiurl'].startswith("/"):
-                apiurl = lsi['apiurl']
-            else:
-                apiurl = _reduce_canon_path(os.path.join(DN(os.path.abspath(filepath)), lsi['apiurl']))
-
-            assert ((_get_id_for_sourcetype(lsi['sourcetype']) == LayerSource.TYPE_LAYERINDEX) or apiurl.startswith("/")), (lsi['sourcetype'],apiurl)
-
-            try:
-                ls, created = LayerSource.objects.get_or_create(sourcetype = _get_id_for_sourcetype(lsi['sourcetype']), apiurl = apiurl)
-                ls.name = lsi['name']
-                ls.save()
-            except IntegrityError as e:
-                logger.warning("IntegrityError %s \nWhile setting name %s for layer source %s " % (e, lsi['name'], ls))
-
-
-            layerbranches = []
-            for branchname in lsi['branches']:
-                bo, created = Branch.objects.get_or_create(layer_source = ls, name = branchname)
-                layerbranches.append(bo)
-
-            if 'layers' in lsi:
-                for layerinfo in lsi['layers']:
-                    lo, created = Layer.objects.get_or_create(layer_source = ls, name = layerinfo['name'])
-                    if layerinfo['local_path'].startswith("/"):
-                        lo.local_path = layerinfo['local_path']
-                    else:
-                        lo.local_path = _reduce_canon_path(os.path.join(ls.apiurl, layerinfo['local_path']))
-
-                    if not os.path.exists(lo.local_path):
-                        logger.error("Local layer path %s must exists. Are you trying to import a layer that does not exist ? Check your local toasterconf.json" % lo.local_path)
-
-                    if layerinfo['vcs_url'].startswith("remote:"):
-                        lo.vcs_url = _read_git_url_from_local_repository(layerinfo['vcs_url'])
-                        if lo.vcs_url is None:
-                            logger.error("The toaster config file references the local git repo, but Toaster cannot detect it.\nYour local configuration for layer %s is invalid. Make sure that the toasterconf.json file is correct." % layerinfo['name'])
-
-                    if lo.vcs_url is None:
-                        lo.vcs_url = layerinfo['vcs_url']
-
-                    if 'layer_index_url' in layerinfo:
-                        lo.layer_index_url = layerinfo['layer_index_url']
-                    lo.save()
-
-                    for branch in layerbranches:
-                        lvo, created = Layer_Version.objects.get_or_create(layer_source = ls,
-                                up_branch = branch,
-                                commit = branch.name,
-                                layer = lo)
-                        lvo.dirpath = layerinfo['dirpath']
-                        lvo.save()
-        # set releases
-        for ri in data['releases']:
-            bvo = BitbakeVersion.objects.get(name = ri['bitbake'])
-            assert bvo is not None
-
-            ro, created = Release.objects.get_or_create(name = ri['name'], bitbake_version = bvo, branch_name = ri['branch'])
-            ro.description = ri['description']
-            ro.helptext = ri['helptext']
-            ro.save()
-
-            # save layer source priority for release
-            for ls_name in ri['layersourcepriority'].keys():
-                rlspo, created = ReleaseLayerSourcePriority.objects.get_or_create(release = ro, layer_source = LayerSource.objects.get(name=ls_name))
-                rlspo.priority = ri['layersourcepriority'][ls_name]
-                rlspo.save()
-
-            for dli in ri['defaultlayers']:
-                # find layers with the same name
-                ReleaseDefaultLayer.objects.get_or_create( release = ro, layer_name = dli)
-
-        # set default release
-        if ToasterSetting.objects.filter(name = "DEFAULT_RELEASE").count() > 0:
-            ToasterSetting.objects.filter(name = "DEFAULT_RELEASE").update(value = data['defaultrelease'])
-        else:
-            ToasterSetting.objects.create(name = "DEFAULT_RELEASE", value = data['defaultrelease'])
-
-        # set default config variables
-        for configname in data['config']:
-            if ToasterSetting.objects.filter(name = "DEFCONF_" + configname).count() > 0:
-                ToasterSetting.objects.filter(name = "DEFCONF_" + configname).update(value = data['config'][configname])
-            else:
-                ToasterSetting.objects.create(name = "DEFCONF_" + configname, value = data['config'][configname])
-
-
-    def handle(self, *args, **options):
-        if len(args) == 0:
-            raise CommandError("Need a path to the toasterconf.json file")
-        filepath = args[0]
-        self._import_layer_config(filepath)
-
-
-

bitbake/lib/toaster/bldcontrol/migrations/0005_reorder_buildrequest_states.py

@@ -0,0 +1,19 @@
+# -*- coding: utf-8 -*-
+from __future__ import unicode_literals
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('bldcontrol', '0004_auto_20160523_1446'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='buildrequest',
+            name='state',
+            field=models.IntegerField(choices=[(0, 'created'), (1, 'queued'), (2, 'in progress'), (3, 'failed'), (4, 'deleted'), (5, 'cancelling'), (6, 'completed'), (7, 'archive')], default=0),
+        ),
+    ]

bitbake/lib/toaster/bldcontrol/migrations/0006_brlayer_local_source_dir.py

@@ -0,0 +1,19 @@
+# -*- coding: utf-8 -*-
+from __future__ import unicode_literals
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('bldcontrol', '0005_reorder_buildrequest_states'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='brlayer',
+            name='local_source_dir',
+            field=models.CharField(max_length=254, null=True),
+        ),
+    ]

bitbake/lib/toaster/bldcontrol/models.py

@@ -63,20 +63,20 @@ class BuildRequest(models.Model):
     REQ_CREATED = 0
     REQ_QUEUED = 1
     REQ_INPROGRESS = 2
-    REQ_COMPLETED = 3
-    REQ_FAILED = 4
-    REQ_DELETED = 5
-    REQ_CANCELLING = 6
+    REQ_FAILED = 3
+    REQ_DELETED = 4
+    REQ_CANCELLING = 5
+    REQ_COMPLETED = 6
     REQ_ARCHIVE = 7
 
     REQUEST_STATE = (
         (REQ_CREATED, "created"),
         (REQ_QUEUED, "queued"),
         (REQ_INPROGRESS, "in progress"),
-        (REQ_COMPLETED, "completed"),
         (REQ_FAILED, "failed"),
         (REQ_DELETED, "deleted"),
         (REQ_CANCELLING, "cancelling"),
+        (REQ_COMPLETED, "completed"),
         (REQ_ARCHIVE, "archive"),
     )
 
@@ -91,7 +91,7 @@ class BuildRequest(models.Model):
 
     def __init__(self, *args, **kwargs):
         super(BuildRequest, self).__init__(*args, **kwargs)
-        # Save the old state incase it's about to be modified
+        # Save the old state in case it's about to be modified
         self.old_state = self.state
 
     def save(self, *args, **kwargs):
@@ -130,6 +130,7 @@ class BRLayer(models.Model):
     req         = models.ForeignKey(BuildRequest)
     name        = models.CharField(max_length = 100)
     giturl      = models.CharField(max_length = 254)
+    local_source_dir = models.CharField(max_length=254, null=True)
     commit      = models.CharField(max_length = 254)
     dirpath     = models.CharField(max_length = 254)
     layer_version = models.ForeignKey(Layer_Version, null=True)

bitbake/lib/toaster/bldcontrol/tests.py

@@ -139,22 +139,3 @@ class RunBuildsCommandTests(TestCase):
         self.assertTrue(br.state == BuildRequest.REQ_INPROGRESS, "Request is not updated")
         # no more selections possible here
         self.assertRaises(IndexError, command._selectBuildRequest)
-
-
-class UtilityTests(TestCase):
-    def test_reduce_path(self):
-        from bldcontrol.management.commands.loadconf import _reduce_canon_path, _get_id_for_sourcetype
-
-        self.assertTrue( _reduce_canon_path("/") == "/")
-        self.assertTrue( _reduce_canon_path("/home/..") == "/")
-        self.assertTrue( _reduce_canon_path("/home/../ana") == "/ana")
-        self.assertTrue( _reduce_canon_path("/home/../ana/..") == "/")
-        self.assertTrue( _reduce_canon_path("/home/ana/mihai/../maria") == "/home/ana/maria")
-
-    def test_get_id_for_sorucetype(self):
-        from bldcontrol.management.commands.loadconf import _reduce_canon_path, _get_id_for_sourcetype
-        self.assertTrue( _get_id_for_sourcetype("layerindex") == 1)
-        self.assertTrue( _get_id_for_sourcetype("local") == 0)
-        self.assertTrue( _get_id_for_sourcetype("imported") == 2)
-        with self.assertRaises(Exception):
-            _get_id_for_sourcetype("unknown")

bitbake/lib/toaster/orm/fixtures/README

@@ -0,0 +1,30 @@
+# Fixtures directory
+
+Fixtures are data dumps that can be loaded into Toaster's database to provide
+configuration and data.
+
+In this directory we have the fixtures which are loaded the first time you start Toaster.
+This is to provide useful default values and metadata to Toaster.
+
+ - settings.xml This Contains Toaster wide settings, such as the default values for
+   certain bitbake variables.
+
+ - poky.xml This is the default release data for supported poky based setup 
+
+ - oe-core.xml This is the default release data for supported oe-core based setups
+
+# Custom data/configuration
+
+  - custom.xml
+
+To add custom initial data/configuration to Toaster place a file called
+"custom.xml" in this directory. If present it will be loaded into the database.
+We suggest that this is used to overlay any configuration already done.
+All objects loaded with the same primary keys overwrite the existing data.
+Data can be provided in XML, JSON and if installed YAML formats.
+
+# To load data at any point in time
+
+Use the django management command manage.py loaddata <your fixture file>
+For further information see the Django command documentation at:
+https://docs.djangoproject.com/en/1.8/ref/django-admin/#django-admin-loaddata

bitbake/lib/toaster/orm/fixtures/oe-core.xml

@@ -0,0 +1,57 @@
+<?xml version="1.0" encoding="utf-8"?>
+<django-objects version="1.0">
+  <!-- Bitbake versions which correspond to the metadata release -->
+  <object model="orm.bitbakeversion" pk="1">
+    <field type="CharField" name="name">master</field>
+    <field type="CharField" name="giturl">git://git.openembedded.org/bitbake</field>
+    <field type="CharField" name="branch">master</field>
+    <field type="CharField" name="dirpath">bitbake</field>
+  </object>
+  <object model="orm.bitbakeversion" pk="2">
+    <field type="CharField" name="name">HEAD</field>
+    <field type="CharField" name="giturl">git://git.openembedded.org/bitbake</field>
+    <field type="CharField" name="branch">HEAD</field>
+    <field type="CharField" name="dirpath">bitbake</field>
+  </object>
+
+  <!-- Releases available -->
+  <object model="orm.release" pk="1">
+    <field type="CharField" name="name">master</field>
+    <field type="CharField" name="description">Openembedded master</field>
+    <field rel="ManyToOneRel" to="orm.bitbakeversion" name="bitbake_version">1</field>
+    <field type="CharField" name="branch_name">master</field>
+    <field type="TextField" name="helptext">Toaster will run your builds using the tip of the &lt;a href=\"http://cgit.openembedded.org/openembedded-core/log/\"&gt;OpenEmbedded master&lt;/a&gt; branch, where active development takes place. This is not a stable branch, so your builds might not work as expected.</field>
+  </object>
+  <object model="orm.release" pk="2">
+    <field type="CharField" name="name">local</field>
+    <field type="CharField" name="description">Local Openembedded</field>
+    <field rel="ManyToOneRel" to="orm.bitbakeversion" name="bitbake_version">2</field>
+    <field type="CharField" name="branch_name">HEAD</field>
+    <field type="TextField" name="helptext">Toaster will run your builds with the version of OpenEmbedded that you have cloned or downloaded to your computer.</field>
+  </object>
+
+  <!-- Default layers for each release -->
+  <object model="orm.releasedefaultlayer" pk="1">
+    <field rel="ManyToOneRel" to="orm.release" name="release">1</field>
+    <field type="CharField" name="layer_name">openembedded-core</field>
+  </object>
+  <object model="orm.releasedefaultlayer" pk="4">
+    <field rel="ManyToOneRel" to="orm.release" name="release">2</field>
+    <field type="CharField" name="layer_name">openembedded-core</field>
+  </object>
+  
+  <!-- TYPE_LOCAL = 0 Layers for the Local release -->
+  <object model="orm.layer" pk="1">
+    <field type="CharField" name="name">openembedded-core</field>
+    <field type="CharField" name="layer_index_url"></field>
+    <field type="CharField" name="vcs_url">git://git.openembedded.org/openembedded-core</field>
+  </object> 
+  <object model="orm.layer_version" pk="1">
+    <field rel="ManyToOneRel" to="orm.layer" name="layer">1</field>
+    <field type="IntegerField" name="layer_source">0</field>
+    <field rel="ManyToOneRel" to="orm.release" name="release">2</field>
+    <field type="CharField" name="branch">HEAD</field>
+    <field type="CharField" name="commit">HEAD</field>
+  </object>
+
+</django-objects>

bitbake/lib/toaster/orm/fixtures/poky.xml

@@ -0,0 +1,105 @@
+<?xml version="1.0" encoding="utf-8"?>
+<django-objects version="1.0">
+  <!-- Bitbake versions which correspond to the metadata release -->
+  <object model="orm.bitbakeversion" pk="1">
+    <field type="CharField" name="name">master</field>
+    <field type="CharField" name="giturl">git://git.yoctoproject.org/poky</field>
+    <field type="CharField" name="branch">master</field>
+    <field type="CharField" name="dirpath">bitbake</field>
+  </object>
+  <object model="orm.bitbakeversion" pk="2">
+    <field type="CharField" name="name">HEAD</field>
+    <field type="CharField" name="giturl">git://git.yoctoproject.org/poky</field>
+    <field type="CharField" name="branch">HEAD</field>
+    <field type="CharField" name="dirpath">bitbake</field>
+  </object>
+
+  <!-- Releases available -->
+  <object model="orm.release" pk="1">
+    <field type="CharField" name="name">master</field>
+    <field type="CharField" name="description">Yocto Project master</field>
+    <field rel="ManyToOneRel" to="orm.bitbakeversion" name="bitbake_version">1</field>
+    <field type="CharField" name="branch_name">master</field>
+    <field type="TextField" name="helptext">Toaster will run your builds using the tip of the &lt;a href="http://git.yoctoproject.org/cgit/cgit.cgi/poky/log/"&gt;Yocto Project master branch&lt;/a&gt;, where active development takes place. This is not a stable branch, so your builds might not work as expected.</field>
+  </object>
+  <object model="orm.release" pk="2">
+    <field type="CharField" name="name">local</field>
+    <field type="CharField" name="description">Local Yocto Project</field>
+    <field rel="ManyToOneRel" to="orm.bitbakeversion" name="bitbake_version">2</field>
+    <field type="CharField" name="branch_name">HEAD</field>
+    <field type="TextField" name="helptext">Toaster will run your builds with the version of the Yocto Project you have cloned or downloaded to your computer.</field>
+  </object>
+
+  <!-- Default layers for each release -->
+  <object model="orm.releasedefaultlayer" pk="1">
+    <field rel="ManyToOneRel" to="orm.release" name="release">1</field>
+    <field type="CharField" name="layer_name">openembedded-core</field>
+  </object>
+  <object model="orm.releasedefaultlayer" pk="2">
+    <field rel="ManyToOneRel" to="orm.release" name="release">1</field>
+    <field type="CharField" name="layer_name">meta-poky</field>
+  </object>
+  <object model="orm.releasedefaultlayer" pk="3">
+    <field rel="ManyToOneRel" to="orm.release" name="release">1</field>
+    <field type="CharField" name="layer_name">meta-yocto-bsp</field>
+  </object>
+  <object model="orm.releasedefaultlayer" pk="4">
+    <field rel="ManyToOneRel" to="orm.release" name="release">2</field>
+    <field type="CharField" name="layer_name">openembedded-core</field>
+  </object>
+  <object model="orm.releasedefaultlayer" pk="5">
+    <field rel="ManyToOneRel" to="orm.release" name="release">2</field>
+    <field type="CharField" name="layer_name">meta-poky</field>
+  </object>
+  <object model="orm.releasedefaultlayer" pk="6">
+    <field rel="ManyToOneRel" to="orm.release" name="release">2</field>
+    <field type="CharField" name="layer_name">meta-yocto-bsp</field>
+  </object>
+
+  <!-- Layers for the Local release
+       layersource TYPE_LOCAL = 0
+  -->
+  <object model="orm.layer" pk="1">
+    <field type="CharField" name="name">openembedded-core</field>
+    <field type="CharField" name="layer_index_url"></field>
+    <field type="CharField" name="vcs_url">git://git.yoctoproject.org/poky</field>
+  </object>
+  <object model="orm.layer_version" pk="1">
+    <field rel="ManyToOneRel" to="orm.layer" name="layer">1</field>
+    <field type="IntegerField" name="layer_source">0</field>
+    <field rel="ManyToOneRel" to="orm.release" name="release">2</field>
+    <field type="CharField" name="branch">HEAD</field>
+    <field type="CharField" name="commit">HEAD</field>
+    <field type="CharField" name="dirpath">meta</field>
+  </object>
+
+
+  <object model="orm.layer" pk="2">
+    <field type="CharField" name="name">meta-poky</field>
+    <field type="CharField" name="layer_index_url"></field>
+    <field type="CharField" name="vcs_url">git://git.yoctoproject.org/poky</field>
+  </object>
+  <object model="orm.layer_version" pk="2">
+    <field rel="ManyToOneRel" to="orm.layer" name="layer">2</field>
+    <field type="IntegerField" name="layer_source">0</field>
+    <field rel="ManyToOneRel" to="orm.release" name="release">2</field>
+    <field type="CharField" name="branch">HEAD</field>
+    <field type="CharField" name="commit">HEAD</field>
+    <field type="CharField" name="dirpath">meta-poky</field>
+  </object>
+
+
+  <object model="orm.layer" pk="3">
+    <field type="CharField" name="name">meta-yocto-bsp</field>
+    <field type="CharField" name="layer_index_url"></field>
+    <field type="CharField" name="vcs_url">git://git.yoctoproject.org/poky</field>
+  </object>
+  <object model="orm.layer_version" pk="3">
+    <field rel="ManyToOneRel" to="orm.layer" name="layer">3</field>
+    <field type="IntegerField" name="layer_source">0</field>
+    <field rel="ManyToOneRel" to="orm.release" name="release">2</field>
+    <field type="CharField" name="branch">HEAD</field>
+    <field type="CharField" name="commit">HEAD</field>
+    <field type="CharField" name="dirpath">meta-yocto-bsp</field>
+  </object>
+</django-objects>

bitbake/lib/toaster/orm/fixtures/settings.xml

@@ -0,0 +1,36 @@
+<?xml version="1.0" encoding="utf-8"?>
+<django-objects version="1.0">
+  <!-- Default project settings -->
+  <object model="orm.toastersetting" pk="1">
+    <field type="CharField" name="name">DEFAULT_RELEASE</field>
+    <field type="CharField" name="value">master</field>
+  </object>
+  <object model="orm.toastersetting" pk="2">
+    <field type="CharField" name="name">DEFCONF_PACKAGE_CLASSES</field>
+    <field type="CharField" name="value">package_rpm</field>
+  </object>
+  <object model="orm.toastersetting" pk="3">
+    <field type="CharField" name="name">DEFCONF_MACHINE</field>
+    <field type="CharField" name="value">qemux86</field>
+  </object>
+  <object model="orm.toastersetting" pk="4">
+    <field type="CharField" name="name">DEFCONF_SSTATE_DIR</field>
+    <field type="CharField" name="value">${TOPDIR}/../sstate-cache</field>
+  </object>
+  <object model="orm.toastersetting" pk="5">
+    <field type="CharField" name="name">DEFCONF_IMAGE_INSTALL_append</field>
+    <field type="CharField" name="value"></field>
+  </object>
+  <object model="orm.toastersetting" pk="6">
+    <field type="CharField" name="name">DEFCONF_IMAGE_FSTYPES</field>
+    <field type="CharField" name="value">ext3 jffs2 tar.bz2</field>
+  </object>
+  <object model="orm.toastersetting" pk="7">
+    <field type="CharField" name="name">DEFCONF_DISTRO</field>
+    <field type="CharField" name="value">poky</field>
+  </object>
+  <object model="orm.toastersetting" pk="8">
+    <field type="CharField" name="name">DEFCONF_DL_DIR</field>
+    <field type="CharField" name="value">${TOPDIR}/../downloads</field>
+  </object>
+</django-objects>

bitbake/lib/toaster/orm/management/commands/lsupdates.py

@@ -1,12 +1,328 @@
-from django.core.management.base import NoArgsCommand, CommandError
-from orm.models import LayerSource
+#
+# ex:ts=4:sw=4:sts=4:et
+# -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*-
+#
+# BitBake Toaster Implementation
+#
+# Copyright (C) 2016        Intel Corporation
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License version 2 as
+# published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License along
+# with this program; if not, write to the Free Software Foundation, Inc.,
+# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+from django.core.management.base import NoArgsCommand
+
+from orm.models import LayerSource, Layer, Release, Layer_Version
+from orm.models import LayerVersionDependency, Machine, Recipe
+
 import os
+import sys
+
+import json
+import logging
+import threading
+import time
+logger = logging.getLogger("toaster")
+
+DEFAULT_LAYERINDEX_SERVER = "http://layers.openembedded.org/layerindex/api/"
+
+
+class Spinner(threading.Thread):
+    """ A simple progress spinner to indicate download/parsing is happening"""
+    def __init__(self, *args, **kwargs):
+        super(Spinner, self).__init__(*args, **kwargs)
+        self.setDaemon(True)
+        self.signal = True
+
+    def run(self):
+        os.system('setterm -cursor off')
+        while self.signal:
+            for char in ["/", "-", "\\", "|"]:
+                sys.stdout.write("\r" + char)
+                sys.stdout.flush()
+                time.sleep(0.25)
+        os.system('setterm -cursor on')
+
+    def stop(self):
+        self.signal = False
+
 
 class Command(NoArgsCommand):
-    args    = ""
-    help    = "Updates locally cached information from all LayerSources"
+    args = ""
+    help = "Updates locally cached information from a layerindex server"
 
+    def mini_progress(self, what, i, total):
+        i = i + 1
+        pec = (float(i)/float(total))*100
+
+        sys.stdout.write("\rUpdating %s %d%%" %
+                         (what,
+                          pec))
+        sys.stdout.flush()
+        if int(pec) is 100:
+            sys.stdout.write("\n")
+            sys.stdout.flush()
+
+    def update(self):
+        """
+            Fetches layer, recipe and machine information from a layerindex
+            server
+        """
+        os.system('setterm -cursor off')
+
+        self.apiurl = DEFAULT_LAYERINDEX_SERVER
+
+        assert self.apiurl is not None
+        try:
+            from urllib.request import urlopen, URLError
+            from urllib.parse import urlparse
+        except ImportError:
+            from urllib2 import urlopen, URLError
+            from urlparse import urlparse
+
+        proxy_settings = os.environ.get("http_proxy", None)
+        oe_core_layer = 'openembedded-core'
+
+        def _get_json_response(apiurl=DEFAULT_LAYERINDEX_SERVER):
+            http_progress = Spinner()
+            http_progress.start()
+
+            _parsedurl = urlparse(apiurl)
+            path = _parsedurl.path
+
+            # logger.debug("Fetching %s", apiurl)
+            try:
+                res = urlopen(apiurl)
+            except URLError as e:
+                raise Exception("Failed to read %s: %s" % (path, e.reason))
+
+            parsed = json.loads(res.read().decode('utf-8'))
+
+            http_progress.stop()
+            return parsed
+
+        # verify we can get the basic api
+        try:
+            apilinks = _get_json_response()
+        except Exception as e:
+            import traceback
+            if proxy_settings is not None:
+                logger.info("EE: Using proxy %s" % proxy_settings)
+            logger.warning("EE: could not connect to %s, skipping update:"
+                           "%s\n%s" % (self.apiurl, e, traceback.format_exc()))
+            return
+
+        # update branches; only those that we already have names listed in the
+        # Releases table
+        whitelist_branch_names = [rel.branch_name
+                                  for rel in Release.objects.all()]
+        if len(whitelist_branch_names) == 0:
+            raise Exception("Failed to make list of branches to fetch")
+
+        logger.info("Fetching metadata releases for %s",
+                    " ".join(whitelist_branch_names))
+
+        branches_info = _get_json_response(apilinks['branches'] +
+                                           "?filter=name:%s"
+                                           % "OR".join(whitelist_branch_names))
+
+        # Map the layer index branches to toaster releases
+        li_branch_id_to_toaster_release = {}
+
+        total = len(branches_info)
+        for i, branch in enumerate(branches_info):
+            li_branch_id_to_toaster_release[branch['id']] = \
+                    Release.objects.get(name=branch['name'])
+            self.mini_progress("Releases", i, total)
+
+        # keep a track of the layerindex (li) id mappings so that
+        # layer_versions can be created for these layers later on
+        li_layer_id_to_toaster_layer_id = {}
+
+        logger.info("Fetching layers")
+
+        layers_info = _get_json_response(apilinks['layerItems'])
+
+        total = len(layers_info)
+        for i, li in enumerate(layers_info):
+            # Special case for the openembedded-core layer
+            if li['name'] == oe_core_layer:
+                try:
+                    # If we have an existing openembedded-core for example
+                    # from the toasterconf.json augment the info using the
+                    # layerindex rather than duplicate it
+                    oe_core_l = Layer.objects.get(name=oe_core_layer)
+                    # Take ownership of the layer as now coming from the
+                    # layerindex
+                    oe_core_l.summary = li['summary']
+                    oe_core_l.description = li['description']
+                    oe_core_l.save()
+                    li_layer_id_to_toaster_layer_id[li['id']] = oe_core_l.pk
+                    self.mini_progress("layers", i, total)
+                    continue
+
+                except Layer.DoesNotExist:
+                    pass
+
+            try:
+                l, created = Layer.objects.get_or_create(name=li['name'],
+                                                         vcs_url=li['vcs_url'])
+                l.up_date = li['updated']
+                l.vcs_url = li['vcs_url']
+                l.vcs_web_url = li['vcs_web_url']
+                l.vcs_web_tree_base_url = li['vcs_web_tree_base_url']
+                l.vcs_web_file_base_url = li['vcs_web_file_base_url']
+                l.summary = li['summary']
+                l.description = li['description']
+                l.save()
+            except Layer.MultipleObjectsReturned:
+                logger.info("Skipped %s as we found multiple layers and "
+                            "don't know which to update" %
+                            li['name'])
+
+            li_layer_id_to_toaster_layer_id[li['id']] = l.pk
+
+            self.mini_progress("layers", i, total)
+
+        # update layer_versions
+        logger.info("Fetching layer versions")
+        layerbranches_info = _get_json_response(
+            apilinks['layerBranches'] + "?filter=branch__name:%s" %
+            "OR".join(whitelist_branch_names))
+
+        # Map Layer index layer_branch object id to
+        # layer_version toaster object id
+        li_layer_branch_id_to_toaster_lv_id = {}
+
+        total = len(layerbranches_info)
+        for i, lbi in enumerate(layerbranches_info):
+
+            try:
+                lv, created = Layer_Version.objects.get_or_create(
+                    layer_source=LayerSource.TYPE_LAYERINDEX,
+                    layer=Layer.objects.get(
+                        pk=li_layer_id_to_toaster_layer_id[lbi['layer']])
+                )
+            except KeyError:
+                logger.warning(
+                    "No such layerindex layer referenced by layerbranch %d" %
+                    lbi['layer'])
+                continue
+
+            lv.release = li_branch_id_to_toaster_release[lbi['branch']]
+            lv.up_date = lbi['updated']
+            lv.commit = lbi['actual_branch']
+            lv.dirpath = lbi['vcs_subdir']
+            lv.save()
+
+            li_layer_branch_id_to_toaster_lv_id[lbi['id']] =\
+                lv.pk
+            self.mini_progress("layer versions", i, total)
+
+        logger.info("Fetching layer version dependencies")
+        # update layer dependencies
+        layerdependencies_info = _get_json_response(
+            apilinks['layerDependencies'] +
+            "?filter=layerbranch__branch__name:%s" %
+            "OR".join(whitelist_branch_names))
+
+        dependlist = {}
+        for ldi in layerdependencies_info:
+            try:
+                lv = Layer_Version.objects.get(
+                    pk=li_layer_branch_id_to_toaster_lv_id[ldi['layerbranch']])
+            except Layer_Version.DoesNotExist as e:
+                continue
+
+            if lv not in dependlist:
+                dependlist[lv] = []
+            try:
+                layer_id = li_layer_id_to_toaster_layer_id[ldi['dependency']]
+
+                dependlist[lv].append(
+                    Layer_Version.objects.get(
+                        layer_source=LayerSource.TYPE_LAYERINDEX,
+                        layer__pk=layer_id))
+
+            except Layer_Version.DoesNotExist:
+                logger.warning("Cannot find layer version (ls:%s),"
+                               "up_id:%s lv:%s" %
+                               (self, ldi['dependency'], lv))
+
+        total = len(dependlist)
+        for i, lv in enumerate(dependlist):
+            LayerVersionDependency.objects.filter(layer_version=lv).delete()
+            for lvd in dependlist[lv]:
+                LayerVersionDependency.objects.get_or_create(layer_version=lv,
+                                                             depends_on=lvd)
+            self.mini_progress("Layer version dependencies", i, total)
+
+        # update machines
+        logger.info("Fetching machine information")
+        machines_info = _get_json_response(
+            apilinks['machines'] + "?filter=layerbranch__branch__name:%s" %
+            "OR".join(whitelist_branch_names))
+
+        total = len(machines_info)
+        for i, mi in enumerate(machines_info):
+            mo, created = Machine.objects.get_or_create(
+                name=mi['name'],
+                layer_version=Layer_Version.objects.get(
+                    pk=li_layer_branch_id_to_toaster_lv_id[mi['layerbranch']]))
+            mo.up_date = mi['updated']
+            mo.name = mi['name']
+            mo.description = mi['description']
+            mo.save()
+            self.mini_progress("machines", i, total)
+
+        # update recipes; paginate by layer version / layer branch
+        logger.info("Fetching recipe information")
+        recipes_info = _get_json_response(
+            apilinks['recipes'] + "?filter=layerbranch__branch__name:%s" %
+            "OR".join(whitelist_branch_names))
+
+        total = len(recipes_info)
+        for i, ri in enumerate(recipes_info):
+            try:
+                lv_id = li_layer_branch_id_to_toaster_lv_id[ri['layerbranch']]
+                lv = Layer_Version.objects.get(pk=lv_id)
+
+                ro, created = Recipe.objects.get_or_create(
+                    layer_version=lv,
+                    name=ri['pn']
+                )
+
+                ro.layer_version = lv
+                ro.up_date = ri['updated']
+                ro.name = ri['pn']
+                ro.version = ri['pv']
+                ro.summary = ri['summary']
+                ro.description = ri['description']
+                ro.section = ri['section']
+                ro.license = ri['license']
+                ro.homepage = ri['homepage']
+                ro.bugtracker = ri['bugtracker']
+                ro.file_path = ri['filepath'] + "/" + ri['filename']
+                if 'inherits' in ri:
+                    ro.is_image = 'image' in ri['inherits'].split()
+                else:  # workaround for old style layer index
+                    ro.is_image = "-image-" in ri['pn']
+                ro.save()
+            except Exception as e:
+                logger.warning("Failed saving recipe %s", e)
+
+            self.mini_progress("recipes", i, total)
+
+        os.system('setterm -cursor on')
 
     def handle_noargs(self, **options):
-        for ls in LayerSource.objects.all():
-            ls.update()
+        self.update()

bitbake/lib/toaster/orm/migrations/0008_refactor_artifact_models.py

@@ -0,0 +1,39 @@
+# -*- coding: utf-8 -*-
+from __future__ import unicode_literals
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('orm', '0007_auto_20160523_1446'),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name='TargetKernelFile',
+            fields=[
+                ('id', models.AutoField(auto_created=True, primary_key=True, verbose_name='ID', serialize=False)),
+                ('file_name', models.FilePathField()),
+                ('file_size', models.IntegerField()),
+                ('target', models.ForeignKey(to='orm.Target')),
+            ],
+        ),
+        migrations.CreateModel(
+            name='TargetSDKFile',
+            fields=[
+                ('id', models.AutoField(auto_created=True, primary_key=True, verbose_name='ID', serialize=False)),
+                ('file_name', models.FilePathField()),
+                ('file_size', models.IntegerField()),
+                ('target', models.ForeignKey(to='orm.Target')),
+            ],
+        ),
+        migrations.RemoveField(
+            model_name='buildartifact',
+            name='build',
+        ),
+        migrations.DeleteModel(
+            name='BuildArtifact',
+        ),
+    ]

bitbake/lib/toaster/orm/migrations/0009_target_package_manifest_path.py

@@ -0,0 +1,19 @@
+# -*- coding: utf-8 -*-
+from __future__ import unicode_literals
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('orm', '0008_refactor_artifact_models'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='target',
+            name='package_manifest_path',
+            field=models.CharField(null=True, max_length=500),
+        ),
+    ]

bitbake/lib/toaster/orm/migrations/0010_delete_layer_source_references.py

@@ -0,0 +1,118 @@
+# -*- coding: utf-8 -*-
+from __future__ import unicode_literals
+
+from django.db import migrations, models
+import django.utils.timezone
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('orm', '0009_target_package_manifest_path'),
+    ]
+
+    operations = [
+        migrations.AlterUniqueTogether(
+            name='releaselayersourcepriority',
+            unique_together=set([]),
+        ),
+        migrations.RemoveField(
+            model_name='releaselayersourcepriority',
+            name='layer_source',
+        ),
+        migrations.RemoveField(
+            model_name='releaselayersourcepriority',
+            name='release',
+        ),
+        migrations.DeleteModel(
+            name='ImportedLayerSource',
+        ),
+        migrations.DeleteModel(
+            name='LayerIndexLayerSource',
+        ),
+        migrations.DeleteModel(
+            name='LocalLayerSource',
+        ),
+        migrations.RemoveField(
+            model_name='recipe',
+            name='layer_source',
+        ),
+        migrations.RemoveField(
+            model_name='recipe',
+            name='up_id',
+        ),
+        migrations.AlterField(
+            model_name='layer',
+            name='up_date',
+            field=models.DateTimeField(default=django.utils.timezone.now, null=True),
+        ),
+        migrations.AlterField(
+            model_name='layer_version',
+            name='layer_source',
+            field=models.IntegerField(default=0, choices=[(0, 'local'), (1, 'layerindex'), (2, 'imported'), (3, 'build')]),
+        ),
+        migrations.AlterField(
+            model_name='layer_version',
+            name='up_date',
+            field=models.DateTimeField(default=django.utils.timezone.now, null=True),
+        ),
+        migrations.AlterUniqueTogether(
+            name='branch',
+            unique_together=set([]),
+        ),
+        migrations.AlterUniqueTogether(
+            name='layer',
+            unique_together=set([]),
+        ),
+        migrations.AlterUniqueTogether(
+            name='layer_version',
+            unique_together=set([]),
+        ),
+        migrations.AlterUniqueTogether(
+            name='layerversiondependency',
+            unique_together=set([]),
+        ),
+        migrations.AlterUniqueTogether(
+            name='machine',
+            unique_together=set([]),
+        ),
+        migrations.DeleteModel(
+            name='ReleaseLayerSourcePriority',
+        ),
+        migrations.RemoveField(
+            model_name='branch',
+            name='layer_source',
+        ),
+        migrations.RemoveField(
+            model_name='branch',
+            name='up_id',
+        ),
+        migrations.RemoveField(
+            model_name='layer',
+            name='layer_source',
+        ),
+        migrations.RemoveField(
+            model_name='layer',
+            name='up_id',
+        ),
+        migrations.RemoveField(
+            model_name='layer_version',
+            name='up_id',
+        ),
+        migrations.RemoveField(
+            model_name='layerversiondependency',
+            name='layer_source',
+        ),
+        migrations.RemoveField(
+            model_name='layerversiondependency',
+            name='up_id',
+        ),
+        migrations.RemoveField(
+            model_name='machine',
+            name='layer_source',
+        ),
+        migrations.RemoveField(
+            model_name='machine',
+            name='up_id',
+        ),
+    ]

bitbake/lib/toaster/orm/migrations/0011_delete_layersource.py

@@ -0,0 +1,17 @@
+# -*- coding: utf-8 -*-
+from __future__ import unicode_literals
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('orm', '0010_delete_layer_source_references'),
+    ]
+
+    operations = [
+        migrations.DeleteModel(
+            name='LayerSource',
+        ),
+    ]

bitbake/lib/toaster/orm/migrations/0012_use_release_instead_of_up_branch.py

@@ -0,0 +1,62 @@
+# -*- coding: utf-8 -*-
+from __future__ import unicode_literals
+
+from django.db import migrations, models
+from django.db.models import Q
+
+
+def branch_to_release(apps, schema_editor):
+    Layer_Version = apps.get_model('orm', 'Layer_Version')
+    Release = apps.get_model('orm', 'Release')
+
+    print("Converting all layer version up_branches to releases")
+    # Find all the layer versions which have an upbranch and convert them to
+    # the release that they're for.
+    for layer_version in Layer_Version.objects.filter(
+            Q(release=None) & ~Q(up_branch=None)):
+        try:
+            # HEAD and local are equivalent
+            if "HEAD" in layer_version.up_branch.name:
+                release = Release.objects.get(name="local")
+                layer_version.commit = "HEAD"
+                layer_version.branch = "HEAD"
+            else:
+                release = Release.objects.get(
+                    name=layer_version.up_branch.name)
+
+            layer_version.release = release
+            layer_version.save()
+        except Exception as e:
+            print("Couldn't work out an appropriate release for %s "
+                  "the up_branch was %s "
+                  "user the django admin interface to correct it" %
+                  (layer_version.layer.name, layer_version.up_branch.name))
+            print(e)
+
+            continue
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('orm', '0011_delete_layersource'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='layer_version',
+            name='release',
+            field=models.ForeignKey(to='orm.Release', default=None, null=True),
+        ),
+        migrations.RunPython(branch_to_release,
+                             reverse_code=migrations.RunPython.noop),
+
+        migrations.RemoveField(
+            model_name='layer_version',
+            name='up_branch',
+        ),
+
+        migrations.DeleteModel(
+            name='Branch',
+        ),
+    ]

bitbake/lib/toaster/orm/migrations/0013_recipe_parse_progress_fields.py

@@ -0,0 +1,24 @@
+# -*- coding: utf-8 -*-
+from __future__ import unicode_literals
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('orm', '0012_use_release_instead_of_up_branch'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='build',
+            name='recipes_parsed',
+            field=models.IntegerField(default=0),
+        ),
+        migrations.AddField(
+            model_name='build',
+            name='recipes_to_parse',
+            field=models.IntegerField(default=1),
+        ),
+    ]

bitbake/lib/toaster/orm/migrations/0014_allow_empty_buildname.py

@@ -0,0 +1,19 @@
+# -*- coding: utf-8 -*-
+from __future__ import unicode_literals
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('orm', '0013_recipe_parse_progress_fields'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='build',
+            name='build_name',
+            field=models.CharField(default='', max_length=100),
+        ),
+    ]

bitbake/lib/toaster/orm/migrations/0015_layer_local_source_dir.py

@@ -0,0 +1,19 @@
+# -*- coding: utf-8 -*-
+from __future__ import unicode_literals
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('orm', '0014_allow_empty_buildname'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='layer',
+            name='local_source_dir',
+            field=models.TextField(null=True, default=None),
+        ),
+    ]

bitbake/lib/toaster/orm/tests.py

@@ -1,180 +0,0 @@
-#! /usr/bin/env python
-# ex:ts=4:sw=4:sts=4:et
-# -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*-
-#
-# BitBake Toaster Implementation
-#
-# Copyright (C) 2013-2015 Intel Corporation
-#
-# This program is free software; you can redistribute it and/or modify
-# it under the terms of the GNU General Public License version 2 as
-# published by the Free Software Foundation.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License along
-# with this program; if not, write to the Free Software Foundation, Inc.,
-# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
-
-"""Test cases for Toaster ORM."""
-
-from django.test import TestCase, TransactionTestCase
-from orm.models import LocalLayerSource, LayerIndexLayerSource, ImportedLayerSource, LayerSource
-from orm.models import Branch, LayerVersionDependency
-
-from orm.models import Project, Layer, Layer_Version, Branch, ProjectLayer
-from orm.models import Release, ReleaseLayerSourcePriority, BitbakeVersion
-
-from django.db import IntegrityError
-
-import os
-
-# set TTS_LAYER_INDEX to the base url to use a different instance of the layer index
-
-class LayerSourceVerifyInheritanceSaveLoad(TestCase):
-    """
-    Tests to verify inheritance for the LayerSource proxy-inheritance classes.
-    """
-    def test_object_creation(self):
-        """Test LayerSource object creation."""
-        for name, sourcetype in [("a1", LayerSource.TYPE_LOCAL),
-                                 ("a2", LayerSource.TYPE_LAYERINDEX),
-                                 ("a3", LayerSource.TYPE_IMPORTED)]:
-            LayerSource.objects.create(name=name, sourcetype=sourcetype)
-
-        objects = LayerSource.objects.all()
-        self.assertTrue(isinstance(objects[0], LocalLayerSource))
-        self.assertTrue(isinstance(objects[1], LayerIndexLayerSource))
-        self.assertTrue(isinstance(objects[2], ImportedLayerSource))
-
-    def test_duplicate_error(self):
-        """Test creation of duplicate LayerSource objects."""
-        stype = LayerSource.TYPE_LOCAL
-        LayerSource.objects.create(name="a1", sourcetype=stype)
-        with self.assertRaises(IntegrityError):
-            LayerSource.objects.create(name="a1", sourcetype=stype)
-
-
-class LILSUpdateTestCase(TransactionTestCase):
-    """Test Layer Source update."""
-
-    def setUp(self):
-        """Create release."""
-        bbv = BitbakeVersion.objects.create(\
-                  name="master", giturl="git://git.openembedded.org/bitbake")
-        Release.objects.create(name="default-release", bitbake_version=bbv,
-                               branch_name="master")
-
-    def test_update(self):
-        """Check if LayerSource.update can fetch branches."""
-        url = os.getenv("TTS_LAYER_INDEX",
-                        default="http://layers.openembedded.org/")
-
-        lsobj = LayerSource.objects.create(\
-                    name="b1", sourcetype=LayerSource.TYPE_LAYERINDEX,
-                    apiurl=url + "layerindex/api/")
-        lsobj.update()
-        self.assertTrue(lsobj.branch_set.all().count() > 0,
-                        "no branches fetched")
-
-class LayerVersionEquivalenceTestCase(TestCase):
-    """Verify Layer_Version priority selection."""
-
-    def setUp(self):
-        """Create required objects."""
-        # create layer source
-        self.lsrc = LayerSource.objects.create(name="dummy-layersource",
-                                               sourcetype=LayerSource.TYPE_LOCAL)
-        # create release
-        bbv = BitbakeVersion.objects.create(\
-                  name="master", giturl="git://git.openembedded.org/bitbake")
-        self.release = Release.objects.create(name="default-release",
-                                              bitbake_version=bbv,
-                                              branch_name="master")
-        # attach layer source to release
-        ReleaseLayerSourcePriority.objects.create(\
-            release=self.release, layer_source=self.lsrc, priority=1)
-
-        # create a layer version for the layer on the specified branch
-        self.layer = Layer.objects.create(name="meta-testlayer",
-                                          layer_source=self.lsrc)
-        self.branch = Branch.objects.create(name="master", layer_source=self.lsrc)
-        self.lver = Layer_Version.objects.create(\
-            layer=self.layer, layer_source=self.lsrc, up_branch=self.branch)
-
-        # create project and project layer
-        self.project = Project.objects.create_project(name="test-project",
-                                                      release=self.release)
-        ProjectLayer.objects.create(project=self.project,
-                                    layercommit=self.lver)
-
-        # create spoof layer that should not appear in the search results
-        layer = Layer.objects.create(name="meta-notvalid",
-                                     layer_source=self.lsrc)
-        self.lver2 = Layer_Version.objects.create(layer=layer,
-                                                  layer_source=self.lsrc,
-                                                  up_branch=self.branch)
-
-    def test_single_layersource(self):
-        """
-        When we have a single layer version,
-        get_equivalents_wpriority() should return a list with
-        just this layer_version.
-        """
-        equivqs = self.lver.get_equivalents_wpriority(self.project)
-        self.assertEqual(list(equivqs), [self.lver])
-
-    def test_dual_layersource(self):
-        """
-        If we have two layers with the same name, from different layer sources,
-        we expect both layers in, in increasing priority of the layer source.
-        """
-        lsrc2 = LayerSource.objects.create(\
-                    name="dummy-layersource2",
-                    sourcetype=LayerSource.TYPE_LOCAL,
-                    apiurl="test")
-
-        # assign a lower priority for the second layer source
-        self.release.releaselayersourcepriority_set.create(layer_source=lsrc2,
-                                                           priority=2)
-
-        # create a new layer_version for a layer with the same name
-        # coming from the second layer source
-        layer2 = Layer.objects.create(name="meta-testlayer",
-                                      layer_source=lsrc2)
-        lver2 = Layer_Version.objects.create(layer=layer2, layer_source=lsrc2,
-                                             up_branch=self.branch)
-
-        # expect two layer versions, in the priority order
-        equivqs = self.lver.get_equivalents_wpriority(self.project)
-        self.assertEqual(list(equivqs), [lver2, self.lver])
-
-    def test_compatible_layer_versions(self):
-        """
-        When we have a 2 layer versions, get_all_compatible_layerversions()
-        should return a queryset with both.
-        """
-        compat_lv = self.project.get_all_compatible_layer_versions()
-        self.assertEqual(list(compat_lv), [self.lver, self.lver2])
-
-    def test_layerversion_get_alldeps(self):
-        """Test Layer_Version.get_alldeps API."""
-        lvers = {}
-        for i in range(10):
-            name = "layer%d" % i
-            lvers[name] = Layer_Version.objects.create(layer=Layer.objects.create(name=name),
-                                                       project=self.project)
-            if i:
-                LayerVersionDependency.objects.create(layer_version=lvers["layer%d" % (i - 1)],
-                                                      depends_on=lvers[name])
-                # Check dinamically added deps
-                self.assertEqual(lvers['layer0'].get_alldeps(self.project.id),
-                                 [lvers['layer%d' % n] for n in range(1, i+1)])
-
-        # Check chain of deps created in previous loop
-        for i in range(10):
-            self.assertEqual(lvers['layer%d' % i].get_alldeps(self.project.id),
-                             [lvers['layer%d' % n] for n in range(i+1, 10)])

bitbake/lib/toaster/tests/browser/README

@@ -13,7 +13,7 @@ a Selenium test report with a version-specific format.
 To run tests against Chrome:
 
 * Download chromedriver for your host OS from
-  https://code.google.com/p/chromedriver/downloads/list
+  https://sites.google.com/a/chromium.org/chromedriver/downloads
 * On *nix systems, put chromedriver on PATH
 * On Windows, put chromedriver.exe in the same directory as chrome.exe
 
@@ -24,15 +24,30 @@ To run tests against PhantomJS (headless):
 * On *nix systems, put phantomjs on PATH
 * Not tested on Windows
 
-Firefox should work without requiring additional software to be installed.
+To run tests against Firefox, you may need to install the Marionette driver,
+depending on how new your version of Firefox is. One clue that you need to do
+this is if you see an exception like:
 
-The test case will instantiate a Selenium driver set by the
+  selenium.common.exceptions.WebDriverException: Message: The browser
+  appears to have exited before we could connect. If you specified
+  a log_file in the FirefoxBinary constructor, check it for details.
+
+See https://developer.mozilla.org/en-US/docs/Mozilla/QA/Marionette/WebDriver
+for installation instructions. Ensure that the Marionette executable (renamed
+as wires on Linux or wires.exe on Windows) is on your PATH; and use "marionette"
+as the browser string passed via TOASTER_TESTS_BROWSER (see below).
+
+(Note: The Toaster tests have been checked against Firefox 47 with the
+Marionette driver.)
+
+The test cases will instantiate a Selenium driver set by the
 TOASTER_TESTS_BROWSER environment variable, or Chrome if this is not specified.
 
 Available drivers:
 
 * chrome (default)
 * firefox
+* marionette (for newer Firefoxes)
 * ie
 * phantomjs
 

bitbake/lib/toaster/tests/browser/selenium_helpers.py

@@ -27,179 +27,8 @@
 Helper methods for creating Toaster Selenium tests which run within
 the context of Django unit tests.
 """
-
-import os
-import time
-
 from django.contrib.staticfiles.testing import StaticLiveServerTestCase
-from selenium import webdriver
-from selenium.webdriver.support.ui import WebDriverWait
-from selenium.common.exceptions import NoSuchElementException, \
-        StaleElementReferenceException, TimeoutException
+from tests.browser.selenium_helpers_base import SeleniumTestCaseBase
 
-def create_selenium_driver(browser='chrome'):
-    # set default browser string based on env (if available)
-    env_browser = os.environ.get('TOASTER_TESTS_BROWSER')
-    if env_browser:
-        browser = env_browser
-
-    if browser == 'chrome':
-        return webdriver.Chrome(
-            service_args=["--verbose", "--log-path=selenium.log"]
-        )
-    elif browser == 'firefox':
-        return webdriver.Firefox()
-    elif browser == 'ie':
-        return webdriver.Ie()
-    elif browser == 'phantomjs':
-        return webdriver.PhantomJS()
-    else:
-        msg = 'Selenium driver for browser %s is not available' % browser
-        raise RuntimeError(msg)
-
-class Wait(WebDriverWait):
-    """
-    Subclass of WebDriverWait with predetermined timeout and poll
-    frequency. Also deals with a wider variety of exceptions.
-    """
-    _TIMEOUT = 10
-    _POLL_FREQUENCY = 0.5
-
-    def __init__(self, driver):
-        super(Wait, self).__init__(driver, self._TIMEOUT, self._POLL_FREQUENCY)
-
-    def until(self, method, message=''):
-        """
-        Calls the method provided with the driver as an argument until the
-        return value is not False.
-        """
-
-        end_time = time.time() + self._timeout
-        while True:
-            try:
-                value = method(self._driver)
-                if value:
-                    return value
-            except NoSuchElementException:
-                pass
-            except StaleElementReferenceException:
-                pass
-
-            time.sleep(self._poll)
-            if time.time() > end_time:
-                break
-
-        raise TimeoutException(message)
-
-    def until_not(self, method, message=''):
-        """
-        Calls the method provided with the driver as an argument until the
-        return value is False.
-        """
-
-        end_time = time.time() + self._timeout
-        while True:
-            try:
-                value = method(self._driver)
-                if not value:
-                    return value
-            except NoSuchElementException:
-                return True
-            except StaleElementReferenceException:
-                pass
-
-            time.sleep(self._poll)
-            if time.time() > end_time:
-                break
-
-        raise TimeoutException(message)
-
-class SeleniumTestCase(StaticLiveServerTestCase):
-    """
-    NB StaticLiveServerTestCase is used as the base test case so that
-    static files are served correctly in a Selenium test run context; see
-    https://docs.djangoproject.com/en/1.9/ref/contrib/staticfiles/#specialized-test-case-to-support-live-testing
-    """
-
-    @classmethod
-    def setUpClass(cls):
-        """ Create a webdriver driver at the class level """
-
-        super(SeleniumTestCase, cls).setUpClass()
-
-        # instantiate the Selenium webdriver once for all the test methods
-        # in this test case
-        cls.driver = create_selenium_driver()
-        cls.driver.maximize_window()
-
-    @classmethod
-    def tearDownClass(cls):
-        """ Clean up webdriver driver """
-
-        cls.driver.quit()
-        super(SeleniumTestCase, cls).tearDownClass()
-
-    def get(self, url):
-        """
-        Selenium requires absolute URLs, so convert Django URLs returned
-        by resolve() or similar to absolute ones and get using the
-        webdriver instance.
-
-        url: a relative URL
-        """
-        abs_url = '%s%s' % (self.live_server_url, url)
-        self.driver.get(abs_url)
-
-    def find(self, selector):
-        """ Find single element by CSS selector """
-        return self.driver.find_element_by_css_selector(selector)
-
-    def find_all(self, selector):
-        """ Find all elements matching CSS selector """
-        return self.driver.find_elements_by_css_selector(selector)
-
-    def focused_element(self):
-        """ Return the element which currently has focus on the page """
-        return self.driver.switch_to.active_element
-
-    def wait_until_present(self, selector):
-        """ Wait until element matching CSS selector is on the page """
-        is_present = lambda driver: self.find(selector)
-        msg = 'An element matching "%s" should be on the page' % selector
-        element = Wait(self.driver).until(is_present, msg)
-        return element
-
-    def wait_until_visible(self, selector):
-        """ Wait until element matching CSS selector is visible on the page """
-        is_visible = lambda driver: self.find(selector).is_displayed()
-        msg = 'An element matching "%s" should be visible' % selector
-        Wait(self.driver).until(is_visible, msg)
-        return self.find(selector)
-
-    def wait_until_focused(self, selector):
-        """ Wait until element matching CSS selector has focus """
-        is_focused = \
-            lambda driver: self.find(selector) == self.focused_element()
-        msg = 'An element matching "%s" should be focused' % selector
-        Wait(self.driver).until(is_focused, msg)
-        return self.find(selector)
-
-    def enter_text(self, selector, value):
-        """ Insert text into element matching selector """
-        # note that keyup events don't occur until the element is clicked
-        # (in the case of <input type="text"...>, for example), so simulate
-        # user clicking the element before inserting text into it
-        field = self.click(selector)
-
-        field.send_keys(value)
-        return field
-
-    def click(self, selector):
-        """ Click on element which matches CSS selector """
-        element = self.wait_until_visible(selector)
-        element.click()
-        return element
-
-    def get_page_source(self):
-        """ Get raw HTML for the current page """
-        return self.driver.page_source
+class SeleniumTestCase(SeleniumTestCaseBase, StaticLiveServerTestCase):
+    pass

bitbake/lib/toaster/tests/browser/selenium_helpers_base.py

@@ -0,0 +1,218 @@
+#! /usr/bin/env python
+# ex:ts=4:sw=4:sts=4:et
+# -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*-
+#
+# BitBake Toaster Implementation
+#
+# Copyright (C) 2013-2016 Intel Corporation
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License version 2 as
+# published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License along
+# with this program; if not, write to the Free Software Foundation, Inc.,
+# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+#
+# The Wait class and some of SeleniumDriverHelper and SeleniumTestCase are
+# modified from Patchwork, released under the same licence terms as Toaster:
+# https://github.com/dlespiau/patchwork/blob/master/patchwork/tests.browser.py
+
+"""
+Helper methods for creating Toaster Selenium tests which run within
+the context of Django unit tests.
+"""
+
+import os
+import time
+import unittest
+
+from django.contrib.staticfiles.testing import StaticLiveServerTestCase
+from selenium import webdriver
+from selenium.webdriver.support.ui import WebDriverWait
+from selenium.webdriver.common.desired_capabilities import DesiredCapabilities
+from selenium.common.exceptions import NoSuchElementException, \
+        StaleElementReferenceException, TimeoutException
+
+def create_selenium_driver(browser='chrome'):
+    # set default browser string based on env (if available)
+    env_browser = os.environ.get('TOASTER_TESTS_BROWSER')
+    if env_browser:
+        browser = env_browser
+
+    if browser == 'chrome':
+        return webdriver.Chrome(
+            service_args=["--verbose", "--log-path=selenium.log"]
+        )
+    elif browser == 'firefox':
+        return webdriver.Firefox()
+    elif browser == 'marionette':
+        capabilities = DesiredCapabilities.FIREFOX
+        capabilities['marionette'] = True
+        return webdriver.Firefox(capabilities=capabilities)
+    elif browser == 'ie':
+        return webdriver.Ie()
+    elif browser == 'phantomjs':
+        return webdriver.PhantomJS()
+    else:
+        msg = 'Selenium driver for browser %s is not available' % browser
+        raise RuntimeError(msg)
+
+class Wait(WebDriverWait):
+    """
+    Subclass of WebDriverWait with predetermined timeout and poll
+    frequency. Also deals with a wider variety of exceptions.
+    """
+    _TIMEOUT = 10
+    _POLL_FREQUENCY = 0.5
+
+    def __init__(self, driver):
+        super(Wait, self).__init__(driver, self._TIMEOUT, self._POLL_FREQUENCY)
+
+    def until(self, method, message=''):
+        """
+        Calls the method provided with the driver as an argument until the
+        return value is not False.
+        """
+
+        end_time = time.time() + self._timeout
+        while True:
+            try:
+                value = method(self._driver)
+                if value:
+                    return value
+            except NoSuchElementException:
+                pass
+            except StaleElementReferenceException:
+                pass
+
+            time.sleep(self._poll)
+            if time.time() > end_time:
+                break
+
+        raise TimeoutException(message)
+
+    def until_not(self, method, message=''):
+        """
+        Calls the method provided with the driver as an argument until the
+        return value is False.
+        """
+
+        end_time = time.time() + self._timeout
+        while True:
+            try:
+                value = method(self._driver)
+                if not value:
+                    return value
+            except NoSuchElementException:
+                return True
+            except StaleElementReferenceException:
+                pass
+
+            time.sleep(self._poll)
+            if time.time() > end_time:
+                break
+
+        raise TimeoutException(message)
+
+class SeleniumTestCaseBase(unittest.TestCase):
+    """
+    NB StaticLiveServerTestCase is used as the base test case so that
+    static files are served correctly in a Selenium test run context; see
+    https://docs.djangoproject.com/en/1.9/ref/contrib/staticfiles/#specialized-test-case-to-support-live-testing
+    """
+
+    @classmethod
+    def setUpClass(cls):
+        """ Create a webdriver driver at the class level """
+
+        super(SeleniumTestCaseBase, cls).setUpClass()
+
+        # instantiate the Selenium webdriver once for all the test methods
+        # in this test case
+        cls.driver = create_selenium_driver()
+        cls.driver.maximize_window()
+
+    @classmethod
+    def tearDownClass(cls):
+        """ Clean up webdriver driver """
+
+        cls.driver.quit()
+        super(SeleniumTestCaseBase, cls).tearDownClass()
+
+    def get(self, url):
+        """
+        Selenium requires absolute URLs, so convert Django URLs returned
+        by resolve() or similar to absolute ones and get using the
+        webdriver instance.
+
+        url: a relative URL
+        """
+        abs_url = '%s%s' % (self.live_server_url, url)
+        self.driver.get(abs_url)
+
+    def find(self, selector):
+        """ Find single element by CSS selector """
+        return self.driver.find_element_by_css_selector(selector)
+
+    def find_all(self, selector):
+        """ Find all elements matching CSS selector """
+        return self.driver.find_elements_by_css_selector(selector)
+
+    def element_exists(self, selector):
+        """
+        Return True if one element matching selector exists,
+        False otherwise
+        """
+        return len(self.find_all(selector)) == 1
+
+    def focused_element(self):
+        """ Return the element which currently has focus on the page """
+        return self.driver.switch_to.active_element
+
+    def wait_until_present(self, selector):
+        """ Wait until element matching CSS selector is on the page """
+        is_present = lambda driver: self.find(selector)
+        msg = 'An element matching "%s" should be on the page' % selector
+        element = Wait(self.driver).until(is_present, msg)
+        return element
+
+    def wait_until_visible(self, selector):
+        """ Wait until element matching CSS selector is visible on the page """
+        is_visible = lambda driver: self.find(selector).is_displayed()
+        msg = 'An element matching "%s" should be visible' % selector
+        Wait(self.driver).until(is_visible, msg)
+        return self.find(selector)
+
+    def wait_until_focused(self, selector):
+        """ Wait until element matching CSS selector has focus """
+        is_focused = \
+            lambda driver: self.find(selector) == self.focused_element()
+        msg = 'An element matching "%s" should be focused' % selector
+        Wait(self.driver).until(is_focused, msg)
+        return self.find(selector)
+
+    def enter_text(self, selector, value):
+        """ Insert text into element matching selector """
+        # note that keyup events don't occur until the element is clicked
+        # (in the case of <input type="text"...>, for example), so simulate
+        # user clicking the element before inserting text into it
+        field = self.click(selector)
+
+        field.send_keys(value)
+        return field
+
+    def click(self, selector):
+        """ Click on element which matches CSS selector """
+        element = self.wait_until_visible(selector)
+        element.click()
+        return element
+
+    def get_page_source(self):
+        """ Get raw HTML for the current page """
+        return self.driver.page_source

bitbake/lib/toaster/tests/browser/test_all_builds_page.py

@@ -58,6 +58,13 @@ class TestAllBuildsPage(SeleniumTestCase):
             'outcome': Build.SUCCEEDED
         }
 
+        self.project1_build_failure = {
+            'project': self.project1,
+            'started_on': now,
+            'completed_on': now,
+            'outcome': Build.FAILED
+        }
+
         self.default_project_build_success = {
             'project': self.default_project,
             'started_on': now,
@@ -65,6 +72,46 @@ class TestAllBuildsPage(SeleniumTestCase):
             'outcome': Build.SUCCEEDED
         }
 
+    def _get_build_time_element(self, build):
+        """
+        Return the HTML element containing the build time for a build
+        in the recent builds area
+        """
+        selector = 'div[data-latest-build-result="%s"] ' \
+            '[data-role="data-recent-build-buildtime-field"]' % build.id
+
+        # because this loads via Ajax, wait for it to be visible
+        self.wait_until_present(selector)
+
+        build_time_spans = self.find_all(selector)
+
+        self.assertEqual(len(build_time_spans), 1)
+
+        return build_time_spans[0]
+
+    def _get_row_for_build(self, build):
+        """ Get the table row for the build from the all builds table """
+        self.wait_until_present('#allbuildstable')
+
+        rows = self.find_all('#allbuildstable tr')
+
+        # look for the row with a download link on the recipe which matches the
+        # build ID
+        url = reverse('builddashboard', args=(build.id,))
+        selector = 'td.target a[href="%s"]' % url
+
+        found_row = None
+        for row in rows:
+
+            outcome_links = row.find_elements_by_css_selector(selector)
+            if len(outcome_links) == 1:
+                found_row = row
+                break
+
+        self.assertNotEqual(found_row, None)
+
+        return found_row
+
     def test_show_tasks_with_suffix(self):
         """ Task should be shown as suffix on build name """
         build = Build.objects.create(**self.project1_build_success)
@@ -97,13 +144,13 @@ class TestAllBuildsPage(SeleniumTestCase):
         self.get(url)
 
         # shouldn't see a rebuild button for command-line builds
-        selector = 'div[data-latest-build-result="%s"] a.run-again-btn' % default_build.id
+        selector = 'div[data-latest-build-result="%s"] .rebuild-btn' % default_build.id
         run_again_button = self.find_all(selector)
         self.assertEqual(len(run_again_button), 0,
                          'should not see a rebuild button for cli builds')
 
         # should see a rebuild button for non-command-line builds
-        selector = 'div[data-latest-build-result="%s"] a.run-again-btn' % build1.id
+        selector = 'div[data-latest-build-result="%s"] .rebuild-btn' % build1.id
         run_again_button = self.find_all(selector)
         self.assertEqual(len(run_again_button), 1,
                          'should see a rebuild button for non-cli builds')
@@ -142,3 +189,45 @@ class TestAllBuildsPage(SeleniumTestCase):
             else:
                 msg = 'found unexpected project name cell in all builds table'
                 self.fail(msg)
+
+    def test_builds_time_links(self):
+        """
+        Successful builds should have links on the time column and in the
+        recent builds area; failed builds should not have links on the time column,
+        or in the recent builds area
+        """
+        build1 = Build.objects.create(**self.project1_build_success)
+        build2 = Build.objects.create(**self.project1_build_failure)
+
+        # add some targets to these builds so they have recipe links
+        # (and so we can find the row in the ToasterTable corresponding to
+        # a particular build)
+        Target.objects.create(build=build1, target='foo')
+        Target.objects.create(build=build2, target='bar')
+
+        url = reverse('all-builds')
+        self.get(url)
+
+        # test recent builds area for successful build
+        element = self._get_build_time_element(build1)
+        links = element.find_elements_by_css_selector('a')
+        msg = 'should be a link on the build time for a successful recent build'
+        self.assertEquals(len(links), 1, msg)
+
+        # test recent builds area for failed build
+        element = self._get_build_time_element(build2)
+        links = element.find_elements_by_css_selector('a')
+        msg = 'should not be a link on the build time for a failed recent build'
+        self.assertEquals(len(links), 0, msg)
+
+        # test the time column for successful build
+        build1_row = self._get_row_for_build(build1)
+        links = build1_row.find_elements_by_css_selector('td.time a')
+        msg = 'should be a link on the build time for a successful build'
+        self.assertEquals(len(links), 1, msg)
+
+        # test the time column for failed build
+        build2_row = self._get_row_for_build(build2)
+        links = build2_row.find_elements_by_css_selector('td.time a')
+        msg = 'should not be a link on the build time for a failed build'
+        self.assertEquals(len(links), 0, msg)

bitbake/lib/toaster/tests/browser/test_builddashboard_page.py

@@ -25,7 +25,7 @@ from django.utils import timezone
 from tests.browser.selenium_helpers import SeleniumTestCase
 
 from orm.models import Project, Release, BitbakeVersion, Build, LogMessage
-from orm.models import Layer, Layer_Version, Recipe, CustomImageRecipe
+from orm.models import Layer, Layer_Version, Recipe, CustomImageRecipe, Variable
 
 class TestBuildDashboardPage(SeleniumTestCase):
     """ Tests for the build dashboard /build/X """
@@ -42,11 +42,27 @@ class TestBuildDashboardPage(SeleniumTestCase):
 
         self.build1 = Build.objects.create(project=project,
                                            started_on=now,
-                                           completed_on=now)
+                                           completed_on=now,
+                                           outcome=Build.SUCCEEDED)
 
         self.build2 = Build.objects.create(project=project,
                                            started_on=now,
-                                           completed_on=now)
+                                           completed_on=now,
+                                           outcome=Build.SUCCEEDED)
+
+        self.build3 = Build.objects.create(project=project,
+                                           started_on=now,
+                                           completed_on=now,
+                                           outcome=Build.FAILED)
+
+        # add Variable objects to the successful builds, as this is the criterion
+        # used to determine whether the left-hand panel should be displayed
+        Variable.objects.create(build=self.build1,
+                                variable_name='Foo',
+                                variable_value='Bar')
+        Variable.objects.create(build=self.build2,
+                                variable_name='Foo',
+                                variable_value='Bar')
 
         # exception
         msg1 = 'an exception was thrown'
@@ -64,6 +80,22 @@ class TestBuildDashboardPage(SeleniumTestCase):
             message=msg2
         )
 
+        # error on the failed build
+        msg3 = 'an error occurred'
+        self.error_message = LogMessage.objects.create(
+            build=self.build3,
+            level=LogMessage.ERROR,
+            message=msg3
+        )
+
+        # warning on the failed build
+        msg4 = 'DANGER WILL ROBINSON'
+        self.warning_message = LogMessage.objects.create(
+            build=self.build3,
+            level=LogMessage.WARNING,
+            message=msg4
+        )
+
         # recipes related to the build, for testing the edit custom image/new
         # custom image buttons
         layer = Layer.objects.create(name='alayer')
@@ -71,6 +103,11 @@ class TestBuildDashboardPage(SeleniumTestCase):
             layer=layer, build=self.build1
         )
 
+        # non-image recipes related to a build, for testing the new custom
+        # image button
+        layer_version2 = Layer_Version.objects.create(layer=layer,
+            build=self.build3)
+
         # image recipes
         self.image_recipe1 = Recipe.objects.create(
             name='recipeA',
@@ -142,36 +179,45 @@ class TestBuildDashboardPage(SeleniumTestCase):
         self._get_build_dashboard(build)
         return self.find_all('#errors div.alert-danger')
 
-    def _check_for_log_message(self, build, log_message):
+    def _check_for_log_message(self, message_elements, log_message):
+        """
+        Check that the LogMessage <log_message> has a representation in
+        the HTML elements <message_elements>.
+
+        message_elements: WebElements representing the log messages shown
+        in the build dashboard; each should have a <pre> element inside
+        it with a data-log-message-id attribute
+
+        log_message: orm.models.LogMessage instance
+        """
+        expected_text = log_message.message
+        expected_pk = str(log_message.pk)
+
+        found = False
+        for element in message_elements:
+            log_message_text = element.find_element_by_tag_name('pre').text.strip()
+            text_matches = (log_message_text == expected_text)
+
+            log_message_pk = element.get_attribute('data-log-message-id')
+            id_matches = (log_message_pk == expected_pk)
+
+            if text_matches and id_matches:
+                found = True
+                break
+
+        template_vars = (expected_text, expected_pk)
+        assertion_failed_msg = 'message not found: ' \
+            'expected text "%s" and ID %s' % template_vars
+        self.assertTrue(found, assertion_failed_msg)
+
+    def _check_for_error_message(self, build, log_message):
         """
         Check whether the LogMessage instance <log_message> is
         represented as an HTML error in the dashboard page for the Build object
         build
         """
         errors = self._get_build_dashboard_errors(build)
-        self.assertEqual(len(errors), 2)
-
-        expected_text = log_message.message
-        expected_id = str(log_message.id)
-
-        found = False
-        for error in errors:
-            error_text = error.find_element_by_tag_name('pre').text
-            text_matches = (error_text == expected_text)
-
-            error_id = error.get_attribute('data-error')
-            id_matches = (error_id == expected_id)
-
-            if text_matches and id_matches:
-                found = True
-                break
-
-        template_vars = (expected_text, error_text,
-                         expected_id, error_id)
-        assertion_error_msg = 'exception not found as error: ' \
-            'expected text "%s" and got "%s"; ' \
-            'expected ID %s and got %s' % template_vars
-        self.assertTrue(found, assertion_error_msg)
+        self._check_for_log_message(errors, log_message)
 
     def _check_labels_in_modal(self, modal, expected):
         """
@@ -194,14 +240,14 @@ class TestBuildDashboardPage(SeleniumTestCase):
         LogMessages with level EXCEPTION should display in the errors
         section of the page
         """
-        self._check_for_log_message(self.build1, self.exception_message)
+        self._check_for_error_message(self.build1, self.exception_message)
 
     def test_criticals_show_as_errors(self):
         """
         LogMessages with level CRITICAL should display in the errors
         section of the page
         """
-        self._check_for_log_message(self.build1, self.critical_message)
+        self._check_for_error_message(self.build1, self.critical_message)
 
     def test_edit_custom_image_button(self):
         """
@@ -248,3 +294,54 @@ class TestBuildDashboardPage(SeleniumTestCase):
         ]
 
         self._check_labels_in_modal(modal, expected_recipes)
+
+    def test_new_custom_image_button_no_image(self):
+        """
+        Check that a build which builds non-image recipes doesn't show
+        the new custom image button on the dashboard.
+        """
+        self._get_build_dashboard(self.build3)
+        selector = '[data-role="new-custom-image-trigger"]'
+        self.assertFalse(self.element_exists(selector),
+            'new custom image button should not show for builds which ' \
+            'don\'t have any image recipes')
+
+    def test_left_panel(self):
+        """"
+        Builds which succeed should have a left panel and a build summary
+        """
+        self._get_build_dashboard(self.build1)
+
+        left_panel = self.find_all('#nav')
+        self.assertEqual(len(left_panel), 1)
+
+        build_summary = self.find_all('[data-role="build-summary-heading"]')
+        self.assertEqual(len(build_summary), 1)
+
+    def test_failed_no_left_panel(self):
+        """
+        Builds which fail should have no left panel and no build summary
+        """
+        self._get_build_dashboard(self.build3)
+
+        left_panel = self.find_all('#nav')
+        self.assertEqual(len(left_panel), 0)
+
+        build_summary = self.find_all('[data-role="build-summary-heading"]')
+        self.assertEqual(len(build_summary), 0)
+
+    def test_failed_shows_errors_and_warnings(self):
+        """
+        Failed builds should still show error and warning messages
+        """
+        self._get_build_dashboard(self.build3)
+
+        errors = self.find_all('#errors div.alert-danger')
+        self._check_for_log_message(errors, self.error_message)
+
+        # expand the warnings area
+        self.click('#warning-toggle')
+        self.wait_until_visible('#warnings div.alert-warning')
+
+        warnings = self.find_all('#warnings div.alert-warning')
+        self._check_for_log_message(warnings, self.warning_message)

bitbake/lib/toaster/tests/browser/test_builddashboard_page_artifacts.py

@@ -0,0 +1,222 @@
+#! /usr/bin/env python
+# ex:ts=4:sw=4:sts=4:et
+# -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*-
+#
+# BitBake Toaster Implementation
+#
+# Copyright (C) 2013-2016 Intel Corporation
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License version 2 as
+# published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License along
+# with this program; if not, write to the Free Software Foundation, Inc.,
+# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+from django.core.urlresolvers import reverse
+from django.utils import timezone
+
+from tests.browser.selenium_helpers import SeleniumTestCase
+
+from orm.models import Project, Release, BitbakeVersion, Build, Target, Package
+from orm.models import Target_Image_File, TargetSDKFile, TargetKernelFile
+from orm.models import Target_Installed_Package, Variable
+
+class TestBuildDashboardPageArtifacts(SeleniumTestCase):
+    """ Tests for artifacts on the build dashboard /build/X """
+
+    def setUp(self):
+        bbv = BitbakeVersion.objects.create(name='bbv1', giturl='/tmp/',
+                                            branch='master', dirpath="")
+        release = Release.objects.create(name='release1',
+                                         bitbake_version=bbv)
+        self.project = Project.objects.create_project(name='test project',
+                                                      release=release)
+
+    def _get_build_dashboard(self, build):
+        """
+        Navigate to the build dashboard for build
+        """
+        url = reverse('builddashboard', args=(build.id,))
+        self.get(url)
+
+    def _has_build_artifacts_heading(self):
+        """
+        Check whether the "Build artifacts" heading is visible (True if it
+        is, False otherwise).
+        """
+        return self.element_exists('[data-heading="build-artifacts"]')
+
+    def _has_images_menu_option(self):
+        """
+        Try to get the "Images" list element from the left-hand menu in the
+        build dashboard, and return True if it is present, False otherwise.
+        """
+        return self.element_exists('li.nav-header[data-menu-heading="images"]')
+
+    def test_no_artifacts(self):
+        """
+        If a build produced no artifacts, the artifacts heading and images
+        menu option shouldn't show.
+        """
+        now = timezone.now()
+        build = Build.objects.create(project=self.project,
+            started_on=now, completed_on=now, outcome=Build.SUCCEEDED)
+
+        Target.objects.create(is_image=False, build=build, task='',
+            target='mpfr-native')
+
+        self._get_build_dashboard(build)
+
+        # check build artifacts heading
+        msg = 'Build artifacts heading should not be displayed for non-image' \
+            'builds'
+        self.assertFalse(self._has_build_artifacts_heading(), msg)
+
+        # check "Images" option in left-hand menu (should not be there)
+        msg = 'Images option should not be shown in left-hand menu'
+        self.assertFalse(self._has_images_menu_option(), msg)
+
+    def test_sdk_artifacts(self):
+        """
+        If a build produced SDK artifacts, they should be shown, but the section
+        for image files and the images menu option should be hidden.
+
+        The packages count and size should also be hidden.
+        """
+        now = timezone.now()
+        build = Build.objects.create(project=self.project,
+            started_on=now, completed_on=timezone.now(),
+            outcome=Build.SUCCEEDED)
+
+        target = Target.objects.create(is_image=True, build=build,
+            task='populate_sdk', target='core-image-minimal')
+
+        sdk_file1 = TargetSDKFile.objects.create(target=target,
+            file_size=100000,
+            file_name='/home/foo/core-image-minimal.toolchain.sh')
+
+        sdk_file2 = TargetSDKFile.objects.create(target=target,
+            file_size=120000,
+            file_name='/home/foo/x86_64.toolchain.sh')
+
+        self._get_build_dashboard(build)
+
+        # check build artifacts heading
+        msg = 'Build artifacts heading should be displayed for SDK ' \
+            'builds which generate artifacts'
+        self.assertTrue(self._has_build_artifacts_heading(), msg)
+
+        # check "Images" option in left-hand menu (should not be there)
+        msg = 'Images option should not be shown in left-hand menu for ' \
+            'builds which didn\'t generate an image file'
+        self.assertFalse(self._has_images_menu_option(), msg)
+
+        # check links to SDK artifacts
+        sdk_artifact_links = self.find_all('[data-links="sdk-artifacts"] li')
+        self.assertEqual(len(sdk_artifact_links), 2,
+            'should be links to 2 SDK artifacts')
+
+        # package count and size should not be visible, no link on
+        # target name
+        selector = '[data-value="target-package-count"]'
+        self.assertFalse(self.element_exists(selector),
+            'package count should not be shown for non-image builds')
+
+        selector = '[data-value="target-package-size"]'
+        self.assertFalse(self.element_exists(selector),
+            'package size should not be shown for non-image builds')
+
+        selector = '[data-link="target-packages"]'
+        self.assertFalse(self.element_exists(selector),
+            'link to target packages should not be on target heading')
+
+    def test_image_artifacts(self):
+        """
+        If a build produced image files, kernel artifacts, and manifests,
+        they should all be shown, as well as the image link in the left-hand
+        menu.
+
+        The packages count and size should be shown, with a link to the
+        package display page.
+        """
+        now = timezone.now()
+        build = Build.objects.create(project=self.project,
+            started_on=now, completed_on=timezone.now(),
+            outcome=Build.SUCCEEDED)
+
+        # add a variable to the build so that it counts as "started"
+        Variable.objects.create(build=build,
+                                variable_name='Christopher',
+                                variable_value='Lee')
+
+        target = Target.objects.create(is_image=True, build=build,
+            task='', target='core-image-minimal',
+            license_manifest_path='/home/foo/license.manifest',
+            package_manifest_path='/home/foo/package.manifest')
+
+        image_file = Target_Image_File.objects.create(target=target,
+            file_name='/home/foo/core-image-minimal.ext4', file_size=9000)
+
+        kernel_file1 = TargetKernelFile.objects.create(target=target,
+            file_name='/home/foo/bzImage', file_size=2000)
+
+        kernel_file2 = TargetKernelFile.objects.create(target=target,
+            file_name='/home/foo/bzImage', file_size=2000)
+
+        package = Package.objects.create(build=build, name='foo', size=1024,
+            installed_name='foo1')
+        installed_package = Target_Installed_Package.objects.create(
+            target=target, package=package)
+
+        self._get_build_dashboard(build)
+
+        # check build artifacts heading
+        msg = 'Build artifacts heading should be displayed for image ' \
+            'builds'
+        self.assertTrue(self._has_build_artifacts_heading(), msg)
+
+        # check "Images" option in left-hand menu (should be there)
+        msg = 'Images option should be shown in left-hand menu for image builds'
+        self.assertTrue(self._has_images_menu_option(), msg)
+
+        # check link to image file
+        selector = '[data-links="image-artifacts"] li'
+        self.assertTrue(self.element_exists(selector),
+            'should be a link to the image file (selector %s)' % selector)
+
+        # check links to kernel artifacts
+        kernel_artifact_links = \
+            self.find_all('[data-links="kernel-artifacts"] li')
+        self.assertEqual(len(kernel_artifact_links), 2,
+            'should be links to 2 kernel artifacts')
+
+        # check manifest links
+        selector = 'a[data-link="license-manifest"]'
+        self.assertTrue(self.element_exists(selector),
+            'should be a link to the license manifest (selector %s)' % selector)
+
+        selector = 'a[data-link="package-manifest"]'
+        self.assertTrue(self.element_exists(selector),
+            'should be a link to the package manifest (selector %s)' % selector)
+
+        # check package count and size, link on target name
+        selector = '[data-value="target-package-count"]'
+        element = self.find(selector)
+        self.assertEquals(element.text, '1',
+            'package count should be shown for image builds')
+
+        selector = '[data-value="target-package-size"]'
+        element = self.find(selector)
+        self.assertEquals(element.text, '1.0 KB',
+            'package size should be shown for image builds')
+
+        selector = '[data-link="target-packages"]'
+        self.assertTrue(self.element_exists(selector),
+            'link to target packages should be on target heading')

bitbake/lib/toaster/tests/browser/test_builddashboard_page_recipes.py

@@ -0,0 +1,66 @@
+#! /usr/bin/env python
+# ex:ts=4:sw=4:sts=4:et
+# -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*-
+#
+# BitBake Toaster Implementation
+#
+# Copyright (C) 2013-2016 Intel Corporation
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License version 2 as
+# published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License along
+# with this program; if not, write to the Free Software Foundation, Inc.,
+# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+from django.core.urlresolvers import reverse
+from django.utils import timezone
+from tests.browser.selenium_helpers import SeleniumTestCase
+from orm.models import Project, Build, Recipe, Task, Layer, Layer_Version
+from orm.models import Target
+
+class TestBuilddashboardPageRecipes(SeleniumTestCase):
+    """ Test build dashboard recipes sub-page """
+
+    def setUp(self):
+        project = Project.objects.get_or_create_default_project()
+
+        now = timezone.now()
+
+        self.build = Build.objects.create(project=project,
+                                          started_on=now,
+                                          completed_on=now)
+
+        layer = Layer.objects.create()
+
+        layer_version = Layer_Version.objects.create(layer=layer,
+            build=self.build)
+
+        recipe = Recipe.objects.create(layer_version=layer_version)
+
+        task = Task.objects.create(build=self.build, recipe=recipe, order=1)
+
+        Target.objects.create(build=self.build, task=task, target='do_build')
+
+    def test_build_recipes_columns(self):
+        """
+        Check that non-hideable columns of the table on the recipes sub-page
+        are disabled on the edit columns dropdown.
+        """
+        url = reverse('recipes', args=(self.build.id,))
+        self.get(url)
+
+        self.wait_until_visible('#edit-columns-button')
+
+        # check that options for the non-hideable columns are disabled
+        non_hideable = ['name', 'version']
+
+        for column in non_hideable:
+            selector = 'input#checkbox-%s[disabled="disabled"]' % column
+            self.wait_until_present(selector)

bitbake/lib/toaster/tests/browser/test_builddashboard_page_tasks.py

@@ -0,0 +1,65 @@
+#! /usr/bin/env python
+# ex:ts=4:sw=4:sts=4:et
+# -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*-
+#
+# BitBake Toaster Implementation
+#
+# Copyright (C) 2013-2016 Intel Corporation
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License version 2 as
+# published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License along
+# with this program; if not, write to the Free Software Foundation, Inc.,
+# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+from django.core.urlresolvers import reverse
+from django.utils import timezone
+from tests.browser.selenium_helpers import SeleniumTestCase
+from orm.models import Project, Build, Recipe, Task, Layer, Layer_Version
+from orm.models import Target
+
+class TestBuilddashboardPageTasks(SeleniumTestCase):
+    """ Test build dashboard tasks sub-page """
+
+    def setUp(self):
+        project = Project.objects.get_or_create_default_project()
+
+        now = timezone.now()
+
+        self.build = Build.objects.create(project=project,
+                                          started_on=now,
+                                          completed_on=now)
+
+        layer = Layer.objects.create()
+
+        layer_version = Layer_Version.objects.create(layer=layer)
+
+        recipe = Recipe.objects.create(layer_version=layer_version)
+
+        task = Task.objects.create(build=self.build, recipe=recipe, order=1)
+
+        Target.objects.create(build=self.build, task=task, target='do_build')
+
+    def test_build_tasks_columns(self):
+        """
+        Check that non-hideable columns of the table on the tasks sub-page
+        are disabled on the edit columns dropdown.
+        """
+        url = reverse('tasks', args=(self.build.id,))
+        self.get(url)
+
+        self.wait_until_visible('#edit-columns-button')
+
+        # check that options for the non-hideable columns are disabled
+        non_hideable = ['order', 'task_name', 'recipe__name']
+
+        for column in non_hideable:
+            selector = 'input#checkbox-%s[disabled="disabled"]' % column
+            self.wait_until_present(selector)

bitbake/lib/toaster/tests/browser/test_layerdetails_page.py

@@ -0,0 +1,187 @@
+#! /usr/bin/env python
+# ex:ts=4:sw=4:sts=4:et
+# -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*-
+#
+# BitBake Toaster Implementation
+#
+# Copyright (C) 2013-2016 Intel Corporation
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License version 2 as
+# published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License along
+# with this program; if not, write to the Free Software Foundation, Inc.,
+# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+from django.core.urlresolvers import reverse
+from tests.browser.selenium_helpers import SeleniumTestCase
+
+from orm.models import Layer, Layer_Version, Project, LayerSource, Release
+from orm.models import BitbakeVersion
+
+from selenium.webdriver.support import expected_conditions as EC
+from selenium.webdriver.support.ui import WebDriverWait
+from selenium.webdriver.common.by import By
+
+
+class TestLayerDetailsPage(SeleniumTestCase):
+    """ Test layerdetails page works correctly """
+
+    def __init__(self, *args, **kwargs):
+        super(TestLayerDetailsPage, self).__init__(*args, **kwargs)
+
+        self.initial_values = None
+        self.url = None
+        self.imported_layer_version = None
+
+    def setUp(self):
+        release = Release.objects.create(
+            name='baz',
+            bitbake_version=BitbakeVersion.objects.create(name='v1')
+        )
+
+        # project to add new custom images to
+        self.project = Project.objects.create(name='foo', release=release)
+
+        name = "meta-imported"
+        vcs_url = "git://example.com/meta-imported"
+        subdir = "/layer"
+        gitrev = "d33d"
+        summary = "A imported layer"
+        description = "This was imported"
+
+        imported_layer = Layer.objects.create(name=name,
+                                              vcs_url=vcs_url,
+                                              summary=summary,
+                                              description=description)
+
+        self.imported_layer_version = Layer_Version.objects.create(
+            layer=imported_layer,
+            layer_source=LayerSource.TYPE_IMPORTED,
+            branch=gitrev,
+            commit=gitrev,
+            dirpath=subdir,
+            project=self.project)
+
+        self.initial_values = [name, vcs_url, subdir, gitrev, summary,
+                               description]
+        self.url = reverse('layerdetails',
+                           args=(self.project.pk,
+                                 self.imported_layer_version.pk))
+
+    def test_edit_layerdetails(self):
+        """ Edit all the editable fields for the layer refresh the page and
+        check that the new values exist"""
+
+        self.get(self.url)
+
+        self.click("#add-remove-layer-btn")
+
+        # Open every edit box
+        for btn in self.find_all("dd .glyphicon-edit"):
+            btn.click()
+
+        self.wait_until_visible("dd input")
+
+        # Edit each value
+        for inputs in self.find_all("dd input[type=text]") + \
+                self.find_all("dd textarea"):
+            # ignore the tt inputs (twitter typeahead input)
+            if "tt-" in inputs.get_attribute("class"):
+                continue
+
+            value = inputs.get_attribute("value")
+
+            self.assertTrue(value in self.initial_values,
+                            "Expecting any of \"%s\"but got \"%s\"" %
+                            (self.initial_values, value))
+
+            inputs.send_keys("-edited")
+
+        for save_btn in self.find_all(".change-btn"):
+            save_btn.click()
+
+        # Refresh the page to see if the new values are returned
+        self.get(self.url)
+
+        new_values = ["%s-edited" % old_val
+                      for old_val in self.initial_values]
+
+        for inputs in self.find_all('dd input[type="text"]') + \
+                self.find_all('dd textarea'):
+            # ignore the tt inputs (twitter typeahead input)
+            if "tt-" in inputs.get_attribute("class"):
+                continue
+
+            value = inputs.get_attribute("value")
+
+            self.assertTrue(value in new_values,
+                            "Expecting any of \"%s\" but got \"%s\"" %
+                            (new_values, value))
+
+    def test_delete_layer(self):
+        """ Delete the layer """
+
+        self.get(self.url)
+
+        # Wait for the tables to load to avoid a race condition where the
+        # toaster tables have made an async request. If the layer is deleted
+        # before the request finishes it will cause an exception and fail this
+        # test.
+        wait = WebDriverWait(self.driver, 30)
+
+        wait.until(EC.text_to_be_present_in_element(
+            (By.CLASS_NAME,
+             "table-count-recipestable"), "0"))
+
+        wait.until(EC.text_to_be_present_in_element(
+            (By.CLASS_NAME,
+             "table-count-machinestable"), "0"))
+
+        self.click('a[data-target="#delete-layer-modal"]')
+        self.wait_until_visible("#delete-layer-modal")
+        self.click("#layer-delete-confirmed")
+
+        notification = self.wait_until_visible("#change-notification-msg")
+        expected_text = "You have deleted 1 layer from your project: %s" % \
+            self.imported_layer_version.layer.name
+
+        self.assertTrue(expected_text in notification.text,
+                        "Expected notification text \"%s\" not found instead"
+                        "it was \"%s\"" %
+                        (expected_text, notification.text))
+
+    def test_addrm_to_project(self):
+        self.get(self.url)
+
+        # Add the layer
+        self.click("#add-remove-layer-btn")
+
+        notification = self.wait_until_visible("#change-notification-msg")
+
+        expected_text = "You have added 1 layer to your project: %s" % \
+            self.imported_layer_version.layer.name
+
+        self.assertTrue(expected_text in notification.text,
+                        "Expected notification text %s not found was "
+                        " \"%s\" instead" %
+                        (expected_text, notification.text))
+
+        # Remove the layer
+        self.click("#add-remove-layer-btn")
+
+        notification = self.wait_until_visible("#change-notification-msg")
+
+        expected_text = "You have removed 1 layer from your project: %s" % \
+            self.imported_layer_version.layer.name
+
+        self.assertTrue(expected_text in notification.text,
+                        "Expected notification text %s not found was "
+                        " \"%s\" instead" %
+                        (expected_text, notification.text))

bitbake/lib/toaster/tests/browser/test_most_recent_builds_states.py

@@ -0,0 +1,211 @@
+#! /usr/bin/env python
+# ex:ts=4:sw=4:sts=4:et
+# -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*-
+#
+# BitBake Toaster Implementation
+#
+# Copyright (C) 2013-2016 Intel Corporation
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License version 2 as
+# published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License along
+# with this program; if not, write to the Free Software Foundation, Inc.,
+# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+from django.core.urlresolvers import reverse
+from django.utils import timezone
+from tests.browser.selenium_helpers import SeleniumTestCase
+from tests.browser.selenium_helpers_base import Wait
+from orm.models import Project, Build, Task, Recipe, Layer, Layer_Version
+from bldcontrol.models import BuildRequest
+
+class TestMostRecentBuildsStates(SeleniumTestCase):
+    """ Test states update correctly in most recent builds area """
+
+    def _create_build_request(self):
+        project = Project.objects.get_or_create_default_project()
+
+        now = timezone.now()
+
+        build = Build.objects.create(project=project, build_name='fakebuild',
+            started_on=now, completed_on=now)
+
+        return BuildRequest.objects.create(build=build, project=project,
+            state=BuildRequest.REQ_QUEUED)
+
+    def _create_recipe(self):
+        """ Add a recipe to the database and return it """
+        layer = Layer.objects.create()
+        layer_version = Layer_Version.objects.create(layer=layer)
+        return Recipe.objects.create(name='foo', layer_version=layer_version)
+
+    def _check_build_states(self, build_request):
+        recipes_to_parse = 10
+        url = reverse('all-builds')
+        self.get(url)
+
+        build = build_request.build
+        base_selector = '[data-latest-build-result="%s"] ' % build.id
+
+        # build queued; check shown as queued
+        selector = base_selector + '[data-build-state="Queued"]'
+        element = self.wait_until_visible(selector)
+        self.assertRegexpMatches(element.get_attribute('innerHTML'),
+            'Build queued', 'build should show queued status')
+
+        # waiting for recipes to be parsed
+        build.outcome = Build.IN_PROGRESS
+        build.recipes_to_parse = recipes_to_parse
+        build.recipes_parsed = 0
+
+        build_request.state = BuildRequest.REQ_INPROGRESS
+        build_request.save()
+
+        self.get(url)
+
+        selector = base_selector + '[data-build-state="Parsing"]'
+        element = self.wait_until_visible(selector)
+
+        bar_selector = '#recipes-parsed-percentage-bar-%s' % build.id
+        bar_element = element.find_element_by_css_selector(bar_selector)
+        self.assertEqual(bar_element.value_of_css_property('width'), '0px',
+            'recipe parse progress should be at 0')
+
+        # recipes being parsed; check parse progress
+        build.recipes_parsed = 5
+        build.save()
+
+        self.get(url)
+
+        element = self.wait_until_visible(selector)
+        bar_element = element.find_element_by_css_selector(bar_selector)
+        recipe_bar_updated = lambda driver: \
+            bar_element.get_attribute('style') == 'width: 50%;'
+        msg = 'recipe parse progress bar should update to 50%'
+        element = Wait(self.driver).until(recipe_bar_updated, msg)
+
+        # all recipes parsed, task started, waiting for first task to finish;
+        # check status is shown as "Tasks starting..."
+        build.recipes_parsed = recipes_to_parse
+        build.save()
+
+        recipe = self._create_recipe()
+        task1 = Task.objects.create(build=build, recipe=recipe,
+            task_name='Lionel')
+        task2 = Task.objects.create(build=build, recipe=recipe,
+            task_name='Jeffries')
+
+        self.get(url)
+
+        selector = base_selector + '[data-build-state="Starting"]'
+        element = self.wait_until_visible(selector)
+        self.assertRegexpMatches(element.get_attribute('innerHTML'),
+            'Tasks starting', 'build should show "tasks starting" status')
+
+        # first task finished; check tasks progress bar
+        task1.order = 1
+        task1.save()
+
+        self.get(url)
+
+        selector = base_selector + '[data-build-state="In Progress"]'
+        element = self.wait_until_visible(selector)
+
+        bar_selector = '#build-pc-done-bar-%s' % build.id
+        bar_element = element.find_element_by_css_selector(bar_selector)
+
+        task_bar_updated = lambda driver: \
+            bar_element.get_attribute('style') == 'width: 50%;'
+        msg = 'tasks progress bar should update to 50%'
+        element = Wait(self.driver).until(task_bar_updated, msg)
+
+        # last task finished; check tasks progress bar updates
+        task2.order = 2
+        task2.save()
+
+        self.get(url)
+
+        element = self.wait_until_visible(selector)
+        bar_element = element.find_element_by_css_selector(bar_selector)
+        task_bar_updated = lambda driver: \
+            bar_element.get_attribute('style') == 'width: 100%;'
+        msg = 'tasks progress bar should update to 100%'
+        element = Wait(self.driver).until(task_bar_updated, msg)
+
+    def test_states_to_success(self):
+        """
+        Test state transitions in the recent builds area for a build which
+        completes successfully.
+        """
+        build_request = self._create_build_request()
+
+        self._check_build_states(build_request)
+
+        # all tasks complete and build succeeded; check success state shown
+        build = build_request.build
+        build.outcome = Build.SUCCEEDED
+        build.save()
+
+        selector = '[data-latest-build-result="%s"] ' \
+            '[data-build-state="Succeeded"]' % build.id
+        element = self.wait_until_visible(selector)
+
+    def test_states_to_failure(self):
+        """
+        Test state transitions in the recent builds area for a build which
+        completes in a failure.
+        """
+        build_request = self._create_build_request()
+
+        self._check_build_states(build_request)
+
+        # all tasks complete and build succeeded; check fail state shown
+        build = build_request.build
+        build.outcome = Build.FAILED
+        build.save()
+
+        selector = '[data-latest-build-result="%s"] ' \
+            '[data-build-state="Failed"]' % build.id
+        element = self.wait_until_visible(selector)
+
+    def test_states_cancelling(self):
+        """
+        Test that most recent build area updates correctly for a build
+        which is cancelled.
+        """
+        url = reverse('all-builds')
+
+        build_request = self._create_build_request()
+        build = build_request.build
+
+        # cancel the build
+        build_request.state = BuildRequest.REQ_CANCELLING
+        build_request.save()
+
+        self.get(url)
+
+        # check cancelling state
+        selector = '[data-latest-build-result="%s"] ' \
+            '[data-build-state="Cancelling"]' % build.id
+        element = self.wait_until_visible(selector)
+        self.assertRegexpMatches(element.get_attribute('innerHTML'),
+            'Cancelling the build', 'build should show "cancelling" status')
+
+        # check cancelled state
+        build.outcome = Build.CANCELLED
+        build.save()
+
+        self.get(url)
+
+        selector = '[data-latest-build-result="%s"] ' \
+            '[data-build-state="Cancelled"]' % build.id
+        element = self.wait_until_visible(selector)
+        self.assertRegexpMatches(element.get_attribute('innerHTML'),
+            'Build cancelled', 'build should show "cancelled" status')

bitbake/lib/toaster/tests/browser/test_new_project_page.py

@@ -22,6 +22,7 @@
 from django.core.urlresolvers import reverse
 from tests.browser.selenium_helpers import SeleniumTestCase
 from selenium.webdriver.support.ui import Select
+from selenium.common.exceptions import InvalidElementStateException
 
 from orm.models import Project, Release, BitbakeVersion
 
@@ -102,7 +103,10 @@ class TestNewProjectPage(SeleniumTestCase):
 
         # Try and click it anyway, if it submits we'll have a new project in
         # the db and assert then
-        self.click("#create-project-button")
+        try:
+            self.click("#create-project-button")
+        except InvalidElementStateException:
+            pass
 
         self.assertTrue(
             (Project.objects.filter(name=project_name).count() == 1),

bitbake/lib/toaster/tests/browser/test_project_config_page.py

@@ -31,6 +31,9 @@ class TestProjectConfigsPage(SeleniumTestCase):
     """ Test data at /project/X/builds is displayed correctly """
 
     PROJECT_NAME = 'test project'
+    INVALID_PATH_START_TEXT = 'The directory path should either start with a /'
+    INVALID_PATH_CHAR_TEXT = 'The directory path cannot include spaces or ' \
+        'any of these characters'
 
     def setUp(self):
         bbv = BitbakeVersion.objects.create(name='bbv1', giturl='/tmp/',
@@ -116,3 +119,113 @@ class TestProjectConfigsPage(SeleniumTestCase):
                checkbox.click()
                self.assertTrue(("cpio" not in element.text),
                                "Image still present in the textbox")
+
+    def test_set_download_dir(self):
+        """
+        Validate the allowed and disallowed types in the directory field for
+        DL_DIR
+        """
+
+        ProjectVariable.objects.get_or_create(project=self.project1,
+            name='DL_DIR')
+        url = reverse('projectconf', args=(self.project1.id,))
+        self.get(url)
+
+        # activate the input to edit download dir
+        self.click('#change-dl_dir-icon')
+        self.wait_until_visible('#new-dl_dir')
+
+        # downloads dir path doesn't start with / or ${...}
+        self.enter_text('#new-dl_dir', 'home/foo')
+        element = self.wait_until_visible('#hintError-initialChar-dl_dir')
+
+        msg = 'downloads directory path starts with invalid character but ' \
+            'treated as valid'
+        self.assertTrue((self.INVALID_PATH_START_TEXT in element.text), msg)
+
+        # downloads dir path has a space
+        self.driver.find_element_by_id('new-dl_dir').clear()
+        self.enter_text('#new-dl_dir', '/foo/bar a')
+
+        element = self.wait_until_visible('#hintError-dl_dir')
+        msg = 'downloads directory path characters invalid but treated as valid'
+        self.assertTrue((self.INVALID_PATH_CHAR_TEXT in element.text), msg)
+
+        # downloads dir path starts with ${...} but has a space
+        self.driver.find_element_by_id('new-dl_dir').clear()
+        self.enter_text('#new-dl_dir', '${TOPDIR}/down foo')
+
+        element = self.wait_until_visible('#hintError-dl_dir')
+        msg = 'downloads directory path characters invalid but treated as valid'
+        self.assertTrue((self.INVALID_PATH_CHAR_TEXT in element.text), msg)
+
+        # downloads dir path starts with /
+        self.driver.find_element_by_id('new-dl_dir').clear()
+        self.enter_text('#new-dl_dir', '/bar/foo')
+
+        hidden_element = self.driver.find_element_by_id('hintError-dl_dir')
+        self.assertEqual(hidden_element.is_displayed(), False,
+            'downloads directory path valid but treated as invalid')
+
+        # downloads dir path starts with ${...}
+        self.driver.find_element_by_id('new-dl_dir').clear()
+        self.enter_text('#new-dl_dir', '${TOPDIR}/down')
+
+        hidden_element = self.driver.find_element_by_id('hintError-dl_dir')
+        self.assertEqual(hidden_element.is_displayed(), False,
+            'downloads directory path valid but treated as invalid')
+
+    def test_set_sstate_dir(self):
+        """
+        Validate the allowed and disallowed types in the directory field for
+        SSTATE_DIR
+        """
+
+        ProjectVariable.objects.get_or_create(project=self.project1,
+            name='SSTATE_DIR')
+        url = reverse('projectconf', args=(self.project1.id,))
+        self.get(url)
+
+        self.click('#change-sstate_dir-icon')
+
+        self.wait_until_visible('#new-sstate_dir')
+
+        # path doesn't start with / or ${...}
+        self.enter_text('#new-sstate_dir', 'home/foo')
+        element = self.wait_until_visible('#hintError-initialChar-sstate_dir')
+
+        msg = 'sstate directory path starts with invalid character but ' \
+            'treated as valid'
+        self.assertTrue((self.INVALID_PATH_START_TEXT in element.text), msg)
+
+        # path has a space
+        self.driver.find_element_by_id('new-sstate_dir').clear()
+        self.enter_text('#new-sstate_dir', '/foo/bar a')
+
+        element = self.wait_until_visible('#hintError-sstate_dir')
+        msg = 'sstate directory path characters invalid but treated as valid'
+        self.assertTrue((self.INVALID_PATH_CHAR_TEXT in element.text), msg)
+
+        # path starts with ${...} but has a space
+        self.driver.find_element_by_id('new-sstate_dir').clear()
+        self.enter_text('#new-sstate_dir', '${TOPDIR}/down foo')
+
+        element = self.wait_until_visible('#hintError-sstate_dir')
+        msg = 'sstate directory path characters invalid but treated as valid'
+        self.assertTrue((self.INVALID_PATH_CHAR_TEXT in element.text), msg)
+
+        # path starts with /
+        self.driver.find_element_by_id('new-sstate_dir').clear()
+        self.enter_text('#new-sstate_dir', '/bar/foo')
+
+        hidden_element = self.driver.find_element_by_id('hintError-sstate_dir')
+        self.assertEqual(hidden_element.is_displayed(), False,
+            'sstate directory path valid but treated as invalid')
+
+        # paths starts with ${...}
+        self.driver.find_element_by_id('new-sstate_dir').clear()
+        self.enter_text('#new-sstate_dir', '${TOPDIR}/down')
+
+        hidden_element = self.driver.find_element_by_id('hintError-sstate_dir')
+        self.assertEqual(hidden_element.is_displayed(), False,
+            'sstate directory path valid but treated as invalid')

bitbake/lib/toaster/tests/browser/test_task_page.py

@@ -0,0 +1,76 @@
+#! /usr/bin/env python
+# ex:ts=4:sw=4:sts=4:et
+# -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*-
+#
+# BitBake Toaster Implementation
+#
+# Copyright (C) 2013-2016 Intel Corporation
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License version 2 as
+# published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License along
+# with this program; if not, write to the Free Software Foundation, Inc.,
+# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+from django.core.urlresolvers import reverse
+from django.utils import timezone
+from tests.browser.selenium_helpers import SeleniumTestCase
+from orm.models import Project, Build, Layer, Layer_Version, Recipe, Target
+from orm.models import Task, Task_Dependency
+
+class TestTaskPage(SeleniumTestCase):
+    """ Test page which shows an individual task """
+    RECIPE_NAME = 'bar'
+    RECIPE_VERSION = '0.1'
+    TASK_NAME = 'do_da_doo_ron_ron'
+
+    def setUp(self):
+        now = timezone.now()
+
+        project = Project.objects.get_or_create_default_project()
+
+        self.build = Build.objects.create(project=project, started_on=now,
+            completed_on=now)
+
+        Target.objects.create(target='foo', build=self.build)
+
+        layer = Layer.objects.create()
+
+        layer_version = Layer_Version.objects.create(layer=layer)
+
+        recipe = Recipe.objects.create(name=TestTaskPage.RECIPE_NAME,
+            layer_version=layer_version, version=TestTaskPage.RECIPE_VERSION)
+
+        self.task = Task.objects.create(build=self.build, recipe=recipe,
+            order=1, outcome=Task.OUTCOME_COVERED, task_executed=False,
+            task_name=TestTaskPage.TASK_NAME)
+
+    def test_covered_task(self):
+        """
+        Check that covered tasks are displayed for tasks which have
+        dependencies on themselves
+        """
+
+        # the infinite loop which of bug 9952 was down to tasks which
+        # depend on themselves, so add self-dependent tasks to replicate the
+        # situation which caused the infinite loop (now fixed)
+        Task_Dependency.objects.create(task=self.task, depends_on=self.task)
+
+        url = reverse('task', args=(self.build.id, self.task.id,))
+        self.get(url)
+
+        # check that we see the task name
+        self.wait_until_visible('.page-header h1')
+
+        heading = self.find('.page-header h1')
+        expected_heading = '%s_%s %s' % (TestTaskPage.RECIPE_NAME,
+            TestTaskPage.RECIPE_VERSION, TestTaskPage.TASK_NAME)
+        self.assertEqual(heading.text, expected_heading,
+            'Heading should show recipe name, version and task')

bitbake/lib/toaster/tests/browser/test_toastertable_ui.py

@@ -0,0 +1,160 @@
+#! /usr/bin/env python
+# ex:ts=4:sw=4:sts=4:et
+# -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*-
+#
+# BitBake Toaster Implementation
+#
+# Copyright (C) 2013-2016 Intel Corporation
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License version 2 as
+# published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License along
+# with this program; if not, write to the Free Software Foundation, Inc.,
+# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+from datetime import datetime
+
+from django.core.urlresolvers import reverse
+from django.utils import timezone
+from tests.browser.selenium_helpers import SeleniumTestCase
+from orm.models import BitbakeVersion, Release, Project, Build
+
+class TestToasterTableUI(SeleniumTestCase):
+    """
+    Tests for the UI elements of ToasterTable (sorting etc.);
+    note that the tests cover generic functionality of ToasterTable which
+    manifests as UI elements in the browser, and can only be tested via
+    Selenium.
+    """
+
+    def setUp(self):
+        pass
+
+    def _get_orderby_heading(self, table):
+        """
+        Get the current order by finding the column heading in <table> with
+        the sorted class on it.
+
+        table: WebElement for a ToasterTable
+        """
+        selector = 'thead a.sorted'
+        heading = table.find_element_by_css_selector(selector)
+        return heading.get_attribute('innerHTML').strip()
+
+    def _get_datetime_from_cell(self, row, selector):
+        """
+        Return the value in the cell selected by <selector> on <row> as a
+        datetime.
+
+        row: <tr> WebElement for a row in the ToasterTable
+        selector: CSS selector to use to find the cell containing the date time
+        string
+        """
+        cell = row.find_element_by_css_selector(selector)
+        cell_text = cell.get_attribute('innerHTML').strip()
+        return datetime.strptime(cell_text, '%d/%m/%y %H:%M')
+
+    def test_revert_orderby(self):
+        """
+        Test that sort order for a table reverts to the default sort order
+        if the current sort column is hidden.
+        """
+        now = timezone.now()
+        later = now + timezone.timedelta(hours=1)
+        even_later = later + timezone.timedelta(hours=1)
+
+        bbv = BitbakeVersion.objects.create(name='test bbv', giturl='/tmp/',
+                                            branch='master', dirpath='')
+        release = Release.objects.create(name='test release',
+                                         branch_name='master',
+                                         bitbake_version=bbv)
+
+        project = Project.objects.create_project('project', release)
+
+        # set up two builds which will order differently when sorted by
+        # started_on or completed_on
+
+        # started first, finished last
+        build1 = Build.objects.create(project=project,
+                                      started_on=now,
+                                      completed_on=even_later,
+                                      outcome=Build.SUCCEEDED)
+
+        # started second, finished first
+        build2 = Build.objects.create(project=project,
+                                      started_on=later,
+                                      completed_on=later,
+                                      outcome=Build.SUCCEEDED)
+
+        url = reverse('all-builds')
+        self.get(url)
+        table = self.wait_until_visible('#allbuildstable')
+
+        # check ordering (default is by -completed_on); so build1 should be
+        # first as it finished last
+        active_heading = self._get_orderby_heading(table)
+        self.assertEqual(active_heading, 'Completed on',
+            'table should be sorted by "Completed on" by default')
+
+        row_selector = '#allbuildstable tbody tr'
+        cell_selector = 'td.completed_on'
+
+        rows = self.find_all(row_selector)
+        row1_completed_on = self._get_datetime_from_cell(rows[0], cell_selector)
+        row2_completed_on = self._get_datetime_from_cell(rows[1], cell_selector)
+        self.assertTrue(row1_completed_on > row2_completed_on,
+            'table should be sorted by -completed_on')
+
+        # turn on started_on column
+        self.click('#edit-columns-button')
+        self.click('#checkbox-started_on')
+
+        # sort by started_on column
+        links = table.find_elements_by_css_selector('th.started_on a')
+        for link in links:
+            if link.get_attribute('innerHTML').strip() == 'Started on':
+                link.click()
+                break
+
+        # wait for table data to reload in response to new sort
+        self.wait_until_visible('#allbuildstable')
+
+        # check ordering; build1 should be first
+        active_heading = self._get_orderby_heading(table)
+        self.assertEqual(active_heading, 'Started on',
+            'table should be sorted by "Started on"')
+
+        cell_selector = 'td.started_on'
+
+        rows = self.find_all(row_selector)
+        row1_started_on = self._get_datetime_from_cell(rows[0], cell_selector)
+        row2_started_on = self._get_datetime_from_cell(rows[1], cell_selector)
+        self.assertTrue(row1_started_on < row2_started_on,
+            'table should be sorted by started_on')
+
+        # turn off started_on column
+        self.click('#edit-columns-button')
+        self.click('#checkbox-started_on')
+
+        # wait for table data to reload in response to new sort
+        self.wait_until_visible('#allbuildstable')
+
+        # check ordering (should revert to completed_on); build2 should be first
+        active_heading = self._get_orderby_heading(table)
+        self.assertEqual(active_heading, 'Completed on',
+            'table should be sorted by "Completed on" after hiding sort column')
+
+        cell_selector = 'td.completed_on'
+
+        rows = self.find_all(row_selector)
+        row1_completed_on = self._get_datetime_from_cell(rows[0], cell_selector)
+        row2_completed_on = self._get_datetime_from_cell(rows[1], cell_selector)
+        self.assertTrue(row1_completed_on > row2_completed_on,
+            'table should be sorted by -completed_on')

bitbake/lib/toaster/toastergui/api.py

@@ -20,11 +20,20 @@
 # Temporary home for the UI's misc API
 import re
 
-from orm.models import Project, ProjectTarget, Build
+from orm.models import Project, ProjectTarget, Build, Layer_Version
+from orm.models import LayerVersionDependency, LayerSource, ProjectLayer
 from bldcontrol.models import BuildRequest
 from bldcontrol import bbcontroller
 from django.http import HttpResponse, JsonResponse
 from django.views.generic import View
+from django.core.urlresolvers import reverse
+from django.core import serializers
+from django.utils import timezone
+from django.template.defaultfilters import date
+from toastergui.templatetags.projecttags import json, sectohms, get_tasks
+
+def error_response(error):
+    return JsonResponse({"error": error})
 
 
 class XhrBuildRequest(View):
@@ -78,20 +87,24 @@ class XhrBuildRequest(View):
                     br.save()
 
                 except BuildRequest.DoesNotExist:
-                    return JsonResponse({'error':'No such build id %s' % i})
+                    return error_response('No such build request id %s' % i)
 
-            return JsonResponse({'error': 'ok'})
+            return error_response('ok')
 
         if 'buildDelete' in request.POST:
             for i in request.POST['buildDelete'].strip().split(" "):
                 try:
-                    BuildRequest.objects.select_for_update().get(project = project, pk = i, state__lte = BuildRequest.REQ_DELETED).delete()
+                    BuildRequest.objects.select_for_update().get(
+                        project=project,
+                        pk=i,
+                        state__lte=BuildRequest.REQ_DELETED).delete()
+
                 except BuildRequest.DoesNotExist:
                     pass
-            return JsonResponse({'error': 'ok' })
+            return error_response("ok")
 
         if 'targets' in request.POST:
-            ProjectTarget.objects.filter(project = project).delete()
+            ProjectTarget.objects.filter(project=project).delete()
             s = str(request.POST['targets'])
             for t in re.sub(r'[;%|"]', '', s).split(" "):
                 if ":" in t:
@@ -99,13 +112,210 @@ class XhrBuildRequest(View):
                 else:
                     target = t
                     task = ""
-                ProjectTarget.objects.create(project = project,
-                                             target = target,
-                                             task = task)
+                ProjectTarget.objects.create(project=project,
+                                             target=target,
+                                             task=task)
             project.schedule_build()
 
-            return JsonResponse({'error': 'ok' })
+            return error_response('ok')
 
         response = HttpResponse()
         response.status_code = 500
         return response
+
+
+class XhrLayer(View):
+    """ Get and Update Layer information """
+
+    def post(self, request, *args, **kwargs):
+        """
+          Update a layer
+
+          Entry point: /xhr_layer/<layerversion_id>
+          Method: POST
+
+          Args:
+              vcs_url, dirpath, commit, up_branch, summary, description
+
+              add_dep = append a layerversion_id as a dependency
+              rm_dep = remove a layerversion_id as a depedency
+          Returns:
+              {"error": "ok"}
+            or
+              {"error": <error message>}
+        """
+
+        try:
+            # We currently only allow Imported layers to be edited
+            layer_version = Layer_Version.objects.get(
+                id=kwargs['layerversion_id'],
+                project=kwargs['pid'],
+                layer_source=LayerSource.TYPE_IMPORTED)
+
+        except Layer_Version.DoesNotExist:
+            return error_response("Cannot find imported layer to update")
+
+        if "vcs_url" in request.POST:
+            layer_version.layer.vcs_url = request.POST["vcs_url"]
+        if "dirpath" in request.POST:
+            layer_version.dirpath = request.POST["dirpath"]
+        if "commit" in request.POST:
+            layer_version.commit = request.POST["commit"]
+            layer_version.branch = request.POST["commit"]
+        if "summary" in request.POST:
+            layer_version.layer.summary = request.POST["summary"]
+        if "description" in request.POST:
+            layer_version.layer.description = request.POST["description"]
+
+        if "add_dep" in request.POST:
+            lvd = LayerVersionDependency(
+                layer_version=layer_version,
+                depends_on_id=request.POST["add_dep"])
+            lvd.save()
+
+        if "rm_dep" in request.POST:
+            rm_dep = LayerVersionDependency.objects.get(
+                layer_version=layer_version,
+                depends_on_id=request.POST["rm_dep"])
+            rm_dep.delete()
+
+        try:
+            layer_version.layer.save()
+            layer_version.save()
+        except Exception as e:
+            return error_response("Could not update layer version entry: %s"
+                                  % e)
+
+        return JsonResponse({"error": "ok"})
+
+    def delete(self, request, *args, **kwargs):
+        try:
+            # We currently only allow Imported layers to be deleted
+            layer_version = Layer_Version.objects.get(
+                id=kwargs['layerversion_id'],
+                project=kwargs['pid'],
+                layer_source=LayerSource.TYPE_IMPORTED)
+        except Layer_Version.DoesNotExist:
+            return error_response("Cannot find imported layer to delete")
+
+        try:
+            ProjectLayer.objects.get(project=kwargs['pid'],
+                                     layercommit=layer_version).delete()
+        except ProjectLayer.DoesNotExist:
+            pass
+
+        layer_version.layer.delete()
+        layer_version.delete()
+
+        return JsonResponse({
+            "error": "ok",
+            "redirect": reverse('project', args=(kwargs['pid'],))
+        })
+
+class MostRecentBuildsView(View):
+    def _was_yesterday_or_earlier(self, completed_on):
+        now = timezone.now()
+        delta = now - completed_on
+
+        if delta.days >= 1:
+            return True
+
+        return False
+
+    def get(self, request, *args, **kwargs):
+        """
+        Returns a list of builds in JSON format.
+        """
+        mrb_type = 'all'
+        project = None
+
+        project_id = request.GET.get('project_id', None)
+        if project_id:
+            try:
+                mrb_type = 'project'
+                project = Project.objects.get(pk=project_id)
+            except:
+                # if project lookup fails, assume no project
+                pass
+
+        recent_build_objs = Build.get_recent(project)
+        recent_builds = []
+
+        # for timezone conversion
+        tz = timezone.get_current_timezone()
+
+        for build_obj in recent_build_objs:
+            dashboard_url = reverse('builddashboard', args=(build_obj.pk,))
+            buildtime_url = reverse('buildtime', args=(build_obj.pk,))
+            rebuild_url = \
+                reverse('xhr_buildrequest', args=(build_obj.project.pk,))
+            cancel_url = \
+                reverse('xhr_buildrequest', args=(build_obj.project.pk,))
+
+            build = {}
+            build['id'] = build_obj.pk
+            build['dashboard_url'] = dashboard_url
+
+            buildrequest_id = None
+            if hasattr(build_obj, 'buildrequest'):
+                buildrequest_id = build_obj.buildrequest.pk
+            build['buildrequest_id'] = buildrequest_id
+
+            build['recipes_parsed_percentage'] = \
+                int((build_obj.recipes_parsed / build_obj.recipes_to_parse) * 100)
+
+            tasks_complete_percentage = 0
+            if build_obj.outcome in (Build.SUCCEEDED, Build.FAILED):
+                tasks_complete_percentage = 100
+            elif build_obj.outcome == Build.IN_PROGRESS:
+                tasks_complete_percentage = build_obj.completeper()
+            build['tasks_complete_percentage'] = tasks_complete_percentage
+
+            build['state'] = build_obj.get_state()
+
+            build['errors'] = build_obj.errors.count()
+            build['dashboard_errors_url'] = dashboard_url + '#errors'
+
+            build['warnings'] = build_obj.warnings.count()
+            build['dashboard_warnings_url'] = dashboard_url + '#warnings'
+
+            build['buildtime'] = sectohms(build_obj.timespent_seconds)
+            build['buildtime_url'] = buildtime_url
+
+            build['rebuild_url'] = rebuild_url
+            build['cancel_url'] = cancel_url
+
+            build['is_default_project_build'] = build_obj.project.is_default
+
+            build['build_targets_json'] = \
+                json(get_tasks(build_obj.target_set.all()))
+
+            # convert completed_on time to user's timezone
+            completed_on = timezone.localtime(build_obj.completed_on)
+
+            completed_on_template = '%H:%M'
+            if self._was_yesterday_or_earlier(completed_on):
+                completed_on_template = '%d/%m/%Y ' + completed_on_template
+            build['completed_on'] = completed_on.strftime(completed_on_template)
+
+            targets = []
+            target_objs = build_obj.get_sorted_target_list()
+            for target_obj in target_objs:
+                if target_obj.task:
+                    targets.append(target_obj.target + ':' + target_obj.task)
+                else:
+                    targets.append(target_obj.target)
+            build['targets'] = ' '.join(targets)
+
+            # abbreviated form of the full target list
+            abbreviated_targets = ''
+            num_targets = len(targets)
+            if num_targets > 0:
+                abbreviated_targets = targets[0]
+            if num_targets > 1:
+                abbreviated_targets += (' +%s' % (num_targets - 1))
+            build['targets_abbreviated'] = abbreviated_targets
+
+            recent_builds.append(build)
+
+        return JsonResponse(recent_builds, safe=False)

bitbake/lib/toaster/toastergui/buildtables.py

@@ -109,15 +109,31 @@ class BuiltPackagesTableBase(tables.PackagesTable):
                         hidden=True,
                         orderable=True)
 
+        layer_branch_template = '''
+        {%if not data.recipe.layer_version.layer.local_source_dir %}
+        <span class="text-muted">{{data.recipe.layer_version.branch}}</span>
+        {% else %}
+        <span class="text-muted">Not applicable</span>
+        <span class="glyphicon glyphicon-question-sign get-help" data-original-title="" title="The source code of {{data.recipe.layer_version.layer.name}} is not in a Git repository, so there is no branch associated with it"> </span>
+        {% endif %}
+        '''
+
         self.add_column(title="Layer branch",
                         field_name="recipe__layer_version__branch",
                         hidden=True,
+                        static_data_name="recipe__layer_version__branch",
+                        static_data_template=layer_branch_template,
                         orderable=True)
 
         git_rev_template = '''
+        {% if not data.recipe.layer_version.layer.local_source_dir %}
         {% with vcs_ref=data.recipe.layer_version.commit %}
         {% include 'snippets/gitrev_popover.html' %}
         {% endwith %}
+        {% else %}
+        <span class="text-muted">Not applicable</span>
+        <span class="glyphicon glyphicon-question-sign get-help" data-original-title="" title="The source code of {{data.recipe.layer_version.layer.name}} is not in a Git repository, so there is no revision associated with it"> </span>
+        {% endif %}
         '''
 
         self.add_column(title="Layer commit",
@@ -250,10 +266,24 @@ class BuiltRecipesTable(BuildTablesMixin):
             '{% if data.pathflags %}<i>({{data.pathflags}})</i>'\
             '{% endif %}'
 
+        git_branch_template = '''
+        {% if data.layer_version.layer.local_source_dir %}
+        <span class="text-muted">Not applicable</span>
+        <span class="glyphicon glyphicon-question-sign get-help" data-original-title="" title="The source code of {{data.layer_version.layer.name}} is not in a Git repository, so there is no branch associated with it"> </span>
+        {% else %}
+        <span>{{data.layer_version.branch}}</span>
+        {% endif %}
+        '''
+
         git_rev_template = '''
+        {% if data.layer_version.layer.local_source_dir %}
+        <span class="text-muted">Not applicable</span>
+        <span class="glyphicon glyphicon-question-sign get-help" data-original-title="" title="The source code of {{data.layer_version.layer.name}} is not in a Git repository, so there is no commit associated with it"> </span>
+        {% else %}
         {% with vcs_ref=data.layer_version.commit %}
         {% include 'snippets/gitrev_popover.html' %}
         {% endwith %}
+        {% endif %}
         '''
 
         depends_on_tmpl = '''
@@ -295,13 +325,15 @@ class BuiltRecipesTable(BuildTablesMixin):
         {% endif %}{% endwith %}{% endwith %}
         '''
 
-        self.add_column(title="Name",
+        self.add_column(title="Recipe",
                         field_name="name",
                         static_data_name='name',
                         orderable=True,
+                        hideable=False,
                         static_data_template=recipe_name_tmpl)
 
         self.add_column(title="Version",
+                        hideable=False,
                         field_name="version")
 
         self.add_column(title="Dependencies",
@@ -340,6 +372,8 @@ class BuiltRecipesTable(BuildTablesMixin):
 
         self.add_column(title="Layer branch",
                         field_name="layer_version__branch",
+                        static_data_name="layer_version__branch",
+                        static_data_template=git_branch_template,
                         orderable=True,
                         hidden=True)
 
@@ -431,17 +465,20 @@ class BuildTasksTable(BuildTablesMixin):
         self.add_column(title="Order",
                         static_data_name="order",
                         static_data_template='{{data.order}}',
+                        hideable=False,
                         orderable=True)
 
         self.add_column(title="Task",
                         static_data_name="task_name",
                         static_data_template=task_link_tmpl(
                             "{{data.task_name}}"),
+                        hideable=False,
                         orderable=True)
 
         self.add_column(title="Recipe",
                         static_data_name='recipe__name',
                         static_data_template=recipe_name_tmpl,
+                        hideable=False,
                         orderable=True)
 
         self.add_column(title="Recipe version",

bitbake/lib/toaster/toastergui/fixtures/toastergui-unittest-data.xml

@@ -1,5 +1,16 @@
 <?xml version="1.0" encoding="utf-8"?>
 <django-objects version="1.0">
+   <object pk="1" model="orm.bitbakeversion">
+      <field type="CharField" name="name">v2.3</field>
+      <field type="GitURLField" name="giturl">git://git.openembedded.org/bitbake</field>
+      <field type="CharField" name="dirpath">b</field>
+      <field type="CharField" name="branch">a</field>
+  </object>
+  <object pk="1" model="orm.release">
+     <field type="CharField" name="name">master</field>
+     <field type="CharField" name="description">master project</field>
+     <field to="orm.bitbake_version" name="bitbake_version">1</field>
+  </object>
   <object pk="1" model="orm.project">
     <field type="CharField" name="name">a test project</field>
     <field type="CharField" name="short_description"></field>
@@ -195,8 +206,6 @@
     <field to="orm.customimagerecipe" name="recipe_appends" rel="ManyToManyRel"><object pk="3"></object></field>
   </object>
   <object pk="1" model="orm.recipe">
-    <field to="orm.layersource" name="layer_source" rel="ManyToOneRel">1</field>
-    <field type="IntegerField" name="up_id"><None></None></field>
     <field type="DateTimeField" name="up_date"><None></None></field>
     <field type="CharField" name="name">z recipe</field>
     <field type="CharField" name="version">5.2</field>
@@ -212,8 +221,6 @@
     <field type="BooleanField" name="is_image">False</field>
   </object>
   <object pk="2" model="orm.recipe">
-    <field to="orm.layersource" name="layer_source" rel="ManyToOneRel">1</field>
-    <field type="IntegerField" name="up_id"><None></None></field>
     <field type="DateTimeField" name="up_date"><None></None></field>
     <field type="CharField" name="name">a recipe</field>
     <field type="CharField" name="version">1.2</field>
@@ -229,8 +236,6 @@
     <field type="BooleanField" name="is_image">False</field>
   </object>
   <object pk="3" model="orm.recipe">
-    <field to="orm.layersource" name="layer_source" rel="ManyToOneRel"><None></None></field>
-    <field type="IntegerField" name="up_id"><None></None></field>
     <field type="DateTimeField" name="up_date"><None></None></field>
     <field type="CharField" name="name">a custom recipe</field>
     <field type="CharField" name="version"></field>
@@ -246,8 +251,6 @@
     <field type="BooleanField" name="is_image">False</field>
   </object>
   <object pk="4" model="orm.recipe">
-    <field to="orm.layersource" name="layer_source" rel="ManyToOneRel">1</field>
-    <field type="IntegerField" name="up_id"><None></None></field>
     <field type="DateTimeField" name="up_date"><None></None></field>
     <field type="CharField" name="name">a image recipe</field>
     <field type="CharField" name="version">1.2</field>
@@ -263,8 +266,6 @@
     <field type="BooleanField" name="is_image">True</field>
   </object>
   <object pk="5" model="orm.recipe">
-    <field to="orm.layersource" name="layer_source" rel="ManyToOneRel">1</field>
-    <field type="IntegerField" name="up_id"><None></None></field>
     <field type="DateTimeField" name="up_date"><None></None></field>
     <field type="CharField" name="name">z image recipe</field>
     <field type="CharField" name="version">1.3</field>
@@ -280,8 +281,6 @@
     <field type="BooleanField" name="is_image">True</field>
   </object>
   <object pk="6" model="orm.recipe">
-    <field to="orm.layersource" name="layer_source" rel="ManyToOneRel"><None></None></field>
-    <field type="IntegerField" name="up_id"><None></None></field>
     <field type="DateTimeField" name="up_date"><None></None></field>
     <field type="CharField" name="name">z custom recipe</field>
     <field type="CharField" name="version"></field>
@@ -307,34 +306,23 @@
   </object>
 
   <object pk="1" model="orm.machine">
-    <field to="orm.layersource" name="layer_source" rel="ManyToOneRel"><None></None></field>
-    <field type="IntegerField" name="up_id"><None></None></field>
     <field type="DateTimeField" name="up_date"><None></None></field>
     <field to="orm.layer_version" name="layer_version" rel="ManyToOneRel">1</field>
     <field type="CharField" name="name">a machine</field>
     <field type="CharField" name="description">a machine</field>
   </object>
   <object pk="2" model="orm.machine">
-    <field to="orm.layersource" name="layer_source" rel="ManyToOneRel"><None></None></field>
-    <field type="IntegerField" name="up_id"><None></None></field>
     <field type="DateTimeField" name="up_date"><None></None></field>
     <field to="orm.layer_version" name="layer_version" rel="ManyToOneRel">2</field>
     <field type="CharField" name="name">z machine</field>
     <field type="CharField" name="description">z machine</field>
   </object>
   <object pk="3" model="orm.machine">
-    <field to="orm.layersource" name="layer_source" rel="ManyToOneRel"><None></None></field>
-    <field type="IntegerField" name="up_id"><None></None></field>
     <field type="DateTimeField" name="up_date"><None></None></field>
     <field to="orm.layer_version" name="layer_version" rel="ManyToOneRel">1</field>
     <field type="CharField" name="name">g machine</field>
     <field type="CharField" name="description">g machine</field>
   </object>
-  <object pk="1" model="orm.layersource">
-    <field type="CharField" name="name">local</field>
-    <field type="IntegerField" name="sourcetype">1</field>
-    <field type="CharField" name="apiurl"></field>
-  </object>
   <object pk="1" model="orm.bitbakeversion">
     <field type="CharField" name="name">test bbv</field>
     <field type="CharField" name="giturl">/tmp/</field>
@@ -355,26 +343,12 @@
     <field type="CharField" name="branch_name">master</field>
     <field type="TextField" name="helptext"><None></None></field>
   </object>
-  <object pk="1" model="orm.releaselayersourcepriority">
-    <field to="orm.release" name="release" rel="ManyToOneRel">1</field>
-    <field to="orm.layersource" name="layer_source" rel="ManyToOneRel">1</field>
-    <field type="IntegerField" name="priority">0</field>
-  </object>
-  <object pk="1" model="orm.branch">
-    <field to="orm.layersource" name="layer_source" rel="ManyToOneRel">1</field>
-    <field type="CharField" name="name">master</field>
-    <field type="CharField" name="short_description"></field>
-  </object>
   <object pk="1" model="orm.layer">
-    <field to="orm.layersource" name="layer_source" rel="ManyToOneRel"><None></None></field>
-    <field type="IntegerField" name="up_id"><None></None></field>
     <field type="DateTimeField" name="up_date"><None></None></field>
     <field type="CharField" name="name">a layer</field>
     <field type="CharField" name="vcs_url">/tmp/</field>
   </object>
   <object pk="2" model="orm.layer">
-    <field to="orm.layersource" name="layer_source" rel="ManyToOneRel">1</field>
-    <field type="IntegerField" name="up_id"><None></None></field>
     <field type="DateTimeField" name="up_date"><None></None></field>
     <field type="CharField" name="name">z layer</field>
     <field type="CharField" name="layer_index_url"></field>
@@ -383,10 +357,8 @@
   <object pk="1" model="orm.layer_version">
     <field to="orm.build" name="build" rel="ManyToOneRel">1</field>
     <field to="orm.layer" name="layer" rel="ManyToOneRel">1</field>
-    <field to="orm.layersource" name="layer_source" rel="ManyToOneRel">1</field>
-    <field type="IntegerField" name="up_id"><None></None></field>
     <field type="DateTimeField" name="up_date"><None></None></field>
-    <field to="orm.branch" name="up_branch" rel="ManyToOneRel">1</field>
+    <field to="orm.release" name="release" rel="ManyToOneRel">1</field>
     <field type="CharField" name="branch">master</field>
     <field type="CharField" name="commit">abcdef123</field>
     <field type="CharField" name="dirpath">/tmp/</field>
@@ -397,10 +369,8 @@
   <object pk="2" model="orm.layer_version">
     <field to="orm.build" name="build" rel="ManyToOneRel"><None></None></field>
     <field to="orm.layer" name="layer" rel="ManyToOneRel">2</field>
-    <field to="orm.layersource" name="layer_source" rel="ManyToOneRel">1</field>
-    <field type="IntegerField" name="up_id"><None></None></field>
     <field type="DateTimeField" name="up_date"><None></None></field>
-    <field to="orm.branch" name="up_branch" rel="ManyToOneRel">1</field>
+    <field to="orm.release" name="release" rel="ManyToOneRel">1</field>
     <field type="CharField" name="branch">testing-branch</field>
     <field type="CharField" name="commit">9876fedcba</field>
     <field type="CharField" name="dirpath"><None></None></field>
@@ -411,10 +381,8 @@
   <object pk="3" model="orm.layer_version">
     <field to="orm.build" name="build" rel="ManyToOneRel">1</field>
     <field to="orm.layer" name="layer" rel="ManyToOneRel">2</field>
-    <field to="orm.layersource" name="layer_source" rel="ManyToOneRel">1</field>
-    <field type="IntegerField" name="up_id"><None></None></field>
     <field type="DateTimeField" name="up_date"><None></None></field>
-    <field to="orm.branch" name="up_branch" rel="ManyToOneRel">1</field>
+    <field to="orm.release" name="release" rel="ManyToOneRel">1</field>
     <field type="CharField" name="branch">testing-branch</field>
     <field type="CharField" name="commit">9876fedcba</field>
     <field type="CharField" name="dirpath"><None></None></field>

bitbake/lib/toaster/toastergui/static/css/default.css

@@ -45,6 +45,7 @@ img.logo { height: 30px; vertical-align: bottom; }
 .alert-link.build-warnings,
 .glyphicon-warning-sign.build-warnings { color: #8a6d3b; }
 .build-result .project-name { margin-top: -10px; margin-bottom: 5px; }
+.rebuild-btn, .cancel-build-btn { cursor: pointer; }
 
 /* Styles for the help information */
 .get-help { color: #CCCCCC; }
@@ -195,17 +196,22 @@ h2 { margin-bottom: 25px; }
 .tt-suggestion:active { background-color: #f5f5f5; cursor: pointer; }
 
 /* Style the import layer form controls*/
-legend { border: none; }
+legend { border: none; margin-top: 20px; }
+.radioLegend { margin-bottom: 0; }
 #layer-name-ctrl { margin-top: 20px; }
 #import-layer-name,
 #layer-subdir { width: 20%; }
 #layer-git-repo-url { width: 40%; }
 #layer-git-ref { width: 32%; }
+#local-dir-path { width: 45%; }
 #layer-dependency { width: 16em; }
+#layer-deps-list { margin-top: 0; }
 #form-actions { margin-bottom: 30px; }
 #duplicate-layer-info dl { margin-top: 10px; }
 #duplicate-layer-info dd { margin-bottom: 10px; }
 .help-inline { color: #737373; margin-left: 10px; }
+.radio-help { width: 50%; margin-left: 20px; }
+#repo-select div:nth-of-type(2) { margin-top: 15px; }
 
 /* Give some padding to the in-cell tooltips we use for notifications in tables */
 td > .tooltip-inner,
@@ -249,6 +255,18 @@ code { color: #333; background-color: transparent; }
 /* Style the special no results message in the custom image details page */
 [id^="no-results-special-"] > .alert-warning > ol { margin-top: 10px; }
 
+/* style the loading spinner in the new custom image dialog */
+#create-new-custom-image-btn [data-role="loading-state"] {
+  padding-left: 16px;
+}
+
+/* icon has to be absolutely positioned, otherwise the spin animation doesn't work */
+#create-new-custom-image-btn [data-role="loading-state"] .icon-spinner {
+  position: absolute;
+  left: 26px;
+  bottom: 26px;
+}
+
 /* Style the content of modal dialogs */
 .modal-footer { text-align: left; }
 .date-filter-controls { margin-top: 10px; }

bitbake/lib/toaster/toastergui/static/js/importlayer.js

@@ -10,6 +10,7 @@ function importLayerPageInit (ctx) {
   var layerDepInput = $("#layer-dependency");
   var layerNameCtrl = $("#layer-name-ctrl");
   var duplicatedLayerName = $("#duplicated-layer-name-hint");
+  var localDirPath = $("#local-dir-path");
 
   var layerDeps = {};
   var layerDepsDeps = {};
@@ -35,6 +36,9 @@ function importLayerPageInit (ctx) {
     }
   });
 
+  // Disable local dir repo when page is loaded.
+  $('#local-dir').hide();
+
   // disable the "Add layer" button when the layer input typeahead is empty
   // or not in the typeahead choices
   layerDepInput.on("input change", function () {
@@ -168,8 +172,16 @@ function importLayerPageInit (ctx) {
         dir_path: $("#layer-subdir").val(),
         project_id: libtoaster.ctx.projectId,
         layer_deps: layerDepsCsv,
+        local_source_dir: $('#local-dir-path').val(),
       };
 
+      if ($('input[name=repo]:checked').val() == "git") {
+        layerData.local_source_dir = "";
+      } else {
+        layerData.vcs_url = "";
+        layerData.git_ref = "";
+      }
+
       $.ajax({
           type: "POST",
           url: ctx.xhrImportLayerUrl,
@@ -208,25 +220,45 @@ function importLayerPageInit (ctx) {
   function check_form() {
     var valid = false;
     var inputs = $("input:required");
+    var inputStr = inputs.val().split("");
 
-    for (var i=0; i<inputs.length; i++){
-      if (!(valid = inputs[i].value)){
+    for (var i=0; i<inputs.val().length; i++){
+      if (!(valid = inputStr[i])){
         enable_import_btn(false);
         break;
       }
     }
 
-    if (valid)
-      enable_import_btn(true);
+    if (valid) {
+      if ($("#local-dir-radio").prop("checked") && localDirPath.val().length > 0) {
+        enable_import_btn(true);
+      }
+      if ($("#git-repo-radio").prop("checked") && vcsURLInput.val().length > 0 && gitRefInput.val().length > 0) {
+        enable_import_btn(true);
+      }
+    }
+
+    if (inputs.val().length == 0)
+      enable_import_btn(false);
   }
 
   function layerExistsError(layer){
     var dupLayerInfo = $("#duplicate-layer-info");
-    dupLayerInfo.find(".dup-layer-name").text(layer.name);
-    dupLayerInfo.find(".dup-layer-link").attr("href", layer.layerdetailurl);
-    dupLayerInfo.find("#dup-layer-vcs-url").text(layer.vcs_url);
-    dupLayerInfo.find("#dup-layer-revision").text(layer.vcs_reference);
 
+    if (layer.local_source_dir) {
+      $("#git-layer-dup").hide();
+      $("#local-layer-dup").fadeIn();
+      dupLayerInfo.find(".dup-layer-name").text(layer.name);
+      dupLayerInfo.find(".dup-layer-link").attr("href", layer.layerdetailurl);
+      dupLayerInfo.find("#dup-local-source-dir-name").text(layer.local_source_dir);
+    } else {
+      $("#git-layer-dup").fadeIn();
+      $("#local-layer-dup").hide();
+      dupLayerInfo.find(".dup-layer-name").text(layer.name);
+      dupLayerInfo.find(".dup-layer-link").attr("href", layer.layerdetailurl);
+      dupLayerInfo.find("#dup-layer-vcs-url").text(layer.vcs_url);
+      dupLayerInfo.find("#dup-layer-revision").text(layer.vcs_reference);
+    }
     $(".fields-apart-from-layer-name").fadeOut(function(){
 
       dupLayerInfo.fadeIn();
@@ -271,10 +303,13 @@ function importLayerPageInit (ctx) {
 
     if ($("#duplicate-layer-info").css("display") != "None"){
       $("#duplicate-layer-info").fadeOut(function(){
-        $(".fields-apart-from-layer-name").show();
-      });
+      $(".fields-apart-from-layer-name").show();
+      radioDisplay();
+    });
 
-    }
+  }
+
+    radioDisplay();
 
     /* Don't remove the error class if we're displaying the error for another
      * reason.
@@ -301,4 +336,72 @@ function importLayerPageInit (ctx) {
     }
   });
 
+  function radioDisplay() {
+    if ($('input[name=repo]:checked').val() == "local") {
+      $('#git-repo').hide();
+      $('#import-git-layer-and-add-hint').hide();
+      $('#local-dir').fadeIn();
+      $('#import-local-dir-and-add-hint').fadeIn();
+    } else {
+      $('#local-dir').hide();
+      $('#import-local-dir-and-add-hint').hide();
+      $('#git-repo').fadeIn();
+      $('#import-git-layer-and-add-hint').fadeIn();
+    }
+  }
+
+  $('input:radio[name="repo"]').change(function() {
+    radioDisplay();
+    if ($("#local-dir-radio").prop("checked")) {
+      if (localDirPath.val().length > 0) {
+        enable_import_btn(true);
+      } else {
+        enable_import_btn(false);
+      }
+    }
+    if ($("#git-repo-radio").prop("checked")) {
+      if (vcsURLInput.val().length > 0 && gitRefInput.val().length > 0) {
+        enable_import_btn(true);
+      } else {
+        enable_import_btn(false);
+      }
+    }
+  });
+
+  localDirPath.on('input', function(){
+    if ($(this).val().trim().length == 0) {
+      $('#import-and-add-btn').attr("disabled","disabled");
+      $('#local-dir').addClass('has-error');
+      $('#hintError-dir-abs-path').show();
+      $('#hintError-dir-path-starts-with-slash').show();
+    } else {
+      var input = $(this);
+      var reBeginWithSlash = /^\//;
+      var reCheckVariable = /^\$/;
+      var re = /([ <>\\|":\.%\?\*]+)/;
+
+      var invalidDir = re.test(input.val());
+      var invalidSlash = reBeginWithSlash.test(input.val());
+      var invalidVar = reCheckVariable.test(input.val());
+
+      if (!invalidSlash && !invalidVar) {
+        $('#local-dir').addClass('has-error');
+        $('#import-and-add-btn').attr("disabled","disabled");
+        $('#hintError-dir-abs-path').show();
+        $('#hintError-dir-path-starts-with-slash').show();
+      } else if (invalidDir) {
+        $('#local-dir').addClass('has-error');
+        $('#import-and-add-btn').attr("disabled","disabled");
+        $('#hintError-dir-path').show();
+      } else {
+        $('#local-dir').removeClass('has-error');
+        if (layerNameInput.val().length > 0) {
+          $('#import-and-add-btn').removeAttr("disabled");
+        }
+        $('#hintError-dir-abs-path').hide();
+        $('#hintError-dir-path-starts-with-slash').hide();
+        $('#hintError-dir-path').hide();
+      }
+    }
+  });
 }

bitbake/lib/toaster/toastergui/static/js/layerBtn.js

@@ -55,8 +55,8 @@ function layerBtnsInit() {
     });
   });
 
-  $(".build-recipe-btn").unbind('click');
-  $(".build-recipe-btn").click(function(e){
+  $("td .build-recipe-btn").unbind('click');
+  $("td .build-recipe-btn").click(function(e){
     e.preventDefault();
     var recipe = $(this).data('recipe-name');
 

bitbake/lib/toaster/toastergui/static/js/layerdetails.js

@@ -366,7 +366,7 @@ function layerDetailsPageInit (ctx) {
     if ($(this).is("dt")) {
       var dd = $(this).next("dd");
       if (!dd.children("form:visible")|| !dd.find(".current-value").html()){
-        if (ctx.layerVersion.sourceId == 3){
+        if (ctx.layerVersion.layer_source == ctx.layerSourceTypes.TYPE_IMPORTED){
         /* There's no current value and the layer is editable
          * so show the "Not set" and hide the delete icon
          */
@@ -397,6 +397,26 @@ function layerDetailsPageInit (ctx) {
     $(this).parents("form").submit();
   });
 
+  $("#layer-delete-confirmed").click(function(){
+    $.cookie("layer-deleted", ctx.layerVersion.name, { path: '/'});
+
+    $.ajax({
+        type: "DELETE",
+        url: ctx.xhrUpdateLayerUrl,
+        headers: { 'X-CSRFToken' : $.cookie('csrftoken')},
+        success: function(data) {
+          if (data.error != "ok") {
+            console.warn(data.error);
+          } else {
+            window.location = data.redirect + "?notify=layer-deleted";
+          }
+        },
+        error: function(data) {
+          console.warn("Call failed");
+          console.warn(data);
+        }
+    });
+  });
 
   layerDepsList.find(".glyphicon-trash").click(layerDepRemoveClick);
   layerDepsList.find("a").tooltip();

bitbake/lib/toaster/toastergui/static/js/libtoaster.js

@@ -148,6 +148,21 @@ var libtoaster = (function () {
     });
   }
 
+  function _getMostRecentBuilds(url, onsuccess, onfail) {
+    $.ajax({
+      url: url,
+      type: 'GET',
+      data : {format: 'json'},
+      headers: {'X-CSRFToken': $.cookie('csrftoken')},
+      success: function (data) {
+        onsuccess ? onsuccess(data) : console.log(data);
+      },
+      error: function (data) {
+        onfail ? onfail(data) : console.error(data);
+      }
+    });
+  }
+
   /* Get a project's configuration info */
   function _getProjectInfo(url, onsuccess, onfail){
     $.ajax({
@@ -421,11 +436,27 @@ var libtoaster = (function () {
      });
   }
 
+  // if true, the loading spinner for Ajax requests will be displayed
+  // if requests take more than 1200ms
+  var ajaxLoadingTimerEnabled = true;
+
+  // turn on the page-level loading spinner for Ajax requests
+  function _enableAjaxLoadingTimer() {
+    ajaxLoadingTimerEnabled = true;
+  }
+
+  // turn off the page-level loading spinner for Ajax requests
+  function _disableAjaxLoadingTimer() {
+    ajaxLoadingTimerEnabled = false;
+  }
 
   return {
+    enableAjaxLoadingTimer: _enableAjaxLoadingTimer,
+    disableAjaxLoadingTimer: _disableAjaxLoadingTimer,
     reload_params : reload_params,
     startABuild : _startABuild,
     cancelABuild : _cancelABuild,
+    getMostRecentBuilds: _getMostRecentBuilds,
     makeTypeahead : _makeTypeahead,
     getProjectInfo: _getProjectInfo,
     getLayerDepsForProject : _getLayerDepsForProject,
@@ -469,7 +500,6 @@ function reload_params(params) {
     window.location.href = url+"?"+callparams.join('&');
 }
 
-
 /* Things that happen for all pages */
 $(document).ready(function() {
 
@@ -628,7 +658,9 @@ $(document).ready(function() {
         window.clearTimeout(ajaxLoadingTimer);
 
       ajaxLoadingTimer = window.setTimeout(function() {
-        $("#loading-notification").fadeIn();
+        if (libtoaster.ajaxLoadingTimerEnabled) {
+          $("#loading-notification").fadeIn();
+        }
       }, 1200);
     });
 

bitbake/lib/toaster/toastergui/static/js/mrbsection.js

@@ -1,33 +1,19 @@
 
 function mrbSectionInit(ctx){
-
-  var projectBuilds;
-
-  if (ctx.mrbType === 'project')
-    projectBuilds = true;
-
-  $(".cancel-build-btn").click(function(e){
+  $('#latest-builds').on('click', '.cancel-build-btn', function(e){
+    e.stopImmediatePropagation();
     e.preventDefault();
 
     var url = $(this).data('request-url');
     var buildReqIds = $(this).data('buildrequest-id');
-    var banner = $(this).parents(".alert");
 
-    banner.find(".progress-info").fadeOut().promise().done(function(){
-      $("#cancelling-msg-" + buildReqIds).show();
-      console.log("cancel build");
-      libtoaster.cancelABuild(url, buildReqIds, function(){
-        if (projectBuilds == false){
-          /* the all builds page is not 'self updating' like thei
-           * project Builds
-           */
-          window.location.reload();
-        }
-      }, null);
-    });
+    libtoaster.cancelABuild(url, buildReqIds, function () {
+      window.location.reload();
+    }, null);
   });
 
-  $(".run-again-btn").click(function(e){
+  $('#latest-builds').on('click', '.rebuild-btn', function(e){
+    e.stopImmediatePropagation();
     e.preventDefault();
 
     var url = $(this).data('request-url');
@@ -38,58 +24,112 @@ function mrbSectionInit(ctx){
     }, null);
   });
 
+  // cached version of buildData, so we can determine whether a build has
+  // changed since it was last fetched, and update the DOM appropriately
+  var buildData = {};
 
-  var progressTimer;
+  // returns the cached version of this build, or {} is there isn't a cached one
+  function getCached(build) {
+    return buildData[build.id] || {};
+  }
 
-  if (projectBuilds === true){
-    progressTimer = window.setInterval(function() {
-      libtoaster.getProjectInfo(libtoaster.ctx.projectPageUrl,
-        function(prjInfo){
-          /* These two are needed because a build can be 100% and still
-           * in progress due to the fact that the % done is updated at the
-           * start of a task so it can be doing the last task at 100%
-           */
-          var inProgress = 0;
-          var allPercentDone = 0;
-          if (prjInfo.builds.length === 0)
-            return
+  // returns true if a build's state changed to "Succeeded", "Failed"
+  // or "Cancelled" from some other value
+  function buildFinished(build) {
+    var cached = getCached(build);
+    return cached.state &&
+      cached.state !== build.state &&
+      (build.state == 'Succeeded' || build.state == 'Failed' ||
+       build.state == 'Cancelled');
+  }
 
-          for (var i in prjInfo.builds){
-            var build = prjInfo.builds[i];
+  // returns true if the state changed
+  function stateChanged(build) {
+    var cached = getCached(build);
+    return (cached.state !== build.state);
+  }
 
-            if (build.outcomeText === "In Progress" ||
-               $(".progress .bar").length > 0){
-              /* Update the build progress */
-              var percentDone;
+  // returns true if the tasks_complete_percentage changed
+  function tasksProgressChanged(build) {
+    var cached = getCached(build);
+    return (cached.tasks_complete_percentage !== build.tasks_complete_percentage);
+  }
 
-              if (build.outcomeText !== "In Progress"){
-                /* We have to ignore the value when it's Succeeded because it
-                *   goes back to 0
-                */
-                percentDone = 100;
-              } else {
-                percentDone = build.percentDone;
-                inProgress++;
-              }
+  // returns true if the number of recipes parsed/to parse changed
+  function recipeProgressChanged(build) {
+    var cached = getCached(build);
+    return (cached.recipes_parsed_percentage !== build.recipes_parsed_percentage);
+  }
 
-              $("#build-pc-done-" + build.id).text(percentDone);
-              $("#build-pc-done-title-" + build.id).attr("title", percentDone);
-              $("#build-pc-done-bar-" + build.id).css("width",
-                String(percentDone) + "%");
+  function refreshMostRecentBuilds(){
+    libtoaster.getMostRecentBuilds(
+      libtoaster.ctx.mostRecentBuildsUrl,
 
-              allPercentDone += percentDone;
-            }
+      // success callback
+      function (data) {
+        var build;
+        var tmpl;
+        var container;
+        var selector;
+        var colourClass;
+        var elements;
+
+        for (var i = 0; i < data.length; i++) {
+          build = data[i];
+
+          if (buildFinished(build)) {
+            // a build finished: reload the whole page so that the build
+            // shows up in the builds table
+            window.location.reload();
+          }
+          else if (stateChanged(build)) {
+            // update the whole template
+            build.warnings_pluralise = (build.warnings !== 1 ? 's' : '');
+            build.errors_pluralise = (build.errors !== 1 ? 's' : '');
+
+            tmpl = $.templates("#build-template");
+
+            html = $(tmpl.render(build));
+
+            selector = '[data-latest-build-result="' + build.id + '"] ' +
+              '[data-role="build-status-container"]';
+            container = $(selector);
+
+            // initialize bootstrap tooltips in the new HTML
+            html.find('span.glyphicon-question-sign').tooltip();
+
+            container.html(html);
+          }
+          else if (tasksProgressChanged(build)) {
+            // update the task progress text
+            selector = '#build-pc-done-' + build.id;
+            $(selector).html(build.tasks_complete_percentage);
+
+            // update the task progress bar
+            selector = '#build-pc-done-bar-' + build.id;
+            $(selector).width(build.tasks_complete_percentage + '%');
+          }
+          else if (recipeProgressChanged(build)) {
+            // update the recipe progress text
+            selector = '#recipes-parsed-percentage-' + build.id;
+            $(selector).html(build.recipes_parsed_percentage);
+
+            // update the recipe progress bar
+            selector = '#recipes-parsed-percentage-bar-' + build.id;
+            $(selector).width(build.recipes_parsed_percentage + '%');
           }
 
-          if (allPercentDone === (100 * prjInfo.builds.length) && !inProgress)
-            window.location.reload();
+          buildData[build.id] = build;
+        }
+      },
 
-          /* Our progress bar is not still showing so shutdown the polling. */
-          if ($(".progress .bar").length === 0)
-            window.clearInterval(progressTimer);
-
-      });
-    }, 1500);
+      // fail callback
+      function (data) {
+        console.error(data);
+      }
+    );
   }
-}
 
+  window.setInterval(refreshMostRecentBuilds, 1500);
+  refreshMostRecentBuilds();
+}

bitbake/lib/toaster/toastergui/static/js/newcustomimage_modal.js

@@ -25,7 +25,11 @@ function newCustomImageModalInit(){
   var duplicateNameMsg = "An image with this name already exists. Image names must be unique.";
   var duplicateImageInProjectMsg = "An image with this name already exists in this project."
   var invalidBaseRecipeIdMsg = "Please select an image to customise.";
-	
+
+  // set button to "submit" state and enable text entry so user can
+  // enter the custom recipe name
+  showSubmitState();
+
   /* capture clicks on radio buttons inside the modal; when one is selected,
    * set the recipe on the modal
    */
@@ -40,6 +44,9 @@ function newCustomImageModalInit(){
   });
 
   newCustomImgBtn.click(function(e){
+    // disable the button and text entry
+    showLoadingState();
+
     e.preventDefault();
 
     var baseRecipeId = imgCustomModal.data('recipe');
@@ -69,12 +76,33 @@ function newCustomImageModalInit(){
           }
         } else {
           imgCustomModal.modal('hide');
+          imgCustomModal.one('hidden.bs.modal', showSubmitState);
           window.location.replace(ret.url + '?notify=new');
         }
       });
     }
   });
 
+  // enable text entry, show "Create image" button text
+  function showSubmitState() {
+    libtoaster.enableAjaxLoadingTimer();
+    newCustomImgBtn.find('[data-role="loading-state"]').hide();
+    newCustomImgBtn.find('[data-role="submit-state"]').show();
+    newCustomImgBtn.removeAttr('disabled');
+    nameInput.removeAttr('disabled');
+  }
+
+  // disable text entry, show "Creating image..." button text;
+  // we also disabled the page-level ajax loading spinner while this spinner
+  // is active
+  function showLoadingState() {
+    libtoaster.disableAjaxLoadingTimer();
+    newCustomImgBtn.find('[data-role="submit-state"]').hide();
+    newCustomImgBtn.find('[data-role="loading-state"]').show();
+    newCustomImgBtn.attr('disabled', 'disabled');
+    nameInput.attr('disabled', 'disabled');
+  }
+
   function showNameError(text){
     invalidNameHelp.text(text);
     invalidNameHelp.show();
@@ -167,6 +195,5 @@ function newCustomImageModalSetRecipes(baseRecipes) {
 
     // show the radio button container
     imageSelector.show();
-	 
-  }	
+  }
 }

bitbake/lib/toaster/toastergui/static/js/projectpage.js

@@ -58,12 +58,30 @@ function projectPageInit(ctx) {
         case 'layer-imported':
           layerImportedNotification();
           break;
+        case 'layer-deleted':
+          layerDeletedNotification();
         default:
           break;
       }
     }
   })();
 
+  /* Layer deleted notification */
+  function layerDeletedNotification(){
+    var layer = $.cookie("layer-deleted");
+
+    if (!layer)
+      return;
+
+    var message = "You have deleted <strong>1</strong> layer from your ";
+    message += "project: <strong>" + layer + "</strong>";
+
+    libtoaster.showChangeNotification(message);
+
+    $.removeCookie("layer-deleted", { path: "/"});
+  }
+
+
   /* Layer imported notification */
   function layerImportedNotification(){
     var imported = $.cookie("layer-imported-alert");
@@ -154,7 +172,12 @@ function projectPageInit(ctx) {
 
       link.attr("href", layerObj.layerdetailurl);
       link.text(layerObj.name);
-      link.tooltip({title: layerObj.vcs_url + " | "+ layerObj.vcs_reference, placement: "right"});
+
+      if (layerObj.local_source_dir) {
+        link.tooltip({title: layerObj.local_source_dir, placement: "right"});
+      } else {
+        link.tooltip({title: layerObj.vcs_url + " | "+ layerObj.vcs_reference, placement: "right"});
+      }
 
       var trashItem = projectLayer.children("span");
       trashItem.click(function (e) {

bitbake/lib/toaster/toastergui/static/js/table.js

@@ -15,6 +15,7 @@ function tableInit(ctx){
     orderby : null,
     filter : null,
     search : null,
+    default_orderby: null,
   };
 
   var defaultHiddenCols = [];
@@ -192,6 +193,8 @@ function tableInit(ctx){
     tableHeadRow.html("");
     editColMenu.html("");
 
+    tableParams.default_orderby = tableData.default_orderby;
+
     if (!tableParams.orderby && tableData.default_orderby){
       tableParams.orderby = tableData.default_orderby;
     }
@@ -217,6 +220,7 @@ function tableInit(ctx){
         var title = $('<a href=\"#\" ></a>');
 
         title.data('field-name', col.field_name);
+        title.attr('data-sort-field', col.field_name);
         title.text(col.title);
         title.click(sortColumnClicked);
 
@@ -344,29 +348,65 @@ function tableInit(ctx){
     }
   }
 
+  /* Apply an ordering to the current table.
+   *
+   * 1. Find the column heading matching the sortSpecifier
+   * 2. Set its up/down arrow and add .sorted
+   *
+   * orderby: e.g. "-started_on", "completed_on"
+   * colHeading: column heading element to activate (by showing the caret
+   * up/down, depending on sort order); if not set, the correct column
+   * heading is selected from the DOM using orderby as a key
+   */
+  function applyOrderby(orderby, colHeading) {
+    if (!orderby) {
+      return;
+    }
+
+    // We only have one sort at a time so remove existing sort indicators
+    $("#" + ctx.tableName + " th .icon-caret-down").hide();
+    $("#" + ctx.tableName + " th .icon-caret-up").hide();
+    $("#" + ctx.tableName + " th a").removeClass("sorted");
+
+    // normalise the orderby so we can use it to find the link we want
+    // to style
+    var fieldName = orderby;
+    if (fieldName.indexOf('-') === 0) {
+      fieldName = fieldName.slice(1);
+    }
+
+    // find the table header element which corresponds to the sort field
+    // (if we don't already have it)
+    if (!colHeading) {
+      colHeading = $('[data-sort-field="' + fieldName + '"]');
+    }
+
+    colHeading.addClass("sorted");
+
+    var parent = colHeading.parent();
+
+    if (orderby.indexOf('-') === 0) {
+      parent.children('.icon-caret-up').show();
+    }
+    else {
+      parent.children('.icon-caret-down').show();
+    }
+
+    tableParams.orderby = orderby;
+    loadData(tableParams);
+  }
+
   function sortColumnClicked(e){
     e.preventDefault();
 
-    /* We only have one sort at a time so remove any existing sort indicators */
-    $("#"+ctx.tableName+" th .icon-caret-down").hide();
-    $("#"+ctx.tableName+" th .icon-caret-up").hide();
-    $("#"+ctx.tableName+" th a").removeClass("sorted");
-
-    var fieldName = $(this).data('field-name');
-
     /* if we're already sorted sort the other way */
-    if (tableParams.orderby === fieldName &&
+    var orderby = $(this).data('field-name');
+    if (tableParams.orderby === orderby &&
         tableParams.orderby.indexOf('-') === -1) {
-      tableParams.orderby = '-' + $(this).data('field-name');
-      $(this).parent().children('.icon-caret-up').show();
-    } else {
-      tableParams.orderby = $(this).data('field-name');
-      $(this).parent().children('.icon-caret-down').show();
+      orderby = '-' + orderby;
     }
 
-    $(this).addClass("sorted");
-
-    loadData(tableParams);
+    applyOrderby(orderby, $(this));
   }
 
   function pageButtonClicked(e) {
@@ -385,11 +425,13 @@ function tableInit(ctx){
       table.find("."+col).show();
     }  else {
       table.find("."+col).hide();
-      /* If we're ordered by the column we're hiding remove the order by */
+      // If we're ordered by the column we're hiding remove the order by
+      // and apply the default one instead
       if (col === tableParams.orderby ||
           '-' + col === tableParams.orderby){
         tableParams.orderby = null;
-        $("#"+ctx.tableName +" .default-orderby").click();
+
+        applyOrderby(tableParams.default_orderby);
       }
     }
 

bitbake/lib/toaster/toastergui/tables.py

@@ -114,8 +114,12 @@ class LayersTable(ToasterTable):
 
         git_url_template = '''
         <a href="{% url 'layerdetails' extra.pid data.id %}">
+        {% if data.layer.local_source_dir %}
+          <code>{{data.layer.local_source_dir}}</code>
+        {% else %}
           <code>{{data.layer.vcs_url}}</code>
         </a>
+        {% endif %}
         {% if data.get_vcs_link_url %}
         <a target="_blank" href="{{ data.get_vcs_link_url }}">
            <span class="glyphicon glyphicon-new-window"></span>
@@ -123,16 +127,21 @@ class LayersTable(ToasterTable):
         {% endif %}
         '''
 
-        self.add_column(title="Git repository URL",
-                        help_text="The Git repository for the layer source code",
+        self.add_column(title="Layer source code location",
+                        help_text="A Git repository or an absolute path to a directory",
                         hidden=True,
                         static_data_name="layer__vcs_url",
                         static_data_template=git_url_template)
 
         git_dir_template = '''
+        {% if data.layer.local_source_dir %}
+        <span class="text-muted">Not applicable</span>
+        <span class="glyphicon glyphicon-question-sign get-help" data-original-title="" title="The source code of {{data.layer.name}} is not in a Git repository, so there is no subdirectory associated with it"> </span>
+        {% else %}
         <a href="{% url 'layerdetails' extra.pid data.id %}">
          <code>{{data.dirpath}}</code>
         </a>
+        {% endif %}
         {% if data.dirpath and data.get_vcs_dirpath_link_url %}
         <a target="_blank" href="{{ data.get_vcs_dirpath_link_url }}">
           <span class="glyphicon glyphicon-new-window"></span>
@@ -146,9 +155,14 @@ class LayersTable(ToasterTable):
                         static_data_template=git_dir_template)
 
         revision_template =  '''
+        {% if data.layer.local_source_dir %}
+        <span class="text-muted">Not applicable</span>
+        <span class="glyphicon glyphicon-question-sign get-help" data-original-title="" title="The source code of {{data.layer.name}} is not in a Git repository, so there is no revision associated with it"> </span>
+        {% else %}
         {% with vcs_ref=data.get_vcs_reference %}
         {% include 'snippets/gitrev_popover.html' %}
         {% endwith %}
+        {% endif %}
         '''
 
         self.add_column(title="Git revision",
@@ -413,9 +427,19 @@ class RecipesTable(ToasterTable):
                         orderable=True,
                         field_name="license")
 
+        revision_link_template = '''
+        {% if data.layer_version.layer.local_source_dir %}
+        <span class="text-muted">Not applicable</span>
+        <span class="glyphicon glyphicon-question-sign get-help" data-original-title="" title="The source code of {{data.layer_version.layer.name}} is not in a Git repository, so there is no revision associated with it"> </span>
+        {% else %}
+        {{data.layer_version.get_vcs_reference}}
+        {% endif %}
+        '''
+
         self.add_column(title="Git revision",
                         hidden=True,
-                        field_name="layer_version__get_vcs_reference")
+                        static_data_name="layer_version__get_vcs_reference",
+                        static_data_template=revision_link_template)
 
 
 class LayerRecipesTable(RecipesTable):
@@ -1197,9 +1221,13 @@ class BuildsTable(ToasterTable):
 
         time_template = '''
         {% load projecttags %}
-        <a href="{% url "buildtime" data.id %}">
+        {% if data.outcome == extra.Build.SUCCEEDED %}
+            <a href="{% url "buildtime" data.id %}">
+                {{data.timespent_seconds | sectohms}}
+            </a>
+        {% else %}
             {{data.timespent_seconds | sectohms}}
-        </a>
+        {% endif %}
         '''
 
         image_files_template = '''

bitbake/lib/toaster/toastergui/templates/base.html

@@ -22,6 +22,8 @@
     </script>
     <script src="{% static 'js/typeahead.jquery.js' %}">
     </script>
+    <script src="{% static 'js/jsrender.min.js' %}">
+    </script>
     <script src="{% static 'js/prettify.js' %}">
     </script>
     <script src="{% static 'js/libtoaster.js' %}">
@@ -32,6 +34,8 @@
     </script>
     {% endif %}
     <script>
+      $.views.settings.delimiters("<%", "%>");
+
       libtoaster.ctx = {
         jsUrl : "{% static 'js/' %}",
         htmlUrl : "{% static 'html/' %}",
@@ -48,7 +52,9 @@
         xhrCustomRecipeUrl : "{% url 'xhr_customrecipe' %}",
         projectId : {{project.id}},
         xhrBuildRequestUrl: "{% url 'xhr_buildrequest' project.id %}",
+        mostRecentBuildsUrl: "{% url 'most_recent_builds' %}?project_id={{project.id}}",
         {% else %}
+        mostRecentBuildsUrl: "{% url 'most_recent_builds' %}",
         projectId : undefined,
         projectPageUrl : undefined,
         projectName : undefined,

bitbake/lib/toaster/toastergui/templates/basebuildpage.html

@@ -33,7 +33,7 @@
 
         $("#build-menu li a").each(function(){
           /* Set the page active state in the Build menu */
-          var currentUrl = window.location.href.split('?')[0]; 
+          var currentUrl = window.location.href.split('?')[0];
           if (currentUrl === $(this).prop("href")){
             $(this).parent().addClass("active");
           } else {
@@ -52,100 +52,106 @@
   </div>
 </div>
 
-  <div class="row">
-    <!-- begin left sidebar container -->
-    <div id="nav" class="col-md-2">
-      <ul class="nav nav-pills nav-stacked" id="build-menu">
-        <li
-          {% if request.resolver_match.url_name == 'builddashboard'  %}
-            class="active"
-          {% endif %} >
-          <a  href="{% url 'builddashboard' build.pk %}">Build summary</a>
-        </li>
-        {% if build.target_set.all.0.is_image and build.outcome == 0 %}
-          <li class="nav-header">Images</li>
-          {% block nav-target %}
-            {% for t in build.get_sorted_target_list %}
-            <li id="menu-{{t.target}}"><a href="{% url 'target' build.pk t.pk %}">{{t.target}}</a><li>
-            {% endfor %}
-          {% endblock %}
-        {% endif %}
-        <li class="nav-header">Build</li>
-          <li id="menu-configuration"><a href="{% url 'configuration' build.pk %}">Configuration</a></li>
-          <li><a href="{% url 'tasks' build.pk %}">Tasks</a></li>
-          <li><a href="{% url 'recipes' build.pk %}">Recipes</a></li>
-          <li><a href="{% url 'packages' build.pk %}">Packages</a></li>
-          <li class="nav-header">Performance</li>
-          <li><a href="{% url 'buildtime' build.pk %}">Time</a></li>
-          <li><a href="{% url 'cputime' build.pk %}">CPU usage</a></li>
-          <li><a href="{% url 'diskio' build.pk %}">Disk I/O</a></li>
+  <!-- begin left sidebar container for builds which started properly -->
+  {% if build.started %}
+    <div class="row">
+      <div id="nav" class="col-md-2">
+        <ul class="nav nav-pills nav-stacked" id="build-menu">
+          <li
+            {% if request.resolver_match.url_name == 'builddashboard'  %}
+              class="active"
+            {% endif %} >
+            <a  href="{% url 'builddashboard' build.pk %}">Build summary</a>
+          </li>
+          {% if build.has_images and build.outcome == build.SUCCEEDED %}
+            <li class="nav-header" data-menu-heading="images">Images</li>
+            {% block nav-target %}
+              {% for t in build.get_sorted_target_list %}
+                {% if t.has_images %}
+                  <li id="menu-{{t.target}}"><a href="{% url 'target' build.pk t.pk %}">{{t.target}}</a><li>
+                {% endif %}
+              {% endfor %}
+            {% endblock %}
+          {% endif %}
+          <li class="nav-header">Build</li>
+            <li id="menu-configuration"><a href="{% url 'configuration' build.pk %}">Configuration</a></li>
+            <li><a href="{% url 'tasks' build.pk %}">Tasks</a></li>
+            <li><a href="{% url 'recipes' build.pk %}">Recipes</a></li>
+            <li><a href="{% url 'packages' build.pk %}">Packages</a></li>
+            <li class="nav-header">Performance</li>
+            <li><a href="{% url 'buildtime' build.pk %}">Time</a></li>
+            <li><a href="{% url 'cputime' build.pk %}">CPU usage</a></li>
+            <li><a href="{% url 'diskio' build.pk %}">Disk I/O</a></li>
 
-	<li class="nav-header">Actions</li>
-       	<a class="btn btn-default btn-block navbar-btn" href="{% url 'build_artifact' build.id 'cookerlog' build.id %}">Download build log</a>
+	  <li class="nav-header">Actions</li>
+         	<a class="btn btn-default btn-block navbar-btn" href="{% url 'build_artifact' build.id 'cookerlog' build.id %}">Download build log</a>
 
-        {% with build.get_custom_image_recipes as custom_image_recipes %}
-          {% if custom_image_recipes.count > 0 %}
-            <!-- edit custom image built during this build -->
-              <button class="btn btn-default btn-block navbar-btn" data-role="edit-custom-image-trigger">Edit custom image</button>
-              {% include 'editcustomimage_modal.html' %}
+          {% with build.get_custom_image_recipes as custom_image_recipes %}
+            {% if custom_image_recipes.count > 0 %}
+              <!-- edit custom image built during this build -->
+                <button class="btn btn-default btn-block navbar-btn" data-role="edit-custom-image-trigger">Edit custom image</button>
+                {% include 'editcustomimage_modal.html' %}
+                <script>
+                  var editableCustomImageRecipes = {{ custom_image_recipes | objects_to_dictionaries:"id,name" | json }};
+
+                  $(document).ready(function () {
+                    var editCustomImageTrigger = $('[data-role="edit-custom-image-trigger"]');
+                    var editCustomImageModal = $('#edit-custom-image-modal');
+
+                    // edit custom image which was built during this build
+                    editCustomImageTrigger.click(function () {
+                      // single editable custom image: redirect to the edit page
+                      // for that image
+                      if (editableCustomImageRecipes.length === 1) {
+                        var url = '{% url "customrecipe" build.project.id custom_image_recipes.first.id %}';
+                        document.location.href = url;
+                      }
+                      // multiple editable custom images: show modal to select
+                      // one of them for editing
+                      else {
+                        editCustomImageModal.modal('show');
+                      }
+                    });
+                  });
+                </script>
+            {% endif %}
+          {% endwith %}
+
+            <!-- new custom image from image recipe in this build -->
+            {% if build.has_image_recipes %}
+              <button class="btn btn-default btn-block navbar-btn" data-role="new-custom-image-trigger">New custom image</button>
+              {% include 'newcustomimage_modal.html' %}
               <script>
-                var editableCustomImageRecipes = {{ custom_image_recipes | objects_to_dictionaries:"id,name" | json }};
+                // imageRecipes includes both custom image recipes and built-in
+                // image recipes, any of which can be used as the basis for a
+                // new custom image
+                var imageRecipes = {{ build.get_image_recipes | objects_to_dictionaries:"id,name" | json }};
 
                 $(document).ready(function () {
-                  var editCustomImageTrigger = $('[data-role="edit-custom-image-trigger"]');
-                  var editCustomImageModal = $('#edit-custom-image-modal');
+                  var newCustomImageModal = $('#new-custom-image-modal');
+                  var newCustomImageTrigger = $('[data-role="new-custom-image-trigger"]');
 
-                  // edit custom image which was built during this build
-                  editCustomImageTrigger.click(function () {
-                    // single editable custom image: redirect to the edit page
-                    // for that image
-                    if (editableCustomImageRecipes.length === 1) {
-                      var url = '{% url "customrecipe" build.project.id custom_image_recipes.first.id %}';
-                      document.location.href = url;
-                    }
-                    // multiple editable custom images: show modal to select
-                    // one of them for editing
-                    else {
-                      editCustomImageModal.modal('show');
+                  // show create new custom image modal to select an image built
+                  // during this build as the basis for the custom recipe
+                  newCustomImageTrigger.click(function () {
+                    if (!imageRecipes.length) {
+                      return;
                     }
+
+                    newCustomImageModalSetRecipes(imageRecipes);
+                    newCustomImageModal.modal('show');
                   });
                 });
               </script>
-          {% endif %}
-        {% endwith %}
-
-          <!-- new custom image from image recipe in this build -->
-          <button class="btn btn-default btn-block navbar-btn" data-role="new-custom-image-trigger">New custom image</button>
-          {% include 'newcustomimage_modal.html' %}
-          <script>
-            // imageRecipes includes both custom image recipes and built-in
-            // image recipes, any of which can be used as the basis for a
-            // new custom image
-            var imageRecipes = {{ build.get_image_recipes | objects_to_dictionaries:"id,name" | json }};
-
-            $(document).ready(function () {
-              var newCustomImageModal = $('#new-custom-image-modal');
-              var newCustomImageTrigger = $('[data-role="new-custom-image-trigger"]');
-
-              // show create new custom image modal to select an image built
-              // during this build as the basis for the custom recipe
-              newCustomImageTrigger.click(function () {
-                if (!imageRecipes.length) {
-                  return;
-                }
-
-                newCustomImageModalSetRecipes(imageRecipes);
-                newCustomImageModal.modal('show');
-              });
-            });
-          </script>
-      </ul>
+            {% endif %}
+        </ul>
+      </div>
+      <!-- end left sidebar container -->
+    {% endif %}
 
+    <!-- right container; need class="row" for builds without left-hand menu -->
+    <div{% if not build.started %} class="row"{% endif %}>
+      {% block buildinfomain %}{% endblock %}
     </div>
-    <!-- end left sidebar container -->
-
-    <!-- begin right container -->
-    {% block buildinfomain %}{% endblock %}
-    <!-- end right container -->
   </div>
 {% endblock %}

bitbake/lib/toaster/toastergui/templates/builddashboard.html

@@ -1,8 +1,9 @@
 {% extends "basebuildpage.html" %}
 {% load humanize %}
 {% load projecttags %}
+{% load field_values_filter %}
 
-{% block title %} {{build.target_set.all|dictsort:"target"|join:", "}} {{build.machine}} - {{build.project.name}} - Toaster {% endblock %}
+{% block title %} {{build.get_sorted_target_list|field_values:"target"|join:", "}} {{build.machine}} - {{build.project.name}} - Toaster {% endblock %}
 {% block parentbreadcrumb %}
 {% if build.get_sorted_target_list.count > 0 %}
   {{build.get_sorted_target_list.0.target}}
@@ -13,9 +14,9 @@
 
 {% block buildinfomain %}
 <!-- page title -->
-<div class="col-md-10">
+<div class="{% if build.started %}col-md-10{% else %}col-md-12{% endif %}">
  <div class="page-header build-data">
-     <h1>{{build.target_set.all|dictsort:"target"|join:", "}} {{build.machine}}</h1>
+     <h1>{{build.get_sorted_target_list|field_values:"target"|join:", "}} {{build.machine}}</h1>
  </div>
 
 <!-- build result bar -->
@@ -37,8 +38,15 @@
 			{% endif %}
 			<span class="pull-right">
 				Build time:
-				<a class="alert-link" href="{% url 'buildtime' build.pk %}">{{ build.timespent_seconds|sectohms }}</a>
-			</span>
+			  <span data-build-field="buildtime">
+          {% if build.outcome == build.SUCCEEDED %}
+            <a href="{% url 'buildtime' build.pk %}">{{ build.timespent_seconds|sectohms }}</a>
+          {% else %}
+            {{ build.timespent_seconds|sectohms }}
+          {% endif %}
+        </span>
+      </span>
+
 		{%endif%}
 </div>
 
@@ -54,9 +62,9 @@
     </div>
     <div class="panel-collapse collapse in" id="error-info">
       <div class="panel-body">
-        <div class="col-md-10">
+        <div class="{% if build.started %}col-md-10{% else %}col-md-12{% endif %}">
           {% for error in build.errors %}
-            <div class="alert alert-danger" data-error="{{ error.id }}">
+            <div class="alert alert-danger" data-log-message-id="{{error.pk}}">
               <pre>{{error.message}}</pre>
             </div>
           {% endfor %}
@@ -68,52 +76,58 @@
 
 {%if build.outcome == build.SUCCEEDED%}
 <!-- built images -->
-{% if hasImages %}
-    <h2>Images</h2>
+  {% if hasArtifacts %}
+    <h2 data-heading="build-artifacts">Build artifacts</h2>
     {% for target in targets %}
         {% if target.target.is_image %}
-    <div class="well well-transparent dashboard-section">
-        <h3><a href="{% url 'target' build.pk target.target.pk %}">{{target.target}}</a></h3>
-        <dl class="dl-horizontal">
-            <dt>Packages included</dt>
-            <dd><a href="{% url 'target' build.pk target.target.pk %}">{{target.npkg}}</a></dd>
-            <dt>Total package size</dt>
-            <dd>{{target.pkgsz|filtered_filesizeformat}}</dd>
-                        {% if target.targetHasNoImages %}
-                </dl>
-                <div class="row">
-                  <div class="col-md-7">
-                    <div class="alert alert-info">
-                      <p>
-                      <strong>This build did not create any image files</strong>
-                      </p>
-                      <p>
-                      This is probably because valid image and license manifest
-                      files from a previous build already exist in your
-                      <code>build/tmp/deploy</code>
-                      directory. You can
-                      also <a href="{% url 'target' build.pk target.target.pk %}">view the
-                        license manifest information</a> in Toaster.
-                      </p>
-                    </div>
-                  </div>
-                </div>
+    <div class="well well-transparent dashboard-section" data-artifacts-for-target="{{target.target.pk}}">
+        {% if target.npkg > 0 %}
+            <h3>
+                <a href="{% url 'target' build.pk target.target.pk %}" data-link="target-packages">
+                  {{target.target.target}}
+                </a>
+            </h3>
+            <dl class="dl-horizontal">
+                <dt>Packages included</dt>
+                <dd>
+                    <a href="{% url 'target' build.pk target.target.pk %}">
+                        <span data-value="target-package-count">{{target.npkg}}</span>
+                    </a>
+                </dd>
+                <dt>Total package size</dt>
+                <dd>
+                    <span data-value="target-package-size">{{target.pkgsz|filtered_filesizeformat}}</span>
+                </dd>
+            </dl>
         {% else %}
-            <dt>
-                <span class="glyphicon glyphicon-question-sign get-help" title="The location in disk of the license manifest, a document listing all packages installed in your image and their licenses"></span>
+            <h3>{{target.target.target}}</h3>
+        {% endif %}
 
-                License manifest
+        {% if target.targetHasImages %}
+          <dl class="dl-horizontal">
+            <dt>
+              Manifests
             </dt>
+
             <dd>
-                <a href="{% url 'target' build.pk target.target.pk %}">View in Toaster</a> |
-                <a href="{% url 'build_artifact' build.pk 'licensemanifest' target.target.pk %}">Download</a></dd>
+              <a data-link="license-manifest" href="{% url 'build_artifact' build.pk 'licensemanifest' target.target.pk %}">License manifest</a>
+            </dd>
+
+            {% if target.target.package_manifest_path %}
+              <dd>
+                <a data-link="package-manifest" href="{% url 'build_artifact' build.pk 'packagemanifest' target.target.pk %}">Package manifest</a>
+              </dd>
+            {% endif %}
+          </dl>
+
+          <dl class="dl-horizontal">
             <dt>
                 <span class="glyphicon glyphicon-question-sign get-help" title="Image files are stored in <code>build/tmp/deploy/images/</code>"></span>
                 Image files
             </dt>
             <dd>
-                <ul class="list-unstyled">
-                    {% for i in target.imageFiles %}
+                <ul class="list-unstyled" data-links="image-artifacts">
+                    {% for i in target.imageFiles|dictsort:"suffix" %}
                         <li>
                             <a href="{% url 'build_artifact' build.pk 'imagefile' i.id %}">
                                 {{i.suffix}}
@@ -123,12 +137,42 @@
                     {% endfor %}
                 </ul>
             </dd>
-        </dl>
+            <dt>
+                Kernel artifacts
+            </dt>
+            <dd>
+                <ul class="list-unstyled" data-links="kernel-artifacts">
+                    {% for artifact in target.target_kernel_artifacts|dictsort:"basename" %}
+                        <li>
+                            <a href="{% url 'build_artifact' build.id 'targetkernelartifact' artifact.id %}">{{artifact.basename}}</a>
+                            ({{artifact.file_size|filtered_filesizeformat}})
+                        </li>
+                    {% endfor %}
+                  </ul>
+            </dd>
+          </dl>
+        {% endif %}
+        {% if target.target_sdk_artifacts_count > 0 %}
+          <dl class="dl-horizontal">
+            <dt>
+              SDK artifacts
+            </dt>
+            <dd>
+              <ul class="list-unstyled" data-links="sdk-artifacts">
+                {% for artifact in target.target_sdk_artifacts|dictsort:"basename" %}
+                <li>
+                  <a href="{% url 'build_artifact' build.id 'targetsdkartifact' artifact.id %}">{{artifact.basename}}</a>
+                  ({{artifact.file_size|filtered_filesizeformat}})
+                </li>
+                {% endfor %}
+              </ul>
+            </dd>
+          </dl>
         {% endif %}
     </div>
         {% endif %}
     {% endfor %}
-{% endif %}
+  {% endif %}
 
 {%else%}
 <!-- error dump -->
@@ -157,79 +201,82 @@
     </div>
 
 {% endif %}
-<!-- build summary -->
-<h2>Build summary</h2>
-  <div class="row">
-    <div class="col-md-4 dashboard-section">
-      <div class="well well-transparent">
-        <h3><a href="{%url 'configuration' build.pk%}">Configuration</a></h3>
-            <dl>
-        <dt>Machine</dt><dd>{{build.machine}}</dd>
-        <dt>Distro</dt><dd>{{build.distro}}</dd>
-        <dt>Layers</dt><dd><ul class="list-unstyled">{% for i in build.layer_version_build.all|dictsort:"layer.name" %}<li>{{i.layer.name}}</li>{%endfor%}</ul></dd>
-            </dl>
-      </div>
-    </div>
-    <div class="col-md-4 dashboard-section">
-      <div class="well well-transparent">
-        <h3><a href="{%url 'tasks' build.pk%}">Tasks</a></h3>
-            <dl>
-            {% query build.task_build outcome=4 order__gt=0 as exectask%}
-            {% if exectask.count > 0 %}
-                <dt>Failed tasks</dt>
-                <dd>
-                {% if exectask.count == 1 %}
-                    <a class="text-danger" href="{% url "task" build.id exectask.0.id %}">
-                        {{exectask.0.recipe.name}}
-                        <span class="task-name">{{exectask.0.task_name}}</span>
-                    </a>
 
-                        <a href="{% url 'build_artifact' build.id "tasklogfile" exectask.0.id %}">
-                            <span class="glyphicon glyphicon-download-alt get-help" title="Download task log file"></i>
-                        </a>
-
-                {% elif exectask.count > 1%}
-                    <a class="text-danger" href="{% url "tasks" build.id %}?filter=outcome%3A4">{{exectask.count}}</a>
-                {% endif %}
-                </dd>
-            {% endif %}
-        <dt>Total number of tasks</dt><dd><a href="{% url 'tasks' build.pk %}">{% query build.task_build order__gt=0 as alltasks %}{{alltasks.count}}</a></dd>
-        <dt>
-            Tasks executed
-            <span class="glyphicon glyphicon-question-sign get-help" title="'Executed' tasks are those that need to be run in order to generate the task output"></span>
-        </dt>
-        <dd><a href="{% url 'tasks' build.pk %}?filter=task_executed%3A1&amp;count=25&amp;search=&amp;page=1&amp;orderby=order%3A%2B">{% query build.task_build task_executed=1 order__gt=0 as exectask%}{{exectask.count}}</a></dd>
-        <dt>
-            Tasks not executed
-            <span class="glyphicon glyphicon-question-sign get-help" title="'Not executed' tasks don't need to run because their outcome is provided by another task"></span>
-        </dt>
-        <dd><a href="{% url 'tasks' build.pk %}?filter=task_executed%3A0&amp;count=25&amp;search=&amp;page=1&amp;orderby=order%3A%2B">{% query build.task_build task_executed=0 order__gt=0 as noexectask%}{{noexectask.count}}</a></dd>
-        <dt>
-            Reuse
-            <span class="glyphicon glyphicon-question-sign get-help" title="The percentage of 'not executed' tasks over the total number of tasks, which is a measure of the efficiency of your build"></span>
-        </dt>
-        <dd>
-{% query build.task_build order__gt=0 as texec %}
-{% if noexectask.count|multiply:100|divide:texec.count < 0 %}
-0
-{% else %}
-{{noexectask.count|multiply:100|divide:texec.count}}
-{% endif %}
-%
-        </dd>
-            </dl>
+{% if build.started %}
+  <!-- build summary -->
+  <h2 data-role="build-summary-heading">Build summary</h2>
+    <div class="row">
+      <div class="col-md-4 dashboard-section">
+        <div class="well well-transparent">
+          <h3><a href="{%url 'configuration' build.pk%}">Configuration</a></h3>
+              <dl>
+          <dt>Machine</dt><dd>{{build.machine}}</dd>
+          <dt>Distro</dt><dd>{{build.distro}}</dd>
+          <dt>Layers</dt><dd><ul class="list-unstyled">{% for i in build.layer_version_build.all|dictsort:"layer.name" %}<li>{{i.layer.name}}</li>{%endfor%}</ul></dd>
+              </dl>
+        </div>
+      </div>
+      <div class="col-md-4 dashboard-section">
+        <div class="well well-transparent">
+          <h3><a href="{%url 'tasks' build.pk%}">Tasks</a></h3>
+              <dl>
+              {% query build.task_build outcome=4 order__gt=0 as exectask%}
+              {% if exectask.count > 0 %}
+                  <dt>Failed tasks</dt>
+                  <dd>
+                  {% if exectask.count == 1 %}
+                      <a class="text-danger" href="{% url "task" build.id exectask.0.id %}">
+                          {{exectask.0.recipe.name}}
+                          <span class="task-name">{{exectask.0.task_name}}</span>
+                      </a>
+
+                          <a href="{% url 'build_artifact' build.id "tasklogfile" exectask.0.id %}">
+                              <span class="glyphicon glyphicon-download-alt get-help" title="Download task log file"></i>
+                          </a>
+
+                  {% elif exectask.count > 1%}
+                      <a class="text-danger" href="{% url "tasks" build.id %}?filter=outcome%3A4">{{exectask.count}}</a>
+                  {% endif %}
+                  </dd>
+              {% endif %}
+          <dt>Total number of tasks</dt><dd><a href="{% url 'tasks' build.pk %}">{% query build.task_build order__gt=0 as alltasks %}{{alltasks.count}}</a></dd>
+          <dt>
+              Tasks executed
+              <span class="glyphicon glyphicon-question-sign get-help" title="'Executed' tasks are those that need to be run in order to generate the task output"></span>
+          </dt>
+          <dd><a href="{% url 'tasks' build.pk %}?filter=task_executed%3A1&amp;count=25&amp;search=&amp;page=1&amp;orderby=order%3A%2B">{% query build.task_build task_executed=1 order__gt=0 as exectask%}{{exectask.count}}</a></dd>
+          <dt>
+              Tasks not executed
+              <span class="glyphicon glyphicon-question-sign get-help" title="'Not executed' tasks don't need to run because their outcome is provided by another task"></span>
+          </dt>
+          <dd><a href="{% url 'tasks' build.pk %}?filter=task_executed%3A0&amp;count=25&amp;search=&amp;page=1&amp;orderby=order%3A%2B">{% query build.task_build task_executed=0 order__gt=0 as noexectask%}{{noexectask.count}}</a></dd>
+          <dt>
+              Reuse
+              <span class="glyphicon glyphicon-question-sign get-help" title="The percentage of 'not executed' tasks over the total number of tasks, which is a measure of the efficiency of your build"></span>
+          </dt>
+          <dd>
+  {% query build.task_build order__gt=0 as texec %}
+  {% if noexectask.count|multiply:100|divide:texec.count < 0 %}
+  0
+  {% else %}
+  {{noexectask.count|multiply:100|divide:texec.count}}
+  {% endif %}
+  %
+          </dd>
+              </dl>
+        </div>
+      </div>
+      <div class="col-md-4 dashboard-section">
+        <div class="well well-transparent">
+          <h3><a href="{% url 'recipes' build.pk %}">Recipes</a> & <a href="{% url 'packages' build.pk %}">Packages</a></h3>
+              <dl>
+          <dt>Recipes built</dt><dd><a href="{% url 'recipes' build.pk %}">{{recipecount}}</a></dd>
+          <dt>Packages built</dt><dd><a href="{% url 'packages' build.pk %}">{{packagecount}}</a></dd>
+              </dl>
       </div>
-    </div>
-    <div class="col-md-4 dashboard-section">
-      <div class="well well-transparent">
-        <h3><a href="{% url 'recipes' build.pk %}">Recipes</a> & <a href="{% url 'packages' build.pk %}">Packages</a></h3>
-            <dl>
-        <dt>Recipes built</dt><dd><a href="{% url 'recipes' build.pk %}">{{recipecount}}</a></dd>
-        <dt>Packages built</dt><dd><a href="{% url 'packages' build.pk %}">{{packagecount}}</a></dd>
-            </dl>
     </div>
   </div>
-</div>
+{% endif %} <!-- end build summary -->
 
 {% if build.warnings.count %}
   <div class="panel panel-default" id="warnings">
@@ -241,9 +288,9 @@
     </div>
     <div class="panel-collapse collapse" id="warning-info">
       <div class="panel-body">
-        <div class="col-md-10">
+        <div class="{% if build.started %}col-md-10{% else %}col-md-12{% endif %}">
           {% for warning in logmessages %}{% if warning.level == 1 %}
-            <div class="alert alert-warning">
+            <div class="alert alert-warning" data-log-message-id="{{warning.pk}}">
               <pre>{{warning.message}}</pre>
             </div>
           {% endif %}{% endfor %}
@@ -261,7 +308,7 @@
         if (location.href.search('#warnings') > -1) {
             $('#warning-info').addClass('in');
         }
-	
+
 	//show warnings section when requested from the build outcome
 	$(".show-warnings").click(function() {
 	    $('#warning-info').addClass('in');

bitbake/lib/toaster/toastergui/templates/buildrequestdetails.html

@@ -1,64 +0,0 @@
-{% extends "baseprojectpage.html" %}
-
-{% load static %}
-{% load projecttags %}
-{% load humanize %}
-
-
-{% block projectinfomain %}
-      <!-- begin content -->
-
-          <div class="row">
-
-            <!-- end left sidebar container -->
-            <!-- Begin right container -->
-            <div class="col-md-10">
-              <div class="page-header">
-                <h1>
- <span data-toggle="tooltip" {%if buildrequest.brtarget_set.all.count > 1%}title="Targets: {%for target in buildrequest.brtarget_set.all%}{{target.target}} {%endfor%}"{%endif%}>{{buildrequest.brtarget_set.all.0.target}} {%if buildrequest.brtarget_set.all.count > 1%}(+ {{buildrequest.brtarget_set.all.count|add:"-1"}}){%endif%} {{buildrequest.get_machine}} </span>
-
-        </h1>
-              </div>
-              <div class="alert alert-error">
-                <p class="lead">
-                  <strong>Failed</strong>
-                    on {{ buildrequest.updated|date:'d/m/y H:i' }}
-            with
-
-                                    <i class="icon-minus-sign error" style="margin-left:6px;"></i>
-                  <strong><a class="error accordion-toggle toggle-errors" href="#errors">
-                        {{buildrequest.brerror_set.all.count}} error{{buildrequest.brerror_set.all.count|pluralize}}
-              </a></strong>
-                  <span class="pull-right">Build time: {{buildrequest.get_duration|sectohms}}</span>
-                </p>
-              </div>
-
-              <div class="accordion" id="errors">
-                <div class="accordion-group">
-                  <div class="accordion-heading">
-                    <a class="accordion-toggle error toggle-errors">
-                      <h2>
-                        <i class="icon-minus-sign"></i>
-                        {{buildrequest.brerror_set.all.count}} error{{buildrequest.brerror_set.all.count|pluralize}}
-                      </h2>
-                    </a>
-                  </div>
-                  <div class="accordion-body collapse in" id="collapse-errors">
-                    <div class="accordion-inner">
-                      <div class="col-md-10">
-               {% for error in buildrequest.brerror_set.all %}
-                        <div class="alert alert-error">
-                          ERROR: <div class="air well"><pre>{{error.errmsg}}</pre></div>
-                        </div>
-                      {% endfor %}
-                      </div>
-                    </div>
-                  </div>
-
-                </div>
-              </div>
-            </div>
-          </div> <!-- end of row -->
-
-
-{%endblock%}

bitbake/lib/toaster/toastergui/templates/configuration.html

@@ -53,11 +53,25 @@
           <tbody>{% for lv in build.layer_version_build.all|dictsort:"layer.name" %}
           <tr>
             <td>{{lv.layer.name}}</td>
+	    {% if lv.layer.local_source_dir %}
+	    <td>
+	      <span class="text-muted">Not applicable</span>
+	      <span class="glyphicon glyphicon-question-sign get-help" data-original-title="" title="The source code of {{lv.layer.name}} is not in a Git repository, so there is no branch associated with it"> </span>
+	    </td>
+	    {% else %}
             <td>{{lv.branch}}</td>
+	    {% endif %}
+	    {% if lv.layer.local_source_dir %}
+	    <td>
+	      <span class="text-muted">Not applicable</span>
+	      <span class="glyphicon glyphicon-question-sign get-help" data-original-title="" title="The source code of {{lv.layer.name}} is not in a Git repository, so there is no commit associated with it"> </span>
+	    </td>
+	    {% else %}
             <td> <a class="btn btn-default" data-content="<ul class='list-unstyled'>
                   <li>{{lv.commit}}</li> </ul>">
                 {{lv.commit|truncatechars:13}}
             </a></td>
+	    {% endif %}
           </tr>{% endfor %}
           </tbody>
         </table>

bitbake/lib/toaster/toastergui/templates/importlayer.html

@@ -26,16 +26,14 @@
   </script>
 
   <form class="col-md-11">
+    <span class="help-block">The layer you are importing must be compatible with <strong>{{project.release.description}}</strong>, which is the release you are using in this project.</span>
+    <div class="alert alert-error" id="import-error" style="display:none">
+      <button type="button" class="close" data-dismiss="alert">&times;</button>
+      <h3>&nbsp;</h3>
+      <p></p>
+      <ul></ul>
+    </div>
     <fieldset>
-      <h2>Layer repository information</h2>
-      <span class="help-block">The layer you are importing must be compatible with <strong>{{project.release.description}}</strong>, which is the release you are using in this project.</span>
-      <div class="alert alert-error" id="import-error" style="display:none">
-        <button type="button" class="close" data-dismiss="alert">&times;</button>
-        <h3>&nbsp;</h3>
-        <p></p>
-        <ul></ul>
-      </div>
-
       <div class="form-group" id="layer-name-ctrl">
         <label class="control-label" for="import-layer-name">
           Layer name
@@ -45,82 +43,112 @@
         <span class="help-block" style="display: none;" id="invalid-layer-name-hint">A valid layer name can only include letters, numbers and dashes</span>
         <span class="help-inline" style="display: none;" id="duplicated-layer-name-hint"></span>
       </div>
-
-      <div id="duplicate-layer-info" style="display:none">
-        <div class="alert alert-warning">
-          <h3>A layer called <a href="" class="dup-layer-link"><span class="dup-layer-name"></span></a> already exists</h3>
-          <p>Layer names must be unqiue. Please use a different layer name.</p>
-          <dl>
-            <dt>
-            The <span class="dup-layer-name"></span> repository url is
-            </dt>
-            <dd>
-            <span id="dup-layer-vcs-url"></span>
-            </dd>
-            <dt>
-            The <span class="dup-layer-name"></span> revision is
-            </dt>
-            <dd>
-            <span id="dup-layer-revision"></span>
-            </dd>
-          </dl>
-          <p><a href="" class="dup-layer-link">View the <span class="dup-layer-name"></span> layer information</a></p>
-        </div>
-      </div>
-
-      <div class="fields-apart-from-layer-name">
-        <div class="form-group">
-          <label for="layer-git-repo-url">
-            Git repository URL
-            <span class="glyphicon glyphicon-question-sign get-help" title="Fetch/clone URL of the repository. Currently, Toaster only supports Git repositories." ></span>
-          </label>
-
-          <input type="text" id="layer-git-repo-url" class="form-control" required>
-        </div>
-        <div class="form-group">
-          <label for="layer-subdir">
-            Repository subdirectory
-            <span class="text-muted">(optional)</span>
-            <span class="glyphicon glyphicon-question-sign get-help" title="Subdirectory within the repository where the layer is located, if not in the root (usually only used if the repository contains more than one layer)"></span>
-          </label>
-          <input type="text" class="form-control" id="layer-subdir">
-        </div>
-        <div class="form-group" id="layer-revision-ctrl">
-          <label for="layer-git-ref">Git revision
-            <span class="glyphicon glyphicon-question-sign get-help" title="You can provide a Git branch, a tag or a commit SHA as the revision"></span>
-          </label>
-          <input type="text" class="form-control" id="layer-git-ref" required>
-          <span class="help-inline" style="diaply:none;" id="invalid-layer-revision-hint"></span>
-        </div>
-      </div>
-
     </fieldset>
-
-    <div class="fields-apart-from-layer-name">
-      <fieldset>
-        <h2>
-          Layer dependencies
-          <small class="text-muted">(optional)</small>
-          <span class="glyphicon glyphicon-question-sign get-help heading-help" title="Other layers this layer depends upon"></span>
-        </h2>
-        <ul class="list-unstyled lead" id="layer-deps-list">
-        </ul>
-        <div class="form-inline">
-          <div class="form-group">
-            <input type="text" autocomplete="off" data-minLength="1" data-autocomplete="off" data-provide="typeahead" placeholder="Type a layer name" id="layer-dependency" class="form-control">
-          </div>
-          <button class="btn btn-default" id="add-layer-dependency-btn">
-            Add layer
-          </button>
-          <span class="help-inline">You can only add layers Toaster knows about</span>
-        </div>
-      </fieldset>
-      <div class="top-air" id="form-actions">
-        <button class="btn btn-primary btn-lg" data-toggle="modal" id="import-and-add-btn" data-target="#dependencies-message" disabled>Import and add to project</button>
-        <span class="help-inline" id="import-and-add-hint" style="vertical-align: middle;">To import a layer you need to enter a layer name, a Git repository URL and a Git revision (branch, tag or commit)</span>
+    <div id="duplicate-layer-info" style="display:none">
+      <div class="alert alert-warning">
+        <h3>A layer called <a href="" class="dup-layer-link"><span class="dup-layer-name"></span></a> already exists</h3>
+        <p>Layer names must be unqiue. Please use a different layer name.</p>
+        <dl id="git-layer-dup" style="display:none;">
+          <dt>
+          The <span class="dup-layer-name"></span> repository url is
+          </dt>
+          <dd>
+          <span id="dup-layer-vcs-url"></span>
+          </dd>
+          <dt>
+          The <span class="dup-layer-name"></span> revision is
+          </dt>
+          <dd>
+          <span id="dup-layer-revision"></span>
+          </dd>
+        </dl>
+	<dl id="local-layer-dup" style="display:none;">
+	  <dt>
+	    The <span class="dup-layer-name"></span> directory is
+	  </dt>
+	  <dd>
+	    <span id="dup-local-source-dir-name"></span>
+	  </dd>
+	</dl>
+        <p><a href="" class="dup-layer-link">View the <span class="dup-layer-name"></span> layer information</a></p>
       </div>
     </div>
-  </form>
+    <fieldset class="fields-apart-from-layer-name" id="repo-select">
+      <legend class="radioLegend">Where is the layer source code?</legend>
+      <div class="radio">
+	<label>
+          <input type="radio" id="git-repo-radio" name="repo" value="git" checked="checked">
+	  In a <strong>Git repository</strong>
+	</label>
+	<p class="help-block radio-help">To build the layer Toaster must be able to access the Git repository, otherwise builds will fail. Toaster will fetch and checkout your chosen Git revision every time you start a build.</p>
+      </div>
+      <div class="radio">
+	<label>
+          <input type="radio" id="local-dir-radio" name="repo" value="local">
+	  In a <strong>directory</strong>
+	</label>
+	<p class="help-block radio-help">Use this option for quick layer development, by simply providing the path to the layer source code.</p>
+      </div>
+    </fieldset>
+    <fieldset class="fields-apart-from-layer-name" id="git-repo">
+      <legend>Git repository information</legend>
+      <div class="form-group">
+        <label for="layer-git-repo-url">
+          Git repository URL
+          <span class="glyphicon glyphicon-question-sign get-help" title="Fetch/clone URL of the repository. Currently, Toaster only supports Git repositories." ></span>
+        </label>
+        <input type="text" id="layer-git-repo-url" class="form-control" required>
+      </div>
+      <div class="form-group">
+        <label for="layer-subdir">
+          Repository subdirectory
+          <span class="text-muted">(optional)</span>
+          <span class="glyphicon glyphicon-question-sign get-help" title="Subdirectory within the repository where the layer is located, if not in the root (usually only used if the repository contains more than one layer)"></span>
+        </label>
+        <input type="text" class="form-control" id="layer-subdir">
+      </div>
+      <div class="form-group" id="layer-revision-ctrl">
+        <label for="layer-git-ref">
+          Git revision
+          <span class="glyphicon glyphicon-question-sign get-help" title="You can provide a Git branch, a tag or a commit SHA as the revision"></span>
+        </label>
+        <input type="text" class="form-control" id="layer-git-ref" required>
+        <span class="help-inline" style="diaply:none;" id="invalid-layer-revision-hint"></span>
+      </div>
+    </fieldset>
+
+    <fieldset class="fields-apart-from-layer-name" id="local-dir" style="display:none;">
+      <legend>Layer directory information</legend>
+	<label for="local-dir-path" class="control-label">Enter the absolute path to the layer directory</label>
+	<input type="text" class="form-control" id="local-dir-path" required/>
+	<p class="help-block" id="hintError-dir-path-starts-with-slash" style="display:none;">The absolute path must start with "/".</p>
+	<p class="help-block" id="hintError-dir-path" style="display:none;">The directory path cannot include spaces or any of these characters: . \ ? % * : | " " &lt; &gt;</p>
+    </fieldset>
+
+    <fieldset class="fields-apart-from-layer-name">
+      <legend>
+        Layer dependencies
+        <small class="text-muted">(optional)</small>
+        <span class="glyphicon glyphicon-question-sign get-help heading-help" title="Other layers this layer depends upon"></span>
+      </legend>
+      <ul class="list-unstyled lead" id="layer-deps-list">
+      </ul>
+      <div class="form-inline">
+        <div class="form-group">
+          <input type="text" autocomplete="off" data-minLength="1" data-autocomplete="off" data-provide="typeahead" placeholder="Type a layer name" id="layer-dependency" class="form-control">
+        </div>
+        <button class="btn btn-default" id="add-layer-dependency-btn">
+          Add layer
+        </button>
+        <span class="help-inline">You can only add layers Toaster knows about</span>
+    </fieldset>
+    <div class="top-air fields-apart-from-layer-name" id="form-actions">
+      <button class="btn btn-primary btn-lg" data-toggle="modal" id="import-and-add-btn" data-target="#dependencies-message" disabled>Import and add to project</button>
+      <span class="help-inline" id="import-git-layer-and-add-hint" style="vertical-align: middle;">To import a layer you need to enter a layer name, a Git repository URL and a Git revision (branch, tag or commit)</span>
+      <span class="help-inline" id="import-local-dir-and-add-hint" style="vertical-align: middle;display:none;">To import a layer you need to enter a layer name and the absolute path to the layer directory</span>
+    </div>
+  </div>
+</form>
 
   {% else %} {#project and project release#}
   <div class="page-header">

bitbake/lib/toaster/toastergui/templates/layerdetails.html

@@ -6,6 +6,20 @@
 {% block title %} {{layerversion.layer.name}} - {{project.name}} - Toaster {% endblock %}
 {% block pagecontent %}
 
+<div id="delete-layer-modal" class="modal fade" tabindex="-1" role="dialog">
+  <div class="modal-dialog" role="document">
+    <div class="modal-content">
+      <div class="modal-body">
+		  Are you sure you want to delete the <strong>{{layerversion.layer.name}}</strong> layer?
+      </div>
+      <div class="modal-footer">
+        <button type="button" id="layer-delete-confirmed" class="btn btn-primary">Delete</button>
+        <button type="button" class="btn btn-default btn-link" data-dismiss="modal">Cancel</button>
+      </div>
+    </div>
+  </div>
+</div>
+
 <div class="row">
   <div class="col-md-12">
     <ul class="breadcrumb">
@@ -22,7 +36,7 @@
     </ul>
 
     {# If this is not an imported layer then hide the edit ui #}
-    {% if not layerversion.layer_source_id or layerversion.layer_source.sourcetype != layerversion.layer_source.TYPE_IMPORTED %}
+    {% if layerversion.layer_source != layer_source.TYPE_IMPORTED %}
     <style scoped>
     .glyphicon-edit {
       display:none;
@@ -44,7 +58,7 @@
 
     $(document).ready(function(){
       var ctx = {
-        xhrUpdateLayerUrl : "{% url 'xhr_updatelayer' %}",
+        xhrUpdateLayerUrl : "{% url 'xhr_layer' project.id layerversion.pk %}",
         layerVersion : {
           name : "{{layerversion.layer.name}}",
           id : {{layerversion.id}},
@@ -55,8 +69,9 @@
           inCurrentPrj : false,
           {% endif %}
           layerdetailurl : "{% url 'layerdetails' project.id layerversion.id  %}",
-          sourceId: {{layerversion.layer_source_id|json}},
-        }
+          layer_source: {{layerversion.layer_source|json}},
+        },
+        layerSourceTypes: {{layer_source|json}},
       };
 
       try {
@@ -75,7 +90,6 @@
         {% endif %}>({{layerversion.get_vcs_reference|truncatechars:13}})</small>
       </h1>
     </div>
-
     <div class="row">
       <!-- container for tabs -->
       <div class="col-md-8 tabbable">
@@ -263,14 +277,18 @@
             <span class="glyphicon glyphicon-edit"></span>
             <span class="glyphicon glyphicon-trash delete-current-value" data-toggle="tooltip" title="Delete"></span>
             </dd>
-            {% if layerversion.layer.up_id %}
+            {% if layerversion.layer_source == layer_source.TYPE_LAYERINDEX  %}
             <dt>Layer index</dt>
             <dd>
-            <a href="http://layers.openembedded.org/layerindex/branch/{{layerversion.up_branch.name}}/layer/{{layerversion.layer.name}}">layer index link</a>
-
+            <a href="http://layers.openembedded.org/layerindex/branch/{{layerversion.release.name}}/layer/{{layerversion.layer.name}}">Layer index {{layerversion.layer.name}}</a>
             </dd>
             {% endif %}
           </dl>
+          {# Only show delete link for imported layers #}
+          {% if layerversion.layer_source == layer_source.TYPE_IMPORTED %}
+          <i class="icon-trash text-danger"></i>
+          <a href="#delete-layer-modal"  role="button" class="text-danger" data-toggle="modal" data-target="#delete-layer-modal">Delete {{layerversion.layer.name}}</a>
+          {% endif %}
         </div>
       </div>
     </div>

bitbake/lib/toaster/toastergui/templates/mrb_section.html

@@ -1,8 +1,263 @@
 {% load static %}
-{% load projecttags %}
-{% load project_url_tag %}
 {% load humanize %}
+{% load project_url_tag %}
 <script src="{% static 'js/mrbsection.js' %}"></script>
+
+{% if mru %}
+  {% if mrb_type == 'project' %}
+    <h2>
+      Latest project builds
+
+      {% if project.is_default %}
+        <span class="glyphicon glyphicon-question-sign get-help heading-help" data-original-title="Builds in this project cannot be started from Toaster: they are started from the command line"></span>
+        {% endif %}
+    </h2>
+  {% else %}
+    <div class="page-header">
+      <h1>Latest builds</h1>
+    </div>
+  {% endif %}
+
+  <div id="latest-builds">
+    {% for build in mru %}
+      <div data-latest-build-result="{{build.id}}" class="alert build-result {% if build.outcome == build.SUCCEEDED %}alert-success{% elif build.outcome == build.FAILED %}alert-danger{% else %}alert-info{% endif %}">
+        <!-- project title -->
+        {% if mrb_type != 'project' %}
+          <div class="row project-name">
+            <div class="col-md-12">
+              <small>
+                <a class="alert-link text-uppercase" href="{% project_url build.project %}">
+                  {{build.project.name}}
+                </a>
+              </small>
+            </div>
+          </div>
+        {% endif %}
+
+        <div class="row" data-role="build-status-container">
+          <div class="col-md-12">
+            Loading...
+          </div>
+        </div>
+      </div>
+    {% endfor %}
+  </div>
+{% endif %}
+
+<!-- build main template -->
+<script id="build-template" type="text/x-jsrender">
+  <div class="col-md-3">
+    <!-- only show link for completed builds -->
+    <%if state == 'Succeeded' || state == 'Failed'%>
+      <a class="alert-link" href="<%:dashboard_url%>">
+        <span data-toggle="tooltip" data-role="targets-text" title="Recipes: <%:targets%>">
+          <%:targets_abbreviated%>
+        </span>
+      </a>
+    <%else targets_abbreviated !== ''%>
+      <span data-toggle="tooltip" data-role="targets-text" title="Recipes: <%:targets%>">
+        <%:targets_abbreviated%>
+      </span>
+    <%else%>
+      Fetching recipe names...
+    <%/if%>
+  </div>
+
+  <div data-build-state="<%:state%>">
+    <%if state == 'Parsing'%>
+      <%include tmpl='#parsing-recipes-build-template'/%>
+    <%else state == 'Queued'%>
+      <%include tmpl='#queued-build-template'/%>
+    <%else state == 'Succeeded' || state == 'Failed'%>
+      <%include tmpl='#succeeded-or-failed-build-template'/%>
+    <%else state == 'Cancelling'%>
+      <%include tmpl='#cancelling-build-template'/%>
+    <%else state == 'Starting'%>
+      <%include tmpl='#starting-template'/%>
+    <%else state == 'In Progress'%>
+      <%include tmpl='#in-progress-build-template'/%>
+    <%else state == 'Cancelled'%>
+      <%include tmpl='#cancelled-build-template'/%>
+    <%/if%>
+  </div>
+</script>
+
+<!-- queued build -->
+<script id="queued-build-template" type="text/x-jsrender">
+  <div class="col-md-5">
+    <span class="glyphicon glyphicon-question-sign get-help get-help-blue"
+          title="This build is waiting for the build directory to become available">
+    </span>
+
+    Build queued
+  </div>
+
+  <div class="col-md-4">
+    <!-- cancel button -->
+    <%include tmpl='#cancel-template'/%>
+  </div>
+</script>
+
+<!-- parsing recipes build -->
+<script id="parsing-recipes-build-template" type="text/x-jsrender">
+  <!-- progress bar and parse completion percentage -->
+  <div data-role="build-status" class="col-md-4 col-md-offset-1 progress-info">
+    <!-- progress bar -->
+    <div class="progress">
+      <div id="recipes-parsed-percentage-bar-<%:id%>"
+           style="width: <%:recipes_parsed_percentage%>%;"
+           class="progress-bar">
+      </div>
+    </div>
+  </div>
+
+  <div class="col-md-4 progress-info">
+    <!-- parse completion percentage -->
+    <span class="glyphicon glyphicon-question-sign get-help get-help-blue"
+          title="BitBake is parsing the layers required for your build">
+    </span>
+
+    Parsing <span id="recipes-parsed-percentage-<%:id%>"><%:recipes_parsed_percentage%></span>% complete
+
+    <%include tmpl='#cancel-template'/%>
+  </div>
+</script>
+
+<!-- in progress build; tasks still starting -->
+<script id="starting-template" type="text/x-jsrender">
+  <div class="col-md-5">
+    <span class="glyphicon glyphicon-question-sign get-help get-help-blue"
+          title="This build is waiting for tasks to start">
+    </span>
+
+    Tasks starting...
+  </div>
+
+  <div class="col-md-4">
+    <!-- cancel button -->
+    <%include tmpl='#cancel-template'/%>
+  </div>
+</script>
+
+<!-- in progress build; at least one task finished -->
+<script id="in-progress-build-template" type="text/x-jsrender">
+  <!-- progress bar and task completion percentage -->
+  <div data-role="build-status" class="col-md-4 col-md-offset-1 progress-info">
+    <!-- progress bar -->
+    <div class="progress" id="build-pc-done-title-<%:id%>">
+      <div id="build-pc-done-bar-<%:id%>"
+           style="width: <%:tasks_complete_percentage%>%;"
+           class="progress-bar">
+      </div>
+    </div>
+  </div>
+
+  <div class="col-md-4 progress-info">
+    <!-- task completion percentage -->
+    <span id="build-pc-done-<%:id%>"><%:tasks_complete_percentage%></span>% of
+    tasks complete
+
+    <!-- cancel button -->
+    <%include tmpl='#cancel-template'/%>
+  </div>
+</script>
+
+<!-- cancelling build -->
+<script id="cancelling-build-template" type="text/x-jsrender">
+  <div class="col-md-9">
+    Cancelling the build ...
+  </div>
+</script>
+
+<!-- succeeded or failed build -->
+<script id="succeeded-or-failed-build-template" type="text/x-jsrender">
+  <!-- completed_on -->
+  <div class="col-md-2">
+    <%:completed_on%>
+  </div>
+
+  <!-- errors -->
+  <div class="col-md-2">
+    <%if errors%>
+      <span class="glyphicon glyphicon-minus-sign"></span>
+      <a href="<%:dashboard_errors_url%>" class="alert-link">
+        <%:errors%> error<%:errors_pluralise%>
+      </a>
+    <%/if%>
+  </div>
+
+  <!-- warnings -->
+  <div class="col-md-2">
+    <%if warnings%>
+      <span class="glyphicon glyphicon-warning-sign build-warnings"></span>
+      <a href="<%:dashboard_warnings_url%>" class="alert-link build-warnings">
+        <%:warnings%> warning<%:warnings_pluralise%>
+      </a>
+    <%/if%>
+  </div>
+
+  <!-- build time -->
+  <div class="col-md-3">
+    Build time:
+
+    <span data-role="data-recent-build-buildtime-field">
+      <%if state == 'Succeeded'%>
+        <a class="alert-link" href="<%:buildtime_url%>"><%:buildtime%></a>
+      <%else%>
+        <%:buildtime%>
+      <%/if%>
+    </span>
+
+    <!-- rebuild button -->
+    <%include tmpl='#rebuild-template'/%>
+  </div>
+</script>
+
+<!-- cancelled build -->
+<script id="cancelled-build-template" type="text/x-jsrender">
+  <!-- build cancelled message -->
+  <div class="col-md-6">
+    Build cancelled
+  </div>
+
+  <!-- rebuild button -->
+  <div class="col-md-3">
+    <%include tmpl='#rebuild-template'/%>
+  </div>
+</script>
+
+<!-- rebuild button or no rebuild icon -->
+<script id="rebuild-template" type="text/x-jsrender">
+  <%if is_default_project_build%>
+    <!-- no rebuild info icon -->
+    <span class="pull-right glyphicon glyphicon-question-sign get-help <%if state == 'Succeeded'%>get-help-green<%else state == 'Failed'%>get-help-red<%else%>get-help-blue<%/if%>"
+    title="Builds in this project cannot be started from Toaster: they are started from the command line">
+    </span>
+  <%else%>
+    <!-- rebuild button -->
+    <span class="rebuild-btn alert-link <%if state == 'Success'%>success<%else state == 'Failed'%>danger<%else%>info<%/if%> pull-right"
+    data-request-url="<%:rebuild_url%>" data-target='<%:build_targets_json%>'>
+      <span class="glyphicon glyphicon-repeat"></span>
+      Rebuild
+    </span>
+  <%/if%>
+</script>
+
+<!-- cancel button or no cancel icon -->
+<script id="cancel-template" type="text/x-jsrender">
+  <%if is_default_project_build%>
+    <!-- no cancel icon -->
+    <span class="glyphicon glyphicon-question-sign get-help get-help-blue pull-right" title="Builds in this project cannot be cancelled from Toaster: they can only be cancelled from the command line"></span>
+  <%else%>
+    <!-- cancel button -->
+    <span class="cancel-build-btn pull-right alert-link"
+    data-buildrequest-id="<%:buildrequest_id%>" data-request-url="<%:cancel_url%>">
+      <span class="glyphicon glyphicon-remove-circle"></span>
+      Cancel
+    </span>
+  <%/if%>
+</script>
+
 <script>
   $(document).ready(function () {
     var ctx = {
@@ -17,166 +272,3 @@
     }
   });
 </script>
-
-{%if mru %}
-
-  {%if mrb_type == 'project' %}
-  <h2>
-    Latest project builds
-
-    {% if project.is_default %}
-    <span class="glyphicon glyphicon-question-sign get-help heading-help" title="" data-original-title="Builds in this project cannot be started from Toaster: they are started from the command line"></span>
-    {% endif %}
-  </h2>
-  {% else %}
-    <div class="page-header">
-      <h1>
-      Latest builds
-      </h1>
-    </div>
-  {% endif %}
-  <div id="latest-builds">
-  {% for build in mru %}
-  <div data-latest-build-result="{{ build.id }}" class="alert build-result {%if build.outcome == build.SUCCEEDED%}alert-success{%elif build.outcome == build.FAILED%}alert-danger{%else%}alert-info{%endif%}">
-        {% if mrb_type != 'project' %}
-        <div class="row project-name">
-          <div class="col-md-12">
-            <small>
-              <a class="alert-link text-uppercase" href={% project_url build.project %}>{{build.project.name}}</a>
-            </small>
-          </div>
-        </div>
-        {% endif %}
-        <div class="row">
-          <div class="col-md-3">
-    {%if build.outcome == build.SUCCEEDED or build.outcome == build.FAILED %}
-              <a href="{%url 'builddashboard' build.pk%}" class="alert-link">
-    {% endif %}
-            {% if build.target_set.all.count > 0 %}
-                <span data-toggle="tooltip"
-                  {% if build.target_set.all.count > 1 %}
-                    {{build.get_sorted_target_list.0.target}}
-                    title="Recipes:
-                    {% for target in build.get_sorted_target_list %}
-                        {% if target.task %}
-                            {{target.target}}:{{target.task}}
-                        {% else %}
-                            {{target.target}}
-                        {% endif %}
-                    {% endfor %}"
-                  {% endif %}
-                >
-                {% if build.target_set.all.0.task %}
-                    {{build.get_sorted_target_list.0.target}}:{{build.target_set.all.0.task}}
-                {% else %}
-                    {{build.get_sorted_target_list.0.target}}
-                {% endif %}
-                {% if build.target_set.all.count > 1 %}
-                    (+{{build.target_set.all.count|add:"-1"}})
-                {% endif %}
-                </span>
-             {% endif %}
-    {%if build.outcome == build.SUCCEEDED or build.outcome == build.FAILED %}
-                </a>
-    {% endif %}
-            </div>
-    {% if build.outcome == build.SUCCEEDED or build.outcome == build.FAILED %}
-            <div class="col-md-2">
-                {% if build.completed_on|format_build_date  %}
-                    {{ build.completed_on|date:'d/m/y H:i' }}
-                {% else %}
-                    {{ build.completed_on|date:'H:i' }}
-                {% endif %}
-            </div>
-    {% endif %}
-    {%if build.outcome == build.SUCCEEDED or build.outcome == build.FAILED %}
-            <div class="col-md-2">
-      {% if  build.errors.count %}
-                <span class="glyphicon glyphicon-minus-sign"></span>
-                <a href="{%url 'builddashboard' build.pk%}#errors" class="alert-link">{{build.errors.count}} error{{build.errors.count|pluralize}}</a>
-      {% endif %}
-            </div>
-            <div class="col-md-2">
-      {% if  build.warnings.count %}
-                <span class="glyphicon glyphicon-warning-sign build-warnings"></span>
-                <a href="{%url 'builddashboard' build.pk%}#warnings" class="alert-link build-warnings">{{build.warnings.count}} warning{{build.warnings.count|pluralize}}</a>
-      {% endif %}
-            </div>
-            <div class="col-md-3">
-              Build time: <a class="alert-link" href="{% url 'buildtime' build.pk %}">{{ build.timespent_seconds|sectohms }}
-              </a>
-              {% if build.project.is_default %}
-                  <span class="pull-right glyphicon glyphicon-question-sign get-help
-                  {% if build.outcome == build.SUCCEEDED %}
-                      get-help-green
-                  {% elif build.outcome == build.FAILED %}
-                      get-help-red
-                  {% else %}
-                      get-help-blue
-                  {% endif %}
-                  " title="Builds in this project cannot be started from Toaster: they are started from the command line">
-                  </span>
-              {% else %}
-                  <a href="#" class="run-again-btn alert-link
-                      {% if build.outcome == build.SUCCEEDED %}
-                          success
-                      {% elif build.outcome == build.FAILED %}
-                          danger
-                      {% else %}
-                          info
-                      {%endif%}
-                      pull-right"
-                      data-request-url="{% url 'xhr_buildrequest' build.project.pk %}"
-                      data-target='{{build.target_set.all|get_tasks|json}}'>
-                      <span class="glyphicon glyphicon-repeat"></span>
-                      Rebuild
-                  </a>
-              {% endif %}
-            </div>
-    {%endif%}
-
-    {%if build.outcome == build.IN_PROGRESS %}
-    <div class="col-md-4" style="display:none" id="cancelling-msg-{{build.buildrequest.pk}}">
-      Cancelling the build ...
-    </div>
-    <div class="col-md-4 col-md-offset-1 progress-info">
-      <div class="progress" id="build-pc-done-title-{{build.pk}}">
-        <div id="build-pc-done-bar-{{build.pk}}" style="width: {{build.completeper}}%;" class="progress-bar"></div>
-      </div>
-    </div>
-    <div class="col-md-4 progress-info"><span id="build-pc-done-{{build.pk}}">{{build.completeper}}</span>% of tasks complete
-    {# No build cancel for command line builds project #}
-    {% if build.project.is_default %}
-    <span class="glyphicon glyphicon-question-sign get-help get-help-blue pull-right" title="Builds in this project cannot be cancelled from Toaster: they can only be cancelled from the command line"></span>
-    {% else %}
-      <a href="#" class="cancel-build-btn pull-right alert-link"
-          data-buildrequest-id={{build.buildrequest.pk}}
-          data-request-url="{% url 'xhr_buildrequest' build.project.pk %}" >
-        <span class="glyphicon glyphicon-remove-circle"></span>
-        Cancel
-      </a>
-    </div>
-    {% endif %}
-
-    {%endif%} {# end if in progress #}
-
-    {% if build.outcome == build.CANCELLED %}
-    <div class="col-md-6">
-      Build cancelled
-    </div>
-    <div class="col-md-3">
-      <a href="#" class="info pull-right run-again-btn alert-link"
-        data-request-url="{% url 'xhr_buildrequest' build.project.pk %}"
-        data-target='{{build.target_set.all|get_tasks|json}}'>
-        <span class="glyphicon glyphicon-repeat"></span>
-        Rebuild
-      </a>
-    </div>
-    {% endif %}
-  </div>
-</div>
-  {% endfor %}
-  </div>
-
-{%endif%}
-

bitbake/lib/toaster/toastergui/templates/newcustomimage_modal.html

@@ -48,7 +48,12 @@
       </div>
 
       <div class="modal-footer">
-        <button id="create-new-custom-image-btn" class="btn btn-primary btn-lg" data-original-title="" title="" disabled>Create custom image</button>
+        <button id="create-new-custom-image-btn" class="btn btn-primary btn-large" disabled>
+          <span data-role="submit-state">Create custom image</span>
+          <span data-role="loading-state" style="display:none">
+            <i class="fa-pulse icon-spinner"></i>&nbsp;Creating custom image...
+          </span>
+        </button>
       </div>
     </div>
   </div>

bitbake/lib/toaster/toastergui/templates/package_detail_base.html

@@ -133,19 +133,33 @@
                     {% endcomment %}
                 {% endif %}
             </dd>
-            {% if package.recipe.layer_version.branch %}
             <dt>
                 Layer branch
                 <span class="glyphicon glyphicon-question-sign get-help" title="The Git branch of the layer providing the recipe that builds this package"></span>
+                {%if package.recipe.layer_version.layer.local_source_dir %}
+		<dd>
+		<span class="text-muted">Not applicable</span>
+		<span class="glyphicon glyphicon-question-sign get-help" title="The source code of {{package.recipe.layer_version.layer.name}} is not in a Git repository, so there is no branch associated with it"></span>
+		</dd>
+		{% endif %}
             </dt>
+	    {% if not package.recipe.layer_version.layer.local_source_dir %}
             <dd>{{package.recipe.layer_version.branch}}</dd>
-            {% endif %}
+	    {% endif %}
             <dt>
                 Layer commit
                 <span class="glyphicon glyphicon-question-sign get-help" title="The Git commit of the layer providing the recipe that builds this package"></span>
+                {%if package.recipe.layer_version.layer.local_source_dir %}
+		<dd>
+		<span class="text-muted">Not applicable</span>
+		<span class="glyphicon glyphicon-question-sign get-help" title="The source code of {{package.recipe.layer_version.layer.name}} is not in a Git repository, so there is no commit associated with it"></span>
+		</dd>
+		{% endif %}
             </dt>
 
+	    {% if not package.recipe.layer_version.layer.local_source_dir %}
             <dd class="iscommit">{{package.recipe.layer_version.commit}}</dd>
+	    {% endif %}
 
         </dl>
       <div> <!-- end well -->