buildtools/uninative-tarball: Fix deployment overlap issues

We still have problems where deploying SDKMACHINE=i686 can cause removal
of SDKMACHINE=x86_64 artefacts.

The reason is that x86_64 is a BUILD_ARCH as well as an SDK_ARCH and
the manifest namespaces overlap. To fix this, set PACKAGE_ARCH and
the stamp-extra-into to include SDK_OS. SDK_OS may not be entirely correct
but it is what sstate.bbclass uses for nativesdk and fixing that is
a separate issue.

This is confirmed to resolve artefact problems on the AB which have been
delaying a new uninative release.

(From OE-Core rev: 1dbc6ec4ca061570d2482c9abebcf720298db9b7)

Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>

bitbake/LICENSE

@@ -10,4 +10,6 @@ Foundation and individual contributors.
 
 * Twitter typeahead.js redistributed under the MIT license. Note that the JS source has one small modification, so the full unminified file is currently included to make it obvious where this is.
 
+* jsrender is redistributed under the MIT license.
+
 * QUnit is redistributed under the MIT license.

bitbake/bin/bitbake

@@ -38,7 +38,7 @@ from bb.main import bitbake_main, BitBakeConfigParameters, BBMainException
 if sys.getfilesystemencoding() != "utf-8":
     sys.exit("Please use a locale setting which supports utf-8.\nPython can't change the filesystem locale after loading so we need a utf-8 when python starts or things won't work.")
 
-__version__ = "1.31.0"
+__version__ = "1.31.1"
 
 if __name__ == "__main__":
     if __version__ != bb.__version__:

bitbake/bin/bitbake-diffsigs

@@ -115,9 +115,9 @@ parser.add_option("-t", "--task",
 options, args = parser.parse_args(sys.argv)
 
 if options.taskargs:
-    tinfoil = bb.tinfoil.Tinfoil()
-    tinfoil.prepare(config_only = True)
-    find_compare_task(tinfoil, options.taskargs[0], options.taskargs[1])
+    with bb.tinfoil.Tinfoil() as tinfoil:
+        tinfoil.prepare(config_only=True)
+        find_compare_task(tinfoil, options.taskargs[0], options.taskargs[1])
 else:
     if len(args) == 1:
         parser.print_help()

bitbake/bin/bitbake-layers

@@ -87,31 +87,34 @@ def main():
 
     plugins = []
     tinfoil = tinfoil_init(False)
-    for path in ([topdir] +
-                 tinfoil.config_data.getVar('BBPATH', True).split(':')):
-        pluginpath = os.path.join(path, 'lib', 'bblayers')
-        bb.utils.load_plugins(logger, plugins, pluginpath)
+    try:
+        for path in ([topdir] +
+                    tinfoil.config_data.getVar('BBPATH', True).split(':')):
+            pluginpath = os.path.join(path, 'lib', 'bblayers')
+            bb.utils.load_plugins(logger, plugins, pluginpath)
 
-    registered = False
-    for plugin in plugins:
-        if hasattr(plugin, 'register_commands'):
-            registered = True
-            plugin.register_commands(subparsers)
-        if hasattr(plugin, 'tinfoil_init'):
-            plugin.tinfoil_init(tinfoil)
+        registered = False
+        for plugin in plugins:
+            if hasattr(plugin, 'register_commands'):
+                registered = True
+                plugin.register_commands(subparsers)
+            if hasattr(plugin, 'tinfoil_init'):
+                plugin.tinfoil_init(tinfoil)
 
-    if not registered:
-        logger.error("No commands registered - missing plugins?")
-        sys.exit(1)
+        if not registered:
+            logger.error("No commands registered - missing plugins?")
+            sys.exit(1)
 
-    args = parser.parse_args(unparsed_args, namespace=global_args)
+        args = parser.parse_args(unparsed_args, namespace=global_args)
 
-    if getattr(args, 'parserecipes', False):
-        tinfoil.config_data.disableTracking()
-        tinfoil.parseRecipes()
-        tinfoil.config_data.enableTracking()
+        if getattr(args, 'parserecipes', False):
+            tinfoil.config_data.disableTracking()
+            tinfoil.parseRecipes()
+            tinfoil.config_data.enableTracking()
 
-    return args.func(args)
+        return args.func(args)
+    finally:
+        tinfoil.shutdown()
 
 
 if __name__ == "__main__":

bitbake/bin/bitbake-selftest

@@ -25,31 +25,48 @@ try:
 except RuntimeError as exc:
     sys.exit(str(exc))
 
-def usage():
-    print('usage: [BB_SKIP_NETTESTS=yes] %s [-v] [testname1 [testname2]...]' % os.path.basename(sys.argv[0]))
-
-verbosity = 1
-
-tests = sys.argv[1:]
-if '-v' in sys.argv:
-    tests.remove('-v')
-    verbosity = 2
-
-if tests:
-    if '--help' in sys.argv[1:]:
-        usage()
-        sys.exit(0)
-else:
-    tests = ["bb.tests.codeparser",
-             "bb.tests.cow",
-             "bb.tests.data",
-             "bb.tests.fetch",
-             "bb.tests.parse",
-             "bb.tests.utils"]
+tests = ["bb.tests.codeparser",
+         "bb.tests.cow",
+         "bb.tests.data",
+         "bb.tests.fetch",
+         "bb.tests.parse",
+         "bb.tests.utils"]
 
 for t in tests:
     t = '.'.join(t.split('.')[:3])
     __import__(t)
 
-unittest.main(argv=["bitbake-selftest"] + tests, verbosity=verbosity)
 
+# Set-up logging
+class StdoutStreamHandler(logging.StreamHandler):
+    """Special handler so that unittest is able to capture stdout"""
+    def __init__(self):
+        # Override __init__() because we don't want to set self.stream here
+        logging.Handler.__init__(self)
+
+    @property
+    def stream(self):
+        # We want to dynamically write wherever sys.stdout is pointing to
+        return sys.stdout
+
+
+handler = StdoutStreamHandler()
+bb.logger.addHandler(handler)
+bb.logger.setLevel(logging.DEBUG)
+
+
+ENV_HELP = """\
+Environment variables:
+  BB_SKIP_NETTESTS      set to 'yes' in order to skip tests using network
+                        connection
+  BB_TMPDIR_NOCLEAN     set to 'yes' to preserve test tmp directories
+"""
+
+class main(unittest.main):
+    def _print_help(self, *args, **kwargs):
+        super(main, self)._print_help(*args, **kwargs)
+        print(ENV_HELP)
+
+
+if __name__ == '__main__':
+        main(defaultTest=tests, buffer=True)

bitbake/bin/bitbake-worker

@@ -115,7 +115,7 @@ def sigterm_handler(signum, frame):
     os.killpg(0, signal.SIGTERM)
     sys.exit()
 
-def fork_off_task(cfg, data, workerdata, fn, task, taskname, appends, taskdepdata, quieterrors=False):
+def fork_off_task(cfg, data, databuilder, workerdata, fn, task, taskname, appends, taskdepdata, quieterrors=False):
     # We need to setup the environment BEFORE the fork, since
     # a fork() or exec*() activates PSEUDO...
 
@@ -193,15 +193,19 @@ def fork_off_task(cfg, data, workerdata, fn, task, taskname, appends, taskdepdat
             if umask:
                 os.umask(umask)
 
-            data.setVar("BB_WORKERCONTEXT", "1")
-            data.setVar("BB_TASKDEPDATA", taskdepdata)
-            data.setVar("BUILDNAME", workerdata["buildname"])
-            data.setVar("DATE", workerdata["date"])
-            data.setVar("TIME", workerdata["time"])
-            bb.parse.siggen.set_taskdata(workerdata["sigdata"])
-            ret = 0
             try:
-                the_data = bb.cache.Cache.loadDataFull(fn, appends, data)
+                bb_cache = bb.cache.NoCache(databuilder)
+                (realfn, virtual, mc) = bb.cache.virtualfn2realfn(fn)
+                the_data = databuilder.mcdata[mc]
+                the_data.setVar("BB_WORKERCONTEXT", "1")
+                the_data.setVar("BB_TASKDEPDATA", taskdepdata)
+                the_data.setVar("BUILDNAME", workerdata["buildname"])
+                the_data.setVar("DATE", workerdata["date"])
+                the_data.setVar("TIME", workerdata["time"])
+                bb.parse.siggen.set_taskdata(workerdata["sigdata"])
+                ret = 0
+
+                the_data = bb_cache.loadDataFull(fn, appends)
                 the_data.setVar('BB_TASKHASH', workerdata["runq_hash"][task])
 
                 bb.utils.set_process_name("%s:%s" % (the_data.getVar("PN", True), taskname.replace("do_", "")))
@@ -371,7 +375,8 @@ class BitbakeWorker(object):
         bb.msg.loggerDefaultVerbose = self.workerdata["logdefaultverbose"]
         bb.msg.loggerVerboseLogs = self.workerdata["logdefaultverboselogs"]
         bb.msg.loggerDefaultDomains = self.workerdata["logdefaultdomain"]
-        self.data.setVar("PRSERV_HOST", self.workerdata["prhost"])
+        for mc in self.databuilder.mcdata:
+            self.databuilder.mcdata[mc].setVar("PRSERV_HOST", self.workerdata["prhost"])
 
     def handle_ping(self, _):
         workerlog_write("Handling ping\n")
@@ -389,7 +394,7 @@ class BitbakeWorker(object):
         fn, task, taskname, quieterrors, appends, taskdepdata = pickle.loads(data)
         workerlog_write("Handling runtask %s %s %s\n" % (task, fn, taskname))
 
-        pid, pipein, pipeout = fork_off_task(self.cookercfg, self.data, self.workerdata, fn, task, taskname, appends, taskdepdata, quieterrors)
+        pid, pipein, pipeout = fork_off_task(self.cookercfg, self.data, self.databuilder, self.workerdata, fn, task, taskname, appends, taskdepdata, quieterrors)
 
         self.build_pids[pid] = task
         self.build_pipes[pid] = runQueueWorkerPipe(pipein, pipeout)

bitbake/bin/toaster

@@ -33,9 +33,6 @@ webserverKillAll()
             while kill -0 $pid 2>/dev/null; do
                 kill -SIGTERM -$pid 2>/dev/null
                 sleep 1
-                # Kill processes if they are still running - may happen
-                # in interactive shells
-                ps fux | grep "python.*manage.py runserver" | awk '{print $2}' | xargs kill
             done
             rm  ${pidfile}
         fi
@@ -59,7 +56,8 @@ webserverStartAll()
         echo "Failed migrations, aborting system start" 1>&2
         return $retval
     fi
-
+    # Make sure that checksettings can pick up any value for TEMPLATECONF
+    export TEMPLATECONF
     $MANAGE checksettings --traceback || retval=1
 
     if [ $retval -eq 1 ]; then
@@ -94,13 +92,8 @@ stop_system()
     # prevent reentry
     if [ $INSTOPSYSTEM -eq 1 ]; then return; fi
     INSTOPSYSTEM=1
-    if [ -f ${BUILDDIR}/.toasterui.pid ]; then
-        kill `cat ${BUILDDIR}/.toasterui.pid` 2>/dev/null
-        rm ${BUILDDIR}/.toasterui.pid
-    fi
     webserverKillAll
     # unset exported variables
-    unset TOASTER_CONF
     unset TOASTER_DIR
     unset BITBAKE_UI
     unset BBBASEDIR
@@ -153,23 +146,9 @@ if [ -n "$TEMPLATECONF" ]; then
         if [ -d "$OEROOT/$TEMPLATECONF" ]; then
             TEMPLATECONF="$OEROOT/$TEMPLATECONF"
         fi
-        if [ ! -d "$TEMPLATECONF" ]; then
-            echo >&2 "Error: '$TEMPLATECONF' must be a directory containing toasterconf.json"
-            return 1
-        fi
     fi
 fi
 
-if [ "$TOASTER_CONF" = "" ]; then
-    TOASTER_CONF="$TEMPLATECONF/toasterconf.json"
-    export TOASTER_CONF=$(python3 -c "import os; print(os.path.realpath('$TOASTER_CONF'))")
-fi
-
-if [ ! -f $TOASTER_CONF ]; then
-    echo "$TOASTER_CONF configuration file not found. Set TOASTER_CONF to specify file or fix .templateconf"
-    return 1
-fi
-
 # this defines the dir toaster will use for
 # 1) clones of layers (in _toaster_clones )
 # 2) the build dir (in build)
@@ -203,6 +182,10 @@ for param in $*; do
                 ADDR_PORT="localhost:$PORT"
             fi
     ;;
+    --help)
+            echo "$HELP"
+            return 0
+    ;;
     *)
             echo "$HELP"
             return 1
@@ -259,14 +242,6 @@ case $CMD in
             fi
         fi
 
-        # kill Toaster web server if it's alive
-        if [ -e $BUILDDIR/.toastermain.pid ] && kill -0 `cat $BUILDDIR/.toastermain.pid`; then
-            echo "Warning: bitbake appears to be dead, but the Toaster web server is running." 1>&2
-            echo " Something fishy is going on." 1>&2
-            echo "Cleaning up the web server to start from a clean slate."
-            webserverKillAll
-        fi
-
         # Create configuration file
         conf=${BUILDDIR}/conf/local.conf
         line='INHERIT+="toaster buildhistory"'

bitbake/contrib/dump_cache.py

@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
 # ex:ts=4:sw=4:sts=4:et
 # -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*-
 #

bitbake/doc/bitbake-user-manual/bitbake-user-manual-execution.xml

@@ -596,7 +596,7 @@
             "<link linkend='checksums'>Checksums (Signatures)</link>"
             section for information).
             It is also possible to append extra metadata to the stamp using
-            the "stamp-extra-info" task flag.
+            the <filename>[stamp-extra-info]</filename> task flag.
             For example, OpenEmbedded uses this flag to make some tasks machine-specific.
         </para>
 
@@ -653,7 +653,8 @@
             </itemizedlist>
             It is possible to have functions run before and after a task's main
             function.
-            This is done using the "prefuncs" and "postfuncs" flags of the task
+            This is done using the <filename>[prefuncs]</filename>
+            and <filename>[postfuncs]</filename> flags of the task
             that lists the functions to run.
         </para>
     </section>
@@ -827,7 +828,7 @@
             itself.
             The simplest parameter to pass is "none", which causes a
             set of signature information to be written out into
-            <filename>STAMP_DIR</filename>
+            <filename>STAMPS_DIR</filename>
             corresponding to the targets specified.
             The other currently available parameter is "printdiff",
             which causes BitBake to try to establish the closest

bitbake/doc/bitbake-user-manual/bitbake-user-manual-intro.xml

@@ -648,26 +648,7 @@
                     recipe and <filename>taskB</filename> for the second
                     recipe:
                     <literallayout class='monospaced'>
-     $ bitbake myfirstrecipe recipe:do_taskA mysecondrecipe recipe:do_taskB
-                    </literallayout>
-                </para>
-            </section>
-
-            <section id='executing-a-list-of-task-and-recipe-combinations'>
-                <title>Executing a List of Task and Recipe Combinations</title>
-
-                <para>
-                    The BitBake command line supports specifying different
-                    tasks for individual targets when you specify multiple
-                    targets.
-                    For example, suppose you had two targets (or recipes)
-                    <filename>myfirstrecipe</filename> and
-                    <filename>mysecondrecipe</filename> and you needed
-                    BitBake to run <filename>taskA</filename> for the first
-                    recipe and <filename>taskB</filename> for the second
-                    recipe:
-                    <literallayout class='monospaced'>
-     $ bitbake myfirstrecipe recipe:do_taskA mysecondrecipe recipe:do_taskB
+     $ bitbake myfirstrecipe:do_taskA mysecondrecipe:do_taskB
                     </literallayout>
                 </para>
             </section>

bitbake/doc/bitbake-user-manual/bitbake-user-manual-metadata.xml

@@ -44,28 +44,73 @@
      VARIABLE = " "
                 </literallayout>
             </para>
+
+            <para>
+                You can use single quotes instead of double quotes
+                when setting a variable's value.
+                Doing so allows you to use values that contain the double
+                character:
+                <literallayout class='monospaced'>
+     VARIABLE = 'I have a " in my value'
+                </literallayout>
+                <note>
+                    Unlike in Bourne shells, single quotes work identically
+                    to double quotes in all other ways.
+                    They do not suppress variable expansions.
+                </note>
+            </para>
         </section>
 
         <section id='variable-expansion'>
             <title>Variable Expansion</title>
 
             <para>
-                BitBake supports variables referencing one another's
-                contents using a syntax that is similar to shell scripting.
-                Following is an example that results in <filename>A</filename>
-                containing "aval" and <filename>B</filename> evaluating to
-                "preavalpost" based on that current value of
-                <filename>A</filename>.
+                Variables can reference the contents of other variables
+                using a syntax that is similar to variable expansion in
+                Bourne shells.
+                The following assignments
+                result in A containing "aval" and B evaluating to "preavalpost".
                 <literallayout class='monospaced'>
      A = "aval"
      B = "pre${A}post"
                 </literallayout>
-                You should realize that whenever <filename>B</filename> is
-                referenced, its evaluation will depend on the state of
-                <filename>A</filename> at that time.
-                Thus, later evaluations of <filename>B</filename> in the
-                previous example could result in different values
-                depending on the value of <filename>A</filename>.
+                <note>
+                    Unlike in Bourne shells, the curly braces are mandatory:
+                    Only <filename>${FOO}</filename> and not
+                    <filename>$FOO</filename> is recognized as an expansion of
+                    <filename>FOO</filename>.
+                </note>
+                The "=" operator does not immediately expand variable
+                references in the right-hand side.
+                Instead, expansion is deferred until the variable assigned to
+                is actually used.
+                The result depends on the current values of the referenced
+                variables.
+                The following example should clarify this behavior:
+                <literallayout class='monospaced'>
+     A = "${B} baz"
+     B = "${C} bar"
+     C = "foo"
+     *At this point, ${A} equals "foo bar baz"*
+     C = "qux"
+     *At this point, ${A} equals "qux bar baz"*
+     B = "norf"
+     *At this point, ${A} equals "norf baz"*
+                </literallayout>
+                Contrast this behavior with the
+                <link linkend='immediate-variable-expansion'>immediate variable expansion</link>
+                operator (i.e. ":=").
+            </para>
+
+            <para>
+                If the variable expansion syntax is used on a variable that
+                does not exist, the string is kept as is.
+                For example, given the following assignment,
+                <filename>BAR</filename> expands to the literal string
+                "${FOO}" as long as <filename>FOO</filename> does not exist.
+                <literallayout class='monospaced'>
+     BAR = "${FOO}"
+                </literallayout>
             </para>
         </section>
 
@@ -259,6 +304,60 @@
                 "789 123456" and <filename>FOO2</filename> becomes
                 "ghi abcdef".
             </para>
+
+            <para>
+                Like "_append" and "_prepend", "_remove"
+                is deferred until after parsing completes.
+            </para>
+        </section>
+
+        <section id='override-style-operation-advantages'>
+            <title>Override Style Operation Advantages</title>
+
+            <para>
+                An advantage of the override style operations
+                "_append", "_prepend", and "_remove" as compared to the
+                "+=" and "=+" operators is that the override style
+                operators provide guaranteed operations.
+                For example, consider a class <filename>foo.bbclass</filename>
+                that needs to add the value "val" to the variable
+                <filename>FOO</filename>, and a recipe that uses
+                <filename>foo.bbclass</filename> as follows:
+                <literallayout class='monospaced'>
+     inherit foo
+
+     FOO = "initial"
+                </literallayout>
+                If <filename>foo.bbclass</filename> uses the "+=" operator,
+                as follows, then the final value of <filename>FOO</filename>
+                will be "initial", which is not what is desired:
+                <literallayout class='monospaced'>
+     FOO += "val"
+                </literallayout>
+                If, on the other hand, <filename>foo.bbclass</filename>
+                uses the "_append" operator, then the final value of
+                <filename>FOO</filename> will be "initial val", as intended:
+                <literallayout class='monospaced'>
+     FOO_append = " val"
+                </literallayout>
+                <note>
+                    It is never necessary to use "+=" together with "_append".
+                    The following sequence of assignments appends "barbaz" to
+                    <filename>FOO</filename>:
+                    <literallayout class='monospaced'>
+     FOO_append = "bar"
+     FOO_append = "baz"
+                    </literallayout>
+                    The only effect of changing the second assignment in the
+                    previous example to use "+=" would be to add a space before
+                    "baz" in the appended value (due to how the "+=" operator
+                    works).
+                </note>
+                Another advantage of the override style operations is that
+                you can combine them with other overrides as described in the
+                "<link linkend='conditional-syntax-overrides'>Conditional Syntax (Overrides)</link>"
+                section.
+            </para>
         </section>
 
         <section id='variable-flag-syntax'>
@@ -277,8 +376,7 @@
                 You can define, append, and prepend values to variable flags.
                 All the standard syntax operations previously mentioned work
                 for variable flags except for override style syntax
-                (i.e. <filename>_prepend</filename>, <filename>_append</filename>,
-                and <filename>_remove</filename>).
+                (i.e. "_prepend", "_append", and "_remove").
             </para>
 
             <para>
@@ -289,9 +387,9 @@
      FOO[a] += "456"
                 </literallayout>
                 The variable <filename>FOO</filename> has two flags:
-                <filename>a</filename> and <filename>b</filename>.
+                <filename>[a]</filename> and <filename>[b]</filename>.
                 The flags are immediately set to "abc" and "123", respectively.
-                The <filename>a</filename> flag becomes "abc 456".
+                The <filename>[a]</filename> flag becomes "abc 456".
             </para>
 
             <para>
@@ -330,9 +428,45 @@
      PN = "${@bb.parse.BBHandler.vars_from_file(d.getVar('FILE', False),d)[0] or 'defaultpkgname'}"
      PV = "${@bb.parse.BBHandler.vars_from_file(d.getVar('FILE', False),d)[1] or '1.0'}"
                 </literallayout>
+                <note>
+                    Inline Python expressions work just like variable expansions
+                    insofar as the "=" and ":=" operators are concerned.
+                    Given the following assignment, <filename>foo()</filename>
+                    is called each time <filename>FOO</filename> is expanded:
+                    <literallayout class='monospaced'>
+     FOO = "${@foo()}"
+                    </literallayout>
+                    Contrast this with the following immediate assignment, where
+                    <filename>foo()</filename> is only called once, while the
+                    assignment is parsed:
+                    <literallayout class='monospaced'>
+     FOO := "${@foo()}"
+                    </literallayout>
+                </note>
+                For a different way to set variables with Python code during
+                parsing, see the
+                "<link linkend='anonymous-python-functions'>Anonymous Python Functions</link>"
+                section.
             </para>
         </section>
 
+        <section id='unsetting-variables'>
+            <title>Unseting variables</title>
+
+            <para>
+                It is possible to completely remove a variable or a variable flag 
+                from BitBake's internal data dictionary by using the "unset" keyword.
+                Here is an example:
+                <literallayout class='monospaced'>
+        unset DATE
+        unset do_fetch[noexec]
+                </literallayout>
+                These two statements remove the <filename>DATE</filename> and the 
+                <filename>do_fetch[noexec]</filename> flag.
+            </para>
+
+        </section>
+
         <section id='providing-pathnames'>
             <title>Providing Pathnames</title>
 
@@ -357,6 +491,53 @@
         </section>
     </section>
 
+    <section id='exporting-variables-to-the-environment'>
+        <title>Exporting Variables to the Environment</title>
+
+        <para>
+            You can export variables to the environment of running
+            tasks by using the <filename>export</filename> keyword.
+            For example, in the following example, the
+            <filename>do_foo</filename> task prints "value from
+            the environment" when run:
+            <literallayout class='monospaced'>
+     export ENV_VARIABLE
+     ENV_VARIABLE = "value from the environment"
+
+     do_foo() {
+         bbplain "$ENV_VARIABLE"
+     }
+            </literallayout>
+            <note>
+                BitBake does not expand <filename>$ENV_VARIABLE</filename>
+                in this case because it lacks the obligatory
+                <filename>{}</filename>.
+                Rather, <filename>$ENV_VARIABLE</filename> is expanded
+                by the shell.
+            </note>
+            It does not matter whether
+            <filename>export ENV_VARIABLE</filename> appears before or
+            after assignments to <filename>ENV_VARIABLE</filename>.
+        </para>
+
+        <para>
+            It is also possible to combine <filename>export</filename>
+            with setting a value for the variable.
+            Here is an example:
+            <literallayout class='monospaced'>
+     export ENV_VARIABLE = "<replaceable>variable-value</replaceable>"
+            </literallayout>
+            In the output of <filename>bitbake -e</filename>, variables
+            that are exported to the environment are preceded by "export".
+        </para>
+
+        <para>
+            Among the variables commonly exported to the environment
+            are <filename>CC</filename> and <filename>CFLAGS</filename>,
+            which are picked up by many build systems.
+        </para>
+    </section>
+
     <section id='conditional-syntax-overrides'>
         <title>Conditional Syntax (Overrides)</title>
 
@@ -455,6 +636,34 @@
      KERNEL_FEATURES_append_qemux86-64=" cfg/sound.scc cfg/paravirt_kvm.scc"
                         </literallayout>
                         </para></listitem>
+                    <listitem><para><emphasis>Setting a Variable for a Single Task:</emphasis>
+                        BitBake supports setting a variable just for the
+                        duration of a single task.
+                        Here is an example:
+                        <literallayout class='monospaced'>
+     FOO_task-configure = "val 1"
+     FOO_task-compile = "val 2"
+                        </literallayout>
+                        In the previous example, <filename>FOO</filename>
+                        has the value "val 1" while the
+                        <filename>do_configure</filename> task is executed,
+                        and the value "val 2" while the
+                        <filename>do_compile</filename> task is executed.
+                        </para>
+
+                        <para>Internally, this is implemented by prepending
+                        the task (e.g. "task-compile:") to the value of
+                        <link linkend='var-OVERRIDES'><filename>OVERRIDES</filename></link>
+                        for the local datastore of the <filename>do_compile</filename>
+                        task.</para>
+
+                        <para>You can also use this syntax with other combinations
+                        (e.g. "<filename>_prepend</filename>") as shown in the
+                        following example:
+                        <literallayout class='monospaced'>
+     EXTRA_OEMAKE_prepend_task-compile = "${PARALLEL_MAKE} "
+                        </literallayout>
+                        </para></listitem>
                 </itemizedlist>
             </para>
         </section>
@@ -545,15 +754,15 @@
      OVERRIDES = "foo"
      A = "Y"
      A_foo_append = "Z"
-     A_foo_append += "X"
+     A_foo_append = "X"
                 </literallayout>
                 For this case, before any overrides are resolved,
                 <filename>A</filename> is set to "Y" using an immediate assignment.
                 After this immediate assignment, <filename>A_foo</filename> is set
                 to "Z", and then further appended with
-                "X" leaving the variable set to "Z X".
+                "X" leaving the variable set to "ZX".
                 Finally, applying the override for "foo" results in the conditional
-                variable <filename>A</filename> becoming "Z X" (i.e.
+                variable <filename>A</filename> becoming "ZX" (i.e.
                 <filename>A</filename> is replaced with <filename>A_foo</filename>).
             </para>
 
@@ -572,7 +781,7 @@
                 Initially, <filename>A</filename> is set to "1 45" because
                 of the three statements that use immediate operators.
                 After these assignments are made, BitBake applies the
-                <filename>_append</filename> operations.
+                "_append" operations.
                 Those operations result in <filename>A</filename> becoming "1 4523".
             </para>
         </section>
@@ -965,32 +1174,81 @@
             <title>Anonymous Python Functions</title>
 
             <para>
-                Sometimes it is useful to run some code during
-                parsing to set variables or to perform other operations
-                programmatically.
-                To do this, you can define an anonymous Python function.
-                Here is an example that conditionally sets a
-                variable based on the value of another variable:
-                <literallayout class='monospaced'>
-     python __anonymous () {
-         if d.getVar('SOMEVAR', True) == 'value':
-             d.setVar('ANOTHERVAR', 'value2')
-     }
-                </literallayout>
-                The "__anonymous" function name is optional, so the
-                following example is functionally equivalent to the above:
+                Sometimes it is useful to set variables or perform
+                other operations programmatically during parsing.
+                To do this, you can define special Python functions,
+                called anonymous Python functions, that run at the
+                end of parsing.
+                For example, the following conditionally sets a variable
+                based on the value of  another variable:
                 <literallayout class='monospaced'>
      python () {
          if d.getVar('SOMEVAR', True) == 'value':
              d.setVar('ANOTHERVAR', 'value2')
      }
                 </literallayout>
-                Because unlike other Python functions anonymous
-                Python functions are executed during parsing, the
-                "d" variable within an anonymous Python function represents
-                the datastore for the entire recipe.
-                Consequently, you can set variable values here and
-                those values can be picked up by other functions.
+                An equivalent way to mark a function as an anonymous
+                function is to give it the name "__anonymous", rather
+                than no name.
+            </para>
+
+            <para>
+                Anonymous Python functions always run at the end
+                of parsing, regardless of where they are defined.
+                If a recipe contains many anonymous functions, they
+                run in the same order as they are defined within the
+                recipe.
+                As an example, consider the following snippet:
+                <literallayout class='monospaced'>
+     python () {
+         d.setVar('FOO', 'foo 2')
+     }
+
+     FOO = "foo 1"
+
+     python () {
+         d.appendVar('BAR', ' bar 2')
+     }
+
+     BAR = "bar 1"
+                </literallayout>
+                The previous example is conceptually equivalent to the
+                following snippet:
+                <literallayout class='monospaced'>
+     FOO = "foo 1"
+     BAR = "bar 1"
+     FOO = "foo 2"
+     BAR += "bar 2"
+                </literallayout>
+                <filename>FOO</filename> ends up with the value "foo 2",
+                and <filename>BAR</filename> with the value "bar 1 bar 2".
+                Just as in the second snippet, the values set for the
+                variables within the anonymous functions become available
+                to tasks, which always run after parsing.
+            </para>
+
+            <para>
+                Overrides and override-style operators such as
+                "<filename>_append</filename>" are applied before
+                anonymous functions run.
+                In the following example, <filename>FOO</filename> ends
+                up with the value "foo from anonymous":
+                <literallayout class='monospaced'>
+     FOO = "foo"
+     FOO_append = " from outside"
+
+     python () {
+         d.setVar("FOO", "foo from anonymous")
+     }
+                </literallayout>
+                For methods you can use with anonymous Python functions,
+                see the
+                "<link linkend='functions-you-can-call-from-within-python'>Functions You Can Call From Within Python</link>"
+                section.
+                For a different method to run Python code during parsing,
+                see the
+                "<link linkend='inline-python-variable-expansion'>Inline Python Variable Expansion</link>"
+                section.
             </para>
         </section>
 
@@ -1172,7 +1430,7 @@
 
             <para>
                 If you want dependencies such as these to remain intact, use
-                the <filename>noexec</filename> varflag to disable the task
+                the <filename>[noexec]</filename> varflag to disable the task
                 instead of using the <filename>deltask</filename> command to
                 delete it:
                 <literallayout class='monospaced'>
@@ -1295,10 +1553,13 @@
             Tasks support a number of these flags which control various
             functionality of the task:
             <itemizedlist>
-                <listitem><para><emphasis>cleandirs:</emphasis>
-                    Empty directories that should created before the task runs.
+                <listitem><para><emphasis><filename>[cleandirs]</filename>:</emphasis>
+                    Empty directories that should be created before the
+                    task runs.
+                    Directories that already exist are removed and recreated
+                    to empty them.
                     </para></listitem>
-                <listitem><para><emphasis>depends:</emphasis>
+                <listitem><para><emphasis><filename>[depends]</filename>:</emphasis>
                     Controls inter-task dependencies.
                     See the
                     <link linkend='var-DEPENDS'><filename>DEPENDS</filename></link>
@@ -1306,7 +1567,7 @@
                     "<link linkend='inter-task-dependencies'>Inter-Task Dependencies</link>"
                     section for more information.
                     </para></listitem>
-                <listitem><para><emphasis>deptask:</emphasis>
+                <listitem><para><emphasis><filename>[deptask]</filename>:</emphasis>
                     Controls task build-time dependencies.
                     See the
                     <link linkend='var-DEPENDS'><filename>DEPENDS</filename></link>
@@ -1314,12 +1575,13 @@
                     "<link linkend='build-dependencies'>Build Dependencies</link>"
                     section for more information.
                     </para></listitem>
-                <listitem><para><emphasis>dirs:</emphasis>
+                <listitem><para><emphasis><filename>[dirs]</filename>:</emphasis>
                     Directories that should be created before the task runs.
-                    The last directory listed will be used as the work directory
-                    for the task.
+                    Directories that already exist are left as is.
+                    The last directory listed is used as the
+                    current working directory for the task.
                     </para></listitem>
-                <listitem><para><emphasis>lockfiles:</emphasis>
+                <listitem><para><emphasis><filename>[lockfiles]</filename>:</emphasis>
                      Specifies one or more lockfiles to lock while the task
                      executes.
                      Only one task may hold a lockfile, and any task that
@@ -1328,23 +1590,23 @@
                      You can use this variable flag to accomplish mutual
                      exclusion.
                     </para></listitem>
-                <listitem><para><emphasis>noexec:</emphasis>
+                <listitem><para><emphasis><filename>[noexec]</filename>:</emphasis>
                     Marks the tasks as being empty and no execution required.
-                    The <filename>noexec</filename> flag can be used to set up
+                    The <filename>[noexec]</filename> flag can be used to set up
                     tasks as dependency placeholders, or to disable tasks defined
                     elsewhere that are not needed in a particular recipe.
                     </para></listitem>
-                <listitem><para><emphasis>nostamp:</emphasis>
+                <listitem><para><emphasis><filename>[nostamp]</filename>:</emphasis>
                     Tells BitBake to not generate a stamp file for a task,
                     which implies the task should always be executed.
                     </para></listitem>
-                <listitem><para><emphasis>postfuncs:</emphasis>
+                <listitem><para><emphasis><filename>[postfuncs]</filename>:</emphasis>
                     List of functions to call after the completion of the task.
                     </para></listitem>
-                <listitem><para><emphasis>prefuncs:</emphasis>
+                <listitem><para><emphasis><filename>[prefuncs]</filename>:</emphasis>
                     List of functions to call before the task executes.
                     </para></listitem>
-                <listitem><para><emphasis>rdepends:</emphasis>
+                <listitem><para><emphasis><filename>[rdepends]</filename>:</emphasis>
                     Controls inter-task runtime dependencies.
                     See the
                     <link linkend='var-RDEPENDS'><filename>RDEPENDS</filename></link>
@@ -1354,7 +1616,7 @@
                     "<link linkend='inter-task-dependencies'>Inter-Task Dependencies</link>"
                     section for more information.
                     </para></listitem>
-                <listitem><para><emphasis>rdeptask:</emphasis>
+                <listitem><para><emphasis><filename>[rdeptask]</filename>:</emphasis>
                     Controls task runtime dependencies.
                     See the
                     <link linkend='var-RDEPENDS'><filename>RDEPENDS</filename></link>
@@ -1364,12 +1626,12 @@
                     "<link linkend='runtime-dependencies'>Runtime Dependencies</link>"
                     section for more information.
                     </para></listitem>
-                <listitem><para><emphasis>recideptask:</emphasis>
+                <listitem><para><emphasis><filename>[recideptask]</filename>:</emphasis>
                     When set in conjunction with
                     <filename>recrdeptask</filename>, specifies a task that
                     should be inspected for additional dependencies.
                     </para></listitem>
-                <listitem><para><emphasis>recrdeptask:</emphasis>
+                <listitem><para><emphasis><filename>[recrdeptask]</filename>:</emphasis>
                     Controls task recursive runtime dependencies.
                     See the
                     <link linkend='var-RDEPENDS'><filename>RDEPENDS</filename></link>
@@ -1379,12 +1641,12 @@
                     "<link linkend='recursive-dependencies'>Recursive Dependencies</link>"
                     section for more information.
                     </para></listitem>
-                <listitem><para><emphasis>stamp-extra-info:</emphasis>
+                <listitem><para><emphasis><filename>[stamp-extra-info]</filename>:</emphasis>
                     Extra stamp information to append to the task's stamp.
                     As an example, OpenEmbedded uses this flag to allow
                     machine-specific tasks.
                     </para></listitem>
-                <listitem><para><emphasis>umask:</emphasis>
+                <listitem><para><emphasis><filename>[umask]</filename>:</emphasis>
                     The umask to run the task under.
                     </para></listitem>
             </itemizedlist>
@@ -1397,7 +1659,7 @@
             "<link linkend='checksums'>Checksums (Signatures)</link>"
             section.
             <itemizedlist>
-                <listitem><para><emphasis>vardeps:</emphasis>
+                <listitem><para><emphasis><filename>[vardeps]</filename>:</emphasis>
                     Specifies a space-separated list of additional
                     variables to add to a variable's dependencies
                     for the purposes of calculating its signature.
@@ -1406,17 +1668,17 @@
                     does not allow BitBake to automatically determine
                     that the variable is referred to.
                     </para></listitem>
-                <listitem><para><emphasis>vardepsexclude:</emphasis>
+                <listitem><para><emphasis><filename>[vardepsexclude]</filename>:</emphasis>
                     Specifies a space-separated list of variables
                     that should be excluded from a variable's dependencies
                     for the purposes of calculating its signature.
                     </para></listitem>
-                <listitem><para><emphasis>vardepvalue:</emphasis>
+                <listitem><para><emphasis><filename>[vardepvalue]</filename>:</emphasis>
                     If set, instructs BitBake to ignore the actual
                     value of the variable and instead use the specified
                     value when calculating the variable's signature.
                     </para></listitem>
-                <listitem><para><emphasis>vardepvalueexclude:</emphasis>
+                <listitem><para><emphasis><filename>[vardepvalueexclude]</filename>:</emphasis>
                     Specifies a pipe-separated list of strings to exclude
                     from the variable's value when calculating the
                     variable's signature.
@@ -1665,44 +1927,48 @@
                 <literallayout class='monospaced'>
      addtask printdate after do_fetch before do_build
                 </literallayout>
-                In this example, the <filename>printdate</filename> task is
-                depends on the completion of the <filename>do_fetch</filename>
+                In this example, the <filename>do_printdate</filename>
+                task depends on the completion of the
+                <filename>do_fetch</filename> task, and the
+                <filename>do_build</filename> task depends on the
+                completion of the <filename>do_printdate</filename>
                 task.
-                And, the <filename>do_build</filename> depends on the completion
-                of the <filename>printdate</filename> task.
-                <note>
-                    Recipes are built by having their
-                    <filename>do_build</filename> (not to be confused with
-                    <filename>do_compile</filename>) tasks executed.
-                    For a task to run when a recipe is built, the task must
-                    therefore be a direct or indirect dependency of
-                    <filename>do_build</filename>.
-                    For illustration, here are some examples:
+                <note><para>
+                    For a task to run, it must be a direct or indirect
+                    dependency of some other task that is scheduled to
+                    run.</para>
+
+                    <para>For illustration, here are some examples:
                     <itemizedlist>
                         <listitem><para>
                             The directive
-                            <filename>addtask mytask before do_build</filename>
-                            causes <filename>mytask</filename> to run when the
-                            recipe is built.
-                            In this example, <filename>mytask</filename> is run
-                            at an unspecified time relative to other tasks within
-                            the recipe, since <filename>after</filename> is not used.
+                            <filename>addtask mytask before do_configure</filename>
+                            causes <filename>do_mytask</filename> to run before
+                            <filename>do_configure</filename> runs.
+                            Be aware that <filename>do_mytask</filename> still only
+                            runs if its <link linkend='checksums'>input checksum</link>
+                            has changed since the last time it was run.
+                            Changes to the input checksum of
+                            <filename>do_mytask</filename> also indirectly cause
+                            <filename>do_configure</filename> to run.
                             </para></listitem>
                         <listitem><para>
                             The directive
                             <filename>addtask mytask after do_configure</filename>
-                            by itself does not cause <filename>mytask</filename>
-                            to run when the recipe is built.
-                            The task can still be run manually using the following:
+                            by itself never causes <filename>do_mytask</filename>
+                            to run.
+                            <filename>do_mytask</filename> can still be run manually
+                            as follows:
                             <literallayout class='monospaced'>
      $ bitbake <replaceable>recipe</replaceable> -c mytask
                             </literallayout>
-                            <filename>mytask</filename> could also be declared as
-                            a dependency of some other task.
-                            Regardless, the task is run after
+                            Declaring <filename>do_mytask</filename> as a dependency
+                            of some other task that is scheduled to run also causes
+                            it to run.
+                            Regardless, the task runs after
                             <filename>do_configure</filename>.
                             </para></listitem>
-                    </itemizedlist>
+                    </itemizedlist></para>
                 </note>
             </para>
         </section>
@@ -1714,7 +1980,8 @@
                 BitBake uses the
                 <link linkend='var-DEPENDS'><filename>DEPENDS</filename></link>
                 variable to manage build time dependencies.
-                The "deptask" varflag for tasks signifies the task of each
+                The <filename>[deptask]</filename> varflag for tasks
+                signifies the task of each
                 item listed in <filename>DEPENDS</filename> that must
                 complete before that task can be executed.
                 Here is an example:
@@ -1743,7 +2010,8 @@
                 packages.
                 Each of those packages can have <filename>RDEPENDS</filename> and
                 <filename>RRECOMMENDS</filename> runtime dependencies.
-                The "rdeptask" flag for tasks is used to signify the task of each
+                The <filename>[rdeptask]</filename> flag for tasks is used to
+                signify the task of each
                 item runtime dependency which must have completed before that
                 task can be executed.
                 <literallayout class='monospaced'>
@@ -1759,7 +2027,7 @@
             <title>Recursive Dependencies</title>
 
             <para>
-                BitBake uses the "recrdeptask" flag to manage
+                BitBake uses the <filename>[recrdeptask]</filename> flag to manage
                 recursive task dependencies.
                 BitBake looks through the build-time and runtime
                 dependencies of the current recipe, looks through
@@ -1773,7 +2041,8 @@
             </para>
 
             <para>
-                The "recrdeptask" flag is most commonly used in high-level
+                The <filename>[recrdeptask]</filename> flag is most commonly
+                used in high-level
                 recipes that need to wait for some task to finish "globally".
                 For example, <filename>image.bbclass</filename> has the following:
                 <literallayout class='monospaced'>
@@ -1803,7 +2072,8 @@
             <title>Inter-Task Dependencies</title>
 
             <para>
-                BitBake uses the "depends" flag in a more generic form
+                BitBake uses the <filename>[depends]</filename>
+                flag in a more generic form
                 to manage inter-task dependencies.
                 This more generic form allows for inter-dependency
                 checks for specific tasks rather than checks for
@@ -1819,109 +2089,138 @@
             </para>
 
             <para>
-                The "rdepends" flag works in a similar way but takes targets
+                The <filename>[rdepends]</filename> flag works in a similar
+                way but takes targets
                 in the runtime namespace instead of the build-time dependency
                 namespace.
             </para>
         </section>
     </section>
 
-    <section id='accessing-datastore-variables-using-python'>
-        <title>Accessing Datastore Variables Using Python</title>
+    <section id='functions-you-can-call-from-within-python'>
+        <title>Functions You Can Call From Within Python</title>
 
         <para>
-            It is often necessary to access variables in the
-            BitBake datastore using Python functions.
-            The Bitbake datastore has an API that allows you this
-            access.
-            Here is a list of available operations:
+            BitBake provides many functions you can call from
+            within Python functions.
+            This section lists the most commonly used functions,
+            and mentions where to find others.
         </para>
 
-        <para>
-            <informaltable frame='none'>
-                <tgroup cols='2' align='left' colsep='1' rowsep='1'>
-                    <colspec colname='c1' colwidth='1*'/>
-                    <colspec colname='c2' colwidth='1*'/>
-                    <thead>
-                        <row>
-                            <entry align="left"><emphasis>Operation</emphasis></entry>
-                            <entry align="left"><emphasis>Description</emphasis></entry>
-                        </row>
-                    </thead>
-                    <tbody>
-                        <row>
-                            <entry align="left"><filename>d.getVar("X", expand)</filename></entry>
-                            <entry align="left">Returns the value of variable "X".
-                            Using "expand=True" expands the value.</entry>
-                        </row>
-                        <row>
-                            <entry align="left"><filename>d.setVar("X", "value")</filename></entry>
-                            <entry align="left">Sets the variable "X" to "value".</entry>
-                        </row>
-                        <row>
-                            <entry align="left"><filename>d.appendVar("X", "value")</filename></entry>
-                            <entry align="left">Adds "value" to the end of the variable "X".</entry>
-                        </row>
-                        <row>
-                            <entry align="left"><filename>d.prependVar("X", "value")</filename></entry>
-                            <entry align="left">Adds "value" to the start of the variable "X".</entry>
-                        </row>
-                        <row>
-                            <entry align="left"><filename>d.delVar("X")</filename></entry>
-                            <entry align="left">Deletes the variable "X" from the datastore.</entry>
-                        </row>
-                        <row>
-                            <entry align="left"><filename>d.renameVar("X", "Y")</filename></entry>
-                            <entry align="left">Renames the variable "X" to "Y".</entry>
-                        </row>
-                        <row>
-                            <entry align="left"><filename>d.getVarFlag("X", flag, expand)</filename></entry>
-                            <entry align="left">Returns the value of variable "X".
+        <section id='functions-for-accessing-datastore-variables'>
+            <title>Functions for Accessing Datastore Variables</title>
+
+            <para>
+                It is often necessary to access variables in the
+                BitBake datastore using Python functions.
+                The Bitbake datastore has an API that allows you this
+                access.
+                Here is a list of available operations:
+            </para>
+
+            <para>
+                <informaltable frame='none'>
+                    <tgroup cols='2' align='left' colsep='1' rowsep='1'>
+                        <colspec colname='c1' colwidth='1*'/>
+                        <colspec colname='c2' colwidth='1*'/>
+                        <thead>
+                            <row>
+                                <entry align="left"><emphasis>Operation</emphasis></entry>
+                                <entry align="left"><emphasis>Description</emphasis></entry>
+                            </row>
+                        </thead>
+                        <tbody>
+                            <row>
+                                <entry align="left"><filename>d.getVar("X", expand)</filename></entry>
+                                <entry align="left">Returns the value of variable "X".
                                 Using "expand=True" expands the value.</entry>
-                        </row>
-                        <row>
-                            <entry align="left"><filename>d.setVarFlag("X", flag, "value")</filename></entry>
-                            <entry align="left">Sets the named flag for variable "X" to "value".</entry>
-                        </row>
-                        <row>
-                            <entry align="left"><filename>d.appendVarFlag("X", flag, "value")</filename></entry>
-                            <entry align="left">Appends "value" to the named flag on the
-                            variable "X".</entry>
-                        </row>
-                        <row>
-                            <entry align="left"><filename>d.prependVarFlag("X", flag, "value")</filename></entry>
-                            <entry align="left">Prepends "value" to the named flag on
-                               the variable "X".</entry>
-                        </row>
-                        <row>
-                            <entry align="left"><filename>d.delVarFlag("X", flag)</filename></entry>
-                            <entry align="left">Deletes the named flag on the variable
-                                "X" from the datastore.</entry>
-                        </row>
-                        <row>
-                            <entry align="left"><filename>d.setVarFlags("X", flagsdict)</filename></entry>
-                            <entry align="left">Sets the flags specified in
-                                the <filename>flagsdict()</filename> parameter.
-                                <filename>setVarFlags</filename> does not clear previous flags.
-                                Think of this operation as <filename>addVarFlags</filename>.</entry>
-                        </row>
-                        <row>
-                            <entry align="left"><filename>d.getVarFlags("X")</filename></entry>
-                            <entry align="left">Returns a <filename>flagsdict</filename> of the flags for
-                                the variable "X".</entry>
-                        </row>
-                        <row>
-                            <entry align="left"><filename>d.delVarFlags("X")</filename></entry>
-                            <entry align="left">Deletes all the flags for the variable "X".</entry>
-                        </row>
-                        <row>
-                            <entry align="left"><filename>d.expand(expression)</filename></entry>
-                            <entry align="left">Expands variable references in the specified string expression.</entry>
-                        </row>
-                    </tbody>
-                </tgroup>
-            </informaltable>
-        </para>
+                            </row>
+                            <row>
+                                <entry align="left"><filename>d.setVar("X", "value")</filename></entry>
+                                <entry align="left">Sets the variable "X" to "value".</entry>
+                            </row>
+                            <row>
+                                <entry align="left"><filename>d.appendVar("X", "value")</filename></entry>
+                                <entry align="left">Adds "value" to the end of the variable "X".</entry>
+                            </row>
+                            <row>
+                                <entry align="left"><filename>d.prependVar("X", "value")</filename></entry>
+                                <entry align="left">Adds "value" to the start of the variable "X".</entry>
+                            </row>
+                            <row>
+                                <entry align="left"><filename>d.delVar("X")</filename></entry>
+                                <entry align="left">Deletes the variable "X" from the datastore.</entry>
+                            </row>
+                            <row>
+                                <entry align="left"><filename>d.renameVar("X", "Y")</filename></entry>
+                                <entry align="left">Renames the variable "X" to "Y".</entry>
+                            </row>
+                            <row>
+                                <entry align="left"><filename>d.getVarFlag("X", flag, expand)</filename></entry>
+                                <entry align="left">Returns the value of variable "X".
+                                    Using "expand=True" expands the value.</entry>
+                            </row>
+                            <row>
+                                <entry align="left"><filename>d.setVarFlag("X", flag, "value")</filename></entry>
+                                <entry align="left">Sets the named flag for variable "X" to "value".</entry>
+                            </row>
+                            <row>
+                                <entry align="left"><filename>d.appendVarFlag("X", flag, "value")</filename></entry>
+                                <entry align="left">Appends "value" to the named flag on the
+                                variable "X".</entry>
+                            </row>
+                            <row>
+                                <entry align="left"><filename>d.prependVarFlag("X", flag, "value")</filename></entry>
+                                <entry align="left">Prepends "value" to the named flag on
+                                   the variable "X".</entry>
+                            </row>
+                            <row>
+                                <entry align="left"><filename>d.delVarFlag("X", flag)</filename></entry>
+                                <entry align="left">Deletes the named flag on the variable
+                                    "X" from the datastore.</entry>
+                            </row>
+                            <row>
+                                <entry align="left"><filename>d.setVarFlags("X", flagsdict)</filename></entry>
+                                <entry align="left">Sets the flags specified in
+                                    the <filename>flagsdict()</filename> parameter.
+                                    <filename>setVarFlags</filename> does not clear previous flags.
+                                    Think of this operation as <filename>addVarFlags</filename>.</entry>
+                            </row>
+                            <row>
+                                <entry align="left"><filename>d.getVarFlags("X")</filename></entry>
+                                <entry align="left">Returns a <filename>flagsdict</filename> of the flags for
+                                    the variable "X".</entry>
+                            </row>
+                            <row>
+                                <entry align="left"><filename>d.delVarFlags("X")</filename></entry>
+                                <entry align="left">Deletes all the flags for the variable "X".</entry>
+                            </row>
+                            <row>
+                                <entry align="left"><filename>d.expand(expression)</filename></entry>
+                                <entry align="left">Expands variable references in the specified string expression.</entry>
+                            </row>
+                        </tbody>
+                    </tgroup>
+                </informaltable>
+            </para>
+        </section>
+
+        <section id='other-functions'>
+            <title>Other Functions</title>
+
+            <para>
+                You can find many other functions that can be called
+                from Python by looking at the source code of the
+                <filename>bb</filename> module, which is in
+                <filename>bitbake/lib/bb</filename>.
+                For example,
+                <filename>bitbake/lib/bb/utils.py</filename> includes
+                the commonly used functions
+                <filename>bb.utils.contains()</filename> and
+                <filename>bb.utils.mkdirhier()</filename>, which come
+                with docstrings.
+            </para>
+        </section>
     </section>
 
     <section id='task-checksums-and-setscene'>

bitbake/doc/bitbake-user-manual/bitbake-user-manual-ref-variables.xml

@@ -1976,6 +1976,27 @@
                     The <filename>PROVIDES</filename> statement results in
                     the "libav" recipe also being known as "libpostproc".
                 </para>
+
+                <para>
+                    In addition to providing recipes under alternate names,
+                    the <filename>PROVIDES</filename> mechanism is also used
+                    to implement virtual targets.
+                    A virtual target is a name that corresponds to some
+                    particular functionality (e.g. a Linux kernel).
+                    Recipes that provide the functionality in question list the
+                    virtual target in <filename>PROVIDES</filename>.
+                    Recipes that depend on the functionality in question can
+                    include the virtual target in
+                    <link linkend='var-DEPENDS'><filename>DEPENDS</filename></link>
+                    to leave the choice of provider open.
+                </para>
+
+                <para>
+                    Conventionally, virtual targets have names on the form
+                    "virtual/function" (e.g. "virtual/kernel").
+                    The slash is simply part of the name and has no
+                    syntactical significance.
+                </para>
             </glossdef>
         </glossentry>
 

bitbake/lib/bb/__init__.py

@@ -21,7 +21,7 @@
 # with this program; if not, write to the Free Software Foundation, Inc.,
 # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
 
-__version__ = "1.31.0"
+__version__ = "1.31.1"
 
 import sys
 if sys.version_info < (3, 4, 0):

bitbake/lib/bb/build.py

@@ -193,12 +193,6 @@ def exec_func(func, d, dirs = None, pythonexception=False):
     except:
         oldcwd = None
 
-    body = d.getVar(func, False)
-    if not body:
-        if body is None:
-            logger.warning("Function %s doesn't exist", func)
-        return
-
     flags = d.getVarFlags(func)
     cleandirs = flags.get('cleandirs')
     if cleandirs:
@@ -217,6 +211,13 @@ def exec_func(func, d, dirs = None, pythonexception=False):
         adir = dirs[-1]
     else:
         adir = None
+
+    body = d.getVar(func, False)
+    if not body:
+        if body is None:
+            logger.warning("Function %s doesn't exist", func)
+        return
+
     ispython = flags.get('python')
 
     lockflag = flags.get('lockfiles')
@@ -385,39 +386,44 @@ exit $ret
         else:
             bb.warn('%s: invalid task progress varflag value "%s", ignoring' % (func, progress))
 
+    fifobuffer = bytearray()
     def readfifo(data):
-        lines = data.split(b'\0')
-        for line in lines:
-            # Just skip empty commands
-            if not line:
-                continue
-            splitval = line.split(b' ', 1)
-            cmd = splitval[0].decode("utf-8")
-            if len(splitval) > 1:
-                value = splitval[1].decode("utf-8")
+        nonlocal fifobuffer
+        fifobuffer.extend(data)
+        while fifobuffer:
+            message, token, nextmsg = fifobuffer.partition(b"\00")
+            if token:
+                splitval = message.split(b' ', 1)
+                cmd = splitval[0].decode("utf-8")
+                if len(splitval) > 1:
+                    value = splitval[1].decode("utf-8")
+                else:
+                    value = ''
+                if cmd == 'bbplain':
+                    bb.plain(value)
+                elif cmd == 'bbnote':
+                    bb.note(value)
+                elif cmd == 'bbwarn':
+                    bb.warn(value)
+                elif cmd == 'bberror':
+                    bb.error(value)
+                elif cmd == 'bbfatal':
+                    # The caller will call exit themselves, so bb.error() is
+                    # what we want here rather than bb.fatal()
+                    bb.error(value)
+                elif cmd == 'bbfatal_log':
+                    bb.error(value, forcelog=True)
+                elif cmd == 'bbdebug':
+                    splitval = value.split(' ', 1)
+                    level = int(splitval[0])
+                    value = splitval[1]
+                    bb.debug(level, value)
+                else:
+                    bb.warn("Unrecognised command '%s' on FIFO" % cmd)
+                fifobuffer = nextmsg
             else:
-                value = ''
-            if cmd == 'bbplain':
-                bb.plain(value)
-            elif cmd == 'bbnote':
-                bb.note(value)
-            elif cmd == 'bbwarn':
-                bb.warn(value)
-            elif cmd == 'bberror':
-                bb.error(value)
-            elif cmd == 'bbfatal':
-                # The caller will call exit themselves, so bb.error() is
-                # what we want here rather than bb.fatal()
-                bb.error(value)
-            elif cmd == 'bbfatal_log':
-                bb.error(value, forcelog=True)
-            elif cmd == 'bbdebug':
-                splitval = value.split(' ', 1)
-                level = int(splitval[0])
-                value = splitval[1]
-                bb.debug(level, value)
-            else:
-                bb.warn("Unrecognised command '%s' on FIFO" % cmd)
+                break
+
     tempdir = d.getVar('T', True)
     fifopath = os.path.join(tempdir, 'fifo.%s' % os.getpid())
     if os.path.exists(fifopath):
@@ -569,6 +575,9 @@ def _exec_task(fn, task, d, quieterr):
             logger.error(str(exc))
             event.fire(TaskFailed(task, logfn, localdata, errprinted), localdata)
         return 1
+    except bb.BBHandledException:
+        event.fire(TaskFailed(task, logfn, localdata, True), localdata)
+        return 1
     finally:
         sys.stdout.flush()
         sys.stderr.flush()
@@ -628,7 +637,7 @@ def exec_task(fn, task, d, profile = False):
             event.fire(failedevent, d)
         return 1
 
-def stamp_internal(taskname, d, file_name, baseonly=False):
+def stamp_internal(taskname, d, file_name, baseonly=False, noextra=False):
     """
     Internal stamp helper function
     Makes sure the stamp directory exists
@@ -651,6 +660,8 @@ def stamp_internal(taskname, d, file_name, baseonly=False):
 
     if baseonly:
         return stamp
+    if noextra:
+        extrainfo = ""
 
     if not stamp:
         return
@@ -746,12 +757,12 @@ def write_taint(task, d, file_name = None):
     with open(taintfn, 'w') as taintf:
         taintf.write(str(uuid.uuid4()))
 
-def stampfile(taskname, d, file_name = None):
+def stampfile(taskname, d, file_name = None, noextra=False):
     """
     Return the stamp for a given task
     (d can be a data dict or dataCache)
     """
-    return stamp_internal(taskname, d, file_name)
+    return stamp_internal(taskname, d, file_name, noextra=noextra)
 
 def add_tasks(tasklist, d):
     task_deps = d.getVar('_task_deps', False)

bitbake/lib/bb/cache.py

@@ -244,14 +244,136 @@ class CoreRecipeInfo(RecipeInfoCommon):
         cachedata.fakerootdirs[fn] = self.fakerootdirs
         cachedata.extradepsfunc[fn] = self.extradepsfunc
 
+def virtualfn2realfn(virtualfn):
+    """
+    Convert a virtual file name to a real one + the associated subclass keyword
+    """
+    mc = ""
+    if virtualfn.startswith('multiconfig:'):
+        elems = virtualfn.split(':')
+        mc = elems[1]
+        virtualfn = ":".join(elems[2:])
+
+    fn = virtualfn
+    cls = ""
+    if virtualfn.startswith('virtual:'):
+        elems = virtualfn.split(':')
+        cls = ":".join(elems[1:-1])
+        fn = elems[-1]
+
+    return (fn, cls, mc)
+
+def realfn2virtual(realfn, cls, mc):
+    """
+    Convert a real filename + the associated subclass keyword to a virtual filename
+    """
+    if cls:
+        realfn = "virtual:" + cls + ":" + realfn
+    if mc:
+        realfn = "multiconfig:" + mc + ":" + realfn
+    return realfn
+
+def variant2virtual(realfn, variant):
+    """
+    Convert a real filename + the associated subclass keyword to a virtual filename
+    """
+    if variant == "":
+        return realfn
+    if variant.startswith("multiconfig:"):
+        elems = variant.split(":")
+        if elems[2]:
+            return "multiconfig:" + elems[1] + ":virtual:" + ":".join(elems[2:]) + ":" + realfn
+        return "multiconfig:" + elems[1] + ":" + realfn
+    return "virtual:" + variant + ":" + realfn
+
+def parse_recipe(bb_data, bbfile, appends, mc=''):
+    """
+    Parse a recipe
+    """
+
+    chdir_back = False
+
+    bb_data.setVar("__BBMULTICONFIG", mc)
+
+    # expand tmpdir to include this topdir
+    bb_data.setVar('TMPDIR', bb_data.getVar('TMPDIR', True) or "")
+    bbfile_loc = os.path.abspath(os.path.dirname(bbfile))
+    oldpath = os.path.abspath(os.getcwd())
+    bb.parse.cached_mtime_noerror(bbfile_loc)
+
+    # The ConfHandler first looks if there is a TOPDIR and if not
+    # then it would call getcwd().
+    # Previously, we chdir()ed to bbfile_loc, called the handler
+    # and finally chdir()ed back, a couple of thousand times. We now
+    # just fill in TOPDIR to point to bbfile_loc if there is no TOPDIR yet.
+    if not bb_data.getVar('TOPDIR', False):
+        chdir_back = True
+        bb_data.setVar('TOPDIR', bbfile_loc)
+    try:
+        if appends:
+            bb_data.setVar('__BBAPPEND', " ".join(appends))
+        bb_data = bb.parse.handle(bbfile, bb_data)
+        if chdir_back:
+            os.chdir(oldpath)
+        return bb_data
+    except:
+        if chdir_back:
+            os.chdir(oldpath)
+        raise
 
 
-class Cache(object):
+
+class NoCache(object):
+
+    def __init__(self, databuilder):
+        self.databuilder = databuilder
+        self.data = databuilder.data
+
+    def loadDataFull(self, virtualfn, appends):
+        """
+        Return a complete set of data for fn.
+        To do this, we need to parse the file.
+        """
+        logger.debug(1, "Parsing %s (full)" % virtualfn)
+        (fn, virtual, mc) = virtualfn2realfn(virtualfn)
+        bb_data = self.load_bbfile(virtualfn, appends, virtonly=True)
+        return bb_data[virtual]
+
+    def load_bbfile(self, bbfile, appends, virtonly = False):
+        """
+        Load and parse one .bb build file
+        Return the data and whether parsing resulted in the file being skipped
+        """
+
+        if virtonly:
+            (bbfile, virtual, mc) = virtualfn2realfn(bbfile)
+            bb_data = self.databuilder.mcdata[mc].createCopy()
+            bb_data.setVar("__ONLYFINALISE", virtual or "default")
+            datastores = parse_recipe(bb_data, bbfile, appends, mc)
+            return datastores
+
+        bb_data = self.data.createCopy()
+        datastores = parse_recipe(bb_data, bbfile, appends)
+
+        for mc in self.databuilder.mcdata:
+            if not mc:
+                continue
+            bb_data = self.databuilder.mcdata[mc].createCopy()
+            newstores = parse_recipe(bb_data, bbfile, appends, mc)
+            for ns in newstores:
+                datastores["multiconfig:%s:%s" % (mc, ns)] = newstores[ns]
+
+        return datastores
+
+class Cache(NoCache):
     """
     BitBake Cache implementation
     """
 
-    def __init__(self, data, data_hash, caches_array):
+    def __init__(self, databuilder, data_hash, caches_array):
+        super().__init__(databuilder)
+        data = databuilder.data
+
         # Pass caches_array information into Cache Constructor
         # It will be used later for deciding whether we 
         # need extra cache file dump/load support 
@@ -260,7 +382,6 @@ class Cache(object):
         self.clean = set()
         self.checked = set()
         self.depends_cache = {}
-        self.data = None
         self.data_fn = None
         self.cacheclean = True
         self.data_hash = data_hash
@@ -280,72 +401,74 @@ class Cache(object):
         cache_ok = True
         if self.caches_array:
             for cache_class in self.caches_array:
-                if type(cache_class) is type and issubclass(cache_class, RecipeInfoCommon):
-                    cachefile = getCacheFile(self.cachedir, cache_class.cachefile, self.data_hash)
-                    cache_ok = cache_ok and os.path.exists(cachefile)
-                    cache_class.init_cacheData(self)
+                cachefile = getCacheFile(self.cachedir, cache_class.cachefile, self.data_hash)
+                cache_ok = cache_ok and os.path.exists(cachefile)
+                cache_class.init_cacheData(self)
         if cache_ok:
             self.load_cachefile()
         elif os.path.isfile(self.cachefile):
             logger.info("Out of date cache found, rebuilding...")
 
     def load_cachefile(self):
-        # Firstly, using core cache file information for
-        # valid checking
-        with open(self.cachefile, "rb") as cachefile:
-            pickled = pickle.Unpickler(cachefile)
-            try:
-                cache_ver = pickled.load()
-                bitbake_ver = pickled.load()
-            except Exception:
-                logger.info('Invalid cache, rebuilding...')
-                return
-
-            if cache_ver != __cache_version__:
-                logger.info('Cache version mismatch, rebuilding...')
-                return
-            elif bitbake_ver != bb.__version__:
-                logger.info('Bitbake version mismatch, rebuilding...')
-                return
-
-
         cachesize = 0
         previous_progress = 0
         previous_percent = 0
 
         # Calculate the correct cachesize of all those cache files
         for cache_class in self.caches_array:
-            if type(cache_class) is type and issubclass(cache_class, RecipeInfoCommon):
-                cachefile = getCacheFile(self.cachedir, cache_class.cachefile, self.data_hash)
-                with open(cachefile, "rb") as cachefile:
-                    cachesize += os.fstat(cachefile.fileno()).st_size
+            cachefile = getCacheFile(self.cachedir, cache_class.cachefile, self.data_hash)
+            with open(cachefile, "rb") as cachefile:
+                cachesize += os.fstat(cachefile.fileno()).st_size
 
         bb.event.fire(bb.event.CacheLoadStarted(cachesize), self.data)
         
         for cache_class in self.caches_array:
-            if type(cache_class) is type and issubclass(cache_class, RecipeInfoCommon):
-                cachefile = getCacheFile(self.cachedir, cache_class.cachefile, self.data_hash)
-                with open(cachefile, "rb") as cachefile:
-                    pickled = pickle.Unpickler(cachefile)                    
-                    while cachefile:
-                        try:
-                            key = pickled.load()
-                            value = pickled.load()
-                        except Exception:
-                            break
-                        if key in self.depends_cache:
-                            self.depends_cache[key].append(value)
-                        else:
-                            self.depends_cache[key] = [value]
-                        # only fire events on even percentage boundaries
-                        current_progress = cachefile.tell() + previous_progress
-                        current_percent = 100 * current_progress / cachesize
-                        if current_percent > previous_percent:
-                            previous_percent = current_percent
-                            bb.event.fire(bb.event.CacheLoadProgress(current_progress, cachesize),
-                                          self.data)
+            cachefile = getCacheFile(self.cachedir, cache_class.cachefile, self.data_hash)
+            with open(cachefile, "rb") as cachefile:
+                pickled = pickle.Unpickler(cachefile)
+                # Check cache version information
+                try:
+                    cache_ver = pickled.load()
+                    bitbake_ver = pickled.load()
+                except Exception:
+                    logger.info('Invalid cache, rebuilding...')
+                    return
 
-                    previous_progress += current_progress
+                if cache_ver != __cache_version__:
+                    logger.info('Cache version mismatch, rebuilding...')
+                    return
+                elif bitbake_ver != bb.__version__:
+                     logger.info('Bitbake version mismatch, rebuilding...')
+                     return
+
+                # Load the rest of the cache file
+                current_progress = 0
+                while cachefile:
+                    try:
+                        key = pickled.load()
+                        value = pickled.load()
+                    except Exception:
+                        break
+                    if not isinstance(key, str):
+                        bb.warn("%s from extras cache is not a string?" % key)
+                        break
+                    if not isinstance(value, RecipeInfoCommon):
+                        bb.warn("%s from extras cache is not a RecipeInfoCommon class?" % value)
+                        break
+
+                    if key in self.depends_cache:
+                        self.depends_cache[key].append(value)
+                    else:
+                        self.depends_cache[key] = [value]
+                    # only fire events on even percentage boundaries
+                    current_progress = cachefile.tell() + previous_progress
+                    current_percent = 100 * current_progress / cachesize
+                    if current_percent > previous_percent:
+                        previous_percent = current_percent
+                        bb.event.fire(bb.event.CacheLoadProgress(current_progress, cachesize),
+                                      self.data)
+
+                previous_progress += current_progress
 
         # Note: depends cache number is corresponding to the parsing file numbers.
         # The same file has several caches, still regarded as one item in the cache
@@ -353,69 +476,33 @@ class Cache(object):
                                                   len(self.depends_cache)),
                       self.data)
 
-    
-    @staticmethod
-    def virtualfn2realfn(virtualfn):
-        """
-        Convert a virtual file name to a real one + the associated subclass keyword
-        """
-
-        fn = virtualfn
-        cls = ""
-        if virtualfn.startswith('virtual:'):
-            elems = virtualfn.split(':')
-            cls = ":".join(elems[1:-1])
-            fn = elems[-1]
-        return (fn, cls)
-
-    @staticmethod
-    def realfn2virtual(realfn, cls):
-        """
-        Convert a real filename + the associated subclass keyword to a virtual filename
-        """
-        if cls == "":
-            return realfn
-        return "virtual:" + cls + ":" + realfn
-
-    @classmethod
-    def loadDataFull(cls, virtualfn, appends, cfgData):
-        """
-        Return a complete set of data for fn.
-        To do this, we need to parse the file.
-        """
-
-        (fn, virtual) = cls.virtualfn2realfn(virtualfn)
-
-        logger.debug(1, "Parsing %s (full)", fn)
-
-        cfgData.setVar("__ONLYFINALISE", virtual or "default")
-        bb_data = cls.load_bbfile(fn, appends, cfgData)
-        return bb_data[virtual]
-
-    @classmethod
-    def parse(cls, filename, appends, configdata, caches_array):
+    def parse(self, filename, appends):
         """Parse the specified filename, returning the recipe information"""
+        logger.debug(1, "Parsing %s", filename)
         infos = []
-        datastores = cls.load_bbfile(filename, appends, configdata)
+        datastores = self.load_bbfile(filename, appends)
         depends = []
+        variants = []
+        # Process the "real" fn last so we can store variants list
         for variant, data in sorted(datastores.items(),
                                     key=lambda i: i[0],
                                     reverse=True):
-            virtualfn = cls.realfn2virtual(filename, variant)
+            virtualfn = variant2virtual(filename, variant)
+            variants.append(variant)
             depends = depends + (data.getVar("__depends", False) or [])
             if depends and not variant:
                 data.setVar("__depends", depends)
-
+            if virtualfn == filename:
+                data.setVar("__VARIANTS", " ".join(variants))
             info_array = []
-            for cache_class in caches_array:
-                if type(cache_class) is type and issubclass(cache_class, RecipeInfoCommon):
-                    info = cache_class(filename, data)
-                    info_array.append(info)
+            for cache_class in self.caches_array:
+                info = cache_class(filename, data)
+                info_array.append(info)
             infos.append((virtualfn, info_array))
 
         return infos
 
-    def load(self, filename, appends, configdata):
+    def load(self, filename, appends):
         """Obtain the recipe information for the specified filename,
         using cached values if available, otherwise parsing.
 
@@ -429,21 +516,20 @@ class Cache(object):
             # info_array item is a list of [CoreRecipeInfo, XXXRecipeInfo]
             info_array = self.depends_cache[filename]
             for variant in info_array[0].variants:
-                virtualfn = self.realfn2virtual(filename, variant)
+                virtualfn = variant2virtual(filename, variant)
                 infos.append((virtualfn, self.depends_cache[virtualfn]))
         else:
-            logger.debug(1, "Parsing %s", filename)
             return self.parse(filename, appends, configdata, self.caches_array)
 
         return cached, infos
 
-    def loadData(self, fn, appends, cfgData, cacheData):
+    def loadData(self, fn, appends, cacheData):
         """Load the recipe info for the specified filename,
         parsing and adding to the cache if necessary, and adding
         the recipe information to the supplied CacheData instance."""
         skipped, virtuals = 0, 0
 
-        cached, infos = self.load(fn, appends, cfgData)
+        cached, infos = self.load(fn, appends)
         for virtualfn, info_array in infos:
             if info_array[0].skipped:
                 logger.debug(1, "Skipping %s: %s", virtualfn, info_array[0].skipreason)
@@ -551,16 +637,19 @@ class Cache(object):
 
         invalid = False
         for cls in info_array[0].variants:
-            virtualfn = self.realfn2virtual(fn, cls)
+            virtualfn = variant2virtual(fn, cls)
             self.clean.add(virtualfn)
             if virtualfn not in self.depends_cache:
                 logger.debug(2, "Cache: %s is not cached", virtualfn)
                 invalid = True
+            elif len(self.depends_cache[virtualfn]) != len(self.caches_array):
+                logger.debug(2, "Cache: Extra caches missing for %s?" % virtualfn)
+                invalid = True
 
         # If any one of the variants is not present, mark as invalid for all
         if invalid:
             for cls in info_array[0].variants:
-                virtualfn = self.realfn2virtual(fn, cls)
+                virtualfn = variant2virtual(fn, cls)
                 if virtualfn in self.clean:
                     logger.debug(2, "Cache: Removing %s from cache", virtualfn)
                     self.clean.remove(virtualfn)
@@ -597,30 +686,19 @@ class Cache(object):
             logger.debug(2, "Cache is clean, not saving.")
             return
 
-        file_dict = {}
-        pickler_dict = {}
         for cache_class in self.caches_array:
-            if type(cache_class) is type and issubclass(cache_class, RecipeInfoCommon):
-                cache_class_name = cache_class.__name__
-                cachefile = getCacheFile(self.cachedir, cache_class.cachefile, self.data_hash)
-                file_dict[cache_class_name] = open(cachefile, "wb")
-                pickler_dict[cache_class_name] =  pickle.Pickler(file_dict[cache_class_name], pickle.HIGHEST_PROTOCOL)
-                   
-        pickler_dict['CoreRecipeInfo'].dump(__cache_version__)
-        pickler_dict['CoreRecipeInfo'].dump(bb.__version__)
+            cache_class_name = cache_class.__name__
+            cachefile = getCacheFile(self.cachedir, cache_class.cachefile, self.data_hash)
+            with open(cachefile, "wb") as f:
+                p = pickle.Pickler(f, pickle.HIGHEST_PROTOCOL)
+                p.dump(__cache_version__)
+                p.dump(bb.__version__)
 
-        try:
-            for key, info_array in self.depends_cache.items():
-                for info in info_array:
-                    if isinstance(info, RecipeInfoCommon):
-                        cache_class_name = info.__class__.__name__
-                        pickler_dict[cache_class_name].dump(key)
-                        pickler_dict[cache_class_name].dump(info)
-        finally:
-            for cache_class in self.caches_array:
-                if type(cache_class) is type and issubclass(cache_class, RecipeInfoCommon):
-                    cache_class_name = cache_class.__name__
-                    file_dict[cache_class_name].close()
+                for key, info_array in self.depends_cache.items():
+                    for info in info_array:
+                        if isinstance(info, RecipeInfoCommon) and info.__class__.__name__ == cache_class_name:
+                            p.dump(key)
+                            p.dump(info)
 
         del self.depends_cache
 
@@ -648,50 +726,13 @@ class Cache(object):
         Save data we need into the cache
         """
 
-        realfn = self.virtualfn2realfn(file_name)[0]
+        realfn = virtualfn2realfn(file_name)[0]
 
         info_array = []
         for cache_class in self.caches_array:
-            if type(cache_class) is type and issubclass(cache_class, RecipeInfoCommon):
-                info_array.append(cache_class(realfn, data))
+            info_array.append(cache_class(realfn, data))
         self.add_info(file_name, info_array, cacheData, parsed)
 
-    @staticmethod
-    def load_bbfile(bbfile, appends, config):
-        """
-        Load and parse one .bb build file
-        Return the data and whether parsing resulted in the file being skipped
-        """
-        chdir_back = False
-
-        from bb import parse
-
-        # expand tmpdir to include this topdir
-        config.setVar('TMPDIR', config.getVar('TMPDIR', True) or "")
-        bbfile_loc = os.path.abspath(os.path.dirname(bbfile))
-        oldpath = os.path.abspath(os.getcwd())
-        parse.cached_mtime_noerror(bbfile_loc)
-        bb_data = config.createCopy()
-        # The ConfHandler first looks if there is a TOPDIR and if not
-        # then it would call getcwd().
-        # Previously, we chdir()ed to bbfile_loc, called the handler
-        # and finally chdir()ed back, a couple of thousand times. We now
-        # just fill in TOPDIR to point to bbfile_loc if there is no TOPDIR yet.
-        if not bb_data.getVar('TOPDIR', False):
-            chdir_back = True
-            bb_data.setVar('TOPDIR', bbfile_loc)
-        try:
-            if appends:
-                bb_data.setVar('__BBAPPEND', " ".join(appends))
-            bb_data = parse.handle(bbfile, bb_data)
-            if chdir_back:
-                os.chdir(oldpath)
-            return bb_data
-        except:
-            if chdir_back:
-                os.chdir(oldpath)
-            raise
-
 
 def init(cooker):
     """
@@ -721,8 +762,9 @@ class CacheData(object):
     def __init__(self, caches_array):
         self.caches_array = caches_array
         for cache_class in self.caches_array:
-            if type(cache_class) is type and issubclass(cache_class, RecipeInfoCommon):
-                cache_class.init_cacheData(self)        
+            if not issubclass(cache_class, RecipeInfoCommon):
+                bb.error("Extra cache data class %s should subclass RecipeInfoCommon class" % cache_class)
+            cache_class.init_cacheData(self)
 
         # Direct cache variables
         self.task_queues = {}

bitbake/lib/bb/checksum.py

@@ -120,13 +120,15 @@ class FileChecksumCache(MultiProcessCache):
                             checksums.extend(checksum_dir(f))
                     else:
                         checksum = checksum_file(f)
-                        checksums.append((f, checksum))
+                        if checksum:
+                            checksums.append((f, checksum))
             elif os.path.isdir(pth):
                 if not os.path.islink(pth):
                     checksums.extend(checksum_dir(pth))
             else:
                 checksum = checksum_file(pth)
-                checksums.append((pth, checksum))
+                if checksum:
+                    checksums.append((pth, checksum))
 
         checksums.sort(key=operator.itemgetter(1))
         return checksums

bitbake/lib/bb/cooker.py

@@ -30,7 +30,7 @@ import logging
 import multiprocessing
 import sre_constants
 import threading
-from io import StringIO
+from io import StringIO, UnsupportedOperation
 from contextlib import closing
 from functools import wraps
 from collections import defaultdict, namedtuple
@@ -141,7 +141,7 @@ class EventWriter:
         else:
             # init on bb.event.BuildStarted
             name = "%s.%s" % (event.__module__, event.__class__.__name__)
-            if name == "bb.event.BuildStarted":
+            if name in ("bb.event.BuildStarted", "bb.cooker.CookerExit"):
                 with open(self.eventfile, "w") as f:
                     f.write("%s\n" % json.dumps({ "allvariables" : self.cooker.getAllKeysWithFlags(["doc", "func"])}))
 
@@ -166,7 +166,7 @@ class BBCooker:
     """
 
     def __init__(self, configuration, featureSet=None):
-        self.recipecache = None
+        self.recipecaches = None
         self.skiplist = {}
         self.featureset = CookerFeatures()
         if featureSet:
@@ -230,14 +230,17 @@ class BBCooker:
             pass
 
         # TOSTOP must not be set or our children will hang when they output
-        fd = sys.stdout.fileno()
-        if os.isatty(fd):
-            import termios
-            tcattr = termios.tcgetattr(fd)
-            if tcattr[3] & termios.TOSTOP:
-                buildlog.info("The terminal had the TOSTOP bit set, clearing...")
-                tcattr[3] = tcattr[3] & ~termios.TOSTOP
-                termios.tcsetattr(fd, termios.TCSANOW, tcattr)
+        try:
+            fd = sys.stdout.fileno()
+            if os.isatty(fd):
+                import termios
+                tcattr = termios.tcgetattr(fd)
+                if tcattr[3] & termios.TOSTOP:
+                    buildlog.info("The terminal had the TOSTOP bit set, clearing...")
+                    tcattr[3] = tcattr[3] & ~termios.TOSTOP
+                    termios.tcsetattr(fd, termios.TCSANOW, tcattr)
+        except UnsupportedOperation:
+            pass
 
         self.command = bb.command.Command(self)
         self.state = state.initial
@@ -521,11 +524,14 @@ class BBCooker:
             nice = int(nice) - curnice
             buildlog.verbose("Renice to %s " % os.nice(nice))
 
-        if self.recipecache:
-            del self.recipecache
-        self.recipecache = bb.cache.CacheData(self.caches_array)
+        if self.recipecaches:
+            del self.recipecaches
+        self.multiconfigs = self.databuilder.mcdata.keys()
+        self.recipecaches = {}
+        for mc in self.multiconfigs:
+            self.recipecaches[mc] = bb.cache.CacheData(self.caches_array)
 
-        self.handleCollections( self.data.getVar("BBFILE_COLLECTIONS", True) )
+        self.handleCollections(self.data.getVar("BBFILE_COLLECTIONS", True))
 
     def updateConfigOpts(self, options, environment):
         clean = True
@@ -569,8 +575,8 @@ class BBCooker:
 
     def showVersions(self):
 
-        pkg_pn = self.recipecache.pkg_pn
-        (latest_versions, preferred_versions) = bb.providers.findProviders(self.data, self.recipecache, pkg_pn)
+        pkg_pn = self.recipecaches[''].pkg_pn
+        (latest_versions, preferred_versions) = bb.providers.findProviders(self.data, self.recipecaches[''], pkg_pn)
 
         logger.plain("%-35s %25s %25s", "Recipe Name", "Latest Version", "Preferred Version")
         logger.plain("%-35s %25s %25s\n", "===========", "==============", "=================")
@@ -601,23 +607,25 @@ class BBCooker:
             # this showEnvironment() code path doesn't use the cache
             self.parseConfiguration()
 
-            fn, cls = bb.cache.Cache.virtualfn2realfn(buildfile)
+            fn, cls, mc = bb.cache.virtualfn2realfn(buildfile)
             fn = self.matchFile(fn)
-            fn = bb.cache.Cache.realfn2virtual(fn, cls)
+            fn = bb.cache.realfn2virtual(fn, cls, mc)
         elif len(pkgs_to_build) == 1:
             ignore = self.expanded_data.getVar("ASSUME_PROVIDED", True) or ""
             if pkgs_to_build[0] in set(ignore.split()):
                 bb.fatal("%s is in ASSUME_PROVIDED" % pkgs_to_build[0])
 
-            taskdata, runlist, pkgs_to_build = self.buildTaskData(pkgs_to_build, None, self.configuration.abort, allowincomplete=True)
+            taskdata, runlist = self.buildTaskData(pkgs_to_build, None, self.configuration.abort, allowincomplete=True)
 
-            fn = taskdata.build_targets[pkgs_to_build[0]][0]
+            mc = runlist[0][0]
+            fn = runlist[0][3]
         else:
             envdata = self.data
 
         if fn:
             try:
-                envdata = bb.cache.Cache.loadDataFull(fn, self.collection.get_file_appends(fn), self.data)
+                bb_cache = bb.cache.Cache(self.databuilder, self.data_hash, self.caches_array)
+                envdata = bb_cache.loadDataFull(fn, self.collection.get_file_appends(fn))
             except Exception as e:
                 parselog.exception("Unable to read %s", fn)
                 raise
@@ -650,30 +658,44 @@ class BBCooker:
         if task is None:
             task = self.configuration.cmd
 
-        fulltargetlist = self.checkPackages(pkgs_to_build)
+        fulltargetlist = self.checkPackages(pkgs_to_build, task)
+        taskdata = {}
+        localdata = {}
 
-        localdata = data.createCopy(self.data)
-        bb.data.update_data(localdata)
-        bb.data.expandKeys(localdata)
-        taskdata = bb.taskdata.TaskData(abort, skiplist=self.skiplist, allowincomplete=allowincomplete)
+        for mc in self.multiconfigs:
+            taskdata[mc] = bb.taskdata.TaskData(abort, skiplist=self.skiplist, allowincomplete=allowincomplete)
+            localdata[mc] = data.createCopy(self.databuilder.mcdata[mc])
+            bb.data.update_data(localdata[mc])
+            bb.data.expandKeys(localdata[mc])
 
         current = 0
         runlist = []
         for k in fulltargetlist:
+            mc = ""
+            if k.startswith("multiconfig:"):
+                mc = k.split(":")[1]
+                k = ":".join(k.split(":")[2:])
             ktask = task
             if ":do_" in k:
                 k2 = k.split(":do_")
                 k = k2[0]
                 ktask = k2[1]
-            taskdata.add_provider(localdata, self.recipecache, k)
+            taskdata[mc].add_provider(localdata[mc], self.recipecaches[mc], k)
             current += 1
             if not ktask.startswith("do_"):
                 ktask = "do_%s" % ktask
-            runlist.append([k, ktask])
+            if k not in taskdata[mc].build_targets or not taskdata[mc].build_targets[k]:
+                # e.g. in ASSUME_PROVIDED
+                continue
+            fn = taskdata[mc].build_targets[k][0]
+            runlist.append([mc, k, ktask, fn])
             bb.event.fire(bb.event.TreeDataPreparationProgress(current, len(fulltargetlist)), self.data)
-        taskdata.add_unresolved(localdata, self.recipecache)
+
+        for mc in self.multiconfigs:
+            taskdata[mc].add_unresolved(localdata[mc], self.recipecaches[mc])
+
         bb.event.fire(bb.event.TreeDataPreparationCompleted(len(fulltargetlist)), self.data)
-        return taskdata, runlist, fulltargetlist
+        return taskdata, runlist
 
     def prepareTreeData(self, pkgs_to_build, task):
         """
@@ -682,7 +704,7 @@ class BBCooker:
 
         # We set abort to False here to prevent unbuildable targets raising
         # an exception when we're just generating data
-        taskdata, runlist, pkgs_to_build = self.buildTaskData(pkgs_to_build, task, False, allowincomplete=True)
+        taskdata, runlist = self.buildTaskData(pkgs_to_build, task, False, allowincomplete=True)
 
         return runlist, taskdata
 
@@ -694,10 +716,15 @@ class BBCooker:
         information.
         """
         runlist, taskdata = self.prepareTreeData(pkgs_to_build, task)
-        rq = bb.runqueue.RunQueue(self, self.data, self.recipecache, taskdata, runlist)
+        rq = bb.runqueue.RunQueue(self, self.data, self.recipecaches, taskdata, runlist)
         rq.rqdata.prepare()
         return self.buildDependTree(rq, taskdata)
 
+    @staticmethod
+    def add_mc_prefix(mc, pn):
+        if mc:
+            return "multiconfig:%s.%s" % (mc, pn)
+        return pn
 
     def buildDependTree(self, rq, taskdata):
         seen_fns = []
@@ -710,24 +737,27 @@ class BBCooker:
         depend_tree["rdepends-pkg"] = {}
         depend_tree["rrecs-pkg"] = {}
         depend_tree['providermap'] = {}
-        depend_tree["layer-priorities"] = self.recipecache.bbfile_config_priorities
+        depend_tree["layer-priorities"] = self.bbfile_config_priorities
 
-        for name, fn in list(taskdata.get_providermap().items()):
-            pn = self.recipecache.pkg_fn[fn]
-            if name != pn:
-                version = "%s:%s-%s" % self.recipecache.pkg_pepvpr[fn]
-                depend_tree['providermap'][name] = (pn, version)
+        for mc in taskdata:
+            for name, fn in list(taskdata[mc].get_providermap().items()):
+                pn = self.recipecaches[mc].pkg_fn[fn]
+                pn = self.add_mc_prefix(mc, pn)
+                if name != pn:
+                    version = "%s:%s-%s" % self.recipecaches[mc].pkg_pepvpr[fn]
+                    depend_tree['providermap'][name] = (pn, version)
 
         for tid in rq.rqdata.runtaskentries:
-            taskname = bb.runqueue.taskname_from_tid(tid)
-            fn = bb.runqueue.fn_from_tid(tid)
-            pn = self.recipecache.pkg_fn[fn]
-            version  = "%s:%s-%s" % self.recipecache.pkg_pepvpr[fn]
+            (mc, fn, taskname) = bb.runqueue.split_tid(tid)
+            taskfn = bb.runqueue.taskfn_fromtid(tid)
+            pn = self.recipecaches[mc].pkg_fn[taskfn]
+            pn = self.add_mc_prefix(mc, pn)
+            version  = "%s:%s-%s" % self.recipecaches[mc].pkg_pepvpr[taskfn]
             if pn not in depend_tree["pn"]:
                 depend_tree["pn"][pn] = {}
-                depend_tree["pn"][pn]["filename"] = fn
+                depend_tree["pn"][pn]["filename"] = taskfn
                 depend_tree["pn"][pn]["version"] = version
-                depend_tree["pn"][pn]["inherits"] = self.recipecache.inherits.get(fn, None)
+                depend_tree["pn"][pn]["inherits"] = self.recipecaches[mc].inherits.get(taskfn, None)
 
                 # if we have extra caches, list all attributes they bring in
                 extra_info = []
@@ -738,36 +768,37 @@ class BBCooker:
 
                 # for all attributes stored, add them to the dependency tree
                 for ei in extra_info:
-                    depend_tree["pn"][pn][ei] = vars(self.recipecache)[ei][fn]
+                    depend_tree["pn"][pn][ei] = vars(self.recipecaches[mc])[ei][taskfn]
 
 
             for dep in rq.rqdata.runtaskentries[tid].depends:
-                depfn = bb.runqueue.fn_from_tid(dep)
-                deppn = self.recipecache.pkg_fn[depfn]
+                (depmc, depfn, deptaskname) = bb.runqueue.split_tid(dep)
+                deptaskfn = bb.runqueue.taskfn_fromtid(dep)
+                deppn = self.recipecaches[mc].pkg_fn[deptaskfn]
                 dotname = "%s.%s" % (pn, bb.runqueue.taskname_from_tid(tid))
                 if not dotname in depend_tree["tdepends"]:
                     depend_tree["tdepends"][dotname] = []
                 depend_tree["tdepends"][dotname].append("%s.%s" % (deppn, bb.runqueue.taskname_from_tid(dep)))
-            if fn not in seen_fns:
-                seen_fns.append(fn)
+            if taskfn not in seen_fns:
+                seen_fns.append(taskfn)
                 packages = []
 
                 depend_tree["depends"][pn] = []
-                for dep in taskdata.depids[fn]:
+                for dep in taskdata[mc].depids[taskfn]:
                     depend_tree["depends"][pn].append(dep)
 
                 depend_tree["rdepends-pn"][pn] = []
-                for rdep in taskdata.rdepids[fn]:
+                for rdep in taskdata[mc].rdepids[taskfn]:
                     depend_tree["rdepends-pn"][pn].append(rdep)
 
-                rdepends = self.recipecache.rundeps[fn]
+                rdepends = self.recipecaches[mc].rundeps[taskfn]
                 for package in rdepends:
                     depend_tree["rdepends-pkg"][package] = []
                     for rdepend in rdepends[package]:
                         depend_tree["rdepends-pkg"][package].append(rdepend)
                     packages.append(package)
 
-                rrecs = self.recipecache.runrecs[fn]
+                rrecs = self.recipecaches[mc].runrecs[taskfn]
                 for package in rrecs:
                     depend_tree["rrecs-pkg"][package] = []
                     for rdepend in rrecs[package]:
@@ -779,7 +810,7 @@ class BBCooker:
                     if package not in depend_tree["packages"]:
                         depend_tree["packages"][package] = {}
                         depend_tree["packages"][package]["pn"] = pn
-                        depend_tree["packages"][package]["filename"] = fn
+                        depend_tree["packages"][package]["filename"] = taskfn
                         depend_tree["packages"][package]["version"] = version
 
         return depend_tree
@@ -806,44 +837,54 @@ class BBCooker:
                 cachefields = getattr(cache_class, 'cachefields', [])
                 extra_info = extra_info + cachefields
 
-        for tid in taskdata.taskentries:
-            fn = bb.runqueue.fn_from_tid(tid)
-            pn = self.recipecache.pkg_fn[fn]
+        tids = []
+        for mc in taskdata:
+            for tid in taskdata[mc].taskentries:
+                tids.append(tid)
+
+        for tid in tids:
+            (mc, fn, taskname) = bb.runqueue.split_tid(tid)
+            taskfn = bb.runqueue.taskfn_fromtid(tid)
+
+            pn = self.recipecaches[mc].pkg_fn[taskfn]
+            pn = self.add_mc_prefix(mc, pn)
 
             if pn not in depend_tree["pn"]:
                 depend_tree["pn"][pn] = {}
-                depend_tree["pn"][pn]["filename"] = fn
-                version  = "%s:%s-%s" % self.recipecache.pkg_pepvpr[fn]
+                depend_tree["pn"][pn]["filename"] = taskfn
+                version  = "%s:%s-%s" % self.recipecaches[mc].pkg_pepvpr[taskfn]
                 depend_tree["pn"][pn]["version"] = version
-                rdepends = self.recipecache.rundeps[fn]
-                rrecs = self.recipecache.runrecs[fn]
-                depend_tree["pn"][pn]["inherits"] = self.recipecache.inherits.get(fn, None)
+                rdepends = self.recipecaches[mc].rundeps[taskfn]
+                rrecs = self.recipecaches[mc].runrecs[taskfn]
+                depend_tree["pn"][pn]["inherits"] = self.recipecaches[mc].inherits.get(taskfn, None)
 
                 # for all extra attributes stored, add them to the dependency tree
                 for ei in extra_info:
-                    depend_tree["pn"][pn][ei] = vars(self.recipecache)[ei][fn]
+                    depend_tree["pn"][pn][ei] = vars(self.recipecaches[mc])[ei][taskfn]
 
-            if fn not in seen_fns:
-                seen_fns.append(fn)
+            if taskfn not in seen_fns:
+                seen_fns.append(taskfn)
 
                 depend_tree["depends"][pn] = []
-                for item in taskdata.depids[fn]:
+                for item in taskdata[mc].depids[taskfn]:
                     pn_provider = ""
-                    if dep in taskdata.build_targets and taskdata.build_targets[dep]:
-                        fn_provider = taskdata.build_targets[dep][0]
-                        pn_provider = self.recipecache.pkg_fn[fn_provider]
+                    if dep in taskdata[mc].build_targets and taskdata[mc].build_targets[dep]:
+                        fn_provider = taskdata[mc].build_targets[dep][0]
+                        pn_provider = self.recipecaches[mc].pkg_fn[fn_provider]
                     else:
                         pn_provider = item
+                    pn_provider = self.add_mc_prefix(mc, pn_provider)
                     depend_tree["depends"][pn].append(pn_provider)
 
                 depend_tree["rdepends-pn"][pn] = []
-                for rdep in taskdata.rdepids[fn]:
+                for rdep in taskdata[mc].rdepids[taskfn]:
                     pn_rprovider = ""
-                    if rdep in taskdata.run_targets and taskdata.run_targets[rdep]:
-                        fn_rprovider = taskdata.run_targets[rdep][0]
-                        pn_rprovider = self.recipecache.pkg_fn[fn_rprovider]
+                    if rdep in taskdata[mc].run_targets and taskdata[mc].run_targets[rdep]:
+                        fn_rprovider = taskdata[mc].run_targets[rdep][0]
+                        pn_rprovider = self.recipecaches[mc].pkg_fn[fn_rprovider]
                     else:
                         pn_rprovider = rdep
+                    pn_rprovider = self.add_mc_prefix(mc, pn_rprovider)
                     depend_tree["rdepends-pn"][pn].append(pn_rprovider)
 
                 depend_tree["rdepends-pkg"].update(rdepends)
@@ -927,7 +968,7 @@ class BBCooker:
         # Determine which bbappends haven't been applied
 
         # First get list of recipes, including skipped
-        recipefns = list(self.recipecache.pkg_fn.keys())
+        recipefns = list(self.recipecaches[''].pkg_fn.keys())
         recipefns.extend(self.skiplist.keys())
 
         # Work out list of bbappends that have been applied
@@ -951,20 +992,21 @@ class BBCooker:
 
     def handlePrefProviders(self):
 
-        localdata = data.createCopy(self.data)
-        bb.data.update_data(localdata)
-        bb.data.expandKeys(localdata)
+        for mc in self.multiconfigs:
+            localdata = data.createCopy(self.databuilder.mcdata[mc])
+            bb.data.update_data(localdata)
+            bb.data.expandKeys(localdata)
 
-        # Handle PREFERRED_PROVIDERS
-        for p in (localdata.getVar('PREFERRED_PROVIDERS', True) or "").split():
-            try:
-                (providee, provider) = p.split(':')
-            except:
-                providerlog.critical("Malformed option in PREFERRED_PROVIDERS variable: %s" % p)
-                continue
-            if providee in self.recipecache.preferred and self.recipecache.preferred[providee] != provider:
-                providerlog.error("conflicting preferences for %s: both %s and %s specified", providee, provider, self.recipecache.preferred[providee])
-            self.recipecache.preferred[providee] = provider
+            # Handle PREFERRED_PROVIDERS
+            for p in (localdata.getVar('PREFERRED_PROVIDERS', True) or "").split():
+                try:
+                    (providee, provider) = p.split(':')
+                except:
+                    providerlog.critical("Malformed option in PREFERRED_PROVIDERS variable: %s" % p)
+                    continue
+                if providee in self.recipecaches[mc].preferred and self.recipecaches[mc].preferred[providee] != provider:
+                    providerlog.error("conflicting preferences for %s: both %s and %s specified", providee, provider, self.recipecaches[mc].preferred[providee])
+                self.recipecaches[mc].preferred[providee] = provider
 
     def findCoreBaseFiles(self, subdir, configfile):
         corebase = self.data.getVar('COREBASE', True) or ""
@@ -1059,10 +1101,10 @@ class BBCooker:
         """
         pkg_list = []
 
-        for pfn in self.recipecache.pkg_fn:
-            inherits = self.recipecache.inherits.get(pfn, None)
+        for pfn in self.recipecaches[''].pkg_fn:
+            inherits = self.recipecaches[''].inherits.get(pfn, None)
             if inherits and klass in inherits:
-                pkg_list.append(self.recipecache.pkg_fn[pfn])
+                pkg_list.append(self.recipecaches[''].pkg_fn[pfn])
 
         return pkg_list
 
@@ -1095,16 +1137,18 @@ class BBCooker:
             shell.start( self )
 
 
-    def handleCollections( self, collections ):
+    def handleCollections(self, collections):
         """Handle collections"""
         errors = False
-        self.recipecache.bbfile_config_priorities = []
+        self.bbfile_config_priorities = []
         if collections:
             collection_priorities = {}
             collection_depends = {}
             collection_list = collections.split()
             min_prio = 0
             for c in collection_list:
+                bb.debug(1,'Processing %s in collection list' % (c))
+
                 # Get collection priority if defined explicitly
                 priority = self.data.getVar("BBFILE_PRIORITY_%s" % c, True)
                 if priority:
@@ -1123,10 +1167,10 @@ class BBCooker:
                 deps = self.data.getVar("LAYERDEPENDS_%s" % c, True)
                 if deps:
                     try:
-                        deplist = bb.utils.explode_dep_versions2(deps)
+                        depDict = bb.utils.explode_dep_versions2(deps)
                     except bb.utils.VersionStringException as vse:
                         bb.fatal('Error parsing LAYERDEPENDS_%s: %s' % (c, str(vse)))
-                    for dep, oplist in list(deplist.items()):
+                    for dep, oplist in list(depDict.items()):
                         if dep in collection_list:
                             for opstr in oplist:
                                 layerver = self.data.getVar("LAYERVERSION_%s" % dep, True)
@@ -1145,10 +1189,39 @@ class BBCooker:
                         else:
                             parselog.error("Layer '%s' depends on layer '%s', but this layer is not enabled in your configuration", c, dep)
                             errors = True
-                    collection_depends[c] = deplist.keys()
+                    collection_depends[c] = list(depDict.keys())
                 else:
                     collection_depends[c] = []
 
+                # Check recommends and store information for priority calculation
+                recs = self.data.getVar("LAYERRECOMMENDS_%s" % c, True)
+                if recs:
+                    try:
+                        recDict = bb.utils.explode_dep_versions2(recs)
+                    except bb.utils.VersionStringException as vse:
+                        bb.fatal('Error parsing LAYERRECOMMENDS_%s: %s' % (c, str(vse)))
+                    for rec, oplist in list(recDict.items()):
+                        if rec in collection_list:
+                            if oplist:
+                                opstr = oplist[0]
+                                layerver = self.data.getVar("LAYERVERSION_%s" % rec, True)
+                                if layerver:
+                                    (op, recver) = opstr.split()
+                                    try:
+                                        res = bb.utils.vercmp_string_op(layerver, recver, op)
+                                    except bb.utils.VersionStringException as vse:
+                                        bb.fatal('Error parsing LAYERRECOMMENDS_%s: %s' % (c, str(vse)))
+                                    if not res:
+                                        parselog.debug(3,"Layer '%s' recommends version %s of layer '%s', but version %s is currently enabled in your configuration. Check that you are using the correct matching versions/branches of these two layers.", c, opstr, rec, layerver)
+                                        continue
+                                else:
+                                    parselog.debug(3,"Layer '%s' recommends version %s of layer '%s', which exists in your configuration but does not specify a version. Check that you are using the correct matching versions/branches of these two layers.", c, opstr, rec)
+                                    continue
+                            parselog.debug(3,"Layer '%s' recommends layer '%s', so we are adding it", c, rec)
+                            collection_depends[c].append(rec)
+                        else:
+                            parselog.debug(3,"Layer '%s' recommends layer '%s', but this layer is not enabled in your configuration", c, rec)
+
             # Recursively work out collection priorities based on dependencies
             def calc_layer_priority(collection):
                 if not collection_priorities[collection]:
@@ -1176,7 +1249,7 @@ class BBCooker:
                     parselog.error("BBFILE_PATTERN_%s \"%s\" is not a valid regular expression", c, regex)
                     errors = True
                     continue
-                self.recipecache.bbfile_config_priorities.append((c, regex, cre, collection_priorities[c]))
+                self.bbfile_config_priorities.append((c, regex, cre, collection_priorities[c]))
         if errors:
             # We've already printed the actual error(s)
             raise CollectionError("Errors during parsing layer configuration")
@@ -1199,7 +1272,7 @@ class BBCooker:
         if bf.startswith("/") or bf.startswith("../"):
             bf = os.path.abspath(bf)
 
-        self.collection = CookerCollectFiles(self.recipecache.bbfile_config_priorities)
+        self.collection = CookerCollectFiles(self.bbfile_config_priorities)
         filelist, masked = self.collection.collect_bbfiles(self.data, self.expanded_data)
         try:
             os.stat(bf)
@@ -1235,6 +1308,7 @@ class BBCooker:
         """
         Build the file matching regexp buildfile
         """
+        bb.event.fire(bb.event.BuildInit(), self.expanded_data)
 
         # Too many people use -b because they think it's how you normally
         # specify a target to be built, so show a warning
@@ -1248,17 +1322,17 @@ class BBCooker:
         if (task == None):
             task = self.configuration.cmd
 
-        fn, cls = bb.cache.Cache.virtualfn2realfn(buildfile)
+        fn, cls, mc = bb.cache.virtualfn2realfn(buildfile)
         fn = self.matchFile(fn)
 
         self.buildSetVars()
 
-        infos = bb.cache.Cache.parse(fn, self.collection.get_file_appends(fn), \
-                                     self.data,
-                                     self.caches_array)
+        bb_cache = bb.cache.Cache(self.databuilder, self.data_hash, self.caches_array)
+
+        infos = bb_cache.parse(fn, self.collection.get_file_appends(fn))
         infos = dict(infos)
 
-        fn = bb.cache.Cache.realfn2virtual(fn, cls)
+        fn = bb.cache.realfn2virtual(fn, cls, mc)
         try:
             info_array = infos[fn]
         except KeyError:
@@ -1267,29 +1341,30 @@ class BBCooker:
         if info_array[0].skipped:
             bb.fatal("%s was skipped: %s" % (fn, info_array[0].skipreason))
 
-        self.recipecache.add_from_recipeinfo(fn, info_array)
+        self.recipecaches[mc].add_from_recipeinfo(fn, info_array)
 
         # Tweak some variables
         item = info_array[0].pn
-        self.recipecache.ignored_dependencies = set()
-        self.recipecache.bbfile_priority[fn] = 1
+        self.recipecaches[mc].ignored_dependencies = set()
+        self.recipecaches[mc].bbfile_priority[fn] = 1
 
         # Remove external dependencies
-        self.recipecache.task_deps[fn]['depends'] = {}
-        self.recipecache.deps[fn] = []
-        self.recipecache.rundeps[fn] = []
-        self.recipecache.runrecs[fn] = []
+        self.recipecaches[mc].task_deps[fn]['depends'] = {}
+        self.recipecaches[mc].deps[fn] = []
+        self.recipecaches[mc].rundeps[fn] = []
+        self.recipecaches[mc].runrecs[fn] = []
 
         # Invalidate task for target if force mode active
         if self.configuration.force:
             logger.verbose("Invalidate task %s, %s", task, fn)
             if not task.startswith("do_"):
                 task = "do_%s" % task
-            bb.parse.siggen.invalidate_task(task, self.recipecache, fn)
+            bb.parse.siggen.invalidate_task(task, self.recipecaches[mc], fn)
 
         # Setup taskdata structure
-        taskdata = bb.taskdata.TaskData(self.configuration.abort)
-        taskdata.add_provider(self.data, self.recipecache, item)
+        taskdata = {}
+        taskdata[mc] = bb.taskdata.TaskData(self.configuration.abort)
+        taskdata[mc].add_provider(self.data, self.recipecaches[mc], item)
 
         buildname = self.data.getVar("BUILDNAME", True)
         bb.event.fire(bb.event.BuildStarted(buildname, [item]), self.expanded_data)
@@ -1297,9 +1372,9 @@ class BBCooker:
         # Execute the runqueue
         if not task.startswith("do_"):
             task = "do_%s" % task
-        runlist = [[item, task]]
+        runlist = [[mc, item, task, fn]]
 
-        rq = bb.runqueue.RunQueue(self, self.data, self.recipecache, taskdata, runlist)
+        rq = bb.runqueue.RunQueue(self, self.data, self.recipecaches, taskdata, runlist)
 
         def buildFileIdle(server, rq, abort):
 
@@ -1377,23 +1452,24 @@ class BBCooker:
         if not task.startswith("do_"):
             task = "do_%s" % task
 
-        taskdata, runlist, fulltargetlist = self.buildTaskData(targets, task, self.configuration.abort)
+        packages = [target if ':' in target else '%s:%s' % (target, task) for target in targets]
+
+        bb.event.fire(bb.event.BuildInit(packages), self.expanded_data)
+
+        taskdata, runlist = self.buildTaskData(targets, task, self.configuration.abort)
 
         buildname = self.data.getVar("BUILDNAME", False)
 
         # make targets to always look as <target>:do_<task>
         ntargets = []
-        for target in fulltargetlist:
-            if ":" in target:
-                if ":do_" not in target:
-                    target = "%s:do_%s" % tuple(target.split(":", 1))
-            else:
-                target = "%s:%s" % (target, task)
-            ntargets.append(target)
+        for target in runlist:
+            if target[0]:
+                ntargets.append("multiconfig:%s:%s:%s" % (target[0], target[1], target[2]))
+            ntargets.append("%s:%s" % (target[1], target[2]))
 
         bb.event.fire(bb.event.BuildStarted(buildname, ntargets), self.data)
 
-        rq = bb.runqueue.RunQueue(self, self.data, self.recipecache, taskdata, runlist)
+        rq = bb.runqueue.RunQueue(self, self.data, self.recipecaches, taskdata, runlist)
         if 'universe' in targets:
             rq.rqdata.warn_multi_bb = True
 
@@ -1508,13 +1584,14 @@ class BBCooker:
             if CookerFeatures.SEND_SANITYEVENTS in self.featureset:
                 bb.event.fire(bb.event.SanityCheck(False), self.data)
 
-            ignore = self.expanded_data.getVar("ASSUME_PROVIDED", True) or ""
-            self.recipecache.ignored_dependencies = set(ignore.split())
+            for mc in self.multiconfigs:
+                ignore = self.databuilder.mcdata[mc].getVar("ASSUME_PROVIDED", True) or ""
+                self.recipecaches[mc].ignored_dependencies = set(ignore.split())
 
-            for dep in self.configuration.extra_assume_provided:
-                self.recipecache.ignored_dependencies.add(dep)
+                for dep in self.configuration.extra_assume_provided:
+                    self.recipecaches[mc].ignored_dependencies.add(dep)
 
-            self.collection = CookerCollectFiles(self.recipecache.bbfile_config_priorities)
+            self.collection = CookerCollectFiles(self.bbfile_config_priorities)
             (filelist, masked) = self.collection.collect_bbfiles(self.data, self.expanded_data)
 
             self.parser = CookerParser(self, filelist, masked)
@@ -1528,18 +1605,20 @@ class BBCooker:
                 raise bb.BBHandledException()
             self.show_appends_with_no_recipes()
             self.handlePrefProviders()
-            self.recipecache.bbfile_priority = self.collection.collection_priorities(self.recipecache.pkg_fn, self.data)
+            for mc in self.multiconfigs:
+                self.recipecaches[mc].bbfile_priority = self.collection.collection_priorities(self.recipecaches[mc].pkg_fn, self.data)
             self.state = state.running
 
             # Send an event listing all stamps reachable after parsing
             # which the metadata may use to clean up stale data
-            event = bb.event.ReachableStamps(self.recipecache.stamp)
-            bb.event.fire(event, self.expanded_data)
+            for mc in self.multiconfigs:
+                event = bb.event.ReachableStamps(self.recipecaches[mc].stamp)
+                bb.event.fire(event, self.databuilder.mcdata[mc])
             return None
 
         return True
 
-    def checkPackages(self, pkgs_to_build):
+    def checkPackages(self, pkgs_to_build, task=None):
 
         # Return a copy, don't modify the original
         pkgs_to_build = pkgs_to_build[:]
@@ -1553,23 +1632,26 @@ class BBCooker:
                 parselog.warning("Explicit target \"%s\" is in ASSUME_PROVIDED, ignoring" % pkg)
 
         if 'world' in pkgs_to_build:
-            bb.providers.buildWorldTargetList(self.recipecache)
             pkgs_to_build.remove('world')
-            for t in self.recipecache.world_target:
-                pkgs_to_build.append(t)
+            for mc in self.multiconfigs:
+                bb.providers.buildWorldTargetList(self.recipecaches[mc], task)
+                for t in self.recipecaches[mc].world_target:
+                    if mc:
+                        t = "multiconfig:" + mc + ":" + t
+                    pkgs_to_build.append(t)
 
         if 'universe' in pkgs_to_build:
             parselog.warning("The \"universe\" target is only intended for testing and may produce errors.")
             parselog.debug(1, "collating packages for \"universe\"")
             pkgs_to_build.remove('universe')
-            for t in self.recipecache.universe_target:
-                pkgs_to_build.append(t)
+            for mc in self.multiconfigs:
+                for t in self.recipecaches[mc].universe_target:
+                    if mc:
+                        t = "multiconfig:" + mc + ":" + t
+                    pkgs_to_build.append(t)
 
         return pkgs_to_build
 
-
-
-
     def pre_serve(self):
         # Empty the environment. The environment will be populated as
         # necessary from the data store.
@@ -1818,7 +1900,7 @@ class CookerCollectFiles(object):
         # Calculate priorities for each file
         matched = set()
         for p in pkgfns:
-            realfn, cls = bb.cache.Cache.virtualfn2realfn(p)
+            realfn, cls, mc = bb.cache.virtualfn2realfn(p)
             priorities[p] = self.calc_bbfile_priority(realfn, matched)
 
         # Don't show the warning if the BBFILE_PATTERN did match .bbappend files
@@ -1939,7 +2021,7 @@ class Parser(multiprocessing.Process):
             except queue.Full:
                 pending.append(result)
 
-    def parse(self, filename, appends, caches_array):
+    def parse(self, filename, appends):
         try:
             # Record the filename we're parsing into any events generated
             def parse_filter(self, record):
@@ -1952,7 +2034,7 @@ class Parser(multiprocessing.Process):
             bb.event.set_class_handlers(self.handlers.copy())
             bb.event.LogHandler.filter = parse_filter
 
-            return True, bb.cache.Cache.parse(filename, appends, self.cfg, caches_array)
+            return True, self.bb_cache.parse(filename, appends)
         except Exception as exc:
             tb = sys.exc_info()[2]
             exc.recipe = filename
@@ -1970,6 +2052,7 @@ class CookerParser(object):
         self.cooker = cooker
         self.cfgdata = cooker.data
         self.cfghash = cooker.data_hash
+        self.cfgbuilder = cooker.databuilder
 
         # Accounting statistics
         self.parsed = 0
@@ -1984,13 +2067,13 @@ class CookerParser(object):
         self.current = 0
         self.process_names = []
 
-        self.bb_cache = bb.cache.Cache(self.cfgdata, self.cfghash, cooker.caches_array)
+        self.bb_cache = bb.cache.Cache(self.cfgbuilder, self.cfghash, cooker.caches_array)
         self.fromcache = []
         self.willparse = []
         for filename in self.filelist:
             appends = self.cooker.collection.get_file_appends(filename)
             if not self.bb_cache.cacheValid(filename, appends):
-                self.willparse.append((filename, appends, cooker.caches_array))
+                self.willparse.append((filename, appends))
             else:
                 self.fromcache.append((filename, appends))
         self.toparse = self.total - len(self.fromcache)
@@ -2008,7 +2091,7 @@ class CookerParser(object):
         if self.toparse:
             bb.event.fire(bb.event.ParseStarted(self.toparse), self.cfgdata)
             def init():
-                Parser.cfg = self.cfgdata
+                Parser.bb_cache = self.bb_cache
                 bb.utils.set_process_name(multiprocessing.current_process().name)
                 multiprocessing.util.Finalize(None, bb.codeparser.parser_cache_save, exitpriority=1)
                 multiprocessing.util.Finalize(None, bb.fetch.fetcher_parse_save, exitpriority=1)
@@ -2079,7 +2162,7 @@ class CookerParser(object):
 
     def load_cached(self):
         for filename, appends in self.fromcache:
-            cached, infos = self.bb_cache.load(filename, appends, self.cfgdata)
+            cached, infos = self.bb_cache.load(filename, appends)
             yield not cached, infos
 
     def parse_generator(self):
@@ -2158,13 +2241,13 @@ class CookerParser(object):
             if info_array[0].skipped:
                 self.skipped += 1
                 self.cooker.skiplist[virtualfn] = SkippedPackage(info_array[0])
-            self.bb_cache.add_info(virtualfn, info_array, self.cooker.recipecache,
+            (fn, cls, mc) = bb.cache.virtualfn2realfn(virtualfn)
+            self.bb_cache.add_info(virtualfn, info_array, self.cooker.recipecaches[mc],
                                         parsed=parsed, watcher = self.cooker.add_filewatch)
         return True
 
     def reparse(self, filename):
-        infos = self.bb_cache.parse(filename,
-                                    self.cooker.collection.get_file_appends(filename),
-                                    self.cfgdata, self.cooker.caches_array)
+        infos = self.bb_cache.parse(filename, self.cooker.collection.get_file_appends(filename))
         for vfn, info_array in infos:
-            self.cooker.recipecache.add_from_recipeinfo(vfn, info_array)
+            (fn, cls, mc) = bb.cache.virtualfn2realfn(vfn)
+            self.cooker.recipecaches[mc].add_from_recipeinfo(vfn, info_array)

bitbake/lib/bb/cookerdata.py

@@ -237,9 +237,9 @@ class CookerDataBuilder(object):
 
         bb.utils.set_context(bb.utils.clean_context())
         bb.event.set_class_handlers(bb.event.clean_class_handlers())
-        self.data = bb.data.init()
+        self.basedata = bb.data.init()
         if self.tracking:
-            self.data.enableTracking()
+            self.basedata.enableTracking()
 
         # Keep a datastore of the initial environment variables and their
         # values from when BitBake was launched to enable child processes
@@ -250,16 +250,49 @@ class CookerDataBuilder(object):
             self.savedenv.setVar(k, cookercfg.env[k])
 
         filtered_keys = bb.utils.approved_variables()
-        bb.data.inheritFromOS(self.data, self.savedenv, filtered_keys)
-        self.data.setVar("BB_ORIGENV", self.savedenv)
+        bb.data.inheritFromOS(self.basedata, self.savedenv, filtered_keys)
+        self.basedata.setVar("BB_ORIGENV", self.savedenv)
         
         if worker:
-            self.data.setVar("BB_WORKERCONTEXT", "1")
+            self.basedata.setVar("BB_WORKERCONTEXT", "1")
+
+        self.data = self.basedata
+        self.mcdata = {}
 
     def parseBaseConfiguration(self):
         try:
-            self.parseConfigurationFiles(self.prefiles, self.postfiles)
-        except SyntaxError:
+            bb.parse.init_parser(self.basedata)
+            self.data = self.parseConfigurationFiles(self.prefiles, self.postfiles)
+
+            if self.data.getVar("BB_WORKERCONTEXT", False) is None:
+                bb.fetch.fetcher_init(self.data)
+            bb.codeparser.parser_cache_init(self.data)
+
+            bb.event.fire(bb.event.ConfigParsed(), self.data)
+
+            reparse_cnt = 0
+            while self.data.getVar("BB_INVALIDCONF", False) is True:
+                if reparse_cnt > 20:
+                    logger.error("Configuration has been re-parsed over 20 times, "
+                                 "breaking out of the loop...")
+                    raise Exception("Too deep config re-parse loop. Check locations where "
+                                    "BB_INVALIDCONF is being set (ConfigParsed event handlers)")
+                self.data.setVar("BB_INVALIDCONF", False)
+                self.data = self.parseConfigurationFiles(self.prefiles, self.postfiles)
+                reparse_cnt += 1
+                bb.event.fire(bb.event.ConfigParsed(), self.data)
+
+            bb.parse.init_parser(self.data)
+            self.data_hash = self.data.get_hash()
+            self.mcdata[''] = self.data
+
+            multiconfig = (self.data.getVar("BBMULTICONFIG", True) or "").split()
+            for config in multiconfig:
+                mcdata = self.parseConfigurationFiles(['conf/multiconfig/%s.conf' % config] + self.prefiles, self.postfiles)
+                bb.event.fire(bb.event.ConfigParsed(), mcdata)
+                self.mcdata[config] = mcdata
+
+        except (SyntaxError, bb.BBHandledException):
             raise bb.BBHandledException
         except bb.data_smart.ExpansionError as e:
             logger.error(str(e))
@@ -272,8 +305,7 @@ class CookerDataBuilder(object):
         return findConfigFile("bblayers.conf", data)
 
     def parseConfigurationFiles(self, prefiles, postfiles):
-        data = self.data
-        bb.parse.init_parser(data)
+        data = bb.data.createCopy(self.basedata)
 
         # Parse files for loading *before* bitbake.conf and any includes
         for f in prefiles:
@@ -333,23 +365,13 @@ class CookerDataBuilder(object):
         # We register any handlers we've found so far here...
         for var in data.getVar('__BBHANDLERS', False) or []:
             handlerfn = data.getVarFlag(var, "filename", False)
+            if not handlerfn:
+                parselog.critical("Undefined event handler function '%s'" % var)
+                sys.exit(1)
             handlerln = int(data.getVarFlag(var, "lineno", False))
             bb.event.register(var, data.getVar(var, False),  (data.getVarFlag(var, "eventmask", True) or "").split(), handlerfn, handlerln)
 
-        if data.getVar("BB_WORKERCONTEXT", False) is None:
-            bb.fetch.fetcher_init(data)
-        bb.codeparser.parser_cache_init(data)
-        bb.event.fire(bb.event.ConfigParsed(), data)
-
-        if data.getVar("BB_INVALIDCONF", False) is True:
-            data.setVar("BB_INVALIDCONF", False)
-            self.parseConfigurationFiles(self.prefiles, self.postfiles)
-            return
-
-        bb.parse.init_parser(data)
         data.setVar('BBINCLUDED',bb.parse.get_file_depends(data))
-        self.data = data
-        self.data_hash = data.get_hash()
-
 
+        return data
 

bitbake/lib/bb/event.py

@@ -119,21 +119,28 @@ def print_ui_queue():
     logger = logging.getLogger("BitBake")
     if not _uiready:
         from bb.msg import BBLogFormatter
-        console = logging.StreamHandler(sys.stdout)
-        console.setFormatter(BBLogFormatter("%(levelname)s: %(message)s"))
-        logger.handlers = [console]
+        stdout = logging.StreamHandler(sys.stdout)
+        stderr = logging.StreamHandler(sys.stderr)
+        formatter = BBLogFormatter("%(levelname)s: %(message)s")
+        stdout.setFormatter(formatter)
+        stderr.setFormatter(formatter)
 
         # First check to see if we have any proper messages
         msgprint = False
         for event in ui_queue:
             if isinstance(event, logging.LogRecord):
                 if event.levelno > logging.DEBUG:
+                    if event.levelno >= logging.WARNING:
+                        logger.handlers = [stderr]
+                    else:
+                        logger.handlers = [stdout]
                     logger.handle(event)
                     msgprint = True
         if msgprint:
             return
 
         # Nope, so just print all of the messages we have (including debug messages)
+        logger.handlers = [stdout]
         for event in ui_queue:
             if isinstance(event, logging.LogRecord):
                 logger.handle(event)
@@ -382,7 +389,11 @@ class BuildBase(Event):
 
 
 
-
+class BuildInit(BuildBase):
+    """buildFile or buildTargets was invoked"""
+    def __init__(self, p=[]):
+        name = None
+        BuildBase.__init__(self, name, p)
 
 class BuildStarted(BuildBase, OperationStarted):
     """bbmake build run started"""

bitbake/lib/bb/fetch2/__init__.py

@@ -664,7 +664,7 @@ def verify_donestamp(ud, d, origud=None):
         # as an upgrade path from the previous done stamp file format.
         if checksums != precomputed_checksums:
             with open(ud.donestamp, "wb") as cachefile:
-                p = pickle.Pickler(cachefile, pickle.HIGHEST_PROTOCOL)
+                p = pickle.Pickler(cachefile, 2)
                 p.dump(checksums)
         return True
     except ChecksumError as e:
@@ -698,7 +698,7 @@ def update_stamp(ud, d):
             checksums = verify_checksum(ud, d)
             # Store the checksums for later re-verification against the recipe
             with open(ud.donestamp, "wb") as cachefile:
-                p = pickle.Pickler(cachefile, pickle.HIGHEST_PROTOCOL)
+                p = pickle.Pickler(cachefile, 2)
                 p.dump(checksums)
         except ChecksumError as e:
             # Checksums failed to verify, trigger re-download and remove the
@@ -761,6 +761,7 @@ def get_srcrev(d, method_name='sortable_revision'):
     if not format:
         raise FetchError("The SRCREV_FORMAT variable must be set when multiple SCMs are used.")
 
+    name_to_rev = {}
     seenautoinc = False
     for scm in scms:
         ud = urldata[scm]
@@ -769,7 +770,16 @@ def get_srcrev(d, method_name='sortable_revision'):
             seenautoinc = seenautoinc or autoinc
             if len(rev) > 10:
                 rev = rev[:10]
-            format = format.replace(name, rev)
+            name_to_rev[name] = rev
+    # Replace names by revisions in the SRCREV_FORMAT string. The approach used
+    # here can handle names being prefixes of other names and names appearing
+    # as substrings in revisions (in which case the name should not be
+    # expanded). The '|' regular expression operator tries matches from left to
+    # right, so we need to sort the names with the longest ones first.
+    names_descending_len = sorted(name_to_rev, key=len, reverse=True)
+    name_to_rev_re = "|".join(re.escape(name) for name in names_descending_len)
+    format = re.sub(name_to_rev_re, lambda match: name_to_rev[match.group(0)], format)
+
     if seenautoinc:
        format = "AUTOINC+" + format
 
@@ -779,7 +789,7 @@ def localpath(url, d):
     fetcher = bb.fetch2.Fetch([url], d)
     return fetcher.localpath(url)
 
-def runfetchcmd(cmd, d, quiet=False, cleanup=None, log=None):
+def runfetchcmd(cmd, d, quiet=False, cleanup=None, log=None, workdir=None):
     """
     Run cmd returning the command output
     Raise an error if interrupted or cmd fails
@@ -821,7 +831,7 @@ def runfetchcmd(cmd, d, quiet=False, cleanup=None, log=None):
     error_message = ""
 
     try:
-        (output, errors) = bb.process.run(cmd, log=log, shell=True, stderr=subprocess.PIPE)
+        (output, errors) = bb.process.run(cmd, log=log, shell=True, stderr=subprocess.PIPE, cwd=workdir)
         success = True
     except bb.process.NotFoundError as e:
         error_message = "Fetch command %s" % (e.command)
@@ -832,7 +842,7 @@ def runfetchcmd(cmd, d, quiet=False, cleanup=None, log=None):
             output = "output:\n%s" % e.stderr
         else:
             output = "no output"
-        error_message = "Fetch command failed with exit code %s, %s" % (e.exitcode, output)
+        error_message = "Fetch command %s failed with exit code %s, %s" % (e.command, e.exitcode, output)
     except bb.process.CmdError as e:
         error_message = "Fetch command %s could not be run:\n%s" % (e.command, e.msg)
     if not success:
@@ -935,8 +945,6 @@ def try_mirror_url(fetch, origud, ud, ld, check = False):
                 return found
             return False
 
-        os.chdir(ld.getVar("DL_DIR", True))
-
         if not verify_donestamp(ud, ld, origud) or ud.method.need_update(ud, ld):
             ud.method.download(ud, ld)
             if hasattr(ud.method,"build_mirror_data"):
@@ -1412,7 +1420,13 @@ class FetchMethod(object):
 
         # If 'subdir' param exists, create a dir and use it as destination for unpack cmd
         if 'subdir' in urldata.parm:
-            unpackdir = '%s/%s' % (rootdir, urldata.parm.get('subdir'))
+            subdir = urldata.parm.get('subdir')
+            if os.path.isabs(subdir):
+                if not os.path.realpath(subdir).startswith(os.path.realpath(rootdir)):
+                    raise UnpackError("subdir argument isn't a subdirectory of unpack root %s" % rootdir, urldata.url)
+                unpackdir = subdir
+            else:
+                unpackdir = os.path.join(rootdir, subdir)
             bb.utils.mkdirhier(unpackdir)
         else:
             unpackdir = rootdir
@@ -1431,22 +1445,16 @@ class FetchMethod(object):
                     if urlpath.find("/") != -1:
                         destdir = urlpath.rsplit("/", 1)[0] + '/'
                         bb.utils.mkdirhier("%s/%s" % (unpackdir, destdir))
-                cmd = 'cp -fpPR %s %s' % (file, destdir)
+                cmd = 'cp -fpPRH %s %s' % (file, destdir)
 
         if not cmd:
             return
 
-        # Change to unpackdir before executing command
-        save_cwd = os.getcwd();
-        os.chdir(unpackdir)
-
         path = data.getVar('PATH', True)
         if path:
             cmd = "PATH=\"%s\" %s" % (path, cmd)
-        bb.note("Unpacking %s to %s/" % (file, os.getcwd()))
-        ret = subprocess.call(cmd, preexec_fn=subprocess_setup, shell=True)
-
-        os.chdir(save_cwd)
+        bb.note("Unpacking %s to %s/" % (file, unpackdir))
+        ret = subprocess.call(cmd, preexec_fn=subprocess_setup, shell=True, cwd=unpackdir)
 
         if ret != 0:
             raise UnpackError("Unpack command %s failed with return value %s" % (cmd, ret), urldata.url)
@@ -1514,8 +1522,9 @@ class Fetch(object):
         self.connection_cache = connection_cache
 
         fn = d.getVar('FILE', True)
-        if cache and fn and fn in urldata_cache:
-            self.ud = urldata_cache[fn]
+        mc = d.getVar('__BBMULTICONFIG', True) or ""
+        if cache and fn and mc + fn in urldata_cache:
+            self.ud = urldata_cache[mc + fn]
 
         for url in urls:
             if url not in self.ud:
@@ -1527,7 +1536,7 @@ class Fetch(object):
                         pass
 
         if fn and cache:
-            urldata_cache[fn] = self.ud
+            urldata_cache[mc + fn] = self.ud
 
     def localpath(self, url):
         if url not in self.urls:
@@ -1581,8 +1590,6 @@ class Fetch(object):
                 if premirroronly:
                     self.d.setVar("BB_NO_NETWORK", "1")
 
-                os.chdir(self.d.getVar("DL_DIR", True))
-
                 firsterr = None
                 verified_stamp = verify_donestamp(ud, self.d)
                 if not localpath and (not verified_stamp or m.need_update(ud, self.d)):

bitbake/lib/bb/fetch2/bzr.py

@@ -88,28 +88,25 @@ class Bzr(FetchMethod):
             bzrcmd = self._buildbzrcommand(ud, d, "update")
             logger.debug(1, "BZR Update %s", ud.url)
             bb.fetch2.check_network_access(d, bzrcmd, ud.url)
-            os.chdir(os.path.join (ud.pkgdir, os.path.basename(ud.path)))
-            runfetchcmd(bzrcmd, d)
+            runfetchcmd(bzrcmd, d, workdir=os.path.join(ud.pkgdir, os.path.basename(ud.path)))
         else:
             bb.utils.remove(os.path.join(ud.pkgdir, os.path.basename(ud.pkgdir)), True)
             bzrcmd = self._buildbzrcommand(ud, d, "fetch")
             bb.fetch2.check_network_access(d, bzrcmd, ud.url)
             logger.debug(1, "BZR Checkout %s", ud.url)
             bb.utils.mkdirhier(ud.pkgdir)
-            os.chdir(ud.pkgdir)
             logger.debug(1, "Running %s", bzrcmd)
-            runfetchcmd(bzrcmd, d)
-
-        os.chdir(ud.pkgdir)
+            runfetchcmd(bzrcmd, d, workdir=ud.pkgdir)
 
         scmdata = ud.parm.get("scmdata", "")
         if scmdata == "keep":
             tar_flags = ""
         else:
-            tar_flags = "--exclude '.bzr' --exclude '.bzrtags'"
+            tar_flags = "--exclude='.bzr' --exclude='.bzrtags'"
 
         # tar them up to a defined filename
-        runfetchcmd("tar %s -czf %s %s" % (tar_flags, ud.localpath, os.path.basename(ud.pkgdir)), d, cleanup = [ud.localpath])
+        runfetchcmd("tar %s -czf %s %s" % (tar_flags, ud.localpath, os.path.basename(ud.pkgdir)),
+                    d, cleanup=[ud.localpath], workdir=ud.pkgdir)
 
     def supports_srcrev(self):
         return True

bitbake/lib/bb/fetch2/clearcase.py

@@ -202,11 +202,10 @@ class ClearCase(FetchMethod):
 
     def _remove_view(self, ud, d):
         if os.path.exists(ud.viewdir):
-            os.chdir(ud.ccasedir)
             cmd = self._build_ccase_command(ud, 'rmview');
             logger.info("cleaning up [VOB=%s label=%s view=%s]", ud.vob, ud.label, ud.viewname)
             bb.fetch2.check_network_access(d, cmd, ud.url)
-            output = runfetchcmd(cmd, d)
+            output = runfetchcmd(cmd, d, workdir=ud.ccasedir)
             logger.info("rmview output: %s", output)
 
     def need_update(self, ud, d):
@@ -241,11 +240,10 @@ class ClearCase(FetchMethod):
                 raise e
 
         # Set configspec: Setting the configspec effectively fetches the files as defined in the configspec
-        os.chdir(ud.viewdir)
         cmd = self._build_ccase_command(ud, 'setcs');
         logger.info("fetching data [VOB=%s label=%s view=%s]", ud.vob, ud.label, ud.viewname)
         bb.fetch2.check_network_access(d, cmd, ud.url)
-        output = runfetchcmd(cmd, d)
+        output = runfetchcmd(cmd, d, workdir=ud.viewdir)
         logger.info("%s", output)
 
         # Copy the configspec to the viewdir so we have it in our source tarball later

bitbake/lib/bb/fetch2/cvs.py

@@ -123,22 +123,23 @@ class Cvs(FetchMethod):
         pkg = d.getVar('PN', True)
         pkgdir = os.path.join(d.getVar('CVSDIR', True), pkg)
         moddir = os.path.join(pkgdir, localdir)
+        workdir = None
         if os.access(os.path.join(moddir, 'CVS'), os.R_OK):
             logger.info("Update " + ud.url)
             bb.fetch2.check_network_access(d, cvsupdatecmd, ud.url)
             # update sources there
-            os.chdir(moddir)
+            workdir = moddir
             cmd = cvsupdatecmd
         else:
             logger.info("Fetch " + ud.url)
             # check out sources there
             bb.utils.mkdirhier(pkgdir)
-            os.chdir(pkgdir)
+            workdir = pkgdir
             logger.debug(1, "Running %s", cvscmd)
             bb.fetch2.check_network_access(d, cvscmd, ud.url)
             cmd = cvscmd
 
-        runfetchcmd(cmd, d, cleanup = [moddir])
+        runfetchcmd(cmd, d, cleanup=[moddir], workdir=workdir)
 
         if not os.access(moddir, os.R_OK):
             raise FetchError("Directory %s was not readable despite sucessful fetch?!" % moddir, ud.url)
@@ -147,18 +148,18 @@ class Cvs(FetchMethod):
         if scmdata == "keep":
             tar_flags = ""
         else:
-            tar_flags = "--exclude 'CVS'"
+            tar_flags = "--exclude='CVS'"
 
         # tar them up to a defined filename
+        workdir = None
         if 'fullpath' in ud.parm:
-            os.chdir(pkgdir)
+            workdir = pkgdir
             cmd = "tar %s -czf %s %s" % (tar_flags, ud.localpath, localdir)
         else:
-            os.chdir(moddir)
-            os.chdir('..')
+            workdir = os.path.dirname(os.path.realpath(moddir))
             cmd = "tar %s -czf %s %s" % (tar_flags, ud.localpath, os.path.basename(moddir))
 
-        runfetchcmd(cmd, d, cleanup = [ud.localpath])
+        runfetchcmd(cmd, d, cleanup=[ud.localpath], workdir=workdir)
 
     def clean(self, ud, d):
         """ Clean CVS Files and tarballs """

bitbake/lib/bb/fetch2/git.py

@@ -49,6 +49,10 @@ Supported SRC_URI options are:
    referring to commit which is valid in tag instead of branch.
    The default is "0", set nobranch=1 if needed.
 
+- usehead
+   For local git:// urls to use the current branch HEAD as the revsion for use with
+   AUTOREV. Implies nobranch.
+
 """
 
 #Copyright (C) 2005 Richard Purdie
@@ -153,6 +157,13 @@ class Git(FetchMethod):
 
         ud.nobranch = ud.parm.get("nobranch","0") == "1"
 
+        # usehead implies nobranch
+        ud.usehead = ud.parm.get("usehead","0") == "1"
+        if ud.usehead:
+            if ud.proto != "file":
+                 raise bb.fetch2.ParameterError("The usehead option is only for use with local ('protocol=file') git repositories", ud.url)
+            ud.nobranch = 1
+
         # bareclone implies nocheckout
         ud.bareclone = ud.parm.get("bareclone","0") == "1"
         if ud.bareclone:
@@ -168,6 +179,9 @@ class Git(FetchMethod):
             ud.branches[name] = branch
             ud.unresolvedrev[name] = branch
 
+        if ud.usehead:
+            ud.unresolvedrev['default'] = 'HEAD'
+
         ud.basecmd = data.getVar("FETCHCMD_git", d, True) or "git -c core.fsyncobjectfiles=0"
 
         ud.write_tarballs = ((data.getVar("BB_GENERATE_MIRROR_TARBALLS", d, True) or "0") != "0") or ud.rebaseable
@@ -205,9 +219,8 @@ class Git(FetchMethod):
     def need_update(self, ud, d):
         if not os.path.exists(ud.clonedir):
             return True
-        os.chdir(ud.clonedir)
         for name in ud.names:
-            if not self._contains_ref(ud, d, name):
+            if not self._contains_ref(ud, d, name, ud.clonedir):
                 return True
         if ud.write_tarballs and not os.path.exists(ud.fullmirror):
             return True
@@ -228,8 +241,7 @@ class Git(FetchMethod):
         # If the checkout doesn't exist and the mirror tarball does, extract it
         if not os.path.exists(ud.clonedir) and os.path.exists(ud.fullmirror):
             bb.utils.mkdirhier(ud.clonedir)
-            os.chdir(ud.clonedir)
-            runfetchcmd("tar -xzf %s" % (ud.fullmirror), d)
+            runfetchcmd("tar -xzf %s" % (ud.fullmirror), d, workdir=ud.clonedir)
 
         repourl = self._get_repo_url(ud)
 
@@ -244,34 +256,32 @@ class Git(FetchMethod):
             progresshandler = GitProgressHandler(d)
             runfetchcmd(clone_cmd, d, log=progresshandler)
 
-        os.chdir(ud.clonedir)
         # Update the checkout if needed
         needupdate = False
         for name in ud.names:
-            if not self._contains_ref(ud, d, name):
+            if not self._contains_ref(ud, d, name, ud.clonedir):
                 needupdate = True
         if needupdate:
             try: 
-                runfetchcmd("%s remote rm origin" % ud.basecmd, d) 
+                runfetchcmd("%s remote rm origin" % ud.basecmd, d, workdir=ud.clonedir)
             except bb.fetch2.FetchError:
                 logger.debug(1, "No Origin")
 
-            runfetchcmd("%s remote add --mirror=fetch origin %s" % (ud.basecmd, repourl), d)
+            runfetchcmd("%s remote add --mirror=fetch origin %s" % (ud.basecmd, repourl), d, workdir=ud.clonedir)
             fetch_cmd = "LANG=C %s fetch -f --prune --progress %s refs/*:refs/*" % (ud.basecmd, repourl)
             if ud.proto.lower() != 'file':
                 bb.fetch2.check_network_access(d, fetch_cmd, ud.url)
             progresshandler = GitProgressHandler(d)
-            runfetchcmd(fetch_cmd, d, log=progresshandler)
-            runfetchcmd("%s prune-packed" % ud.basecmd, d)
-            runfetchcmd("%s pack-redundant --all | xargs -r rm" % ud.basecmd, d)
+            runfetchcmd(fetch_cmd, d, log=progresshandler, workdir=ud.clonedir)
+            runfetchcmd("%s prune-packed" % ud.basecmd, d, workdir=ud.clonedir)
+            runfetchcmd("%s pack-redundant --all | xargs -r rm" % ud.basecmd, d, workdir=ud.clonedir)
             try:
                 os.unlink(ud.fullmirror)
             except OSError as exc:
                 if exc.errno != errno.ENOENT:
                     raise
-        os.chdir(ud.clonedir)
         for name in ud.names:
-            if not self._contains_ref(ud, d, name):
+            if not self._contains_ref(ud, d, name, ud.clonedir):
                 raise bb.fetch2.FetchError("Unable to find revision %s in branch %s even from upstream" % (ud.revisions[name], ud.branches[name]))
 
     def build_mirror_data(self, ud, d):
@@ -281,10 +291,9 @@ class Git(FetchMethod):
             if os.path.islink(ud.fullmirror):
                 os.unlink(ud.fullmirror)
 
-            os.chdir(ud.clonedir)
             logger.info("Creating tarball of git repository")
-            runfetchcmd("tar -czf %s %s" % (ud.fullmirror, os.path.join(".") ), d)
-            runfetchcmd("touch %s.done" % (ud.fullmirror), d)
+            runfetchcmd("tar -czf %s %s" % (ud.fullmirror, os.path.join(".") ), d, workdir=ud.clonedir)
+            runfetchcmd("touch %s.done" % (ud.fullmirror), d, workdir=ud.clonedir)
 
     def unpack(self, ud, destdir, d):
         """ unpack the downloaded src to destdir"""
@@ -307,21 +316,21 @@ class Git(FetchMethod):
             cloneflags += " --mirror"
 
         runfetchcmd("%s clone %s %s/ %s" % (ud.basecmd, cloneflags, ud.clonedir, destdir), d)
-        os.chdir(destdir)
         repourl = self._get_repo_url(ud)
-        runfetchcmd("%s remote set-url origin %s" % (ud.basecmd, repourl), d)
+        runfetchcmd("%s remote set-url origin %s" % (ud.basecmd, repourl), d, workdir=destdir)
         if not ud.nocheckout:
             if subdir != "":
-                runfetchcmd("%s read-tree %s%s" % (ud.basecmd, ud.revisions[ud.names[0]], readpathspec), d)
-                runfetchcmd("%s checkout-index -q -f -a" % ud.basecmd, d)
+                runfetchcmd("%s read-tree %s%s" % (ud.basecmd, ud.revisions[ud.names[0]], readpathspec), d,
+                            workdir=destdir)
+                runfetchcmd("%s checkout-index -q -f -a" % ud.basecmd, d, workdir=destdir)
             elif not ud.nobranch:
                 branchname =  ud.branches[ud.names[0]]
                 runfetchcmd("%s checkout -B %s %s" % (ud.basecmd, branchname, \
-                            ud.revisions[ud.names[0]]), d)
+                            ud.revisions[ud.names[0]]), d, workdir=destdir)
                 runfetchcmd("%s branch --set-upstream %s origin/%s" % (ud.basecmd, branchname, \
-                            branchname), d)
+                            branchname), d, workdir=destdir)
             else:
-                runfetchcmd("%s checkout %s" % (ud.basecmd, ud.revisions[ud.names[0]]), d)
+                runfetchcmd("%s checkout %s" % (ud.basecmd, ud.revisions[ud.names[0]]), d, workdir=destdir)
 
         return True
 
@@ -335,7 +344,7 @@ class Git(FetchMethod):
     def supports_srcrev(self):
         return True
 
-    def _contains_ref(self, ud, d, name):
+    def _contains_ref(self, ud, d, name, wd):
         cmd = ""
         if ud.nobranch:
             cmd = "%s log --pretty=oneline -n 1 %s -- 2> /dev/null | wc -l" % (
@@ -344,7 +353,7 @@ class Git(FetchMethod):
             cmd =  "%s branch --contains %s --list %s 2> /dev/null | wc -l" % (
                 ud.basecmd, ud.revisions[name], ud.branches[name])
         try:
-            output = runfetchcmd(cmd, d, quiet=True)
+            output = runfetchcmd(cmd, d, quiet=True, workdir=wd)
         except bb.fetch2.FetchError:
             return False
         if len(output.split()) > 1:
@@ -387,7 +396,7 @@ class Git(FetchMethod):
         """
         output = self._lsremote(ud, d, "")
         # Tags of the form ^{} may not work, need to fallback to other form
-        if ud.unresolvedrev[name][:5] == "refs/":
+        if ud.unresolvedrev[name][:5] == "refs/" or ud.usehead:
             head = ud.unresolvedrev[name]
             tag = ud.unresolvedrev[name]
         else:

bitbake/lib/bb/fetch2/gitannex.py

@@ -34,43 +34,42 @@ class GitANNEX(Git):
         """
         return ud.type in ['gitannex']
 
-    def uses_annex(self, ud, d):
+    def uses_annex(self, ud, d, wd):
         for name in ud.names:
             try:
-                runfetchcmd("%s rev-list git-annex" % (ud.basecmd), d, quiet=True)
+                runfetchcmd("%s rev-list git-annex" % (ud.basecmd), d, quiet=True, workdir=wd)
                 return True
             except bb.fetch.FetchError:
                 pass
 
         return False
 
-    def update_annex(self, ud, d):
+    def update_annex(self, ud, d, wd):
         try:
-            runfetchcmd("%s annex get --all" % (ud.basecmd), d, quiet=True)
+            runfetchcmd("%s annex get --all" % (ud.basecmd), d, quiet=True, workdir=wd)
         except bb.fetch.FetchError:
             return False
-        runfetchcmd("chmod u+w -R %s/annex" % (ud.clonedir), d, quiet=True)
+        runfetchcmd("chmod u+w -R %s/annex" % (ud.clonedir), d, quiet=True, workdir=wd)
 
         return True
 
     def download(self, ud, d):
         Git.download(self, ud, d)
 
-        os.chdir(ud.clonedir)
-        annex = self.uses_annex(ud, d)
+        annex = self.uses_annex(ud, d, ud.clonedir)
         if annex:
-            self.update_annex(ud, d)
+            self.update_annex(ud, d, ud.clonedir)
 
     def unpack(self, ud, destdir, d):
         Git.unpack(self, ud, destdir, d)
 
-        os.chdir(ud.destdir)
         try:
-            runfetchcmd("%s annex sync" % (ud.basecmd), d)
+            runfetchcmd("%s annex init" % (ud.basecmd), d, workdir=ud.destdir)
         except bb.fetch.FetchError:
             pass
 
-        annex = self.uses_annex(ud, d)
+        annex = self.uses_annex(ud, d, ud.destdir)
         if annex:
-            runfetchcmd("%s annex get" % (ud.basecmd), d)
-            runfetchcmd("chmod u+w -R %s/.git/annex" % (ud.destdir), d, quiet=True)
+            runfetchcmd("%s annex get" % (ud.basecmd), d, workdir=ud.destdir)
+            runfetchcmd("chmod u+w -R %s/.git/annex" % (ud.destdir), d, quiet=True, workdir=ud.destdir)
+

bitbake/lib/bb/fetch2/gitsm.py

@@ -43,10 +43,10 @@ class GitSM(Git):
         """
         return ud.type in ['gitsm']
 
-    def uses_submodules(self, ud, d):
+    def uses_submodules(self, ud, d, wd):
         for name in ud.names:
             try:
-                runfetchcmd("%s show %s:.gitmodules" % (ud.basecmd, ud.revisions[name]), d, quiet=True)
+                runfetchcmd("%s show %s:.gitmodules" % (ud.basecmd, ud.revisions[name]), d, quiet=True, workdir=wd)
                 return True
             except bb.fetch.FetchError:
                 pass
@@ -107,28 +107,25 @@ class GitSM(Git):
         os.mkdir(tmpclonedir)
         os.rename(ud.clonedir, gitdir)
         runfetchcmd("sed " + gitdir + "/config -i -e 's/bare.*=.*true/bare = false/'", d)
-        os.chdir(tmpclonedir)
-        runfetchcmd(ud.basecmd + " reset --hard", d)
-        runfetchcmd(ud.basecmd + " checkout " + ud.revisions[ud.names[0]], d)
-        runfetchcmd(ud.basecmd + " submodule update --init --recursive", d)
+        runfetchcmd(ud.basecmd + " reset --hard", d, workdir=tmpclonedir)
+        runfetchcmd(ud.basecmd + " checkout " + ud.revisions[ud.names[0]], d, workdir=tmpclonedir)
+        runfetchcmd(ud.basecmd + " submodule update --init --recursive", d, workdir=tmpclonedir)
         self._set_relative_paths(tmpclonedir)
-        runfetchcmd("sed " + gitdir + "/config -i -e 's/bare.*=.*false/bare = true/'", d)
+        runfetchcmd("sed " + gitdir + "/config -i -e 's/bare.*=.*false/bare = true/'", d, workdir=tmpclonedir)
         os.rename(gitdir, ud.clonedir,)
         bb.utils.remove(tmpclonedir, True)
 
     def download(self, ud, d):
         Git.download(self, ud, d)
 
-        os.chdir(ud.clonedir)
-        submodules = self.uses_submodules(ud, d)
+        submodules = self.uses_submodules(ud, d, ud.clonedir)
         if submodules:
             self.update_submodules(ud, d)
 
     def unpack(self, ud, destdir, d):
         Git.unpack(self, ud, destdir, d)
         
-        os.chdir(ud.destdir)
-        submodules = self.uses_submodules(ud, d)
+        submodules = self.uses_submodules(ud, d, ud.destdir)
         if submodules:
-            runfetchcmd(ud.basecmd + " checkout " + ud.revisions[ud.names[0]], d)
-            runfetchcmd(ud.basecmd + " submodule update --init --recursive", d)
+            runfetchcmd(ud.basecmd + " checkout " + ud.revisions[ud.names[0]], d, workdir=ud.destdir)
+            runfetchcmd(ud.basecmd + " submodule update --init --recursive", d, workdir=ud.destdir)

bitbake/lib/bb/fetch2/hg.py

@@ -169,25 +169,22 @@ class Hg(FetchMethod):
         # If the checkout doesn't exist and the mirror tarball does, extract it
         if not os.path.exists(ud.pkgdir) and os.path.exists(ud.fullmirror):
             bb.utils.mkdirhier(ud.pkgdir)
-            os.chdir(ud.pkgdir)
-            runfetchcmd("tar -xzf %s" % (ud.fullmirror), d)
+            runfetchcmd("tar -xzf %s" % (ud.fullmirror), d, workdir=ud.pkgdir)
 
         if os.access(os.path.join(ud.moddir, '.hg'), os.R_OK):
             # Found the source, check whether need pull
             updatecmd = self._buildhgcommand(ud, d, "update")
-            os.chdir(ud.moddir)
             logger.debug(1, "Running %s", updatecmd)
             try:
-                runfetchcmd(updatecmd, d)
+                runfetchcmd(updatecmd, d, workdir=ud.moddir)
             except bb.fetch2.FetchError:
                 # Runnning pull in the repo
                 pullcmd = self._buildhgcommand(ud, d, "pull")
                 logger.info("Pulling " + ud.url)
                 # update sources there
-                os.chdir(ud.moddir)
                 logger.debug(1, "Running %s", pullcmd)
                 bb.fetch2.check_network_access(d, pullcmd, ud.url)
-                runfetchcmd(pullcmd, d)
+                runfetchcmd(pullcmd, d, workdir=ud.moddir)
                 try:
                     os.unlink(ud.fullmirror)
                 except OSError as exc:
@@ -200,17 +197,15 @@ class Hg(FetchMethod):
             logger.info("Fetch " + ud.url)
             # check out sources there
             bb.utils.mkdirhier(ud.pkgdir)
-            os.chdir(ud.pkgdir)
             logger.debug(1, "Running %s", fetchcmd)
             bb.fetch2.check_network_access(d, fetchcmd, ud.url)
-            runfetchcmd(fetchcmd, d)
+            runfetchcmd(fetchcmd, d, workdir=ud.pkgdir)
 
         # Even when we clone (fetch), we still need to update as hg's clone
         # won't checkout the specified revision if its on a branch
         updatecmd = self._buildhgcommand(ud, d, "update")
-        os.chdir(ud.moddir)
         logger.debug(1, "Running %s", updatecmd)
-        runfetchcmd(updatecmd, d)
+        runfetchcmd(updatecmd, d, workdir=ud.moddir)
 
     def clean(self, ud, d):
         """ Clean the hg dir """
@@ -246,10 +241,9 @@ class Hg(FetchMethod):
             if os.path.islink(ud.fullmirror):
                 os.unlink(ud.fullmirror)
 
-            os.chdir(ud.pkgdir)
             logger.info("Creating tarball of hg repository")
-            runfetchcmd("tar -czf %s %s" % (ud.fullmirror, ud.module), d)
-            runfetchcmd("touch %s.done" % (ud.fullmirror), d)
+            runfetchcmd("tar -czf %s %s" % (ud.fullmirror, ud.module), d, workdir=ud.pkgdir)
+            runfetchcmd("touch %s.done" % (ud.fullmirror), d, workdir=ud.pkgdir)
 
     def localpath(self, ud, d):
         return ud.pkgdir
@@ -269,10 +263,8 @@ class Hg(FetchMethod):
                 logger.debug(2, "Unpack: creating new hg repository in '" + codir + "'")
                 runfetchcmd("%s init %s" % (ud.basecmd, codir), d)
             logger.debug(2, "Unpack: updating source in '" + codir + "'")
-            os.chdir(codir)
-            runfetchcmd("%s pull %s" % (ud.basecmd, ud.moddir), d)
-            runfetchcmd("%s up -C %s" % (ud.basecmd, revflag), d)
+            runfetchcmd("%s pull %s" % (ud.basecmd, ud.moddir), d, workdir=codir)
+            runfetchcmd("%s up -C %s" % (ud.basecmd, revflag), d, workdir=codir)
         else:
             logger.debug(2, "Unpack: extracting source to '" + codir + "'")
-            os.chdir(ud.moddir)
-            runfetchcmd("%s archive -t files %s %s" % (ud.basecmd, revflag, codir), d)
+            runfetchcmd("%s archive -t files %s %s" % (ud.basecmd, revflag, codir), d, workdir=ud.moddir)

bitbake/lib/bb/fetch2/npm.py

@@ -13,7 +13,7 @@ Usage in the recipe:
     - name
     - version
 
-    npm://registry.npmjs.org/${PN}/-/${PN}-${PV}.tgz  would become npm://registry.npmjs.org;name=${PN};ver=${PV}
+    npm://registry.npmjs.org/${PN}/-/${PN}-${PV}.tgz  would become npm://registry.npmjs.org;name=${PN};version=${PV}
     The fetcher all triggers off the existence of ud.localpath. If that exists and has the ".done" stamp, its assumed the fetch is good/done
 
 """
@@ -88,7 +88,7 @@ class Npm(FetchMethod):
         ud.localpath = d.expand("${DL_DIR}/npm/%s" % ud.bbnpmmanifest)
 
         self.basecmd = d.getVar("FETCHCMD_wget", True) or "/usr/bin/env wget -O -t 2 -T 30 -nv --passive-ftp --no-check-certificate "
-        self.basecmd += " --directory-prefix=%s " % prefixdir
+        ud.prefixdir = prefixdir
 
         ud.write_tarballs = ((data.getVar("BB_GENERATE_MIRROR_TARBALLS", d, True) or "0") != "0")
         ud.mirrortarball = 'npm_%s-%s.tar.xz' % (ud.pkgname, ud.version)
@@ -102,7 +102,8 @@ class Npm(FetchMethod):
     def _runwget(self, ud, d, command, quiet):
         logger.debug(2, "Fetching %s using command '%s'" % (ud.url, command))
         bb.fetch2.check_network_access(d, command)
-        runfetchcmd(command, d, quiet)
+        dldir = d.getVar("DL_DIR", True)
+        runfetchcmd(command, d, quiet, workdir=dldir)
 
     def _unpackdep(self, ud, pkg, data, destdir, dldir, d):
         file = data[pkg]['tgz']
@@ -113,16 +114,13 @@ class Npm(FetchMethod):
             bb.fatal("NPM package %s downloaded not a tarball!" % file)
 
         # Change to subdir before executing command
-        save_cwd = os.getcwd()
         if not os.path.exists(destdir):
             os.makedirs(destdir)
-        os.chdir(destdir)
         path = d.getVar('PATH', True)
         if path:
             cmd = "PATH=\"%s\" %s" % (path, cmd)
-        bb.note("Unpacking %s to %s/" % (file, os.getcwd()))
-        ret = subprocess.call(cmd, preexec_fn=subprocess_setup, shell=True)
-        os.chdir(save_cwd)
+        bb.note("Unpacking %s to %s/" % (file, destdir))
+        ret = subprocess.call(cmd, preexec_fn=subprocess_setup, shell=True, cwd=destdir)
 
         if ret != 0:
             raise UnpackError("Unpack command %s failed with return value %s" % (cmd, ret), ud.url)
@@ -140,7 +138,12 @@ class Npm(FetchMethod):
             workobj = json.load(datafile)
         dldir = "%s/%s" % (os.path.dirname(ud.localpath), ud.pkgname)
 
-        self._unpackdep(ud, ud.pkgname, workobj,  "%s/npmpkg" % destdir, dldir, d)
+        if 'subdir' in ud.parm:
+            unpackdir = '%s/%s' % (destdir, ud.parm.get('subdir'))
+        else:
+            unpackdir = '%s/npmpkg' % destdir
+
+        self._unpackdep(ud, ud.pkgname, workobj, unpackdir, dldir, d)
 
     def _parse_view(self, output):
         '''
@@ -162,7 +165,9 @@ class Npm(FetchMethod):
             pdata = json.loads('\n'.join(datalines))
         return pdata
 
-    def _getdependencies(self, pkg, data, version, d, ud, optional=False):
+    def _getdependencies(self, pkg, data, version, d, ud, optional=False, fetchedlist=None):
+        if fetchedlist is None:
+            fetchedlist = []
         pkgfullname = pkg
         if version != '*' and not '/' in version:
             pkgfullname += "@'%s'" % version
@@ -184,7 +189,9 @@ class Npm(FetchMethod):
         outputurl = pdata['dist']['tarball']
         data[pkg] = {}
         data[pkg]['tgz'] = os.path.basename(outputurl)
-        self._runwget(ud, d, "%s %s" % (self.basecmd, outputurl), False)
+        if not outputurl in fetchedlist:
+            self._runwget(ud, d, "%s --directory-prefix=%s %s" % (self.basecmd, ud.prefixdir, outputurl), False)
+            fetchedlist.append(outputurl)
 
         dependencies = pdata.get('dependencies', {})
         optionalDependencies = pdata.get('optionalDependencies', {})
@@ -197,12 +204,19 @@ class Npm(FetchMethod):
             else:
                 depsfound[dep] = dependencies[dep]
         for dep, version in optdepsfound.items():
-            self._getdependencies(dep, data[pkg]['deps'], version, d, ud, optional=True)
+            self._getdependencies(dep, data[pkg]['deps'], version, d, ud, optional=True, fetchedlist=fetchedlist)
         for dep, version in depsfound.items():
-            self._getdependencies(dep, data[pkg]['deps'], version, d, ud)
+            self._getdependencies(dep, data[pkg]['deps'], version, d, ud, fetchedlist=fetchedlist)
 
-    def _getshrinkeddependencies(self, pkg, data, version, d, ud, lockdown, manifest):
+    def _getshrinkeddependencies(self, pkg, data, version, d, ud, lockdown, manifest, toplevel=True):
         logger.debug(2, "NPM shrinkwrap file is %s" % data)
+        if toplevel:
+            name = data.get('name', None)
+            if name and name != pkg:
+                for obj in data.get('dependencies', []):
+                    if obj == pkg:
+                        self._getshrinkeddependencies(obj, data['dependencies'][obj], data['dependencies'][obj]['version'], d, ud, lockdown, manifest, False)
+                        return
         outputurl = "invalid"
         if ('resolved' not in data) or (not data['resolved'].startswith('http')):
             # will be the case for ${PN}
@@ -211,7 +225,7 @@ class Npm(FetchMethod):
             outputurl = runfetchcmd(fetchcmd, d, True)
         else:
             outputurl = data['resolved']
-        self._runwget(ud, d, "%s %s" % (self.basecmd, outputurl), False)
+        self._runwget(ud, d, "%s --directory-prefix=%s %s" % (self.basecmd, ud.prefixdir, outputurl), False)
         manifest[pkg] = {}
         manifest[pkg]['tgz'] = os.path.basename(outputurl).rstrip()
         manifest[pkg]['deps'] = {}
@@ -228,7 +242,7 @@ class Npm(FetchMethod):
         if 'dependencies' in data:
             for obj in data['dependencies']:
                 logger.debug(2, "Found dep is %s" % str(obj))
-                self._getshrinkeddependencies(obj, data['dependencies'][obj], data['dependencies'][obj]['version'], d, ud, lockdown, manifest[pkg]['deps'])
+                self._getshrinkeddependencies(obj, data['dependencies'][obj], data['dependencies'][obj]['version'], d, ud, lockdown, manifest[pkg]['deps'], False)
 
     def download(self, ud, d):
         """Fetch url"""
@@ -239,10 +253,7 @@ class Npm(FetchMethod):
         if not os.listdir(ud.pkgdatadir) and os.path.exists(ud.fullmirror):
             dest = d.getVar("DL_DIR", True)
             bb.utils.mkdirhier(dest)
-            save_cwd = os.getcwd()
-            os.chdir(dest)
-            runfetchcmd("tar -xJf %s" % (ud.fullmirror), d)
-            os.chdir(save_cwd)
+            runfetchcmd("tar -xJf %s" % (ud.fullmirror), d, workdir=dest)
             return
 
         shwrf = d.getVar('NPM_SHRINKWRAP', True)
@@ -275,10 +286,8 @@ class Npm(FetchMethod):
             if os.path.islink(ud.fullmirror):
                 os.unlink(ud.fullmirror)
 
-            save_cwd = os.getcwd()
-            os.chdir(d.getVar("DL_DIR", True))
+            dldir = d.getVar("DL_DIR", True)
             logger.info("Creating tarball of npm data")
-            runfetchcmd("tar -cJf %s npm/%s npm/%s" % (ud.fullmirror, ud.bbnpmmanifest, ud.pkgname), d)
-            runfetchcmd("touch %s.done" % (ud.fullmirror), d)
-            os.chdir(save_cwd)
-
+            runfetchcmd("tar -cJf %s npm/%s npm/%s" % (ud.fullmirror, ud.bbnpmmanifest, ud.pkgname), d,
+                        workdir=dldir)
+            runfetchcmd("touch %s.done" % (ud.fullmirror), d, workdir=dldir)

bitbake/lib/bb/fetch2/osc.py

@@ -88,23 +88,21 @@ class Osc(FetchMethod):
             oscupdatecmd = self._buildosccommand(ud, d, "update")
             logger.info("Update "+ ud.url)
             # update sources there
-            os.chdir(ud.moddir)
             logger.debug(1, "Running %s", oscupdatecmd)
             bb.fetch2.check_network_access(d, oscupdatecmd, ud.url)
-            runfetchcmd(oscupdatecmd, d)
+            runfetchcmd(oscupdatecmd, d, workdir=ud.moddir)
         else:
             oscfetchcmd = self._buildosccommand(ud, d, "fetch")
             logger.info("Fetch " + ud.url)
             # check out sources there
             bb.utils.mkdirhier(ud.pkgdir)
-            os.chdir(ud.pkgdir)
             logger.debug(1, "Running %s", oscfetchcmd)
             bb.fetch2.check_network_access(d, oscfetchcmd, ud.url)
-            runfetchcmd(oscfetchcmd, d)
+            runfetchcmd(oscfetchcmd, d, workdir=ud.pkgdir)
 
-        os.chdir(os.path.join(ud.pkgdir + ud.path))
         # tar them up to a defined filename
-        runfetchcmd("tar -czf %s %s" % (ud.localpath, ud.module), d, cleanup = [ud.localpath])
+        runfetchcmd("tar -czf %s %s" % (ud.localpath, ud.module), d,
+                    cleanup=[ud.localpath], workdir=os.path.join(ud.pkgdir + ud.path))
 
     def supports_srcrev(self):
         return False

bitbake/lib/bb/fetch2/perforce.py

@@ -168,15 +168,13 @@ class Perforce(FetchMethod):
 
         bb.utils.remove(ud.pkgdir, True)
         bb.utils.mkdirhier(ud.pkgdir)
-        os.chdir(ud.pkgdir)
 
         for afile in filelist:
             p4fetchcmd = self._buildp4command(ud, d, 'print', afile)
             bb.fetch2.check_network_access(d, p4fetchcmd)
-            runfetchcmd(p4fetchcmd, d)
+            runfetchcmd(p4fetchcmd, d, workdir=ud.pkgdir)
 
-        os.chdir(ud.pkgdir)
-        runfetchcmd('tar -czf %s p4' % (ud.localpath), d, cleanup = [ud.localpath])
+        runfetchcmd('tar -czf %s p4' % (ud.localpath), d, cleanup=[ud.localpath], workdir=ud.pkgdir)
 
     def clean(self, ud, d):
         """ Cleanup p4 specific files and dirs"""

bitbake/lib/bb/fetch2/repo.py

@@ -69,24 +69,23 @@ class Repo(FetchMethod):
         else:
             username = ""
 
-        bb.utils.mkdirhier(os.path.join(codir, "repo"))
-        os.chdir(os.path.join(codir, "repo"))
-        if not os.path.exists(os.path.join(codir, "repo", ".repo")):
+        repodir = os.path.join(codir, "repo")
+        bb.utils.mkdirhier(repodir)
+        if not os.path.exists(os.path.join(repodir, ".repo")):
             bb.fetch2.check_network_access(d, "repo init -m %s -b %s -u %s://%s%s%s" % (ud.manifest, ud.branch, ud.proto, username, ud.host, ud.path), ud.url)
-            runfetchcmd("repo init -m %s -b %s -u %s://%s%s%s" % (ud.manifest, ud.branch, ud.proto, username, ud.host, ud.path), d)
+            runfetchcmd("repo init -m %s -b %s -u %s://%s%s%s" % (ud.manifest, ud.branch, ud.proto, username, ud.host, ud.path), d, workdir=repodir)
 
         bb.fetch2.check_network_access(d, "repo sync %s" % ud.url, ud.url)
-        runfetchcmd("repo sync", d)
-        os.chdir(codir)
+        runfetchcmd("repo sync", d, workdir=repodir)
 
         scmdata = ud.parm.get("scmdata", "")
         if scmdata == "keep":
             tar_flags = ""
         else:
-            tar_flags = "--exclude '.repo' --exclude '.git'"
+            tar_flags = "--exclude='.repo' --exclude='.git'"
 
         # Create a cache
-        runfetchcmd("tar %s -czf %s %s" % (tar_flags, ud.localpath, os.path.join(".", "*") ), d)
+        runfetchcmd("tar %s -czf %s %s" % (tar_flags, ud.localpath, os.path.join(".", "*") ), d, workdir=codir)
 
     def supports_srcrev(self):
         return False

bitbake/lib/bb/fetch2/svn.py

@@ -126,35 +126,32 @@ class Svn(FetchMethod):
         if os.access(os.path.join(ud.moddir, '.svn'), os.R_OK):
             svnupdatecmd = self._buildsvncommand(ud, d, "update")
             logger.info("Update " + ud.url)
-            # update sources there
-            os.chdir(ud.moddir)
             # We need to attempt to run svn upgrade first in case its an older working format
             try:
-                runfetchcmd(ud.basecmd + " upgrade", d)
+                runfetchcmd(ud.basecmd + " upgrade", d, workdir=ud.moddir)
             except FetchError:
                 pass
             logger.debug(1, "Running %s", svnupdatecmd)
             bb.fetch2.check_network_access(d, svnupdatecmd, ud.url)
-            runfetchcmd(svnupdatecmd, d)
+            runfetchcmd(svnupdatecmd, d, workdir=ud.moddir)
         else:
             svnfetchcmd = self._buildsvncommand(ud, d, "fetch")
             logger.info("Fetch " + ud.url)
             # check out sources there
             bb.utils.mkdirhier(ud.pkgdir)
-            os.chdir(ud.pkgdir)
             logger.debug(1, "Running %s", svnfetchcmd)
             bb.fetch2.check_network_access(d, svnfetchcmd, ud.url)
-            runfetchcmd(svnfetchcmd, d)
+            runfetchcmd(svnfetchcmd, d, workdir=ud.pkgdir)
 
         scmdata = ud.parm.get("scmdata", "")
         if scmdata == "keep":
             tar_flags = ""
         else:
-            tar_flags = "--exclude '.svn'"
+            tar_flags = "--exclude='.svn'"
 
-        os.chdir(ud.pkgdir)
         # tar them up to a defined filename
-        runfetchcmd("tar %s -czf %s %s" % (tar_flags, ud.localpath, ud.path_spec), d, cleanup = [ud.localpath])
+        runfetchcmd("tar %s -czf %s %s" % (tar_flags, ud.localpath, ud.path_spec), d,
+                    cleanup=[ud.localpath], workdir=ud.pkgdir)
 
     def clean(self, ud, d):
         """ Clean SVN specific files and dirs """

bitbake/lib/bb/fetch2/wget.py

@@ -108,6 +108,10 @@ class Wget(FetchMethod):
             bb.utils.mkdirhier(os.path.dirname(dldir + os.sep + ud.localfile))
             fetchcmd += " -O " + dldir + os.sep + ud.localfile
 
+        if ud.user:
+            up = ud.user.split(":")
+            fetchcmd += " --user=%s --password=%s --auth-no-challenge" % (up[0],up[1])
+
         uri = ud.url.split(";")[0]
         if os.path.exists(ud.localpath):
             # file exists, but we didnt complete it.. trying again..
@@ -300,6 +304,13 @@ class Wget(FetchMethod):
             uri = ud.url.split(";")[0]
             r = urllib.request.Request(uri)
             r.get_method = lambda: "HEAD"
+
+            if ud.user:
+                import base64
+                encodeuser = base64.b64encode(ud.user.encode('utf-8')).decode("utf-8")
+                authheader =  "Basic %s" % encodeuser
+                r.add_header("Authorization", authheader)
+
             opener.open(r)
         except urllib.error.URLError as e:
             if try_again:

bitbake/lib/bb/parse/ast.py

@@ -69,6 +69,33 @@ class ExportNode(AstNode):
     def eval(self, data):
         data.setVarFlag(self.var, "export", 1, op = 'exported')
 
+class UnsetNode(AstNode):
+    def __init__(self, filename, lineno, var):
+        AstNode.__init__(self, filename, lineno)
+        self.var = var
+
+    def eval(self, data):
+        loginfo = {
+            'variable': self.var,
+            'file': self.filename,
+            'line': self.lineno,
+        }
+        data.delVar(self.var,**loginfo)
+
+class UnsetFlagNode(AstNode):
+    def __init__(self, filename, lineno, var, flag):
+        AstNode.__init__(self, filename, lineno)
+        self.var = var
+        self.flag = flag
+
+    def eval(self, data):
+        loginfo = {
+            'variable': self.var,
+            'file': self.filename,
+            'line': self.lineno,
+        }
+        data.delVarFlag(self.var, self.flag, **loginfo)
+
 class DataNode(AstNode):
     """
     Various data related updates. For the sake of sanity
@@ -270,6 +297,12 @@ def handleInclude(statements, filename, lineno, m, force):
 def handleExport(statements, filename, lineno, m):
     statements.append(ExportNode(filename, lineno, m.group(1)))
 
+def handleUnset(statements, filename, lineno, m):
+    statements.append(UnsetNode(filename, lineno, m.group(1)))
+
+def handleUnsetFlag(statements, filename, lineno, m):
+    statements.append(UnsetFlagNode(filename, lineno, m.group(1), m.group(2)))
+
 def handleData(statements, filename, lineno, groupd):
     statements.append(DataNode(filename, lineno, groupd))
 
@@ -311,6 +344,8 @@ def finalize(fn, d, variant = None):
     for var in d.getVar('__BBHANDLERS', False) or []:
         # try to add the handler
         handlerfn = d.getVarFlag(var, "filename", False)
+        if not handlerfn:
+            bb.fatal("Undefined event handler function '%s'" % var)
         handlerln = int(d.getVarFlag(var, "lineno", False))
         bb.event.register(var, d.getVar(var, False), (d.getVarFlag(var, "eventmask", True) or "").split(), handlerfn, handlerln)
 
@@ -469,9 +504,5 @@ def multi_finalize(fn, d):
             except bb.parse.SkipRecipe as e:
                 datastores[variant].setVar("__SKIPPED", e.args[0])
 
-    if len(datastores) > 1:
-        variants = filter(None, datastores.keys())
-        safe_d.setVar("__VARIANTS", " ".join(variants))
-
     datastores[""] = d
     return datastores

bitbake/lib/bb/parse/parse_py/ConfHandler.py

@@ -57,6 +57,8 @@ __config_regexp__  = re.compile( r"""
 __include_regexp__ = re.compile( r"include\s+(.+)" )
 __require_regexp__ = re.compile( r"require\s+(.+)" )
 __export_regexp__ = re.compile( r"export\s+([a-zA-Z0-9\-_+.${}/]+)$" )
+__unset_regexp__ = re.compile( r"unset\s+([a-zA-Z0-9\-_+.${}/]+)$" )
+__unset_flag_regexp__ = re.compile( r"unset\s+([a-zA-Z0-9\-_+.${}/]+)\[([a-zA-Z0-9\-_+.${}/]+)\]$" )
 
 def init(data):
     topdir = data.getVar('TOPDIR', False)
@@ -185,6 +187,16 @@ def feeder(lineno, s, fn, statements):
         ast.handleExport(statements, fn, lineno, m)
         return
 
+    m = __unset_regexp__.match(s)
+    if m:
+        ast.handleUnset(statements, fn, lineno, m)
+        return
+
+    m = __unset_flag_regexp__.match(s)
+    if m:
+        ast.handleUnsetFlag(statements, fn, lineno, m)
+        return
+
     raise ParseError("unparsed line: '%s'" % s, fn, lineno);
 
 # Add us to the handlers list

bitbake/lib/bb/persist_data.py

@@ -131,14 +131,14 @@ class SQLTable(collections.MutableMapping):
         return [row[1] for row in data]
 
     def values(self):
-        return list(self.values())
+        return list(self.itervalues())
 
     def itervalues(self):
         data = self._execute("SELECT value FROM %s;" % self.table)
         return (row[0] for row in data)
 
     def items(self):
-        return list(self.items())
+        return list(self.iteritems())
 
     def iteritems(self):
         return self._execute("SELECT * FROM %s;" % self.table)

bitbake/lib/bb/progress.py

@@ -234,10 +234,13 @@ class MultiStageProcessProgressReporter(MultiStageProgressReporter):
     """
     def __init__(self, d, processname, stage_weights, debug=False):
         self._processname = processname
+        self._started = False
         MultiStageProgressReporter.__init__(self, d, stage_weights, debug)
 
     def start(self):
-        bb.event.fire(bb.event.ProcessStarted(self._processname, 100), self._data)
+        if not self._started:
+            bb.event.fire(bb.event.ProcessStarted(self._processname, 100), self._data)
+            self._started = True
 
     def _fire_progress(self, taskprogress):
         if taskprogress == 0:

bitbake/lib/bb/providers.py

@@ -402,7 +402,7 @@ def getRuntimeProviders(dataCache, rdepend):
     return rproviders
 
 
-def buildWorldTargetList(dataCache):
+def buildWorldTargetList(dataCache, task=None):
     """
     Build package list for "bitbake world"
     """
@@ -413,6 +413,9 @@ def buildWorldTargetList(dataCache):
     for f in dataCache.possible_world:
         terminal = True
         pn = dataCache.pkg_fn[f]
+        if task and task not in dataCache.task_deps[f]['tasks']:
+            logger.debug(2, "World build skipping %s as task %s doesn't exist", f, task)
+            terminal = False
 
         for p in dataCache.pn_provides[pn]:
             if p.startswith('virtual/'):

bitbake/lib/bb/siggen.py

@@ -144,8 +144,9 @@ class SignatureGeneratorBasic(SignatureGenerator):
 
     def finalise(self, fn, d, variant):
 
-        if variant:
-            fn = "virtual:" + variant + ":" + fn
+        mc = d.getVar("__BBMULTICONFIG", False) or ""
+        if variant or mc:
+            fn = bb.cache.realfn2virtual(fn, variant, mc)
 
         try:
             taskdeps = self._build_data(fn, d)
@@ -293,23 +294,25 @@ class SignatureGeneratorBasic(SignatureGenerator):
 
         computed_basehash = calc_basehash(data)
         if computed_basehash != self.basehash[k]:
-            bb.error("Basehash mismatch %s verses %s for %s" % (computed_basehash, self.basehash[k], k))
+            bb.error("Basehash mismatch %s versus %s for %s" % (computed_basehash, self.basehash[k], k))
         if runtime and k in self.taskhash:
             computed_taskhash = calc_taskhash(data)
             if computed_taskhash != self.taskhash[k]:
-                bb.error("Taskhash mismatch %s verses %s for %s" % (computed_taskhash, self.taskhash[k], k))
+                bb.error("Taskhash mismatch %s versus %s for %s" % (computed_taskhash, self.taskhash[k], k))
 
 
-    def dump_sigs(self, dataCache, options):
+    def dump_sigs(self, dataCaches, options):
         for fn in self.taskdeps:
             for task in self.taskdeps[fn]:
+                tid = fn + ":" + task
+                (mc, _, _) = bb.runqueue.split_tid(tid)
                 k = fn + "." + task
                 if k not in self.taskhash:
                     continue
-                if dataCache.basetaskhash[k] != self.basehash[k]:
+                if dataCaches[mc].basetaskhash[k] != self.basehash[k]:
                     bb.error("Bitbake's cached basehash does not match the one we just generated (%s)!" % k)
-                    bb.error("The mismatched hashes were %s and %s" % (dataCache.basetaskhash[k], self.basehash[k]))
-                self.dump_sigtask(fn, task, dataCache.stamp[fn], True)
+                    bb.error("The mismatched hashes were %s and %s" % (dataCaches[mc].basetaskhash[k], self.basehash[k]))
+                self.dump_sigtask(fn, task, dataCaches[mc].stamp[fn], True)
 
 class SignatureGeneratorBasicHash(SignatureGeneratorBasic):
     name = "basichash"
@@ -363,10 +366,12 @@ def clean_basepaths_list(a):
 def compare_sigfiles(a, b, recursecb = None):
     output = []
 
-    p1 = pickle.Unpickler(open(a, "rb"))
-    a_data = p1.load()
-    p2 = pickle.Unpickler(open(b, "rb"))
-    b_data = p2.load()
+    with open(a, 'rb') as f:
+        p1 = pickle.Unpickler(f)
+        a_data = p1.load()
+    with open(b, 'rb') as f:
+        p2 = pickle.Unpickler(f)
+        b_data = p2.load()
 
     def dict_diff(a, b, whitelist=set()):
         sa = set(a.keys())
@@ -563,8 +568,9 @@ def calc_taskhash(sigdata):
 def dump_sigfile(a):
     output = []
 
-    p1 = pickle.Unpickler(open(a, "rb"))
-    a_data = p1.load()
+    with open(a, 'rb') as f:
+        p1 = pickle.Unpickler(f)
+        a_data = p1.load()
 
     output.append("basewhitelist: %s" % (a_data['basewhitelist']))
 

bitbake/lib/bb/tests/fetch.py

@@ -360,7 +360,10 @@ class FetcherTest(unittest.TestCase):
 
     def tearDown(self):
         os.chdir(self.origdir)
-        bb.utils.prunedir(self.tempdir)
+        if os.environ.get("BB_TMPDIR_NOCLEAN") == "yes":
+            print("Not cleaning up %s. Please remove manually." % self.tempdir)
+        else:
+            bb.utils.prunedir(self.tempdir)
 
 class MirrorUriTest(FetcherTest):
 
@@ -505,6 +508,15 @@ class FetcherLocalTest(FetcherTest):
         tree = self.fetchUnpack(['file://dir/subdir/e;subdir=bar'])
         self.assertEqual(tree, ['bar/dir/subdir/e'])
 
+    def test_local_absolutedir(self):
+        # Unpacking to an absolute path that is a subdirectory of the root
+        # should work
+        tree = self.fetchUnpack(['file://a;subdir=%s' % os.path.join(self.unpackdir, 'bar')])
+
+        # Unpacking to an absolute path outside of the root should fail
+        with self.assertRaises(bb.fetch2.UnpackError):
+            self.fetchUnpack(['file://a;subdir=/bin/sh'])
+
 class FetcherNetworkTest(FetcherTest):
 
     if os.environ.get("BB_SKIP_NETTESTS") == "yes":
@@ -585,6 +597,36 @@ class FetcherNetworkTest(FetcherTest):
             url1 = url2 = "git://git.openembedded.org/bitbake;rev=270a05b0b4ba0959fe0624d2a4885d7b70426da5;tag=270a05b0b4ba0959fe0624d2a4885d7b70426da5"
             self.assertRaises(bb.fetch.FetchError, self.gitfetcher, url1, url2)
 
+        def test_gitfetch_localusehead(self):
+            # Create dummy local Git repo
+            src_dir = tempfile.mkdtemp(dir=self.tempdir,
+                                       prefix='gitfetch_localusehead_')
+            src_dir = os.path.abspath(src_dir)
+            bb.process.run("git init", cwd=src_dir)
+            bb.process.run("git commit --allow-empty -m'Dummy commit'",
+                           cwd=src_dir)
+            # Use other branch than master
+            bb.process.run("git checkout -b my-devel", cwd=src_dir)
+            bb.process.run("git commit --allow-empty -m'Dummy commit 2'",
+                           cwd=src_dir)
+            stdout = bb.process.run("git rev-parse HEAD", cwd=src_dir)
+            orig_rev = stdout[0].strip()
+
+            # Fetch and check revision
+            self.d.setVar("SRCREV", "AUTOINC")
+            url = "git://" + src_dir + ";protocol=file;usehead=1"
+            fetcher = bb.fetch.Fetch([url], self.d)
+            fetcher.download()
+            fetcher.unpack(self.unpackdir)
+            stdout = bb.process.run("git rev-parse HEAD",
+                                    cwd=os.path.join(self.unpackdir, 'git'))
+            unpack_rev = stdout[0].strip()
+            self.assertEqual(orig_rev, unpack_rev)
+
+        def test_gitfetch_remoteusehead(self):
+            url = "git://git.openembedded.org/bitbake;usehead=1"
+            self.assertRaises(bb.fetch.ParameterError, self.gitfetcher, url, url)
+
         def test_gitfetch_premirror(self):
             url1 = "git://git.openembedded.org/bitbake"
             url2 = "git://someserver.org/bitbake"
@@ -768,7 +810,6 @@ class FetchLatestVersionTest(FetcherTest):
 
 class FetchCheckStatusTest(FetcherTest):
     test_wget_uris = ["http://www.cups.org/software/1.7.2/cups-1.7.2-source.tar.bz2",
-                      "http://www.cups.org/software/ipptool/ipptool-20130731-linux-ubuntu-i686.tar.gz",
                       "http://www.cups.org/",
                       "http://downloads.yoctoproject.org/releases/sato/sato-engine-0.1.tar.gz",
                       "http://downloads.yoctoproject.org/releases/sato/sato-engine-0.2.tar.gz",

bitbake/lib/bb/tests/parse.py

@@ -68,6 +68,23 @@ C = "3"
         with self.assertRaises(bb.parse.ParseError):
             d = bb.parse.handle(f.name, self.d)['']
 
+    unsettest = """
+A = "1"
+B = "2"
+B[flag] = "3"
+
+unset A
+unset B[flag]
+"""
+
+    def test_parse_unset(self):
+        f = self.parsehelper(self.unsettest)
+        d = bb.parse.handle(f.name, self.d)['']
+        self.assertEqual(d.getVar("A", True), None)
+        self.assertEqual(d.getVarFlag("A","flag", True), None)
+        self.assertEqual(d.getVar("B", True), "2")
+        
+
     overridetest = """
 RRECOMMENDS_${PN} = "a"
 RRECOMMENDS_${PN}_libc = "b"

bitbake/lib/bb/tinfoil.py

@@ -59,6 +59,12 @@ class Tinfoil:
     def register_idle_function(self, function, data):
         pass
 
+    def __enter__(self):
+        return self
+
+    def __exit__(self, type, value, traceback):
+        self.shutdown()
+
     def parseRecipes(self):
         sys.stderr.write("Parsing recipes..")
         self.logger.setLevel(logging.WARNING)
@@ -74,16 +80,52 @@ class Tinfoil:
         self.logger.setLevel(logging.INFO)
         sys.stderr.write("done.\n")
 
-        self.cooker_data = self.cooker.recipecache
+        self.cooker_data = self.cooker.recipecaches['']
 
     def prepare(self, config_only = False):
         if not self.cooker_data:
             if config_only:
                 self.cooker.parseConfiguration()
-                self.cooker_data = self.cooker.recipecache
+                self.cooker_data = self.cooker.recipecaches['']
             else:
                 self.parseRecipes()
 
+    def parse_recipe_file(self, fn, appends=True, appendlist=None, config_data=None):
+        """
+        Parse the specified recipe file (with or without bbappends)
+        and return a datastore object representing the environment
+        for the recipe.
+        Parameters:
+            fn: recipe file to parse - can be a file path or virtual
+                specification
+            appends: True to apply bbappends, False otherwise
+            appendlist: optional list of bbappend files to apply, if you
+                        want to filter them
+            config_data: custom config datastore to use. NOTE: if you
+                         specify config_data then you cannot use a virtual
+                         specification for fn.
+        """
+        if appends and appendlist == []:
+            appends = False
+        if appends:
+            if appendlist:
+                appendfiles = appendlist
+            else:
+                if not hasattr(self.cooker, 'collection'):
+                    raise Exception('You must call tinfoil.prepare() with config_only=False in order to get bbappends')
+                appendfiles = self.cooker.collection.get_file_appends(fn)
+        else:
+            appendfiles = None
+        if config_data:
+            # We have to use a different function here if we're passing in a datastore
+            localdata = bb.data.createCopy(config_data)
+            envdata = bb.cache.parse_recipe(localdata, fn, appendfiles)['']
+        else:
+            # Use the standard path
+            parser = bb.cache.NoCache(self.cooker.databuilder)
+            envdata = parser.loadDataFull(fn, appendfiles)
+        return envdata
+
     def shutdown(self):
         self.cooker.shutdown(force=True)
         self.cooker.post_serve()

bitbake/lib/bb/ui/buildinfohelper.py

@@ -43,6 +43,7 @@ from orm.models import Package, Package_File, Target_Installed_Package, Target_F
 from orm.models import Task_Dependency, Package_Dependency
 from orm.models import Recipe_Dependency, Provides
 from orm.models import Project, CustomImagePackage, CustomImageRecipe
+from orm.models import signal_runbuilds
 
 from bldcontrol.models import BuildEnvironment, BuildRequest
 
@@ -150,55 +151,33 @@ class ORMWrapper(object):
     # pylint: disable=bad-continuation
     # we do not follow the python conventions for continuation indentation due to long lines here
 
-    def create_build_object(self, build_info, brbe, project_id):
-        assert 'machine' in build_info
-        assert 'distro' in build_info
-        assert 'distro_version' in build_info
-        assert 'started_on' in build_info
-        assert 'cooker_log_path' in build_info
-        assert 'build_name' in build_info
-        assert 'bitbake_version' in build_info
-
+    def get_or_create_build_object(self, brbe):
         prj = None
         buildrequest = None
-        if brbe is not None:            # this build was triggered by a request from a user
+        if brbe is not None:
+            # Toaster-triggered build
             logger.debug(1, "buildinfohelper: brbe is %s" % brbe)
             br, _ = brbe.split(":")
-            buildrequest = BuildRequest.objects.get(pk = br)
+            buildrequest = BuildRequest.objects.get(pk=br)
             prj = buildrequest.project
-
-        elif project_id is not None:    # this build was triggered by an external system for a specific project
-            logger.debug(1, "buildinfohelper: project is %s" % prj)
-            prj = Project.objects.get(pk = project_id)
-
-        else:                           # this build was triggered by a legacy system, or command line interactive mode
+        else:
+            # CLI build
             prj = Project.objects.get_or_create_default_project()
             logger.debug(1, "buildinfohelper: project is not specified, defaulting to %s" % prj)
 
-
         if buildrequest is not None:
+            # reuse existing Build object
             build = buildrequest.build
-            logger.info("Updating existing build, with %s", build_info)
             build.project = prj
-            build.machine=build_info['machine']
-            build.distro=build_info['distro']
-            build.distro_version=build_info['distro_version']
-            build.cooker_log_path=build_info['cooker_log_path']
-            build.build_name=build_info['build_name']
-            build.bitbake_version=build_info['bitbake_version']
             build.save()
-
         else:
+            # create new Build object
+            now = timezone.now()
             build = Build.objects.create(
-                                    project = prj,
-                                    machine=build_info['machine'],
-                                    distro=build_info['distro'],
-                                    distro_version=build_info['distro_version'],
-                                    started_on=build_info['started_on'],
-                                    completed_on=build_info['started_on'],
-                                    cooker_log_path=build_info['cooker_log_path'],
-                                    build_name=build_info['build_name'],
-                                    bitbake_version=build_info['bitbake_version'])
+                project=prj,
+                started_on=now,
+                completed_on=now,
+                build_name='')
 
         logger.debug(1, "buildinfohelper: build is created %s" % build)
 
@@ -208,6 +187,11 @@ class ORMWrapper(object):
 
         return build
 
+    def update_build(self, build, data_dict):
+        for key in data_dict:
+            setattr(build, key, data_dict[key])
+        build.save()
+
     @staticmethod
     def get_or_create_targets(target_info):
         """
@@ -230,7 +214,7 @@ class ORMWrapper(object):
             result.append(obj)
         return result
 
-    def update_build_object(self, build, errors, warnings, taskfailures):
+    def update_build_stats_and_outcome(self, build, errors, warnings, taskfailures):
         assert isinstance(build,Build)
         assert isinstance(errors, int)
         assert isinstance(warnings, int)
@@ -251,6 +235,7 @@ class ORMWrapper(object):
         build.completed_on = timezone.now()
         build.outcome = outcome
         build.save()
+        signal_runbuilds()
 
     def update_target_set_license_manifest(self, target, license_manifest_path):
         target.license_manifest_path = license_manifest_path
@@ -393,7 +378,7 @@ class ORMWrapper(object):
             layer_copy, c = Layer_Version.objects.get_or_create(
                 build=build_obj,
                 layer=layer_obj.layer,
-                up_branch=layer_obj.up_branch,
+                release=layer_obj.release,
                 branch=layer_version_information['branch'],
                 commit=layer_version_information['commit'],
                 local_path=layer_version_information['local_path'],
@@ -436,13 +421,24 @@ class ORMWrapper(object):
         assert 'name' in layer_information
         assert 'layer_index_url' in layer_information
 
+        # From command line builds we have no brbe as the request is directly
+        # from bitbake
         if brbe is None:
-            layer_object, _ = Layer.objects.get_or_create(
-                                name=layer_information['name'],
-                                layer_index_url=layer_information['layer_index_url'])
+            # If we don't have git commit sha then we're using a non-git
+            # layer so set the layer_source_dir to identify it as such
+            if not layer_information['version']['commit']:
+                local_source_dir = layer_information["local_path"]
+            else:
+                local_source_dir = None
+
+            layer_object, _ = \
+                Layer.objects.get_or_create(
+                    name=layer_information['name'],
+                    local_source_dir=local_source_dir,
+                    layer_index_url=layer_information['layer_index_url'])
+
             return layer_object
         else:
-            # we are under managed mode; we must match the layer used in the Project Layer
             br_id, be_id = brbe.split(":")
 
             # find layer by checkout path;
@@ -455,7 +451,11 @@ class ORMWrapper(object):
             # note that this is different
             buildrequest = BuildRequest.objects.get(pk = br_id)
             for brl in buildrequest.brlayer_set.all():
-                localdirname = os.path.join(bc.getGitCloneDirectory(brl.giturl, brl.commit), brl.dirpath)
+                if brl.local_source_dir:
+                    localdirname = os.path.join(brl.local_source_dir,
+                                                brl.dirpath)
+                else:
+                    localdirname = os.path.join(bc.getGitCloneDirectory(brl.giturl, brl.commit), brl.dirpath)
                 # we get a relative path, unless running in HEAD mode where the path is absolute
                 if not localdirname.startswith("/"):
                     localdirname = os.path.join(bc.be.sourcedir, localdirname)
@@ -467,6 +467,11 @@ class ORMWrapper(object):
                     if brl.layer_version:
                         return brl.layer_version
 
+                # This might be a local layer (i.e. no git info) so try
+                # matching local_source_dir
+                if brl.local_source_dir and brl.local_source_dir == layer_information["local_path"]:
+                    return brl.layer_version
+
                     # we matched the BRLayer, but we need the layer_version that generated this BR; reverse of the Project.schedule_build()
                     #logger.debug(1, "Matched %s to BRlayer %s" % (pformat(layer_information["local_path"]), localdirname))
 
@@ -625,8 +630,8 @@ class ORMWrapper(object):
                         Recipe,
                         name=built_recipe.name,
                         layer_version__build=None,
-                        layer_version__up_branch=
-                        built_recipe.layer_version.up_branch,
+                        layer_version__release=
+                        built_recipe.layer_version.release,
                         file_path=built_recipe.file_path,
                         version=built_recipe.version
                     )
@@ -915,22 +920,55 @@ class BuildInfoHelper(object):
     ###################
     ## methods to convert event/external info into objects that the ORM layer uses
 
+    def _ensure_build(self):
+        """
+        Ensure the current build object exists and is up to date with
+        data on the bitbake server
+        """
+        if not 'build' in self.internal_state or not self.internal_state['build']:
+            # create the Build object
+            self.internal_state['build'] = \
+                self.orm_wrapper.get_or_create_build_object(self.brbe)
 
-    def _get_build_information(self, build_log_path):
+        build = self.internal_state['build']
+
+        # update missing fields on the Build object with found data
         build_info = {}
-        build_info['machine'] = self.server.runCommand(["getVariable", "MACHINE"])[0]
-        build_info['distro'] = self.server.runCommand(["getVariable", "DISTRO"])[0]
-        build_info['distro_version'] = self.server.runCommand(["getVariable", "DISTRO_VERSION"])[0]
-        build_info['started_on'] = timezone.now()
-        build_info['completed_on'] = timezone.now()
-        build_info['cooker_log_path'] = build_log_path
-        build_info['build_name'] = self.server.runCommand(["getVariable", "BUILDNAME"])[0]
-        build_info['bitbake_version'] = self.server.runCommand(["getVariable", "BB_VERSION"])[0]
-        build_info['project'] = self.project = self.server.runCommand(["getVariable", "TOASTER_PROJECT"])[0]
-        return build_info
+
+        # set to True if at least one field is going to be set
+        changed = False
+
+        if not build.build_name:
+            build_name = self.server.runCommand(["getVariable", "BUILDNAME"])[0]
+
+            # only reset the build name if the one on the server is actually
+            # a valid value for the build_name field
+            if build_name != None:
+                build_info['build_name'] = build_name
+                changed = True
+
+        if not build.machine:
+            build_info['machine'] = self.server.runCommand(["getVariable", "MACHINE"])[0]
+            changed = True
+
+        if not build.distro:
+            build_info['distro'] = self.server.runCommand(["getVariable", "DISTRO"])[0]
+            changed = True
+
+        if not build.distro_version:
+            build_info['distro_version'] = self.server.runCommand(["getVariable", "DISTRO_VERSION"])[0]
+            changed = True
+
+        if not build.bitbake_version:
+            build_info['bitbake_version'] = self.server.runCommand(["getVariable", "BB_VERSION"])[0]
+            changed = True
+
+        if changed:
+            self.orm_wrapper.update_build(self.internal_state['build'], build_info)
 
     def _get_task_information(self, event, recipe):
         assert 'taskname' in vars(event)
+        self._ensure_build()
 
         task_information = {}
         task_information['build'] = self.internal_state['build']
@@ -945,8 +983,7 @@ class BuildInfoHelper(object):
         return task_information
 
     def _get_layer_version_for_path(self, path):
-        assert path.startswith("/")
-        assert 'build' in self.internal_state
+        self._ensure_build()
 
         def _slkey_interactive(layer_version):
             assert isinstance(layer_version, Layer_Version)
@@ -957,6 +994,9 @@ class BuildInfoHelper(object):
             # we can match to the recipe file path
             if path.startswith(lvo.local_path):
                 return lvo
+            if lvo.layer.local_source_dir and \
+               path.startswith(lvo.layer.local_source_dir):
+                return lvo
 
         #if we get here, we didn't read layers correctly; dump whatever information we have on the error log
         logger.warning("Could not match layer version for recipe path %s : %s", path, self.orm_wrapper.layer_version_objects)
@@ -990,6 +1030,8 @@ class BuildInfoHelper(object):
         return recipe_info
 
     def _get_path_information(self, task_object):
+        self._ensure_build()
+
         assert isinstance(task_object, Task)
         build_stats_format = "{tmpdir}/buildstats/{buildname}/{package}/"
         build_stats_path = []
@@ -1032,17 +1074,31 @@ class BuildInfoHelper(object):
             except NotExisting as nee:
                 logger.warning("buildinfohelper: cannot identify layer exception:%s ", nee)
 
+    def store_started_build(self):
+        self._ensure_build()
 
-    def store_started_build(self, event, build_log_path):
+    def save_build_log_file_path(self, build_log_path):
+        self._ensure_build()
+
+        if not self.internal_state['build'].cooker_log_path:
+            data_dict = {'cooker_log_path': build_log_path}
+            self.orm_wrapper.update_build(self.internal_state['build'], data_dict)
+
+    def save_build_targets(self, event):
+        self._ensure_build()
+
+        # create target information
         assert '_pkgs' in vars(event)
-        build_information = self._get_build_information(build_log_path)
+        target_information = {}
+        target_information['targets'] = event._pkgs
+        target_information['build'] = self.internal_state['build']
 
-        # Update brbe and project as they can be changed for every build
-        self.project = build_information['project']
+        self.internal_state['targets'] = self.orm_wrapper.get_or_create_targets(target_information)
 
-        build_obj = self.orm_wrapper.create_build_object(build_information, self.brbe, self.project)
+    def save_build_layers_and_variables(self):
+        self._ensure_build()
 
-        self.internal_state['build'] = build_obj
+        build_obj = self.internal_state['build']
 
         # save layer version information for this build
         if not 'lvs' in self.internal_state:
@@ -1053,13 +1109,6 @@ class BuildInfoHelper(object):
 
             del self.internal_state['lvs']
 
-        # create target information
-        target_information = {}
-        target_information['targets'] = event._pkgs
-        target_information['build'] = build_obj
-
-        self.internal_state['targets'] = self.orm_wrapper.get_or_create_targets(target_information)
-
         # Save build configuration
         data = self.server.runCommand(["getAllKeysWithFlags", ["doc", "func"]])[0]
 
@@ -1082,7 +1131,8 @@ class BuildInfoHelper(object):
                         abs_file_name = vh['file']
                         for pp in path_prefixes:
                             if abs_file_name.startswith(pp + "/"):
-                                vh['file']=abs_file_name[len(pp + "/"):]
+                                # preserve layer name in relative path
+                                vh['file']=abs_file_name[pp.rfind("/")+1:]
                                 break
 
         # save the variables
@@ -1090,9 +1140,48 @@ class BuildInfoHelper(object):
 
         return self.brbe
 
+    def set_recipes_to_parse(self, num_recipes):
+        """
+        Set the number of recipes which need to be parsed for this build.
+        This is set the first time ParseStarted is received by toasterui.
+        """
+        self._ensure_build()
+        self.internal_state['build'].recipes_to_parse = num_recipes
+        self.internal_state['build'].save()
+
+    def set_recipes_parsed(self, num_recipes):
+        """
+        Set the number of recipes parsed so far for this build; this is updated
+        each time a ParseProgress or ParseCompleted event is received by
+        toasterui.
+        """
+        self._ensure_build()
+        if num_recipes <= self.internal_state['build'].recipes_to_parse:
+            self.internal_state['build'].recipes_parsed = num_recipes
+            self.internal_state['build'].save()
+
+    def update_target_image_file(self, event):
+        evdata = BuildInfoHelper._get_data_from_event(event)
+
+        for t in self.internal_state['targets']:
+            if t.is_image == True:
+                output_files = list(evdata.keys())
+                for output in output_files:
+                    if t.target in output and 'rootfs' in output and not output.endswith(".manifest"):
+                        self.orm_wrapper.save_target_image_file_information(t, output, evdata[output])
+
+    def update_artifact_image_file(self, event):
+        self._ensure_build()
+        evdata = BuildInfoHelper._get_data_from_event(event)
+        for artifact_path in evdata.keys():
+            self.orm_wrapper.save_artifact_information(
+                self.internal_state['build'], artifact_path,
+                evdata[artifact_path])
+
     def update_build_information(self, event, errors, warnings, taskfailures):
-        if 'build' in self.internal_state:
-            self.orm_wrapper.update_build_object(self.internal_state['build'], errors, warnings, taskfailures)
+        self._ensure_build()
+        self.orm_wrapper.update_build_stats_and_outcome(
+            self.internal_state['build'], errors, warnings, taskfailures)
 
     def store_started_task(self, event):
         assert isinstance(event, (bb.runqueue.sceneQueueTaskStarted, bb.runqueue.runQueueTaskStarted, bb.runqueue.runQueueTaskSkipped))
@@ -1135,6 +1224,7 @@ class BuildInfoHelper(object):
 
 
     def store_tasks_stats(self, event):
+        self._ensure_build()
         task_data = BuildInfoHelper._get_data_from_event(event)
 
         for (task_file, task_name, task_stats, recipe_name) in task_data:
@@ -1230,6 +1320,8 @@ class BuildInfoHelper(object):
 
 
     def store_target_package_data(self, event):
+        self._ensure_build()
+
         # for all image targets
         for target in self.internal_state['targets']:
             if target.is_image:
@@ -1263,10 +1355,10 @@ class BuildInfoHelper(object):
         note that this only gets called for command line builds which are
         interrupted, so it doesn't touch any BuildRequest objects
         """
-        build = self.internal_state['build']
-        if build:
-            build.outcome = Build.CANCELLED
-            build.save()
+        self._ensure_build()
+        self.internal_state['build'].outcome = Build.CANCELLED
+        self.internal_state['build'].save()
+        signal_runbuilds()
 
     def store_dependency_information(self, event):
         assert '_depgraph' in vars(event)
@@ -1412,6 +1504,8 @@ class BuildInfoHelper(object):
 
 
     def store_build_package_information(self, event):
+        self._ensure_build()
+
         package_info = BuildInfoHelper._get_data_from_event(event)
         self.orm_wrapper.save_build_package_information(
             self.internal_state['build'],
@@ -1427,10 +1521,12 @@ class BuildInfoHelper(object):
 
     def _store_build_done(self, errorcode):
         logger.info("Build exited with errorcode %d", errorcode)
+
+        if not self.brbe:
+            return
+
         br_id, be_id = self.brbe.split(":")
-        be = BuildEnvironment.objects.get(pk = be_id)
-        be.lock = BuildEnvironment.LOCK_LOCK
-        be.save()
+
         br = BuildRequest.objects.get(pk = br_id)
 
         # if we're 'done' because we got cancelled update the build outcome
@@ -1448,6 +1544,10 @@ class BuildInfoHelper(object):
             br.state = BuildRequest.REQ_FAILED
         br.save()
 
+        be = BuildEnvironment.objects.get(pk = be_id)
+        be.lock = BuildEnvironment.LOCK_FREE
+        be.save()
+        signal_runbuilds()
 
     def store_log_error(self, text):
         mockevent = MockEvent()
@@ -1467,24 +1567,22 @@ class BuildInfoHelper(object):
 
 
     def store_log_event(self, event):
+        self._ensure_build()
+
         if event.levelno < formatter.WARNING:
             return
 
         if 'args' in vars(event):
             event.msg = event.msg % event.args
 
-        if not 'build' in self.internal_state:
-            if self.brbe is None:
-                if not 'backlog' in self.internal_state:
-                    self.internal_state['backlog'] = []
-                self.internal_state['backlog'].append(event)
-                return
-            else:   # we're under Toaster control, the build is already created
-                br, _ = self.brbe.split(":")
-                buildrequest = BuildRequest.objects.get(pk = br)
-                self.internal_state['build'] = buildrequest.build
+        # early return for CLI builds
+        if self.brbe is None:
+            if not 'backlog' in self.internal_state:
+                self.internal_state['backlog'] = []
+            self.internal_state['backlog'].append(event)
+            return
 
-        if 'build' in self.internal_state and 'backlog' in self.internal_state:
+        if 'backlog' in self.internal_state:
             # if we have a backlog of events, do our best to save them here
             if len(self.internal_state['backlog']):
                 tempevent = self.internal_state['backlog'].pop()
@@ -1527,7 +1625,10 @@ class BuildInfoHelper(object):
                 if line.startswith('FILES'):
                     files_str = line.split(':')[1].strip()
                     files_str = re.sub(r' {2,}', ' ', files_str)
-                    files = files_str.split(' ')
+
+                    # ignore lines like "FILES:" with no filenames
+                    if files_str:
+                        files += files_str.split(' ')
         return files
 
     def _endswith(self, str_to_test, endings):
@@ -1645,9 +1746,9 @@ class BuildInfoHelper(object):
                 real_image_name,
                 'image_license.manifest')
 
-            # if image_license.manifest exists, we can read the names of bzImage
-            # and modules files for this build from it, then look for them
-            # in the DEPLOY_DIR_IMAGE; note that this file is only produced
+            # if image_license.manifest exists, we can read the names of
+            # bzImage, modules etc. files for this build from it, then look for
+            # them in the DEPLOY_DIR_IMAGE; note that this file is only produced
             # if an image file was produced
             if os.path.isfile(image_license_manifest_path):
                 has_files = True
@@ -1813,18 +1914,12 @@ class BuildInfoHelper(object):
                         sdk_target)
 
     def close(self, errorcode):
-        if self.brbe is not None:
-            self._store_build_done(errorcode)
+        self._store_build_done(errorcode)
 
         if 'backlog' in self.internal_state:
-            if 'build' in self.internal_state:
-                # we save missed events in the database for the current build
-                tempevent = self.internal_state['backlog'].pop()
-                self.store_log_event(tempevent)
-            else:
-                # we have no build, and we still have events; something amazingly wrong happend
-                for event in self.internal_state['backlog']:
-                    logger.error("UNSAVED log: %s", event.msg)
+            # we save missed events in the database for the current build
+            tempevent = self.internal_state['backlog'].pop()
+            self.store_log_event(tempevent)
 
         if not connection.features.autocommits_when_autocommit_is_off:
             transaction.set_autocommit(True)
@@ -1833,3 +1928,7 @@ class BuildInfoHelper(object):
         # being incorrectly attached to the previous Toaster-triggered build;
         # see https://bugzilla.yoctoproject.org/show_bug.cgi?id=9021
         self.brbe = None
+
+        # unset the internal Build object to prevent it being reused for the
+        # next build
+        self.internal_state['build'] = None

bitbake/lib/bb/ui/knotty.py

@@ -40,9 +40,9 @@ logger = logging.getLogger("BitBake")
 interactive = sys.stdout.isatty()
 
 class BBProgress(progressbar.ProgressBar):
-    def __init__(self, msg, maxval, widgets=None):
+    def __init__(self, msg, maxval, widgets=None, extrapos=-1):
         self.msg = msg
-        self.extrapos = -1
+        self.extrapos = extrapos
         if not widgets:
             widgets = [progressbar.Percentage(), ' ', progressbar.Bar(), ' ',
             progressbar.ETA()]
@@ -69,15 +69,16 @@ class BBProgress(progressbar.ProgressBar):
         self.widgets[0] = msg
 
     def setextra(self, extra):
-        if extra:
-            extrastr = str(extra)
-            if extrastr[0] != ' ':
-                extrastr = ' ' + extrastr
-            if extrastr[-1] != ' ':
-                extrastr += ' '
-        else:
-            extrastr = ' '
-        self.widgets[self.extrapos] = extrastr
+        if self.extrapos > -1:
+            if extra:
+                extrastr = str(extra)
+                if extrastr[0] != ' ':
+                    extrastr = ' ' + extrastr
+                if extrastr[-1] != ' ':
+                    extrastr += ' '
+            else:
+                extrastr = ' '
+            self.widgets[self.extrapos] = extrastr
 
     def _need_update(self):
         # We always want the bar to print when update() is called
@@ -169,6 +170,7 @@ class TerminalFilter(object):
         self.interactive = sys.stdout.isatty()
         self.footer_present = False
         self.lastpids = []
+        self.lasttime = None
         self.quiet = quiet
 
         if not self.interactive:
@@ -225,6 +227,10 @@ class TerminalFilter(object):
         activetasks = self.helper.running_tasks
         failedtasks = self.helper.failed_tasks
         runningpids = self.helper.running_pids
+        currenttime = time.time()
+        if not self.lasttime or (currenttime - self.lasttime > 5):
+            self.helper.needUpdate = True
+            self.lasttime = currenttime
         if self.footer_present and not self.helper.needUpdate:
             return
         self.helper.needUpdate = False
@@ -241,31 +247,40 @@ class TerminalFilter(object):
                 start_time = activetasks[t].get("starttime", None)
                 if not pbar or pbar.bouncing != (progress < 0):
                     if progress < 0:
-                        pbar = BBProgress("0: %s (pid %s) " % (activetasks[t]["title"], t), 100, widgets=[progressbar.BouncingSlider()])
+                        pbar = BBProgress("0: %s (pid %s) " % (activetasks[t]["title"], t), 100, widgets=[progressbar.BouncingSlider(), ''], extrapos=2)
                         pbar.bouncing = True
                     else:
-                        pbar = BBProgress("0: %s (pid %s) " % (activetasks[t]["title"], t), 100)
+                        pbar = BBProgress("0: %s (pid %s) " % (activetasks[t]["title"], t), 100, widgets=[progressbar.Percentage(), ' ', progressbar.Bar(), ''], extrapos=4)
                         pbar.bouncing = False
                     activetasks[t]["progressbar"] = pbar
                 tasks.append((pbar, progress, rate, start_time))
             else:
-                tasks.append("%s (pid %s)" % (activetasks[t]["title"], t))
+                start_time = activetasks[t].get("starttime", None)
+                if start_time:
+                    tasks.append("%s - %ds (pid %s)" % (activetasks[t]["title"], currenttime - start_time, t))
+                else:
+                    tasks.append("%s (pid %s)" % (activetasks[t]["title"], t))
 
         if self.main.shutdown:
             content = "Waiting for %s running tasks to finish:" % len(activetasks)
             print(content)
         else:
-            if not len(activetasks):
+            if self.quiet:
+                content = "Running tasks (%s of %s)" % (self.helper.tasknumber_current, self.helper.tasknumber_total)
+            elif not len(activetasks):
                 content = "No currently running tasks (%s of %s)" % (self.helper.tasknumber_current, self.helper.tasknumber_total)
             else:
                 content = "Currently %2s running tasks (%s of %s)" % (len(activetasks), self.helper.tasknumber_current, self.helper.tasknumber_total)
-            maxtask = self.helper.tasknumber_total + 1
+            maxtask = self.helper.tasknumber_total
             if not self.main_progress or self.main_progress.maxval != maxtask:
                 widgets = [' ', progressbar.Percentage(), ' ', progressbar.Bar()]
                 self.main_progress = BBProgress("Running tasks", maxtask, widgets=widgets)
                 self.main_progress.start(False)
             self.main_progress.setmessage(content)
-            self.main_progress.update(self.helper.tasknumber_current)
+            progress = self.helper.tasknumber_current - 1
+            if progress < 0:
+                progress = 0
+            content = self.main_progress.update(progress)
             print('')
         lines = 1 + int(len(content) / (self.columns + 1))
         if not self.quiet:
@@ -279,11 +294,10 @@ class TerminalFilter(object):
                     pbar.setmessage('%s:%s' % (tasknum, pbar.msg.split(':', 1)[1]))
                     if progress > -1:
                         pbar.setextra(rate)
-                        output = pbar.update(progress)
+                        content = pbar.update(progress)
                     else:
-                        output = pbar.update(1)
-                    if not output or (len(output) <= pbar.term_width):
-                        print('')
+                        content = pbar.update(1)
+                    print('')
                 else:
                     content = "%s: %s" % (tasknum, task)
                     print(content)
@@ -582,23 +596,23 @@ def main(server, eventHandler, params, tf = TerminalFilter):
                     tasktype = 'noexec task'
                 else:
                     tasktype = 'task'
-                logger.info("Running %s %s of %s (ID: %s, %s)",
+                logger.info("Running %s %d of %d (%s)",
                             tasktype,
                             event.stats.completed + event.stats.active +
                                 event.stats.failed + 1,
-                            event.stats.total, event.taskid, event.taskstring)
+                            event.stats.total, event.taskstring)
                 continue
 
             if isinstance(event, bb.runqueue.runQueueTaskFailed):
                 return_value = 1
                 taskfailures.append(event.taskstring)
-                logger.error("Task %s (%s) failed with exit code '%s'",
-                             event.taskid, event.taskstring, event.exitcode)
+                logger.error("Task (%s) failed with exit code '%s'",
+                             event.taskstring, event.exitcode)
                 continue
 
             if isinstance(event, bb.runqueue.sceneQueueTaskFailed):
-                logger.warning("Setscene task %s (%s) failed with exit code '%s' - real task will be run instead",
-                               event.taskid, event.taskstring, event.exitcode)
+                logger.warning("Setscene task (%s) failed with exit code '%s' - real task will be run instead",
+                               event.taskstring, event.exitcode)
                 continue
 
             if isinstance(event, bb.event.DepTreeGenerated):

bitbake/lib/bb/ui/toasterui.py

@@ -102,6 +102,7 @@ _evt_list = [
     "bb.command.CommandExit",
     "bb.command.CommandFailed",
     "bb.cooker.CookerExit",
+    "bb.event.BuildInit",
     "bb.event.BuildCompleted",
     "bb.event.BuildStarted",
     "bb.event.CacheLoadCompleted",
@@ -115,6 +116,7 @@ _evt_list = [
     "bb.event.NoProvider",
     "bb.event.ParseCompleted",
     "bb.event.ParseProgress",
+    "bb.event.ParseStarted",
     "bb.event.RecipeParsed",
     "bb.event.SanityCheck",
     "bb.event.SanityCheckPassed",
@@ -231,19 +233,35 @@ def main(server, eventHandler, params):
             # pylint: disable=protected-access
             # the code will look into the protected variables of the event; no easy way around this
 
-            # we treat ParseStarted as the first event of toaster-triggered
-            # builds; that way we get the Build Configuration included in the log
-            # and any errors that occur before BuildStarted is fired
             if isinstance(event, bb.event.ParseStarted):
                 if not (build_log and build_log_file_path):
                     build_log, build_log_file_path = _open_build_log(log_dir)
+
+                buildinfohelper.store_started_build()
+                buildinfohelper.save_build_log_file_path(build_log_file_path)
+                buildinfohelper.set_recipes_to_parse(event.total)
                 continue
 
-            if isinstance(event, bb.event.BuildStarted):
+            # create a build object in buildinfohelper from either BuildInit
+            # (if available) or BuildStarted (for jethro and previous versions)
+            if isinstance(event, (bb.event.BuildStarted, bb.event.BuildInit)):
                 if not (build_log and build_log_file_path):
                     build_log, build_log_file_path = _open_build_log(log_dir)
 
-                buildinfohelper.store_started_build(event, build_log_file_path)
+                buildinfohelper.save_build_targets(event)
+                buildinfohelper.save_build_log_file_path(build_log_file_path)
+
+                # get additional data from BuildStarted
+                if isinstance(event, bb.event.BuildStarted):
+                    buildinfohelper.save_build_layers_and_variables()
+                continue
+
+            if isinstance(event, bb.event.ParseProgress):
+                buildinfohelper.set_recipes_parsed(event.current)
+                continue
+
+            if isinstance(event, bb.event.ParseCompleted):
+                buildinfohelper.set_recipes_parsed(event.total)
                 continue
 
             if isinstance(event, (bb.build.TaskStarted, bb.build.TaskSucceeded, bb.build.TaskFailedSilent)):
@@ -289,10 +307,6 @@ def main(server, eventHandler, params):
             # timing and error informations from the parsing phase in Toaster
             if isinstance(event, (bb.event.SanityCheckPassed, bb.event.SanityCheck)):
                 continue
-            if isinstance(event, bb.event.ParseProgress):
-                continue
-            if isinstance(event, bb.event.ParseCompleted):
-                continue
             if isinstance(event, bb.event.CacheLoadStarted):
                 continue
             if isinstance(event, bb.event.CacheLoadProgress):
@@ -344,8 +358,8 @@ def main(server, eventHandler, params):
             if isinstance(event, bb.runqueue.runQueueTaskFailed):
                 buildinfohelper.update_and_store_task(event)
                 taskfailures.append(event.taskstring)
-                logger.error("Task %s (%s) failed with exit code '%s'",
-                             event.taskid, event.taskstring, event.exitcode)
+                logger.error("Task (%s) failed with exit code '%s'",
+                             event.taskstring, event.exitcode)
                 continue
 
             if isinstance(event, (bb.runqueue.sceneQueueTaskCompleted, bb.runqueue.sceneQueueTaskFailed)):

bitbake/lib/bb/ui/uihelper.py

@@ -35,25 +35,25 @@ class BBUIHelper:
             self.running_tasks[event.pid] = { 'title' : "%s %s" % (event._package, event._task), 'starttime' : time.time() }
             self.running_pids.append(event.pid)
             self.needUpdate = True
-        if isinstance(event, bb.build.TaskSucceeded):
+        elif isinstance(event, bb.build.TaskSucceeded):
             del self.running_tasks[event.pid]
             self.running_pids.remove(event.pid)
             self.needUpdate = True
-        if isinstance(event, bb.build.TaskFailedSilent):
+        elif isinstance(event, bb.build.TaskFailedSilent):
             del self.running_tasks[event.pid]
             self.running_pids.remove(event.pid)
             # Don't add to the failed tasks list since this is e.g. a setscene task failure
             self.needUpdate = True
-        if isinstance(event, bb.build.TaskFailed):
+        elif isinstance(event, bb.build.TaskFailed):
             del self.running_tasks[event.pid]
             self.running_pids.remove(event.pid)
             self.failed_tasks.append( { 'title' : "%s %s" % (event._package, event._task)})
             self.needUpdate = True
-        if isinstance(event, bb.runqueue.runQueueTaskStarted) or isinstance(event, bb.runqueue.sceneQueueTaskStarted):
+        elif isinstance(event, bb.runqueue.runQueueTaskStarted) or isinstance(event, bb.runqueue.sceneQueueTaskStarted):
             self.tasknumber_current = event.stats.completed + event.stats.active + event.stats.failed + 1
             self.tasknumber_total = event.stats.total
             self.needUpdate = True
-        if isinstance(event, bb.build.TaskProgress):
+        elif isinstance(event, bb.build.TaskProgress):
             if event.pid > 0:
                 self.running_tasks[event.pid]['progress'] = event.progress
                 self.running_tasks[event.pid]['rate'] = event.rate
@@ -62,4 +62,3 @@ class BBUIHelper:
     def getTasks(self):
         self.needUpdate = False
         return (self.running_tasks, self.failed_tasks)
-

bitbake/lib/bb/utils.py

@@ -375,6 +375,12 @@ def _print_exception(t, value, tb, realfile, text, context):
             level = level + 1
 
         error.append("Exception: %s" % ''.join(exception))
+
+        # If the exception is from spwaning a task, let's be helpful and display
+        # the output (which hopefully includes stderr).
+        if isinstance(value, subprocess.CalledProcessError):
+            error.append("Subprocess output:")
+            error.append(value.output.decode("utf-8", errors="ignore"))
     finally:
         logger.error("\n".join(error))
 
@@ -1075,7 +1081,7 @@ def edit_metadata(meta_lines, variables, varfunc, match_overrides=False):
                 newlines: list of lines up to this point. You can use
                     this to prepend lines before this variable setting
                     if you wish.
-            and should return a three-element tuple:
+            and should return a four-element tuple:
                 newvalue: new value to substitute in, or None to drop
                     the variable setting entirely. (If the removal
                     results in two consecutive blank lines, one of the
@@ -1089,6 +1095,8 @@ def edit_metadata(meta_lines, variables, varfunc, match_overrides=False):
                     multi-line value to continue on the same line as
                     the assignment, False to indent before the first
                     element.
+            To clarify, if you wish not to change the value, then you
+            would return like this: return origvalue, None, 0, True
         match_overrides: True to match items with _overrides on the end,
             False otherwise
     Returns a tuple:
@@ -1455,7 +1463,8 @@ def export_proxies(d):
     import os
 
     variables = ['http_proxy', 'HTTP_PROXY', 'https_proxy', 'HTTPS_PROXY',
-                    'ftp_proxy', 'FTP_PROXY', 'no_proxy', 'NO_PROXY']
+                    'ftp_proxy', 'FTP_PROXY', 'no_proxy', 'NO_PROXY',
+                    'GIT_PROXY_COMMAND']
     exported = False
 
     for v in variables:

bitbake/lib/bblayers/action.py

@@ -173,7 +173,7 @@ build results (as the layer priority order has effectively changed).
         # have come from)
         first_regex = None
         layerdir = layers[0]
-        for layername, pattern, regex, _ in self.tinfoil.cooker.recipecache.bbfile_config_priorities:
+        for layername, pattern, regex, _ in self.tinfoil.cooker.bbfile_config_priorities:
             if regex.match(os.path.join(layerdir, 'test')):
                 first_regex = regex
                 break

bitbake/lib/bblayers/query.py

@@ -23,7 +23,7 @@ class QueryPlugin(LayerPlugin):
         """show current configured layers."""
         logger.plain("%s  %s  %s" % ("layer".ljust(20), "path".ljust(40), "priority"))
         logger.plain('=' * 74)
-        for layer, _, regex, pri in self.tinfoil.cooker.recipecache.bbfile_config_priorities:
+        for layer, _, regex, pri in self.tinfoil.cooker.bbfile_config_priorities:
             layerdir = self.bbfile_collections.get(layer, None)
             layername = self.get_layer_name(layerdir)
             logger.plain("%s  %s  %d" % (layername.ljust(20), layerdir.ljust(40), pri))
@@ -121,9 +121,9 @@ skipped recipes will also be listed, with a " (skipped)" suffix.
                     logger.error('No class named %s found in BBPATH', classfile)
                     sys.exit(1)
 
-        pkg_pn = self.tinfoil.cooker.recipecache.pkg_pn
-        (latest_versions, preferred_versions) = bb.providers.findProviders(self.tinfoil.config_data, self.tinfoil.cooker.recipecache, pkg_pn)
-        allproviders = bb.providers.allProviders(self.tinfoil.cooker.recipecache)
+        pkg_pn = self.tinfoil.cooker.recipecaches[''].pkg_pn
+        (latest_versions, preferred_versions) = bb.providers.findProviders(self.tinfoil.config_data, self.tinfoil.cooker.recipecaches[''], pkg_pn)
+        allproviders = bb.providers.allProviders(self.tinfoil.cooker.recipecaches[''])
 
         # Ensure we list skipped recipes
         # We are largely guessing about PN, PV and the preferred version here,
@@ -170,13 +170,13 @@ skipped recipes will also be listed, with a " (skipped)" suffix.
 
             if len(allproviders[p]) > 1 or not show_multi_provider_only:
                 pref = preferred_versions[p]
-                realfn = bb.cache.Cache.virtualfn2realfn(pref[1])
+                realfn = bb.cache.virtualfn2realfn(pref[1])
                 preffile = realfn[0]
 
                 # We only display once per recipe, we should prefer non extended versions of the
                 # recipe if present (so e.g. in OpenEmbedded, openssl rather than nativesdk-openssl
                 # which would otherwise sort first).
-                if realfn[1] and realfn[0] in self.tinfoil.cooker.recipecache.pkg_fn:
+                if realfn[1] and realfn[0] in self.tinfoil.cooker.recipecaches[''].pkg_fn:
                     continue
 
                 if inherits:
@@ -200,7 +200,7 @@ skipped recipes will also be listed, with a " (skipped)" suffix.
                     same_ver = True
                     provs = []
                     for prov in allproviders[p]:
-                        provfile = bb.cache.Cache.virtualfn2realfn(prov[1])[0]
+                        provfile = bb.cache.virtualfn2realfn(prov[1])[0]
                         provlayer = self.get_file_layer(provfile)
                         provs.append((provfile, provlayer, prov[0]))
                         if provlayer != preflayer:
@@ -297,7 +297,7 @@ Lists recipes with the bbappends that apply to them as subitems.
     def get_appends_for_files(self, filenames):
         appended, notappended = [], []
         for filename in filenames:
-            _, cls = bb.cache.Cache.virtualfn2realfn(filename)
+            _, cls, _ = bb.cache.virtualfn2realfn(filename)
             if cls:
                 continue
 
@@ -328,7 +328,7 @@ NOTE: .bbappend files can impact the dependencies.
 
         # The bb's DEPENDS and RDEPENDS
         for f in pkg_fn:
-            f = bb.cache.Cache.virtualfn2realfn(f)[0]
+            f = bb.cache.virtualfn2realfn(f)[0]
             # Get the layername that the file is in
             layername = self.get_file_layer(f)
 
@@ -471,7 +471,7 @@ NOTE: .bbappend files can impact the dependencies.
 
     def check_cross_depends(self, keyword, layername, f, needed_file, show_filenames, ignore_layers):
         """Print the DEPENDS/RDEPENDS file that crosses a layer boundary"""
-        best_realfn = bb.cache.Cache.virtualfn2realfn(needed_file)[0]
+        best_realfn = bb.cache.virtualfn2realfn(needed_file)[0]
         needed_layername = self.get_file_layer(best_realfn)
         if needed_layername != layername and not needed_layername in ignore_layers:
             if not show_filenames:

bitbake/lib/toaster/bldcollector/admin.py

@@ -1,33 +1,33 @@
 from django.contrib import admin
-from django.contrib.admin.filters import RelatedFieldListFilter
-from orm.models import BitbakeVersion, Release, LayerSource, ToasterSetting
-from django.forms.widgets import Textarea
+from orm.models import BitbakeVersion, Release, ToasterSetting, Layer_Version
 from django import forms
 import django.db.models as models
 
-from django.contrib.admin import widgets, helpers
-
-class LayerSourceAdmin(admin.ModelAdmin):
-    pass
 
 class BitbakeVersionAdmin(admin.ModelAdmin):
 
-    # we override the formfield for db URLField because of broken URL validation
+    # we override the formfield for db URLField
+    # because of broken URL validation
 
     def formfield_for_dbfield(self, db_field, **kwargs):
         if isinstance(db_field, models.fields.URLField):
             return forms.fields.CharField()
-        return super(BitbakeVersionAdmin, self).formfield_for_dbfield(db_field, **kwargs)
-
+        return super(BitbakeVersionAdmin, self).formfield_for_dbfield(
+            db_field, **kwargs)
 
 
 class ReleaseAdmin(admin.ModelAdmin):
     pass
 
+
 class ToasterSettingAdmin(admin.ModelAdmin):
     pass
 
-admin.site.register(LayerSource, LayerSourceAdmin)
+
+class LayerVersionsAdmin(admin.ModelAdmin):
+    pass
+
+admin.site.register(Layer_Version, LayerVersionsAdmin)
 admin.site.register(BitbakeVersion, BitbakeVersionAdmin)
 admin.site.register(Release, ReleaseAdmin)
 admin.site.register(ToasterSetting, ToasterSettingAdmin)

bitbake/lib/toaster/bldcontrol/localhostbecontroller.py

@@ -89,19 +89,34 @@ class LocalhostBEController(BuildEnvironmentController):
         """ a word of attention: by convention, the first layer for any build will be poky! """
 
         assert self.be.sourcedir is not None
+
+        layerlist = []
+        nongitlayerlist = []
+
         # set layers in the layersource
 
         # 1. get a list of repos with branches, and map dirpaths for each layer
         gitrepos = {}
 
-        gitrepos[(bitbake.giturl, bitbake.commit)] = []
-        gitrepos[(bitbake.giturl, bitbake.commit)].append( ("bitbake", bitbake.dirpath) )
+        # if we're using a remotely fetched version of bitbake add its git
+        # details to the list of repos to clone
+        if bitbake.giturl and bitbake.commit:
+            gitrepos[(bitbake.giturl, bitbake.commit)] = []
+            gitrepos[(bitbake.giturl, bitbake.commit)].append(
+                ("bitbake", bitbake.dirpath))
 
         for layer in layers:
             # We don't need to git clone the layer for the CustomImageRecipe
             # as it's generated by us layer on if needed
             if CustomImageRecipe.LAYER_NAME in layer.name:
                 continue
+
+            # If we have local layers then we don't need clone them
+            # For local layers giturl will be empty
+            if not layer.giturl:
+                nongitlayerlist.append(layer.layer_version.layer.local_source_dir)
+                continue
+
             if not (layer.giturl, layer.commit) in gitrepos:
                 gitrepos[(layer.giturl, layer.commit)] = []
             gitrepos[(layer.giturl, layer.commit)].append( (layer.name, layer.dirpath) )
@@ -131,19 +146,22 @@ class LocalhostBEController(BuildEnvironmentController):
 
         logger.info("Using pre-checked out source for layer %s", cached_layers)
 
-        layerlist = []
-
-
         # 3. checkout the repositories
         for giturl, commit in gitrepos.keys():
             localdirname = os.path.join(self.be.sourcedir, self.getGitCloneDirectory(giturl, commit))
             logger.debug("localhostbecontroller: giturl %s:%s checking out in current directory %s" % (giturl, commit, localdirname))
 
-            # make sure our directory is a git repository
+            # see if our directory is a git repository
             if os.path.exists(localdirname):
-                localremotes = self._shellcmd("git remote -v", localdirname)
-                if not giturl in localremotes:
-                    raise BuildSetupException("Existing git repository at %s, but with different remotes ('%s', expected '%s'). Toaster will not continue out of fear of damaging something." % (localdirname, ", ".join(localremotes.split("\n")), giturl))
+                try:
+                    localremotes = self._shellcmd("git remote -v",
+                                                  localdirname)
+                    if not giturl in localremotes and commit != 'HEAD':
+                        raise BuildSetupException("Existing git repository at %s, but with different remotes ('%s', expected '%s'). Toaster will not continue out of fear of damaging something." % (localdirname, ", ".join(localremotes.split("\n")), giturl))
+                except ShellCmdException:
+                    # our localdirname might not be a git repository
+                    #- that's fine
+                    pass
             else:
                 if giturl in cached_layers:
                     logger.debug("localhostbecontroller git-copying %s to %s" % (cached_layers[giturl], localdirname))
@@ -245,6 +263,7 @@ class LocalhostBEController(BuildEnvironmentController):
             layerlist.append(layerpath)
 
         self.islayerset = True
+        layerlist.extend(nongitlayerlist)
         return layerlist
 
     def readServerLogFile(self):

bitbake/lib/toaster/bldcontrol/management/commands/checksettings.py

@@ -1,10 +1,15 @@
 from django.core.management.base import NoArgsCommand, CommandError
 from django.db import transaction
+
+from django.core.management import call_command
 from bldcontrol.bbcontroller import getBuildEnvironmentController, ShellCmdException
 from bldcontrol.models import BuildRequest, BuildEnvironment, BRError
-from orm.models import ToasterSetting, Build
+from orm.models import ToasterSetting, Build, Layer
+
 import os
 import traceback
+import warnings
+
 
 def DN(path):
     if path is None:
@@ -21,39 +26,6 @@ class Command(NoArgsCommand):
         super(Command, self).__init__(*args, **kwargs)
         self.guesspath = DN(DN(DN(DN(DN(DN(DN(__file__)))))))
 
-    def _find_first_path_for_file(self, startdirectory, filename, level=0):
-        if level < 0:
-            return None
-        dirs = []
-        for i in os.listdir(startdirectory):
-            j = os.path.join(startdirectory, i)
-            if os.path.isfile(j):
-                if i == filename:
-                    return startdirectory
-            elif os.path.isdir(j):
-                dirs.append(j)
-        for j in dirs:
-            ret = self._find_first_path_for_file(j, filename, level - 1)
-            if ret is not None:
-                return ret
-        return None
-
-    def _recursive_list_directories(self, startdirectory, level=0):
-        if level < 0:
-            return []
-        dirs = []
-        try:
-            for i in os.listdir(startdirectory):
-                j = os.path.join(startdirectory, i)
-                if os.path.isdir(j):
-                    dirs.append(j)
-        except OSError:
-            pass
-        for j in dirs:
-            dirs = dirs + self._recursive_list_directories(j, level - 1)
-        return dirs
-
-
     def _verify_build_environment(self):
         # provide a local build env. This will be extended later to include non local
         if BuildEnvironment.objects.count() == 0:
@@ -94,30 +66,60 @@ class Command(NoArgsCommand):
                     print("\n -- Validation: The build directory must to be set to an absolute path.")
                     is_changed = _update_builddir()
 
-
                 if is_changed:
                     print("\nBuild configuration saved")
                     be.save()
                     return True
 
-
                 if be.needs_import:
                     try:
-                        config_file = os.environ.get('TOASTER_CONF')
-                        print("\nImporting file: %s" % config_file)
-                        from .loadconf import Command as LoadConfigCommand
+                        print("Loading default settings")
+                        call_command("loaddata", "settings")
+                        template_conf = os.environ.get("TEMPLATECONF", "")
+
+                        if "poky" in template_conf:
+                            print("Loading poky configuration")
+                            call_command("loaddata", "poky")
+                        else:
+                            print("Loading OE-Core configuration")
+                            call_command("loaddata", "oe-core")
+                            if template_conf:
+                                oe_core_path = os.path.realpath(
+                                    template_conf +
+                                    "/../")
+                            else:
+                                print("TEMPLATECONF not found. You may have to"
+                                      " manually configure layer paths")
+                                oe_core_path = input("Please enter the path of"
+                                                     " your openembedded-core "
+                                                     "layer: ")
+                            # Update the layer instances of openemebedded-core
+                            for layer in Layer.objects.filter(
+                                    name="openembedded-core",
+                                    local_source_dir="OE-CORE-LAYER-DIR"):
+                                layer.local_path = oe_core_path
+                                layer.save()
+
+                        # Import the custom fixture if it's present
+                        with warnings.catch_warnings():
+                            warnings.filterwarnings(
+                                action="ignore",
+                                message="^.*No fixture named.*$")
+                            print("Importing custom settings if present")
+                            call_command("loaddata", "custom")
 
-                        LoadConfigCommand()._import_layer_config(config_file)
                         # we run lsupdates after config update
-                        print("\nLayer configuration imported. Updating information from the layer sources, please wait.\nYou can re-update any time later by running bitbake/lib/toaster/manage.py lsupdates")
-                        from django.core.management import call_command
+                        print("\nFetching information from the layer index, "
+                              "please wait.\nYou can re-update any time later "
+                              "by running bitbake/lib/toaster/manage.py "
+                              "lsupdates\n")
                         call_command("lsupdates")
 
                         # we don't look for any other config files
                         return is_changed
                     except Exception as e:
-                        print("Failure while trying to import the toaster config file %s: %s" %\
-                            (config_file, e))
+                        print("Failure while trying to setup toaster: %s"
+                              % e)
                         traceback.print_exc()
 
                 return is_changed

bitbake/lib/toaster/bldcontrol/management/commands/loadconf.py

@@ -1,183 +0,0 @@
-from django.core.management.base import BaseCommand, CommandError
-from orm.models import LayerSource, ToasterSetting, Branch, Layer, Layer_Version
-from orm.models import BitbakeVersion, Release, ReleaseDefaultLayer, ReleaseLayerSourcePriority
-from django.db import IntegrityError
-import os
-
-from .checksettings import DN
-
-import logging
-logger = logging.getLogger("toaster")
-
-def _reduce_canon_path(path):
-    components = []
-    for c in path.split("/"):
-        if c == "..":
-            del components[-1]
-        elif c == ".":
-            pass
-        else:
-            components.append(c)
-    if len(components) < 2:
-        components.append('')
-    return "/".join(components)
-
-def _get_id_for_sourcetype(s):
-    for i in LayerSource.SOURCE_TYPE:
-        if s == i[1]:
-            return i[0]
-    raise Exception("Could not find definition for sourcetype '%s'. Valid source types are %s" % (str(s), ', '.join(map(lambda x: "'%s'" % x[1], LayerSource.SOURCE_TYPE ))))
-
-class Command(BaseCommand):
-    help = "Loads a toasterconf.json file in the database"
-    args = "filepath"
-
-
-
-    def _import_layer_config(self, filepath):
-        if not os.path.exists(filepath) or not os.path.isfile(filepath):
-            raise Exception("Failed to find toaster config file %s ." % filepath)
-
-        import json
-        data = json.loads(open(filepath, "r").read())
-
-        # verify config file validity before updating settings
-        for i in ['bitbake', 'releases', 'defaultrelease', 'config', 'layersources']:
-            assert i in data
-
-        def _read_git_url_from_local_repository(address):
-            url = None
-            # we detect the remote name at runtime
-            import subprocess
-            (remote, remote_name) = address.split(":", 1)
-            cmd = subprocess.Popen("git remote -v", shell=True, cwd = os.path.dirname(filepath), stdout=subprocess.PIPE, stderr = subprocess.PIPE)
-            (out,err) = cmd.communicate()
-            if cmd.returncode != 0:
-                logging.warning("Error while importing layer vcs_url: git error: %s" % err)
-            for line in out.decode('utf-8').split("\n"):
-                try:
-                    (name, path) = line.split("\t", 1)
-                    if name == remote_name:
-                        url = path.split(" ")[0]
-                        break
-                except ValueError:
-                    pass
-            if url == None:
-                logging.warning("Error while looking for remote \"%s\" in \"%s\"" % (remote_name, out))
-            return url
-
-
-        # import bitbake data
-        for bvi in data['bitbake']:
-            bvo, created = BitbakeVersion.objects.get_or_create(name=bvi['name'])
-            if bvi['giturl'].startswith("remote:"):
-                bvo.giturl = _read_git_url_from_local_repository(bvi['giturl'])
-                if bvo.giturl is None:
-                    logger.error("The toaster config file references the local git repo, but Toaster cannot detect it.\nYour local configuration for bitbake version %s is invalid. Make sure that the toasterconf.json file is correct." % bvi['name'])
-
-            if bvo.giturl is None:
-                bvo.giturl = bvi['giturl']
-            bvo.branch = bvi['branch']
-            bvo.dirpath = bvi['dirpath']
-            bvo.save()
-
-        # set the layer sources
-        for lsi in data['layersources']:
-            assert 'sourcetype' in lsi
-            assert 'apiurl' in lsi
-            assert 'name' in lsi
-            assert 'branches' in lsi
-
-
-            if _get_id_for_sourcetype(lsi['sourcetype']) == LayerSource.TYPE_LAYERINDEX or lsi['apiurl'].startswith("/"):
-                apiurl = lsi['apiurl']
-            else:
-                apiurl = _reduce_canon_path(os.path.join(DN(os.path.abspath(filepath)), lsi['apiurl']))
-
-            assert ((_get_id_for_sourcetype(lsi['sourcetype']) == LayerSource.TYPE_LAYERINDEX) or apiurl.startswith("/")), (lsi['sourcetype'],apiurl)
-
-            try:
-                ls, created = LayerSource.objects.get_or_create(sourcetype = _get_id_for_sourcetype(lsi['sourcetype']), apiurl = apiurl)
-                ls.name = lsi['name']
-                ls.save()
-            except IntegrityError as e:
-                logger.warning("IntegrityError %s \nWhile setting name %s for layer source %s " % (e, lsi['name'], ls))
-
-
-            layerbranches = []
-            for branchname in lsi['branches']:
-                bo, created = Branch.objects.get_or_create(layer_source = ls, name = branchname)
-                layerbranches.append(bo)
-
-            if 'layers' in lsi:
-                for layerinfo in lsi['layers']:
-                    lo, created = Layer.objects.get_or_create(layer_source = ls, name = layerinfo['name'])
-                    if layerinfo['local_path'].startswith("/"):
-                        lo.local_path = layerinfo['local_path']
-                    else:
-                        lo.local_path = _reduce_canon_path(os.path.join(ls.apiurl, layerinfo['local_path']))
-
-                    if not os.path.exists(lo.local_path):
-                        logger.error("Local layer path %s must exists. Are you trying to import a layer that does not exist ? Check your local toasterconf.json" % lo.local_path)
-
-                    if layerinfo['vcs_url'].startswith("remote:"):
-                        lo.vcs_url = _read_git_url_from_local_repository(layerinfo['vcs_url'])
-                        if lo.vcs_url is None:
-                            logger.error("The toaster config file references the local git repo, but Toaster cannot detect it.\nYour local configuration for layer %s is invalid. Make sure that the toasterconf.json file is correct." % layerinfo['name'])
-
-                    if lo.vcs_url is None:
-                        lo.vcs_url = layerinfo['vcs_url']
-
-                    if 'layer_index_url' in layerinfo:
-                        lo.layer_index_url = layerinfo['layer_index_url']
-                    lo.save()
-
-                    for branch in layerbranches:
-                        lvo, created = Layer_Version.objects.get_or_create(layer_source = ls,
-                                up_branch = branch,
-                                commit = branch.name,
-                                layer = lo)
-                        lvo.dirpath = layerinfo['dirpath']
-                        lvo.save()
-        # set releases
-        for ri in data['releases']:
-            bvo = BitbakeVersion.objects.get(name = ri['bitbake'])
-            assert bvo is not None
-
-            ro, created = Release.objects.get_or_create(name = ri['name'], bitbake_version = bvo, branch_name = ri['branch'])
-            ro.description = ri['description']
-            ro.helptext = ri['helptext']
-            ro.save()
-
-            # save layer source priority for release
-            for ls_name in ri['layersourcepriority'].keys():
-                rlspo, created = ReleaseLayerSourcePriority.objects.get_or_create(release = ro, layer_source = LayerSource.objects.get(name=ls_name))
-                rlspo.priority = ri['layersourcepriority'][ls_name]
-                rlspo.save()
-
-            for dli in ri['defaultlayers']:
-                # find layers with the same name
-                ReleaseDefaultLayer.objects.get_or_create( release = ro, layer_name = dli)
-
-        # set default release
-        if ToasterSetting.objects.filter(name = "DEFAULT_RELEASE").count() > 0:
-            ToasterSetting.objects.filter(name = "DEFAULT_RELEASE").update(value = data['defaultrelease'])
-        else:
-            ToasterSetting.objects.create(name = "DEFAULT_RELEASE", value = data['defaultrelease'])
-
-        # set default config variables
-        for configname in data['config']:
-            if ToasterSetting.objects.filter(name = "DEFCONF_" + configname).count() > 0:
-                ToasterSetting.objects.filter(name = "DEFCONF_" + configname).update(value = data['config'][configname])
-            else:
-                ToasterSetting.objects.create(name = "DEFCONF_" + configname, value = data['config'][configname])
-
-
-    def handle(self, *args, **options):
-        if len(args) == 0:
-            raise CommandError("Need a path to the toasterconf.json file")
-        filepath = args[0]
-        self._import_layer_config(filepath)
-
-
-

bitbake/lib/toaster/bldcontrol/management/commands/runbuilds.py

@@ -1,31 +1,27 @@
-from django.core.management.base import NoArgsCommand, CommandError
+from django.core.management.base import NoArgsCommand
 from django.db import transaction
 from django.db.models import Q
 
 from bldcontrol.bbcontroller import getBuildEnvironmentController
-from bldcontrol.bbcontroller import ShellCmdException, BuildSetupException
 from bldcontrol.models import BuildRequest, BuildEnvironment
 from bldcontrol.models import BRError, BRVariable
 
-from orm.models import Build, ToasterSetting, LogMessage, Target
+from orm.models import Build, LogMessage, Target
 
-import os
 import logging
-import time
-import sys
 import traceback
+import signal
 
 logger = logging.getLogger("toaster")
 
 class Command(NoArgsCommand):
-    args    = ""
-    help    = "Schedules and executes build requests as possible."
-    "Does not return (interrupt with Ctrl-C)"
-
+    args = ""
+    help = "Schedules and executes build requests as possible. "\
+           "Does not return (interrupt with Ctrl-C)"
 
     @transaction.atomic
     def _selectBuildEnvironment(self):
-        bec = getBuildEnvironmentController(lock = BuildEnvironment.LOCK_FREE)
+        bec = getBuildEnvironmentController(lock=BuildEnvironment.LOCK_FREE)
         bec.be.lock = BuildEnvironment.LOCK_LOCK
         bec.be.save()
         return bec
@@ -75,17 +71,15 @@ class Command(NoArgsCommand):
             else:
                 errmsg = str(e)
 
-            BRError.objects.create(req = br,
-                    errtype = str(type(e)),
-                    errmsg = errmsg,
-                    traceback = traceback.format_exc())
+            BRError.objects.create(req=br, errtype=str(type(e)), errmsg=errmsg,
+                                   traceback=traceback.format_exc())
             br.state = BuildRequest.REQ_FAILED
             br.save()
             bec.be.lock = BuildEnvironment.LOCK_FREE
             bec.be.save()
 
     def archive(self):
-        for br in BuildRequest.objects.filter(state = BuildRequest.REQ_ARCHIVE):
+        for br in BuildRequest.objects.filter(state=BuildRequest.REQ_ARCHIVE):
             if br.build == None:
                 br.state = BuildRequest.REQ_FAILED
             else:
@@ -102,14 +96,13 @@ class Command(NoArgsCommand):
                                        BuildRequest.REQ_COMPLETED,
                                        BuildRequest.REQ_CANCELLING]) &
             Q(lock=BuildEnvironment.LOCK_LOCK) &
-            Q(updated__lt=timezone.now() - timedelta(seconds = 30))
+            Q(updated__lt=timezone.now() - timedelta(seconds=30))
         ).update(lock=BuildEnvironment.LOCK_FREE)
 
 
         # update all Builds that were in progress and failed to start
-        for br in BuildRequest.objects.filter(
-            state=BuildRequest.REQ_FAILED,
-            build__outcome=Build.IN_PROGRESS):
+        for br in BuildRequest.objects.filter(state=BuildRequest.REQ_FAILED,
+                                              build__outcome=Build.IN_PROGRESS):
             # transpose the launch errors in ToasterExceptions
             br.build.outcome = Build.FAILED
             for brerror in br.brerror_set.all():
@@ -126,7 +119,7 @@ class Command(NoArgsCommand):
 
 
         # update all BuildRequests without a build created
-        for br in BuildRequest.objects.filter(build = None):
+        for br in BuildRequest.objects.filter(build=None):
             br.build = Build.objects.create(project=br.project,
                                             completed_on=br.updated,
                                             started_on=br.created)
@@ -151,10 +144,10 @@ class Command(NoArgsCommand):
 
         # Make sure the LOCK is removed for builds which have been fully
         # cancelled
-        for br in BuildRequest.objects.filter(
-            Q(build__outcome=Build.CANCELLED) &
-            Q(state=BuildRequest.REQ_CANCELLING) &
-            ~Q(environment=None)):
+        for br in BuildRequest.objects.filter(\
+                      Q(build__outcome=Build.CANCELLED) &
+                      Q(state=BuildRequest.REQ_CANCELLING) &
+                      ~Q(environment=None)):
             br.environment.lock = BuildEnvironment.LOCK_FREE
             br.environment.save()
 
@@ -175,6 +168,10 @@ class Command(NoArgsCommand):
             logger.warn("runbuilds: schedule exception %s" % str(e))
 
     def handle_noargs(self, **options):
+        self.runbuild()
+
+        signal.signal(signal.SIGUSR1, lambda sig, frame: None)
+
         while True:
+            signal.pause()
             self.runbuild()
-            time.sleep(1)

bitbake/lib/toaster/bldcontrol/migrations/0005_reorder_buildrequest_states.py

@@ -0,0 +1,19 @@
+# -*- coding: utf-8 -*-
+from __future__ import unicode_literals
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('bldcontrol', '0004_auto_20160523_1446'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='buildrequest',
+            name='state',
+            field=models.IntegerField(choices=[(0, 'created'), (1, 'queued'), (2, 'in progress'), (3, 'failed'), (4, 'deleted'), (5, 'cancelling'), (6, 'completed'), (7, 'archive')], default=0),
+        ),
+    ]

bitbake/lib/toaster/bldcontrol/migrations/0006_brlayer_local_source_dir.py

@@ -0,0 +1,19 @@
+# -*- coding: utf-8 -*-
+from __future__ import unicode_literals
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('bldcontrol', '0005_reorder_buildrequest_states'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='brlayer',
+            name='local_source_dir',
+            field=models.CharField(max_length=254, null=True),
+        ),
+    ]

bitbake/lib/toaster/bldcontrol/migrations/0007_brlayers_optional_gitinfo.py

@@ -0,0 +1,29 @@
+# -*- coding: utf-8 -*-
+from __future__ import unicode_literals
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('bldcontrol', '0006_brlayer_local_source_dir'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='brlayer',
+            name='commit',
+            field=models.CharField(max_length=254, null=True),
+        ),
+        migrations.AlterField(
+            model_name='brlayer',
+            name='dirpath',
+            field=models.CharField(max_length=254, null=True),
+        ),
+        migrations.AlterField(
+            model_name='brlayer',
+            name='giturl',
+            field=models.CharField(max_length=254, null=True),
+        ),
+    ]

bitbake/lib/toaster/bldcontrol/models.py

@@ -63,20 +63,20 @@ class BuildRequest(models.Model):
     REQ_CREATED = 0
     REQ_QUEUED = 1
     REQ_INPROGRESS = 2
-    REQ_COMPLETED = 3
-    REQ_FAILED = 4
-    REQ_DELETED = 5
-    REQ_CANCELLING = 6
+    REQ_FAILED = 3
+    REQ_DELETED = 4
+    REQ_CANCELLING = 5
+    REQ_COMPLETED = 6
     REQ_ARCHIVE = 7
 
     REQUEST_STATE = (
         (REQ_CREATED, "created"),
         (REQ_QUEUED, "queued"),
         (REQ_INPROGRESS, "in progress"),
-        (REQ_COMPLETED, "completed"),
         (REQ_FAILED, "failed"),
         (REQ_DELETED, "deleted"),
         (REQ_CANCELLING, "cancelling"),
+        (REQ_COMPLETED, "completed"),
         (REQ_ARCHIVE, "archive"),
     )
 
@@ -91,7 +91,7 @@ class BuildRequest(models.Model):
 
     def __init__(self, *args, **kwargs):
         super(BuildRequest, self).__init__(*args, **kwargs)
-        # Save the old state incase it's about to be modified
+        # Save the old state in case it's about to be modified
         self.old_state = self.state
 
     def save(self, *args, **kwargs):
@@ -126,12 +126,14 @@ class BuildRequest(models.Model):
 # These tables specify the settings for running an actual build.
 # They MUST be kept in sync with the tables in orm.models.Project*
 
+
 class BRLayer(models.Model):
-    req         = models.ForeignKey(BuildRequest)
-    name        = models.CharField(max_length = 100)
-    giturl      = models.CharField(max_length = 254)
-    commit      = models.CharField(max_length = 254)
-    dirpath     = models.CharField(max_length = 254)
+    req = models.ForeignKey(BuildRequest)
+    name = models.CharField(max_length=100)
+    giturl = models.CharField(max_length=254, null=True)
+    local_source_dir = models.CharField(max_length=254, null=True)
+    commit = models.CharField(max_length=254, null=True)
+    dirpath = models.CharField(max_length=254, null=True)
     layer_version = models.ForeignKey(Layer_Version, null=True)
 
 class BRBitbake(models.Model):

bitbake/lib/toaster/bldcontrol/tests.py

@@ -139,22 +139,3 @@ class RunBuildsCommandTests(TestCase):
         self.assertTrue(br.state == BuildRequest.REQ_INPROGRESS, "Request is not updated")
         # no more selections possible here
         self.assertRaises(IndexError, command._selectBuildRequest)
-
-
-class UtilityTests(TestCase):
-    def test_reduce_path(self):
-        from bldcontrol.management.commands.loadconf import _reduce_canon_path, _get_id_for_sourcetype
-
-        self.assertTrue( _reduce_canon_path("/") == "/")
-        self.assertTrue( _reduce_canon_path("/home/..") == "/")
-        self.assertTrue( _reduce_canon_path("/home/../ana") == "/ana")
-        self.assertTrue( _reduce_canon_path("/home/../ana/..") == "/")
-        self.assertTrue( _reduce_canon_path("/home/ana/mihai/../maria") == "/home/ana/maria")
-
-    def test_get_id_for_sorucetype(self):
-        from bldcontrol.management.commands.loadconf import _reduce_canon_path, _get_id_for_sourcetype
-        self.assertTrue( _get_id_for_sourcetype("layerindex") == 1)
-        self.assertTrue( _get_id_for_sourcetype("local") == 0)
-        self.assertTrue( _get_id_for_sourcetype("imported") == 2)
-        with self.assertRaises(Exception):
-            _get_id_for_sourcetype("unknown")

bitbake/lib/toaster/orm/fixtures/README

@@ -0,0 +1,30 @@
+# Fixtures directory
+
+Fixtures are data dumps that can be loaded into Toaster's database to provide
+configuration and data.
+
+In this directory we have the fixtures which are loaded the first time you start Toaster.
+This is to provide useful default values and metadata to Toaster.
+
+ - settings.xml This Contains Toaster wide settings, such as the default values for
+   certain bitbake variables.
+
+ - poky.xml This is the default release data for supported poky based setup 
+
+ - oe-core.xml This is the default release data for supported oe-core based setups
+
+# Custom data/configuration
+
+  - custom.xml
+
+To add custom initial data/configuration to Toaster place a file called
+"custom.xml" in this directory. If present it will be loaded into the database.
+We suggest that this is used to overlay any configuration already done.
+All objects loaded with the same primary keys overwrite the existing data.
+Data can be provided in XML, JSON and if installed YAML formats.
+
+# To load data at any point in time
+
+Use the django management command manage.py loaddata <your fixture file>
+For further information see the Django command documentation at:
+https://docs.djangoproject.com/en/1.8/ref/django-admin/#django-admin-loaddata

bitbake/lib/toaster/orm/fixtures/oe-core.xml

@@ -0,0 +1,59 @@
+<?xml version="1.0" encoding="utf-8"?>
+<django-objects version="1.0">
+  <!-- Set the project default value for DISTRO -->
+  <object model="orm.toastersetting" pk="1">
+    <field type="CharField" name="name">DEFCONF_DISTRO</field>
+    <field type="CharField" name="value">nodistro</field>
+  </object>
+
+  <!-- Bitbake versions which correspond to the metadata release -->
+  <object model="orm.bitbakeversion" pk="1">
+    <field type="CharField" name="name">master</field>
+    <field type="CharField" name="giturl">git://git.openembedded.org/bitbake</field>
+    <field type="CharField" name="branch">master</field>
+  </object>
+  <object model="orm.bitbakeversion" pk="2">
+    <field type="CharField" name="name">HEAD</field>
+  </object>
+
+  <!-- Releases available -->
+  <object model="orm.release" pk="1">
+    <field type="CharField" name="name">master</field>
+    <field type="CharField" name="description">Openembedded master</field>
+    <field rel="ManyToOneRel" to="orm.bitbakeversion" name="bitbake_version">1</field>
+    <field type="CharField" name="branch_name">master</field>
+    <field type="TextField" name="helptext">Toaster will run your builds using the tip of the &lt;a href=\"http://cgit.openembedded.org/openembedded-core/log/\"&gt;OpenEmbedded master&lt;/a&gt; branch, where active development takes place. This is not a stable branch, so your builds might not work as expected.</field>
+  </object>
+  <object model="orm.release" pk="2">
+    <field type="CharField" name="name">local</field>
+    <field type="CharField" name="description">Local Openembedded</field>
+    <field rel="ManyToOneRel" to="orm.bitbakeversion" name="bitbake_version">2</field>
+    <field type="CharField" name="branch_name">HEAD</field>
+    <field type="TextField" name="helptext">Toaster will run your builds with the version of OpenEmbedded that you have cloned or downloaded to your computer.</field>
+  </object>
+
+  <!-- Default layers for each release -->
+  <object model="orm.releasedefaultlayer" pk="1">
+    <field rel="ManyToOneRel" to="orm.release" name="release">1</field>
+    <field type="CharField" name="layer_name">openembedded-core</field>
+  </object>
+  <object model="orm.releasedefaultlayer" pk="4">
+    <field rel="ManyToOneRel" to="orm.release" name="release">2</field>
+    <field type="CharField" name="layer_name">openembedded-core</field>
+  </object>
+  
+  <!-- TYPE_LOCAL = 0 Layers for the Local release -->
+  <object model="orm.layer" pk="1">
+    <field type="CharField" name="name">openembedded-core</field>
+    <field type="CharField" name="vcs_url">git://git.openembedded.org/openembedded-core</field>
+  </object> 
+  <object model="orm.layer_version" pk="1">
+    <field rel="ManyToOneRel" to="orm.layer" name="layer">1</field>
+    <field rel="ManyToOneRel" to="orm.release" name="release">2</field>
+    <field type="CharField" name="local_path">OE-CORE-LAYER-DIR</field>
+    <field type="CharField" name="branch">HEAD</field>
+    <field type="CharField" name="dirpath">meta</field>
+    <field type="IntegerField" name="layer_source">0</field>
+  </object>
+
+</django-objects>

bitbake/lib/toaster/orm/fixtures/poky.xml

@@ -0,0 +1,111 @@
+<?xml version="1.0" encoding="utf-8"?>
+<django-objects version="1.0">
+  <!-- Set the project default value for DISTRO -->
+  <object model="orm.toastersetting" pk="1">
+    <field type="CharField" name="name">DEFCONF_DISTRO</field>
+    <field type="CharField" name="value">poky</field>
+  </object>
+
+  <!-- Bitbake versions which correspond to the metadata release -->
+  <object model="orm.bitbakeversion" pk="1">
+    <field type="CharField" name="name">master</field>
+    <field type="CharField" name="giturl">git://git.yoctoproject.org/poky</field>
+    <field type="CharField" name="branch">master</field>
+    <field type="CharField" name="dirpath">bitbake</field>
+  </object>
+  <object model="orm.bitbakeversion" pk="2">
+    <field type="CharField" name="name">HEAD</field>
+    <field type="CharField" name="giturl">git://git.yoctoproject.org/poky</field>
+    <field type="CharField" name="branch">HEAD</field>
+    <field type="CharField" name="dirpath">bitbake</field>
+  </object>
+
+  <!-- Releases available -->
+  <object model="orm.release" pk="1">
+    <field type="CharField" name="name">master</field>
+    <field type="CharField" name="description">Yocto Project master</field>
+    <field rel="ManyToOneRel" to="orm.bitbakeversion" name="bitbake_version">1</field>
+    <field type="CharField" name="branch_name">master</field>
+    <field type="TextField" name="helptext">Toaster will run your builds using the tip of the &lt;a href="http://git.yoctoproject.org/cgit/cgit.cgi/poky/log/"&gt;Yocto Project master branch&lt;/a&gt;, where active development takes place. This is not a stable branch, so your builds might not work as expected.</field>
+  </object>
+  <object model="orm.release" pk="2">
+    <field type="CharField" name="name">local</field>
+    <field type="CharField" name="description">Local Yocto Project</field>
+    <field rel="ManyToOneRel" to="orm.bitbakeversion" name="bitbake_version">2</field>
+    <field type="CharField" name="branch_name">HEAD</field>
+    <field type="TextField" name="helptext">Toaster will run your builds with the version of the Yocto Project you have cloned or downloaded to your computer.</field>
+  </object>
+
+  <!-- Default layers for each release -->
+  <object model="orm.releasedefaultlayer" pk="1">
+    <field rel="ManyToOneRel" to="orm.release" name="release">1</field>
+    <field type="CharField" name="layer_name">openembedded-core</field>
+  </object>
+  <object model="orm.releasedefaultlayer" pk="2">
+    <field rel="ManyToOneRel" to="orm.release" name="release">1</field>
+    <field type="CharField" name="layer_name">meta-poky</field>
+  </object>
+  <object model="orm.releasedefaultlayer" pk="3">
+    <field rel="ManyToOneRel" to="orm.release" name="release">1</field>
+    <field type="CharField" name="layer_name">meta-yocto-bsp</field>
+  </object>
+  <object model="orm.releasedefaultlayer" pk="4">
+    <field rel="ManyToOneRel" to="orm.release" name="release">2</field>
+    <field type="CharField" name="layer_name">openembedded-core</field>
+  </object>
+  <object model="orm.releasedefaultlayer" pk="5">
+    <field rel="ManyToOneRel" to="orm.release" name="release">2</field>
+    <field type="CharField" name="layer_name">meta-poky</field>
+  </object>
+  <object model="orm.releasedefaultlayer" pk="6">
+    <field rel="ManyToOneRel" to="orm.release" name="release">2</field>
+    <field type="CharField" name="layer_name">meta-yocto-bsp</field>
+  </object>
+
+  <!-- Layers for the Local release
+       layersource TYPE_LOCAL = 0
+  -->
+  <object model="orm.layer" pk="1">
+    <field type="CharField" name="name">openembedded-core</field>
+    <field type="CharField" name="layer_index_url"></field>
+    <field type="CharField" name="vcs_url">git://git.yoctoproject.org/poky</field>
+  </object>
+  <object model="orm.layer_version" pk="1">
+    <field rel="ManyToOneRel" to="orm.layer" name="layer">1</field>
+    <field type="IntegerField" name="layer_source">0</field>
+    <field rel="ManyToOneRel" to="orm.release" name="release">2</field>
+    <field type="CharField" name="branch">HEAD</field>
+    <field type="CharField" name="commit">HEAD</field>
+    <field type="CharField" name="dirpath">meta</field>
+  </object>
+
+
+  <object model="orm.layer" pk="2">
+    <field type="CharField" name="name">meta-poky</field>
+    <field type="CharField" name="layer_index_url"></field>
+    <field type="CharField" name="vcs_url">git://git.yoctoproject.org/poky</field>
+  </object>
+  <object model="orm.layer_version" pk="2">
+    <field rel="ManyToOneRel" to="orm.layer" name="layer">2</field>
+    <field type="IntegerField" name="layer_source">0</field>
+    <field rel="ManyToOneRel" to="orm.release" name="release">2</field>
+    <field type="CharField" name="branch">HEAD</field>
+    <field type="CharField" name="commit">HEAD</field>
+    <field type="CharField" name="dirpath">meta-poky</field>
+  </object>
+
+
+  <object model="orm.layer" pk="3">
+    <field type="CharField" name="name">meta-yocto-bsp</field>
+    <field type="CharField" name="layer_index_url"></field>
+    <field type="CharField" name="vcs_url">git://git.yoctoproject.org/poky</field>
+  </object>
+  <object model="orm.layer_version" pk="3">
+    <field rel="ManyToOneRel" to="orm.layer" name="layer">3</field>
+    <field type="IntegerField" name="layer_source">0</field>
+    <field rel="ManyToOneRel" to="orm.release" name="release">2</field>
+    <field type="CharField" name="branch">HEAD</field>
+    <field type="CharField" name="commit">HEAD</field>
+    <field type="CharField" name="dirpath">meta-yocto-bsp</field>
+  </object>
+</django-objects>

bitbake/lib/toaster/orm/fixtures/settings.xml

@@ -0,0 +1,33 @@
+<?xml version="1.0" encoding="utf-8"?>
+<django-objects version="1.0">
+  <!-- Default project settings -->
+  <!-- pk=1 is DISTRO -->
+  <object model="orm.toastersetting" pk="2">
+    <field type="CharField" name="name">DEFAULT_RELEASE</field>
+    <field type="CharField" name="value">master</field>
+  </object>
+  <object model="orm.toastersetting" pk="3">
+    <field type="CharField" name="name">DEFCONF_PACKAGE_CLASSES</field>
+    <field type="CharField" name="value">package_rpm</field>
+  </object>
+  <object model="orm.toastersetting" pk="4">
+    <field type="CharField" name="name">DEFCONF_MACHINE</field>
+    <field type="CharField" name="value">qemux86</field>
+  </object>
+  <object model="orm.toastersetting" pk="5">
+    <field type="CharField" name="name">DEFCONF_SSTATE_DIR</field>
+    <field type="CharField" name="value">${TOPDIR}/../sstate-cache</field>
+  </object>
+  <object model="orm.toastersetting" pk="6">
+    <field type="CharField" name="name">DEFCONF_IMAGE_INSTALL_append</field>
+    <field type="CharField" name="value"></field>
+  </object>
+  <object model="orm.toastersetting" pk="7">
+    <field type="CharField" name="name">DEFCONF_IMAGE_FSTYPES</field>
+    <field type="CharField" name="value">ext3 jffs2 tar.bz2</field>
+  </object>
+  <object model="orm.toastersetting" pk="8">
+    <field type="CharField" name="name">DEFCONF_DL_DIR</field>
+    <field type="CharField" name="value">${TOPDIR}/../downloads</field>
+  </object>
+</django-objects>

bitbake/lib/toaster/orm/management/commands/lsupdates.py

@@ -1,12 +1,334 @@
-from django.core.management.base import NoArgsCommand, CommandError
-from orm.models import LayerSource
+#
+# ex:ts=4:sw=4:sts=4:et
+# -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*-
+#
+# BitBake Toaster Implementation
+#
+# Copyright (C) 2016        Intel Corporation
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License version 2 as
+# published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License along
+# with this program; if not, write to the Free Software Foundation, Inc.,
+# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+from django.core.management.base import NoArgsCommand
+
+from orm.models import LayerSource, Layer, Release, Layer_Version
+from orm.models import LayerVersionDependency, Machine, Recipe
+
 import os
+import sys
+
+import json
+import logging
+import threading
+import time
+logger = logging.getLogger("toaster")
+
+DEFAULT_LAYERINDEX_SERVER = "http://layers.openembedded.org/layerindex/api/"
+
+
+class Spinner(threading.Thread):
+    """ A simple progress spinner to indicate download/parsing is happening"""
+    def __init__(self, *args, **kwargs):
+        super(Spinner, self).__init__(*args, **kwargs)
+        self.setDaemon(True)
+        self.signal = True
+
+    def run(self):
+        os.system('setterm -cursor off')
+        while self.signal:
+            for char in ["/", "-", "\\", "|"]:
+                sys.stdout.write("\r" + char)
+                sys.stdout.flush()
+                time.sleep(0.25)
+        os.system('setterm -cursor on')
+
+    def stop(self):
+        self.signal = False
+
 
 class Command(NoArgsCommand):
-    args    = ""
-    help    = "Updates locally cached information from all LayerSources"
+    args = ""
+    help = "Updates locally cached information from a layerindex server"
 
+    def mini_progress(self, what, i, total):
+        i = i + 1
+        pec = (float(i)/float(total))*100
+
+        sys.stdout.write("\rUpdating %s %d%%" %
+                         (what,
+                          pec))
+        sys.stdout.flush()
+        if int(pec) is 100:
+            sys.stdout.write("\n")
+            sys.stdout.flush()
+
+    def update(self):
+        """
+            Fetches layer, recipe and machine information from a layerindex
+            server
+        """
+        os.system('setterm -cursor off')
+
+        self.apiurl = DEFAULT_LAYERINDEX_SERVER
+
+        assert self.apiurl is not None
+        try:
+            from urllib.request import urlopen, URLError
+            from urllib.parse import urlparse
+        except ImportError:
+            from urllib2 import urlopen, URLError
+            from urlparse import urlparse
+
+        proxy_settings = os.environ.get("http_proxy", None)
+        oe_core_layer = 'openembedded-core'
+
+        def _get_json_response(apiurl=DEFAULT_LAYERINDEX_SERVER):
+            http_progress = Spinner()
+            http_progress.start()
+
+            _parsedurl = urlparse(apiurl)
+            path = _parsedurl.path
+
+            # logger.debug("Fetching %s", apiurl)
+            try:
+                res = urlopen(apiurl)
+            except URLError as e:
+                raise Exception("Failed to read %s: %s" % (path, e.reason))
+
+            parsed = json.loads(res.read().decode('utf-8'))
+
+            http_progress.stop()
+            return parsed
+
+        # verify we can get the basic api
+        try:
+            apilinks = _get_json_response()
+        except Exception as e:
+            import traceback
+            if proxy_settings is not None:
+                logger.info("EE: Using proxy %s" % proxy_settings)
+            logger.warning("EE: could not connect to %s, skipping update:"
+                           "%s\n%s" % (self.apiurl, e, traceback.format_exc()))
+            return
+
+        # update branches; only those that we already have names listed in the
+        # Releases table
+        whitelist_branch_names = [rel.branch_name
+                                  for rel in Release.objects.all()]
+        if len(whitelist_branch_names) == 0:
+            raise Exception("Failed to make list of branches to fetch")
+
+        logger.info("Fetching metadata releases for %s",
+                    " ".join(whitelist_branch_names))
+
+        branches_info = _get_json_response(apilinks['branches'] +
+                                           "?filter=name:%s"
+                                           % "OR".join(whitelist_branch_names))
+
+        # Map the layer index branches to toaster releases
+        li_branch_id_to_toaster_release = {}
+
+        total = len(branches_info)
+        for i, branch in enumerate(branches_info):
+            li_branch_id_to_toaster_release[branch['id']] = \
+                    Release.objects.get(name=branch['name'])
+            self.mini_progress("Releases", i, total)
+
+        # keep a track of the layerindex (li) id mappings so that
+        # layer_versions can be created for these layers later on
+        li_layer_id_to_toaster_layer_id = {}
+
+        logger.info("Fetching layers")
+
+        layers_info = _get_json_response(apilinks['layerItems'])
+
+        total = len(layers_info)
+        for i, li in enumerate(layers_info):
+            # Special case for the openembedded-core layer
+            if li['name'] == oe_core_layer:
+                try:
+                    # If we have an existing openembedded-core for example
+                    # from the toasterconf.json augment the info using the
+                    # layerindex rather than duplicate it
+                    oe_core_l = Layer.objects.get(name=oe_core_layer)
+                    # Take ownership of the layer as now coming from the
+                    # layerindex
+                    oe_core_l.summary = li['summary']
+                    oe_core_l.description = li['description']
+                    oe_core_l.vcs_web_url = li['vcs_web_url']
+                    oe_core_l.vcs_web_tree_base_url = \
+                        li['vcs_web_tree_base_url']
+                    oe_core_l.vcs_web_file_base_url = \
+                        li['vcs_web_file_base_url']
+
+                    oe_core_l.save()
+                    li_layer_id_to_toaster_layer_id[li['id']] = oe_core_l.pk
+                    self.mini_progress("layers", i, total)
+                    continue
+
+                except Layer.DoesNotExist:
+                    pass
+
+            try:
+                l, created = Layer.objects.get_or_create(name=li['name'],
+                                                         vcs_url=li['vcs_url'])
+                l.up_date = li['updated']
+                l.vcs_url = li['vcs_url']
+                l.vcs_web_url = li['vcs_web_url']
+                l.vcs_web_tree_base_url = li['vcs_web_tree_base_url']
+                l.vcs_web_file_base_url = li['vcs_web_file_base_url']
+                l.summary = li['summary']
+                l.description = li['description']
+                l.save()
+            except Layer.MultipleObjectsReturned:
+                logger.info("Skipped %s as we found multiple layers and "
+                            "don't know which to update" %
+                            li['name'])
+
+            li_layer_id_to_toaster_layer_id[li['id']] = l.pk
+
+            self.mini_progress("layers", i, total)
+
+        # update layer_versions
+        logger.info("Fetching layer versions")
+        layerbranches_info = _get_json_response(
+            apilinks['layerBranches'] + "?filter=branch__name:%s" %
+            "OR".join(whitelist_branch_names))
+
+        # Map Layer index layer_branch object id to
+        # layer_version toaster object id
+        li_layer_branch_id_to_toaster_lv_id = {}
+
+        total = len(layerbranches_info)
+        for i, lbi in enumerate(layerbranches_info):
+
+            try:
+                lv, created = Layer_Version.objects.get_or_create(
+                    layer_source=LayerSource.TYPE_LAYERINDEX,
+                    layer=Layer.objects.get(
+                        pk=li_layer_id_to_toaster_layer_id[lbi['layer']])
+                )
+            except KeyError:
+                logger.warning(
+                    "No such layerindex layer referenced by layerbranch %d" %
+                    lbi['layer'])
+                continue
+
+            lv.release = li_branch_id_to_toaster_release[lbi['branch']]
+            lv.up_date = lbi['updated']
+            lv.commit = lbi['actual_branch']
+            lv.dirpath = lbi['vcs_subdir']
+            lv.save()
+
+            li_layer_branch_id_to_toaster_lv_id[lbi['id']] =\
+                lv.pk
+            self.mini_progress("layer versions", i, total)
+
+        logger.info("Fetching layer version dependencies")
+        # update layer dependencies
+        layerdependencies_info = _get_json_response(
+            apilinks['layerDependencies'] +
+            "?filter=layerbranch__branch__name:%s" %
+            "OR".join(whitelist_branch_names))
+
+        dependlist = {}
+        for ldi in layerdependencies_info:
+            try:
+                lv = Layer_Version.objects.get(
+                    pk=li_layer_branch_id_to_toaster_lv_id[ldi['layerbranch']])
+            except Layer_Version.DoesNotExist as e:
+                continue
+
+            if lv not in dependlist:
+                dependlist[lv] = []
+            try:
+                layer_id = li_layer_id_to_toaster_layer_id[ldi['dependency']]
+
+                dependlist[lv].append(
+                    Layer_Version.objects.get(
+                        layer_source=LayerSource.TYPE_LAYERINDEX,
+                        layer__pk=layer_id))
+
+            except Layer_Version.DoesNotExist:
+                logger.warning("Cannot find layer version (ls:%s),"
+                               "up_id:%s lv:%s" %
+                               (self, ldi['dependency'], lv))
+
+        total = len(dependlist)
+        for i, lv in enumerate(dependlist):
+            LayerVersionDependency.objects.filter(layer_version=lv).delete()
+            for lvd in dependlist[lv]:
+                LayerVersionDependency.objects.get_or_create(layer_version=lv,
+                                                             depends_on=lvd)
+            self.mini_progress("Layer version dependencies", i, total)
+
+        # update machines
+        logger.info("Fetching machine information")
+        machines_info = _get_json_response(
+            apilinks['machines'] + "?filter=layerbranch__branch__name:%s" %
+            "OR".join(whitelist_branch_names))
+
+        total = len(machines_info)
+        for i, mi in enumerate(machines_info):
+            mo, created = Machine.objects.get_or_create(
+                name=mi['name'],
+                layer_version=Layer_Version.objects.get(
+                    pk=li_layer_branch_id_to_toaster_lv_id[mi['layerbranch']]))
+            mo.up_date = mi['updated']
+            mo.name = mi['name']
+            mo.description = mi['description']
+            mo.save()
+            self.mini_progress("machines", i, total)
+
+        # update recipes; paginate by layer version / layer branch
+        logger.info("Fetching recipe information")
+        recipes_info = _get_json_response(
+            apilinks['recipes'] + "?filter=layerbranch__branch__name:%s" %
+            "OR".join(whitelist_branch_names))
+
+        total = len(recipes_info)
+        for i, ri in enumerate(recipes_info):
+            try:
+                lv_id = li_layer_branch_id_to_toaster_lv_id[ri['layerbranch']]
+                lv = Layer_Version.objects.get(pk=lv_id)
+
+                ro, created = Recipe.objects.get_or_create(
+                    layer_version=lv,
+                    name=ri['pn']
+                )
+
+                ro.layer_version = lv
+                ro.up_date = ri['updated']
+                ro.name = ri['pn']
+                ro.version = ri['pv']
+                ro.summary = ri['summary']
+                ro.description = ri['description']
+                ro.section = ri['section']
+                ro.license = ri['license']
+                ro.homepage = ri['homepage']
+                ro.bugtracker = ri['bugtracker']
+                ro.file_path = ri['filepath'] + "/" + ri['filename']
+                if 'inherits' in ri:
+                    ro.is_image = 'image' in ri['inherits'].split()
+                else:  # workaround for old style layer index
+                    ro.is_image = "-image-" in ri['pn']
+                ro.save()
+            except Exception as e:
+                logger.warning("Failed saving recipe %s", e)
+
+            self.mini_progress("recipes", i, total)
+
+        os.system('setterm -cursor on')
 
     def handle_noargs(self, **options):
-        for ls in LayerSource.objects.all():
-            ls.update()
+        self.update()

bitbake/lib/toaster/orm/migrations/0010_delete_layer_source_references.py

@@ -0,0 +1,118 @@
+# -*- coding: utf-8 -*-
+from __future__ import unicode_literals
+
+from django.db import migrations, models
+import django.utils.timezone
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('orm', '0009_target_package_manifest_path'),
+    ]
+
+    operations = [
+        migrations.AlterUniqueTogether(
+            name='releaselayersourcepriority',
+            unique_together=set([]),
+        ),
+        migrations.RemoveField(
+            model_name='releaselayersourcepriority',
+            name='layer_source',
+        ),
+        migrations.RemoveField(
+            model_name='releaselayersourcepriority',
+            name='release',
+        ),
+        migrations.DeleteModel(
+            name='ImportedLayerSource',
+        ),
+        migrations.DeleteModel(
+            name='LayerIndexLayerSource',
+        ),
+        migrations.DeleteModel(
+            name='LocalLayerSource',
+        ),
+        migrations.RemoveField(
+            model_name='recipe',
+            name='layer_source',
+        ),
+        migrations.RemoveField(
+            model_name='recipe',
+            name='up_id',
+        ),
+        migrations.AlterField(
+            model_name='layer',
+            name='up_date',
+            field=models.DateTimeField(default=django.utils.timezone.now, null=True),
+        ),
+        migrations.AlterField(
+            model_name='layer_version',
+            name='layer_source',
+            field=models.IntegerField(default=0, choices=[(0, 'local'), (1, 'layerindex'), (2, 'imported'), (3, 'build')]),
+        ),
+        migrations.AlterField(
+            model_name='layer_version',
+            name='up_date',
+            field=models.DateTimeField(default=django.utils.timezone.now, null=True),
+        ),
+        migrations.AlterUniqueTogether(
+            name='branch',
+            unique_together=set([]),
+        ),
+        migrations.AlterUniqueTogether(
+            name='layer',
+            unique_together=set([]),
+        ),
+        migrations.AlterUniqueTogether(
+            name='layer_version',
+            unique_together=set([]),
+        ),
+        migrations.AlterUniqueTogether(
+            name='layerversiondependency',
+            unique_together=set([]),
+        ),
+        migrations.AlterUniqueTogether(
+            name='machine',
+            unique_together=set([]),
+        ),
+        migrations.DeleteModel(
+            name='ReleaseLayerSourcePriority',
+        ),
+        migrations.RemoveField(
+            model_name='branch',
+            name='layer_source',
+        ),
+        migrations.RemoveField(
+            model_name='branch',
+            name='up_id',
+        ),
+        migrations.RemoveField(
+            model_name='layer',
+            name='layer_source',
+        ),
+        migrations.RemoveField(
+            model_name='layer',
+            name='up_id',
+        ),
+        migrations.RemoveField(
+            model_name='layer_version',
+            name='up_id',
+        ),
+        migrations.RemoveField(
+            model_name='layerversiondependency',
+            name='layer_source',
+        ),
+        migrations.RemoveField(
+            model_name='layerversiondependency',
+            name='up_id',
+        ),
+        migrations.RemoveField(
+            model_name='machine',
+            name='layer_source',
+        ),
+        migrations.RemoveField(
+            model_name='machine',
+            name='up_id',
+        ),
+    ]

bitbake/lib/toaster/orm/migrations/0011_delete_layersource.py

@@ -0,0 +1,17 @@
+# -*- coding: utf-8 -*-
+from __future__ import unicode_literals
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('orm', '0010_delete_layer_source_references'),
+    ]
+
+    operations = [
+        migrations.DeleteModel(
+            name='LayerSource',
+        ),
+    ]

bitbake/lib/toaster/orm/migrations/0012_use_release_instead_of_up_branch.py

@@ -0,0 +1,62 @@
+# -*- coding: utf-8 -*-
+from __future__ import unicode_literals
+
+from django.db import migrations, models
+from django.db.models import Q
+
+
+def branch_to_release(apps, schema_editor):
+    Layer_Version = apps.get_model('orm', 'Layer_Version')
+    Release = apps.get_model('orm', 'Release')
+
+    print("Converting all layer version up_branches to releases")
+    # Find all the layer versions which have an upbranch and convert them to
+    # the release that they're for.
+    for layer_version in Layer_Version.objects.filter(
+            Q(release=None) & ~Q(up_branch=None)):
+        try:
+            # HEAD and local are equivalent
+            if "HEAD" in layer_version.up_branch.name:
+                release = Release.objects.get(name="local")
+                layer_version.commit = "HEAD"
+                layer_version.branch = "HEAD"
+            else:
+                release = Release.objects.get(
+                    name=layer_version.up_branch.name)
+
+            layer_version.release = release
+            layer_version.save()
+        except Exception as e:
+            print("Couldn't work out an appropriate release for %s "
+                  "the up_branch was %s "
+                  "user the django admin interface to correct it" %
+                  (layer_version.layer.name, layer_version.up_branch.name))
+            print(e)
+
+            continue
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('orm', '0011_delete_layersource'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='layer_version',
+            name='release',
+            field=models.ForeignKey(to='orm.Release', default=None, null=True),
+        ),
+        migrations.RunPython(branch_to_release,
+                             reverse_code=migrations.RunPython.noop),
+
+        migrations.RemoveField(
+            model_name='layer_version',
+            name='up_branch',
+        ),
+
+        migrations.DeleteModel(
+            name='Branch',
+        ),
+    ]

bitbake/lib/toaster/orm/migrations/0013_recipe_parse_progress_fields.py

@@ -0,0 +1,24 @@
+# -*- coding: utf-8 -*-
+from __future__ import unicode_literals
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('orm', '0012_use_release_instead_of_up_branch'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='build',
+            name='recipes_parsed',
+            field=models.IntegerField(default=0),
+        ),
+        migrations.AddField(
+            model_name='build',
+            name='recipes_to_parse',
+            field=models.IntegerField(default=1),
+        ),
+    ]

bitbake/lib/toaster/orm/migrations/0014_allow_empty_buildname.py

@@ -0,0 +1,19 @@
+# -*- coding: utf-8 -*-
+from __future__ import unicode_literals
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('orm', '0013_recipe_parse_progress_fields'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='build',
+            name='build_name',
+            field=models.CharField(default='', max_length=100),
+        ),
+    ]

bitbake/lib/toaster/orm/migrations/0015_layer_local_source_dir.py

@@ -0,0 +1,19 @@
+# -*- coding: utf-8 -*-
+from __future__ import unicode_literals
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('orm', '0014_allow_empty_buildname'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='layer',
+            name='local_source_dir',
+            field=models.TextField(null=True, default=None),
+        ),
+    ]

bitbake/lib/toaster/orm/models.py

@@ -21,8 +21,8 @@
 
 from __future__ import unicode_literals
 
-from django.db import models, IntegrityError
-from django.db.models import F, Q, Avg, Max, Sum, Count
+from django.db import models, IntegrityError, DataError
+from django.db.models import F, Q, Sum, Count
 from django.utils import timezone
 from django.utils.encoding import force_bytes
 
@@ -33,9 +33,10 @@ from django.conf import settings
 import django.db.models.signals
 
 import sys
-import os.path
+import os
 import re
 import itertools
+from signal import SIGUSR1
 
 import logging
 logger = logging.getLogger("toaster")
@@ -78,7 +79,7 @@ if 'sqlite' in settings.DATABASES['default']['ENGINE']:
         try:
             obj = self.create(**params)
             return obj, True
-        except IntegrityError:
+        except (IntegrityError, DataError):
             exc_info = sys.exc_info()
             try:
                 return self.get(**lookup), False
@@ -117,39 +118,48 @@ class ToasterSetting(models.Model):
     def __unicode__(self):
         return "Setting %s = %s" % (self.name, self.value)
 
+
 class ProjectManager(models.Manager):
     def create_project(self, name, release):
         if release is not None:
-            prj = self.model(name = name, bitbake_version = release.bitbake_version, release = release)
+            prj = self.model(name=name,
+                             bitbake_version=release.bitbake_version,
+                             release=release)
         else:
-            prj = self.model(name = name, bitbake_version = None, release = None)
+            prj = self.model(name=name,
+                             bitbake_version=None,
+                             release=None)
 
         prj.save()
 
-        for defaultconf in ToasterSetting.objects.filter(name__startswith="DEFCONF_"):
+        for defaultconf in ToasterSetting.objects.filter(
+                name__startswith="DEFCONF_"):
             name = defaultconf.name[8:]
-            ProjectVariable.objects.create( project = prj,
-                name = name,
-                value = defaultconf.value)
+            ProjectVariable.objects.create(project=prj,
+                                           name=name,
+                                           value=defaultconf.value)
 
         if release is None:
             return prj
 
         for rdl in release.releasedefaultlayer_set.all():
-            try:
-                lv = Layer_Version.objects.filter(layer__name = rdl.layer_name, up_branch__name = release.branch_name)[0].get_equivalents_wpriority(prj)[0]
-                ProjectLayer.objects.create( project = prj,
-                        layercommit = lv,
-                        optional = False )
-            except IndexError:
-                # we may have no valid layer version objects, and that's ok
-                pass
+            lv = Layer_Version.objects.filter(
+                layer__name=rdl.layer_name,
+                release=release).first()
+
+            if lv:
+                ProjectLayer.objects.create(project=prj,
+                                            layercommit=lv,
+                                            optional=False)
+            else:
+                logger.warning("Default project layer %s not found" %
+                               rdl.layer_name)
 
         return prj
 
     # return single object with is_default = True
     def get_or_create_default_project(self):
-        projects = super(ProjectManager, self).filter(is_default = True)
+        projects = super(ProjectManager, self).filter(is_default=True)
 
         if len(projects) > 1:
             raise Exception('Inconsistent project data: multiple ' +
@@ -157,7 +167,8 @@ class ProjectManager(models.Manager):
         elif len(projects) < 1:
             options = {
                 'name': 'Command line builds',
-                'short_description': 'Project for builds started outside Toaster',
+                'short_description':
+                'Project for builds started outside Toaster',
                 'is_default': True
             }
             project = Project.objects.create(**options)
@@ -270,7 +281,7 @@ class Project(models.Model):
         # guard on release, as it can be null
         if self.release:
             queryset = Layer_Version.objects.filter(
-                (Q(up_branch__name=self.release.branch_name) &
+                (Q(release=self.release) &
                  Q(build=None) &
                  Q(project=None)) |
                  Q(project=self))
@@ -336,7 +347,15 @@ class Project(models.Model):
             for l in self.projectlayer_set.all().order_by("pk"):
                 commit = l.layercommit.get_vcs_reference()
                 print("ii Building layer ", l.layercommit.layer.name, " at vcs point ", commit)
-                BRLayer.objects.create(req = br, name = l.layercommit.layer.name, giturl = l.layercommit.layer.vcs_url, commit = commit, dirpath = l.layercommit.dirpath, layer_version=l.layercommit)
+                BRLayer.objects.create(
+                    req=br,
+                    name=l.layercommit.layer.name,
+                    giturl=l.layercommit.layer.vcs_url,
+                    commit=commit,
+                    dirpath=l.layercommit.dirpath,
+                    layer_version=l.layercommit,
+                    local_source_dir=l.layercommit.layer.local_source_dir
+                )
 
             br.state = BuildRequest.REQ_QUEUED
             now = timezone.now()
@@ -358,6 +377,8 @@ class Project(models.Model):
             except ProjectVariable.DoesNotExist:
                 pass
             br.save()
+            signal_runbuilds()
+
         except Exception:
             # revert the build request creation since we're not done cleanly
             br.delete()
@@ -387,9 +408,15 @@ class Build(models.Model):
     completed_on = models.DateTimeField()
     outcome = models.IntegerField(choices=BUILD_OUTCOME, default=IN_PROGRESS)
     cooker_log_path = models.CharField(max_length=500)
-    build_name = models.CharField(max_length=100)
+    build_name = models.CharField(max_length=100, default='')
     bitbake_version = models.CharField(max_length=50)
 
+    # number of recipes to parse for this build
+    recipes_to_parse = models.IntegerField(default=1)
+
+    # number of recipes parsed so far for this build
+    recipes_parsed = models.IntegerField(default=0)
+
     @staticmethod
     def get_recent(project=None):
         """
@@ -420,6 +447,21 @@ class Build(models.Model):
 
         return recent_builds
 
+    def started(self):
+        """
+        As build variables are only added for a build when its BuildStarted event
+        is received, a build with no build variables is counted as
+        "in preparation" and not properly started yet. This method
+        will return False if a build has no build variables (it never properly
+        started), or True otherwise.
+
+        Note that this is a temporary workaround for the fact that we don't
+        have a fine-grained state variable on a build which would allow us
+        to record "in progress" (BuildStarted received) vs. "in preparation".
+        """
+        variables = Variable.objects.filter(build=self)
+        return len(variables) > 0
+
     def completeper(self):
         tf = Task.objects.filter(build = self)
         tfc = tf.count()
@@ -582,22 +624,64 @@ class Build(models.Model):
 
         return target_labels
 
-    def get_current_status(self):
-        """
-        get the status string from the build request if the build
-        has one, or the text for the build outcome if it doesn't
-        """
-
-        from bldcontrol.models import BuildRequest
-
-        build_request = None
+    def get_buildrequest(self):
+        buildrequest = None
         if hasattr(self, 'buildrequest'):
-            build_request = self.buildrequest
+            buildrequest = self.buildrequest
+        return buildrequest
 
-        if (build_request
-                and build_request.state != BuildRequest.REQ_INPROGRESS
-                and self.outcome == Build.IN_PROGRESS):
-            return self.buildrequest.get_state_display()
+    def is_queued(self):
+        from bldcontrol.models import BuildRequest
+        buildrequest = self.get_buildrequest()
+        if buildrequest:
+            return buildrequest.state == BuildRequest.REQ_QUEUED
+        else:
+            return False
+
+    def is_cancelling(self):
+        from bldcontrol.models import BuildRequest
+        buildrequest = self.get_buildrequest()
+        if buildrequest:
+            return self.outcome == Build.IN_PROGRESS and \
+                buildrequest.state == BuildRequest.REQ_CANCELLING
+        else:
+            return False
+
+    def is_parsing(self):
+        """
+        True if the build is still parsing recipes
+        """
+        return self.outcome == Build.IN_PROGRESS and \
+            self.recipes_parsed < self.recipes_to_parse
+
+    def is_starting(self):
+        """
+        True if the build has no completed tasks yet and is still just starting
+        tasks.
+
+        Note that the mechanism for testing whether a Task is "done" is whether
+        its order field is set, as per the completeper() method.
+        """
+        return self.outcome == Build.IN_PROGRESS and \
+            self.task_build.filter(order__isnull=False).count() == 0
+
+    def get_state(self):
+        """
+        Get the state of the build; one of 'Succeeded', 'Failed', 'In Progress',
+        'Cancelled' (Build outcomes); or 'Queued', 'Cancelling' (states
+        dependent on the BuildRequest state).
+
+        This works around the fact that we have BuildRequest states as well
+        as Build states, but really we just want to know the state of the build.
+        """
+        if self.is_cancelling():
+            return 'Cancelling';
+        elif self.is_queued():
+            return 'Queued'
+        elif self.is_parsing():
+            return 'Parsing'
+        elif self.is_starting():
+            return 'Starting'
         else:
             return self.get_outcome_text()
 
@@ -795,9 +879,10 @@ class Target_Image_File(models.Model):
     SUFFIXES = {
         'btrfs', 'cpio', 'cpio.gz', 'cpio.lz4', 'cpio.lzma', 'cpio.xz',
         'cramfs', 'elf', 'ext2', 'ext2.bz2', 'ext2.gz', 'ext2.lzma', 'ext4',
-        'ext4.gz', 'ext3', 'ext3.gz', 'hddimg', 'iso', 'jffs2', 'jffs2.sum',
-        'squashfs', 'squashfs-lzo', 'squashfs-xz', 'tar.bz2', 'tar.lz4',
-        'tar.xz', 'tartar.gz', 'ubi', 'ubifs', 'vmdk'
+        'ext4.gz', 'ext3', 'ext3.gz', 'hdddirect', 'hddimg', 'iso', 'jffs2',
+        'jffs2.sum', 'multiubi', 'qcow2', 'squashfs', 'squashfs-lzo',
+        'squashfs-xz', 'tar', 'tar.bz2', 'tar.gz', 'tar.lz4', 'tar.xz', 'ubi',
+        'ubifs', 'vdi', 'vmdk', 'wic', 'wic.bz2', 'wic.gz', 'wic.lzma'
     }
 
     target = models.ForeignKey(Target)
@@ -1126,21 +1211,27 @@ class Target_Installed_Package(models.Model):
     target = models.ForeignKey(Target)
     package = models.ForeignKey(Package, related_name='buildtargetlist_package')
 
+
 class Package_File(models.Model):
     package = models.ForeignKey(Package, related_name='buildfilelist_package')
     path = models.FilePathField(max_length=255, blank=True)
     size = models.IntegerField()
 
+
 class Recipe(models.Model):
-    search_allowed_fields = ['name', 'version', 'file_path', 'section', 'summary', 'description', 'license', 'layer_version__layer__name', 'layer_version__branch', 'layer_version__commit', 'layer_version__local_path', 'layer_version__layer_source__name']
+    search_allowed_fields = ['name', 'version', 'file_path', 'section',
+                             'summary', 'description', 'license',
+                             'layer_version__layer__name',
+                             'layer_version__branch', 'layer_version__commit',
+                             'layer_version__local_path',
+                             'layer_version__layer_source']
 
-    layer_source = models.ForeignKey('LayerSource', default = None, null = True)  # from where did we get this recipe
-    up_id = models.IntegerField(null = True, default = None)                    # id of entry in the source
-    up_date = models.DateTimeField(null = True, default = None)
+    up_date = models.DateTimeField(null=True, default=None)
 
-    name = models.CharField(max_length=100, blank=True)                 # pn
-    version = models.CharField(max_length=100, blank=True)              # pv
-    layer_version = models.ForeignKey('Layer_Version', related_name='recipe_layer_version')
+    name = models.CharField(max_length=100, blank=True)
+    version = models.CharField(max_length=100, blank=True)
+    layer_version = models.ForeignKey('Layer_Version',
+                                      related_name='recipe_layer_version')
     summary = models.TextField(blank=True)
     description = models.TextField(blank=True)
     section = models.CharField(max_length=100, blank=True)
@@ -1151,13 +1242,6 @@ class Recipe(models.Model):
     pathflags = models.CharField(max_length=200, blank=True)
     is_image = models.BooleanField(default=False)
 
-    def get_layersource_view_url(self):
-        if self.layer_source is None:
-            return ""
-
-        url = self.layer_source.get_object_view(self.layer_version.up_branch, "recipes", self.name)
-        return url
-
     def __unicode__(self):
         return "Recipe " + self.name + ":" + self.version
 
@@ -1203,8 +1287,6 @@ class Recipe_Dependency(models.Model):
 
 class Machine(models.Model):
     search_allowed_fields = ["name", "description", "layer_version__layer__name"]
-    layer_source = models.ForeignKey('LayerSource', default = None, null = True)  # from where did we get this machine
-    up_id = models.IntegerField(null = True, default = None)                      # id of entry in the source
     up_date = models.DateTimeField(null = True, default = None)
 
     layer_version = models.ForeignKey('Layer_Version')
@@ -1219,293 +1301,9 @@ class Machine(models.Model):
     def __unicode__(self):
         return "Machine " + self.name + "(" + self.description + ")"
 
-    class Meta:
-        unique_together = ("layer_source", "up_id")
 
 
-from django.db.models.base import ModelBase
 
-class InheritanceMetaclass(ModelBase):
-    def __call__(cls, *args, **kwargs):
-        obj = super(InheritanceMetaclass, cls).__call__(*args, **kwargs)
-        return obj.get_object()
-
-
-class LayerSource(models.Model):
-    __metaclass__ = InheritanceMetaclass
-
-    class Meta:
-        unique_together = (('sourcetype', 'apiurl'), )
-
-    TYPE_LOCAL = 0
-    TYPE_LAYERINDEX = 1
-    TYPE_IMPORTED = 2
-    SOURCE_TYPE = (
-        (TYPE_LOCAL, "local"),
-        (TYPE_LAYERINDEX, "layerindex"),
-        (TYPE_IMPORTED, "imported"),
-      )
-
-    name = models.CharField(max_length=63, unique = True)
-    sourcetype = models.IntegerField(choices=SOURCE_TYPE)
-    apiurl = models.CharField(max_length=255, null=True, default=None)
-
-    def __init__(self, *args, **kwargs):
-        super(LayerSource, self).__init__(*args, **kwargs)
-        if self.sourcetype == LayerSource.TYPE_LOCAL:
-            self.__class__ = LocalLayerSource
-        elif self.sourcetype == LayerSource.TYPE_LAYERINDEX:
-            self.__class__ = LayerIndexLayerSource
-        elif self.sourcetype == LayerSource.TYPE_IMPORTED:
-            self.__class__ = ImportedLayerSource
-        elif self.sourcetype == None:
-            raise Exception("Unknown LayerSource-derived class. If you added a new layer source type, fill out all code stubs.")
-
-
-    def update(self):
-        """
-            Updates the local database information from the upstream layer source
-        """
-        raise Exception("Abstract, update() must be implemented by all LayerSource-derived classes (object is %s)" % str(vars(self)))
-
-    def save(self, *args, **kwargs):
-        return super(LayerSource, self).save(*args, **kwargs)
-
-    def get_object(self):
-        # preset an un-initilized object
-        if None == self.name:
-            self.name=""
-        if None == self.apiurl:
-            self.apiurl=""
-        if None == self.sourcetype:
-            self.sourcetype=LayerSource.TYPE_LOCAL
-
-        if self.sourcetype == LayerSource.TYPE_LOCAL:
-            self.__class__ = LocalLayerSource
-        elif self.sourcetype == LayerSource.TYPE_LAYERINDEX:
-            self.__class__ = LayerIndexLayerSource
-        elif self.sourcetype == LayerSource.TYPE_IMPORTED:
-            self.__class__ = ImportedLayerSource
-        else:
-            raise Exception("Unknown LayerSource type. If you added a new layer source type, fill out all code stubs.")
-        return self
-
-    def __unicode__(self):
-        return "%s (%s)" % (self.name, self.sourcetype)
-
-
-class LocalLayerSource(LayerSource):
-    class Meta(LayerSource._meta.__class__):
-        proxy = True
-
-    def __init__(self, *args, **kwargs):
-        super(LocalLayerSource, self).__init__(args, kwargs)
-        self.sourcetype = LayerSource.TYPE_LOCAL
-
-    def update(self):
-        """
-            Fetches layer, recipe and machine information from local repository
-        """
-        pass
-
-class ImportedLayerSource(LayerSource):
-    class Meta(LayerSource._meta.__class__):
-        proxy = True
-
-    def __init__(self, *args, **kwargs):
-        super(ImportedLayerSource, self).__init__(args, kwargs)
-        self.sourcetype = LayerSource.TYPE_IMPORTED
-
-    def update(self):
-        """
-            Fetches layer, recipe and machine information from local repository
-        """
-        pass
-
-
-class LayerIndexLayerSource(LayerSource):
-    class Meta(LayerSource._meta.__class__):
-        proxy = True
-
-    def __init__(self, *args, **kwargs):
-        super(LayerIndexLayerSource, self).__init__(args, kwargs)
-        self.sourcetype = LayerSource.TYPE_LAYERINDEX
-
-    def get_object_view(self, branch, objectype, upid):
-        return self.apiurl + "../branch/" + branch.name + "/" + objectype + "/?q=" + str(upid)
-
-    def update(self):
-        """
-            Fetches layer, recipe and machine information from remote repository
-        """
-        assert self.apiurl is not None
-        from django.db import transaction, connection
-
-        import json
-        import os
-
-        try:
-            from urllib.request import urlopen, URLError
-            from urllib.parse import urlparse
-        except ImportError:
-            from urllib2 import urlopen, URLError
-            from urlparse import urlparse
-
-        proxy_settings = os.environ.get("http_proxy", None)
-        oe_core_layer = 'openembedded-core'
-
-        def _get_json_response(apiurl = self.apiurl):
-            _parsedurl = urlparse(apiurl)
-            path = _parsedurl.path
-
-            try:
-                res = urlopen(apiurl)
-            except URLError as e:
-                raise Exception("Failed to read %s: %s" % (path, e.reason))
-
-            return json.loads(res.read().decode('utf-8'))
-
-        # verify we can get the basic api
-        try:
-            apilinks = _get_json_response()
-        except Exception as e:
-            import traceback
-            if proxy_settings is not None:
-                logger.info("EE: Using proxy %s" % proxy_settings)
-            logger.warning("EE: could not connect to %s, skipping update: %s\n%s" % (self.apiurl, e, traceback.format_exc()))
-            return
-
-        # update branches; only those that we already have names listed in the
-        # Releases table
-        whitelist_branch_names = [rel.branch_name for rel in Release.objects.all()]
-        if len(whitelist_branch_names) == 0:
-            raise Exception("Failed to make list of branches to fetch")
-
-        logger.debug("Fetching branches")
-        branches_info = _get_json_response(apilinks['branches']
-            + "?filter=name:%s" % "OR".join(whitelist_branch_names))
-        for bi in branches_info:
-            b, created = Branch.objects.get_or_create(layer_source = self, name = bi['name'])
-            b.up_id = bi['id']
-            b.up_date = bi['updated']
-            b.name = bi['name']
-            b.short_description = bi['short_description']
-            b.save()
-
-        # update layers
-        layers_info = _get_json_response(apilinks['layerItems'])
-
-        for li in layers_info:
-            # Special case for the openembedded-core layer
-            if li['name'] == oe_core_layer:
-                try:
-                    # If we have an existing openembedded-core for example
-                    # from the toasterconf.json augment the info using the
-                    # layerindex rather than duplicate it
-                    oe_core_l =  Layer.objects.get(name=oe_core_layer)
-                    # Take ownership of the layer as now coming from the
-                    # layerindex
-                    oe_core_l.layer_source = self
-                    oe_core_l.up_id = li['id']
-                    oe_core_l.summary = li['summary']
-                    oe_core_l.description = li['description']
-                    oe_core_l.save()
-                    continue
-
-                except Layer.DoesNotExist:
-                    pass
-
-            l, created = Layer.objects.get_or_create(layer_source = self, name = li['name'])
-            l.up_id = li['id']
-            l.up_date = li['updated']
-            l.vcs_url = li['vcs_url']
-            l.vcs_web_url = li['vcs_web_url']
-            l.vcs_web_tree_base_url = li['vcs_web_tree_base_url']
-            l.vcs_web_file_base_url = li['vcs_web_file_base_url']
-            l.summary = li['summary']
-            l.description = li['description']
-            l.save()
-
-        # update layerbranches/layer_versions
-        logger.debug("Fetching layer information")
-        layerbranches_info = _get_json_response(apilinks['layerBranches']
-                + "?filter=branch:%s" % "OR".join(map(lambda x: str(x.up_id), [i for i in Branch.objects.filter(layer_source = self) if i.up_id is not None] ))
-            )
-
-        for lbi in layerbranches_info:
-            lv, created = Layer_Version.objects.get_or_create(layer_source = self,
-                    up_id = lbi['id'],
-                    layer=Layer.objects.get(layer_source = self, up_id = lbi['layer'])
-                )
-
-            lv.up_date = lbi['updated']
-            lv.up_branch = Branch.objects.get(layer_source = self, up_id = lbi['branch'])
-            lv.branch = lbi['actual_branch']
-            lv.commit = lbi['actual_branch']
-            lv.dirpath = lbi['vcs_subdir']
-            lv.save()
-
-        # update layer dependencies
-        layerdependencies_info = _get_json_response(apilinks['layerDependencies'])
-        dependlist = {}
-        for ldi in layerdependencies_info:
-            try:
-                lv = Layer_Version.objects.get(layer_source = self, up_id = ldi['layerbranch'])
-            except Layer_Version.DoesNotExist as e:
-                continue
-
-            if lv not in dependlist:
-                dependlist[lv] = []
-            try:
-                dependlist[lv].append(Layer_Version.objects.get(layer_source = self, layer__up_id = ldi['dependency'], up_branch = lv.up_branch))
-            except Layer_Version.DoesNotExist:
-                logger.warning("Cannot find layer version (ls:%s), up_id:%s lv:%s" % (self, ldi['dependency'], lv))
-
-        for lv in dependlist:
-            LayerVersionDependency.objects.filter(layer_version = lv).delete()
-            for lvd in dependlist[lv]:
-                LayerVersionDependency.objects.get_or_create(layer_version = lv, depends_on = lvd)
-
-
-        # update machines
-        logger.debug("Fetching machine information")
-        machines_info = _get_json_response(apilinks['machines']
-                + "?filter=layerbranch:%s" % "OR".join(map(lambda x: str(x.up_id), Layer_Version.objects.filter(layer_source = self)))
-            )
-
-        for mi in machines_info:
-            mo, created = Machine.objects.get_or_create(layer_source = self, up_id = mi['id'], layer_version = Layer_Version.objects.get(layer_source = self, up_id = mi['layerbranch']))
-            mo.up_date = mi['updated']
-            mo.name = mi['name']
-            mo.description = mi['description']
-            mo.save()
-
-        # update recipes; paginate by layer version / layer branch
-        logger.debug("Fetching target information")
-        recipes_info = _get_json_response(apilinks['recipes']
-                + "?filter=layerbranch:%s" % "OR".join(map(lambda x: str(x.up_id), Layer_Version.objects.filter(layer_source = self)))
-            )
-        for ri in recipes_info:
-            try:
-                ro, created = Recipe.objects.get_or_create(layer_source = self, up_id = ri['id'], layer_version = Layer_Version.objects.get(layer_source = self, up_id = ri['layerbranch']))
-                ro.up_date = ri['updated']
-                ro.name = ri['pn']
-                ro.version = ri['pv']
-                ro.summary = ri['summary']
-                ro.description = ri['description']
-                ro.section = ri['section']
-                ro.license = ri['license']
-                ro.homepage = ri['homepage']
-                ro.bugtracker = ri['bugtracker']
-                ro.file_path = ri['filepath'] + "/" + ri['filename']
-                if 'inherits' in ri:
-                    ro.is_image = 'image' in ri['inherits'].split()
-                else: # workaround for old style layer index
-                    ro.is_image = "-image-" in ri['pn']
-                ro.save()
-            except IntegrityError as e:
-                logger.debug("Failed saving recipe, ignoring: %s (%s:%s)" % (e, ro.layer_version, ri['filepath']+"/"+ri['filename']))
-                ro.delete()
 
 class BitbakeVersion(models.Model):
 
@@ -1529,87 +1327,94 @@ class Release(models.Model):
     def __unicode__(self):
         return "%s (%s)" % (self.name, self.branch_name)
 
-class ReleaseLayerSourcePriority(models.Model):
-    """ Each release selects layers from the set up layer sources, ordered by priority """
-    release = models.ForeignKey("Release")
-    layer_source = models.ForeignKey("LayerSource")
-    priority = models.IntegerField(default = 0)
-
-    def __unicode__(self):
-        return "%s-%s:%d" % (self.release.name, self.layer_source.name, self.priority)
-    class Meta:
-        unique_together = (('release', 'layer_source'),)
-
+    def __str__(self):
+        return self.name
 
 class ReleaseDefaultLayer(models.Model):
     release = models.ForeignKey(Release)
     layer_name = models.CharField(max_length=100, default="")
 
 
-# Branch class is synced with layerindex.Branch, branches can only come from remote layer indexes
-class Branch(models.Model):
-    layer_source = models.ForeignKey('LayerSource', null = True, default = True)
-    up_id = models.IntegerField(null = True, default = None)                    # id of branch in the source
-    up_date = models.DateTimeField(null = True, default = None)
+class LayerSource(object):
+    """ Where the layer metadata came from """
+    TYPE_LOCAL = 0
+    TYPE_LAYERINDEX = 1
+    TYPE_IMPORTED = 2
+    TYPE_BUILD = 3
 
-    name = models.CharField(max_length=50)
-    short_description = models.CharField(max_length=50, blank=True)
+    SOURCE_TYPE = (
+        (TYPE_LOCAL, "local"),
+        (TYPE_LAYERINDEX, "layerindex"),
+        (TYPE_IMPORTED, "imported"),
+        (TYPE_BUILD, "build"),
+    )
 
-    class Meta:
-        verbose_name_plural = "Branches"
-        unique_together = (('layer_source', 'name'),('layer_source', 'up_id'))
-
-    def __unicode__(self):
-        return self.name
+    def types_dict():
+        """ Turn the TYPES enums into a simple dictionary """
+        dictionary = {}
+        for key in LayerSource.__dict__:
+            if "TYPE" in key:
+                dictionary[key] = getattr(LayerSource, key)
+        return dictionary
 
 
-# Layer class synced with layerindex.LayerItem
 class Layer(models.Model):
-    layer_source = models.ForeignKey(LayerSource, null = True, default = None)  # from where did we got this layer
-    up_id = models.IntegerField(null = True, default = None)                    # id of layer in the remote source
-    up_date = models.DateTimeField(null = True, default = None)
+
+    up_date = models.DateTimeField(null=True, default=timezone.now)
 
     name = models.CharField(max_length=100)
     layer_index_url = models.URLField()
-    vcs_url = GitURLField(default = None, null = True)
-    vcs_web_url = models.URLField(null = True, default = None)
-    vcs_web_tree_base_url = models.URLField(null = True, default = None)
-    vcs_web_file_base_url = models.URLField(null = True, default = None)
+    vcs_url = GitURLField(default=None, null=True)
+    local_source_dir = models.TextField(null = True, default = None)
+    vcs_web_url = models.URLField(null=True, default=None)
+    vcs_web_tree_base_url = models.URLField(null=True, default=None)
+    vcs_web_file_base_url = models.URLField(null=True, default=None)
 
-    summary = models.TextField(help_text='One-line description of the layer', null = True, default = None)
-    description = models.TextField(null = True, default = None)
+    summary = models.TextField(help_text='One-line description of the layer',
+                               null=True, default=None)
+    description = models.TextField(null=True, default=None)
 
     def __unicode__(self):
-        return "%s / %s " % (self.name, self.layer_source)
-
-    class Meta:
-        unique_together = (("layer_source", "up_id"), ("layer_source", "name"))
+        return "%s / %s " % (self.name, self.summary)
 
 
-# LayerCommit class is synced with layerindex.LayerBranch
 class Layer_Version(models.Model):
     """
     A Layer_Version either belongs to a single project or no project
     """
-    search_allowed_fields = ["layer__name", "layer__summary", "layer__description", "layer__vcs_url", "dirpath", "up_branch__name", "commit", "branch"]
-    build = models.ForeignKey(Build, related_name='layer_version_build', default = None, null = True)
+    search_allowed_fields = ["layer__name", "layer__summary",
+                             "layer__description", "layer__vcs_url",
+                             "dirpath", "release__name", "commit", "branch"]
+
+    build = models.ForeignKey(Build, related_name='layer_version_build',
+                              default=None, null=True)
+
     layer = models.ForeignKey(Layer, related_name='layer_version_layer')
 
-    layer_source = models.ForeignKey(LayerSource, null = True, default = None)                   # from where did we get this Layer Version
-    up_id = models.IntegerField(null = True, default = None)        # id of layerbranch in the remote source
-    up_date = models.DateTimeField(null = True, default = None)
-    up_branch = models.ForeignKey(Branch, null = True, default = None)
+    layer_source = models.IntegerField(choices=LayerSource.SOURCE_TYPE,
+                                       default=0)
 
-    branch = models.CharField(max_length=80)            # LayerBranch.actual_branch
-    commit = models.CharField(max_length=100)           # LayerBranch.vcs_last_rev
-    dirpath = models.CharField(max_length=255, null = True, default = None)          # LayerBranch.vcs_subdir
-    priority = models.IntegerField(default = 0)         # if -1, this is a default layer
+    up_date = models.DateTimeField(null=True, default=timezone.now)
 
-    local_path = models.FilePathField(max_length=1024, default = "/")  # where this layer was checked-out
+    # To which metadata release does this layer version belong to
+    release = models.ForeignKey(Release, null=True, default=None)
 
-    project = models.ForeignKey('Project', null = True, default = None)   # Set if this layer is project-specific; always set for imported layers, and project-set branches
+    branch = models.CharField(max_length=80)
+    commit = models.CharField(max_length=100)
+    # If the layer is in a subdir
+    dirpath = models.CharField(max_length=255, null=True, default=None)
 
-    # code lifted, with adaptations, from the layerindex-web application https://git.yoctoproject.org/cgit/cgit.cgi/layerindex-web/
+    # if -1, this is a default layer
+    priority = models.IntegerField(default=0)
+
+    # where this layer exists on the filesystem
+    local_path = models.FilePathField(max_length=1024, default="/")
+
+    # Set if this layer is restricted to a particular project
+    project = models.ForeignKey('Project', null=True, default=None)
+
+    # code lifted, with adaptations, from the layerindex-web application
+    # https://git.yoctoproject.org/cgit/cgit.cgi/layerindex-web/
     def _handle_url_path(self, base_url, path):
         import re, posixpath
         if base_url:
@@ -1626,7 +1431,7 @@ class Layer_Version(models.Model):
                     extra_path = self.dirpath
             else:
                 extra_path = path
-            branchname = self.up_branch.name
+            branchname = self.release.name
             url = base_url.replace('%branch%', branchname)
 
             # If there's a % in the path (e.g. a wildcard bbappend) we need to encode it
@@ -1651,23 +1456,19 @@ class Layer_Version(models.Model):
     def get_vcs_file_link_url(self, file_path=""):
         if self.layer.vcs_web_file_base_url is None:
             return None
-        return self._handle_url_path(self.layer.vcs_web_file_base_url, file_path)
+        return self._handle_url_path(self.layer.vcs_web_file_base_url,
+                                     file_path)
 
     def get_vcs_dirpath_link_url(self):
         if self.layer.vcs_web_tree_base_url is None:
             return None
         return self._handle_url_path(self.layer.vcs_web_tree_base_url, '')
 
-    def get_equivalents_wpriority(self, project):
-        layer_versions = project.get_all_compatible_layer_versions()
-        filtered = layer_versions.filter(layer__name = self.layer.name)
-        return filtered.order_by("-layer_source__releaselayersourcepriority__priority")
-
     def get_vcs_reference(self):
         if self.branch is not None and len(self.branch) > 0:
             return self.branch
-        if self.up_branch is not None:
-            return self.up_branch.name
+        if self.release is not None:
+            return self.release.name
         if self.commit is not None and len(self.commit) > 0:
             return self.commit
         return 'N/A'
@@ -1695,20 +1496,23 @@ class Layer_Version(models.Model):
         return sorted(result, key=lambda x: x.layer.name)
 
     def __unicode__(self):
-        return "%d %s (VCS %s, Project %s)" % (self.pk, str(self.layer), self.get_vcs_reference(), self.build.project if self.build is not None else "No project")
+        return ("id %d belongs to layer: %s" % (self.pk, self.layer.name))
+
+    def __str__(self):
+        if self.release:
+            release = self.release.name
+        else:
+            release = "No release set"
+
+        return "%d %s (%s)" % (self.pk, self.layer.name, release)
 
-    class Meta:
-        unique_together = ("layer_source", "up_id")
 
 class LayerVersionDependency(models.Model):
-    layer_source = models.ForeignKey(LayerSource, null = True, default = None)  # from where did we got this layer
-    up_id = models.IntegerField(null = True, default = None)                    # id of layerbranch in the remote source
 
-    layer_version = models.ForeignKey(Layer_Version, related_name="dependencies")
-    depends_on = models.ForeignKey(Layer_Version, related_name="dependees")
-
-    class Meta:
-        unique_together = ("layer_source", "up_id")
+    layer_version = models.ForeignKey(Layer_Version,
+                                      related_name="dependencies")
+    depends_on = models.ForeignKey(Layer_Version,
+                                   related_name="dependees")
 
 class ProjectLayer(models.Model):
     project = models.ForeignKey(Project)
@@ -1935,6 +1739,11 @@ def invalidate_cache(**kwargs):
     except Exception as e:
       logger.warning("Problem with cache backend: Failed to clear cache: %s" % e)
 
+def signal_runbuilds():
+    """Send SIGUSR1 to runbuilds process"""
+    with open(os.path.join(os.getenv('BUILDDIR'), '.runbuilds.pid')) as pidf:
+        os.kill(int(pidf.read()), SIGUSR1)
+
 django.db.models.signals.post_save.connect(invalidate_cache)
 django.db.models.signals.post_delete.connect(invalidate_cache)
 django.db.models.signals.m2m_changed.connect(invalidate_cache)

bitbake/lib/toaster/orm/tests.py

@@ -1,180 +0,0 @@
-#! /usr/bin/env python
-# ex:ts=4:sw=4:sts=4:et
-# -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*-
-#
-# BitBake Toaster Implementation
-#
-# Copyright (C) 2013-2015 Intel Corporation
-#
-# This program is free software; you can redistribute it and/or modify
-# it under the terms of the GNU General Public License version 2 as
-# published by the Free Software Foundation.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License along
-# with this program; if not, write to the Free Software Foundation, Inc.,
-# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
-
-"""Test cases for Toaster ORM."""
-
-from django.test import TestCase, TransactionTestCase
-from orm.models import LocalLayerSource, LayerIndexLayerSource, ImportedLayerSource, LayerSource
-from orm.models import Branch, LayerVersionDependency
-
-from orm.models import Project, Layer, Layer_Version, Branch, ProjectLayer
-from orm.models import Release, ReleaseLayerSourcePriority, BitbakeVersion
-
-from django.db import IntegrityError
-
-import os
-
-# set TTS_LAYER_INDEX to the base url to use a different instance of the layer index
-
-class LayerSourceVerifyInheritanceSaveLoad(TestCase):
-    """
-    Tests to verify inheritance for the LayerSource proxy-inheritance classes.
-    """
-    def test_object_creation(self):
-        """Test LayerSource object creation."""
-        for name, sourcetype in [("a1", LayerSource.TYPE_LOCAL),
-                                 ("a2", LayerSource.TYPE_LAYERINDEX),
-                                 ("a3", LayerSource.TYPE_IMPORTED)]:
-            LayerSource.objects.create(name=name, sourcetype=sourcetype)
-
-        objects = LayerSource.objects.all()
-        self.assertTrue(isinstance(objects[0], LocalLayerSource))
-        self.assertTrue(isinstance(objects[1], LayerIndexLayerSource))
-        self.assertTrue(isinstance(objects[2], ImportedLayerSource))
-
-    def test_duplicate_error(self):
-        """Test creation of duplicate LayerSource objects."""
-        stype = LayerSource.TYPE_LOCAL
-        LayerSource.objects.create(name="a1", sourcetype=stype)
-        with self.assertRaises(IntegrityError):
-            LayerSource.objects.create(name="a1", sourcetype=stype)
-
-
-class LILSUpdateTestCase(TransactionTestCase):
-    """Test Layer Source update."""
-
-    def setUp(self):
-        """Create release."""
-        bbv = BitbakeVersion.objects.create(\
-                  name="master", giturl="git://git.openembedded.org/bitbake")
-        Release.objects.create(name="default-release", bitbake_version=bbv,
-                               branch_name="master")
-
-    def test_update(self):
-        """Check if LayerSource.update can fetch branches."""
-        url = os.getenv("TTS_LAYER_INDEX",
-                        default="http://layers.openembedded.org/")
-
-        lsobj = LayerSource.objects.create(\
-                    name="b1", sourcetype=LayerSource.TYPE_LAYERINDEX,
-                    apiurl=url + "layerindex/api/")
-        lsobj.update()
-        self.assertTrue(lsobj.branch_set.all().count() > 0,
-                        "no branches fetched")
-
-class LayerVersionEquivalenceTestCase(TestCase):
-    """Verify Layer_Version priority selection."""
-
-    def setUp(self):
-        """Create required objects."""
-        # create layer source
-        self.lsrc = LayerSource.objects.create(name="dummy-layersource",
-                                               sourcetype=LayerSource.TYPE_LOCAL)
-        # create release
-        bbv = BitbakeVersion.objects.create(\
-                  name="master", giturl="git://git.openembedded.org/bitbake")
-        self.release = Release.objects.create(name="default-release",
-                                              bitbake_version=bbv,
-                                              branch_name="master")
-        # attach layer source to release
-        ReleaseLayerSourcePriority.objects.create(\
-            release=self.release, layer_source=self.lsrc, priority=1)
-
-        # create a layer version for the layer on the specified branch
-        self.layer = Layer.objects.create(name="meta-testlayer",
-                                          layer_source=self.lsrc)
-        self.branch = Branch.objects.create(name="master", layer_source=self.lsrc)
-        self.lver = Layer_Version.objects.create(\
-            layer=self.layer, layer_source=self.lsrc, up_branch=self.branch)
-
-        # create project and project layer
-        self.project = Project.objects.create_project(name="test-project",
-                                                      release=self.release)
-        ProjectLayer.objects.create(project=self.project,
-                                    layercommit=self.lver)
-
-        # create spoof layer that should not appear in the search results
-        layer = Layer.objects.create(name="meta-notvalid",
-                                     layer_source=self.lsrc)
-        self.lver2 = Layer_Version.objects.create(layer=layer,
-                                                  layer_source=self.lsrc,
-                                                  up_branch=self.branch)
-
-    def test_single_layersource(self):
-        """
-        When we have a single layer version,
-        get_equivalents_wpriority() should return a list with
-        just this layer_version.
-        """
-        equivqs = self.lver.get_equivalents_wpriority(self.project)
-        self.assertEqual(list(equivqs), [self.lver])
-
-    def test_dual_layersource(self):
-        """
-        If we have two layers with the same name, from different layer sources,
-        we expect both layers in, in increasing priority of the layer source.
-        """
-        lsrc2 = LayerSource.objects.create(\
-                    name="dummy-layersource2",
-                    sourcetype=LayerSource.TYPE_LOCAL,
-                    apiurl="test")
-
-        # assign a lower priority for the second layer source
-        self.release.releaselayersourcepriority_set.create(layer_source=lsrc2,
-                                                           priority=2)
-
-        # create a new layer_version for a layer with the same name
-        # coming from the second layer source
-        layer2 = Layer.objects.create(name="meta-testlayer",
-                                      layer_source=lsrc2)
-        lver2 = Layer_Version.objects.create(layer=layer2, layer_source=lsrc2,
-                                             up_branch=self.branch)
-
-        # expect two layer versions, in the priority order
-        equivqs = self.lver.get_equivalents_wpriority(self.project)
-        self.assertEqual(list(equivqs), [lver2, self.lver])
-
-    def test_compatible_layer_versions(self):
-        """
-        When we have a 2 layer versions, get_all_compatible_layerversions()
-        should return a queryset with both.
-        """
-        compat_lv = self.project.get_all_compatible_layer_versions()
-        self.assertEqual(list(compat_lv), [self.lver, self.lver2])
-
-    def test_layerversion_get_alldeps(self):
-        """Test Layer_Version.get_alldeps API."""
-        lvers = {}
-        for i in range(10):
-            name = "layer%d" % i
-            lvers[name] = Layer_Version.objects.create(layer=Layer.objects.create(name=name),
-                                                       project=self.project)
-            if i:
-                LayerVersionDependency.objects.create(layer_version=lvers["layer%d" % (i - 1)],
-                                                      depends_on=lvers[name])
-                # Check dinamically added deps
-                self.assertEqual(lvers['layer0'].get_alldeps(self.project.id),
-                                 [lvers['layer%d' % n] for n in range(1, i+1)])
-
-        # Check chain of deps created in previous loop
-        for i in range(10):
-            self.assertEqual(lvers['layer%d' % i].get_alldeps(self.project.id),
-                             [lvers['layer%d' % n] for n in range(i+1, 10)])

bitbake/lib/toaster/tests/browser/README

@@ -13,7 +13,7 @@ a Selenium test report with a version-specific format.
 To run tests against Chrome:
 
 * Download chromedriver for your host OS from
-  https://code.google.com/p/chromedriver/downloads/list
+  https://sites.google.com/a/chromium.org/chromedriver/downloads
 * On *nix systems, put chromedriver on PATH
 * On Windows, put chromedriver.exe in the same directory as chrome.exe
 
@@ -24,15 +24,30 @@ To run tests against PhantomJS (headless):
 * On *nix systems, put phantomjs on PATH
 * Not tested on Windows
 
-Firefox should work without requiring additional software to be installed.
+To run tests against Firefox, you may need to install the Marionette driver,
+depending on how new your version of Firefox is. One clue that you need to do
+this is if you see an exception like:
 
-The test case will instantiate a Selenium driver set by the
+  selenium.common.exceptions.WebDriverException: Message: The browser
+  appears to have exited before we could connect. If you specified
+  a log_file in the FirefoxBinary constructor, check it for details.
+
+See https://developer.mozilla.org/en-US/docs/Mozilla/QA/Marionette/WebDriver
+for installation instructions. Ensure that the Marionette executable (renamed
+as wires on Linux or wires.exe on Windows) is on your PATH; and use "marionette"
+as the browser string passed via TOASTER_TESTS_BROWSER (see below).
+
+(Note: The Toaster tests have been checked against Firefox 47 with the
+Marionette driver.)
+
+The test cases will instantiate a Selenium driver set by the
 TOASTER_TESTS_BROWSER environment variable, or Chrome if this is not specified.
 
 Available drivers:
 
 * chrome (default)
 * firefox
+* marionette (for newer Firefoxes)
 * ie
 * phantomjs
 

bitbake/lib/toaster/tests/browser/selenium_helpers_base.py

@@ -0,0 +1,218 @@
+#! /usr/bin/env python
+# ex:ts=4:sw=4:sts=4:et
+# -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*-
+#
+# BitBake Toaster Implementation
+#
+# Copyright (C) 2013-2016 Intel Corporation
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License version 2 as
+# published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License along
+# with this program; if not, write to the Free Software Foundation, Inc.,
+# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+#
+# The Wait class and some of SeleniumDriverHelper and SeleniumTestCase are
+# modified from Patchwork, released under the same licence terms as Toaster:
+# https://github.com/dlespiau/patchwork/blob/master/patchwork/tests.browser.py
+
+"""
+Helper methods for creating Toaster Selenium tests which run within
+the context of Django unit tests.
+"""
+
+import os
+import time
+import unittest
+
+from django.contrib.staticfiles.testing import StaticLiveServerTestCase
+from selenium import webdriver
+from selenium.webdriver.support.ui import WebDriverWait
+from selenium.webdriver.common.desired_capabilities import DesiredCapabilities
+from selenium.common.exceptions import NoSuchElementException, \
+        StaleElementReferenceException, TimeoutException
+
+def create_selenium_driver(browser='chrome'):
+    # set default browser string based on env (if available)
+    env_browser = os.environ.get('TOASTER_TESTS_BROWSER')
+    if env_browser:
+        browser = env_browser
+
+    if browser == 'chrome':
+        return webdriver.Chrome(
+            service_args=["--verbose", "--log-path=selenium.log"]
+        )
+    elif browser == 'firefox':
+        return webdriver.Firefox()
+    elif browser == 'marionette':
+        capabilities = DesiredCapabilities.FIREFOX
+        capabilities['marionette'] = True
+        return webdriver.Firefox(capabilities=capabilities)
+    elif browser == 'ie':
+        return webdriver.Ie()
+    elif browser == 'phantomjs':
+        return webdriver.PhantomJS()
+    else:
+        msg = 'Selenium driver for browser %s is not available' % browser
+        raise RuntimeError(msg)
+
+class Wait(WebDriverWait):
+    """
+    Subclass of WebDriverWait with predetermined timeout and poll
+    frequency. Also deals with a wider variety of exceptions.
+    """
+    _TIMEOUT = 10
+    _POLL_FREQUENCY = 0.5
+
+    def __init__(self, driver):
+        super(Wait, self).__init__(driver, self._TIMEOUT, self._POLL_FREQUENCY)
+
+    def until(self, method, message=''):
+        """
+        Calls the method provided with the driver as an argument until the
+        return value is not False.
+        """
+
+        end_time = time.time() + self._timeout
+        while True:
+            try:
+                value = method(self._driver)
+                if value:
+                    return value
+            except NoSuchElementException:
+                pass
+            except StaleElementReferenceException:
+                pass
+
+            time.sleep(self._poll)
+            if time.time() > end_time:
+                break
+
+        raise TimeoutException(message)
+
+    def until_not(self, method, message=''):
+        """
+        Calls the method provided with the driver as an argument until the
+        return value is False.
+        """
+
+        end_time = time.time() + self._timeout
+        while True:
+            try:
+                value = method(self._driver)
+                if not value:
+                    return value
+            except NoSuchElementException:
+                return True
+            except StaleElementReferenceException:
+                pass
+
+            time.sleep(self._poll)
+            if time.time() > end_time:
+                break
+
+        raise TimeoutException(message)
+
+class SeleniumTestCaseBase(unittest.TestCase):
+    """
+    NB StaticLiveServerTestCase is used as the base test case so that
+    static files are served correctly in a Selenium test run context; see
+    https://docs.djangoproject.com/en/1.9/ref/contrib/staticfiles/#specialized-test-case-to-support-live-testing
+    """
+
+    @classmethod
+    def setUpClass(cls):
+        """ Create a webdriver driver at the class level """
+
+        super(SeleniumTestCaseBase, cls).setUpClass()
+
+        # instantiate the Selenium webdriver once for all the test methods
+        # in this test case
+        cls.driver = create_selenium_driver()
+        cls.driver.maximize_window()
+
+    @classmethod
+    def tearDownClass(cls):
+        """ Clean up webdriver driver """
+
+        cls.driver.quit()
+        super(SeleniumTestCaseBase, cls).tearDownClass()
+
+    def get(self, url):
+        """
+        Selenium requires absolute URLs, so convert Django URLs returned
+        by resolve() or similar to absolute ones and get using the
+        webdriver instance.
+
+        url: a relative URL
+        """
+        abs_url = '%s%s' % (self.live_server_url, url)
+        self.driver.get(abs_url)
+
+    def find(self, selector):
+        """ Find single element by CSS selector """
+        return self.driver.find_element_by_css_selector(selector)
+
+    def find_all(self, selector):
+        """ Find all elements matching CSS selector """
+        return self.driver.find_elements_by_css_selector(selector)
+
+    def element_exists(self, selector):
+        """
+        Return True if one element matching selector exists,
+        False otherwise
+        """
+        return len(self.find_all(selector)) == 1
+
+    def focused_element(self):
+        """ Return the element which currently has focus on the page """
+        return self.driver.switch_to.active_element
+
+    def wait_until_present(self, selector):
+        """ Wait until element matching CSS selector is on the page """
+        is_present = lambda driver: self.find(selector)
+        msg = 'An element matching "%s" should be on the page' % selector
+        element = Wait(self.driver).until(is_present, msg)
+        return element
+
+    def wait_until_visible(self, selector):
+        """ Wait until element matching CSS selector is visible on the page """
+        is_visible = lambda driver: self.find(selector).is_displayed()
+        msg = 'An element matching "%s" should be visible' % selector
+        Wait(self.driver).until(is_visible, msg)
+        return self.find(selector)
+
+    def wait_until_focused(self, selector):
+        """ Wait until element matching CSS selector has focus """
+        is_focused = \
+            lambda driver: self.find(selector) == self.focused_element()
+        msg = 'An element matching "%s" should be focused' % selector
+        Wait(self.driver).until(is_focused, msg)
+        return self.find(selector)
+
+    def enter_text(self, selector, value):
+        """ Insert text into element matching selector """
+        # note that keyup events don't occur until the element is clicked
+        # (in the case of <input type="text"...>, for example), so simulate
+        # user clicking the element before inserting text into it
+        field = self.click(selector)
+
+        field.send_keys(value)
+        return field
+
+    def click(self, selector):
+        """ Click on element which matches CSS selector """
+        element = self.wait_until_visible(selector)
+        element.click()
+        return element
+
+    def get_page_source(self):
+        """ Get raw HTML for the current page """
+        return self.driver.page_source

bitbake/lib/toaster/tests/browser/test_all_builds_page.py

@@ -58,6 +58,13 @@ class TestAllBuildsPage(SeleniumTestCase):
             'outcome': Build.SUCCEEDED
         }
 
+        self.project1_build_failure = {
+            'project': self.project1,
+            'started_on': now,
+            'completed_on': now,
+            'outcome': Build.FAILED
+        }
+
         self.default_project_build_success = {
             'project': self.default_project,
             'started_on': now,
@@ -65,6 +72,46 @@ class TestAllBuildsPage(SeleniumTestCase):
             'outcome': Build.SUCCEEDED
         }
 
+    def _get_build_time_element(self, build):
+        """
+        Return the HTML element containing the build time for a build
+        in the recent builds area
+        """
+        selector = 'div[data-latest-build-result="%s"] ' \
+            '[data-role="data-recent-build-buildtime-field"]' % build.id
+
+        # because this loads via Ajax, wait for it to be visible
+        self.wait_until_present(selector)
+
+        build_time_spans = self.find_all(selector)
+
+        self.assertEqual(len(build_time_spans), 1)
+
+        return build_time_spans[0]
+
+    def _get_row_for_build(self, build):
+        """ Get the table row for the build from the all builds table """
+        self.wait_until_present('#allbuildstable')
+
+        rows = self.find_all('#allbuildstable tr')
+
+        # look for the row with a download link on the recipe which matches the
+        # build ID
+        url = reverse('builddashboard', args=(build.id,))
+        selector = 'td.target a[href="%s"]' % url
+
+        found_row = None
+        for row in rows:
+
+            outcome_links = row.find_elements_by_css_selector(selector)
+            if len(outcome_links) == 1:
+                found_row = row
+                break
+
+        self.assertNotEqual(found_row, None)
+
+        return found_row
+
     def test_show_tasks_with_suffix(self):
         """ Task should be shown as suffix on build name """
         build = Build.objects.create(**self.project1_build_success)
@@ -97,13 +144,13 @@ class TestAllBuildsPage(SeleniumTestCase):
         self.get(url)
 
         # shouldn't see a rebuild button for command-line builds
-        selector = 'div[data-latest-build-result="%s"] a.run-again-btn' % default_build.id
+        selector = 'div[data-latest-build-result="%s"] .rebuild-btn' % default_build.id
         run_again_button = self.find_all(selector)
         self.assertEqual(len(run_again_button), 0,
                          'should not see a rebuild button for cli builds')
 
         # should see a rebuild button for non-command-line builds
-        selector = 'div[data-latest-build-result="%s"] a.run-again-btn' % build1.id
+        selector = 'div[data-latest-build-result="%s"] .rebuild-btn' % build1.id
         run_again_button = self.find_all(selector)
         self.assertEqual(len(run_again_button), 1,
                          'should see a rebuild button for non-cli builds')
@@ -142,3 +189,45 @@ class TestAllBuildsPage(SeleniumTestCase):
             else:
                 msg = 'found unexpected project name cell in all builds table'
                 self.fail(msg)
+
+    def test_builds_time_links(self):
+        """
+        Successful builds should have links on the time column and in the
+        recent builds area; failed builds should not have links on the time column,
+        or in the recent builds area
+        """
+        build1 = Build.objects.create(**self.project1_build_success)
+        build2 = Build.objects.create(**self.project1_build_failure)
+
+        # add some targets to these builds so they have recipe links
+        # (and so we can find the row in the ToasterTable corresponding to
+        # a particular build)
+        Target.objects.create(build=build1, target='foo')
+        Target.objects.create(build=build2, target='bar')
+
+        url = reverse('all-builds')
+        self.get(url)
+
+        # test recent builds area for successful build
+        element = self._get_build_time_element(build1)
+        links = element.find_elements_by_css_selector('a')
+        msg = 'should be a link on the build time for a successful recent build'
+        self.assertEquals(len(links), 1, msg)
+
+        # test recent builds area for failed build
+        element = self._get_build_time_element(build2)
+        links = element.find_elements_by_css_selector('a')
+        msg = 'should not be a link on the build time for a failed recent build'
+        self.assertEquals(len(links), 0, msg)
+
+        # test the time column for successful build
+        build1_row = self._get_row_for_build(build1)
+        links = build1_row.find_elements_by_css_selector('td.time a')
+        msg = 'should be a link on the build time for a successful build'
+        self.assertEquals(len(links), 1, msg)
+
+        # test the time column for failed build
+        build2_row = self._get_row_for_build(build2)
+        links = build2_row.find_elements_by_css_selector('td.time a')
+        msg = 'should not be a link on the build time for a failed build'
+        self.assertEquals(len(links), 0, msg)

bitbake/lib/toaster/tests/browser/test_builddashboard_page.py

@@ -25,7 +25,7 @@ from django.utils import timezone
 from tests.browser.selenium_helpers import SeleniumTestCase
 
 from orm.models import Project, Release, BitbakeVersion, Build, LogMessage
-from orm.models import Layer, Layer_Version, Recipe, CustomImageRecipe
+from orm.models import Layer, Layer_Version, Recipe, CustomImageRecipe, Variable
 
 class TestBuildDashboardPage(SeleniumTestCase):
     """ Tests for the build dashboard /build/X """
@@ -42,15 +42,27 @@ class TestBuildDashboardPage(SeleniumTestCase):
 
         self.build1 = Build.objects.create(project=project,
                                            started_on=now,
-                                           completed_on=now)
+                                           completed_on=now,
+                                           outcome=Build.SUCCEEDED)
 
         self.build2 = Build.objects.create(project=project,
                                            started_on=now,
-                                           completed_on=now)
+                                           completed_on=now,
+                                           outcome=Build.SUCCEEDED)
 
         self.build3 = Build.objects.create(project=project,
                                            started_on=now,
-                                           completed_on=now)
+                                           completed_on=now,
+                                           outcome=Build.FAILED)
+
+        # add Variable objects to the successful builds, as this is the criterion
+        # used to determine whether the left-hand panel should be displayed
+        Variable.objects.create(build=self.build1,
+                                variable_name='Foo',
+                                variable_value='Bar')
+        Variable.objects.create(build=self.build2,
+                                variable_name='Foo',
+                                variable_value='Bar')
 
         # exception
         msg1 = 'an exception was thrown'
@@ -68,6 +80,22 @@ class TestBuildDashboardPage(SeleniumTestCase):
             message=msg2
         )
 
+        # error on the failed build
+        msg3 = 'an error occurred'
+        self.error_message = LogMessage.objects.create(
+            build=self.build3,
+            level=LogMessage.ERROR,
+            message=msg3
+        )
+
+        # warning on the failed build
+        msg4 = 'DANGER WILL ROBINSON'
+        self.warning_message = LogMessage.objects.create(
+            build=self.build3,
+            level=LogMessage.WARNING,
+            message=msg4
+        )
+
         # recipes related to the build, for testing the edit custom image/new
         # custom image buttons
         layer = Layer.objects.create(name='alayer')
@@ -151,36 +179,45 @@ class TestBuildDashboardPage(SeleniumTestCase):
         self._get_build_dashboard(build)
         return self.find_all('#errors div.alert-danger')
 
-    def _check_for_log_message(self, build, log_message):
+    def _check_for_log_message(self, message_elements, log_message):
+        """
+        Check that the LogMessage <log_message> has a representation in
+        the HTML elements <message_elements>.
+
+        message_elements: WebElements representing the log messages shown
+        in the build dashboard; each should have a <pre> element inside
+        it with a data-log-message-id attribute
+
+        log_message: orm.models.LogMessage instance
+        """
+        expected_text = log_message.message
+        expected_pk = str(log_message.pk)
+
+        found = False
+        for element in message_elements:
+            log_message_text = element.find_element_by_tag_name('pre').text.strip()
+            text_matches = (log_message_text == expected_text)
+
+            log_message_pk = element.get_attribute('data-log-message-id')
+            id_matches = (log_message_pk == expected_pk)
+
+            if text_matches and id_matches:
+                found = True
+                break
+
+        template_vars = (expected_text, expected_pk)
+        assertion_failed_msg = 'message not found: ' \
+            'expected text "%s" and ID %s' % template_vars
+        self.assertTrue(found, assertion_failed_msg)
+
+    def _check_for_error_message(self, build, log_message):
         """
         Check whether the LogMessage instance <log_message> is
         represented as an HTML error in the dashboard page for the Build object
         build
         """
         errors = self._get_build_dashboard_errors(build)
-        self.assertEqual(len(errors), 2)
-
-        expected_text = log_message.message
-        expected_id = str(log_message.id)
-
-        found = False
-        for error in errors:
-            error_text = error.find_element_by_tag_name('pre').text
-            text_matches = (error_text == expected_text)
-
-            error_id = error.get_attribute('data-error')
-            id_matches = (error_id == expected_id)
-
-            if text_matches and id_matches:
-                found = True
-                break
-
-        template_vars = (expected_text, error_text,
-                         expected_id, error_id)
-        assertion_error_msg = 'exception not found as error: ' \
-            'expected text "%s" and got "%s"; ' \
-            'expected ID %s and got %s' % template_vars
-        self.assertTrue(found, assertion_error_msg)
+        self._check_for_log_message(errors, log_message)
 
     def _check_labels_in_modal(self, modal, expected):
         """
@@ -203,14 +240,14 @@ class TestBuildDashboardPage(SeleniumTestCase):
         LogMessages with level EXCEPTION should display in the errors
         section of the page
         """
-        self._check_for_log_message(self.build1, self.exception_message)
+        self._check_for_error_message(self.build1, self.exception_message)
 
     def test_criticals_show_as_errors(self):
         """
         LogMessages with level CRITICAL should display in the errors
         section of the page
         """
-        self._check_for_log_message(self.build1, self.critical_message)
+        self._check_for_error_message(self.build1, self.critical_message)
 
     def test_edit_custom_image_button(self):
         """
@@ -268,3 +305,43 @@ class TestBuildDashboardPage(SeleniumTestCase):
         self.assertFalse(self.element_exists(selector),
             'new custom image button should not show for builds which ' \
             'don\'t have any image recipes')
+
+    def test_left_panel(self):
+        """"
+        Builds which succeed should have a left panel and a build summary
+        """
+        self._get_build_dashboard(self.build1)
+
+        left_panel = self.find_all('#nav')
+        self.assertEqual(len(left_panel), 1)
+
+        build_summary = self.find_all('[data-role="build-summary-heading"]')
+        self.assertEqual(len(build_summary), 1)
+
+    def test_failed_no_left_panel(self):
+        """
+        Builds which fail should have no left panel and no build summary
+        """
+        self._get_build_dashboard(self.build3)
+
+        left_panel = self.find_all('#nav')
+        self.assertEqual(len(left_panel), 0)
+
+        build_summary = self.find_all('[data-role="build-summary-heading"]')
+        self.assertEqual(len(build_summary), 0)
+
+    def test_failed_shows_errors_and_warnings(self):
+        """
+        Failed builds should still show error and warning messages
+        """
+        self._get_build_dashboard(self.build3)
+
+        errors = self.find_all('#errors div.alert-danger')
+        self._check_for_log_message(errors, self.error_message)
+
+        # expand the warnings area
+        self.click('#warning-toggle')
+        self.wait_until_visible('#warnings div.alert-warning')
+
+        warnings = self.find_all('#warnings div.alert-warning')
+        self._check_for_log_message(warnings, self.warning_message)

bitbake/lib/toaster/tests/browser/test_builddashboard_page_artifacts.py

@@ -26,7 +26,7 @@ from tests.browser.selenium_helpers import SeleniumTestCase
 
 from orm.models import Project, Release, BitbakeVersion, Build, Target, Package
 from orm.models import Target_Image_File, TargetSDKFile, TargetKernelFile
-from orm.models import Target_Installed_Package
+from orm.models import Target_Installed_Package, Variable
 
 class TestBuildDashboardPageArtifacts(SeleniumTestCase):
     """ Tests for artifacts on the build dashboard /build/X """
@@ -151,6 +151,11 @@ class TestBuildDashboardPageArtifacts(SeleniumTestCase):
             started_on=now, completed_on=timezone.now(),
             outcome=Build.SUCCEEDED)
 
+        # add a variable to the build so that it counts as "started"
+        Variable.objects.create(build=build,
+                                variable_name='Christopher',
+                                variable_value='Lee')
+
         target = Target.objects.create(is_image=True, build=build,
             task='', target='core-image-minimal',
             license_manifest_path='/home/foo/license.manifest',

bitbake/lib/toaster/tests/browser/test_builddashboard_page_recipes.py

@@ -0,0 +1,66 @@
+#! /usr/bin/env python
+# ex:ts=4:sw=4:sts=4:et
+# -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*-
+#
+# BitBake Toaster Implementation
+#
+# Copyright (C) 2013-2016 Intel Corporation
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License version 2 as
+# published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License along
+# with this program; if not, write to the Free Software Foundation, Inc.,
+# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+from django.core.urlresolvers import reverse
+from django.utils import timezone
+from tests.browser.selenium_helpers import SeleniumTestCase
+from orm.models import Project, Build, Recipe, Task, Layer, Layer_Version
+from orm.models import Target
+
+class TestBuilddashboardPageRecipes(SeleniumTestCase):
+    """ Test build dashboard recipes sub-page """
+
+    def setUp(self):
+        project = Project.objects.get_or_create_default_project()
+
+        now = timezone.now()
+
+        self.build = Build.objects.create(project=project,
+                                          started_on=now,
+                                          completed_on=now)
+
+        layer = Layer.objects.create()
+
+        layer_version = Layer_Version.objects.create(layer=layer,
+            build=self.build)
+
+        recipe = Recipe.objects.create(layer_version=layer_version)
+
+        task = Task.objects.create(build=self.build, recipe=recipe, order=1)
+
+        Target.objects.create(build=self.build, task=task, target='do_build')
+
+    def test_build_recipes_columns(self):
+        """
+        Check that non-hideable columns of the table on the recipes sub-page
+        are disabled on the edit columns dropdown.
+        """
+        url = reverse('recipes', args=(self.build.id,))
+        self.get(url)
+
+        self.wait_until_visible('#edit-columns-button')
+
+        # check that options for the non-hideable columns are disabled
+        non_hideable = ['name', 'version']
+
+        for column in non_hideable:
+            selector = 'input#checkbox-%s[disabled="disabled"]' % column
+            self.wait_until_present(selector)

bitbake/lib/toaster/tests/browser/test_builddashboard_page_tasks.py

@@ -0,0 +1,65 @@
+#! /usr/bin/env python
+# ex:ts=4:sw=4:sts=4:et
+# -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*-
+#
+# BitBake Toaster Implementation
+#
+# Copyright (C) 2013-2016 Intel Corporation
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License version 2 as
+# published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License along
+# with this program; if not, write to the Free Software Foundation, Inc.,
+# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+from django.core.urlresolvers import reverse
+from django.utils import timezone
+from tests.browser.selenium_helpers import SeleniumTestCase
+from orm.models import Project, Build, Recipe, Task, Layer, Layer_Version
+from orm.models import Target
+
+class TestBuilddashboardPageTasks(SeleniumTestCase):
+    """ Test build dashboard tasks sub-page """
+
+    def setUp(self):
+        project = Project.objects.get_or_create_default_project()
+
+        now = timezone.now()
+
+        self.build = Build.objects.create(project=project,
+                                          started_on=now,
+                                          completed_on=now)
+
+        layer = Layer.objects.create()
+
+        layer_version = Layer_Version.objects.create(layer=layer)
+
+        recipe = Recipe.objects.create(layer_version=layer_version)
+
+        task = Task.objects.create(build=self.build, recipe=recipe, order=1)
+
+        Target.objects.create(build=self.build, task=task, target='do_build')
+
+    def test_build_tasks_columns(self):
+        """
+        Check that non-hideable columns of the table on the tasks sub-page
+        are disabled on the edit columns dropdown.
+        """
+        url = reverse('tasks', args=(self.build.id,))
+        self.get(url)
+
+        self.wait_until_visible('#edit-columns-button')
+
+        # check that options for the non-hideable columns are disabled
+        non_hideable = ['order', 'task_name', 'recipe__name']
+
+        for column in non_hideable:
+            selector = 'input#checkbox-%s[disabled="disabled"]' % column
+            self.wait_until_present(selector)

bitbake/lib/toaster/tests/browser/test_layerdetails_page.py

@@ -49,9 +49,6 @@ class TestLayerDetailsPage(SeleniumTestCase):
         # project to add new custom images to
         self.project = Project.objects.create(name='foo', release=release)
 
-        layer_source = LayerSource.objects.create(
-            sourcetype=LayerSource.TYPE_IMPORTED)
-
         name = "meta-imported"
         vcs_url = "git://example.com/meta-imported"
         subdir = "/layer"
@@ -66,7 +63,7 @@ class TestLayerDetailsPage(SeleniumTestCase):
 
         self.imported_layer_version = Layer_Version.objects.create(
             layer=imported_layer,
-            layer_source=layer_source,
+            layer_source=LayerSource.TYPE_IMPORTED,
             branch=gitrev,
             commit=gitrev,
             dirpath=subdir,
@@ -85,15 +82,17 @@ class TestLayerDetailsPage(SeleniumTestCase):
         self.get(self.url)
 
         self.click("#add-remove-layer-btn")
+        self.click("#edit-layer-source")
+        self.click("#repo")
+
+        self.wait_until_visible("#layer-git-repo-url")
 
         # Open every edit box
         for btn in self.find_all("dd .glyphicon-edit"):
             btn.click()
 
-        self.wait_until_visible("dd input")
-
         # Edit each value
-        for inputs in self.find_all("dd input[type=text]") + \
+        for inputs in self.find_all("#layer-git input[type=text]") + \
                 self.find_all("dd textarea"):
             # ignore the tt inputs (twitter typeahead input)
             if "tt-" in inputs.get_attribute("class"):
@@ -107,17 +106,21 @@ class TestLayerDetailsPage(SeleniumTestCase):
 
             inputs.send_keys("-edited")
 
+        # Save the new values
         for save_btn in self.find_all(".change-btn"):
             save_btn.click()
 
+        self.click("#save-changes-for-switch")
+        self.wait_until_visible("#edit-layer-source")
+
         # Refresh the page to see if the new values are returned
         self.get(self.url)
 
         new_values = ["%s-edited" % old_val
                       for old_val in self.initial_values]
 
-        for inputs in self.find_all("dd input[type=text]") + \
-                self.find_all("dd textarea"):
+        for inputs in self.find_all('#layer-git input[type="text"]') + \
+                self.find_all('dd textarea'):
             # ignore the tt inputs (twitter typeahead input)
             if "tt-" in inputs.get_attribute("class"):
                 continue
@@ -125,8 +128,26 @@ class TestLayerDetailsPage(SeleniumTestCase):
             value = inputs.get_attribute("value")
 
             self.assertTrue(value in new_values,
-                            "Expecting any of \"%s\"but got \"%s\"" %
-                            (self.initial_values, value))
+                            "Expecting any of \"%s\" but got \"%s\"" %
+                            (new_values, value))
+
+        # Now convert it to a local layer
+        self.click("#edit-layer-source")
+        self.click("#dir")
+        dir_input = self.wait_until_visible("#layer-dir-path-in-details")
+
+        new_dir = "/home/test/my-meta-dir"
+        dir_input.send_keys(new_dir)
+
+        self.click("#save-changes-for-switch")
+        self.wait_until_visible("#edit-layer-source")
+
+        # Refresh the page to see if the new values are returned
+        self.get(self.url)
+        dir_input = self.find("#layer-dir-path-in-details")
+        self.assertTrue(new_dir in dir_input.get_attribute("value"),
+                        "Expected %s in the dir value for layer directory" %
+                        new_dir)
 
     def test_delete_layer(self):
         """ Delete the layer """

bitbake/lib/toaster/tests/browser/test_most_recent_builds_states.py

@@ -0,0 +1,211 @@
+#! /usr/bin/env python
+# ex:ts=4:sw=4:sts=4:et
+# -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*-
+#
+# BitBake Toaster Implementation
+#
+# Copyright (C) 2013-2016 Intel Corporation
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License version 2 as
+# published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License along
+# with this program; if not, write to the Free Software Foundation, Inc.,
+# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+from django.core.urlresolvers import reverse
+from django.utils import timezone
+from tests.browser.selenium_helpers import SeleniumTestCase
+from tests.browser.selenium_helpers_base import Wait
+from orm.models import Project, Build, Task, Recipe, Layer, Layer_Version
+from bldcontrol.models import BuildRequest
+
+class TestMostRecentBuildsStates(SeleniumTestCase):
+    """ Test states update correctly in most recent builds area """
+
+    def _create_build_request(self):
+        project = Project.objects.get_or_create_default_project()
+
+        now = timezone.now()
+
+        build = Build.objects.create(project=project, build_name='fakebuild',
+            started_on=now, completed_on=now)
+
+        return BuildRequest.objects.create(build=build, project=project,
+            state=BuildRequest.REQ_QUEUED)
+
+    def _create_recipe(self):
+        """ Add a recipe to the database and return it """
+        layer = Layer.objects.create()
+        layer_version = Layer_Version.objects.create(layer=layer)
+        return Recipe.objects.create(name='foo', layer_version=layer_version)
+
+    def _check_build_states(self, build_request):
+        recipes_to_parse = 10
+        url = reverse('all-builds')
+        self.get(url)
+
+        build = build_request.build
+        base_selector = '[data-latest-build-result="%s"] ' % build.id
+
+        # build queued; check shown as queued
+        selector = base_selector + '[data-build-state="Queued"]'
+        element = self.wait_until_visible(selector)
+        self.assertRegexpMatches(element.get_attribute('innerHTML'),
+            'Build queued', 'build should show queued status')
+
+        # waiting for recipes to be parsed
+        build.outcome = Build.IN_PROGRESS
+        build.recipes_to_parse = recipes_to_parse
+        build.recipes_parsed = 0
+
+        build_request.state = BuildRequest.REQ_INPROGRESS
+        build_request.save()
+
+        self.get(url)
+
+        selector = base_selector + '[data-build-state="Parsing"]'
+        element = self.wait_until_visible(selector)
+
+        bar_selector = '#recipes-parsed-percentage-bar-%s' % build.id
+        bar_element = element.find_element_by_css_selector(bar_selector)
+        self.assertEqual(bar_element.value_of_css_property('width'), '0px',
+            'recipe parse progress should be at 0')
+
+        # recipes being parsed; check parse progress
+        build.recipes_parsed = 5
+        build.save()
+
+        self.get(url)
+
+        element = self.wait_until_visible(selector)
+        bar_element = element.find_element_by_css_selector(bar_selector)
+        recipe_bar_updated = lambda driver: \
+            bar_element.get_attribute('style') == 'width: 50%;'
+        msg = 'recipe parse progress bar should update to 50%'
+        element = Wait(self.driver).until(recipe_bar_updated, msg)
+
+        # all recipes parsed, task started, waiting for first task to finish;
+        # check status is shown as "Tasks starting..."
+        build.recipes_parsed = recipes_to_parse
+        build.save()
+
+        recipe = self._create_recipe()
+        task1 = Task.objects.create(build=build, recipe=recipe,
+            task_name='Lionel')
+        task2 = Task.objects.create(build=build, recipe=recipe,
+            task_name='Jeffries')
+
+        self.get(url)
+
+        selector = base_selector + '[data-build-state="Starting"]'
+        element = self.wait_until_visible(selector)
+        self.assertRegexpMatches(element.get_attribute('innerHTML'),
+            'Tasks starting', 'build should show "tasks starting" status')
+
+        # first task finished; check tasks progress bar
+        task1.order = 1
+        task1.save()
+
+        self.get(url)
+
+        selector = base_selector + '[data-build-state="In Progress"]'
+        element = self.wait_until_visible(selector)
+
+        bar_selector = '#build-pc-done-bar-%s' % build.id
+        bar_element = element.find_element_by_css_selector(bar_selector)
+
+        task_bar_updated = lambda driver: \
+            bar_element.get_attribute('style') == 'width: 50%;'
+        msg = 'tasks progress bar should update to 50%'
+        element = Wait(self.driver).until(task_bar_updated, msg)
+
+        # last task finished; check tasks progress bar updates
+        task2.order = 2
+        task2.save()
+
+        self.get(url)
+
+        element = self.wait_until_visible(selector)
+        bar_element = element.find_element_by_css_selector(bar_selector)
+        task_bar_updated = lambda driver: \
+            bar_element.get_attribute('style') == 'width: 100%;'
+        msg = 'tasks progress bar should update to 100%'
+        element = Wait(self.driver).until(task_bar_updated, msg)
+
+    def test_states_to_success(self):
+        """
+        Test state transitions in the recent builds area for a build which
+        completes successfully.
+        """
+        build_request = self._create_build_request()
+
+        self._check_build_states(build_request)
+
+        # all tasks complete and build succeeded; check success state shown
+        build = build_request.build
+        build.outcome = Build.SUCCEEDED
+        build.save()
+
+        selector = '[data-latest-build-result="%s"] ' \
+            '[data-build-state="Succeeded"]' % build.id
+        element = self.wait_until_visible(selector)
+
+    def test_states_to_failure(self):
+        """
+        Test state transitions in the recent builds area for a build which
+        completes in a failure.
+        """
+        build_request = self._create_build_request()
+
+        self._check_build_states(build_request)
+
+        # all tasks complete and build succeeded; check fail state shown
+        build = build_request.build
+        build.outcome = Build.FAILED
+        build.save()
+
+        selector = '[data-latest-build-result="%s"] ' \
+            '[data-build-state="Failed"]' % build.id
+        element = self.wait_until_visible(selector)
+
+    def test_states_cancelling(self):
+        """
+        Test that most recent build area updates correctly for a build
+        which is cancelled.
+        """
+        url = reverse('all-builds')
+
+        build_request = self._create_build_request()
+        build = build_request.build
+
+        # cancel the build
+        build_request.state = BuildRequest.REQ_CANCELLING
+        build_request.save()
+
+        self.get(url)
+
+        # check cancelling state
+        selector = '[data-latest-build-result="%s"] ' \
+            '[data-build-state="Cancelling"]' % build.id
+        element = self.wait_until_visible(selector)
+        self.assertRegexpMatches(element.get_attribute('innerHTML'),
+            'Cancelling the build', 'build should show "cancelling" status')
+
+        # check cancelled state
+        build.outcome = Build.CANCELLED
+        build.save()
+
+        self.get(url)
+
+        selector = '[data-latest-build-result="%s"] ' \
+            '[data-build-state="Cancelled"]' % build.id
+        element = self.wait_until_visible(selector)
+        self.assertRegexpMatches(element.get_attribute('innerHTML'),
+            'Build cancelled', 'build should show "cancelled" status')

bitbake/lib/toaster/tests/browser/test_new_project_page.py

@@ -22,6 +22,7 @@
 from django.core.urlresolvers import reverse
 from tests.browser.selenium_helpers import SeleniumTestCase
 from selenium.webdriver.support.ui import Select
+from selenium.common.exceptions import InvalidElementStateException
 
 from orm.models import Project, Release, BitbakeVersion
 
@@ -102,7 +103,10 @@ class TestNewProjectPage(SeleniumTestCase):
 
         # Try and click it anyway, if it submits we'll have a new project in
         # the db and assert then
-        self.click("#create-project-button")
+        try:
+            self.click("#create-project-button")
+        except InvalidElementStateException:
+            pass
 
         self.assertTrue(
             (Project.objects.filter(name=project_name).count() == 1),

bitbake/lib/toaster/tests/browser/test_task_page.py

@@ -0,0 +1,76 @@
+#! /usr/bin/env python
+# ex:ts=4:sw=4:sts=4:et
+# -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*-
+#
+# BitBake Toaster Implementation
+#
+# Copyright (C) 2013-2016 Intel Corporation
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License version 2 as
+# published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License along
+# with this program; if not, write to the Free Software Foundation, Inc.,
+# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+from django.core.urlresolvers import reverse
+from django.utils import timezone
+from tests.browser.selenium_helpers import SeleniumTestCase
+from orm.models import Project, Build, Layer, Layer_Version, Recipe, Target
+from orm.models import Task, Task_Dependency
+
+class TestTaskPage(SeleniumTestCase):
+    """ Test page which shows an individual task """
+    RECIPE_NAME = 'bar'
+    RECIPE_VERSION = '0.1'
+    TASK_NAME = 'do_da_doo_ron_ron'
+
+    def setUp(self):
+        now = timezone.now()
+
+        project = Project.objects.get_or_create_default_project()
+
+        self.build = Build.objects.create(project=project, started_on=now,
+            completed_on=now)
+
+        Target.objects.create(target='foo', build=self.build)
+
+        layer = Layer.objects.create()
+
+        layer_version = Layer_Version.objects.create(layer=layer)
+
+        recipe = Recipe.objects.create(name=TestTaskPage.RECIPE_NAME,
+            layer_version=layer_version, version=TestTaskPage.RECIPE_VERSION)
+
+        self.task = Task.objects.create(build=self.build, recipe=recipe,
+            order=1, outcome=Task.OUTCOME_COVERED, task_executed=False,
+            task_name=TestTaskPage.TASK_NAME)
+
+    def test_covered_task(self):
+        """
+        Check that covered tasks are displayed for tasks which have
+        dependencies on themselves
+        """
+
+        # the infinite loop which of bug 9952 was down to tasks which
+        # depend on themselves, so add self-dependent tasks to replicate the
+        # situation which caused the infinite loop (now fixed)
+        Task_Dependency.objects.create(task=self.task, depends_on=self.task)
+
+        url = reverse('task', args=(self.build.id, self.task.id,))
+        self.get(url)
+
+        # check that we see the task name
+        self.wait_until_visible('.page-header h1')
+
+        heading = self.find('.page-header h1')
+        expected_heading = '%s_%s %s' % (TestTaskPage.RECIPE_NAME,
+            TestTaskPage.RECIPE_VERSION, TestTaskPage.TASK_NAME)
+        self.assertEqual(heading.text, expected_heading,
+            'Heading should show recipe name, version and task')

bitbake/lib/toaster/tests/browser/test_toastertable_ui.py

@@ -0,0 +1,160 @@
+#! /usr/bin/env python
+# ex:ts=4:sw=4:sts=4:et
+# -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*-
+#
+# BitBake Toaster Implementation
+#
+# Copyright (C) 2013-2016 Intel Corporation
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License version 2 as
+# published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License along
+# with this program; if not, write to the Free Software Foundation, Inc.,
+# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+from datetime import datetime
+
+from django.core.urlresolvers import reverse
+from django.utils import timezone
+from tests.browser.selenium_helpers import SeleniumTestCase
+from orm.models import BitbakeVersion, Release, Project, Build
+
+class TestToasterTableUI(SeleniumTestCase):
+    """
+    Tests for the UI elements of ToasterTable (sorting etc.);
+    note that the tests cover generic functionality of ToasterTable which
+    manifests as UI elements in the browser, and can only be tested via
+    Selenium.
+    """
+
+    def setUp(self):
+        pass
+
+    def _get_orderby_heading(self, table):
+        """
+        Get the current order by finding the column heading in <table> with
+        the sorted class on it.
+
+        table: WebElement for a ToasterTable
+        """
+        selector = 'thead a.sorted'
+        heading = table.find_element_by_css_selector(selector)
+        return heading.get_attribute('innerHTML').strip()
+
+    def _get_datetime_from_cell(self, row, selector):
+        """
+        Return the value in the cell selected by <selector> on <row> as a
+        datetime.
+
+        row: <tr> WebElement for a row in the ToasterTable
+        selector: CSS selector to use to find the cell containing the date time
+        string
+        """
+        cell = row.find_element_by_css_selector(selector)
+        cell_text = cell.get_attribute('innerHTML').strip()
+        return datetime.strptime(cell_text, '%d/%m/%y %H:%M')
+
+    def test_revert_orderby(self):
+        """
+        Test that sort order for a table reverts to the default sort order
+        if the current sort column is hidden.
+        """
+        now = timezone.now()
+        later = now + timezone.timedelta(hours=1)
+        even_later = later + timezone.timedelta(hours=1)
+
+        bbv = BitbakeVersion.objects.create(name='test bbv', giturl='/tmp/',
+                                            branch='master', dirpath='')
+        release = Release.objects.create(name='test release',
+                                         branch_name='master',
+                                         bitbake_version=bbv)
+
+        project = Project.objects.create_project('project', release)
+
+        # set up two builds which will order differently when sorted by
+        # started_on or completed_on
+
+        # started first, finished last
+        build1 = Build.objects.create(project=project,
+                                      started_on=now,
+                                      completed_on=even_later,
+                                      outcome=Build.SUCCEEDED)
+
+        # started second, finished first
+        build2 = Build.objects.create(project=project,
+                                      started_on=later,
+                                      completed_on=later,
+                                      outcome=Build.SUCCEEDED)
+
+        url = reverse('all-builds')
+        self.get(url)
+        table = self.wait_until_visible('#allbuildstable')
+
+        # check ordering (default is by -completed_on); so build1 should be
+        # first as it finished last
+        active_heading = self._get_orderby_heading(table)
+        self.assertEqual(active_heading, 'Completed on',
+            'table should be sorted by "Completed on" by default')
+
+        row_selector = '#allbuildstable tbody tr'
+        cell_selector = 'td.completed_on'
+
+        rows = self.find_all(row_selector)
+        row1_completed_on = self._get_datetime_from_cell(rows[0], cell_selector)
+        row2_completed_on = self._get_datetime_from_cell(rows[1], cell_selector)
+        self.assertTrue(row1_completed_on > row2_completed_on,
+            'table should be sorted by -completed_on')
+
+        # turn on started_on column
+        self.click('#edit-columns-button')
+        self.click('#checkbox-started_on')
+
+        # sort by started_on column
+        links = table.find_elements_by_css_selector('th.started_on a')
+        for link in links:
+            if link.get_attribute('innerHTML').strip() == 'Started on':
+                link.click()
+                break
+
+        # wait for table data to reload in response to new sort
+        self.wait_until_visible('#allbuildstable')
+
+        # check ordering; build1 should be first
+        active_heading = self._get_orderby_heading(table)
+        self.assertEqual(active_heading, 'Started on',
+            'table should be sorted by "Started on"')
+
+        cell_selector = 'td.started_on'
+
+        rows = self.find_all(row_selector)
+        row1_started_on = self._get_datetime_from_cell(rows[0], cell_selector)
+        row2_started_on = self._get_datetime_from_cell(rows[1], cell_selector)
+        self.assertTrue(row1_started_on < row2_started_on,
+            'table should be sorted by started_on')
+
+        # turn off started_on column
+        self.click('#edit-columns-button')
+        self.click('#checkbox-started_on')
+
+        # wait for table data to reload in response to new sort
+        self.wait_until_visible('#allbuildstable')
+
+        # check ordering (should revert to completed_on); build2 should be first
+        active_heading = self._get_orderby_heading(table)
+        self.assertEqual(active_heading, 'Completed on',
+            'table should be sorted by "Completed on" after hiding sort column')
+
+        cell_selector = 'td.completed_on'
+
+        rows = self.find_all(row_selector)
+        row1_completed_on = self._get_datetime_from_cell(rows[0], cell_selector)
+        row2_completed_on = self._get_datetime_from_cell(rows[1], cell_selector)
+        self.assertTrue(row1_completed_on > row2_completed_on,
+            'table should be sorted by -completed_on')

bitbake/lib/toaster/tests/db/test_db.py

@@ -0,0 +1,55 @@
+# The MIT License (MIT)
+#
+# Copyright (c) 2016 Damien Lespiau
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+# SOFTWARE.
+
+import sys
+
+try:
+    from StringIO import StringIO
+except ImportError:
+    from io import StringIO
+
+from contextlib import contextmanager
+
+from django.core import management
+from django.test import TestCase
+
+
+@contextmanager
+def capture(command, *args, **kwargs):
+    out, sys.stdout = sys.stdout, StringIO()
+    command(*args, **kwargs)
+    sys.stdout.seek(0)
+    yield sys.stdout.read()
+    sys.stdout = out
+
+
+def makemigrations():
+    management.call_command('makemigrations')
+
+
+class MigrationTest(TestCase):
+
+    def testPendingMigration(self):
+        """Make sure there's no pending migration."""
+
+        with capture(makemigrations) as output:
+            self.assertEqual(output, "No changes detected\n")

bitbake/lib/toaster/toastergui/api.py

@@ -16,17 +16,28 @@
 # with this program; if not, write to the Free Software Foundation, Inc.,
 # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
 
+# Please run flake8 on this file before sending patches
 
-# Temporary home for the UI's misc API
 import re
+import logging
 
 from orm.models import Project, ProjectTarget, Build, Layer_Version
 from orm.models import LayerVersionDependency, LayerSource, ProjectLayer
+from orm.models import Recipe, CustomImageRecipe, CustomImagePackage
+from orm.models import Layer, Target, Package, Package_Dependency
 from bldcontrol.models import BuildRequest
 from bldcontrol import bbcontroller
+
 from django.http import HttpResponse, JsonResponse
 from django.views.generic import View
 from django.core.urlresolvers import reverse
+from django.utils import timezone
+from django.db.models import Q, F
+from django.db import Error
+from toastergui.templatetags.projecttags import json, sectohms, get_tasks
+from toastergui.templatetags.projecttags import filtered_filesizeformat
+
+logger = logging.getLogger("toaster")
 
 
 def error_response(error):
@@ -84,7 +95,7 @@ class XhrBuildRequest(View):
                     br.save()
 
                 except BuildRequest.DoesNotExist:
-                    return error_response('No such build id %s' % i)
+                    return error_response('No such build request id %s' % i)
 
             return error_response('ok')
 
@@ -132,7 +143,8 @@ class XhrLayer(View):
           Method: POST
 
           Args:
-              vcs_url, dirpath, commit, up_branch, summary, description
+              vcs_url, dirpath, commit, up_branch, summary, description,
+              local_source_dir
 
               add_dep = append a layerversion_id as a dependency
               rm_dep = remove a layerversion_id as a depedency
@@ -147,7 +159,7 @@ class XhrLayer(View):
             layer_version = Layer_Version.objects.get(
                 id=kwargs['layerversion_id'],
                 project=kwargs['pid'],
-                layer_source__sourcetype=LayerSource.TYPE_IMPORTED)
+                layer_source=LayerSource.TYPE_IMPORTED)
 
         except Layer_Version.DoesNotExist:
             return error_response("Cannot find imported layer to update")
@@ -159,12 +171,13 @@ class XhrLayer(View):
         if "commit" in request.POST:
             layer_version.commit = request.POST["commit"]
             layer_version.branch = request.POST["commit"]
-        if "up_branch" in request.POST:
-            layer_version.up_branch_id = int(request.POST["up_branch"])
         if "summary" in request.POST:
             layer_version.layer.summary = request.POST["summary"]
         if "description" in request.POST:
             layer_version.layer.description = request.POST["description"]
+        if "local_source_dir" in request.POST:
+            layer_version.layer.local_source_dir = \
+                request.POST["local_source_dir"]
 
         if "add_dep" in request.POST:
             lvd = LayerVersionDependency(
@@ -193,7 +206,7 @@ class XhrLayer(View):
             layer_version = Layer_Version.objects.get(
                 id=kwargs['layerversion_id'],
                 project=kwargs['pid'],
-                layer_source__sourcetype=LayerSource.TYPE_IMPORTED)
+                layer_source=LayerSource.TYPE_IMPORTED)
         except Layer_Version.DoesNotExist:
             return error_response("Cannot find imported layer to delete")
 
@@ -210,3 +223,552 @@ class XhrLayer(View):
             "error": "ok",
             "redirect": reverse('project', args=(kwargs['pid'],))
         })
+
+
+class MostRecentBuildsView(View):
+    def _was_yesterday_or_earlier(self, completed_on):
+        now = timezone.now()
+        delta = now - completed_on
+
+        if delta.days >= 1:
+            return True
+
+        return False
+
+    def get(self, request, *args, **kwargs):
+        """
+        Returns a list of builds in JSON format.
+        """
+        project = None
+
+        project_id = request.GET.get('project_id', None)
+        if project_id:
+            try:
+                project = Project.objects.get(pk=project_id)
+            except:
+                # if project lookup fails, assume no project
+                pass
+
+        recent_build_objs = Build.get_recent(project)
+        recent_builds = []
+
+        for build_obj in recent_build_objs:
+            dashboard_url = reverse('builddashboard', args=(build_obj.pk,))
+            buildtime_url = reverse('buildtime', args=(build_obj.pk,))
+            rebuild_url = \
+                reverse('xhr_buildrequest', args=(build_obj.project.pk,))
+            cancel_url = \
+                reverse('xhr_buildrequest', args=(build_obj.project.pk,))
+
+            build = {}
+            build['id'] = build_obj.pk
+            build['dashboard_url'] = dashboard_url
+
+            buildrequest_id = None
+            if hasattr(build_obj, 'buildrequest'):
+                buildrequest_id = build_obj.buildrequest.pk
+            build['buildrequest_id'] = buildrequest_id
+
+            build['recipes_parsed_percentage'] = \
+                int((build_obj.recipes_parsed /
+                     build_obj.recipes_to_parse) * 100)
+
+            tasks_complete_percentage = 0
+            if build_obj.outcome in (Build.SUCCEEDED, Build.FAILED):
+                tasks_complete_percentage = 100
+            elif build_obj.outcome == Build.IN_PROGRESS:
+                tasks_complete_percentage = build_obj.completeper()
+            build['tasks_complete_percentage'] = tasks_complete_percentage
+
+            build['state'] = build_obj.get_state()
+
+            build['errors'] = build_obj.errors.count()
+            build['dashboard_errors_url'] = dashboard_url + '#errors'
+
+            build['warnings'] = build_obj.warnings.count()
+            build['dashboard_warnings_url'] = dashboard_url + '#warnings'
+
+            build['buildtime'] = sectohms(build_obj.timespent_seconds)
+            build['buildtime_url'] = buildtime_url
+
+            build['rebuild_url'] = rebuild_url
+            build['cancel_url'] = cancel_url
+
+            build['is_default_project_build'] = build_obj.project.is_default
+
+            build['build_targets_json'] = \
+                json(get_tasks(build_obj.target_set.all()))
+
+            # convert completed_on time to user's timezone
+            completed_on = timezone.localtime(build_obj.completed_on)
+
+            completed_on_template = '%H:%M'
+            if self._was_yesterday_or_earlier(completed_on):
+                completed_on_template = '%d/%m/%Y ' + completed_on_template
+            build['completed_on'] = completed_on.strftime(
+                completed_on_template)
+
+            targets = []
+            target_objs = build_obj.get_sorted_target_list()
+            for target_obj in target_objs:
+                if target_obj.task:
+                    targets.append(target_obj.target + ':' + target_obj.task)
+                else:
+                    targets.append(target_obj.target)
+            build['targets'] = ' '.join(targets)
+
+            # abbreviated form of the full target list
+            abbreviated_targets = ''
+            num_targets = len(targets)
+            if num_targets > 0:
+                abbreviated_targets = targets[0]
+            if num_targets > 1:
+                abbreviated_targets += (' +%s' % (num_targets - 1))
+            build['targets_abbreviated'] = abbreviated_targets
+
+            recent_builds.append(build)
+
+        return JsonResponse(recent_builds, safe=False)
+
+
+class XhrCustomRecipe(View):
+    """ Create a custom image recipe """
+
+    def post(self, request, *args, **kwargs):
+        """
+        Custom image recipe REST API
+
+        Entry point: /xhr_customrecipe/
+        Method: POST
+
+        Args:
+            name: name of custom recipe to create
+            project: target project id of orm.models.Project
+            base: base recipe id of orm.models.Recipe
+
+        Returns:
+            {"error": "ok",
+             "url": <url of the created recipe>}
+            or
+            {"error": <error message>}
+        """
+        # check if request has all required parameters
+        for param in ('name', 'project', 'base'):
+            if param not in request.POST:
+                return error_response("Missing parameter '%s'" % param)
+
+        # get project and baserecipe objects
+        params = {}
+        for name, model in [("project", Project),
+                            ("base", Recipe)]:
+            value = request.POST[name]
+            try:
+                params[name] = model.objects.get(id=value)
+            except model.DoesNotExist:
+                return error_response("Invalid %s id %s" % (name, value))
+
+        # create custom recipe
+        try:
+
+            # Only allowed chars in name are a-z, 0-9 and -
+            if re.search(r'[^a-z|0-9|-]', request.POST["name"]):
+                return error_response("invalid-name")
+
+            custom_images = CustomImageRecipe.objects.all()
+
+            # Are there any recipes with this name already in our project?
+            existing_image_recipes_in_project = custom_images.filter(
+                name=request.POST["name"], project=params["project"])
+
+            if existing_image_recipes_in_project.count() > 0:
+                return error_response("image-already-exists")
+
+            # Are there any recipes with this name which aren't custom
+            # image recipes?
+            custom_image_ids = custom_images.values_list('id', flat=True)
+            existing_non_image_recipes = Recipe.objects.filter(
+                Q(name=request.POST["name"]) & ~Q(pk__in=custom_image_ids)
+            )
+
+            if existing_non_image_recipes.count() > 0:
+                return error_response("recipe-already-exists")
+
+            # create layer 'Custom layer' and verion if needed
+            layer = Layer.objects.get_or_create(
+                name=CustomImageRecipe.LAYER_NAME,
+                summary="Layer for custom recipes",
+                vcs_url="file:///toaster_created_layer")[0]
+
+            # Check if we have a layer version already
+            # We don't use get_or_create here because the dirpath will change
+            # and is a required field
+            lver = Layer_Version.objects.filter(Q(project=params['project']) &
+                                                Q(layer=layer) &
+                                                Q(build=None)).last()
+            if lver is None:
+                lver, created = Layer_Version.objects.get_or_create(
+                    project=params['project'],
+                    layer=layer,
+                    dirpath="toaster_created_layer")
+
+            # Add a dependency on our layer to the base recipe's layer
+            LayerVersionDependency.objects.get_or_create(
+                layer_version=lver,
+                depends_on=params["base"].layer_version)
+
+            # Add it to our current project if needed
+            ProjectLayer.objects.get_or_create(project=params['project'],
+                                               layercommit=lver,
+                                               optional=False)
+
+            # Create the actual recipe
+            recipe, created = CustomImageRecipe.objects.get_or_create(
+                name=request.POST["name"],
+                base_recipe=params["base"],
+                project=params["project"],
+                layer_version=lver,
+                is_image=True)
+
+            # If we created the object then setup these fields. They may get
+            # overwritten later on and cause the get_or_create to create a
+            # duplicate if they've changed.
+            if created:
+                recipe.file_path = request.POST["name"]
+                recipe.license = "MIT"
+                recipe.version = "0.1"
+                recipe.save()
+
+        except Error as err:
+            return error_response("Can't create custom recipe: %s" % err)
+
+        # Find the package list from the last build of this recipe/target
+        target = Target.objects.filter(Q(build__outcome=Build.SUCCEEDED) &
+                                       Q(build__project=params['project']) &
+                                       (Q(target=params['base'].name) |
+                                        Q(target=recipe.name))).last()
+        if target:
+            # Copy in every package
+            # We don't want these packages to be linked to anything because
+            # that underlying data may change e.g. delete a build
+            for tpackage in target.target_installed_package_set.all():
+                try:
+                    built_package = tpackage.package
+                    # The package had no recipe information so is a ghost
+                    # package skip it
+                    if built_package.recipe is None:
+                        continue
+
+                    config_package = CustomImagePackage.objects.get(
+                        name=built_package.name)
+
+                    recipe.includes_set.add(config_package)
+                except Exception as e:
+                    logger.warning("Error adding package %s %s" %
+                                   (tpackage.package.name, e))
+                    pass
+
+        return JsonResponse(
+            {"error": "ok",
+             "packages": recipe.get_all_packages().count(),
+             "url": reverse('customrecipe', args=(params['project'].pk,
+                                                  recipe.id))})
+
+
+class XhrCustomRecipeId(View):
+    """
+    Set of ReST API processors working with recipe id.
+
+    Entry point: /xhr_customrecipe/<recipe_id>
+
+    Methods:
+        GET - Get details of custom image recipe
+        DELETE - Delete custom image recipe
+
+    Returns:
+        GET:
+        {"error": "ok",
+        "info": dictionary of field name -> value pairs
+        of the CustomImageRecipe model}
+        DELETE:
+        {"error": "ok"}
+          or
+        {"error": <error message>}
+    """
+    @staticmethod
+    def _get_ci_recipe(recipe_id):
+        """ Get Custom Image recipe or return an error response"""
+        try:
+            custom_recipe = \
+                    CustomImageRecipe.objects.get(pk=recipe_id)
+            return custom_recipe, None
+
+        except CustomImageRecipe.DoesNotExist:
+            return None, error_response("Custom recipe with id=%s "
+                                        "not found" % recipe_id)
+
+    def get(self, request, *args, **kwargs):
+        custom_recipe, error = self._get_ci_recipe(kwargs['recipe_id'])
+        if error:
+            return error
+
+        if request.method == 'GET':
+            info = {"id": custom_recipe.id,
+                    "name": custom_recipe.name,
+                    "base_recipe_id": custom_recipe.base_recipe.id,
+                    "project_id": custom_recipe.project.id}
+
+            return JsonResponse({"error": "ok", "info": info})
+
+    def delete(self, request, *args, **kwargs):
+        custom_recipe, error = self._get_ci_recipe(kwargs['recipe_id'])
+        if error:
+            return error
+
+        custom_recipe.delete()
+        return JsonResponse({"error": "ok"})
+
+
+class XhrCustomRecipePackages(View):
+    """
+    ReST API to add/remove packages to/from custom recipe.
+
+    Entry point: /xhr_customrecipe/<recipe_id>/packages/<package_id>
+    Methods:
+         PUT - Add package to the recipe
+         DELETE - Delete package from the recipe
+         GET - Get package information
+
+     Returns:
+         {"error": "ok"}
+          or
+          {"error": <error message>}
+    """
+    @staticmethod
+    def _get_package(package_id):
+        try:
+            package = CustomImagePackage.objects.get(pk=package_id)
+            return package, None
+        except Package.DoesNotExist:
+            return None, error_response("Package with id=%s "
+                                        "not found" % package_id)
+
+    def _traverse_dependents(self, next_package_id,
+                             rev_deps, all_current_packages, tree_level=0):
+        """
+        Recurse through reverse dependency tree for next_package_id.
+        Limit the reverse dependency search to packages not already scanned,
+        that is, not already in rev_deps.
+        Limit the scan to a depth (tree_level) not exceeding the count of
+        all packages in the custom image, and if that depth is exceeded
+        return False, pop out of the recursion, and write a warning
+        to the log, but this is unlikely, suggesting a dependency loop
+        not caught by bitbake.
+        On return, the input/output arg rev_deps is appended with queryset
+        dictionary elements, annotated for use in the customimage template.
+        The list has unsorted, but unique elements.
+        """
+        max_dependency_tree_depth = all_current_packages.count()
+        if tree_level >= max_dependency_tree_depth:
+            logger.warning(
+                "The number of reverse dependencies "
+                "for this package exceeds " + max_dependency_tree_depth +
+                " and the remaining reverse dependencies will not be removed")
+            return True
+
+        package = CustomImagePackage.objects.get(id=next_package_id)
+        dependents = \
+            package.package_dependencies_target.annotate(
+                name=F('package__name'),
+                pk=F('package__pk'),
+                size=F('package__size'),
+            ).values("name", "pk", "size").exclude(
+                ~Q(pk__in=all_current_packages)
+            )
+
+        for pkg in dependents:
+            if pkg in rev_deps:
+                # already seen, skip dependent search
+                continue
+
+            rev_deps.append(pkg)
+            if (self._traverse_dependents(pkg["pk"], rev_deps,
+                                          all_current_packages,
+                                          tree_level+1)):
+                return True
+
+        return False
+
+    def _get_all_dependents(self, package_id, all_current_packages):
+        """
+        Returns sorted list of recursive reverse dependencies for package_id,
+        as a list of dictionary items, by recursing through dependency
+        relationships.
+        """
+        rev_deps = []
+        self._traverse_dependents(package_id, rev_deps, all_current_packages)
+        rev_deps = sorted(rev_deps, key=lambda x: x["name"])
+        return rev_deps
+
+    def get(self, request, *args, **kwargs):
+        recipe, error = XhrCustomRecipeId._get_ci_recipe(
+            kwargs['recipe_id'])
+        if error:
+            return error
+
+        # If no package_id then list all the current packages
+        if not kwargs['package_id']:
+            total_size = 0
+            packages = recipe.get_all_packages().values("id",
+                                                        "name",
+                                                        "version",
+                                                        "size")
+            for package in packages:
+                package['size_formatted'] = \
+                    filtered_filesizeformat(package['size'])
+                total_size += package['size']
+
+            return JsonResponse({"error": "ok",
+                                 "packages": list(packages),
+                                 "total": len(packages),
+                                 "total_size": total_size,
+                                 "total_size_formatted":
+                                 filtered_filesizeformat(total_size)})
+        else:
+            package, error = XhrCustomRecipePackages._get_package(
+                kwargs['package_id'])
+            if error:
+                return error
+
+            all_current_packages = recipe.get_all_packages()
+
+            # Dependencies for package which aren't satisfied by the
+            # current packages in the custom image recipe
+            deps = package.package_dependencies_source.for_target_or_none(
+                recipe.name)['packages'].annotate(
+                name=F('depends_on__name'),
+                pk=F('depends_on__pk'),
+                size=F('depends_on__size'),
+                ).values("name", "pk", "size").filter(
+                # There are two depends types we don't know why
+                (Q(dep_type=Package_Dependency.TYPE_TRDEPENDS) |
+                 Q(dep_type=Package_Dependency.TYPE_RDEPENDS)) &
+                ~Q(pk__in=all_current_packages)
+                )
+
+            # Reverse dependencies which are needed by packages that are
+            # in the image. Recursive search providing all dependents,
+            # not just immediate dependents.
+            reverse_deps = self._get_all_dependents(kwargs['package_id'],
+                                                    all_current_packages)
+            total_size_deps = 0
+            total_size_reverse_deps = 0
+
+            for dep in deps:
+                dep['size_formatted'] = \
+                    filtered_filesizeformat(dep['size'])
+                total_size_deps += dep['size']
+
+            for dep in reverse_deps:
+                dep['size_formatted'] = \
+                    filtered_filesizeformat(dep['size'])
+                total_size_reverse_deps += dep['size']
+
+            return JsonResponse(
+                {"error": "ok",
+                 "id": package.pk,
+                 "name": package.name,
+                 "version": package.version,
+                 "unsatisfied_dependencies": list(deps),
+                 "unsatisfied_dependencies_size": total_size_deps,
+                 "unsatisfied_dependencies_size_formatted":
+                 filtered_filesizeformat(total_size_deps),
+                 "reverse_dependencies": list(reverse_deps),
+                 "reverse_dependencies_size": total_size_reverse_deps,
+                 "reverse_dependencies_size_formatted":
+                 filtered_filesizeformat(total_size_reverse_deps)})
+
+    def put(self, request, *args, **kwargs):
+        recipe, error = XhrCustomRecipeId._get_ci_recipe(kwargs['recipe_id'])
+        package, error = self._get_package(kwargs['package_id'])
+        if error:
+            return error
+
+        included_packages = recipe.includes_set.values_list('pk',
+                                                            flat=True)
+
+        # If we're adding back a package which used to be included in this
+        # image all we need to do is remove it from the excludes
+        if package.pk in included_packages:
+            try:
+                recipe.excludes_set.remove(package)
+                return {"error": "ok"}
+            except Package.DoesNotExist:
+                return error_response("Package %s not found in excludes"
+                                      " but was in included list" %
+                                      package.name)
+
+        else:
+            recipe.appends_set.add(package)
+            # Make sure that package is not in the excludes set
+            try:
+                recipe.excludes_set.remove(package)
+            except:
+                pass
+            # Add the dependencies we think will be added to the recipe
+            # as a result of appending this package.
+            # TODO this should recurse down the entire deps tree
+            for dep in package.package_dependencies_source.all_depends():
+                try:
+                    cust_package = CustomImagePackage.objects.get(
+                        name=dep.depends_on.name)
+
+                    recipe.includes_set.add(cust_package)
+                    try:
+                        # When adding the pre-requisite package, make
+                        # sure it's not in the excluded list from a
+                        # prior removal.
+                        recipe.excludes_set.remove(cust_package)
+                    except package.DoesNotExist:
+                        # Don't care if the package had never been excluded
+                        pass
+                except:
+                    logger.warning("Could not add package's suggested"
+                                   "dependencies to the list")
+        return JsonResponse({"error": "ok"})
+
+    def delete(self, request, *args, **kwargs):
+        recipe, error = XhrCustomRecipeId._get_ci_recipe(kwargs['recipe_id'])
+        package, error = self._get_package(kwargs['package_id'])
+        if error:
+            return error
+
+        try:
+            included_packages = recipe.includes_set.values_list('pk',
+                                                                flat=True)
+            # If we're deleting a package which is included we need to
+            # Add it to the excludes list.
+            if package.pk in included_packages:
+                recipe.excludes_set.add(package)
+            else:
+                recipe.appends_set.remove(package)
+                all_current_packages = recipe.get_all_packages()
+
+                reverse_deps_dictlist = self._get_all_dependents(
+                    package.pk,
+                    all_current_packages)
+
+                ids = [entry['pk'] for entry in reverse_deps_dictlist]
+                reverse_deps = CustomImagePackage.objects.filter(id__in=ids)
+                for r in reverse_deps:
+                    try:
+                        if r.id in included_packages:
+                            recipe.excludes_set.add(r)
+                        else:
+                            recipe.appends_set.remove(r)
+                    except:
+                        pass
+
+            return JsonResponse({"error": "ok"})
+        except CustomImageRecipe.DoesNotExist:
+            return error_response("Tried to remove package that wasn't"
+                                  " present")

bitbake/lib/toaster/toastergui/buildtables.py

@@ -109,15 +109,31 @@ class BuiltPackagesTableBase(tables.PackagesTable):
                         hidden=True,
                         orderable=True)
 
+        layer_branch_template = '''
+        {%if not data.recipe.layer_version.layer.local_source_dir %}
+        <span class="text-muted">{{data.recipe.layer_version.branch}}</span>
+        {% else %}
+        <span class="text-muted">Not applicable</span>
+        <span class="glyphicon glyphicon-question-sign get-help" data-original-title="" title="The source code of {{data.recipe.layer_version.layer.name}} is not in a Git repository, so there is no branch associated with it"> </span>
+        {% endif %}
+        '''
+
         self.add_column(title="Layer branch",
                         field_name="recipe__layer_version__branch",
                         hidden=True,
+                        static_data_name="recipe__layer_version__branch",
+                        static_data_template=layer_branch_template,
                         orderable=True)
 
         git_rev_template = '''
+        {% if not data.recipe.layer_version.layer.local_source_dir %}
         {% with vcs_ref=data.recipe.layer_version.commit %}
         {% include 'snippets/gitrev_popover.html' %}
         {% endwith %}
+        {% else %}
+        <span class="text-muted">Not applicable</span>
+        <span class="glyphicon glyphicon-question-sign get-help" data-original-title="" title="The source code of {{data.recipe.layer_version.layer.name}} is not in a Git repository, so there is no revision associated with it"> </span>
+        {% endif %}
         '''
 
         self.add_column(title="Layer commit",
@@ -250,10 +266,24 @@ class BuiltRecipesTable(BuildTablesMixin):
             '{% if data.pathflags %}<i>({{data.pathflags}})</i>'\
             '{% endif %}'
 
+        git_branch_template = '''
+        {% if data.layer_version.layer.local_source_dir %}
+        <span class="text-muted">Not applicable</span>
+        <span class="glyphicon glyphicon-question-sign get-help" data-original-title="" title="The source code of {{data.layer_version.layer.name}} is not in a Git repository, so there is no branch associated with it"> </span>
+        {% else %}
+        <span>{{data.layer_version.branch}}</span>
+        {% endif %}
+        '''
+
         git_rev_template = '''
+        {% if data.layer_version.layer.local_source_dir %}
+        <span class="text-muted">Not applicable</span>
+        <span class="glyphicon glyphicon-question-sign get-help" data-original-title="" title="The source code of {{data.layer_version.layer.name}} is not in a Git repository, so there is no commit associated with it"> </span>
+        {% else %}
         {% with vcs_ref=data.layer_version.commit %}
         {% include 'snippets/gitrev_popover.html' %}
         {% endwith %}
+        {% endif %}
         '''
 
         depends_on_tmpl = '''
@@ -295,13 +325,15 @@ class BuiltRecipesTable(BuildTablesMixin):
         {% endif %}{% endwith %}{% endwith %}
         '''
 
-        self.add_column(title="Name",
+        self.add_column(title="Recipe",
                         field_name="name",
                         static_data_name='name',
                         orderable=True,
+                        hideable=False,
                         static_data_template=recipe_name_tmpl)
 
         self.add_column(title="Version",
+                        hideable=False,
                         field_name="version")
 
         self.add_column(title="Dependencies",
@@ -340,6 +372,8 @@ class BuiltRecipesTable(BuildTablesMixin):
 
         self.add_column(title="Layer branch",
                         field_name="layer_version__branch",
+                        static_data_name="layer_version__branch",
+                        static_data_template=git_branch_template,
                         orderable=True,
                         hidden=True)
 
@@ -431,21 +465,25 @@ class BuildTasksTable(BuildTablesMixin):
         self.add_column(title="Order",
                         static_data_name="order",
                         static_data_template='{{data.order}}',
+                        hideable=False,
                         orderable=True)
 
         self.add_column(title="Task",
                         static_data_name="task_name",
                         static_data_template=task_link_tmpl(
                             "{{data.task_name}}"),
+                        hideable=False,
                         orderable=True)
 
         self.add_column(title="Recipe",
                         static_data_name='recipe__name',
                         static_data_template=recipe_name_tmpl,
+                        hideable=False,
                         orderable=True)
 
         self.add_column(title="Recipe version",
-                        field_name='recipe__version')
+                        field_name='recipe__version',
+                        hidden=True)
 
         self.add_column(title="Executed",
                         static_data_name="task_executed",

bitbake/lib/toaster/toastergui/fixtures/toastergui-unittest-data.xml

@@ -1,5 +1,16 @@
 <?xml version="1.0" encoding="utf-8"?>
 <django-objects version="1.0">
+   <object pk="1" model="orm.bitbakeversion">
+      <field type="CharField" name="name">v2.3</field>
+      <field type="GitURLField" name="giturl">git://git.openembedded.org/bitbake</field>
+      <field type="CharField" name="dirpath">b</field>
+      <field type="CharField" name="branch">a</field>
+  </object>
+  <object pk="1" model="orm.release">
+     <field type="CharField" name="name">master</field>
+     <field type="CharField" name="description">master project</field>
+     <field to="orm.bitbake_version" name="bitbake_version">1</field>
+  </object>
   <object pk="1" model="orm.project">
     <field type="CharField" name="name">a test project</field>
     <field type="CharField" name="short_description"></field>
@@ -195,8 +206,6 @@
     <field to="orm.customimagerecipe" name="recipe_appends" rel="ManyToManyRel"><object pk="3"></object></field>
   </object>
   <object pk="1" model="orm.recipe">
-    <field to="orm.layersource" name="layer_source" rel="ManyToOneRel">1</field>
-    <field type="IntegerField" name="up_id"><None></None></field>
     <field type="DateTimeField" name="up_date"><None></None></field>
     <field type="CharField" name="name">z recipe</field>
     <field type="CharField" name="version">5.2</field>
@@ -212,8 +221,6 @@
     <field type="BooleanField" name="is_image">False</field>
   </object>
   <object pk="2" model="orm.recipe">
-    <field to="orm.layersource" name="layer_source" rel="ManyToOneRel">1</field>
-    <field type="IntegerField" name="up_id"><None></None></field>
     <field type="DateTimeField" name="up_date"><None></None></field>
     <field type="CharField" name="name">a recipe</field>
     <field type="CharField" name="version">1.2</field>
@@ -229,8 +236,6 @@
     <field type="BooleanField" name="is_image">False</field>
   </object>
   <object pk="3" model="orm.recipe">
-    <field to="orm.layersource" name="layer_source" rel="ManyToOneRel"><None></None></field>
-    <field type="IntegerField" name="up_id"><None></None></field>
     <field type="DateTimeField" name="up_date"><None></None></field>
     <field type="CharField" name="name">a custom recipe</field>
     <field type="CharField" name="version"></field>
@@ -246,8 +251,6 @@
     <field type="BooleanField" name="is_image">False</field>
   </object>
   <object pk="4" model="orm.recipe">
-    <field to="orm.layersource" name="layer_source" rel="ManyToOneRel">1</field>
-    <field type="IntegerField" name="up_id"><None></None></field>
     <field type="DateTimeField" name="up_date"><None></None></field>
     <field type="CharField" name="name">a image recipe</field>
     <field type="CharField" name="version">1.2</field>
@@ -263,8 +266,6 @@
     <field type="BooleanField" name="is_image">True</field>
   </object>
   <object pk="5" model="orm.recipe">
-    <field to="orm.layersource" name="layer_source" rel="ManyToOneRel">1</field>
-    <field type="IntegerField" name="up_id"><None></None></field>
     <field type="DateTimeField" name="up_date"><None></None></field>
     <field type="CharField" name="name">z image recipe</field>
     <field type="CharField" name="version">1.3</field>
@@ -280,8 +281,6 @@
     <field type="BooleanField" name="is_image">True</field>
   </object>
   <object pk="6" model="orm.recipe">
-    <field to="orm.layersource" name="layer_source" rel="ManyToOneRel"><None></None></field>
-    <field type="IntegerField" name="up_id"><None></None></field>
     <field type="DateTimeField" name="up_date"><None></None></field>
     <field type="CharField" name="name">z custom recipe</field>
     <field type="CharField" name="version"></field>
@@ -307,34 +306,23 @@
   </object>
 
   <object pk="1" model="orm.machine">
-    <field to="orm.layersource" name="layer_source" rel="ManyToOneRel"><None></None></field>
-    <field type="IntegerField" name="up_id"><None></None></field>
     <field type="DateTimeField" name="up_date"><None></None></field>
     <field to="orm.layer_version" name="layer_version" rel="ManyToOneRel">1</field>
     <field type="CharField" name="name">a machine</field>
     <field type="CharField" name="description">a machine</field>
   </object>
   <object pk="2" model="orm.machine">
-    <field to="orm.layersource" name="layer_source" rel="ManyToOneRel"><None></None></field>
-    <field type="IntegerField" name="up_id"><None></None></field>
     <field type="DateTimeField" name="up_date"><None></None></field>
     <field to="orm.layer_version" name="layer_version" rel="ManyToOneRel">2</field>
     <field type="CharField" name="name">z machine</field>
     <field type="CharField" name="description">z machine</field>
   </object>
   <object pk="3" model="orm.machine">
-    <field to="orm.layersource" name="layer_source" rel="ManyToOneRel"><None></None></field>
-    <field type="IntegerField" name="up_id"><None></None></field>
     <field type="DateTimeField" name="up_date"><None></None></field>
     <field to="orm.layer_version" name="layer_version" rel="ManyToOneRel">1</field>
     <field type="CharField" name="name">g machine</field>
     <field type="CharField" name="description">g machine</field>
   </object>
-  <object pk="1" model="orm.layersource">
-    <field type="CharField" name="name">local</field>
-    <field type="IntegerField" name="sourcetype">1</field>
-    <field type="CharField" name="apiurl"></field>
-  </object>
   <object pk="1" model="orm.bitbakeversion">
     <field type="CharField" name="name">test bbv</field>
     <field type="CharField" name="giturl">/tmp/</field>
@@ -355,26 +343,12 @@
     <field type="CharField" name="branch_name">master</field>
     <field type="TextField" name="helptext"><None></None></field>
   </object>
-  <object pk="1" model="orm.releaselayersourcepriority">
-    <field to="orm.release" name="release" rel="ManyToOneRel">1</field>
-    <field to="orm.layersource" name="layer_source" rel="ManyToOneRel">1</field>
-    <field type="IntegerField" name="priority">0</field>
-  </object>
-  <object pk="1" model="orm.branch">
-    <field to="orm.layersource" name="layer_source" rel="ManyToOneRel">1</field>
-    <field type="CharField" name="name">master</field>
-    <field type="CharField" name="short_description"></field>
-  </object>
   <object pk="1" model="orm.layer">
-    <field to="orm.layersource" name="layer_source" rel="ManyToOneRel"><None></None></field>
-    <field type="IntegerField" name="up_id"><None></None></field>
     <field type="DateTimeField" name="up_date"><None></None></field>
     <field type="CharField" name="name">a layer</field>
     <field type="CharField" name="vcs_url">/tmp/</field>
   </object>
   <object pk="2" model="orm.layer">
-    <field to="orm.layersource" name="layer_source" rel="ManyToOneRel">1</field>
-    <field type="IntegerField" name="up_id"><None></None></field>
     <field type="DateTimeField" name="up_date"><None></None></field>
     <field type="CharField" name="name">z layer</field>
     <field type="CharField" name="layer_index_url"></field>
@@ -383,10 +357,8 @@
   <object pk="1" model="orm.layer_version">
     <field to="orm.build" name="build" rel="ManyToOneRel">1</field>
     <field to="orm.layer" name="layer" rel="ManyToOneRel">1</field>
-    <field to="orm.layersource" name="layer_source" rel="ManyToOneRel">1</field>
-    <field type="IntegerField" name="up_id"><None></None></field>
     <field type="DateTimeField" name="up_date"><None></None></field>
-    <field to="orm.branch" name="up_branch" rel="ManyToOneRel">1</field>
+    <field to="orm.release" name="release" rel="ManyToOneRel">1</field>
     <field type="CharField" name="branch">master</field>
     <field type="CharField" name="commit">abcdef123</field>
     <field type="CharField" name="dirpath">/tmp/</field>
@@ -397,10 +369,8 @@
   <object pk="2" model="orm.layer_version">
     <field to="orm.build" name="build" rel="ManyToOneRel"><None></None></field>
     <field to="orm.layer" name="layer" rel="ManyToOneRel">2</field>
-    <field to="orm.layersource" name="layer_source" rel="ManyToOneRel">1</field>
-    <field type="IntegerField" name="up_id"><None></None></field>
     <field type="DateTimeField" name="up_date"><None></None></field>
-    <field to="orm.branch" name="up_branch" rel="ManyToOneRel">1</field>
+    <field to="orm.release" name="release" rel="ManyToOneRel">1</field>
     <field type="CharField" name="branch">testing-branch</field>
     <field type="CharField" name="commit">9876fedcba</field>
     <field type="CharField" name="dirpath"><None></None></field>
@@ -411,10 +381,8 @@
   <object pk="3" model="orm.layer_version">
     <field to="orm.build" name="build" rel="ManyToOneRel">1</field>
     <field to="orm.layer" name="layer" rel="ManyToOneRel">2</field>
-    <field to="orm.layersource" name="layer_source" rel="ManyToOneRel">1</field>
-    <field type="IntegerField" name="up_id"><None></None></field>
     <field type="DateTimeField" name="up_date"><None></None></field>
-    <field to="orm.branch" name="up_branch" rel="ManyToOneRel">1</field>
+    <field to="orm.release" name="release" rel="ManyToOneRel">1</field>
     <field type="CharField" name="branch">testing-branch</field>
     <field type="CharField" name="commit">9876fedcba</field>
     <field type="CharField" name="dirpath"><None></None></field>

bitbake/lib/toaster/toastergui/static/css/default.css

@@ -45,6 +45,7 @@ img.logo { height: 30px; vertical-align: bottom; }
 .alert-link.build-warnings,
 .glyphicon-warning-sign.build-warnings { color: #8a6d3b; }
 .build-result .project-name { margin-top: -10px; margin-bottom: 5px; }
+.rebuild-btn, .cancel-build-btn { cursor: pointer; }
 
 /* Styles for the help information */
 .get-help { color: #CCCCCC; }
@@ -160,7 +161,13 @@ dd .glyphicon-edit { margin-left: 5px; }
 
 /* Style the forms and definition lists in the layer details pages */
 #change-repo-form .form-control { width: 17em; }
+#information { margin-bottom: 5em; }
 #information dd > form { margin-bottom: 5px; margin-top: 5px; }
+#edit-layer-source-form fieldset { margin-top: 20px; }
+#directory-info,
+#git-repo-info { margin-top: 20px; }
+#layer-dir-path-in-details { width: 55%; }
+.add-deps .form-control { width: 15em; }
 
 /* Style the forms and definition lists in the BitBake variables page */
 .variable-list { margin-bottom: 20px; }
@@ -175,6 +182,7 @@ dd.variable-list form { margin-top: 10px; }
 .scrolling.has-error { border-color: #a94442; }
 .help-block.text-danger { color: #a94442; }
 .tooltip-inner code { color: #fff; }
+.text-danger > code { color: #a94442; }
 dd.variable-list .glyphicon-question-sign { font-size: 14px; }
 dd.variable-list .glyphicon-edit { font-size: 16px; }
 dt .glyphicon-trash { margin-left: 5px; font-size: 16px; }
@@ -196,16 +204,22 @@ h2 { margin-bottom: 25px; }
 
 /* Style the import layer form controls*/
 legend { border: none; }
+fieldset.fields-apart-from-layer-name { margin-top: 20px; }
+.radioLegend { margin-bottom: 0; }
 #layer-name-ctrl { margin-top: 20px; }
 #import-layer-name,
 #layer-subdir { width: 20%; }
 #layer-git-repo-url { width: 40%; }
 #layer-git-ref { width: 32%; }
+#local-dir-path { width: 45%; }
 #layer-dependency { width: 16em; }
+#layer-deps-list { margin-top: 0; }
 #form-actions { margin-bottom: 30px; }
 #duplicate-layer-info dl { margin-top: 10px; }
 #duplicate-layer-info dd { margin-bottom: 10px; }
 .help-inline { color: #737373; margin-left: 10px; }
+.radio-help { width: 50%; margin-left: 20px; }
+#repo-select div:nth-of-type(2) { margin-top: 15px; }
 
 /* Give some padding to the in-cell tooltips we use for notifications in tables */
 td > .tooltip-inner,
@@ -249,6 +263,18 @@ code { color: #333; background-color: transparent; }
 /* Style the special no results message in the custom image details page */
 [id^="no-results-special-"] > .alert-warning > ol { margin-top: 10px; }
 
+/* style the loading spinner in the new custom image dialog */
+#create-new-custom-image-btn [data-role="loading-state"] {
+  padding-left: 16px;
+}
+
+/* icon has to be absolutely positioned, otherwise the spin animation doesn't work */
+#create-new-custom-image-btn [data-role="loading-state"] .icon-spinner {
+  position: absolute;
+  left: 26px;
+  bottom: 26px;
+}
+
 /* Style the content of modal dialogs */
 .modal-footer { text-align: left; }
 .date-filter-controls { margin-top: 10px; }
@@ -298,6 +324,8 @@ h2.panel-title { font-size: 30px; }
 /* Make the help in tables insivisble until you hover over the right cell */
 .hover-help { visibility: hidden; }
 
+#add-remove-layer-btn { margin-bottom: 20px; }
+
 /* Blue hightlight animation for tasks and directory structure tables */
 .highlight { -webkit-animation: target-fade 15s 1; -moz-animation: target-fade 15s 1; animation: target-fade 15s 1; }
 @-webkit-keyframes target-fade { 0% { background-color: #D9EDF7; } 25% { background-color: #D9EDF7; } 100% { background-color: white; } }

bitbake/lib/toaster/toastergui/static/js/importlayer.js

@@ -10,6 +10,7 @@ function importLayerPageInit (ctx) {
   var layerDepInput = $("#layer-dependency");
   var layerNameCtrl = $("#layer-name-ctrl");
   var duplicatedLayerName = $("#duplicated-layer-name-hint");
+  var localDirPath = $("#local-dir-path");
 
   var layerDeps = {};
   var layerDepsDeps = {};
@@ -35,6 +36,9 @@ function importLayerPageInit (ctx) {
     }
   });
 
+  // Disable local dir repo when page is loaded.
+  $('#local-dir').hide();
+
   // disable the "Add layer" button when the layer input typeahead is empty
   // or not in the typeahead choices
   layerDepInput.on("input change", function () {
@@ -168,8 +172,16 @@ function importLayerPageInit (ctx) {
         dir_path: $("#layer-subdir").val(),
         project_id: libtoaster.ctx.projectId,
         layer_deps: layerDepsCsv,
+        local_source_dir: $('#local-dir-path').val(),
       };
 
+      if ($('input[name=repo]:checked').val() == "git") {
+        layerData.local_source_dir = "";
+      } else {
+        layerData.vcs_url = "";
+        layerData.git_ref = "";
+      }
+
       $.ajax({
           type: "POST",
           url: ctx.xhrImportLayerUrl,
@@ -208,25 +220,45 @@ function importLayerPageInit (ctx) {
   function check_form() {
     var valid = false;
     var inputs = $("input:required");
+    var inputStr = inputs.val().split("");
 
-    for (var i=0; i<inputs.length; i++){
-      if (!(valid = inputs[i].value)){
+    for (var i=0; i<inputs.val().length; i++){
+      if (!(valid = inputStr[i])){
         enable_import_btn(false);
         break;
       }
     }
 
-    if (valid)
-      enable_import_btn(true);
+    if (valid) {
+      if ($("#local-dir-radio").prop("checked") && localDirPath.val().length > 0) {
+        enable_import_btn(true);
+      }
+      if ($("#git-repo-radio").prop("checked") && vcsURLInput.val().length > 0 && gitRefInput.val().length > 0) {
+        enable_import_btn(true);
+      }
+    }
+
+    if (inputs.val().length == 0)
+      enable_import_btn(false);
   }
 
   function layerExistsError(layer){
     var dupLayerInfo = $("#duplicate-layer-info");
-    dupLayerInfo.find(".dup-layer-name").text(layer.name);
-    dupLayerInfo.find(".dup-layer-link").attr("href", layer.layerdetailurl);
-    dupLayerInfo.find("#dup-layer-vcs-url").text(layer.vcs_url);
-    dupLayerInfo.find("#dup-layer-revision").text(layer.vcs_reference);
 
+    if (layer.local_source_dir) {
+      $("#git-layer-dup").hide();
+      $("#local-layer-dup").fadeIn();
+      dupLayerInfo.find(".dup-layer-name").text(layer.name);
+      dupLayerInfo.find(".dup-layer-link").attr("href", layer.layerdetailurl);
+      dupLayerInfo.find("#dup-local-source-dir-name").text(layer.local_source_dir);
+    } else {
+      $("#git-layer-dup").fadeIn();
+      $("#local-layer-dup").hide();
+      dupLayerInfo.find(".dup-layer-name").text(layer.name);
+      dupLayerInfo.find(".dup-layer-link").attr("href", layer.layerdetailurl);
+      dupLayerInfo.find("#dup-layer-vcs-url").text(layer.vcs_url);
+      dupLayerInfo.find("#dup-layer-revision").text(layer.vcs_reference);
+    }
     $(".fields-apart-from-layer-name").fadeOut(function(){
 
       dupLayerInfo.fadeIn();
@@ -271,10 +303,13 @@ function importLayerPageInit (ctx) {
 
     if ($("#duplicate-layer-info").css("display") != "None"){
       $("#duplicate-layer-info").fadeOut(function(){
-        $(".fields-apart-from-layer-name").show();
-      });
+      $(".fields-apart-from-layer-name").show();
+      radioDisplay();
+    });
 
-    }
+  }
+
+    radioDisplay();
 
     /* Don't remove the error class if we're displaying the error for another
      * reason.
@@ -301,4 +336,72 @@ function importLayerPageInit (ctx) {
     }
   });
 
+  function radioDisplay() {
+    if ($('input[name=repo]:checked').val() == "local") {
+      $('#git-repo').hide();
+      $('#import-git-layer-and-add-hint').hide();
+      $('#local-dir').fadeIn();
+      $('#import-local-dir-and-add-hint').fadeIn();
+    } else {
+      $('#local-dir').hide();
+      $('#import-local-dir-and-add-hint').hide();
+      $('#git-repo').fadeIn();
+      $('#import-git-layer-and-add-hint').fadeIn();
+    }
+  }
+
+  $('input:radio[name="repo"]').change(function() {
+    radioDisplay();
+    if ($("#local-dir-radio").prop("checked")) {
+      if (localDirPath.val().length > 0) {
+        enable_import_btn(true);
+      } else {
+        enable_import_btn(false);
+      }
+    }
+    if ($("#git-repo-radio").prop("checked")) {
+      if (vcsURLInput.val().length > 0 && gitRefInput.val().length > 0) {
+        enable_import_btn(true);
+      } else {
+        enable_import_btn(false);
+      }
+    }
+  });
+
+  localDirPath.on('input', function(){
+    if ($(this).val().trim().length == 0) {
+      $('#import-and-add-btn').attr("disabled","disabled");
+      $('#local-dir').addClass('has-error');
+      $('#hintError-dir-abs-path').show();
+      $('#hintError-dir-path-starts-with-slash').show();
+    } else {
+      var input = $(this);
+      var reBeginWithSlash = /^\//;
+      var reCheckVariable = /^\$/;
+      var re = /([ <>\\|":\.%\?\*]+)/;
+
+      var invalidDir = re.test(input.val());
+      var invalidSlash = reBeginWithSlash.test(input.val());
+      var invalidVar = reCheckVariable.test(input.val());
+
+      if (!invalidSlash && !invalidVar) {
+        $('#local-dir').addClass('has-error');
+        $('#import-and-add-btn').attr("disabled","disabled");
+        $('#hintError-dir-abs-path').show();
+        $('#hintError-dir-path-starts-with-slash').show();
+      } else if (invalidDir) {
+        $('#local-dir').addClass('has-error');
+        $('#import-and-add-btn').attr("disabled","disabled");
+        $('#hintError-dir-path').show();
+      } else {
+        $('#local-dir').removeClass('has-error');
+        if (layerNameInput.val().length > 0) {
+          $('#import-and-add-btn').removeAttr("disabled");
+        }
+        $('#hintError-dir-abs-path').hide();
+        $('#hintError-dir-path-starts-with-slash').hide();
+        $('#hintError-dir-path').hide();
+      }
+    }
+  });
 }

bitbake/lib/toaster/toastergui/static/js/layerdetails.js

@@ -10,11 +10,20 @@ function layerDetailsPageInit (ctx) {
   var targetTab = $("#targets-tab");
   var machineTab = $("#machines-tab");
   var detailsTab = $("#details-tab");
+  var editLayerSource = $("#edit-layer-source");
+  var saveSourceChangesBtn = $("#save-changes-for-switch");
+  var layerGitRefInput = $("#layer-git-ref");
+  var layerSubDirInput = $('#layer-subdir');
+
+  targetTab.on('show.bs.tab', targetsTabShow);
+  detailsTab.on('show.bs.tab', detailsTabShow);
+  machineTab.on('show.bs.tab', machinesTabShow);
 
   /* setup the dependencies typeahead */
-  libtoaster.makeTypeahead(layerDepInput, libtoaster.ctx.layersTypeAheadUrl, { include_added: "true" }, function(item){
+  libtoaster.makeTypeahead(layerDepInput,
+                           libtoaster.ctx.layersTypeAheadUrl,
+                           { include_added: "true" }, function(item){
     currentLayerDepSelection = item;
-
     layerDepBtn.removeAttr("disabled");
   });
 
@@ -25,20 +34,6 @@ function layerDetailsPageInit (ctx) {
     }
   });
 
-  $(window).on('hashchange', function(e){
-    switch(window.location.hash){
-      case '#machines':
-        machineTab.tab('show');
-        break;
-      case '#recipes':
-        targetTab.tab('show');
-        break;
-      default:
-        detailsTab.tab('show');
-        break;
-    }
-  });
-
   function addRemoveDep(depLayerId, add, doneCb) {
     var data = { layer_version_id : ctx.layerVersion.id };
     if (add)
@@ -150,6 +145,7 @@ function layerDetailsPageInit (ctx) {
     });
   });
 
+
   function defaultAddBtnText(){
       var text = " Add the "+ctx.layerVersion.name+" layer to your project";
       addRmLayerBtn.text(text);
@@ -157,12 +153,12 @@ function layerDetailsPageInit (ctx) {
       addRmLayerBtn.removeClass("btn-danger");
   }
 
-  detailsTab.on('show', function(){
+  function detailsTabShow(){
     if (!ctx.layerVersion.inCurrentPrj)
       defaultAddBtnText();
 
-    window.location.hash = "details";
-  });
+    window.location.hash = "information";
+  }
 
   function targetsTabShow(){
     if (!ctx.layerVersion.inCurrentPrj){
@@ -216,7 +212,6 @@ function layerDetailsPageInit (ctx) {
 
   });
 
-  targetTab.on('show.bs.tab', targetsTabShow);
 
   function machinesTabShow(){
     if (!ctx.layerVersion.inCurrentPrj) {
@@ -233,8 +228,6 @@ function layerDetailsPageInit (ctx) {
     window.location.hash = "machines";
   }
 
-  machineTab.on('show.bs.tab', machinesTabShow);
-
   $(".pagesize").change(function(){
     var search = libtoaster.parseUrlParams();
     search.limit = this.value;
@@ -366,7 +359,7 @@ function layerDetailsPageInit (ctx) {
     if ($(this).is("dt")) {
       var dd = $(this).next("dd");
       if (!dd.children("form:visible")|| !dd.find(".current-value").html()){
-        if (ctx.layerVersion.sourceId == 3){
+        if (ctx.layerVersion.layer_source == ctx.layerSourceTypes.TYPE_IMPORTED){
         /* There's no current value and the layer is editable
          * so show the "Not set" and hide the delete icon
          */
@@ -423,4 +416,101 @@ function layerDetailsPageInit (ctx) {
   $(".glyphicon-trash").tooltip();
   $(".commit").tooltip();
 
+  editLayerSource.click(function() {
+    /* Kindly bring the git layers imported from layerindex to normal page
+     * and not this new page :(
+     */
+    $(this).hide();
+    saveSourceChangesBtn.attr("disabled", "disabled");
+
+    $("#git-repo-info, #directory-info").hide();
+    $("#edit-layer-source-form").fadeIn();
+    if ($("#layer-dir-path-in-details").val() == "") {
+      //Local dir path is empty...
+      $("#repo").prop("checked", true);
+      $("#layer-git").fadeIn();
+      $("#layer-dir").hide();
+    } else {
+      $("#layer-git").hide();
+      $("#layer-dir").fadeIn();
+    }
+  });
+
+  $('input:radio[name="source-location"]').change(function() {
+    if ($('input[name=source-location]:checked').val() == "repo") {
+      $("#layer-git").fadeIn();
+      $("#layer-dir").hide();
+      if ($("#layer-git-repo-url").val().length === 0 && layerGitRefInput.val().length === 0) {
+        saveSourceChangesBtn.attr("disabled", "disabled");
+      }
+    } else {
+      $("#layer-dir").fadeIn();
+      $("#layer-git").hide();
+    }
+  });
+
+  $("#layer-dir-path-in-details").keyup(function() {
+    saveSourceChangesBtn.removeAttr("disabled");
+  });
+
+  $("#layer-git-repo-url").keyup(function() {
+    if ($("#layer-git-repo-url").val().length > 0 && layerGitRefInput.val().length > 0) {
+      saveSourceChangesBtn.removeAttr("disabled");
+    }
+  });
+
+  layerGitRefInput.keyup(function() {
+    if ($("#layer-git-repo-url").val().length > 0 && layerGitRefInput.val().length > 0) {
+      saveSourceChangesBtn.removeAttr("disabled");
+    }
+  });
+
+
+  layerSubDirInput.keyup(function(){
+    if ($(this).val().length > 0){
+      saveSourceChangesBtn.removeAttr("disabled");
+    }
+  });
+
+  $('#cancel-changes-for-switch').click(function() {
+    $("#edit-layer-source-form").hide();
+    $("#directory-info, #git-repo-info").fadeIn();
+    editLayerSource.show();
+  });
+
+  saveSourceChangesBtn.click(function() {
+
+    var layerData = {
+      vcs_url: $('#layer-git-repo-url').val(),
+      commit: layerGitRefInput.val(),
+      dirpath: layerSubDirInput.val(),
+      local_source_dir: $('#layer-dir-path-in-details').val(),
+    };
+
+    if ($('input[name=source-location]:checked').val() == "repo") {
+      layerData.local_source_dir = "";
+    } else {
+      layerData.vcs_url = "";
+      layerData.git_ref = "";
+    }
+
+    $.ajax({
+        type: "POST",
+        url: ctx.xhrUpdateLayerUrl,
+        data: layerData,
+        headers: { 'X-CSRFToken' : $.cookie('csrftoken')},
+        success: function (data) {
+          if (data.error != "ok") {
+            console.warn(data.error);
+          } else {
+            /* success layer property changed */
+            window.location.reload();
+          }
+        },
+        error: function (data) {
+          console.warn("Call failed");
+          console.warn(data);
+        }
+    });
+  });
 }

bitbake/lib/toaster/toastergui/static/js/libtoaster.js

@@ -148,6 +148,21 @@ var libtoaster = (function () {
     });
   }
 
+  function _getMostRecentBuilds(url, onsuccess, onfail) {
+    $.ajax({
+      url: url,
+      type: 'GET',
+      data : {format: 'json'},
+      headers: {'X-CSRFToken': $.cookie('csrftoken')},
+      success: function (data) {
+        onsuccess ? onsuccess(data) : console.log(data);
+      },
+      error: function (data) {
+        onfail ? onfail(data) : console.error(data);
+      }
+    });
+  }
+
   /* Get a project's configuration info */
   function _getProjectInfo(url, onsuccess, onfail){
     $.ajax({
@@ -421,11 +436,27 @@ var libtoaster = (function () {
      });
   }
 
+  // if true, the loading spinner for Ajax requests will be displayed
+  // if requests take more than 1200ms
+  var ajaxLoadingTimerEnabled = true;
+
+  // turn on the page-level loading spinner for Ajax requests
+  function _enableAjaxLoadingTimer() {
+    ajaxLoadingTimerEnabled = true;
+  }
+
+  // turn off the page-level loading spinner for Ajax requests
+  function _disableAjaxLoadingTimer() {
+    ajaxLoadingTimerEnabled = false;
+  }
 
   return {
+    enableAjaxLoadingTimer: _enableAjaxLoadingTimer,
+    disableAjaxLoadingTimer: _disableAjaxLoadingTimer,
     reload_params : reload_params,
     startABuild : _startABuild,
     cancelABuild : _cancelABuild,
+    getMostRecentBuilds: _getMostRecentBuilds,
     makeTypeahead : _makeTypeahead,
     getProjectInfo: _getProjectInfo,
     getLayerDepsForProject : _getLayerDepsForProject,
@@ -469,7 +500,6 @@ function reload_params(params) {
     window.location.href = url+"?"+callparams.join('&');
 }
 
-
 /* Things that happen for all pages */
 $(document).ready(function() {
 
@@ -628,7 +658,9 @@ $(document).ready(function() {
         window.clearTimeout(ajaxLoadingTimer);
 
       ajaxLoadingTimer = window.setTimeout(function() {
-        $("#loading-notification").fadeIn();
+        if (libtoaster.ajaxLoadingTimerEnabled) {
+          $("#loading-notification").fadeIn();
+        }
       }, 1200);
     });
 

bitbake/lib/toaster/toastergui/static/js/mrbsection.js

@@ -1,33 +1,19 @@
 
 function mrbSectionInit(ctx){
-
-  var projectBuilds;
-
-  if (ctx.mrbType === 'project')
-    projectBuilds = true;
-
-  $(".cancel-build-btn").click(function(e){
+  $('#latest-builds').on('click', '.cancel-build-btn', function(e){
+    e.stopImmediatePropagation();
     e.preventDefault();
 
     var url = $(this).data('request-url');
     var buildReqIds = $(this).data('buildrequest-id');
-    var banner = $(this).parents(".alert");
 
-    banner.find(".progress-info").fadeOut().promise().done(function(){
-      $("#cancelling-msg-" + buildReqIds).show();
-      console.log("cancel build");
-      libtoaster.cancelABuild(url, buildReqIds, function(){
-        if (projectBuilds == false){
-          /* the all builds page is not 'self updating' like thei
-           * project Builds
-           */
-          window.location.reload();
-        }
-      }, null);
-    });
+    libtoaster.cancelABuild(url, buildReqIds, function () {
+      window.location.reload();
+    }, null);
   });
 
-  $(".run-again-btn").click(function(e){
+  $('#latest-builds').on('click', '.rebuild-btn', function(e){
+    e.stopImmediatePropagation();
     e.preventDefault();
 
     var url = $(this).data('request-url');
@@ -38,58 +24,112 @@ function mrbSectionInit(ctx){
     }, null);
   });
 
+  // cached version of buildData, so we can determine whether a build has
+  // changed since it was last fetched, and update the DOM appropriately
+  var buildData = {};
 
-  var progressTimer;
+  // returns the cached version of this build, or {} is there isn't a cached one
+  function getCached(build) {
+    return buildData[build.id] || {};
+  }
 
-  if (projectBuilds === true){
-    progressTimer = window.setInterval(function() {
-      libtoaster.getProjectInfo(libtoaster.ctx.projectPageUrl,
-        function(prjInfo){
-          /* These two are needed because a build can be 100% and still
-           * in progress due to the fact that the % done is updated at the
-           * start of a task so it can be doing the last task at 100%
-           */
-          var inProgress = 0;
-          var allPercentDone = 0;
-          if (prjInfo.builds.length === 0)
-            return
+  // returns true if a build's state changed to "Succeeded", "Failed"
+  // or "Cancelled" from some other value
+  function buildFinished(build) {
+    var cached = getCached(build);
+    return cached.state &&
+      cached.state !== build.state &&
+      (build.state == 'Succeeded' || build.state == 'Failed' ||
+       build.state == 'Cancelled');
+  }
 
-          for (var i in prjInfo.builds){
-            var build = prjInfo.builds[i];
+  // returns true if the state changed
+  function stateChanged(build) {
+    var cached = getCached(build);
+    return (cached.state !== build.state);
+  }
 
-            if (build.outcomeText === "In Progress" ||
-               $(".progress .bar").length > 0){
-              /* Update the build progress */
-              var percentDone;
+  // returns true if the tasks_complete_percentage changed
+  function tasksProgressChanged(build) {
+    var cached = getCached(build);
+    return (cached.tasks_complete_percentage !== build.tasks_complete_percentage);
+  }
 
-              if (build.outcomeText !== "In Progress"){
-                /* We have to ignore the value when it's Succeeded because it
-                *   goes back to 0
-                */
-                percentDone = 100;
-              } else {
-                percentDone = build.percentDone;
-                inProgress++;
-              }
+  // returns true if the number of recipes parsed/to parse changed
+  function recipeProgressChanged(build) {
+    var cached = getCached(build);
+    return (cached.recipes_parsed_percentage !== build.recipes_parsed_percentage);
+  }
 
-              $("#build-pc-done-" + build.id).text(percentDone);
-              $("#build-pc-done-title-" + build.id).attr("title", percentDone);
-              $("#build-pc-done-bar-" + build.id).css("width",
-                String(percentDone) + "%");
+  function refreshMostRecentBuilds(){
+    libtoaster.getMostRecentBuilds(
+      libtoaster.ctx.mostRecentBuildsUrl,
 
-              allPercentDone += percentDone;
-            }
+      // success callback
+      function (data) {
+        var build;
+        var tmpl;
+        var container;
+        var selector;
+        var colourClass;
+        var elements;
+
+        for (var i = 0; i < data.length; i++) {
+          build = data[i];
+
+          if (buildFinished(build)) {
+            // a build finished: reload the whole page so that the build
+            // shows up in the builds table
+            window.location.reload();
+          }
+          else if (stateChanged(build)) {
+            // update the whole template
+            build.warnings_pluralise = (build.warnings !== 1 ? 's' : '');
+            build.errors_pluralise = (build.errors !== 1 ? 's' : '');
+
+            tmpl = $.templates("#build-template");
+
+            html = $(tmpl.render(build));
+
+            selector = '[data-latest-build-result="' + build.id + '"] ' +
+              '[data-role="build-status-container"]';
+            container = $(selector);
+
+            // initialize bootstrap tooltips in the new HTML
+            html.find('span.glyphicon-question-sign').tooltip();
+
+            container.html(html);
+          }
+          else if (tasksProgressChanged(build)) {
+            // update the task progress text
+            selector = '#build-pc-done-' + build.id;
+            $(selector).html(build.tasks_complete_percentage);
+
+            // update the task progress bar
+            selector = '#build-pc-done-bar-' + build.id;
+            $(selector).width(build.tasks_complete_percentage + '%');
+          }
+          else if (recipeProgressChanged(build)) {
+            // update the recipe progress text
+            selector = '#recipes-parsed-percentage-' + build.id;
+            $(selector).html(build.recipes_parsed_percentage);
+
+            // update the recipe progress bar
+            selector = '#recipes-parsed-percentage-bar-' + build.id;
+            $(selector).width(build.recipes_parsed_percentage + '%');
           }
 
-          if (allPercentDone === (100 * prjInfo.builds.length) && !inProgress)
-            window.location.reload();
+          buildData[build.id] = build;
+        }
+      },
 
-          /* Our progress bar is not still showing so shutdown the polling. */
-          if ($(".progress .bar").length === 0)
-            window.clearInterval(progressTimer);
-
-      });
-    }, 1500);
+      // fail callback
+      function (data) {
+        console.error(data);
+      }
+    );
   }
-}
 
+  window.setInterval(refreshMostRecentBuilds, 1500);
+  refreshMostRecentBuilds();
+}

bitbake/lib/toaster/toastergui/static/js/newcustomimage_modal.js

@@ -25,7 +25,11 @@ function newCustomImageModalInit(){
   var duplicateNameMsg = "An image with this name already exists. Image names must be unique.";
   var duplicateImageInProjectMsg = "An image with this name already exists in this project."
   var invalidBaseRecipeIdMsg = "Please select an image to customise.";
-	
+
+  // set button to "submit" state and enable text entry so user can
+  // enter the custom recipe name
+  showSubmitState();
+
   /* capture clicks on radio buttons inside the modal; when one is selected,
    * set the recipe on the modal
    */
@@ -40,6 +44,9 @@ function newCustomImageModalInit(){
   });
 
   newCustomImgBtn.click(function(e){
+    // disable the button and text entry
+    showLoadingState();
+
     e.preventDefault();
 
     var baseRecipeId = imgCustomModal.data('recipe');
@@ -69,12 +76,33 @@ function newCustomImageModalInit(){
           }
         } else {
           imgCustomModal.modal('hide');
+          imgCustomModal.one('hidden.bs.modal', showSubmitState);
           window.location.replace(ret.url + '?notify=new');
         }
       });
     }
   });
 
+  // enable text entry, show "Create image" button text
+  function showSubmitState() {
+    libtoaster.enableAjaxLoadingTimer();
+    newCustomImgBtn.find('[data-role="loading-state"]').hide();
+    newCustomImgBtn.find('[data-role="submit-state"]').show();
+    newCustomImgBtn.removeAttr('disabled');
+    nameInput.removeAttr('disabled');
+  }
+
+  // disable text entry, show "Creating image..." button text;
+  // we also disabled the page-level ajax loading spinner while this spinner
+  // is active
+  function showLoadingState() {
+    libtoaster.disableAjaxLoadingTimer();
+    newCustomImgBtn.find('[data-role="submit-state"]').hide();
+    newCustomImgBtn.find('[data-role="loading-state"]').show();
+    newCustomImgBtn.attr('disabled', 'disabled');
+    nameInput.attr('disabled', 'disabled');
+  }
+
   function showNameError(text){
     invalidNameHelp.text(text);
     invalidNameHelp.show();
@@ -167,6 +195,5 @@ function newCustomImageModalSetRecipes(baseRecipes) {
 
     // show the radio button container
     imageSelector.show();
-	 
-  }	
+  }
 }

bitbake/lib/toaster/toastergui/static/js/projectpage.js

@@ -172,7 +172,12 @@ function projectPageInit(ctx) {
 
       link.attr("href", layerObj.layerdetailurl);
       link.text(layerObj.name);
-      link.tooltip({title: layerObj.vcs_url + " | "+ layerObj.vcs_reference, placement: "right"});
+
+      if (layerObj.local_source_dir) {
+        link.tooltip({title: layerObj.local_source_dir, placement: "right"});
+      } else {
+        link.tooltip({title: layerObj.vcs_url + " | "+ layerObj.vcs_reference, placement: "right"});
+      }
 
       var trashItem = projectLayer.children("span");
       trashItem.click(function (e) {

bitbake/lib/toaster/toastergui/static/js/table.js

@@ -15,6 +15,7 @@ function tableInit(ctx){
     orderby : null,
     filter : null,
     search : null,
+    default_orderby: null,
   };
 
   var defaultHiddenCols = [];
@@ -192,6 +193,8 @@ function tableInit(ctx){
     tableHeadRow.html("");
     editColMenu.html("");
 
+    tableParams.default_orderby = tableData.default_orderby;
+
     if (!tableParams.orderby && tableData.default_orderby){
       tableParams.orderby = tableData.default_orderby;
     }
@@ -217,6 +220,7 @@ function tableInit(ctx){
         var title = $('<a href=\"#\" ></a>');
 
         title.data('field-name', col.field_name);
+        title.attr('data-sort-field', col.field_name);
         title.text(col.title);
         title.click(sortColumnClicked);
 
@@ -344,29 +348,65 @@ function tableInit(ctx){
     }
   }
 
+  /* Apply an ordering to the current table.
+   *
+   * 1. Find the column heading matching the sortSpecifier
+   * 2. Set its up/down arrow and add .sorted
+   *
+   * orderby: e.g. "-started_on", "completed_on"
+   * colHeading: column heading element to activate (by showing the caret
+   * up/down, depending on sort order); if not set, the correct column
+   * heading is selected from the DOM using orderby as a key
+   */
+  function applyOrderby(orderby, colHeading) {
+    if (!orderby) {
+      return;
+    }
+
+    // We only have one sort at a time so remove existing sort indicators
+    $("#" + ctx.tableName + " th .icon-caret-down").hide();
+    $("#" + ctx.tableName + " th .icon-caret-up").hide();
+    $("#" + ctx.tableName + " th a").removeClass("sorted");
+
+    // normalise the orderby so we can use it to find the link we want
+    // to style
+    var fieldName = orderby;
+    if (fieldName.indexOf('-') === 0) {
+      fieldName = fieldName.slice(1);
+    }
+
+    // find the table header element which corresponds to the sort field
+    // (if we don't already have it)
+    if (!colHeading) {
+      colHeading = $('[data-sort-field="' + fieldName + '"]');
+    }
+
+    colHeading.addClass("sorted");
+
+    var parent = colHeading.parent();
+
+    if (orderby.indexOf('-') === 0) {
+      parent.children('.icon-caret-up').show();
+    }
+    else {
+      parent.children('.icon-caret-down').show();
+    }
+
+    tableParams.orderby = orderby;
+    loadData(tableParams);
+  }
+
   function sortColumnClicked(e){
     e.preventDefault();
 
-    /* We only have one sort at a time so remove any existing sort indicators */
-    $("#"+ctx.tableName+" th .icon-caret-down").hide();
-    $("#"+ctx.tableName+" th .icon-caret-up").hide();
-    $("#"+ctx.tableName+" th a").removeClass("sorted");
-
-    var fieldName = $(this).data('field-name');
-
     /* if we're already sorted sort the other way */
-    if (tableParams.orderby === fieldName &&
+    var orderby = $(this).data('field-name');
+    if (tableParams.orderby === orderby &&
         tableParams.orderby.indexOf('-') === -1) {
-      tableParams.orderby = '-' + $(this).data('field-name');
-      $(this).parent().children('.icon-caret-up').show();
-    } else {
-      tableParams.orderby = $(this).data('field-name');
-      $(this).parent().children('.icon-caret-down').show();
+      orderby = '-' + orderby;
     }
 
-    $(this).addClass("sorted");
-
-    loadData(tableParams);
+    applyOrderby(orderby, $(this));
   }
 
   function pageButtonClicked(e) {
@@ -385,11 +425,13 @@ function tableInit(ctx){
       table.find("."+col).show();
     }  else {
       table.find("."+col).hide();
-      /* If we're ordered by the column we're hiding remove the order by */
+      // If we're ordered by the column we're hiding remove the order by
+      // and apply the default one instead
       if (col === tableParams.orderby ||
           '-' + col === tableParams.orderby){
         tableParams.orderby = null;
-        $("#"+ctx.tableName +" .default-orderby").click();
+
+        applyOrderby(tableParams.default_orderby);
       }
     }
 

bitbake/lib/toaster/toastergui/tables.py

@@ -114,8 +114,12 @@ class LayersTable(ToasterTable):
 
         git_url_template = '''
         <a href="{% url 'layerdetails' extra.pid data.id %}">
+        {% if data.layer.local_source_dir %}
+          <code>{{data.layer.local_source_dir}}</code>
+        {% else %}
           <code>{{data.layer.vcs_url}}</code>
         </a>
+        {% endif %}
         {% if data.get_vcs_link_url %}
         <a target="_blank" href="{{ data.get_vcs_link_url }}">
            <span class="glyphicon glyphicon-new-window"></span>
@@ -123,16 +127,21 @@ class LayersTable(ToasterTable):
         {% endif %}
         '''
 
-        self.add_column(title="Git repository URL",
-                        help_text="The Git repository for the layer source code",
+        self.add_column(title="Layer source code location",
+                        help_text="A Git repository or an absolute path to a directory",
                         hidden=True,
                         static_data_name="layer__vcs_url",
                         static_data_template=git_url_template)
 
         git_dir_template = '''
+        {% if data.layer.local_source_dir %}
+        <span class="text-muted">Not applicable</span>
+        <span class="glyphicon glyphicon-question-sign get-help" data-original-title="" title="The source code of {{data.layer.name}} is not in a Git repository, so there is no subdirectory associated with it"> </span>
+        {% else %}
         <a href="{% url 'layerdetails' extra.pid data.id %}">
          <code>{{data.dirpath}}</code>
         </a>
+        {% endif %}
         {% if data.dirpath and data.get_vcs_dirpath_link_url %}
         <a target="_blank" href="{{ data.get_vcs_dirpath_link_url }}">
           <span class="glyphicon glyphicon-new-window"></span>
@@ -146,9 +155,14 @@ class LayersTable(ToasterTable):
                         static_data_template=git_dir_template)
 
         revision_template =  '''
+        {% if data.layer.local_source_dir %}
+        <span class="text-muted">Not applicable</span>
+        <span class="glyphicon glyphicon-question-sign get-help" data-original-title="" title="The source code of {{data.layer.name}} is not in a Git repository, so there is no revision associated with it"> </span>
+        {% else %}
         {% with vcs_ref=data.get_vcs_reference %}
         {% include 'snippets/gitrev_popover.html' %}
         {% endwith %}
+        {% endif %}
         '''
 
         self.add_column(title="Git revision",
@@ -413,9 +427,19 @@ class RecipesTable(ToasterTable):
                         orderable=True,
                         field_name="license")
 
+        revision_link_template = '''
+        {% if data.layer_version.layer.local_source_dir %}
+        <span class="text-muted">Not applicable</span>
+        <span class="glyphicon glyphicon-question-sign get-help" data-original-title="" title="The source code of {{data.layer_version.layer.name}} is not in a Git repository, so there is no revision associated with it"> </span>
+        {% else %}
+        {{data.layer_version.get_vcs_reference}}
+        {% endif %}
+        '''
+
         self.add_column(title="Git revision",
                         hidden=True,
-                        field_name="layer_version__get_vcs_reference")
+                        static_data_name="layer_version__get_vcs_reference",
+                        static_data_template=revision_link_template)
 
 
 class LayerRecipesTable(RecipesTable):
@@ -1197,9 +1221,13 @@ class BuildsTable(ToasterTable):
 
         time_template = '''
         {% load projecttags %}
-        <a href="{% url "buildtime" data.id %}">
+        {% if data.outcome == extra.Build.SUCCEEDED %}
+            <a href="{% url "buildtime" data.id %}">
+                {{data.timespent_seconds | sectohms}}
+            </a>
+        {% else %}
             {{data.timespent_seconds | sectohms}}
-        </a>
+        {% endif %}
         '''
 
         image_files_template = '''