build-appliance-image: Update to master head revision

(From OE-Core rev: e3f5ceec690379d64bd38330111a32f6c8fc6353)

Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>

bitbake/bin/bitbake

@@ -38,7 +38,7 @@ from bb.main import bitbake_main, BitBakeConfigParameters, BBMainException
 if sys.getfilesystemencoding() != "utf-8":
     sys.exit("Please use a locale setting which supports UTF-8 (such as LANG=en_US.UTF-8).\nPython can't change the filesystem locale after loading so we need a UTF-8 when Python starts or things won't work.")
 
-__version__ = "1.39.0"
+__version__ = "1.39.1"
 
 if __name__ == "__main__":
     if __version__ != bb.__version__:

bitbake/bin/bitbake-selftest

@@ -22,16 +22,21 @@ sys.path.insert(0, os.path.join(os.path.dirname(os.path.dirname(__file__)), 'lib
 import unittest
 try:
     import bb
+    import layerindexlib
 except RuntimeError as exc:
     sys.exit(str(exc))
 
 tests = ["bb.tests.codeparser",
+         "bb.tests.cooker",
          "bb.tests.cow",
          "bb.tests.data",
          "bb.tests.event",
          "bb.tests.fetch",
          "bb.tests.parse",
-         "bb.tests.utils"]
+         "bb.tests.utils",
+         "layerindexlib.tests.layerindexobj",
+         "layerindexlib.tests.restapi",
+         "layerindexlib.tests.cooker"]
 
 for t in tests:
     t = '.'.join(t.split('.')[:3])

bitbake/bin/toaster

@@ -18,11 +18,12 @@
 # along with this program. If not, see http://www.gnu.org/licenses/.
 
 HELP="
-Usage: source toaster start|stop [webport=<address:port>] [noweb] [nobuild]
+Usage: source toaster start|stop [webport=<address:port>] [noweb] [nobuild] [toasterdir]
     Optional arguments:
         [nobuild] Setup the environment for capturing builds with toaster but disable managed builds
         [noweb] Setup the environment for capturing builds with toaster but don't start the web server
         [webport] Set the development server (default: localhost:8000)
+        [toasterdir] Set absolute path to be used as TOASTER_DIR (default: BUILDDIR/../)
 "
 
 custom_extention()
@@ -186,6 +187,7 @@ unset OE_ROOT
 WEBSERVER=1
 export TOASTER_BUILDSERVER=1
 ADDR_PORT="localhost:8000"
+TOASTERDIR=`dirname $BUILDDIR`
 unset CMD
 for param in $*; do
     case $param in
@@ -211,6 +213,9 @@ for param in $*; do
                 ADDR_PORT="localhost:$PORT"
             fi
     ;;
+    toasterdir=*)
+            TOASTERDIR="${param#*=}"
+    ;;
     --help)
             echo "$HELP"
             return 0
@@ -241,7 +246,7 @@ fi
 # 2) the build dir (in build)
 # 3) the sqlite db if that is being used.
 # 4) pid's we need to clean up on exit/shutdown
-export TOASTER_DIR=`dirname $BUILDDIR`
+export TOASTER_DIR=$TOASTERDIR
 export BB_ENV_EXTRAWHITE="$BB_ENV_EXTRAWHITE TOASTER_DIR"
 
 # Determine the action. If specified by arguments, fine, if not, toggle it

bitbake/doc/bitbake-user-manual/bitbake-user-manual-metadata.xml

@@ -1929,6 +1929,38 @@
                         not careful.
                     </note>
                     </para></listitem>
+                <listitem><para><emphasis><filename>[number_threads]</filename>:</emphasis>
+                    Limits tasks to a specific number of simultaneous threads
+                    during execution.
+                    This varflag is useful when your build host has a large number
+                    of cores but certain tasks need to be rate-limited due to various
+                    kinds of resource constraints (e.g. to avoid network throttling).
+                    <filename>number_threads</filename> works similarly to the
+                    <link linkend='var-BB_NUMBER_THREADS'><filename>BB_NUMBER_THREADS</filename></link>
+                    variable but is task-specific.</para>
+
+                    <para>Set the value globally.
+                    For example, the following makes sure the
+                    <filename>do_fetch</filename> task uses no more than two
+                    simultaneous execution threads:
+                    <literallayout class='monospaced'>
+     do_fetch[number_threads] = "2"
+                    </literallayout>
+                    <note><title>Warnings</title>
+                        <itemizedlist>
+                            <listitem><para>
+                                Setting the varflag in individual recipes rather
+                                than globally can result in unpredictable behavior.
+                                </para></listitem>
+                            <listitem><para>
+                                Setting the varflag to a value greater than the
+                                value used in the <filename>BB_NUMBER_THREADS</filename>
+                                variable causes <filename>number_threads</filename>
+                                to have no effect.
+                                </para></listitem>
+                        </itemizedlist>
+                    </note>
+                    </para></listitem>
                 <listitem><para><emphasis><filename>[postfuncs]</filename>:</emphasis>
                     List of functions to call after the completion of the task.
                     </para></listitem>

bitbake/lib/bb/COW.py

@@ -150,7 +150,7 @@ class COWDictMeta(COWMeta):
                 yield value
             if type == "items":
                 yield (key, value)
-        raise StopIteration()
+        return
 
     def iterkeys(cls):
         return cls.iter("keys")

bitbake/lib/bb/__init__.py

@@ -21,7 +21,7 @@
 # with this program; if not, write to the Free Software Foundation, Inc.,
 # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
 
-__version__ = "1.39.0"
+__version__ = "1.39.1"
 
 import sys
 if sys.version_info < (3, 4, 0):
@@ -63,6 +63,10 @@ class BBLogger(Logger):
     def verbose(self, msg, *args, **kwargs):
         return self.log(logging.INFO - 1, msg, *args, **kwargs)
 
+    def verbnote(self, msg, *args, **kwargs):
+        return self.log(logging.INFO + 2, msg, *args, **kwargs)
+
+
 logging.raiseExceptions = False
 logging.setLoggerClass(BBLogger)
 
@@ -93,6 +97,18 @@ def debug(lvl, *args):
 def note(*args):
     mainlogger.info(''.join(args))
 
+#
+# A higher prioity note which will show on the console but isn't a warning
+#
+# Something is happening the user should be aware of but they probably did
+# something to make it happen
+#
+def verbnote(*args):
+    mainlogger.verbnote(''.join(args))
+
+#
+# Warnings - things the user likely needs to pay attention to and fix
+#
 def warn(*args):
     mainlogger.warning(''.join(args))
 

bitbake/lib/bb/build.py

@@ -41,8 +41,6 @@ from bb import data, event, utils
 bblogger = logging.getLogger('BitBake')
 logger = logging.getLogger('BitBake.Build')
 
-NULL = open(os.devnull, 'r+')
-
 __mtime_cache = {}
 
 def cached_mtime_noerror(f):
@@ -533,7 +531,6 @@ def _exec_task(fn, task, d, quieterr):
                 self.triggered = True
 
     # Handle logfiles
-    si = open('/dev/null', 'r')
     try:
         bb.utils.mkdirhier(os.path.dirname(logfn))
         logfile = open(logfn, 'w')
@@ -547,7 +544,8 @@ def _exec_task(fn, task, d, quieterr):
     ose = [os.dup(sys.stderr.fileno()), sys.stderr.fileno()]
 
     # Replace those fds with our own
-    os.dup2(si.fileno(), osi[1])
+    with open('/dev/null', 'r') as si:
+        os.dup2(si.fileno(), osi[1])
     os.dup2(logfile.fileno(), oso[1])
     os.dup2(logfile.fileno(), ose[1])
 
@@ -608,7 +606,6 @@ def _exec_task(fn, task, d, quieterr):
         os.close(osi[0])
         os.close(oso[0])
         os.close(ose[0])
-        si.close()
 
         logfile.close()
         if os.path.exists(logfn) and os.path.getsize(logfn) == 0:
@@ -803,6 +800,7 @@ def add_tasks(tasklist, d):
             if name in flags:
                 deptask = d.expand(flags[name])
                 task_deps[name][task] = deptask
+        getTask('mcdepends')
         getTask('depends')
         getTask('rdepends')
         getTask('deptask')

bitbake/lib/bb/cache.py

@@ -37,7 +37,7 @@ import bb.utils
 
 logger = logging.getLogger("BitBake.Cache")
 
-__cache_version__ = "151"
+__cache_version__ = "152"
 
 def getCacheFile(path, filename, data_hash):
     return os.path.join(path, filename + "." + data_hash)

bitbake/lib/bb/checksum.py

@@ -97,6 +97,8 @@ class FileChecksumCache(MultiProcessCache):
 
         def checksum_dir(pth):
             # Handle directories recursively
+            if pth == "/":
+                bb.fatal("Refusing to checksum /")
             dirchecksums = []
             for root, dirs, files in os.walk(pth):
                 for name in files:

bitbake/lib/bb/codeparser.py

@@ -140,7 +140,7 @@ class CodeParserCache(MultiProcessCache):
     # so that an existing cache gets invalidated. Additionally you'll need
     # to increment __cache_version__ in cache.py in order to ensure that old
     # recipe caches don't trigger "Taskhash mismatch" errors.
-    CACHE_VERSION = 9
+    CACHE_VERSION = 10
 
     def __init__(self):
         MultiProcessCache.__init__(self)
@@ -214,7 +214,7 @@ class BufferedLogger(Logger):
         self.buffer = []
 
 class PythonParser():
-    getvars = (".getVar", ".appendVar", ".prependVar")
+    getvars = (".getVar", ".appendVar", ".prependVar", "oe.utils.conditional")
     getvarflags = (".getVarFlag", ".appendVarFlag", ".prependVarFlag")
     containsfuncs = ("bb.utils.contains", "base_contains")
     containsanyfuncs = ("bb.utils.contains_any",  "bb.utils.filter")

bitbake/lib/bb/cooker.py

@@ -609,7 +609,14 @@ class BBCooker:
                 k2 = k.split(":do_")
                 k = k2[0]
                 ktask = k2[1]
-            taskdata[mc].add_provider(localdata[mc], self.recipecaches[mc], k)
+            if mc:
+                # Provider might be from another mc
+                for mcavailable in self.multiconfigs:
+                    # The first element is empty
+                    if mcavailable:
+                        taskdata[mcavailable].add_provider(localdata[mcavailable], self.recipecaches[mcavailable], k)
+            else:
+                taskdata[mc].add_provider(localdata[mc], self.recipecaches[mc], k)
             current += 1
             if not ktask.startswith("do_"):
                 ktask = "do_%s" % ktask
@@ -620,6 +627,27 @@ class BBCooker:
             runlist.append([mc, k, ktask, fn])
             bb.event.fire(bb.event.TreeDataPreparationProgress(current, len(fulltargetlist)), self.data)
 
+        mcdeps = taskdata[mc].get_mcdepends()
+        # No need to do check providers if there are no mcdeps or not an mc build
+        if mcdeps and mc:
+            # Make sure we can provide the multiconfig dependency
+            seen = set()
+            new = True
+            while new:
+                new = False
+                for mc in self.multiconfigs:
+                    for k in mcdeps:
+                        if k in seen:
+                            continue
+                        l = k.split(':')
+                        depmc = l[2]
+                        if depmc not in self.multiconfigs:
+                            bb.fatal("Multiconfig dependency %s depends on nonexistent mc configuration %s" % (k,depmc))
+                        else:
+                            logger.debug(1, "Adding providers for multiconfig dependency %s" % l[3])
+                            taskdata[depmc].add_provider(localdata[depmc], self.recipecaches[depmc], l[3])
+                            seen.add(k)
+                            new = True
         for mc in self.multiconfigs:
             taskdata[mc].add_unresolved(localdata[mc], self.recipecaches[mc])
 
@@ -706,8 +734,8 @@ class BBCooker:
             if not dotname in depend_tree["tdepends"]:
                 depend_tree["tdepends"][dotname] = []
             for dep in rq.rqdata.runtaskentries[tid].depends:
-                (depmc, depfn, deptaskname, deptaskfn) = bb.runqueue.split_tid_mcfn(dep)
-                deppn = self.recipecaches[mc].pkg_fn[deptaskfn]
+                (depmc, depfn, _, deptaskfn) = bb.runqueue.split_tid_mcfn(dep)
+                deppn = self.recipecaches[depmc].pkg_fn[deptaskfn]
                 depend_tree["tdepends"][dotname].append("%s.%s" % (deppn, bb.runqueue.taskname_from_tid(dep)))
             if taskfn not in seen_fns:
                 seen_fns.append(taskfn)
@@ -1566,7 +1594,7 @@ class BBCooker:
                     pkgs_to_build.append(t)
 
         if 'universe' in pkgs_to_build:
-            parselog.warning("The \"universe\" target is only intended for testing and may produce errors.")
+            parselog.verbnote("The \"universe\" target is only intended for testing and may produce errors.")
             parselog.debug(1, "collating packages for \"universe\"")
             pkgs_to_build.remove('universe')
             for mc in self.multiconfigs:
@@ -1633,7 +1661,10 @@ class CookerExit(bb.event.Event):
 class CookerCollectFiles(object):
     def __init__(self, priorities):
         self.bbappends = []
-        self.bbfile_config_priorities = priorities
+        # Priorities is a list of tupples, with the second element as the pattern.
+        # We need to sort the list with the longest pattern first, and so on to
+        # the shortest.  This allows nested layers to be properly evaluated.
+        self.bbfile_config_priorities = sorted(priorities, key=lambda tup: tup[1], reverse=True)
 
     def calc_bbfile_priority( self, filename, matched = None ):
         for _, _, regex, pri in self.bbfile_config_priorities:

bitbake/lib/bb/daemonize.py

@@ -16,6 +16,10 @@ def createDaemon(function, logfile):
     background as a daemon, returning control to the caller.
     """
 
+    # Ensure stdout/stderror are flushed before forking to avoid duplicate output
+    sys.stdout.flush()
+    sys.stderr.flush()
+
     try:
         # Fork a child process so the parent can exit.  This returns control to
         # the command-line or shell.  It also guarantees that the child will not
@@ -49,8 +53,8 @@ def createDaemon(function, logfile):
             # exit() or _exit()?
             # _exit is like exit(), but it doesn't call any functions registered
             # with atexit (and on_exit) or any registered signal handlers.  It also
-            # closes any open file descriptors.  Using exit() may cause all stdio
-            # streams to be flushed twice and any temporary files may be unexpectedly
+            # closes any open file descriptors, but doesn't flush any buffered output.
+            # Using exit() may cause all any temporary files to be unexpectedly
             # removed.  It's therefore recommended that child branches of a fork()
             # and the parent branch(es) of a daemon use _exit().
             os._exit(0)
@@ -61,17 +65,19 @@ def createDaemon(function, logfile):
     # The second child.
 
     # Replace standard fds with our own
-    si = open('/dev/null', 'r')
-    os.dup2(si.fileno(), sys.stdin.fileno())
+    with open('/dev/null', 'r') as si:
+        os.dup2(si.fileno(), sys.stdin.fileno())
 
     try:
         so = open(logfile, 'a+')
-        se = so
         os.dup2(so.fileno(), sys.stdout.fileno())
-        os.dup2(se.fileno(), sys.stderr.fileno())
+        os.dup2(so.fileno(), sys.stderr.fileno())
     except io.UnsupportedOperation:
         sys.stdout = open(logfile, 'a+')
-        sys.stderr = sys.stdout
+
+    # Have stdout and stderr be the same so log output matches chronologically
+    # and there aren't two seperate buffers
+    sys.stderr = sys.stdout
 
     try:
         function()
@@ -79,4 +85,9 @@ def createDaemon(function, logfile):
         traceback.print_exc()
     finally:
         bb.event.print_ui_queue()
+        # os._exit() doesn't flush open files like os.exit() does. Manually flush
+        # stdout and stderr so that any logging output will be seen, particularly
+        # exception tracebacks.
+        sys.stdout.flush()
+        sys.stderr.flush()
         os._exit(0)

bitbake/lib/bb/event.py

@@ -141,6 +141,9 @@ def print_ui_queue():
     logger = logging.getLogger("BitBake")
     if not _uiready:
         from bb.msg import BBLogFormatter
+        # Flush any existing buffered content
+        sys.stdout.flush()
+        sys.stderr.flush()
         stdout = logging.StreamHandler(sys.stdout)
         stderr = logging.StreamHandler(sys.stderr)
         formatter = BBLogFormatter("%(levelname)s: %(message)s")
@@ -395,7 +398,7 @@ class RecipeEvent(Event):
         Event.__init__(self)
 
 class RecipePreFinalise(RecipeEvent):
-    """ Recipe Parsing Complete but not yet finialised"""
+    """ Recipe Parsing Complete but not yet finalised"""
 
 class RecipeTaskPreProcess(RecipeEvent):
     """

bitbake/lib/bb/fetch2/__init__.py

@@ -452,8 +452,8 @@ def uri_replace(ud, uri_find, uri_replace, replacements, d, mirrortarball=None):
             # Handle URL parameters
             if i:
                 # Any specified URL parameters must match
-                for k in uri_replace_decoded[loc]:
-                    if uri_decoded[loc][k] != uri_replace_decoded[loc][k]:
+                for k in uri_find_decoded[loc]:
+                    if uri_decoded[loc][k] != uri_find_decoded[loc][k]:
                         return None
             # Overwrite any specified replacement parameters
             for k in uri_replace_decoded[loc]:

bitbake/lib/bb/fetch2/bzr.py

@@ -41,8 +41,9 @@ class Bzr(FetchMethod):
         init bzr specific variable within url data
         """
         # Create paths to bzr checkouts
+        bzrdir = d.getVar("BZRDIR") or (d.getVar("DL_DIR") + "/bzr")
         relpath = self._strip_leading_slashes(ud.path)
-        ud.pkgdir = os.path.join(d.expand('${BZRDIR}'), ud.host, relpath)
+        ud.pkgdir = os.path.join(bzrdir, ud.host, relpath)
 
         ud.setup_revisions(d)
 
@@ -57,7 +58,7 @@ class Bzr(FetchMethod):
         command is "fetch", "update", "revno"
         """
 
-        basecmd = d.expand('${FETCHCMD_bzr}')
+        basecmd = d.getVar("FETCHCMD_bzr") or "/usr/bin/env bzr"
 
         proto =  ud.parm.get('protocol', 'http')
 

bitbake/lib/bb/fetch2/cvs.py

@@ -110,7 +110,7 @@ class Cvs(FetchMethod):
         if ud.tag:
             options.append("-r %s" % ud.tag)
 
-        cvsbasecmd = d.getVar("FETCHCMD_cvs")
+        cvsbasecmd = d.getVar("FETCHCMD_cvs") or "/usr/bin/env cvs"
         cvscmd = cvsbasecmd + " '-d" + cvsroot + "' co " + " ".join(options) + " " + ud.module
         cvsupdatecmd = cvsbasecmd + " '-d" + cvsroot + "' update -d -P " + " ".join(options)
 
@@ -121,7 +121,8 @@ class Cvs(FetchMethod):
         # create module directory
         logger.debug(2, "Fetch: checking for module directory")
         pkg = d.getVar('PN')
-        pkgdir = os.path.join(d.getVar('CVSDIR'), pkg)
+        cvsdir = d.getVar("CVSDIR") or (d.getVar("DL_DIR") + "/cvs")
+        pkgdir = os.path.join(cvsdir, pkg)
         moddir = os.path.join(pkgdir, localdir)
         workdir = None
         if os.access(os.path.join(moddir, 'CVS'), os.R_OK):

bitbake/lib/bb/fetch2/git.py

@@ -261,7 +261,7 @@ class Git(FetchMethod):
                 gitsrcname = gitsrcname + '_' + ud.revisions[name]
 
         dl_dir = d.getVar("DL_DIR")
-        gitdir = d.getVar("GITDIR") or (dl_dir + "/git2/")
+        gitdir = d.getVar("GITDIR") or (dl_dir + "/git2")
         ud.clonedir = os.path.join(gitdir, gitsrcname)
         ud.localfile = ud.clonedir
 
@@ -353,10 +353,9 @@ class Git(FetchMethod):
                 break
 
         if needupdate:
-            try: 
-                runfetchcmd("%s remote rm origin" % ud.basecmd, d, workdir=ud.clonedir)
-            except bb.fetch2.FetchError:
-                logger.debug(1, "No Origin")
+            output = runfetchcmd("%s remote" % ud.basecmd, d, quiet=True, workdir=ud.clonedir)
+            if "origin" in output:
+              runfetchcmd("%s remote rm origin" % ud.basecmd, d, workdir=ud.clonedir)
 
             runfetchcmd("%s remote add --mirror=fetch origin %s" % (ud.basecmd, repourl), d, workdir=ud.clonedir)
             fetch_cmd = "LANG=C %s fetch -f --prune --progress %s refs/*:refs/*" % (ud.basecmd, repourl)

bitbake/lib/bb/fetch2/hg.py

@@ -80,7 +80,7 @@ class Hg(FetchMethod):
         ud.fullmirror = os.path.join(d.getVar("DL_DIR"), mirrortarball)
         ud.mirrortarballs = [mirrortarball]
 
-        hgdir = d.getVar("HGDIR") or (d.getVar("DL_DIR") + "/hg/")
+        hgdir = d.getVar("HGDIR") or (d.getVar("DL_DIR") + "/hg")
         ud.pkgdir = os.path.join(hgdir, hgsrcname)
         ud.moddir = os.path.join(ud.pkgdir, ud.module)
         ud.localfile = ud.moddir

bitbake/lib/bb/fetch2/osc.py

@@ -32,8 +32,9 @@ class Osc(FetchMethod):
         ud.module = ud.parm["module"]
 
         # Create paths to osc checkouts
+        oscdir = d.getVar("OSCDIR") or (d.getVar("DL_DIR") + "/osc")
         relpath = self._strip_leading_slashes(ud.path)
-        ud.pkgdir = os.path.join(d.getVar('OSCDIR'), ud.host)
+        ud.pkgdir = os.path.join(oscdir, ud.host)
         ud.moddir = os.path.join(ud.pkgdir, relpath, ud.module)
 
         if 'rev' in ud.parm:
@@ -54,7 +55,7 @@ class Osc(FetchMethod):
         command is "fetch", "update", "info"
         """
 
-        basecmd = d.expand('${FETCHCMD_osc}')
+        basecmd = d.getVar("FETCHCMD_osc") or "/usr/bin/env osc"
 
         proto = ud.parm.get('protocol', 'ocs')
 

bitbake/lib/bb/fetch2/perforce.py

@@ -43,13 +43,9 @@ class Perforce(FetchMethod):
         provided by the env, use it.  If P4PORT is specified by the recipe, use
         its values, which may override the settings in P4CONFIG.
         """
-        ud.basecmd = d.getVar('FETCHCMD_p4')
-        if not ud.basecmd:
-            ud.basecmd = "/usr/bin/env p4"
+        ud.basecmd = d.getVar("FETCHCMD_p4") or "/usr/bin/env p4"
 
-        ud.dldir = d.getVar('P4DIR')
-        if not ud.dldir:
-            ud.dldir = '%s/%s' % (d.getVar('DL_DIR'), 'p4')
+        ud.dldir = d.getVar("P4DIR") or (d.getVar("DL_DIR") + "/p4")
 
         path = ud.url.split('://')[1]
         path = path.split(';')[0]

bitbake/lib/bb/fetch2/repo.py

@@ -45,6 +45,8 @@ class Repo(FetchMethod):
         "master".
         """
 
+        ud.basecmd = d.getVar("FETCHCMD_repo") or "/usr/bin/env repo"
+
         ud.proto = ud.parm.get('protocol', 'git')
         ud.branch = ud.parm.get('branch', 'master')
         ud.manifest = ud.parm.get('manifest', 'default.xml')
@@ -60,8 +62,8 @@ class Repo(FetchMethod):
             logger.debug(1, "%s already exists (or was stashed). Skipping repo init / sync.", ud.localpath)
             return
 
+        repodir = d.getVar("REPODIR") or (d.getVar("DL_DIR") + "/repo")
         gitsrcname = "%s%s" % (ud.host, ud.path.replace("/", "."))
-        repodir = d.getVar("REPODIR") or os.path.join(d.getVar("DL_DIR"), "repo")
         codir = os.path.join(repodir, gitsrcname, ud.manifest)
 
         if ud.user:
@@ -72,11 +74,11 @@ class Repo(FetchMethod):
         repodir = os.path.join(codir, "repo")
         bb.utils.mkdirhier(repodir)
         if not os.path.exists(os.path.join(repodir, ".repo")):
-            bb.fetch2.check_network_access(d, "repo init -m %s -b %s -u %s://%s%s%s" % (ud.manifest, ud.branch, ud.proto, username, ud.host, ud.path), ud.url)
-            runfetchcmd("repo init -m %s -b %s -u %s://%s%s%s" % (ud.manifest, ud.branch, ud.proto, username, ud.host, ud.path), d, workdir=repodir)
+            bb.fetch2.check_network_access(d, "%s init -m %s -b %s -u %s://%s%s%s" % (ud.basecmd, ud.manifest, ud.branch, ud.proto, username, ud.host, ud.path), ud.url)
+            runfetchcmd("%s init -m %s -b %s -u %s://%s%s%s" % (ud.basecmd, ud.manifest, ud.branch, ud.proto, username, ud.host, ud.path), d, workdir=repodir)
 
-        bb.fetch2.check_network_access(d, "repo sync %s" % ud.url, ud.url)
-        runfetchcmd("repo sync", d, workdir=repodir)
+        bb.fetch2.check_network_access(d, "%s sync %s" % (ud.basecmd, ud.url), ud.url)
+        runfetchcmd("%s sync" % ud.basecmd, d, workdir=repodir)
 
         scmdata = ud.parm.get("scmdata", "")
         if scmdata == "keep":

bitbake/lib/bb/fetch2/svn.py

@@ -49,7 +49,7 @@ class Svn(FetchMethod):
         if not "module" in ud.parm:
             raise MissingParameterError('module', ud.url)
 
-        ud.basecmd = d.getVar('FETCHCMD_svn')
+        ud.basecmd = d.getVar("FETCHCMD_svn") or "/usr/bin/env svn --non-interactive --trust-server-cert"
 
         ud.module = ud.parm["module"]
 
@@ -59,8 +59,9 @@ class Svn(FetchMethod):
             ud.path_spec = ud.parm["path_spec"]
 
         # Create paths to svn checkouts
+        svndir = d.getVar("SVNDIR") or (d.getVar("DL_DIR") + "/svn")
         relpath = self._strip_leading_slashes(ud.path)
-        ud.pkgdir = os.path.join(d.expand('${SVNDIR}'), ud.host, relpath)
+        ud.pkgdir = os.path.join(svndir, ud.host, relpath)
         ud.moddir = os.path.join(ud.pkgdir, ud.module)
 
         ud.setup_revisions(d)

bitbake/lib/bb/msg.py

@@ -40,6 +40,7 @@ class BBLogFormatter(logging.Formatter):
     VERBOSE = logging.INFO - 1
     NOTE = logging.INFO
     PLAIN = logging.INFO + 1
+    VERBNOTE = logging.INFO + 2
     ERROR = logging.ERROR
     WARNING = logging.WARNING
     CRITICAL = logging.CRITICAL
@@ -51,6 +52,7 @@ class BBLogFormatter(logging.Formatter):
         VERBOSE: 'NOTE',
         NOTE    : 'NOTE',
         PLAIN  : '',
+        VERBNOTE: 'NOTE',
         WARNING : 'WARNING',
         ERROR   : 'ERROR',
         CRITICAL: 'ERROR',
@@ -66,6 +68,7 @@ class BBLogFormatter(logging.Formatter):
         VERBOSE : BASECOLOR,
         NOTE    : BASECOLOR,
         PLAIN   : BASECOLOR,
+        VERBNOTE: BASECOLOR,
         WARNING : YELLOW,
         ERROR   : RED,
         CRITICAL: RED,

bitbake/lib/bb/parse/ast.py

@@ -343,30 +343,31 @@ def runAnonFuncs(d):
 
 def finalize(fn, d, variant = None):
     saved_handlers = bb.event.get_handlers().copy()
+    try:
+        for var in d.getVar('__BBHANDLERS', False) or []:
+            # try to add the handler
+            handlerfn = d.getVarFlag(var, "filename", False)
+            if not handlerfn:
+                bb.fatal("Undefined event handler function '%s'" % var)
+            handlerln = int(d.getVarFlag(var, "lineno", False))
+            bb.event.register(var, d.getVar(var, False), (d.getVarFlag(var, "eventmask") or "").split(), handlerfn, handlerln)
 
-    for var in d.getVar('__BBHANDLERS', False) or []:
-        # try to add the handler
-        handlerfn = d.getVarFlag(var, "filename", False)
-        if not handlerfn:
-            bb.fatal("Undefined event handler function '%s'" % var)
-        handlerln = int(d.getVarFlag(var, "lineno", False))
-        bb.event.register(var, d.getVar(var, False), (d.getVarFlag(var, "eventmask") or "").split(), handlerfn, handlerln)
+        bb.event.fire(bb.event.RecipePreFinalise(fn), d)
 
-    bb.event.fire(bb.event.RecipePreFinalise(fn), d)
+        bb.data.expandKeys(d)
+        runAnonFuncs(d)
 
-    bb.data.expandKeys(d)
-    runAnonFuncs(d)
+        tasklist = d.getVar('__BBTASKS', False) or []
+        bb.event.fire(bb.event.RecipeTaskPreProcess(fn, list(tasklist)), d)
+        bb.build.add_tasks(tasklist, d)
 
-    tasklist = d.getVar('__BBTASKS', False) or []
-    bb.event.fire(bb.event.RecipeTaskPreProcess(fn, list(tasklist)), d)
-    bb.build.add_tasks(tasklist, d)
+        bb.parse.siggen.finalise(fn, d, variant)
 
-    bb.parse.siggen.finalise(fn, d, variant)
+        d.setVar('BBINCLUDED', bb.parse.get_file_depends(d))
 
-    d.setVar('BBINCLUDED', bb.parse.get_file_depends(d))
-
-    bb.event.fire(bb.event.RecipeParsed(fn), d)
-    bb.event.set_handlers(saved_handlers)
+        bb.event.fire(bb.event.RecipeParsed(fn), d)
+    finally:
+        bb.event.set_handlers(saved_handlers)
 
 def _create_variants(datastores, names, function, onlyfinalise):
     def create_variant(name, orig_d, arg = None):

bitbake/lib/bb/runqueue.py

@@ -94,13 +94,13 @@ class RunQueueStats:
         self.active = self.active - 1
         self.failed = self.failed + 1
 
-    def taskCompleted(self, number = 1):
-        self.active = self.active - number
-        self.completed = self.completed + number
+    def taskCompleted(self):
+        self.active = self.active - 1
+        self.completed = self.completed + 1
 
-    def taskSkipped(self, number = 1):
-        self.active = self.active + number
-        self.skipped = self.skipped + number
+    def taskSkipped(self):
+        self.active = self.active + 1
+        self.skipped = self.skipped + 1
 
     def taskActive(self):
         self.active = self.active + 1
@@ -134,6 +134,7 @@ class RunQueueScheduler(object):
         self.prio_map = [self.rqdata.runtaskentries.keys()]
 
         self.buildable = []
+        self.skip_maxthread = {}
         self.stamps = {}
         for tid in self.rqdata.runtaskentries:
             (mc, fn, taskname, taskfn) = split_tid_mcfn(tid)
@@ -150,8 +151,25 @@ class RunQueueScheduler(object):
         self.buildable = [x for x in self.buildable if x not in self.rq.runq_running]
         if not self.buildable:
             return None
+
+        # Filter out tasks that have a max number of threads that have been exceeded
+        skip_buildable = {}
+        for running in self.rq.runq_running.difference(self.rq.runq_complete):
+            rtaskname = taskname_from_tid(running)
+            if rtaskname not in self.skip_maxthread:
+                self.skip_maxthread[rtaskname] = self.rq.cfgData.getVarFlag(rtaskname, "number_threads")
+            if not self.skip_maxthread[rtaskname]:
+                continue
+            if rtaskname in skip_buildable:
+                skip_buildable[rtaskname] += 1
+            else:
+                skip_buildable[rtaskname] = 1
+
         if len(self.buildable) == 1:
             tid = self.buildable[0]
+            taskname = taskname_from_tid(tid)
+            if taskname in skip_buildable and skip_buildable[taskname] >= int(self.skip_maxthread[taskname]):
+                return None
             stamp = self.stamps[tid]
             if stamp not in self.rq.build_stamps.values():
                 return tid
@@ -164,6 +182,9 @@ class RunQueueScheduler(object):
         best = None
         bestprio = None
         for tid in self.buildable:
+            taskname = taskname_from_tid(tid)
+            if taskname in skip_buildable and skip_buildable[taskname] >= int(self.skip_maxthread[taskname]):
+                continue
             prio = self.rev_prio_map[tid]
             if bestprio is None or bestprio > prio:
                 stamp = self.stamps[tid]
@@ -178,7 +199,7 @@ class RunQueueScheduler(object):
         """
         Return the id of the task we should build next
         """
-        if self.rq.stats.active < self.rq.number_tasks:
+        if self.rq.can_start_task():
             return self.next_buildable_task()
 
     def newbuildable(self, task):
@@ -581,6 +602,19 @@ class RunQueueData:
                     if t in taskData[mc].taskentries:
                         depends.add(t)
 
+        def add_mc_dependencies(mc, tid):
+            mcdeps = taskData[mc].get_mcdepends()
+            for dep in mcdeps:
+                mcdependency = dep.split(':')
+                pn = mcdependency[3]
+                frommc = mcdependency[1]
+                mcdep = mcdependency[2]
+                deptask = mcdependency[4]
+                if mc == frommc:
+                    fn = taskData[mcdep].build_targets[pn][0]
+                    newdep = '%s:%s' % (fn,deptask)
+                    taskData[mc].taskentries[tid].tdepends.append(newdep)
+
         for mc in taskData:
             for tid in taskData[mc].taskentries:
 
@@ -597,12 +631,16 @@ class RunQueueData:
                 if fn in taskData[mc].failed_fns:
                     continue
 
+                # We add multiconfig dependencies before processing internal task deps (tdepends)
+                if 'mcdepends' in task_deps and taskname in task_deps['mcdepends']:
+                    add_mc_dependencies(mc, tid)
+
                 # Resolve task internal dependencies
                 #
                 # e.g. addtask before X after Y
                 for t in taskData[mc].taskentries[tid].tdepends:
-                    (_, depfn, deptaskname, _) = split_tid_mcfn(t)
-                    depends.add(build_tid(mc, depfn, deptaskname))
+                    (depmc, depfn, deptaskname, _) = split_tid_mcfn(t)
+                    depends.add(build_tid(depmc, depfn, deptaskname))
 
                 # Resolve 'deptask' dependencies
                 #
@@ -1032,7 +1070,7 @@ class RunQueueData:
                     msg += "\n%s has unique rprovides:\n  %s" % (provfn, "\n  ".join(rprovide_results[provfn] - commonrprovs))
 
                 if self.warn_multi_bb:
-                    logger.warning(msg)
+                    logger.verbnote(msg)
                 else:
                     logger.error(msg)
 
@@ -1040,7 +1078,7 @@ class RunQueueData:
 
         # Create a whitelist usable by the stamp checks
         self.stampfnwhitelist = {}
-        for mc in self.taskData: 
+        for mc in self.taskData:
             self.stampfnwhitelist[mc] = []
             for entry in self.stampwhitelist.split():
                 if entry not in self.taskData[mc].build_targets:
@@ -1072,7 +1110,7 @@ class RunQueueData:
                     bb.debug(1, "Task %s is marked nostamp, cannot invalidate this task" % taskname)
             else:
                 logger.verbose("Invalidate task %s, %s", taskname, fn)
-                bb.parse.siggen.invalidate_task(taskname, self.dataCaches[mc], fn)
+                bb.parse.siggen.invalidate_task(taskname, self.dataCaches[mc], taskfn)
 
         self.init_progress_reporter.next_stage()
 
@@ -1716,6 +1754,10 @@ class RunQueueExecute:
         valid = bb.utils.better_eval(call, locs)
         return valid
 
+    def can_start_task(self):
+        can_start = self.stats.active < self.number_tasks
+        return can_start
+
 class RunQueueExecuteDummy(RunQueueExecute):
     def __init__(self, rq):
         self.rq = rq
@@ -1789,13 +1831,14 @@ class RunQueueExecuteTasks(RunQueueExecute):
             bb.build.del_stamp(taskname, self.rqdata.dataCaches[mc], taskfn)
             self.rq.scenequeue_covered.remove(tid)
 
-        toremove = covered_remove
+        toremove = covered_remove | self.rq.scenequeue_notcovered
         for task in toremove:
             logger.debug(1, 'Not skipping task %s due to setsceneverify', task)
         while toremove:
             covered_remove = []
             for task in toremove:
-                removecoveredtask(task)
+                if task in self.rq.scenequeue_covered:
+                    removecoveredtask(task)
                 for deptask in self.rqdata.runtaskentries[task].depends:
                     if deptask not in self.rq.scenequeue_covered:
                         continue
@@ -1865,14 +1908,13 @@ class RunQueueExecuteTasks(RunQueueExecute):
                 continue
             if revdep in self.runq_buildable:
                 continue
-            alldeps = 1
+            alldeps = True
             for dep in self.rqdata.runtaskentries[revdep].depends:
                 if dep not in self.runq_complete:
-                    alldeps = 0
-            if alldeps == 1:
+                    alldeps = False
+                    break
+            if alldeps:
                 self.setbuildable(revdep)
-                fn = fn_from_tid(revdep)
-                taskname = taskname_from_tid(revdep)
                 logger.debug(1, "Marking task %s as buildable", revdep)
 
     def task_complete(self, task):
@@ -1896,8 +1938,8 @@ class RunQueueExecuteTasks(RunQueueExecute):
         self.setbuildable(task)
         bb.event.fire(runQueueTaskSkipped(task, self.stats, self.rq, reason), self.cfgData)
         self.task_completeoutright(task)
-        self.stats.taskCompleted()
         self.stats.taskSkipped()
+        self.stats.taskCompleted()
 
     def execute(self):
         """
@@ -2007,7 +2049,7 @@ class RunQueueExecuteTasks(RunQueueExecute):
             self.build_stamps2.append(self.build_stamps[task])
             self.runq_running.add(task)
             self.stats.taskActive()
-            if self.stats.active < self.number_tasks:
+            if self.can_start_task():
                 return True
 
         if self.stats.active > 0:
@@ -2062,6 +2104,7 @@ class RunQueueExecuteScenequeue(RunQueueExecute):
         # If we don't have any setscene functions, skip this step
         if len(self.rqdata.runq_setscene_tids) == 0:
             rq.scenequeue_covered = set()
+            rq.scenequeue_notcovered = set()
             rq.state = runQueueRunInit
             return
 
@@ -2277,10 +2320,15 @@ class RunQueueExecuteScenequeue(RunQueueExecute):
                 sq_hash.append(self.rqdata.runtaskentries[tid].hash)
                 sq_taskname.append(taskname)
                 sq_task.append(tid)
+
+            self.cooker.data.setVar("BB_SETSCENE_STAMPCURRENT_COUNT", len(stamppresent))
+
             call = self.rq.hashvalidate + "(sq_fn, sq_task, sq_hash, sq_hashfn, d)"
             locs = { "sq_fn" : sq_fn, "sq_task" : sq_taskname, "sq_hash" : sq_hash, "sq_hashfn" : sq_hashfn, "d" : self.cooker.data }
             valid = bb.utils.better_eval(call, locs)
 
+            self.cooker.data.delVar("BB_SETSCENE_STAMPCURRENT_COUNT")
+
             valid_new = stamppresent
             for v in valid:
                 valid_new.append(sq_task[v])
@@ -2342,8 +2390,8 @@ class RunQueueExecuteScenequeue(RunQueueExecute):
     def task_failoutright(self, task):
         self.runq_running.add(task)
         self.runq_buildable.add(task)
-        self.stats.taskCompleted()
         self.stats.taskSkipped()
+        self.stats.taskCompleted()
         self.scenequeue_notcovered.add(task)
         self.scenequeue_updatecounters(task, True)
 
@@ -2351,8 +2399,8 @@ class RunQueueExecuteScenequeue(RunQueueExecute):
         self.runq_running.add(task)
         self.runq_buildable.add(task)
         self.task_completeoutright(task)
-        self.stats.taskCompleted()
         self.stats.taskSkipped()
+        self.stats.taskCompleted()
 
     def execute(self):
         """
@@ -2362,7 +2410,7 @@ class RunQueueExecuteScenequeue(RunQueueExecute):
         self.rq.read_workers()
 
         task = None
-        if self.stats.active < self.number_tasks:
+        if self.can_start_task():
             # Find the next setscene to run
             for nexttask in self.rqdata.runq_setscene_tids:
                 if nexttask in self.runq_buildable and nexttask not in self.runq_running and self.stamps[nexttask] not in self.build_stamps.values():
@@ -2421,7 +2469,7 @@ class RunQueueExecuteScenequeue(RunQueueExecute):
             self.build_stamps2.append(self.build_stamps[task])
             self.runq_running.add(task)
             self.stats.taskActive()
-            if self.stats.active < self.number_tasks:
+            if self.can_start_task():
                 return True
 
         if self.stats.active > 0:

bitbake/lib/bb/server/process.py

@@ -377,11 +377,12 @@ class BitBakeServer(object):
         if os.path.exists(sockname):
             os.unlink(sockname)
 
+        # Place the log in the builddirectory alongside the lock file
+        logfile = os.path.join(os.path.dirname(self.bitbake_lock.name), "bitbake-cookerdaemon.log")
+
         self.sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
         # AF_UNIX has path length issues so chdir here to workaround
         cwd = os.getcwd()
-        logfile = os.path.join(cwd, "bitbake-cookerdaemon.log")
-
         try:
             os.chdir(os.path.dirname(sockname))
             self.sock.bind(os.path.basename(sockname))
@@ -394,11 +395,16 @@ class BitBakeServer(object):
         bb.daemonize.createDaemon(self._startServer, logfile)
         self.sock.close()
         self.bitbake_lock.close()
+        os.close(self.readypipein)
 
         ready = ConnectionReader(self.readypipe)
         r = ready.poll(30)
         if r:
-            r = ready.get()
+            try:
+                r = ready.get()
+            except EOFError:
+                # Trap the child exitting/closing the pipe and error out
+                r = None
         if not r or r != "ready":
             ready.close()
             bb.error("Unable to start bitbake server")
@@ -424,21 +430,16 @@ class BitBakeServer(object):
                         bb.error("Server log for this session (%s):\n%s" % (logfile, "".join(lines)))
             raise SystemExit(1)
         ready.close()
-        os.close(self.readypipein)
 
     def _startServer(self):
         print(self.start_log_format % (os.getpid(), datetime.datetime.now().strftime(self.start_log_datetime_format)))
         server = ProcessServer(self.bitbake_lock, self.sock, self.sockname)
         self.configuration.setServerRegIdleCallback(server.register_idle_function)
+        os.close(self.readypipe)
         writer = ConnectionWriter(self.readypipein)
-        try:
-            self.cooker = bb.cooker.BBCooker(self.configuration, self.featureset)
-            writer.send("ready")
-        except:
-            writer.send("fail")
-            raise
-        finally:
-            os.close(self.readypipein)
+        self.cooker = bb.cooker.BBCooker(self.configuration, self.featureset)
+        writer.send("ready")
+        writer.close()
         server.cooker = self.cooker
         server.server_timeout = self.configuration.server_timeout
         server.xmlrpcinterface = self.configuration.xmlrpcinterface

bitbake/lib/bb/siggen.py

@@ -193,15 +193,24 @@ class SignatureGeneratorBasic(SignatureGenerator):
         return taint
 
     def get_taskhash(self, fn, task, deps, dataCache):
+
+        mc = ''
+        if fn.startswith('multiconfig:'):
+            mc = fn.split(':')[1]
         k = fn + "." + task
+
         data = dataCache.basetaskhash[k]
         self.basehash[k] = data
         self.runtaskdeps[k] = []
         self.file_checksum_values[k] = []
         recipename = dataCache.pkg_fn[fn]
-
         for dep in sorted(deps, key=clean_basepath):
-            depname = dataCache.pkg_fn[self.pkgnameextract.search(dep).group('fn')]
+            pkgname = self.pkgnameextract.search(dep).group('fn')
+            if mc:
+                depmc = pkgname.split(':')[1]
+                if mc != depmc:
+                    continue
+            depname = dataCache.pkg_fn[pkgname]
             if not self.rundep_check(fn, recipename, task, dep, depname, dataCache):
                 continue
             if dep not in self.taskhash:
@@ -347,7 +356,7 @@ class SignatureGeneratorBasicHash(SignatureGeneratorBasic):
 
     def stampcleanmask(self, stampbase, fn, taskname, extrainfo):
         return self.stampfile(stampbase, fn, taskname, extrainfo, clean=True)
-        
+
     def invalidate_task(self, task, d, fn):
         bb.note("Tainting hash to force rebuild of task %s, %s" % (fn, task))
         bb.build.write_taint(task, d, fn)
@@ -636,7 +645,7 @@ def compare_sigfiles(a, b, recursecb=None, color=False, collapsed=False):
                         if collapsed:
                             output.extend(recout)
                         else:
-                            # If a dependent hash changed, might as well print the line above and then defer to the changes in 
+                            # If a dependent hash changed, might as well print the line above and then defer to the changes in
                             # that hash since in all likelyhood, they're the same changes this task also saw.
                             output = [output[-1]] + recout
 

bitbake/lib/bb/taskdata.py

@@ -70,6 +70,8 @@ class TaskData:
 
         self.skiplist = skiplist
 
+        self.mcdepends = []
+
     def add_tasks(self, fn, dataCache):
         """
         Add tasks for a given fn to the database
@@ -88,6 +90,13 @@ class TaskData:
 
         self.add_extra_deps(fn, dataCache)
 
+        def add_mcdepends(task):
+            for dep in task_deps['mcdepends'][task].split():
+                if len(dep.split(':')) != 5:
+                    bb.msg.fatal("TaskData", "Error for %s:%s[%s], multiconfig dependency %s does not contain exactly four  ':' characters.\n Task '%s' should be specified in the form 'multiconfig:fromMC:toMC:packagename:task'" % (fn, task, 'mcdepends', dep, 'mcdepends'))
+                if dep not in self.mcdepends:
+                    self.mcdepends.append(dep)
+
         # Common code for dep_name/depends = 'depends'/idepends and 'rdepends'/irdepends
         def handle_deps(task, dep_name, depends, seen):
             if dep_name in task_deps and task in task_deps[dep_name]:
@@ -110,16 +119,20 @@ class TaskData:
             parentids = []
             for dep in task_deps['parents'][task]:
                 if dep not in task_deps['tasks']:
-                    bb.debug(2, "Not adding dependeny of %s on %s since %s does not exist" % (task, dep, dep))
+                    bb.debug(2, "Not adding dependency of %s on %s since %s does not exist" % (task, dep, dep))
                     continue
                 parentid = "%s:%s" % (fn, dep)
                 parentids.append(parentid)
             self.taskentries[tid].tdepends.extend(parentids)
 
+
             # Touch all intertask dependencies
             handle_deps(task, 'depends', self.taskentries[tid].idepends, self.seen_build_target)
             handle_deps(task, 'rdepends', self.taskentries[tid].irdepends, self.seen_run_target)
 
+            if 'mcdepends' in task_deps and task in task_deps['mcdepends']:
+                add_mcdepends(task)
+
         # Work out build dependencies
         if not fn in self.depids:
             dependids = set()
@@ -537,6 +550,9 @@ class TaskData:
                     provmap[name] = provider[0]
         return provmap
 
+    def get_mcdepends(self):
+        return self.mcdepends
+
     def dump_data(self):
         """
         Dump some debug information on the internal data structures

bitbake/lib/bb/tests/cooker.py

@@ -0,0 +1,83 @@
+# ex:ts=4:sw=4:sts=4:et
+# -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*-
+#
+# BitBake Tests for cooker.py
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License version 2 as
+# published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License along
+# with this program; if not, write to the Free Software Foundation, Inc.,
+# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+#
+
+import unittest
+import tempfile
+import os
+import bb, bb.cooker
+import re
+import logging
+
+# Cooker tests
+class CookerTest(unittest.TestCase):
+    def setUp(self):
+        # At least one variable needs to be set
+        self.d = bb.data.init()
+        topdir = os.path.join(os.path.dirname(os.path.realpath(__file__)), "testdata/cooker")
+        self.d.setVar('TOPDIR', topdir)
+
+    def test_CookerCollectFiles_sublayers(self):
+        '''Test that a sublayer of an existing layer does not trigger
+           No bb files matched ...'''
+
+        def append_collection(topdir, path, d):
+            collection = path.split('/')[-1]
+            pattern = "^" + topdir + "/" + path + "/"
+            regex = re.compile(pattern)
+            priority = 5
+
+            d.setVar('BBFILE_COLLECTIONS', (d.getVar('BBFILE_COLLECTIONS') or "") + " " + collection)
+            d.setVar('BBFILE_PATTERN_%s' % (collection), pattern)
+            d.setVar('BBFILE_PRIORITY_%s' % (collection), priority)
+
+            return (collection, pattern, regex, priority)
+
+        topdir = self.d.getVar("TOPDIR")
+
+        # Priorities: list of (collection, pattern, regex, priority)
+        bbfile_config_priorities = []
+        # Order is important for this test, shortest to longest is typical failure case
+        bbfile_config_priorities.append( append_collection(topdir, 'first', self.d) )
+        bbfile_config_priorities.append( append_collection(topdir, 'second', self.d) )
+        bbfile_config_priorities.append( append_collection(topdir, 'second/third', self.d) )
+
+        pkgfns = [ topdir + '/first/recipes/sample1_1.0.bb',
+                   topdir + '/second/recipes/sample2_1.0.bb',
+                   topdir + '/second/third/recipes/sample3_1.0.bb' ]
+
+        class LogHandler(logging.Handler):
+            def __init__(self):
+                logging.Handler.__init__(self)
+                self.logdata = []
+
+            def emit(self, record):
+                self.logdata.append(record.getMessage())
+
+        # Move cooker to use my special logging
+        logger = bb.cooker.logger
+        log_handler = LogHandler()
+        logger.addHandler(log_handler)
+        collection = bb.cooker.CookerCollectFiles(bbfile_config_priorities)
+        collection.collection_priorities(pkgfns, self.d)
+        logger.removeHandler(log_handler)
+
+        # Should be empty (no generated messages)
+        expected = []
+
+        self.assertEqual(log_handler.logdata, expected)

bitbake/lib/bb/tests/fetch.py

@@ -406,6 +406,8 @@ class MirrorUriTest(FetcherTest):
             : "http://somewhere2.org/somefile_1.2.3.tar.gz",
         ("http://somewhere.org/somedir1/somedir2/somefile_1.2.3.tar.gz", "http://.*/.*", "http://somewhere2.org/")
             : "http://somewhere2.org/somefile_1.2.3.tar.gz",
+        ("git://someserver.org/bitbake;tag=1234567890123456789012345678901234567890;branch=master", "git://someserver.org/bitbake;branch=master", "git://git.openembedded.org/bitbake;protocol=http")
+            : "git://git.openembedded.org/bitbake;tag=1234567890123456789012345678901234567890;branch=master;protocol=http",
 
         #Renaming files doesn't work
         #("http://somewhere.org/somedir1/somefile_1.2.3.tar.gz", "http://somewhere.org/somedir1/somefile_1.2.3.tar.gz", "http://somewhere2.org/somedir3/somefile_2.3.4.tar.gz") : "http://somewhere2.org/somedir3/somefile_2.3.4.tar.gz"
@@ -865,12 +867,12 @@ class FetchLatestVersionTest(FetcherTest):
         ("dtc", "git://git.qemu.org/dtc.git", "65cc4d2748a2c2e6f27f1cf39e07a5dbabd80ebf", "")
             : "1.4.0",
         # combination version pattern
-        ("sysprof", "git://git.gnome.org/sysprof", "cd44ee6644c3641507fb53b8a2a69137f2971219", "")
+        ("sysprof", "git://gitlab.gnome.org/GNOME/sysprof;protocol=https", "cd44ee6644c3641507fb53b8a2a69137f2971219", "")
             : "1.2.0",
         ("u-boot-mkimage", "git://git.denx.de/u-boot.git;branch=master;protocol=git", "62c175fbb8a0f9a926c88294ea9f7e88eb898f6c", "")
             : "2014.01",
         # version pattern "yyyymmdd"
-        ("mobile-broadband-provider-info", "git://git.gnome.org/mobile-broadband-provider-info", "4ed19e11c2975105b71b956440acdb25d46a347d", "")
+        ("mobile-broadband-provider-info", "git://gitlab.gnome.org/GNOME/mobile-broadband-provider-info;protocol=https", "4ed19e11c2975105b71b956440acdb25d46a347d", "")
             : "20120614",
         # packages with a valid UPSTREAM_CHECK_GITTAGREGEX
         ("xf86-video-omap", "git://anongit.freedesktop.org/xorg/driver/xf86-video-omap", "ae0394e687f1a77e966cf72f895da91840dffb8f", "(?P<pver>(\d+\.(\d\.?)*))")

bitbake/lib/bb/utils.py

@@ -524,12 +524,17 @@ def md5_file(filename):
     """
     Return the hex string representation of the MD5 checksum of filename.
     """
-    import hashlib
-    m = hashlib.md5()
+    import hashlib, mmap
 
     with open(filename, "rb") as f:
-        for line in f:
-            m.update(line)
+        m = hashlib.md5()
+        try:
+            with mmap.mmap(f.fileno(), 0, access=mmap.ACCESS_READ) as mm:
+                for chunk in iter(lambda: mm.read(8192), b''):
+                    m.update(chunk)
+        except ValueError:
+            # You can't mmap() an empty file so silence this exception
+            pass
     return m.hexdigest()
 
 def sha256_file(filename):
@@ -901,6 +906,23 @@ def copyfile(src, dest, newmtime = None, sstat = None):
         newmtime = sstat[stat.ST_MTIME]
     return newmtime
 
+def break_hardlinks(src, sstat = None):
+    """
+    Ensures src is the only hardlink to this file.  Other hardlinks,
+    if any, are not affected (other than in their st_nlink value, of
+    course).  Returns true on success and false on failure.
+
+    """
+    try:
+        if not sstat:
+            sstat = os.lstat(src)
+    except Exception as e:
+        logger.warning("break_hardlinks: stat of %s failed (%s)" % (src, e))
+        return False
+    if sstat[stat.ST_NLINK] == 1:
+        return True
+    return copyfile(src, src, sstat=sstat)
+
 def which(path, item, direction = 0, history = False, executable=False):
     """
     Locate `item` in the list of paths `path` (colon separated string like $PATH).
@@ -1285,7 +1307,7 @@ def edit_metadata_file(meta_file, variables, varfunc):
     return updated
 
 
-def edit_bblayers_conf(bblayers_conf, add, remove):
+def edit_bblayers_conf(bblayers_conf, add, remove, edit_cb=None):
     """Edit bblayers.conf, adding and/or removing layers
     Parameters:
         bblayers_conf: path to bblayers.conf file to edit
@@ -1293,6 +1315,8 @@ def edit_bblayers_conf(bblayers_conf, add, remove):
             list to add nothing
         remove: layer path (or list of layer paths) to remove; None or
             empty list to remove nothing
+        edit_cb: optional callback function that will be called after
+            processing adds/removes once per existing entry.
     Returns a tuple:
         notadded: list of layers specified to be added but weren't
             (because they were already in the list)
@@ -1356,6 +1380,17 @@ def edit_bblayers_conf(bblayers_conf, add, remove):
                     bblayers.append(addlayer)
             del addlayers[:]
 
+        if edit_cb:
+            newlist = []
+            for layer in bblayers:
+                res = edit_cb(layer, canonicalise_path(layer))
+                if res != layer:
+                    newlist.append(res)
+                    updated = True
+                else:
+                    newlist.append(layer)
+            bblayers = newlist
+
         if updated:
             if op == '+=' and not bblayers:
                 bblayers = None

bitbake/lib/bblayers/action.py

@@ -45,7 +45,7 @@ class ActionPlugin(LayerPlugin):
             notadded, _ = bb.utils.edit_bblayers_conf(bblayers_conf, layerdirs, None)
             if not (args.force or notadded):
                 try:
-                    self.tinfoil.parseRecipes()
+                    self.tinfoil.run_command('parseConfiguration')
                 except bb.tinfoil.TinfoilUIException:
                     # Restore the back up copy of bblayers.conf
                     shutil.copy2(backup, bblayers_conf)

bitbake/lib/bblayers/layerindex.py

@@ -1,10 +1,9 @@
+import layerindexlib
+
 import argparse
-import http.client
-import json
 import logging
 import os
 import subprocess
-import urllib.parse
 
 from bblayers.action import ActionPlugin
 
@@ -21,110 +20,6 @@ class LayerIndexPlugin(ActionPlugin):
     This class inherits ActionPlugin to get do_add_layer.
     """
 
-    def get_json_data(self, apiurl):
-        proxy_settings = os.environ.get("http_proxy", None)
-        conn = None
-        _parsedurl = urllib.parse.urlparse(apiurl)
-        path = _parsedurl.path
-        query = _parsedurl.query
-
-        def parse_url(url):
-            parsedurl = urllib.parse.urlparse(url)
-            if parsedurl.netloc[0] == '[':
-                host, port = parsedurl.netloc[1:].split(']', 1)
-                if ':' in port:
-                    port = port.rsplit(':', 1)[1]
-                else:
-                    port = None
-            else:
-                if parsedurl.netloc.count(':') == 1:
-                    (host, port) = parsedurl.netloc.split(":")
-                else:
-                    host = parsedurl.netloc
-                    port = None
-            return (host, 80 if port is None else int(port))
-
-        if proxy_settings is None:
-            host, port = parse_url(apiurl)
-            conn = http.client.HTTPConnection(host, port)
-            conn.request("GET", path + "?" + query)
-        else:
-            host, port = parse_url(proxy_settings)
-            conn = http.client.HTTPConnection(host, port)
-            conn.request("GET", apiurl)
-
-        r = conn.getresponse()
-        if r.status != 200:
-            raise Exception("Failed to read " + path + ": %d %s" % (r.status, r.reason))
-        return json.loads(r.read().decode())
-
-    def get_layer_deps(self, layername, layeritems, layerbranches, layerdependencies, branchnum, selfname=False):
-        def layeritems_info_id(items_name, layeritems):
-            litems_id = None
-            for li in layeritems:
-                if li['name'] == items_name:
-                    litems_id = li['id']
-                    break
-            return litems_id
-
-        def layerbranches_info(items_id, layerbranches):
-            lbranch = {}
-            for lb in layerbranches:
-                if lb['layer'] == items_id and lb['branch'] == branchnum:
-                    lbranch['id'] = lb['id']
-                    lbranch['vcs_subdir'] = lb['vcs_subdir']
-                    break
-            return lbranch
-
-        def layerdependencies_info(lb_id, layerdependencies):
-            ld_deps = []
-            for ld in layerdependencies:
-                if ld['layerbranch'] == lb_id and not ld['dependency'] in ld_deps:
-                    ld_deps.append(ld['dependency'])
-            if not ld_deps:
-                logger.error("The dependency of layerDependencies is not found.")
-            return ld_deps
-
-        def layeritems_info_name_subdir(items_id, layeritems):
-            litems = {}
-            for li in layeritems:
-                if li['id'] == items_id:
-                    litems['vcs_url'] = li['vcs_url']
-                    litems['name'] = li['name']
-                    break
-            return litems
-
-        if selfname:
-            selfid = layeritems_info_id(layername, layeritems)
-            lbinfo = layerbranches_info(selfid, layerbranches)
-            if lbinfo:
-                selfsubdir = lbinfo['vcs_subdir']
-            else:
-                logger.error("%s is not found in the specified branch" % layername)
-                return
-            selfurl = layeritems_info_name_subdir(selfid, layeritems)['vcs_url']
-            if selfurl:
-                return selfurl, selfsubdir
-            else:
-                logger.error("Cannot get layer %s git repo and subdir" % layername)
-                return
-        ldict = {}
-        itemsid = layeritems_info_id(layername, layeritems)
-        if not itemsid:
-            return layername, None
-        lbid = layerbranches_info(itemsid, layerbranches)
-        if lbid:
-            lbid = layerbranches_info(itemsid, layerbranches)['id']
-        else:
-            logger.error("%s is not found in the specified branch" % layername)
-            return None, None
-        for dependency in layerdependencies_info(lbid, layerdependencies):
-            lname = layeritems_info_name_subdir(dependency, layeritems)['name']
-            lurl = layeritems_info_name_subdir(dependency, layeritems)['vcs_url']
-            lsubdir = layerbranches_info(dependency, layerbranches)['vcs_subdir']
-            ldict[lname] = lurl, lsubdir
-        return None, ldict
-
     def get_fetch_layer(self, fetchdir, url, subdir, fetch_layer):
         layername = self.get_layer_name(url)
         if os.path.splitext(layername)[1] == '.git':
@@ -136,95 +31,124 @@ class LayerIndexPlugin(ActionPlugin):
                 result = subprocess.call('git clone %s %s' % (url, repodir), shell = True)
                 if result:
                     logger.error("Failed to download %s" % url)
-                    return None, None
+                    return None, None, None
                 else:
-                    return layername, layerdir
+                    return subdir, layername, layerdir
             else:
                 logger.plain("Repository %s needs to be fetched" % url)
-                return layername, layerdir
+                return subdir, layername, layerdir
         elif os.path.exists(layerdir):
-            return layername, layerdir
+            return subdir, layername, layerdir
         else:
             logger.error("%s is not in %s" % (url, subdir))
-        return None, None
+        return None, None, None
 
     def do_layerindex_fetch(self, args):
         """Fetches a layer from a layer index along with its dependent layers, and adds them to conf/bblayers.conf.
 """
-        apiurl = self.tinfoil.config_data.getVar('BBLAYERS_LAYERINDEX_URL')
-        if not apiurl:
-            logger.error("Cannot get BBLAYERS_LAYERINDEX_URL")
-            return 1
-        else:
-            if apiurl[-1] != '/':
-                apiurl += '/'
-            apiurl += "api/"
-        apilinks = self.get_json_data(apiurl)
-        branches = self.get_json_data(apilinks['branches'])
 
-        branchnum = 0
-        for branch in branches:
-            if branch['name'] == args.branch:
-                branchnum = branch['id']
-                break
-        if branchnum == 0:
-            validbranches = ', '.join([branch['name'] for branch in branches])
-            logger.error('Invalid layer branch name "%s". Valid branches: %s' % (args.branch, validbranches))
-            return 1
+        def _construct_url(baseurls, branches):
+            urls = []
+            for baseurl in baseurls:
+                if baseurl[-1] != '/':
+                    baseurl += '/'
+
+                if not baseurl.startswith('cooker'):
+                    baseurl += "api/"
+
+                if branches:
+                    baseurl += ";branch=%s" % ','.join(branches)
+
+                urls.append(baseurl)
+
+            return urls
+
+
+        # Set the default...
+        if args.branch:
+            branches = [args.branch]
+        else:
+            branches = (self.tinfoil.config_data.getVar('LAYERSERIES_CORENAMES') or 'master').split()
+        logger.debug(1, 'Trying branches: %s' % branches)
 
         ignore_layers = []
-        for collection in self.tinfoil.config_data.getVar('BBFILE_COLLECTIONS').split():
-            lname = self.tinfoil.config_data.getVar('BBLAYERS_LAYERINDEX_NAME_%s' % collection)
-            if lname:
-                ignore_layers.append(lname)
-
         if args.ignore:
             ignore_layers.extend(args.ignore.split(','))
 
-        layeritems = self.get_json_data(apilinks['layerItems'])
-        layerbranches = self.get_json_data(apilinks['layerBranches'])
-        layerdependencies = self.get_json_data(apilinks['layerDependencies'])
-        invaluenames = []
-        repourls = {}
-        printlayers = []
+        # Load the cooker DB
+        cookerIndex = layerindexlib.LayerIndex(self.tinfoil.config_data)
+        cookerIndex.load_layerindex('cooker://', load='layerDependencies')
 
-        def query_dependencies(layers, layeritems, layerbranches, layerdependencies, branchnum):
-            depslayer = []
-            for layername in layers:
-                invaluename, layerdict = self.get_layer_deps(layername, layeritems, layerbranches, layerdependencies, branchnum)
-                if layerdict:
-                    repourls[layername] = self.get_layer_deps(layername, layeritems, layerbranches, layerdependencies, branchnum, selfname=True)
-                    for layer in layerdict:
-                        if not layer in ignore_layers:
-                            depslayer.append(layer)
-                        printlayers.append((layername, layer, layerdict[layer][0], layerdict[layer][1]))
-                        if not layer in ignore_layers and not layer in repourls:
-                            repourls[layer] = (layerdict[layer][0], layerdict[layer][1])
-                if invaluename and not invaluename in invaluenames:
-                    invaluenames.append(invaluename)
-            return depslayer
+        # Fast path, check if we already have what has been requested!
+        (dependencies, invalidnames) = cookerIndex.find_dependencies(names=args.layername, ignores=ignore_layers)
+        if not args.show_only and not invalidnames:
+            logger.plain("You already have the requested layer(s): %s" % args.layername)
+            return 0
 
-        depslayers = query_dependencies(args.layername, layeritems, layerbranches, layerdependencies, branchnum)
-        while depslayers:
-            depslayer = query_dependencies(depslayers, layeritems, layerbranches, layerdependencies, branchnum)
-            depslayers = depslayer
-        if invaluenames:
-            for invaluename in invaluenames:
-                logger.error('Layer "%s" not found in layer index' % invaluename)
-            return 1
-        logger.plain("%s  %s  %s  %s" % ("Layer".ljust(19), "Required by".ljust(19), "Git repository".ljust(54), "Subdirectory"))
-        logger.plain('=' * 115)
-        for layername in args.layername:
-            layerurl = repourls[layername]
-            logger.plain("%s %s %s %s" % (layername.ljust(20), '-'.ljust(20), layerurl[0].ljust(55), layerurl[1]))
-        printedlayers = []
-        for layer, dependency, gitrepo, subdirectory in printlayers:
-            if dependency in printedlayers:
-                continue
-            logger.plain("%s %s %s %s" % (dependency.ljust(20), layer.ljust(20), gitrepo.ljust(55), subdirectory))
-            printedlayers.append(dependency)
+        # The information to show is already in the cookerIndex
+        if invalidnames:
+            # General URL to use to access the layer index
+            # While there is ONE right now, we're expect users could enter several
+            apiurl = self.tinfoil.config_data.getVar('BBLAYERS_LAYERINDEX_URL').split()
+            if not apiurl:
+                logger.error("Cannot get BBLAYERS_LAYERINDEX_URL")
+                return 1
 
-        if repourls:
+            remoteIndex = layerindexlib.LayerIndex(self.tinfoil.config_data)
+
+            for remoteurl in _construct_url(apiurl, branches):
+                logger.plain("Loading %s..." % remoteurl)
+                remoteIndex.load_layerindex(remoteurl)
+
+            if remoteIndex.is_empty():
+                logger.error("Remote layer index %s is empty for branches %s" % (apiurl, branches))
+                return 1
+
+            lIndex = cookerIndex + remoteIndex
+
+            (dependencies, invalidnames) = lIndex.find_dependencies(names=args.layername, ignores=ignore_layers)
+
+            if invalidnames:
+                for invaluename in invalidnames:
+                    logger.error('Layer "%s" not found in layer index' % invaluename)
+                return 1
+
+        logger.plain("%s  %s  %s" % ("Layer".ljust(49), "Git repository (branch)".ljust(54), "Subdirectory"))
+        logger.plain('=' * 125)
+
+        for deplayerbranch in dependencies:
+            layerBranch = dependencies[deplayerbranch][0]
+
+            # TODO: Determine display behavior
+            # This is the local content, uncomment to hide local
+            # layers from the display.
+            #if layerBranch.index.config['TYPE'] == 'cooker':
+            #    continue
+
+            layerDeps = dependencies[deplayerbranch][1:]
+
+            requiredby = []
+            recommendedby = []
+            for dep in layerDeps:
+                if dep.required:
+                    requiredby.append(dep.layer.name)
+                else:
+                    recommendedby.append(dep.layer.name)
+
+            logger.plain('%s %s %s' % (("%s:%s:%s" %
+                                  (layerBranch.index.config['DESCRIPTION'],
+                                  layerBranch.branch.name,
+                                  layerBranch.layer.name)).ljust(50),
+                                  ("%s (%s)" % (layerBranch.layer.vcs_url,
+                                  layerBranch.actual_branch)).ljust(55),
+                                  layerBranch.vcs_subdir
+                                               ))
+            if requiredby:
+                logger.plain('  required by: %s' % ' '.join(requiredby))
+            if recommendedby:
+                logger.plain('  recommended by: %s' % ' '.join(recommendedby))
+
+        if dependencies:
             fetchdir = self.tinfoil.config_data.getVar('BBLAYERS_FETCH_DIR')
             if not fetchdir:
                 logger.error("Cannot get BBLAYERS_FETCH_DIR")
@@ -232,26 +156,39 @@ class LayerIndexPlugin(ActionPlugin):
             if not os.path.exists(fetchdir):
                 os.makedirs(fetchdir)
             addlayers = []
-            for repourl, subdir in repourls.values():
-                name, layerdir = self.get_fetch_layer(fetchdir, repourl, subdir, not args.show_only)
+
+            for deplayerbranch in dependencies:
+                layerBranch = dependencies[deplayerbranch][0]
+
+                if layerBranch.index.config['TYPE'] == 'cooker':
+                    # Anything loaded via cooker is already local, skip it
+                    continue
+
+                subdir, name, layerdir = self.get_fetch_layer(fetchdir,
+                                                      layerBranch.layer.vcs_url,
+                                                      layerBranch.vcs_subdir,
+                                                      not args.show_only)
                 if not name:
                     # Error already shown
                     return 1
                 addlayers.append((subdir, name, layerdir))
         if not args.show_only:
-            for subdir, name, layerdir in set(addlayers):
+            localargs = argparse.Namespace()
+            localargs.layerdir = []
+            localargs.force = args.force
+            for subdir, name, layerdir in addlayers:
                 if os.path.exists(layerdir):
                     if subdir:
-                        logger.plain("Adding layer \"%s\" to conf/bblayers.conf" % subdir)
+                        logger.plain("Adding layer \"%s\" (%s) to conf/bblayers.conf" % (subdir, layerdir))
                     else:
-                        logger.plain("Adding layer \"%s\" to conf/bblayers.conf" % name)
-                    localargs = argparse.Namespace()
-                    localargs.layerdir = layerdir
-                    localargs.force = args.force
-                    self.do_add_layer(localargs)
+                        logger.plain("Adding layer \"%s\" (%s) to conf/bblayers.conf" % (name, layerdir))
+                    localargs.layerdir.append(layerdir)
                 else:
                     break
 
+            if localargs.layerdir:
+                self.do_add_layer(localargs)
+
     def do_layerindex_show_depends(self, args):
         """Find layer dependencies from layer index.
 """
@@ -260,12 +197,12 @@ class LayerIndexPlugin(ActionPlugin):
         self.do_layerindex_fetch(args)
 
     def register_commands(self, sp):
-        parser_layerindex_fetch = self.add_command(sp, 'layerindex-fetch', self.do_layerindex_fetch)
+        parser_layerindex_fetch = self.add_command(sp, 'layerindex-fetch', self.do_layerindex_fetch, parserecipes=False)
         parser_layerindex_fetch.add_argument('-n', '--show-only', help='show dependencies and do nothing else', action='store_true')
-        parser_layerindex_fetch.add_argument('-b', '--branch', help='branch name to fetch (default %(default)s)', default='master')
+        parser_layerindex_fetch.add_argument('-b', '--branch', help='branch name to fetch')
         parser_layerindex_fetch.add_argument('-i', '--ignore', help='assume the specified layers do not need to be fetched/added (separate multiple layers with commas, no spaces)', metavar='LAYER')
         parser_layerindex_fetch.add_argument('layername', nargs='+', help='layer to fetch')
 
-        parser_layerindex_show_depends = self.add_command(sp, 'layerindex-show-depends', self.do_layerindex_show_depends)
-        parser_layerindex_show_depends.add_argument('-b', '--branch', help='branch name to fetch (default %(default)s)', default='master')
+        parser_layerindex_show_depends = self.add_command(sp, 'layerindex-show-depends', self.do_layerindex_show_depends, parserecipes=False)
+        parser_layerindex_show_depends.add_argument('-b', '--branch', help='branch name to fetch')
         parser_layerindex_show_depends.add_argument('layername', nargs='+', help='layer to query')

bitbake/lib/layerindexlib/README

@@ -0,0 +1,28 @@
+The layerindexlib module is designed to permit programs to work directly
+with layer index information.  (See layers.openembedded.org...)
+
+The layerindexlib module includes a plugin interface that is used to extend
+the basic functionality.  There are two primary plugins available: restapi
+and cooker.
+
+The restapi plugin works with a web based REST Api compatible with the
+layerindex-web project, as well as the ability to store and retried a
+the information for one or more files on the disk.
+
+The cooker plugin works by reading the information from the current build
+project and processing it as if it were a layer index.
+
+
+TODO:
+
+__init__.py:
+Implement local on-disk caching (using the rest api store/load)
+Implement layer index style query operations on a combined index
+
+common.py:
+Stop network access if BB_NO_NETWORK or allowed hosts is restricted
+
+cooker.py:
+Cooker - Implement recipe parsing
+
+

bitbake/lib/layerindexlib/cooker.py

@@ -0,0 +1,341 @@
+# Copyright (C) 2016-2018 Wind River Systems, Inc.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License version 2 as
+# published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+# See the GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+
+import logging
+import json
+
+from collections import OrderedDict, defaultdict
+
+from urllib.parse import unquote, urlparse
+
+import layerindexlib
+
+import layerindexlib.plugin
+
+logger = logging.getLogger('BitBake.layerindexlib.cooker')
+
+import bb.utils
+
+def plugin_init(plugins):
+    return CookerPlugin()
+
+class CookerPlugin(layerindexlib.plugin.IndexPlugin):
+    def __init__(self):
+        self.type = "cooker"
+
+        self.server_connection = None
+        self.ui_module = None
+        self.server = None
+
+    def _run_command(self, command, path, default=None):
+        try:
+            result, _ = bb.process.run(command, cwd=path)
+            result = result.strip()
+        except bb.process.ExecutionError:
+            result = default
+        return result
+
+    def _handle_git_remote(self, remote):
+        if "://" not in remote:
+            if ':' in remote:
+                # This is assumed to be ssh
+                remote = "ssh://" + remote
+            else:
+                # This is assumed to be a file path
+                remote = "file://" + remote
+        return remote
+
+    def _get_bitbake_info(self):
+        """Return a tuple of bitbake information"""
+
+        # Our path SHOULD be .../bitbake/lib/layerindex/cooker.py
+        bb_path = os.path.dirname(__file__) # .../bitbake/lib/layerindex/cooker.py
+        bb_path = os.path.dirname(bb_path)  # .../bitbake/lib/layerindex
+        bb_path = os.path.dirname(bb_path)  # .../bitbake/lib
+        bb_path = os.path.dirname(bb_path)  # .../bitbake
+        bb_path = self._run_command('git rev-parse --show-toplevel', os.path.dirname(__file__), default=bb_path)
+        bb_branch = self._run_command('git rev-parse --abbrev-ref HEAD', bb_path, default="<unknown>")
+        bb_rev = self._run_command('git rev-parse HEAD', bb_path, default="<unknown>")
+        for remotes in self._run_command('git remote -v', bb_path, default="").split("\n"):
+            remote = remotes.split("\t")[1].split(" ")[0]
+            if "(fetch)" == remotes.split("\t")[1].split(" ")[1]:
+                bb_remote = self._handle_git_remote(remote)
+                break
+        else:
+            bb_remote = self._handle_git_remote(bb_path)
+
+        return (bb_remote, bb_branch, bb_rev, bb_path)
+
+    def _load_bblayers(self, branches=None):
+        """Load the BBLAYERS and related collection information"""
+
+        d = self.layerindex.data
+
+        if not branches:
+            raise LayerIndexFetchError("No branches specified for _load_bblayers!")
+
+        index = layerindexlib.LayerIndexObj()
+
+        branchId = 0
+        index.branches = {}
+
+        layerItemId = 0
+        index.layerItems = {}
+
+        layerBranchId = 0
+        index.layerBranches = {}
+
+        bblayers = d.getVar('BBLAYERS').split()
+
+        if not bblayers:
+            # It's blank!  Nothing to process...
+            return index
+
+        collections = d.getVar('BBFILE_COLLECTIONS')
+        layerconfs = d.varhistory.get_variable_items_files('BBFILE_COLLECTIONS', d)
+        bbfile_collections = {layer: os.path.dirname(os.path.dirname(path)) for layer, path in layerconfs.items()}
+
+        (_, bb_branch, _, _) = self._get_bitbake_info()
+
+        for branch in branches:
+            branchId += 1
+            index.branches[branchId] = layerindexlib.Branch(index, None)
+            index.branches[branchId].define_data(branchId, branch, bb_branch)
+
+        for entry in collections.split():
+            layerpath = entry
+            if entry in bbfile_collections:
+                layerpath = bbfile_collections[entry]
+
+            layername = d.getVar('BBLAYERS_LAYERINDEX_NAME_%s' % entry) or os.path.basename(layerpath)
+            layerversion = d.getVar('LAYERVERSION_%s' % entry) or ""
+            layerurl = self._handle_git_remote(layerpath)
+
+            layersubdir = ""
+            layerrev = "<unknown>"
+            layerbranch = "<unknown>"
+
+            if os.path.isdir(layerpath):
+                layerbasepath = self._run_command('git rev-parse --show-toplevel', layerpath, default=layerpath)
+                if os.path.abspath(layerpath) != os.path.abspath(layerbasepath):
+                    layersubdir = os.path.abspath(layerpath)[len(layerbasepath) + 1:]
+
+                layerbranch = self._run_command('git rev-parse --abbrev-ref HEAD', layerpath, default="<unknown>")
+                layerrev = self._run_command('git rev-parse HEAD', layerpath, default="<unknown>")
+
+                for remotes in self._run_command('git remote -v', layerpath, default="").split("\n"):
+                    remote = remotes.split("\t")[1].split(" ")[0]
+                    if "(fetch)" == remotes.split("\t")[1].split(" ")[1]:
+                        layerurl = self._handle_git_remote(remote)
+                        break
+
+            layerItemId += 1
+            index.layerItems[layerItemId] = layerindexlib.LayerItem(index, None)
+            index.layerItems[layerItemId].define_data(layerItemId, layername, description=layerpath, vcs_url=layerurl)
+
+            for branchId in index.branches:
+                layerBranchId += 1
+                index.layerBranches[layerBranchId] = layerindexlib.LayerBranch(index, None)
+                index.layerBranches[layerBranchId].define_data(layerBranchId, entry, layerversion, layerItemId, branchId,
+                                               vcs_subdir=layersubdir, vcs_last_rev=layerrev, actual_branch=layerbranch)
+
+        return index
+
+
+    def load_index(self, url, load):
+        """
+            Fetches layer information from a build configuration.
+
+            The return value is a dictionary containing API,
+            layer, branch, dependency, recipe, machine, distro, information.
+
+            url type should be 'cooker'.
+            url path is ignored
+        """
+
+        up = urlparse(url)
+
+        if up.scheme != 'cooker':
+            raise layerindexlib.plugin.LayerIndexPluginUrlError(self.type, url)
+
+        d = self.layerindex.data
+
+        params = self.layerindex._parse_params(up.params)
+
+        # Only reason to pass a branch is to emulate them...
+        if 'branch' in params:
+            branches = params['branch'].split(',')
+        else:
+            branches = ['HEAD']
+
+        logger.debug(1, "Loading cooker data branches %s" % branches)
+
+        index = self._load_bblayers(branches=branches)
+
+        index.config = {}
+        index.config['TYPE'] = self.type
+        index.config['URL'] = url
+
+        if 'desc' in params:
+            index.config['DESCRIPTION'] = unquote(params['desc'])
+        else:
+            index.config['DESCRIPTION'] = 'local'
+
+        if 'cache' in params:
+            index.config['CACHE'] = params['cache']
+
+        index.config['BRANCH'] = branches
+
+        # ("layerDependencies", layerindexlib.LayerDependency)
+        layerDependencyId = 0
+        if "layerDependencies" in load:
+            index.layerDependencies = {}
+            for layerBranchId in index.layerBranches:
+                branchName = index.layerBranches[layerBranchId].branch.name
+                collection = index.layerBranches[layerBranchId].collection
+
+                def add_dependency(layerDependencyId, index, deps, required):
+                    try:
+                        depDict = bb.utils.explode_dep_versions2(deps)
+                    except bb.utils.VersionStringException as vse:
+                        bb.fatal('Error parsing LAYERDEPENDS_%s: %s' % (c, str(vse)))
+
+                    for dep, oplist in list(depDict.items()):
+                        # We need to search ourselves, so use the _ version...
+                        depLayerBranch = index.find_collection(dep, branches=[branchName])
+                        if not depLayerBranch:
+                            # Missing dependency?!
+                            logger.error('Missing dependency %s (%s)' % (dep, branchName))
+                            continue
+
+                        # We assume that the oplist matches...
+                        layerDependencyId += 1
+                        layerDependency = layerindexlib.LayerDependency(index, None)
+                        layerDependency.define_data(id=layerDependencyId,
+                                        required=required, layerbranch=layerBranchId,
+                                        dependency=depLayerBranch.layer_id)
+
+                        logger.debug(1, '%s requires %s' % (layerDependency.layer.name, layerDependency.dependency.name))
+                        index.add_element("layerDependencies", [layerDependency])
+
+                    return layerDependencyId
+
+                deps = d.getVar("LAYERDEPENDS_%s" % collection)
+                if deps:
+                    layerDependencyId = add_dependency(layerDependencyId, index, deps, True)
+
+                deps = d.getVar("LAYERRECOMMENDS_%s" % collection)
+                if deps:
+                    layerDependencyId = add_dependency(layerDependencyId, index, deps, False)
+
+        # Need to load recipes here (requires cooker access)
+        recipeId = 0
+        ## TODO: NOT IMPLEMENTED
+        # The code following this is an example of what needs to be
+        # implemented.  However, it does not work as-is.
+        if False and 'recipes' in load:
+            index.recipes = {}
+
+            ret = self.ui_module.main(self.server_connection.connection, self.server_connection.events, config_params)
+
+            all_versions = self._run_command('allProviders')
+
+            all_versions_list = defaultdict(list, all_versions)
+            for pn in all_versions_list:
+                for ((pe, pv, pr), fpath) in all_versions_list[pn]:
+                    realfn = bb.cache.virtualfn2realfn(fpath)
+
+                    filepath = os.path.dirname(realfn[0])
+                    filename = os.path.basename(realfn[0])
+
+                    # This is all HORRIBLY slow, and likely unnecessary
+                    #dscon = self._run_command('parseRecipeFile', fpath, False, [])
+                    #connector = myDataStoreConnector(self, dscon.dsindex)
+                    #recipe_data = bb.data.init()
+                    #recipe_data.setVar('_remote_data', connector)
+
+                    #summary = recipe_data.getVar('SUMMARY')
+                    #description = recipe_data.getVar('DESCRIPTION')
+                    #section = recipe_data.getVar('SECTION')
+                    #license = recipe_data.getVar('LICENSE')
+                    #homepage = recipe_data.getVar('HOMEPAGE')
+                    #bugtracker = recipe_data.getVar('BUGTRACKER')
+                    #provides = recipe_data.getVar('PROVIDES')
+
+                    layer = bb.utils.get_file_layer(realfn[0], self.config_data)
+
+                    depBranchId = collection_layerbranch[layer]
+
+                    recipeId += 1
+                    recipe = layerindexlib.Recipe(index, None)
+                    recipe.define_data(id=recipeId,
+                                   filename=filename, filepath=filepath,
+                                   pn=pn, pv=pv,
+                                   summary=pn, description=pn, section='?',
+                                   license='?', homepage='?', bugtracker='?',
+                                   provides='?', bbclassextend='?', inherits='?',
+                                   blacklisted='?', layerbranch=depBranchId)
+
+                    index = addElement("recipes", [recipe], index)
+
+        # ("machines", layerindexlib.Machine)
+        machineId = 0
+        if 'machines' in load:
+            index.machines = {}
+
+            for layerBranchId in index.layerBranches:
+                # load_bblayers uses the description to cache the actual path...
+                machine_path = index.layerBranches[layerBranchId].getDescription()
+                machine_path = os.path.join(machine_path, 'conf/machine')
+                if os.path.isdir(machine_path):
+                    for (dirpath, _, filenames) in os.walk(machine_path):
+                        # Ignore subdirs...
+                        if not dirpath.endswith('conf/machine'):
+                            continue
+                        for fname in filenames:
+                            if fname.endswith('.conf'):
+                                machineId += 1
+                                machine = layerindexlib.Machine(index, None)
+                                machine.define_data(id=machineId, name=fname[:-5],
+                                                    description=fname[:-5],
+                                                    layerbranch=collection_layerbranch[entry])
+
+                                index.add_element("machines", [machine])
+
+        # ("distros", layerindexlib.Distro)
+        distroId = 0
+        if 'distros' in load:
+            index.distros = {}
+
+            for layerBranchId in index.layerBranches:
+                # load_bblayers uses the description to cache the actual path...
+                distro_path = index.layerBranches[layerBranchId].getDescription()
+                distro_path = os.path.join(distro_path, 'conf/distro')
+                if os.path.isdir(distro_path):
+                    for (dirpath, _, filenames) in os.walk(distro_path):
+                        # Ignore subdirs...
+                        if not dirpath.endswith('conf/distro'):
+                            continue
+                        for fname in filenames:
+                            if fname.endswith('.conf'):
+                                distroId += 1
+                                distro = layerindexlib.Distro(index, None)
+                                distro.define_data(id=distroId, name=fname[:-5],
+                                                    description=fname[:-5],
+                                                    layerbranch=collection_layerbranch[entry])
+
+                                index.add_element("distros", [distro])
+
+        return index

bitbake/lib/layerindexlib/plugin.py

@@ -0,0 +1,60 @@
+# Copyright (C) 2016-2018 Wind River Systems, Inc.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License version 2 as
+# published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+# See the GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+
+# The file contains:
+#   LayerIndex exceptions
+#   Plugin base class
+#   Utility Functions for working on layerindex data
+
+import argparse
+import logging
+import os
+import bb.msg
+
+logger = logging.getLogger('BitBake.layerindexlib.plugin')
+
+class LayerIndexPluginException(Exception):
+    """LayerIndex Generic Exception"""
+    def __init__(self, message):
+         self.msg = message
+         Exception.__init__(self, message)
+
+    def __str__(self):
+         return self.msg
+
+class LayerIndexPluginUrlError(LayerIndexPluginException):
+    """Exception raised when a plugin does not support a given URL type"""
+    def __init__(self, plugin, url):
+        msg = "%s does not support %s:" % (plugin, url)
+        self.plugin = plugin
+        self.url = url
+        LayerIndexPluginException.__init__(self, msg)
+
+class IndexPlugin():
+    def __init__(self):
+        self.type = None
+
+    def init(self, layerindex):
+        self.layerindex = layerindex
+
+    def plugin_type(self):
+        return self.type
+
+    def load_index(self, uri):
+        raise NotImplementedError('load_index is not implemented')
+
+    def store_index(self, uri, index):
+        raise NotImplementedError('store_index is not implemented')
+

bitbake/lib/layerindexlib/restapi.py

@@ -0,0 +1,398 @@
+# Copyright (C) 2016-2018 Wind River Systems, Inc.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License version 2 as
+# published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+# See the GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+
+import logging
+import json
+from urllib.parse import unquote
+from urllib.parse import urlparse
+
+import layerindexlib
+import layerindexlib.plugin
+
+logger = logging.getLogger('BitBake.layerindexlib.restapi')
+
+def plugin_init(plugins):
+    return RestApiPlugin()
+
+class RestApiPlugin(layerindexlib.plugin.IndexPlugin):
+    def __init__(self):
+        self.type = "restapi"
+
+    def load_index(self, url, load):
+        """
+            Fetches layer information from a local or remote layer index.
+
+            The return value is a LayerIndexObj.
+
+            url is the url to the rest api of the layer index, such as:
+            http://layers.openembedded.org/layerindex/api/
+
+            Or a local file...
+        """
+
+        up = urlparse(url)
+
+        if up.scheme == 'file':
+            return self.load_index_file(up, url, load)
+
+        if up.scheme == 'http' or up.scheme == 'https':
+            return self.load_index_web(up, url, load)
+
+        raise layerindexlib.plugin.LayerIndexPluginUrlError(self.type, url)
+
+
+    def load_index_file(self, up, url, load):
+        """
+            Fetches layer information from a local file or directory.
+
+            The return value is a LayerIndexObj.
+
+            ud is the parsed url to the local file or directory.
+        """
+        if not os.path.exists(up.path):
+            raise FileNotFoundError(up.path)
+
+        index = layerindexlib.LayerIndexObj()
+
+        index.config = {}
+        index.config['TYPE'] = self.type
+        index.config['URL'] = url
+
+        params = self.layerindex._parse_params(up.params)
+
+        if 'desc' in params:
+            index.config['DESCRIPTION'] = unquote(params['desc'])
+        else:
+            index.config['DESCRIPTION'] = up.path
+
+        if 'cache' in params:
+            index.config['CACHE'] = params['cache']
+
+        if 'branch' in params:
+            branches = params['branch'].split(',')
+            index.config['BRANCH'] = branches
+        else:
+            branches = ['*']
+
+
+        def load_cache(path, index, branches=[]):
+            logger.debug(1, 'Loading json file %s' % path)
+            with open(path, 'rt', encoding='utf-8') as f:
+                pindex = json.load(f)
+
+            # Filter the branches on loaded files...
+            newpBranch = []
+            for branch in branches:
+                if branch != '*':
+                    if 'branches' in pindex:
+                        for br in pindex['branches']:
+                            if br['name'] == branch:
+                                newpBranch.append(br)
+                else:
+                    if 'branches' in pindex:
+                        for br in pindex['branches']:
+                            newpBranch.append(br)
+
+            if newpBranch:
+                index.add_raw_element('branches', layerindexlib.Branch, newpBranch)
+            else:
+                logger.debug(1, 'No matching branches (%s) in index file(s)' % branches)
+                # No matching branches.. return nothing...
+                return
+
+            for (lName, lType) in [("layerItems", layerindexlib.LayerItem),
+                                   ("layerBranches", layerindexlib.LayerBranch),
+                                   ("layerDependencies", layerindexlib.LayerDependency),
+                                   ("recipes", layerindexlib.Recipe),
+                                   ("machines", layerindexlib.Machine),
+                                   ("distros", layerindexlib.Distro)]:
+                if lName in pindex:
+                    index.add_raw_element(lName, lType, pindex[lName])
+
+
+        if not os.path.isdir(up.path):
+            load_cache(up.path, index, branches)
+            return index
+
+        logger.debug(1, 'Loading from dir %s...' % (up.path))
+        for (dirpath, _, filenames) in os.walk(up.path):
+            for filename in filenames:
+                if not filename.endswith('.json'):
+                    continue
+                fpath = os.path.join(dirpath, filename)
+                load_cache(fpath, index, branches)
+
+        return index
+
+
+    def load_index_web(self, up, url, load):
+        """
+            Fetches layer information from a remote layer index.
+
+            The return value is a LayerIndexObj.
+
+            ud is the parsed url to the rest api of the layer index, such as:
+            http://layers.openembedded.org/layerindex/api/
+        """
+
+        def _get_json_response(apiurl=None, username=None, password=None, retry=True):
+            assert apiurl is not None
+
+            logger.debug(1, "fetching %s" % apiurl)
+
+            up = urlparse(apiurl)
+
+            username=up.username
+            password=up.password
+
+            # Strip username/password and params
+            if up.port:
+                up_stripped = up._replace(params="", netloc="%s:%s" % (up.hostname, up.port))
+            else:
+                up_stripped = up._replace(params="", netloc=up.hostname)
+
+            res = self.layerindex._fetch_url(up_stripped.geturl(), username=username, password=password)
+
+            try:
+                parsed = json.loads(res.read().decode('utf-8'))
+            except ConnectionResetError:
+                if retry:
+                    logger.debug(1, "%s: Connection reset by peer.  Retrying..." % url)
+                    parsed = _get_json_response(apiurl=up_stripped.geturl(), username=username, password=password, retry=False)
+                    logger.debug(1, "%s: retry successful.")
+                else:
+                    raise LayerIndexFetchError('%s: Connection reset by peer.  Is there a firewall blocking your connection?' % apiurl)
+
+            return parsed
+
+        index = layerindexlib.LayerIndexObj()
+
+        index.config = {}
+        index.config['TYPE'] = self.type
+        index.config['URL'] = url
+
+        params = self.layerindex._parse_params(up.params)
+
+        if 'desc' in params:
+            index.config['DESCRIPTION'] = unquote(params['desc'])
+        else:
+            index.config['DESCRIPTION'] = up.hostname
+
+        if 'cache' in params:
+            index.config['CACHE'] = params['cache']
+
+        if 'branch' in params:
+            branches = params['branch'].split(',')
+            index.config['BRANCH'] = branches
+        else:
+            branches = ['*']
+
+        try:
+            index.apilinks = _get_json_response(apiurl=url, username=up.username, password=up.password)
+        except Exception as e:
+            raise layerindexlib.LayerIndexFetchError(url, e)
+
+        # Local raw index set...
+        pindex = {}
+
+        # Load all the requested branches at the same time time,
+        # a special branch of '*' means load all branches
+        filter = ""
+        if "*" not in branches:
+            filter = "?filter=name:%s" % "OR".join(branches)
+
+        logger.debug(1, "Loading %s from %s" % (branches, index.apilinks['branches']))
+
+        # The link won't include username/password, so pull it from the original url
+        pindex['branches'] = _get_json_response(index.apilinks['branches'] + filter,
+                                                    username=up.username, password=up.password)
+        if not pindex['branches']:
+            logger.debug(1, "No valid branches (%s) found at url %s." % (branch, url))
+            return index
+        index.add_raw_element("branches", layerindexlib.Branch, pindex['branches'])
+
+        # Load all of the layerItems (these can not be easily filtered)
+        logger.debug(1, "Loading %s from %s" % ('layerItems', index.apilinks['layerItems']))
+
+
+        # The link won't include username/password, so pull it from the original url
+        pindex['layerItems'] = _get_json_response(index.apilinks['layerItems'],
+                                                  username=up.username, password=up.password)
+        if not pindex['layerItems']:
+            logger.debug(1, "No layers were found at url %s." % (url))
+            return index
+        index.add_raw_element("layerItems", layerindexlib.LayerItem, pindex['layerItems'])
+
+
+	# From this point on load the contents for each branch.  Otherwise we
+	# could run into a timeout.
+        for branch in index.branches:
+            filter = "?filter=branch__name:%s" % index.branches[branch].name
+
+            logger.debug(1, "Loading %s from %s" % ('layerBranches', index.apilinks['layerBranches']))
+
+            # The link won't include username/password, so pull it from the original url
+            pindex['layerBranches'] = _get_json_response(index.apilinks['layerBranches'] + filter,
+                                                  username=up.username, password=up.password)
+            if not pindex['layerBranches']:
+                logger.debug(1, "No valid layer branches (%s) found at url %s." % (branches or "*", url))
+                return index
+            index.add_raw_element("layerBranches", layerindexlib.LayerBranch, pindex['layerBranches'])
+
+
+            # Load the rest, they all have a similar format
+            # Note: the layer index has a few more items, we can add them if necessary
+            # in the future.
+            filter = "?filter=layerbranch__branch__name:%s" % index.branches[branch].name
+            for (lName, lType) in [("layerDependencies", layerindexlib.LayerDependency),
+                                   ("recipes", layerindexlib.Recipe),
+                                   ("machines", layerindexlib.Machine),
+                                   ("distros", layerindexlib.Distro)]:
+                if lName not in load:
+                    continue
+                logger.debug(1, "Loading %s from %s" % (lName, index.apilinks[lName]))
+
+                # The link won't include username/password, so pull it from the original url
+                pindex[lName] = _get_json_response(index.apilinks[lName] + filter,
+                                            username=up.username, password=up.password)
+                index.add_raw_element(lName, lType, pindex[lName])
+
+        return index
+
+    def store_index(self, url, index):
+        """
+            Store layer information into a local file/dir.
+
+            The return value is a dictionary containing API,
+            layer, branch, dependency, recipe, machine, distro, information.
+
+            ud is a parsed url to a directory or file.  If the path is a
+            directory, we will split the files into one file per layer.
+            If the path is to a file (exists or not) the entire DB will be
+            dumped into that one file.
+        """
+
+        up = urlparse(url)
+
+        if up.scheme != 'file':
+            raise layerindexlib.plugin.LayerIndexPluginUrlError(self.type, url)
+
+        logger.debug(1, "Storing to %s..." % up.path)
+
+        try:
+            layerbranches = index.layerBranches
+        except KeyError:
+            logger.error('No layerBranches to write.')
+            return
+
+
+        def filter_item(layerbranchid, objects):
+            filtered = []
+            for obj in getattr(index, objects, None):
+                try:
+                    if getattr(index, objects)[obj].layerbranch_id == layerbranchid:
+                       filtered.append(getattr(index, objects)[obj]._data)
+                except AttributeError:
+                    logger.debug(1, 'No obj.layerbranch_id: %s' % objects)
+                    # No simple filter method, just include it...
+                    try:
+                        filtered.append(getattr(index, objects)[obj]._data)
+                    except AttributeError:
+                        logger.debug(1, 'No obj._data: %s %s' % (objects, type(obj)))
+                        filtered.append(obj)
+            return filtered
+
+
+        # Write out to a single file.
+        # Filter out unnecessary items, then sort as we write for determinism
+        if not os.path.isdir(up.path):
+            pindex = {}
+
+            pindex['branches'] = []
+            pindex['layerItems'] = []
+            pindex['layerBranches'] = []
+
+            for layerbranchid in layerbranches:
+                if layerbranches[layerbranchid].branch._data not in pindex['branches']:
+                    pindex['branches'].append(layerbranches[layerbranchid].branch._data)
+
+                if layerbranches[layerbranchid].layer._data not in pindex['layerItems']:
+                    pindex['layerItems'].append(layerbranches[layerbranchid].layer._data)
+
+                if layerbranches[layerbranchid]._data not in pindex['layerBranches']:
+                    pindex['layerBranches'].append(layerbranches[layerbranchid]._data)
+
+                for entry in index._index:
+                    # Skip local items, apilinks and items already processed
+                    if entry in index.config['local'] or \
+                       entry == 'apilinks' or \
+                       entry == 'branches' or \
+                       entry == 'layerBranches' or \
+                       entry == 'layerItems':
+                        continue
+                    if entry not in pindex:
+                        pindex[entry] = []
+                    pindex[entry].extend(filter_item(layerbranchid, entry))
+
+            bb.debug(1, 'Writing index to %s' % up.path)
+            with open(up.path, 'wt') as f:
+                json.dump(layerindexlib.sort_entry(pindex), f, indent=4)
+            return
+
+
+        # Write out to a directory one file per layerBranch
+        # Prepare all layer related items, to create a minimal file.
+        # We have to sort the entries as we write so they are deterministic
+        for layerbranchid in layerbranches:
+            pindex = {}
+
+            for entry in index._index:
+                # Skip local items, apilinks and items already processed
+                if entry in index.config['local'] or \
+                   entry == 'apilinks' or \
+                   entry == 'branches' or \
+                   entry == 'layerBranches' or \
+                   entry == 'layerItems':
+                    continue
+                pindex[entry] = filter_item(layerbranchid, entry)
+
+            # Add the layer we're processing as the first one...
+            pindex['branches'] = [layerbranches[layerbranchid].branch._data]
+            pindex['layerItems'] = [layerbranches[layerbranchid].layer._data]
+            pindex['layerBranches'] = [layerbranches[layerbranchid]._data]
+
+            # We also need to include the layerbranch for any dependencies...
+            for layerdep in pindex['layerDependencies']:
+                layerdependency = layerindexlib.LayerDependency(index, layerdep)
+
+                layeritem = layerdependency.dependency
+                layerbranch = layerdependency.dependency_layerBranch
+
+                # We need to avoid duplicates...
+                if layeritem._data not in pindex['layerItems']:
+                    pindex['layerItems'].append(layeritem._data)
+
+                if layerbranch._data not in pindex['layerBranches']:
+                    pindex['layerBranches'].append(layerbranch._data)
+
+            # apply mirroring adjustments here....
+
+            fname = index.config['DESCRIPTION'] + '__' + pindex['branches'][0]['name'] + '__' + pindex['layerItems'][0]['name']
+            fname = fname.translate(str.maketrans('/ ', '__'))
+            fpath = os.path.join(up.path, fname)
+
+            bb.debug(1, 'Writing index to %s' % fpath + '.json')
+            with open(fpath + '.json', 'wt') as f:
+                json.dump(layerindexlib.sort_entry(pindex), f, indent=4)

bitbake/lib/layerindexlib/tests/common.py

@@ -0,0 +1,43 @@
+# Copyright (C) 2017-2018 Wind River Systems, Inc.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License version 2 as
+# published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+# See the GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+
+import unittest
+import tempfile
+import os
+import bb
+
+import logging
+
+class LayersTest(unittest.TestCase):
+
+    def setUp(self):
+        self.origdir = os.getcwd()
+        self.d = bb.data.init()
+        # At least one variable needs to be set
+        self.d.setVar('DL_DIR', os.getcwd())
+
+        if os.environ.get("BB_SKIP_NETTESTS") == "yes":
+            self.d.setVar('BB_NO_NETWORK', '1')
+
+        self.tempdir = tempfile.mkdtemp()
+        self.logger = logging.getLogger("BitBake")
+
+    def tearDown(self):
+        os.chdir(self.origdir)
+        if os.environ.get("BB_TMPDIR_NOCLEAN") == "yes":
+            print("Not cleaning up %s. Please remove manually." % self.tempdir)
+        else:
+            bb.utils.prunedir(self.tempdir)
+

bitbake/lib/layerindexlib/tests/cooker.py

@@ -0,0 +1,123 @@
+# Copyright (C) 2018 Wind River Systems, Inc.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License version 2 as
+# published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+# See the GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+
+import unittest
+import tempfile
+import os
+import bb
+
+import layerindexlib
+from layerindexlib.tests.common import LayersTest
+
+import logging
+
+class LayerIndexCookerTest(LayersTest):
+
+    def setUp(self):
+        LayersTest.setUp(self)
+
+        # Note this is NOT a comprehensive test of cooker, as we can't easily
+        # configure the test data.  But we can emulate the basics of the layer.conf
+        # files, so that is what we will do.
+
+        new_topdir = os.path.join(os.path.dirname(__file__), "testdata")
+        new_bbpath = os.path.join(new_topdir, "build")
+
+        self.d.setVar('TOPDIR', new_topdir)
+        self.d.setVar('BBPATH', new_bbpath)
+
+        self.d = bb.parse.handle("%s/conf/bblayers.conf" % new_bbpath, self.d, True)
+        for layer in self.d.getVar('BBLAYERS').split():
+            self.d = bb.parse.handle("%s/conf/layer.conf" % layer, self.d, True)
+
+        self.layerindex = layerindexlib.LayerIndex(self.d)
+        self.layerindex.load_layerindex('cooker://', load=['layerDependencies'])
+
+    def test_layerindex_is_empty(self):
+        self.assertFalse(self.layerindex.is_empty(), msg="Layerindex is not empty!")
+
+    def test_dependency_resolution(self):
+        # Verify depth first searching...
+        (dependencies, invalidnames) = self.layerindex.find_dependencies(names=['meta-python'])
+
+        first = True
+        for deplayerbranch in dependencies:
+            layerBranch = dependencies[deplayerbranch][0]
+            layerDeps = dependencies[deplayerbranch][1:]
+
+            if not first:
+                continue
+
+            first = False
+
+            # Top of the deps should be openembedded-core, since everything depends on it.
+            self.assertEqual(layerBranch.layer.name, "openembedded-core", msg='Top dependency not openembedded-core')
+
+            # meta-python should cause an openembedded-core dependency, if not assert!
+            for dep in layerDeps:
+                if dep.layer.name == 'meta-python':
+                    break
+            else:
+                self.assertTrue(False, msg='meta-python was not found')
+
+            # Only check the first element...
+            break
+        else:
+            if first:
+                # Empty list, this is bad.
+                self.assertTrue(False, msg='Empty list of dependencies')
+
+            # Last dep should be the requested item
+            layerBranch = dependencies[deplayerbranch][0]
+            self.assertEqual(layerBranch.layer.name, "meta-python", msg='Last dependency not meta-python')
+
+    def test_find_collection(self):
+        def _check(collection, expected):
+            self.logger.debug(1, "Looking for collection %s..." % collection)
+            result = self.layerindex.find_collection(collection)
+            if expected:
+                self.assertIsNotNone(result, msg="Did not find %s when it shouldn't be there" % collection)
+            else:
+                self.assertIsNone(result, msg="Found %s when it should be there" % collection)
+
+        tests = [ ('core', True),
+                  ('openembedded-core', False),
+                  ('networking-layer', True),
+                  ('meta-python', True),
+                  ('openembedded-layer', True),
+                  ('notpresent', False) ]
+
+        for collection,result in tests:
+            _check(collection, result)
+
+    def test_find_layerbranch(self):
+        def _check(name, expected):
+            self.logger.debug(1, "Looking for layerbranch %s..." % name)
+            result = self.layerindex.find_layerbranch(name)
+            if expected:
+                self.assertIsNotNone(result, msg="Did not find %s when it shouldn't be there" % collection)
+            else:
+                self.assertIsNone(result, msg="Found %s when it should be there" % collection)
+
+        tests = [ ('openembedded-core', True),
+                  ('core', False),
+                  ('networking-layer', True),
+                  ('meta-python', True),
+                  ('openembedded-layer', True),
+                  ('notpresent', False) ]
+
+        for collection,result in tests:
+            _check(collection, result)
+

bitbake/lib/layerindexlib/tests/layerindexobj.py

@@ -0,0 +1,226 @@
+# Copyright (C) 2017-2018 Wind River Systems, Inc.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License version 2 as
+# published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+# See the GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+
+import unittest
+import tempfile
+import os
+import bb
+
+from layerindexlib.tests.common import LayersTest
+
+import logging
+
+class LayerIndexObjectsTest(LayersTest):
+    def setUp(self):
+        from layerindexlib import LayerIndexObj, Branch, LayerItem, LayerBranch, LayerDependency, Recipe, Machine, Distro
+
+        LayersTest.setUp(self)
+
+        self.index = LayerIndexObj()
+
+        branchId = 0
+        layerItemId = 0
+        layerBranchId = 0
+        layerDependencyId = 0
+        recipeId = 0
+        machineId = 0
+        distroId = 0
+
+        self.index.branches = {}
+        self.index.layerItems = {}
+        self.index.layerBranches = {}
+        self.index.layerDependencies = {}
+        self.index.recipes = {}
+        self.index.machines = {}
+        self.index.distros = {}
+
+        branchId += 1
+        self.index.branches[branchId] = Branch(self.index)
+        self.index.branches[branchId].define_data(branchId,
+                                        'test_branch', 'bb_test_branch')
+        self.index.branches[branchId].lockData()
+
+        layerItemId +=1
+        self.index.layerItems[layerItemId] = LayerItem(self.index)
+        self.index.layerItems[layerItemId].define_data(layerItemId,
+                                        'test_layerItem', vcs_url='git://git_test_url/test_layerItem')
+        self.index.layerItems[layerItemId].lockData()
+
+        layerBranchId +=1
+        self.index.layerBranches[layerBranchId] = LayerBranch(self.index)
+        self.index.layerBranches[layerBranchId].define_data(layerBranchId,
+                                        'test_collection', '99', layerItemId,
+                                        branchId)
+
+        recipeId += 1
+        self.index.recipes[recipeId] = Recipe(self.index)
+        self.index.recipes[recipeId].define_data(recipeId, 'test_git.bb',
+                                        'recipes-test', 'test', 'git',
+                                        layerBranchId)
+
+        machineId += 1
+        self.index.machines[machineId] = Machine(self.index)
+        self.index.machines[machineId].define_data(machineId,
+                                        'test_machine', 'test_machine',
+                                        layerBranchId)
+
+        distroId += 1
+        self.index.distros[distroId] = Distro(self.index)
+        self.index.distros[distroId].define_data(distroId,
+                                        'test_distro', 'test_distro',
+                                        layerBranchId)
+
+        layerItemId +=1
+        self.index.layerItems[layerItemId] = LayerItem(self.index)
+        self.index.layerItems[layerItemId].define_data(layerItemId, 'test_layerItem 2',
+                                        vcs_url='git://git_test_url/test_layerItem')
+
+        layerBranchId +=1
+        self.index.layerBranches[layerBranchId] = LayerBranch(self.index)
+        self.index.layerBranches[layerBranchId].define_data(layerBranchId,
+                                        'test_collection_2', '72', layerItemId,
+                                        branchId, actual_branch='some_other_branch')
+
+        layerDependencyId += 1
+        self.index.layerDependencies[layerDependencyId] = LayerDependency(self.index)
+        self.index.layerDependencies[layerDependencyId].define_data(layerDependencyId,
+                                        layerBranchId, 1)
+
+        layerDependencyId += 1
+        self.index.layerDependencies[layerDependencyId] = LayerDependency(self.index)
+        self.index.layerDependencies[layerDependencyId].define_data(layerDependencyId,
+                                        layerBranchId, 1, required=False)
+
+    def test_branch(self):
+        branch = self.index.branches[1]
+        self.assertEqual(branch.id, 1)
+        self.assertEqual(branch.name, 'test_branch')
+        self.assertEqual(branch.short_description, 'test_branch')
+        self.assertEqual(branch.bitbake_branch, 'bb_test_branch')
+
+    def test_layerItem(self):
+        layerItem = self.index.layerItems[1]
+        self.assertEqual(layerItem.id, 1)
+        self.assertEqual(layerItem.name, 'test_layerItem')
+        self.assertEqual(layerItem.summary, 'test_layerItem')
+        self.assertEqual(layerItem.description, 'test_layerItem')
+        self.assertEqual(layerItem.vcs_url, 'git://git_test_url/test_layerItem')
+        self.assertEqual(layerItem.vcs_web_url, None)
+        self.assertIsNone(layerItem.vcs_web_tree_base_url)
+        self.assertIsNone(layerItem.vcs_web_file_base_url)
+        self.assertIsNotNone(layerItem.updated)
+
+        layerItem = self.index.layerItems[2]
+        self.assertEqual(layerItem.id, 2)
+        self.assertEqual(layerItem.name, 'test_layerItem 2')
+        self.assertEqual(layerItem.summary, 'test_layerItem 2')
+        self.assertEqual(layerItem.description, 'test_layerItem 2')
+        self.assertEqual(layerItem.vcs_url, 'git://git_test_url/test_layerItem')
+        self.assertIsNone(layerItem.vcs_web_url)
+        self.assertIsNone(layerItem.vcs_web_tree_base_url)
+        self.assertIsNone(layerItem.vcs_web_file_base_url)
+        self.assertIsNotNone(layerItem.updated)
+
+    def test_layerBranch(self):
+        layerBranch = self.index.layerBranches[1]
+        self.assertEqual(layerBranch.id, 1)
+        self.assertEqual(layerBranch.collection, 'test_collection')
+        self.assertEqual(layerBranch.version, '99')
+        self.assertEqual(layerBranch.vcs_subdir, '')
+        self.assertEqual(layerBranch.actual_branch, 'test_branch')
+        self.assertIsNotNone(layerBranch.updated)
+        self.assertEqual(layerBranch.layer_id, 1)
+        self.assertEqual(layerBranch.branch_id, 1)
+        self.assertEqual(layerBranch.layer, self.index.layerItems[1])
+        self.assertEqual(layerBranch.branch, self.index.branches[1])
+
+        layerBranch = self.index.layerBranches[2]
+        self.assertEqual(layerBranch.id, 2)
+        self.assertEqual(layerBranch.collection, 'test_collection_2')
+        self.assertEqual(layerBranch.version, '72')
+        self.assertEqual(layerBranch.vcs_subdir, '')
+        self.assertEqual(layerBranch.actual_branch, 'some_other_branch')
+        self.assertIsNotNone(layerBranch.updated)
+        self.assertEqual(layerBranch.layer_id, 2)
+        self.assertEqual(layerBranch.branch_id, 1)
+        self.assertEqual(layerBranch.layer, self.index.layerItems[2])
+        self.assertEqual(layerBranch.branch, self.index.branches[1])
+
+    def test_layerDependency(self):
+        layerDependency = self.index.layerDependencies[1]
+        self.assertEqual(layerDependency.id, 1)
+        self.assertEqual(layerDependency.layerbranch_id, 2)
+        self.assertEqual(layerDependency.layerbranch, self.index.layerBranches[2])
+        self.assertEqual(layerDependency.layer_id, 2)
+        self.assertEqual(layerDependency.layer, self.index.layerItems[2])
+        self.assertTrue(layerDependency.required)
+        self.assertEqual(layerDependency.dependency_id, 1)
+        self.assertEqual(layerDependency.dependency, self.index.layerItems[1])
+        self.assertEqual(layerDependency.dependency_layerBranch, self.index.layerBranches[1])
+
+        layerDependency = self.index.layerDependencies[2]
+        self.assertEqual(layerDependency.id, 2)
+        self.assertEqual(layerDependency.layerbranch_id, 2)
+        self.assertEqual(layerDependency.layerbranch, self.index.layerBranches[2])
+        self.assertEqual(layerDependency.layer_id, 2)
+        self.assertEqual(layerDependency.layer, self.index.layerItems[2])
+        self.assertFalse(layerDependency.required)
+        self.assertEqual(layerDependency.dependency_id, 1)
+        self.assertEqual(layerDependency.dependency, self.index.layerItems[1])
+        self.assertEqual(layerDependency.dependency_layerBranch, self.index.layerBranches[1])
+
+    def test_recipe(self):
+        recipe = self.index.recipes[1]
+        self.assertEqual(recipe.id, 1)
+        self.assertEqual(recipe.layerbranch_id, 1)
+        self.assertEqual(recipe.layerbranch, self.index.layerBranches[1])
+        self.assertEqual(recipe.layer_id, 1)
+        self.assertEqual(recipe.layer, self.index.layerItems[1])
+        self.assertEqual(recipe.filename, 'test_git.bb')
+        self.assertEqual(recipe.filepath, 'recipes-test')
+        self.assertEqual(recipe.fullpath, 'recipes-test/test_git.bb')
+        self.assertEqual(recipe.summary, "")
+        self.assertEqual(recipe.description, "")
+        self.assertEqual(recipe.section, "")
+        self.assertEqual(recipe.pn, 'test')
+        self.assertEqual(recipe.pv, 'git')
+        self.assertEqual(recipe.license, "")
+        self.assertEqual(recipe.homepage, "")
+        self.assertEqual(recipe.bugtracker, "")
+        self.assertEqual(recipe.provides, "")
+        self.assertIsNotNone(recipe.updated)
+        self.assertEqual(recipe.inherits, "")
+
+    def test_machine(self):
+        machine = self.index.machines[1]
+        self.assertEqual(machine.id, 1)
+        self.assertEqual(machine.layerbranch_id, 1)
+        self.assertEqual(machine.layerbranch, self.index.layerBranches[1])
+        self.assertEqual(machine.layer_id, 1)
+        self.assertEqual(machine.layer, self.index.layerItems[1])
+        self.assertEqual(machine.name, 'test_machine')
+        self.assertEqual(machine.description, 'test_machine')
+        self.assertIsNotNone(machine.updated)
+
+    def test_distro(self):
+        distro = self.index.distros[1]
+        self.assertEqual(distro.id, 1)
+        self.assertEqual(distro.layerbranch_id, 1)
+        self.assertEqual(distro.layerbranch, self.index.layerBranches[1])
+        self.assertEqual(distro.layer_id, 1)
+        self.assertEqual(distro.layer, self.index.layerItems[1])
+        self.assertEqual(distro.name, 'test_distro')
+        self.assertEqual(distro.description, 'test_distro')
+        self.assertIsNotNone(distro.updated)

bitbake/lib/layerindexlib/tests/restapi.py

@@ -0,0 +1,174 @@
+# Copyright (C) 2017-2018 Wind River Systems, Inc.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License version 2 as
+# published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+# See the GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+
+import unittest
+import tempfile
+import os
+import bb
+
+import layerindexlib
+from layerindexlib.tests.common import LayersTest
+
+import logging
+
+class LayerIndexWebRestApiTest(LayersTest):
+
+    if os.environ.get("BB_SKIP_NETTESTS") == "yes":
+        print("Unset BB_SKIP_NETTESTS to run network tests")
+    else:
+        def setUp(self):
+            LayersTest.setUp(self)
+            self.layerindex = layerindexlib.LayerIndex(self.d)
+            self.layerindex.load_layerindex('http://layers.openembedded.org/layerindex/api/;branch=sumo', load=['layerDependencies'])
+
+        def test_layerindex_is_empty(self):
+            self.assertFalse(self.layerindex.is_empty(), msg="Layerindex is empty")
+
+        def test_layerindex_store_file(self):
+            self.layerindex.store_layerindex('file://%s/file.json' % self.tempdir, self.layerindex.indexes[0])
+
+            self.assertTrue(os.path.isfile('%s/file.json' % self.tempdir), msg="Temporary file was not created by store_layerindex")
+
+            reload = layerindexlib.LayerIndex(self.d)
+            reload.load_layerindex('file://%s/file.json' % self.tempdir)
+
+            self.assertFalse(reload.is_empty(), msg="Layerindex is empty")
+
+            # Calculate layerItems in original index that should NOT be in reload
+            layerItemNames = []
+            for itemId in self.layerindex.indexes[0].layerItems:
+                layerItemNames.append(self.layerindex.indexes[0].layerItems[itemId].name)
+
+            for layerBranchId in self.layerindex.indexes[0].layerBranches:
+                layerItemNames.remove(self.layerindex.indexes[0].layerBranches[layerBranchId].layer.name)
+
+            for itemId in reload.indexes[0].layerItems:
+                self.assertFalse(reload.indexes[0].layerItems[itemId].name in layerItemNames, msg="Item reloaded when it shouldn't have been")
+
+            # Compare the original to what we wrote...
+            for type in self.layerindex.indexes[0]._index:
+                if type == 'apilinks' or \
+                   type == 'layerItems' or \
+                   type in self.layerindex.indexes[0].config['local']:
+                    continue
+                for id in getattr(self.layerindex.indexes[0], type):
+                    self.logger.debug(1, "type %s" % (type))
+
+                    self.assertTrue(id in getattr(reload.indexes[0], type), msg="Id number not in reloaded index")
+
+                    self.logger.debug(1, "%s ? %s" % (getattr(self.layerindex.indexes[0], type)[id], getattr(reload.indexes[0], type)[id]))
+
+                    self.assertEqual(getattr(self.layerindex.indexes[0], type)[id], getattr(reload.indexes[0], type)[id], msg="Reloaded contents different")
+
+        def test_layerindex_store_split(self):
+            self.layerindex.store_layerindex('file://%s' % self.tempdir, self.layerindex.indexes[0])
+
+            reload = layerindexlib.LayerIndex(self.d)
+            reload.load_layerindex('file://%s' % self.tempdir)
+
+            self.assertFalse(reload.is_empty(), msg="Layer index is empty")
+
+            for type in self.layerindex.indexes[0]._index:
+                if type == 'apilinks' or \
+                   type == 'layerItems' or \
+                   type in self.layerindex.indexes[0].config['local']:
+                    continue
+                for id in getattr(self.layerindex.indexes[0] ,type):
+                    self.logger.debug(1, "type %s" % (type))
+
+                    self.assertTrue(id in getattr(reload.indexes[0], type), msg="Id number missing from reloaded data")
+
+                    self.logger.debug(1, "%s ? %s" % (getattr(self.layerindex.indexes[0] ,type)[id], getattr(reload.indexes[0], type)[id]))
+
+                    self.assertEqual(getattr(self.layerindex.indexes[0] ,type)[id], getattr(reload.indexes[0], type)[id], msg="reloaded data does not match original")
+
+        def test_dependency_resolution(self):
+            # Verify depth first searching...
+            (dependencies, invalidnames) = self.layerindex.find_dependencies(names=['meta-python'])
+
+            first = True
+            for deplayerbranch in dependencies:
+                layerBranch = dependencies[deplayerbranch][0]
+                layerDeps = dependencies[deplayerbranch][1:]
+
+                if not first:
+                    continue
+
+                first = False
+
+                # Top of the deps should be openembedded-core, since everything depends on it.
+                self.assertEqual(layerBranch.layer.name, "openembedded-core", msg='OpenEmbedded-Core is no the first dependency')
+
+                # meta-python should cause an openembedded-core dependency, if not assert!
+                for dep in layerDeps:
+                    if dep.layer.name == 'meta-python':
+                        break
+                else:
+                    self.logger.debug(1, "meta-python was not found")
+                    self.assetTrue(False)
+
+                # Only check the first element...
+                break
+            else:
+                # Empty list, this is bad.
+                self.logger.debug(1, "Empty list of dependencies")
+                self.assertIsNotNone(first, msg="Empty list of dependencies")
+
+                # Last dep should be the requested item
+                layerBranch = dependencies[deplayerbranch][0]
+                self.assertEqual(layerBranch.layer.name, "meta-python", msg="Last dependency not meta-python")
+
+    def test_find_collection(self):
+        def _check(collection, expected):
+            self.logger.debug(1, "Looking for collection %s..." % collection)
+            result = self.layerindex.find_collection(collection)
+            if expected:
+                self.assertIsNotNone(result, msg="Did not find %s when it should be there" % collection)
+            else:
+                self.assertIsNone(result, msg="Found %s when it shouldn't be there" % collection)
+
+        tests = [ ('core', True),
+                  ('openembedded-core', False),
+                  ('networking-layer', True),
+                  ('meta-python', True),
+                  ('openembedded-layer', True),
+                  ('notpresent', False) ]
+
+        for collection,result in tests:
+            _check(collection, result)
+
+    def test_find_layerbranch(self):
+        def _check(name, expected):
+            self.logger.debug(1, "Looking for layerbranch %s..." % name)
+
+            for index in self.layerindex.indexes:
+                for layerbranchid in index.layerBranches:
+                    self.logger.debug(1, "Present: %s" % index.layerBranches[layerbranchid].layer.name)
+            result = self.layerindex.find_layerbranch(name)
+            if expected:
+                self.assertIsNotNone(result, msg="Did not find %s when it should be there" % collection)
+            else:
+                self.assertIsNone(result, msg="Found %s when it shouldn't be there" % collection)
+
+        tests = [ ('openembedded-core', True),
+                  ('core', False),
+                  ('meta-networking', True),
+                  ('meta-python', True),
+                  ('meta-oe', True),
+                  ('notpresent', False) ]
+
+        for collection,result in tests:
+            _check(collection, result)
+

bitbake/lib/layerindexlib/tests/testdata/README

@@ -0,0 +1,11 @@
+This test data is used to verify the 'cooker' module of the layerindex.
+
+The module consists of a faux project bblayers.conf with four layers defined.
+
+layer1 - openembedded-core
+layer2 - networking-layer
+layer3 - meta-python
+layer4 - openembedded-layer (meta-oe)
+
+Since we do not have a fully populated cooker, we use this to test the
+basic index generation, and not any deep recipe based contents.

bitbake/lib/layerindexlib/tests/testdata/build/conf/bblayers.conf

@@ -0,0 +1,15 @@
+LAYERSERIES_CORENAMES = "sumo"
+
+# LAYER_CONF_VERSION is increased each time build/conf/bblayers.conf
+# changes incompatibly
+LCONF_VERSION = "7"
+
+BBPATH = "${TOPDIR}"
+BBFILES ?= ""
+
+BBLAYERS ?= " \
+  ${TOPDIR}/layer1 \
+  ${TOPDIR}/layer2 \
+  ${TOPDIR}/layer3 \
+  ${TOPDIR}/layer4 \
+  "

bitbake/lib/layerindexlib/tests/testdata/layer1/conf/layer.conf

@@ -0,0 +1,17 @@
+# We have a conf and classes directory, add to BBPATH
+BBPATH .= ":${LAYERDIR}"
+# We have recipes-* directories, add to BBFILES
+BBFILES += "${LAYERDIR}/recipes-*/*/*.bb"
+
+BBFILE_COLLECTIONS += "core"
+BBFILE_PATTERN_core = "^${LAYERDIR}/"
+BBFILE_PRIORITY_core = "5"
+
+LAYERSERIES_CORENAMES = "sumo"
+
+# This should only be incremented on significant changes that will
+# cause compatibility issues with other layers
+LAYERVERSION_core = "11"
+LAYERSERIES_COMPAT_core = "sumo"
+
+BBLAYERS_LAYERINDEX_NAME_core = "openembedded-core"

bitbake/lib/layerindexlib/tests/testdata/layer2/conf/layer.conf

@@ -0,0 +1,20 @@
+# We have a conf and classes directory, add to BBPATH
+BBPATH .= ":${LAYERDIR}"
+
+# We have a packages directory, add to BBFILES
+BBFILES += "${LAYERDIR}/recipes-*/*/*.bb \
+            ${LAYERDIR}/recipes-*/*/*.bbappend"
+
+BBFILE_COLLECTIONS += "networking-layer"
+BBFILE_PATTERN_networking-layer := "^${LAYERDIR}/"
+BBFILE_PRIORITY_networking-layer = "5"
+
+# This should only be incremented on significant changes that will
+# cause compatibility issues with other layers
+LAYERVERSION_networking-layer = "1"
+
+LAYERDEPENDS_networking-layer = "core"
+LAYERDEPENDS_networking-layer += "openembedded-layer"
+LAYERDEPENDS_networking-layer += "meta-python"
+
+LAYERSERIES_COMPAT_networking-layer = "sumo"

bitbake/lib/layerindexlib/tests/testdata/layer3/conf/layer.conf

@@ -0,0 +1,19 @@
+# We might have a conf and classes directory, append to BBPATH
+BBPATH .= ":${LAYERDIR}"
+
+# We have recipes directories, add to BBFILES
+BBFILES += "${LAYERDIR}/recipes*/*/*.bb ${LAYERDIR}/recipes*/*/*.bbappend"
+
+BBFILE_COLLECTIONS += "meta-python"
+BBFILE_PATTERN_meta-python := "^${LAYERDIR}/"
+BBFILE_PRIORITY_meta-python = "7"
+
+# This should only be incremented on significant changes that will
+# cause compatibility issues with other layers
+LAYERVERSION_meta-python = "1"
+
+LAYERDEPENDS_meta-python = "core openembedded-layer"
+
+LAYERSERIES_COMPAT_meta-python = "sumo"
+
+LICENSE_PATH += "${LAYERDIR}/licenses"

bitbake/lib/layerindexlib/tests/testdata/layer4/conf/layer.conf

@@ -0,0 +1,22 @@
+# We have a conf and classes directory, append to BBPATH
+BBPATH .= ":${LAYERDIR}"
+
+# We have a recipes directory, add to BBFILES
+BBFILES += "${LAYERDIR}/recipes-*/*/*.bb ${LAYERDIR}/recipes-*/*/*.bbappend"
+
+BBFILE_COLLECTIONS += "openembedded-layer"
+BBFILE_PATTERN_openembedded-layer := "^${LAYERDIR}/"
+
+# Define the priority for recipes (.bb files) from this layer,
+# choosing carefully how this layer interacts with all of the
+# other layers.
+
+BBFILE_PRIORITY_openembedded-layer = "6"
+
+# This should only be incremented on significant changes that will
+# cause compatibility issues with other layers
+LAYERVERSION_openembedded-layer = "1"
+
+LAYERDEPENDS_openembedded-layer = "core"
+
+LAYERSERIES_COMPAT_openembedded-layer = "sumo"

bitbake/lib/toaster/bldcontrol/localhostbecontroller.py

@@ -27,8 +27,9 @@ import shutil
 import time
 from django.db import transaction
 from django.db.models import Q
-from bldcontrol.models import BuildEnvironment, BRLayer, BRVariable, BRTarget, BRBitbake
-from orm.models import CustomImageRecipe, Layer, Layer_Version, ProjectLayer, ToasterSetting
+from bldcontrol.models import BuildEnvironment, BuildRequest, BRLayer, BRVariable, BRTarget, BRBitbake, Build
+from orm.models import CustomImageRecipe, Layer, Layer_Version, Project, ProjectLayer, ToasterSetting
+from orm.models import signal_runbuilds
 import subprocess
 
 from toastermain import settings
@@ -38,6 +39,8 @@ from bldcontrol.bbcontroller import BuildEnvironmentController, ShellCmdExceptio
 import logging
 logger = logging.getLogger("toaster")
 
+install_dir = os.environ.get('TOASTER_DIR')
+
 from pprint import pprint, pformat
 
 class LocalhostBEController(BuildEnvironmentController):
@@ -87,10 +90,10 @@ class LocalhostBEController(BuildEnvironmentController):
         #logger.debug("localhostbecontroller: using HEAD checkout in %s" % local_checkout_path)
         return local_checkout_path
 
-
-    def setCloneStatus(self,bitbake,status,total,current):
+    def setCloneStatus(self,bitbake,status,total,current,repo_name):
         bitbake.req.build.repos_cloned=current
         bitbake.req.build.repos_to_clone=total
+        bitbake.req.build.progress_item=repo_name
         bitbake.req.build.save()
 
     def setLayers(self, bitbake, layers, targets):
@@ -100,6 +103,7 @@ class LocalhostBEController(BuildEnvironmentController):
 
         layerlist = []
         nongitlayerlist = []
+        layer_index = 0
         git_env = os.environ.copy()
         # (note: add custom environment settings here)
 
@@ -113,7 +117,7 @@ class LocalhostBEController(BuildEnvironmentController):
         if bitbake.giturl and bitbake.commit:
             gitrepos[(bitbake.giturl, bitbake.commit)] = []
             gitrepos[(bitbake.giturl, bitbake.commit)].append(
-                ("bitbake", bitbake.dirpath))
+                ("bitbake", bitbake.dirpath, 0))
 
         for layer in layers:
             # We don't need to git clone the layer for the CustomImageRecipe
@@ -124,12 +128,13 @@ class LocalhostBEController(BuildEnvironmentController):
             # If we have local layers then we don't need clone them
             # For local layers giturl will be empty
             if not layer.giturl:
-                nongitlayerlist.append(layer.layer_version.layer.local_source_dir)
+                nongitlayerlist.append( "%03d:%s" % (layer_index,layer.local_source_dir) )
                 continue
 
             if not (layer.giturl, layer.commit) in gitrepos:
                 gitrepos[(layer.giturl, layer.commit)] = []
-            gitrepos[(layer.giturl, layer.commit)].append( (layer.name, layer.dirpath) )
+            gitrepos[(layer.giturl, layer.commit)].append( (layer.name,layer.dirpath,layer_index) )
+            layer_index += 1
 
 
         logger.debug("localhostbecontroller, our git repos are %s" % pformat(gitrepos))
@@ -159,9 +164,9 @@ class LocalhostBEController(BuildEnvironmentController):
         # 3. checkout the repositories
         clone_count=0
         clone_total=len(gitrepos.keys())
-        self.setCloneStatus(bitbake,'Started',clone_total,clone_count)
+        self.setCloneStatus(bitbake,'Started',clone_total,clone_count,'')
         for giturl, commit in gitrepos.keys():
-            self.setCloneStatus(bitbake,'progress',clone_total,clone_count)
+            self.setCloneStatus(bitbake,'progress',clone_total,clone_count,gitrepos[(giturl, commit)][0][0])
             clone_count += 1
 
             localdirname = os.path.join(self.be.sourcedir, self.getGitCloneDirectory(giturl, commit))
@@ -205,21 +210,33 @@ class LocalhostBEController(BuildEnvironmentController):
                     self._shellcmd("git clone -b \"%s\" \"%s\" \"%s\" " % (bitbake.commit, bitbake.giturl, os.path.join(self.pokydirname, 'bitbake')),env=git_env)
 
             # verify our repositories
-            for name, dirpath in gitrepos[(giturl, commit)]:
+            for name, dirpath, index in gitrepos[(giturl, commit)]:
                 localdirpath = os.path.join(localdirname, dirpath)
-                logger.debug("localhostbecontroller: localdirpath expected '%s'" % localdirpath)
+                logger.debug("localhostbecontroller: localdirpath expects '%s'" % localdirpath)
                 if not os.path.exists(localdirpath):
                     raise BuildSetupException("Cannot find layer git path '%s' in checked out repository '%s:%s'. Aborting." % (localdirpath, giturl, commit))
 
                 if name != "bitbake":
-                    layerlist.append(localdirpath.rstrip("/"))
+                    layerlist.append("%03d:%s" % (index,localdirpath.rstrip("/")))
 
-        self.setCloneStatus(bitbake,'complete',clone_total,clone_count)
+        self.setCloneStatus(bitbake,'complete',clone_total,clone_count,'')
         logger.debug("localhostbecontroller: current layer list %s " % pformat(layerlist))
 
-        if self.pokydirname is None and os.path.exists(os.path.join(self.be.sourcedir, "oe-init-build-env")):
-            logger.debug("localhostbecontroller: selected poky dir name %s" % self.be.sourcedir)
-            self.pokydirname = self.be.sourcedir
+        # Resolve self.pokydirname if not resolved yet, consider the scenario
+        # where all layers are local, that's the else clause
+        if self.pokydirname is None:
+            if os.path.exists(os.path.join(self.be.sourcedir, "oe-init-build-env")):
+                logger.debug("localhostbecontroller: selected poky dir name %s" % self.be.sourcedir)
+                self.pokydirname = self.be.sourcedir
+            else:
+                # Alternatively, scan local layers for relative "oe-init-build-env" location
+                for layer in layers:
+                    if os.path.exists(os.path.join(layer.layer_version.layer.local_source_dir,"..","oe-init-build-env")):
+                        logger.debug("localhostbecontroller, setting pokydirname to %s" % (layer.layer_version.layer.local_source_dir))
+                        self.pokydirname = os.path.join(layer.layer_version.layer.local_source_dir,"..")
+                        break
+                else:
+                    logger.error("pokydirname is not set, you will run into trouble!")
 
         # 5. create custom layer and add custom recipes to it
         for target in targets:
@@ -232,7 +249,7 @@ class LocalhostBEController(BuildEnvironmentController):
                     customrecipe, layers)
 
                 if os.path.isdir(custom_layer_path):
-                    layerlist.append(custom_layer_path)
+                    layerlist.append("%03d:%s" % (layer_index,custom_layer_path))
 
             except CustomImageRecipe.DoesNotExist:
                 continue  # not a custom recipe, skip
@@ -240,7 +257,11 @@ class LocalhostBEController(BuildEnvironmentController):
         layerlist.extend(nongitlayerlist)
         logger.debug("\n\nset layers gives this list %s" % pformat(layerlist))
         self.islayerset = True
-        return layerlist
+
+        # restore the order of layer list for bblayers.conf
+        layerlist.sort()
+        sorted_layerlist = [l[4:] for l in layerlist]
+        return sorted_layerlist
 
     def setup_custom_image_recipe(self, customrecipe, layers):
         """ Set up toaster-custom-images layer and recipe files """
@@ -310,41 +331,141 @@ class LocalhostBEController(BuildEnvironmentController):
 
     def triggerBuild(self, bitbake, layers, variables, targets, brbe):
         layers = self.setLayers(bitbake, layers, targets)
+        is_merged_attr = bitbake.req.project.merged_attr
+
+        git_env = os.environ.copy()
+        # (note: add custom environment settings here)
+        try:
+            # insure that the project init/build uses the selected bitbake, and not Toaster's
+            del git_env['TEMPLATECONF']
+            del git_env['BBBASEDIR']
+            del git_env['BUILDDIR']
+        except KeyError:
+            pass
 
         # init build environment from the clone
-        builddir = '%s-toaster-%d' % (self.be.builddir, bitbake.req.project.id)
+        if bitbake.req.project.builddir:
+            builddir = bitbake.req.project.builddir
+        else:
+            builddir = '%s-toaster-%d' % (self.be.builddir, bitbake.req.project.id)
         oe_init = os.path.join(self.pokydirname, 'oe-init-build-env')
         # init build environment
         try:
             custom_script = ToasterSetting.objects.get(name="CUSTOM_BUILD_INIT_SCRIPT").value
             custom_script = custom_script.replace("%BUILDDIR%" ,builddir)
-            self._shellcmd("bash -c 'source %s'" % (custom_script))
+            self._shellcmd("bash -c 'source %s'" % (custom_script),env=git_env)
         except ToasterSetting.DoesNotExist:
             self._shellcmd("bash -c 'source %s %s'" % (oe_init, builddir),
-                       self.be.sourcedir)
+                       self.be.sourcedir,env=git_env)
 
         # update bblayers.conf
-        bblconfpath = os.path.join(builddir, "conf/toaster-bblayers.conf")
-        with open(bblconfpath, 'w') as bblayers:
-            bblayers.write('# line added by toaster build control\n'
-                           'BBLAYERS = "%s"' % ' '.join(layers))
+        if not is_merged_attr:
+            bblconfpath = os.path.join(builddir, "conf/toaster-bblayers.conf")
+            with open(bblconfpath, 'w') as bblayers:
+                bblayers.write('# line added by toaster build control\n'
+                               'BBLAYERS = "%s"' % ' '.join(layers))
 
-        # write configuration file
-        confpath = os.path.join(builddir, 'conf/toaster.conf')
-        with open(confpath, 'w') as conf:
-            for var in variables:
-                conf.write('%s="%s"\n' % (var.name, var.value))
-            conf.write('INHERIT+="toaster buildhistory"')
+            # write configuration file
+            confpath = os.path.join(builddir, 'conf/toaster.conf')
+            with open(confpath, 'w') as conf:
+                for var in variables:
+                    conf.write('%s="%s"\n' % (var.name, var.value))
+                conf.write('INHERIT+="toaster buildhistory"')
+        else:
+            # Append the Toaster-specific values directly to the bblayers.conf
+            bblconfpath = os.path.join(builddir, "conf/bblayers.conf")
+            bblconfpath_save = os.path.join(builddir, "conf/bblayers.conf.save")
+            shutil.copyfile(bblconfpath, bblconfpath_save)
+            with open(bblconfpath) as bblayers:
+                content = bblayers.readlines()
+            do_write = True
+            was_toaster = False
+            with open(bblconfpath,'w') as bblayers:
+                for line in content:
+                    #line = line.strip('\n')
+                    if 'TOASTER_CONFIG_PROLOG' in line:
+                        do_write = False
+                        was_toaster = True
+                    elif 'TOASTER_CONFIG_EPILOG' in line:
+                        do_write = True
+                    elif do_write:
+                        bblayers.write(line)
+                if not was_toaster:
+                    bblayers.write('\n')
+                bblayers.write('#=== TOASTER_CONFIG_PROLOG ===\n')
+                bblayers.write('BBLAYERS = "\\\n')
+                for layer in layers:
+                    bblayers.write('  %s \\\n' % layer)
+                bblayers.write('  "\n')
+                bblayers.write('#=== TOASTER_CONFIG_EPILOG ===\n')
+            # Append the Toaster-specific values directly to the local.conf
+            bbconfpath = os.path.join(builddir, "conf/local.conf")
+            bbconfpath_save = os.path.join(builddir, "conf/local.conf.save")
+            shutil.copyfile(bbconfpath, bbconfpath_save)
+            with open(bbconfpath) as f:
+                content = f.readlines()
+            do_write = True
+            was_toaster = False
+            with open(bbconfpath,'w') as conf:
+                for line in content:
+                    #line = line.strip('\n')
+                    if 'TOASTER_CONFIG_PROLOG' in line:
+                        do_write = False
+                        was_toaster = True
+                    elif 'TOASTER_CONFIG_EPILOG' in line:
+                        do_write = True
+                    elif do_write:
+                        conf.write(line)
+                if not was_toaster:
+                    conf.write('\n')
+                conf.write('#=== TOASTER_CONFIG_PROLOG ===\n')
+                for var in variables:
+                    if (not var.name.startswith("INTERNAL_")) and (not var.name == "BBLAYERS"):
+                        conf.write('%s="%s"\n' % (var.name, var.value))
+                conf.write('#=== TOASTER_CONFIG_EPILOG ===\n')
+
+        # If 'target' is just the project preparation target, then we are done
+        for target in targets:
+            if "_PROJECT_PREPARE_" == target.target:
+                logger.debug('localhostbecontroller: Project has been prepared. Done.')
+                # Update the Build Request and release the build environment
+                bitbake.req.state = BuildRequest.REQ_COMPLETED
+                bitbake.req.save()
+                self.be.lock = BuildEnvironment.LOCK_FREE
+                self.be.save()
+                # Close the project build and progress bar
+                bitbake.req.build.outcome = Build.SUCCEEDED
+                bitbake.req.build.save()
+                # Update the project status
+                bitbake.req.project.set_variable(Project.PROJECT_SPECIFIC_STATUS,Project.PROJECT_SPECIFIC_CLONING_SUCCESS)
+                signal_runbuilds()
+                return
 
         # clean the Toaster to build environment
         env_clean = 'unset BBPATH;' # clean BBPATH for <= YP-2.4.0
 
-        # run bitbake server from the clone
+        # run bitbake server from the clone if available
+        # otherwise pick it from the PATH
         bitbake = os.path.join(self.pokydirname, 'bitbake', 'bin', 'bitbake')
+        if not os.path.exists(bitbake):
+            logger.info("Bitbake not available under %s, will try to use it from PATH" %
+                        self.pokydirname)
+            for path in os.environ["PATH"].split(os.pathsep):
+                if os.path.exists(os.path.join(path, 'bitbake')):
+                    bitbake = os.path.join(path, 'bitbake')
+                    break
+            else:
+                logger.error("Looks like Bitbake is not available, please fix your environment")
+
         toasterlayers = os.path.join(builddir,"conf/toaster-bblayers.conf")
-        self._shellcmd('%s bash -c \"source %s %s; BITBAKE_UI="knotty" %s --read %s --read %s '
-                       '--server-only -B 0.0.0.0:0\"' % (env_clean, oe_init,
-                       builddir, bitbake, confpath, toasterlayers), self.be.sourcedir)
+        if not is_merged_attr:
+            self._shellcmd('%s bash -c \"source %s %s; BITBAKE_UI="knotty" %s --read %s --read %s '
+                           '--server-only -B 0.0.0.0:0\"' % (env_clean, oe_init,
+                           builddir, bitbake, confpath, toasterlayers), self.be.sourcedir)
+        else:
+            self._shellcmd('%s bash -c \"source %s %s; BITBAKE_UI="knotty" %s '
+                           '--server-only -B 0.0.0.0:0\"' % (env_clean, oe_init,
+                           builddir, bitbake), self.be.sourcedir)
 
         # read port number from bitbake.lock
         self.be.bbport = -1
@@ -390,12 +511,20 @@ class LocalhostBEController(BuildEnvironmentController):
         log = os.path.join(builddir, 'toaster_ui.log')
         local_bitbake = os.path.join(os.path.dirname(os.getenv('BBBASEDIR')),
                                      'bitbake')
-        self._shellcmd(['%s bash -c \"(TOASTER_BRBE="%s" BBSERVER="0.0.0.0:%s" '
+        if not is_merged_attr:
+            self._shellcmd(['%s bash -c \"(TOASTER_BRBE="%s" BBSERVER="0.0.0.0:%s" '
                         '%s %s -u toasterui  --read %s --read %s --token="" >>%s 2>&1;'
                         'BITBAKE_UI="knotty" BBSERVER=0.0.0.0:%s %s -m)&\"' \
                         % (env_clean, brbe, self.be.bbport, local_bitbake, bbtargets, confpath, toasterlayers, log,
                         self.be.bbport, bitbake,)],
                         builddir, nowait=True)
+        else:
+            self._shellcmd(['%s bash -c \"(TOASTER_BRBE="%s" BBSERVER="0.0.0.0:%s" '
+                        '%s %s -u toasterui  --token="" >>%s 2>&1;'
+                        'BITBAKE_UI="knotty" BBSERVER=0.0.0.0:%s %s -m)&\"' \
+                        % (env_clean, brbe, self.be.bbport, local_bitbake, bbtargets, log,
+                        self.be.bbport, bitbake,)],
+                        builddir, nowait=True)
 
         logger.debug('localhostbecontroller: Build launched, exiting. '
                      'Follow build logs at %s' % log)

bitbake/lib/toaster/bldcontrol/management/commands/checksettings.py

@@ -74,8 +74,9 @@ class Command(BaseCommand):
                         print("Loading default settings")
                         call_command("loaddata", "settings")
                         template_conf = os.environ.get("TEMPLATECONF", "")
+                        custom_xml_only = os.environ.get("CUSTOM_XML_ONLY")
 
-                        if ToasterSetting.objects.filter(name='CUSTOM_XML_ONLY').count() > 0:
+                        if ToasterSetting.objects.filter(name='CUSTOM_XML_ONLY').count() > 0 or (not custom_xml_only == None):
                             # only use the custom settings
                             pass
                         elif "poky" in template_conf:

bitbake/lib/toaster/bldcontrol/management/commands/runbuilds.py

@@ -49,7 +49,7 @@ class Command(BaseCommand):
                 # we could not find a BEC; postpone the BR
                 br.state = BuildRequest.REQ_QUEUED
                 br.save()
-                logger.debug("runbuilds: No build env")
+                logger.debug("runbuilds: No build env (%s)" % e)
                 return
 
             logger.info("runbuilds: starting build %s, environment %s" %

bitbake/lib/toaster/orm/management/commands/lsupdates.py

@@ -29,7 +29,6 @@ from orm.models import ToasterSetting
 import os
 import sys
 
-import json
 import logging
 import threading
 import time
@@ -37,6 +36,18 @@ logger = logging.getLogger("toaster")
 
 DEFAULT_LAYERINDEX_SERVER = "http://layers.openembedded.org/layerindex/api/"
 
+# Add path to bitbake modules for layerindexlib
+# lib/toaster/orm/management/commands/lsupdates.py (abspath)
+# lib/toaster/orm/management/commands (dirname)
+# lib/toaster/orm/management (dirname)
+# lib/toaster/orm (dirname)
+# lib/toaster/ (dirname)
+# lib/ (dirname)
+path = os.path.dirname(os.path.dirname(os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))))
+sys.path.insert(0, path)
+
+import layerindexlib
+
 
 class Spinner(threading.Thread):
     """ A simple progress spinner to indicate download/parsing is happening"""
@@ -86,45 +97,6 @@ class Command(BaseCommand):
             self.apiurl = ToasterSetting.objects.get(name = 'CUSTOM_LAYERINDEX_SERVER').value
 
         assert self.apiurl is not None
-        try:
-            from urllib.request import urlopen, URLError
-            from urllib.parse import urlparse
-        except ImportError:
-            from urllib2 import urlopen, URLError
-            from urlparse import urlparse
-
-        proxy_settings = os.environ.get("http_proxy", None)
-
-        def _get_json_response(apiurl=None):
-            if None == apiurl:
-                apiurl=self.apiurl
-            http_progress = Spinner()
-            http_progress.start()
-
-            _parsedurl = urlparse(apiurl)
-            path = _parsedurl.path
-
-            # logger.debug("Fetching %s", apiurl)
-            try:
-                res = urlopen(apiurl)
-            except URLError as e:
-                raise Exception("Failed to read %s: %s" % (path, e.reason))
-
-            parsed = json.loads(res.read().decode('utf-8'))
-
-            http_progress.stop()
-            return parsed
-
-        # verify we can get the basic api
-        try:
-            apilinks = _get_json_response()
-        except Exception as e:
-            import traceback
-            if proxy_settings is not None:
-                logger.info("EE: Using proxy %s" % proxy_settings)
-            logger.warning("EE: could not connect to %s, skipping update:"
-                           "%s\n%s" % (self.apiurl, e, traceback.format_exc()))
-            return
 
         # update branches; only those that we already have names listed in the
         # Releases table
@@ -133,112 +105,118 @@ class Command(BaseCommand):
         if len(whitelist_branch_names) == 0:
             raise Exception("Failed to make list of branches to fetch")
 
-        logger.info("Fetching metadata releases for %s",
+        logger.info("Fetching metadata for %s",
                     " ".join(whitelist_branch_names))
 
-        branches_info = _get_json_response(apilinks['branches'] +
-                                           "?filter=name:%s"
-                                           % "OR".join(whitelist_branch_names))
+        # We require a non-empty bb.data, but we can fake it with a dictionary
+        layerindex = layerindexlib.LayerIndex({"DUMMY" : "VALUE"})
+
+        http_progress = Spinner()
+        http_progress.start()
+
+        if whitelist_branch_names:
+            url_branches = ";branch=%s" % ','.join(whitelist_branch_names)
+        else:
+            url_branches = ""
+        layerindex.load_layerindex("%s%s" % (self.apiurl, url_branches))
+
+        http_progress.stop()
+
+        # We know we're only processing one entry, so we reference it here
+        # (this is cheating...)
+        index = layerindex.indexes[0]
 
         # Map the layer index branches to toaster releases
         li_branch_id_to_toaster_release = {}
 
-        total = len(branches_info)
-        for i, branch in enumerate(branches_info):
-            li_branch_id_to_toaster_release[branch['id']] = \
-                    Release.objects.get(name=branch['name'])
+        logger.info("Processing releases")
+
+        total = len(index.branches)
+        for i, id in enumerate(index.branches):
+            li_branch_id_to_toaster_release[id] = \
+                    Release.objects.get(name=index.branches[id].name)
             self.mini_progress("Releases", i, total)
 
         # keep a track of the layerindex (li) id mappings so that
         # layer_versions can be created for these layers later on
         li_layer_id_to_toaster_layer_id = {}
 
-        logger.info("Fetching layers")
+        logger.info("Processing layers")
 
-        layers_info = _get_json_response(apilinks['layerItems'])
-
-        total = len(layers_info)
-        for i, li in enumerate(layers_info):
+        total = len(index.layerItems)
+        for i, id in enumerate(index.layerItems):
             try:
-                l, created = Layer.objects.get_or_create(name=li['name'])
-                l.up_date = li['updated']
-                l.summary = li['summary']
-                l.description = li['description']
+                l, created = Layer.objects.get_or_create(name=index.layerItems[id].name)
+                l.up_date = index.layerItems[id].updated
+                l.summary = index.layerItems[id].summary
+                l.description = index.layerItems[id].description
 
                 if created:
                     # predefined layers in the fixtures (for example poky.xml)
                     # always preempt the Layer Index for these values
-                    l.vcs_url = li['vcs_url']
-                    l.vcs_web_url = li['vcs_web_url']
-                    l.vcs_web_tree_base_url = li['vcs_web_tree_base_url']
-                    l.vcs_web_file_base_url = li['vcs_web_file_base_url']
+                    l.vcs_url = index.layerItems[id].vcs_url
+                    l.vcs_web_url = index.layerItems[id].vcs_web_url
+                    l.vcs_web_tree_base_url = index.layerItems[id].vcs_web_tree_base_url
+                    l.vcs_web_file_base_url = index.layerItems[id].vcs_web_file_base_url
                 l.save()
             except Layer.MultipleObjectsReturned:
                 logger.info("Skipped %s as we found multiple layers and "
                             "don't know which to update" %
-                            li['name'])
+                            index.layerItems[id].name)
 
-            li_layer_id_to_toaster_layer_id[li['id']] = l.pk
+            li_layer_id_to_toaster_layer_id[id] = l.pk
 
             self.mini_progress("layers", i, total)
 
         # update layer_versions
-        logger.info("Fetching layer versions")
-        layerbranches_info = _get_json_response(
-            apilinks['layerBranches'] + "?filter=branch__name:%s" %
-            "OR".join(whitelist_branch_names))
+        logger.info("Processing layer versions")
 
         # Map Layer index layer_branch object id to
         # layer_version toaster object id
         li_layer_branch_id_to_toaster_lv_id = {}
 
-        total = len(layerbranches_info)
-        for i, lbi in enumerate(layerbranches_info):
+        total = len(index.layerBranches)
+        for i, id in enumerate(index.layerBranches):
             # release as defined by toaster map to layerindex branch
-            release = li_branch_id_to_toaster_release[lbi['branch']]
+            release = li_branch_id_to_toaster_release[index.layerBranches[id].branch_id]
 
             try:
                 lv, created = Layer_Version.objects.get_or_create(
                     layer=Layer.objects.get(
-                        pk=li_layer_id_to_toaster_layer_id[lbi['layer']]),
+                        pk=li_layer_id_to_toaster_layer_id[index.layerBranches[id].layer_id]),
                     release=release
                 )
             except KeyError:
                 logger.warning(
                     "No such layerindex layer referenced by layerbranch %d" %
-                    lbi['layer'])
+                    index.layerBranches[id].layer_id)
                 continue
 
             if created:
-                lv.release = li_branch_id_to_toaster_release[lbi['branch']]
-                lv.up_date = lbi['updated']
-                lv.commit = lbi['actual_branch']
-                lv.dirpath = lbi['vcs_subdir']
+                lv.release = li_branch_id_to_toaster_release[index.layerBranches[id].branch_id]
+                lv.up_date = index.layerBranches[id].updated
+                lv.commit = index.layerBranches[id].actual_branch
+                lv.dirpath = index.layerBranches[id].vcs_subdir
                 lv.save()
 
-            li_layer_branch_id_to_toaster_lv_id[lbi['id']] =\
+            li_layer_branch_id_to_toaster_lv_id[index.layerBranches[id].id] =\
                 lv.pk
             self.mini_progress("layer versions", i, total)
 
-        logger.info("Fetching layer version dependencies")
-        # update layer dependencies
-        layerdependencies_info = _get_json_response(
-            apilinks['layerDependencies'] +
-            "?filter=layerbranch__branch__name:%s" %
-            "OR".join(whitelist_branch_names))
+        logger.info("Processing layer version dependencies")
 
         dependlist = {}
-        for ldi in layerdependencies_info:
+        for id in index.layerDependencies:
             try:
                 lv = Layer_Version.objects.get(
-                    pk=li_layer_branch_id_to_toaster_lv_id[ldi['layerbranch']])
+                    pk=li_layer_branch_id_to_toaster_lv_id[index.layerDependencies[id].layerbranch_id])
             except Layer_Version.DoesNotExist as e:
                 continue
 
             if lv not in dependlist:
                 dependlist[lv] = []
             try:
-                layer_id = li_layer_id_to_toaster_layer_id[ldi['dependency']]
+                layer_id = li_layer_id_to_toaster_layer_id[index.layerDependencies[id].dependency_id]
 
                 dependlist[lv].append(
                     Layer_Version.objects.get(layer__pk=layer_id,
@@ -247,7 +225,7 @@ class Command(BaseCommand):
             except Layer_Version.DoesNotExist:
                 logger.warning("Cannot find layer version (ls:%s),"
                                "up_id:%s lv:%s" %
-                               (self, ldi['dependency'], lv))
+                               (self, index.layerDependencies[id].dependency_id, lv))
 
         total = len(dependlist)
         for i, lv in enumerate(dependlist):
@@ -258,73 +236,61 @@ class Command(BaseCommand):
             self.mini_progress("Layer version dependencies", i, total)
 
         # update Distros
-        logger.info("Fetching distro information")
-        distros_info = _get_json_response(
-            apilinks['distros'] + "?filter=layerbranch__branch__name:%s" %
-            "OR".join(whitelist_branch_names))
+        logger.info("Processing distro information")
 
-        total = len(distros_info)
-        for i, di in enumerate(distros_info):
+        total = len(index.distros)
+        for i, id in enumerate(index.distros):
             distro, created = Distro.objects.get_or_create(
-                name=di['name'],
+                name=index.distros[id].name,
                 layer_version=Layer_Version.objects.get(
-                    pk=li_layer_branch_id_to_toaster_lv_id[di['layerbranch']]))
-            distro.up_date = di['updated']
-            distro.name = di['name']
-            distro.description = di['description']
+                    pk=li_layer_branch_id_to_toaster_lv_id[index.distros[id].layerbranch_id]))
+            distro.up_date = index.distros[id].updated
+            distro.name = index.distros[id].name
+            distro.description = index.distros[id].description
             distro.save()
             self.mini_progress("distros", i, total)
 
         # update machines
-        logger.info("Fetching machine information")
-        machines_info = _get_json_response(
-            apilinks['machines'] + "?filter=layerbranch__branch__name:%s" %
-            "OR".join(whitelist_branch_names))
+        logger.info("Processing machine information")
 
-        total = len(machines_info)
-        for i, mi in enumerate(machines_info):
+        total = len(index.machines)
+        for i, id in enumerate(index.machines):
             mo, created = Machine.objects.get_or_create(
-                name=mi['name'],
+                name=index.machines[id].name,
                 layer_version=Layer_Version.objects.get(
-                    pk=li_layer_branch_id_to_toaster_lv_id[mi['layerbranch']]))
-            mo.up_date = mi['updated']
-            mo.name = mi['name']
-            mo.description = mi['description']
+                    pk=li_layer_branch_id_to_toaster_lv_id[index.machines[id].layerbranch_id]))
+            mo.up_date = index.machines[id].updated
+            mo.name = index.machines[id].name
+            mo.description = index.machines[id].description
             mo.save()
             self.mini_progress("machines", i, total)
 
         # update recipes; paginate by layer version / layer branch
-        logger.info("Fetching recipe information")
-        recipes_info = _get_json_response(
-            apilinks['recipes'] + "?filter=layerbranch__branch__name:%s" %
-            "OR".join(whitelist_branch_names))
+        logger.info("Processing recipe information")
 
-        total = len(recipes_info)
-        for i, ri in enumerate(recipes_info):
+        total = len(index.recipes)
+        for i, id in enumerate(index.recipes):
             try:
-                lv_id = li_layer_branch_id_to_toaster_lv_id[ri['layerbranch']]
+                lv_id = li_layer_branch_id_to_toaster_lv_id[index.recipes[id].layerbranch_id]
                 lv = Layer_Version.objects.get(pk=lv_id)
 
                 ro, created = Recipe.objects.get_or_create(
                     layer_version=lv,
-                    name=ri['pn']
+                    name=index.recipes[id].pn
                 )
 
                 ro.layer_version = lv
-                ro.up_date = ri['updated']
-                ro.name = ri['pn']
-                ro.version = ri['pv']
-                ro.summary = ri['summary']
-                ro.description = ri['description']
-                ro.section = ri['section']
-                ro.license = ri['license']
-                ro.homepage = ri['homepage']
-                ro.bugtracker = ri['bugtracker']
-                ro.file_path = ri['filepath'] + "/" + ri['filename']
-                if 'inherits' in ri:
-                    ro.is_image = 'image' in ri['inherits'].split()
-                else:  # workaround for old style layer index
-                    ro.is_image = "-image-" in ri['pn']
+                ro.up_date = index.recipes[id].updated
+                ro.name = index.recipes[id].pn
+                ro.version = index.recipes[id].pv
+                ro.summary = index.recipes[id].summary
+                ro.description = index.recipes[id].description
+                ro.section = index.recipes[id].section
+                ro.license = index.recipes[id].license
+                ro.homepage = index.recipes[id].homepage
+                ro.bugtracker = index.recipes[id].bugtracker
+                ro.file_path = index.recipes[id].fullpath
+                ro.is_image = 'image' in index.recipes[id].inherits.split()
                 ro.save()
             except Exception as e:
                 logger.warning("Failed saving recipe %s", e)

bitbake/lib/toaster/orm/migrations/0018_project_specific.py

@@ -0,0 +1,28 @@
+# -*- coding: utf-8 -*-
+from __future__ import unicode_literals
+
+from django.db import migrations, models
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('orm', '0017_distro_clone'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='Project',
+            name='builddir',
+            field=models.TextField(),
+        ),
+        migrations.AddField(
+            model_name='Project',
+            name='merged_attr',
+            field=models.BooleanField(default=False)
+        ),
+        migrations.AddField(
+            model_name='Build',
+            name='progress_item',
+            field=models.CharField(max_length=40)
+        ),
+    ]

bitbake/lib/toaster/orm/models.py

@@ -121,8 +121,15 @@ class ToasterSetting(models.Model):
 
 
 class ProjectManager(models.Manager):
-    def create_project(self, name, release):
-        if release is not None:
+    def create_project(self, name, release, existing_project=None):
+        if existing_project and (release is not None):
+            prj = existing_project
+            prj.bitbake_version = release.bitbake_version
+            prj.release = release
+            # Delete the previous ProjectLayer mappings
+            for pl in ProjectLayer.objects.filter(project=prj):
+                pl.delete()
+        elif release is not None:
             prj = self.model(name=name,
                              bitbake_version=release.bitbake_version,
                              release=release)
@@ -130,15 +137,14 @@ class ProjectManager(models.Manager):
             prj = self.model(name=name,
                              bitbake_version=None,
                              release=None)
-
         prj.save()
 
         for defaultconf in ToasterSetting.objects.filter(
                 name__startswith="DEFCONF_"):
             name = defaultconf.name[8:]
-            ProjectVariable.objects.create(project=prj,
-                                           name=name,
-                                           value=defaultconf.value)
+            pv,create = ProjectVariable.objects.get_or_create(project=prj,name=name)
+            pv.value = defaultconf.value
+            pv.save()
 
         if release is None:
             return prj
@@ -197,6 +203,11 @@ class Project(models.Model):
     user_id = models.IntegerField(null=True)
     objects = ProjectManager()
 
+    # build directory override (e.g. imported)
+    builddir = models.TextField()
+    # merge the Toaster configure attributes directly into the standard conf files
+    merged_attr = models.BooleanField(default=False)
+
     # set to True for the project which is the default container
     # for builds initiated by the command line etc.
     is_default= models.BooleanField(default=False)
@@ -305,6 +316,15 @@ class Project(models.Model):
             return layer_versions
 
 
+    def get_default_image_recipe(self):
+        try:
+            return self.projectvariable_set.get(name="DEFAULT_IMAGE").value
+        except (ProjectVariable.DoesNotExist,IndexError):
+            return None;
+
+    def get_is_new(self):
+        return self.get_variable(Project.PROJECT_SPECIFIC_ISNEW)
+
     def get_available_machines(self):
         """ Returns QuerySet of all Machines which are provided by the
         Layers currently added to the Project """
@@ -353,6 +373,32 @@ class Project(models.Model):
 
         return queryset
 
+    # Project Specific status management
+    PROJECT_SPECIFIC_STATUS = 'INTERNAL_PROJECT_SPECIFIC_STATUS'
+    PROJECT_SPECIFIC_CALLBACK = 'INTERNAL_PROJECT_SPECIFIC_CALLBACK'
+    PROJECT_SPECIFIC_ISNEW = 'INTERNAL_PROJECT_SPECIFIC_ISNEW'
+    PROJECT_SPECIFIC_DEFAULTIMAGE = 'PROJECT_SPECIFIC_DEFAULTIMAGE'
+    PROJECT_SPECIFIC_NONE = ''
+    PROJECT_SPECIFIC_NEW = '1'
+    PROJECT_SPECIFIC_EDIT = '2'
+    PROJECT_SPECIFIC_CLONING = '3'
+    PROJECT_SPECIFIC_CLONING_SUCCESS = '4'
+    PROJECT_SPECIFIC_CLONING_FAIL = '5'
+
+    def get_variable(self,variable,default_value = ''):
+        try:
+            return self.projectvariable_set.get(name=variable).value
+        except (ProjectVariable.DoesNotExist,IndexError):
+            return default_value
+
+    def set_variable(self,variable,value):
+        pv,create = ProjectVariable.objects.get_or_create(project = self, name = variable)
+        pv.value = value
+        pv.save()
+
+    def get_default_image(self):
+        return self.get_variable(Project.PROJECT_SPECIFIC_DEFAULTIMAGE)
+
     def schedule_build(self):
 
         from bldcontrol.models import BuildRequest, BRTarget, BRLayer
@@ -459,6 +505,9 @@ class Build(models.Model):
     # number of repos cloned so far for this build (default off)
     repos_cloned = models.IntegerField(default=1)
 
+    # Hint on current progress item
+    progress_item = models.CharField(max_length=40)
+
     @staticmethod
     def get_recent(project=None):
         """
@@ -1663,6 +1712,9 @@ class CustomImageRecipe(Recipe):
 
         path_schema_two = self.base_recipe.file_path
 
+        path_schema_three = "%s/%s" % (self.base_recipe.layer_version.layer.local_source_dir,
+                                     self.base_recipe.file_path)
+
         if os.path.exists(path_schema_one):
             return path_schema_one
 
@@ -1670,6 +1722,10 @@ class CustomImageRecipe(Recipe):
         if os.path.exists(path_schema_two):
             return path_schema_two
 
+        # Or a local path if all layers are local
+        if os.path.exists(path_schema_three):
+            return path_schema_three
+
         return None
 
     def generate_recipe_file_contents(self):

bitbake/lib/toaster/toastergui/api.py

@@ -22,7 +22,9 @@ import os
 import re
 import logging
 import json
+import subprocess
 from collections import Counter
+from shutil import copyfile
 
 from orm.models import Project, ProjectTarget, Build, Layer_Version
 from orm.models import LayerVersionDependency, LayerSource, ProjectLayer
@@ -38,6 +40,18 @@ from django.core.urlresolvers import reverse
 from django.db.models import Q, F
 from django.db import Error
 from toastergui.templatetags.projecttags import filtered_filesizeformat
+from django.utils import timezone
+import pytz
+
+# development/debugging support
+verbose = 2
+def _log(msg):
+    if 1 == verbose:
+        print(msg)
+    elif 2 == verbose:
+        f1=open('/tmp/toaster.log', 'a')
+        f1.write("|" + msg + "|\n" )
+        f1.close()
 
 logger = logging.getLogger("toaster")
 
@@ -137,6 +151,130 @@ class XhrBuildRequest(View):
         return response
 
 
+class XhrProjectUpdate(View):
+
+    def get(self, request, *args, **kwargs):
+        return HttpResponse()
+
+    def post(self, request, *args, **kwargs):
+        """
+          Project Update
+
+          Entry point: /xhr_projectupdate/<project_id>
+          Method: POST
+
+          Args:
+              pid: pid of project to update
+
+          Returns:
+              {"error": "ok"}
+            or
+              {"error": <error message>}
+        """
+
+        project = Project.objects.get(pk=kwargs['pid'])
+        logger.debug("ProjectUpdateCallback:project.pk=%d,project.builddir=%s" % (project.pk,project.builddir))
+
+        if 'do_update' in request.POST:
+
+            # Extract any default image recipe
+            if 'default_image' in request.POST:
+                project.set_variable(Project.PROJECT_SPECIFIC_DEFAULTIMAGE,str(request.POST['default_image']))
+            else:
+                project.set_variable(Project.PROJECT_SPECIFIC_DEFAULTIMAGE,'')
+
+            logger.debug("ProjectUpdateCallback:Chain to the build request")
+
+            # Chain to the build request
+            xhrBuildRequest = XhrBuildRequest()
+            return xhrBuildRequest.post(request, *args, **kwargs)
+
+        logger.warning("ERROR:XhrProjectUpdate")
+        response = HttpResponse()
+        response.status_code = 500
+        return response
+
+class XhrSetDefaultImageUrl(View):
+
+    def get(self, request, *args, **kwargs):
+        return HttpResponse()
+
+    def post(self, request, *args, **kwargs):
+        """
+          Project Update
+
+          Entry point: /xhr_setdefaultimage/<project_id>
+          Method: POST
+
+          Args:
+              pid: pid of project to update default image
+
+          Returns:
+              {"error": "ok"}
+            or
+              {"error": <error message>}
+        """
+
+        project = Project.objects.get(pk=kwargs['pid'])
+        logger.debug("XhrSetDefaultImageUrl:project.pk=%d" % (project.pk))
+
+        # set any default image recipe
+        if 'targets' in request.POST:
+            default_target = str(request.POST['targets'])
+            project.set_variable(Project.PROJECT_SPECIFIC_DEFAULTIMAGE,default_target)
+            logger.debug("XhrSetDefaultImageUrl,project.pk=%d,project.builddir=%s" % (project.pk,project.builddir))
+            return error_response('ok')
+
+        logger.warning("ERROR:XhrSetDefaultImageUrl")
+        response = HttpResponse()
+        response.status_code = 500
+        return response
+
+
+#
+# Layer Management
+#
+# Rules for 'local_source_dir' layers
+#  * Layers must have a unique name in the Layers table
+#  * A 'local_source_dir' layer is supposed to be shared
+#    by all projects that use it, so that it can have the
+#    same logical name
+#  * Each project that uses a layer will have its own
+#    LayerVersion and Project Layer for it
+#  * During the Paroject delete process, when the last
+#    LayerVersion for a 'local_source_dir' layer is deleted
+#    then the Layer record is deleted to remove orphans
+#
+
+def scan_layer_content(layer,layer_version):
+    # if this is a local layer directory, we can immediately scan its content
+    if layer.local_source_dir:
+        try:
+            # recipes-*/*/*.bb
+            cmd = '%s %s' % ('ls', os.path.join(layer.local_source_dir,'recipes-*/*/*.bb'))
+            recipes_list = subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE,stderr=subprocess.STDOUT).stdout.read()
+            recipes_list = recipes_list.decode("utf-8").strip()
+            if recipes_list and 'No such' not in recipes_list:
+                for recipe in recipes_list.split('\n'):
+                    recipe_path = recipe[recipe.rfind('recipes-'):]
+                    recipe_name = recipe[recipe.rfind('/')+1:].replace('.bb','')
+                    recipe_ver = recipe_name.rfind('_')
+                    if recipe_ver > 0:
+                        recipe_name = recipe_name[0:recipe_ver]
+                    if recipe_name:
+                        ro, created = Recipe.objects.get_or_create(
+                            layer_version=layer_version,
+                            name=recipe_name
+                        )
+                        if created:
+                            ro.file_path = recipe_path
+                            ro.summary = 'Recipe %s from layer %s' % (recipe_name,layer.name)
+                            ro.description = ro.summary
+                        ro.save()
+
+        except Exception as e:
+            logger.warning("ERROR:scan_layer_content: %s" % e)
+
 class XhrLayer(View):
     """ Delete, Get, Add and Update Layer information
 
@@ -265,6 +403,7 @@ class XhrLayer(View):
             (csv)]
 
         """
+
         try:
             project = Project.objects.get(pk=kwargs['pid'])
 
@@ -285,7 +424,13 @@ class XhrLayer(View):
             if layer_data['name'] in existing_layers:
                 return JsonResponse({"error": "layer-name-exists"})
 
-            layer = Layer.objects.create(name=layer_data['name'])
+            if ('local_source_dir' in layer_data):
+                # Local layer can be shared across projects. They have no 'release'
+                # and are not included in get_all_compatible_layer_versions() above
+                layer,created = Layer.objects.get_or_create(name=layer_data['name'])
+                _log("Local Layer created=%s" % created)
+            else:
+                layer = Layer.objects.create(name=layer_data['name'])
 
             layer_version = Layer_Version.objects.create(
                 layer=layer,
@@ -293,7 +438,7 @@ class XhrLayer(View):
                 layer_source=LayerSource.TYPE_IMPORTED)
 
             # Local layer
-            if ('local_source_dir' in layer_data) and layer.local_source_dir:
+            if ('local_source_dir' in layer_data): ### and layer.local_source_dir:
                 layer.local_source_dir = layer_data['local_source_dir']
             # git layer
             elif 'vcs_url' in layer_data:
@@ -325,6 +470,9 @@ class XhrLayer(View):
                              'layerdetailurl':
                              layer_dep.get_detailspage_url(project.pk)})
 
+            # Scan the layer's content and update components
+            scan_layer_content(layer,layer_version)
+
         except Layer_Version.DoesNotExist:
             return error_response("layer-dep-not-found")
         except Project.DoesNotExist:
@@ -1014,8 +1162,24 @@ class XhrProject(View):
                     state=BuildRequest.REQ_INPROGRESS):
                 XhrBuildRequest.cancel_build(br)
 
+            # gather potential orphaned local layers attached to this project
+            project_local_layer_list = []
+            for pl in ProjectLayer.objects.filter(project=project):
+                if pl.layercommit.layer_source == LayerSource.TYPE_IMPORTED:
+                    project_local_layer_list.append(pl.layercommit.layer)
+
+            # deep delete the project and its dependencies
             project.delete()
 
+            # delete any local layers now orphaned
+            _log("LAYER_ORPHAN_CHECK:Check for orphaned layers")
+            for layer in project_local_layer_list:
+                layer_refs = Layer_Version.objects.filter(layer=layer)
+                _log("LAYER_ORPHAN_CHECK:Ref Count for '%s' = %d" % (layer.name,len(layer_refs)))
+                if 0 == len(layer_refs):
+                    _log("LAYER_ORPHAN_CHECK:DELETE orpahned '%s'" % (layer.name))
+                    Layer.objects.filter(pk=layer.id).delete()
+
         except Project.DoesNotExist:
             return error_response("Project %s does not exist" %
                                   kwargs['project_id'])

bitbake/lib/toaster/toastergui/static/js/layerBtn.js

@@ -67,6 +67,18 @@ function layerBtnsInit() {
       });
   });
 
+  $("td .set-default-recipe-btn").unbind('click');
+  $("td .set-default-recipe-btn").click(function(e){
+    e.preventDefault();
+    var recipe = $(this).data('recipe-name');
+
+    libtoaster.setDefaultImage(null, recipe,
+      function(){
+        /* Success */
+        window.location.replace(libtoaster.ctx.projectSpecificPageUrl);
+      });
+  });
+
 
   $(".customise-btn").unbind('click');
   $(".customise-btn").click(function(e){

bitbake/lib/toaster/toastergui/static/js/layerdetails.js

@@ -359,7 +359,8 @@ function layerDetailsPageInit (ctx) {
     if ($(this).is("dt")) {
       var dd = $(this).next("dd");
       if (!dd.children("form:visible")|| !dd.find(".current-value").html()){
-        if (ctx.layerVersion.layer_source == ctx.layerSourceTypes.TYPE_IMPORTED){
+        if (ctx.layerVersion.layer_source == ctx.layerSourceTypes.TYPE_IMPORTED ||
+            ctx.layerVersion.layer_source == ctx.layerSourceTypes.TYPE_LOCAL) {
         /* There's no current value and the layer is editable
          * so show the "Not set" and hide the delete icon
          */

bitbake/lib/toaster/toastergui/static/js/libtoaster.js

@@ -465,6 +465,108 @@ var libtoaster = (function () {
     $.cookie('toaster-notification', JSON.stringify(data), { path: '/'});
   }
 
+  /* _updateProject:
+   * url: xhrProjectUpdateUrl or null for current project
+   * onsuccess: callback for successful execution
+   * onfail: callback for failed execution
+   */
+  function _updateProject (url, targets, default_image, onsuccess, onfail) {
+
+    if (!url)
+      url = libtoaster.ctx.xhrProjectUpdateUrl;
+
+    /* Flatten the array of targets into a space spearated list */
+    if (targets instanceof Array){
+      targets = targets.reduce(function(prevV, nextV){
+        return prev + ' ' + next;
+      });
+    }
+
+    $.ajax( {
+        type: "POST",
+        url: url,
+        data: { 'do_update' : 'True' , 'targets' : targets , 'default_image' : default_image , },
+        headers: { 'X-CSRFToken' : $.cookie('csrftoken')},
+        success: function (_data) {
+          if (_data.error !== "ok") {
+            console.warn(_data.error);
+          } else {
+            if (onsuccess !== undefined) onsuccess(_data);
+          }
+        },
+        error: function (_data) {
+          console.warn("Call failed");
+          console.warn(_data);
+          if (onfail) onfail(data);
+    } });
+  }
+
+  /* _cancelProject:
+   * url: xhrProjectUpdateUrl or null for current project
+   * onsuccess: callback for successful execution
+   * onfail: callback for failed execution
+   */
+  function _cancelProject (url, onsuccess, onfail) {
+
+    if (!url)
+      url = libtoaster.ctx.xhrProjectCancelUrl;
+
+    $.ajax( {
+        type: "POST",
+        url: url,
+        data: { 'do_cancel' : 'True'  },
+        headers: { 'X-CSRFToken' : $.cookie('csrftoken')},
+        success: function (_data) {
+          if (_data.error !== "ok") {
+            console.warn(_data.error);
+          } else {
+            if (onsuccess !== undefined) onsuccess(_data);
+          }
+        },
+        error: function (_data) {
+          console.warn("Call failed");
+          console.warn(_data);
+          if (onfail) onfail(data);
+    } });
+  }
+
+  /* _setDefaultImage:
+   * url: xhrSetDefaultImageUrl or null for current project
+   * targets: an array or space separated list of targets to set as default
+   * onsuccess: callback for successful execution
+   * onfail: callback for failed execution
+   */
+  function _setDefaultImage (url, targets, onsuccess, onfail) {
+
+    if (!url)
+      url = libtoaster.ctx.xhrSetDefaultImageUrl;
+
+    /* Flatten the array of targets into a space spearated list */
+    if (targets instanceof Array){
+      targets = targets.reduce(function(prevV, nextV){
+        return prev + ' ' + next;
+      });
+    }
+
+    $.ajax( {
+        type: "POST",
+        url: url,
+        data: { 'targets' : targets },
+        headers: { 'X-CSRFToken' : $.cookie('csrftoken')},
+        success: function (_data) {
+          if (_data.error !== "ok") {
+            console.warn(_data.error);
+          } else {
+            if (onsuccess !== undefined) onsuccess(_data);
+          }
+        },
+        error: function (_data) {
+          console.warn("Call failed");
+          console.warn(_data);
+          if (onfail) onfail(data);
+    } });
+  }
+
   return {
     enableAjaxLoadingTimer: _enableAjaxLoadingTimer,
     disableAjaxLoadingTimer: _disableAjaxLoadingTimer,
@@ -485,6 +587,9 @@ var libtoaster = (function () {
     createCustomRecipe: _createCustomRecipe,
     makeProjectNameValidation: _makeProjectNameValidation,
     setNotification: _setNotification,
+    updateProject : _updateProject,
+    cancelProject : _cancelProject,
+    setDefaultImage : _setDefaultImage,
   };
 })();
 

bitbake/lib/toaster/toastergui/static/js/mrbsection.js

@@ -86,7 +86,7 @@ function mrbSectionInit(ctx){
           if (buildFinished(build)) {
             // a build finished: reload the whole page so that the build
             // shows up in the builds table
-            window.location.reload();
+            window.location.reload(true);
           }
           else if (stateChanged(build)) {
             // update the whole template
@@ -110,6 +110,8 @@ function mrbSectionInit(ctx){
             // update the clone progress text
             selector = '#repos-cloned-percentage-' + build.id;
             $(selector).html(build.repos_cloned_percentage);
+            selector = '#repos-cloned-progressitem-' + build.id;
+            $(selector).html('('+build.progress_item+')');
 
             // update the recipe progress bar
             selector = '#repos-cloned-percentage-bar-' + build.id;

bitbake/lib/toaster/toastergui/static/js/projecttopbar.js

@@ -14,6 +14,9 @@ function projectTopBarInit(ctx) {
   var newBuildTargetBuildBtn = $("#build-button");
   var selectedTarget;
 
+  var updateProjectBtn = $("#update-project-button");
+  var cancelProjectBtn = $("#cancel-project-button");
+
   /* Project name change functionality */
   projectNameFormToggle.click(function(e){
     e.preventDefault();
@@ -89,6 +92,25 @@ function projectTopBarInit(ctx) {
     }, null);
   });
 
+  updateProjectBtn.click(function (e) {
+    e.preventDefault();
+
+    selectedTarget = { name: "_PROJECT_PREPARE_" };
+
+    /* Save current default build image, fire off the build */
+    libtoaster.updateProject(null, selectedTarget.name, newBuildTargetInput.val().trim(),
+      function(){
+        window.location.replace(libtoaster.ctx.projectSpecificPageUrl);
+    }, null);
+  });
+
+  cancelProjectBtn.click(function (e) {
+    e.preventDefault();
+
+    /* redirect to 'done/canceled' landing page */
+    window.location.replace(libtoaster.ctx.landingSpecificCancelURL);
+  });
+
   /* Call makeProjectNameValidation function */
   libtoaster.makeProjectNameValidation($("#project-name-change-input"),
       $("#hint-error-project-name"), $("#validate-project-name"),

bitbake/lib/toaster/toastergui/tables.py

@@ -35,6 +35,8 @@ from toastergui.tablefilter import TableFilterActionToggle
 from toastergui.tablefilter import TableFilterActionDateRange
 from toastergui.tablefilter import TableFilterActionDay
 
+import os
+
 class ProjectFilters(object):
     @staticmethod
     def in_project(project_layers):
@@ -339,6 +341,8 @@ class RecipesTable(ToasterTable):
             'filter_name' : "in_current_project",
             'static_data_name' : "add-del-layers",
             'static_data_template' : '{% include "recipe_btn.html" %}'}
+    if '1' == os.environ.get('TOASTER_PROJECTSPECIFIC'):
+            build_col['static_data_template'] = '{% include "recipe_add_btn.html" %}'
 
     def get_context_data(self, **kwargs):
         project = Project.objects.get(pk=kwargs['pid'])

bitbake/lib/toaster/toastergui/templates/base_specific.html

@@ -0,0 +1,128 @@
+<!DOCTYPE html>
+{% load static %}
+{% load projecttags %}
+{% load project_url_tag %}
+<html lang="en">
+  <head>
+    <title>
+      {% block title %} Toaster {% endblock %}
+    </title>
+    <link rel="stylesheet" href="{% static 'css/bootstrap.min.css' %}" type="text/css"/>
+    <!--link rel="stylesheet" href="{% static 'css/bootstrap-theme.css' %}" type="text/css"/-->
+    <link rel="stylesheet" href="{% static 'css/font-awesome.min.css' %}" type='text/css'/>
+    <link rel="stylesheet" href="{% static 'css/default.css' %}" type='text/css'/>
+
+    <meta name="viewport" content="width=device-width, initial-scale=1.0" />
+    <meta http-equiv="Content-Type" content="text/html;charset=UTF-8" />
+    <script src="{% static 'js/jquery-2.0.3.min.js' %}">
+    </script>
+    <script src="{% static 'js/jquery.cookie.js' %}">
+    </script>
+    <script src="{% static 'js/bootstrap.min.js' %}">
+    </script>
+    <script src="{% static 'js/typeahead.jquery.js' %}">
+    </script>
+    <script src="{% static 'js/jsrender.min.js' %}">
+    </script>
+    <script src="{% static 'js/highlight.pack.js' %}">
+    </script>
+    <script src="{% static 'js/libtoaster.js' %}">
+    </script>
+    {% if DEBUG %}
+    <script>
+      libtoaster.debug = true;
+    </script>
+    {% endif %}
+    <script>
+      /* Set JsRender delimiters (mrb_section.html) different than Django's */
+      $.views.settings.delimiters("<%", "%>");
+
+      /* This table allows Django substitutions to be passed to libtoaster.js */
+      libtoaster.ctx = {
+        jsUrl : "{% static 'js/' %}",
+        htmlUrl : "{% static 'html/' %}",
+        projectsUrl : "{% url 'all-projects' %}",
+        projectsTypeAheadUrl: {% url 'xhr_projectstypeahead' as prjurl%}{{prjurl|json}},
+        {% if project.id %}
+        landingSpecificURL : "{% url 'landing_specific' project.id %}",
+        landingSpecificCancelURL : "{% url 'landing_specific_cancel' project.id %}",
+        projectId : {{project.id}},
+        projectPageUrl : {% url 'project' project.id as purl %}{{purl|json}},
+        projectSpecificPageUrl : {% url 'project_specific' project.id as purl %}{{purl|json}},
+        xhrProjectUrl : {% url 'xhr_project' project.id as pxurl %}{{pxurl|json}},
+        projectName : {{project.name|json}},
+        recipesTypeAheadUrl: {% url 'xhr_recipestypeahead' project.id as paturl%}{{paturl|json}},
+        layersTypeAheadUrl: {% url 'xhr_layerstypeahead' project.id as paturl%}{{paturl|json}},
+        machinesTypeAheadUrl: {% url 'xhr_machinestypeahead' project.id as paturl%}{{paturl|json}},
+        distrosTypeAheadUrl: {% url 'xhr_distrostypeahead' project.id as paturl%}{{paturl|json}},
+        projectBuildsUrl: {% url 'projectbuilds' project.id as pburl %}{{pburl|json}},
+        xhrCustomRecipeUrl : "{% url 'xhr_customrecipe' %}",
+        projectId : {{project.id}},
+        xhrBuildRequestUrl: "{% url 'xhr_buildrequest' project.id %}",
+        mostRecentBuildsUrl: "{% url 'most_recent_builds' %}?project_id={{project.id}}",
+        xhrProjectUpdateUrl: "{% url 'xhr_projectupdate' project.id %}",
+        xhrProjectCancelUrl: "{% url 'landing_specific_cancel' project.id %}",
+        xhrSetDefaultImageUrl: "{% url 'xhr_setdefaultimage' project.id %}",
+        {% else %}
+        mostRecentBuildsUrl: "{% url 'most_recent_builds' %}",
+        projectId : undefined,
+        projectPageUrl : undefined,
+        projectName : undefined,
+        {% endif %}
+      };
+    </script>
+    {% block extraheadcontent %}
+    {% endblock %}
+  </head>
+
+  <body>
+
+    {% csrf_token %}
+    <div id="loading-notification" class="alert alert-warning lead text-center" style="display:none">
+      Loading <i class="fa-pulse icon-spinner"></i>
+    </div>
+
+    <div id="change-notification" class="alert alert-info alert-dismissible change-notification" style="display:none">
+      <button type="button" class="close" id="hide-alert" data-toggle="alert">&times;</button>
+      <span id="change-notification-msg"></span>
+    </div>
+
+    <nav class="navbar navbar-default navbar-fixed-top">
+      <div class="container-fluid">
+        <div class="navbar-header">
+          <button type="button" class="navbar-toggle collapsed" data-toggle="collapse" data-target="#global-nav" aria-expanded="false">
+            <span class="sr-only">Toggle navigation</span>
+            <span class="icon-bar"></span>
+            <span class="icon-bar"></span>
+            <span class="icon-bar"></span>
+          </button>
+          <div class="toaster-navbar-brand">
+          	{% if project_specific %}
+              <img class="logo" src="{% static 'img/logo.png' %}" class="" alt="Yocto Project logo"/>
+              Toaster
+          	{% else %}
+            <a href="/">
+            </a>
+            <a href="/">
+              <img class="logo" src="{% static 'img/logo.png' %}" class="" alt="Yocto Project logo"/>
+            </a>
+            <a class="brand" href="/">Toaster</a>
+            {% endif %}
+            {% if DEBUG %}
+            <span class="glyphicon glyphicon-info-sign" title="<strong>Toaster version information</strong>" data-content="<dl><dt>Git branch</dt><dd>{{TOASTER_BRANCH}}</dd><dt>Git revision</dt><dd>{{TOASTER_REVISION}}</dd></dl>"></i>
+            {% endif %}
+          </div>
+        </div>
+        <div class="collapse navbar-collapse" id="global-nav">
+            <ul class="nav navbar-nav">
+            <h3> Project Configuration Page </h3>
+        </div>
+      </div>
+    </nav>
+
+    <div class="container-fluid">
+      {% block pagecontent %}
+      {% endblock %}
+    </div>
+  </body>
+</html>

bitbake/lib/toaster/toastergui/templates/baseprojectspecificpage.html

@@ -0,0 +1,48 @@
+{% extends "base_specific.html" %}
+
+{% load projecttags %}
+{% load humanize %}
+
+{% block title %} {{title}} - {{project.name}} - Toaster {% endblock %}
+
+{% block pagecontent %}
+
+<div class="row">
+  {% include "project_specific_topbar.html" %}
+  <script type="text/javascript">
+$(document).ready(function(){
+    $("#config-nav .nav li a").each(function(){
+      if (window.location.pathname === $(this).attr('href'))
+      $(this).parent().addClass('active');
+      else
+      $(this).parent().removeClass('active');
+      });
+
+    $("#topbar-configuration-tab").addClass("active")
+    });
+  </script>
+
+  <!-- only on config pages -->
+  <div id="config-nav" class="col-md-2">
+    <ul class="nav nav-pills nav-stacked">
+      <li><a class="nav-parent" href="{% url 'project' project.id %}">Configuration</a></li>
+      <li class="nav-header">Compatible metadata</li>
+      <li><a href="{% url 'projectcustomimages' project.id %}">Custom images</a></li>
+      <li><a href="{% url 'projectimagerecipes' project.id %}">Image recipes</a></li>
+      <li><a href="{% url 'projectsoftwarerecipes' project.id %}">Software recipes</a></li>
+      <li><a href="{% url 'projectmachines' project.id %}">Machines</a></li>
+      <li><a href="{% url 'projectlayers' project.id %}">Layers</a></li>
+      <li><a href="{% url 'projectdistros' project.id %}">Distros</a></li>
+      <li class="nav-header">Extra configuration</li>
+      <li><a href="{% url 'projectconf' project.id %}">BitBake variables</a></li>
+
+      <li class="nav-header">Actions</li>
+    </ul>
+  </div>
+  <div class="col-md-10">
+    {% block projectinfomain %}{% endblock %}
+  </div>
+
+</div>
+{% endblock %}
+

bitbake/lib/toaster/toastergui/templates/generic-toastertable-page.html

@@ -1,4 +1,4 @@
-{% extends "baseprojectpage.html" %}
+{% extends project_specific|yesno:"baseprojectspecificpage.html,baseprojectpage.html" %}
 {% load projecttags %}
 {% load humanize %}
 {% load static %}

bitbake/lib/toaster/toastergui/templates/importlayer.html

@@ -1,4 +1,4 @@
-{% extends "base.html" %}
+{% extends project_specific|yesno:"baseprojectspecificpage.html,base.html" %}
 {% load projecttags %}
 {% load humanize %}
 {% load static %}
@@ -6,7 +6,7 @@
 {% block pagecontent %}
 
 <div class="row">
-  {% include "projecttopbar.html" %}
+  {% include project_specific|yesno:"project_specific_topbar.html,projecttopbar.html" %}
   {% if project and project.release %}
   <script src="{% static 'js/layerDepsModal.js' %}"></script>
   <script src="{% static 'js/importlayer.js' %}"></script>

bitbake/lib/toaster/toastergui/templates/landing_specific.html

@@ -0,0 +1,50 @@
+{% extends "base_specific.html" %}
+
+{% load static %}
+{% load projecttags %}
+{% load humanize %}
+
+{% block title %} Welcome to Toaster {% endblock %}
+
+{% block pagecontent %}
+
+  <div class="container">
+   <div class="row">
+    <!-- Empty - no build module -->
+    <div class="page-header top-air">
+     <h1>
+       Configuration {% if status == "cancel" %}Canceled{% else %}Completed{% endif %}! You can now close this window.
+     </h1>
+    </div>
+    <div class="alert alert-info lead">
+     <p>
+     Your project configuration {% if status == "cancel" %}changes have been canceled{% else %}has completed!{% endif %}
+     <br>
+     <br>
+     <ul>
+       <li>
+       The Toaster instance for project configuration has been shut down
+       </li>
+       <li>
+       You can start Toaster independently for advanced project management and analysis:
+         <pre><code>
+         Set up bitbake environment:
+         $ cd {{install_dir}}
+         $ . oe-init-build-env [toaster_server]
+
+         Option 1: Start a local Toaster server, open local browser to "localhost:8000"
+         $ . toaster start webport=8000
+
+         Option 2: Start a shared Toaster server, open any browser to "[host_ip]:8000"
+         $ . toaster start webport=0.0.0.0:8000
+
+         To stop the Toaster server:
+         $ . toaster stop
+         </code></pre>
+       </li>
+     </ul>
+     </p>
+    </div>
+   </div>
+
+{% endblock %}

bitbake/lib/toaster/toastergui/templates/layerdetails.html

@@ -1,4 +1,4 @@
-{% extends "base.html" %}
+{% extends project_specific|yesno:"baseprojectspecificpage.html,base.html" %}
 {% load projecttags %}
 {% load humanize %}
 {% load static %}
@@ -310,6 +310,7 @@
             {% endwith %}
             {% endwith %}
           </div>
+
         </div> <!-- end tab content -->
       </div> <!-- end tabable -->
 

bitbake/lib/toaster/toastergui/templates/mrb_section.html

@@ -119,7 +119,7 @@
           title="Toaster is cloning the repos required for your build">
     </span>
 
-    Cloning <span id="repos-cloned-percentage-<%:id%>"><%:repos_cloned_percentage%></span>% complete
+    Cloning <span id="repos-cloned-percentage-<%:id%>"><%:repos_cloned_percentage%></span>% complete <span id="repos-cloned-progressitem-<%:id%>">(<%:progress_item%>)</span>
 
     <%include tmpl='#cancel-template'/%>
   </div>

bitbake/lib/toaster/toastergui/templates/newcustomimage.html

@@ -1,4 +1,4 @@
-{% extends "base.html" %}
+{% extends project_specific|yesno:"baseprojectspecificpage.html,base.html" %}
 {% load projecttags %}
 {% load humanize %}
 {% load static %}
@@ -8,7 +8,7 @@
 
 <div class="row">
 
-  {% include "projecttopbar.html" %}
+  {% include project_specific|yesno:"project_specific_topbar.html,projecttopbar.html" %}
 
   <div class="col-md-12">
     {% url table_name project.id as xhr_table_url %}

bitbake/lib/toaster/toastergui/templates/newproject.html

@@ -20,23 +20,19 @@
             <input type="text" class="form-control" required id="new-project-name" name="projectname">
           </div>
           <p class="help-block text-danger" style="display: none;" id="hint-error-project-name">A project with this name exists. Project names must be unique.</p>
-<!--
-            <fieldset>
-                <label class="project-form">Project type</label>
-                    <label class="project-form radio"><input type="radio" name="ptype" value="analysis" checked/> Analysis Project</label>
 
+                <label class="project-form">Project type:</label>
                 {% if releases.count > 0 %}
-                    <label class="project-form radio"><input type="radio" name="ptype" value="build" checked /> Build Project</label>
+                    <label class="project-form radio" style="padding-left: 35px;"><input id='type-new'    type="radio" name="ptype" value="new"/> New project</label>
                 {% endif %}
-              </fieldset> -->
-        <input type="hidden" name="ptype" value="build" />
+                    <label class="project-form radio" style="padding-left: 35px;"><input id='type-import' type="radio" name="ptype" value="import"/> Import command line project</label>
 
         {% if releases.count > 0 %}
-				<div class="release form-group">
+          <div class="release form-group">
             {% if releases.count > 1 %}
               <label class="control-label">
                 Release
-                <span class="glyphicon glyphicon-question-sign get-help" title="The version of the build system you want to use"></span>
+                <span class="glyphicon glyphicon-question-sign get-help" title="The version of the build system you want to use for this project"></span>
               </label>
               <select name="projectversion" id="projectversion" class="form-control">
                 {% for release in releases %}
@@ -54,33 +50,31 @@
                     <span class="help-block">{{release.helptext|safe}}</span>
                   </div>
                 {% endfor %}
+                </div>
+              </div>
             {% else %}
               <input type="hidden" name="projectversion" value="{{releases.0.id}}"/>
             {% endif %}
-                </div>
-              </div>
-            </fieldset>
+
+              <input type="checkbox" class="checkbox-mergeattr" name="mergeattr" value="mergeattr">  Merged Toaster settings (Command line user compatibility)
+              <span class="glyphicon glyphicon-question-sign get-help" title="Place the Toaster settings into the standard 'local.conf' and 'bblayers.conf' instead of 'toaster_bblayers.conf' and 'toaster.conf'"></span>
+
+           </div>
         {% endif %}
+
+            <div class="build-import form-group" id="import-project">
+              <label class="control-label">Import existing project directory
+                <span class="glyphicon glyphicon-question-sign get-help" title="Enter a path to an existing build directory, import the existing settings, and create a Toaster Project for it."></span>
+              </label>
+              <input style="width: 33%;"type="text" class="form-control" required id="import-project-dir" name="importdir">
+            </div>
+
             <div class="top-air">
               <input type="submit" id="create-project-button" class="btn btn-primary btn-lg" value="Create project"/>
               <span class="help-inline" style="vertical-align:middle;">To create a project, you need to enter a project name</span>
             </div>
 
         </form>
-        <!--
-        <div class="col-md-5 well">
-                <span class="help-block">
-                 <h4>Toaster project types</h4>
-                 <p>With a <strong>build project</strong> you configure and run your builds from Toaster.</p>
-                 <p>With an <strong>analysis project</strong>, the builds are configured and run by another tool
-                 (something like Buildbot or Jenkins), and the project only collects the information about the
-                 builds (packages, recipes, dependencies, logs, etc). </p>
-                 <p>You can read more on <a href="#">how to set up an analysis project</a>
-                 in the Toaster manual.</p>
-                 <h4>Release</h4>
-                 <p>If you create a <strong>build project</strong>, you will need to select a <strong>release</strong>,
-                 which is the version of the build system you want to use to run your builds.</p>
-             </div> -->
       </div>
     </div>
 
@@ -89,6 +83,7 @@
             // hide the new project button
             $("#new-project-button").hide();
             $('.btn-primary').attr('disabled', 'disabled');
+            $('#type-new').attr('checked', 'checked');
 
             // enable submit button when all required fields are populated
             $("input#new-project-name").on('input', function() {
@@ -118,20 +113,24 @@
                  $(".btn-primary"));
 
 
-/*			// Hide the project release when you select an analysis project
+			// Hide the project release when you select an analysis project
 			function projectType() {
-				if ($("input[type='radio']:checked").val() == 'build') {
+				if ($("input[type='radio']:checked").val() == 'new') {
+					$('.build-import').fadeOut();
 					$('.release').fadeIn();
+                    $('#import-project-dir').removeAttr('required');
 				}
 				else {
 					$('.release').fadeOut();
+					$('.build-import').fadeIn();
+                    $('#import-project-dir').attr('required', 'required');
 				}
 			}
 			projectType();
 
 			$('input:radio').change(function(){
 				projectType();
-			}); */
+			});
         });
     </script>
 

bitbake/lib/toaster/toastergui/templates/newproject_specific.html

@@ -0,0 +1,95 @@
+{% extends "base.html" %}
+{% load projecttags %}
+{% load humanize %}
+
+{% block title %} Create a new project - Toaster {% endblock %}
+
+{% block pagecontent %}
+<div class="row">
+  <div class="col-md-12">
+    <div class="page-header">
+          <h1>Create a new project</h1>
+        </div>
+    {% if alert %}
+      <div class="alert alert-danger" role="alert">{{alert}}</div>
+    {% endif %}
+
+        <form method="POST"  action="{%url "newproject_specific" project_pk %}">{% csrf_token %}
+          <div class="form-group" id="validate-project-name">
+            <label class="control-label">Project name <span class="text-muted">(required)</span></label>
+            <input type="text" class="form-control" required id="new-project-name" name="display_projectname" value="{{projectname}}" disabled>
+          </div>
+          <p class="help-block text-danger" style="display: none;" id="hint-error-project-name">A project with this name exists. Project names must be unique.</p>
+        <input type="hidden" name="ptype" value="build" />
+        <input type="hidden" name="projectname" value="{{projectname}}" />
+
+        {% if releases.count > 0 %}
+				<div class="release form-group">
+            {% if releases.count > 1 %}
+              <label class="control-label">
+                Release
+                <span class="glyphicon glyphicon-question-sign get-help" title="The version of the build system you want to use"></span>
+              </label>
+              <select name="projectversion" id="projectversion" class="form-control">
+                {% for release in releases %}
+                    <option value="{{release.id}}"
+                        {%if defaultbranch == release.name %}
+                            selected
+                        {%endif%}
+                     >{{release.description}}</option>
+                {% endfor %}
+              </select>
+              <div class="row">
+                <div class="col-md-4">
+                {% for release in releases %}
+                  <div class="helptext" id="description-{{release.id}}" style="display: none">
+                    <span class="help-block">{{release.helptext|safe}}</span>
+                  </div>
+                {% endfor %}
+            {% else %}
+              <input type="hidden" name="projectversion" value="{{releases.0.id}}"/>
+            {% endif %}
+                </div>
+              </div>
+            </fieldset>
+        {% endif %}
+            <div class="top-air">
+              <input type="submit" id="create-project-button" class="btn btn-primary btn-lg" value="Create project"/>
+              <span class="help-inline" style="vertical-align:middle;">To create a project, you need to specify the release</span>
+            </div>
+
+        </form>
+      </div>
+    </div>
+
+    <script type="text/javascript">
+        $(document).ready(function () {
+            // hide the new project button, name is preset
+            $("#new-project-button").hide();
+
+            // enable submit button when all required fields are populated
+            $("input#new-project-name").on('input', function() {
+                if ($("input#new-project-name").val().length > 0 ){
+                    $('.btn-primary').removeAttr('disabled');
+                    $(".help-inline").css('visibility','hidden');
+                }
+                else {
+                    $('.btn-primary').attr('disabled', 'disabled');
+                    $(".help-inline").css('visibility','visible');
+                }
+            });
+
+            // show relevant help text for the selected release
+            var selected_release = $('select').val();
+            $("#description-" + selected_release).show();
+
+            $('select').change(function(){
+                var new_release = $('select').val();
+                $(".helptext").hide();
+                $('#description-' + new_release).fadeIn();
+            });
+
+        });
+    </script>
+
+{% endblock %}

bitbake/lib/toaster/toastergui/templates/project.html

@@ -1,4 +1,4 @@
-{% extends "baseprojectpage.html" %}
+{% extends project_specific|yesno:"baseprojectspecificpage.html,baseprojectpage.html" %}
 
 {% load projecttags %}
 {% load humanize %}
@@ -18,7 +18,7 @@
     try {
       projectPageInit(ctx);
     } catch (e) {
-      document.write("Sorry, An error has occurred loading this page");
+      document.write("Sorry, An error has occurred loading this page (project):"+e);
       console.warn(e);
     }
   });
@@ -93,6 +93,7 @@
       </form>
     </div>
 
+	{% if not project_specific %}
     <div class="well well-transparent">
       <h3>Most built recipes</h3>
 
@@ -105,6 +106,7 @@
       </ul>
       <button class="btn btn-primary" id="freq-build-btn" disabled="disabled">Build selected recipes</button>
     </div>
+    {% endif %}
 
     <div class="well well-transparent">
       <h3>Project release</h3>
@@ -157,5 +159,6 @@
       <ul class="list-unstyled lead" id="layers-in-project-list">
       </ul>
   </div>
+
 </div>
 {% endblock %}

bitbake/lib/toaster/toastergui/templates/project_specific.html

@@ -0,0 +1,162 @@
+{% extends "baseprojectspecificpage.html" %}
+
+{% load projecttags %}
+{% load humanize %}
+{% load static %}
+
+{% block title %} Configuration - {{project.name}} - Toaster {% endblock %}
+{% block projectinfomain %}
+
+<script src="{% static 'js/layerDepsModal.js' %}"></script>
+<script src="{% static 'js/projectpage.js' %}"></script>
+<script>
+  $(document).ready(function (){
+    var ctx = {
+      testReleaseChangeUrl: "{% url 'xhr_testreleasechange' project.id %}",
+    };
+
+    try {
+      projectPageInit(ctx);
+    } catch (e) {
+      document.write("Sorry, An error has occurred loading this page");
+      console.warn(e);
+    }
+  });
+</script>
+
+<div id="delete-project-modal" class="modal fade" tabindex="-1" role="dialog" data-backdrop="static" data-keyboard="false">
+  <div class="modal-dialog">
+    <div class="modal-content">
+      <div class="modal-header">
+        <h4>Are you sure you want to delete this project?</h4>
+      </div>
+      <div class="modal-body">
+        <p>Deleting the <strong class="project-name"></strong> project
+        will:</p>
+        <ul>
+          <li>Cancel its builds currently in progress</li>
+          <li>Remove its configuration information</li>
+          <li>Remove its imported layers</li>
+          <li>Remove its custom images</li>
+          <li>Remove all its build information</li>
+        </ul>
+      </div>
+      <div class="modal-footer">
+        <button type="button" class="btn btn-primary" id="delete-project-confirmed">
+          <span data-role="submit-state">Delete project</span>
+          <span data-role="loading-state" style="display:none">
+            <span class="fa-pulse">
+            <i class="fa-pulse icon-spinner"></i>
+          </span>
+            &nbsp;Deleting project...
+          </span>
+        </button>
+        <button type="button" class="btn btn-link" data-dismiss="modal">Cancel</button>
+      </div>
+    </div><!-- /.modal-content -->
+  </div><!-- /.modal-dialog -->
+</div>
+
+
+<div class="row" id="project-page" style="display:none">
+  <div class="col-md-6">
+    <div class="well well-transparent" id="machine-section">
+      <h3>Machine</h3>
+
+      <p class="lead"><span id="project-machine-name"></span> <span class="glyphicon glyphicon-edit" id="change-machine-toggle"></span></p>
+
+      <form id="select-machine-form" style="display:none;" class="form-inline">
+        <span class="help-block">Machine suggestions come from the list of layers added to your project. If you don't see the machine you are looking for, <a href="{% url 'projectmachines' project.id %}">check the full list of machines</a></span>
+        <div class="form-group" id="machine-input-form">
+          <input class="form-control" id="machine-change-input" autocomplete="off" value="" data-provide="typeahead" data-minlength="1" data-autocomplete="off" type="text">
+        </div>
+        <button id="machine-change-btn" class="btn btn-default" type="button">Save</button>
+        <a href="#" id="cancel-machine-change" class="btn btn-link">Cancel</a>
+        <span class="help-block text-danger" id="invalid-machine-name-help" style="display:none">A valid machine name cannot include spaces.</span>
+        <p class="form-link"><a href="{% url 'projectmachines' project.id %}">View compatible machines</a></p>
+      </form>
+    </div>
+
+    <div class="well well-transparent" id="distro-section">
+      <h3>Distro</h3>
+
+      <p class="lead"><span id="project-distro-name"></span> <span class="glyphicon glyphicon-edit" id="change-distro-toggle"></span></p>
+
+      <form id="select-distro-form" style="display:none;" class="form-inline">
+        <span class="help-block">Distro suggestions come from the Layer Index</a></span>
+        <div class="form-group">
+          <input class="form-control" id="distro-change-input" autocomplete="off" value="" data-provide="typeahead" data-minlength="1" data-autocomplete="off" type="text">
+        </div>
+        <button id="distro-change-btn" class="btn btn-default" type="button">Save</button>
+        <a href="#" id="cancel-distro-change" class="btn btn-link">Cancel</a>
+        <p class="form-link"><a href="{% url 'projectdistros' project.id %}">View compatible distros</a></p>
+      </form>
+    </div>
+
+    <div class="well well-transparent">
+      <h3>Most built recipes</h3>
+
+      <div class="alert alert-info" style="display:none" id="no-most-built">
+        <h4>You haven't built any recipes yet</h4>
+        <p class="form-link"><a href="{% url 'projectimagerecipes' project.id %}">Choose a recipe to build</a></p>
+      </div>
+
+      <ul class="list-unstyled lead" id="freq-build-list">
+      </ul>
+      <button class="btn btn-primary" id="freq-build-btn" disabled="disabled">Build selected recipes</button>
+    </div>
+
+    <div class="well well-transparent">
+      <h3>Project release</h3>
+
+      <p class="lead"><span id="project-release-title"></span>
+
+      <!-- Comment out the ability to change the project release, until we decide what to do with this functionality -->
+
+      <!--i title="" data-original-title="" id="release-change-toggle" class="icon-pencil"></i-->
+      </p>
+
+      <!-- Comment out the ability to change the project release, until we decide what to do with this functionality -->
+
+      <!--form class="form-inline" id="change-release-form" style="display:none;">
+        <select></select>
+        <button class="btn" style="margin-left:5px;" id="change-release-btn">Change</button> <a href="#" id="cancel-release-change" class="btn btn-link">Cancel</a>
+      </form-->
+    </div>
+  </div>
+
+  <div class="col-md-6">
+    <div class="well well-transparent" id="layer-container">
+      <h3>Layers <span class="counter">(<span id="project-layers-count"></span>)</span>
+        <span title="OpenEmbedded organises recipes and machines into thematic groups called <strong>layers</strong>. Click on a layer name to see the recipes and machines it includes." class="glyphicon glyphicon-question-sign get-help"></span>
+      </h3>
+
+      <div class="alert alert-warning" id="no-layers-in-project" style="display:none">
+        <h4>This project has no layers</h4>
+        In order to build this project you need to add some layers first. For that you can:
+        <ul>
+          <li><a href="{% url 'projectlayers' project.id %}">Choose from the layers compatible with this project</a></li>
+          <li><a href="{% url 'importlayer' project.id %}">Import a layer</a></li>
+          <li><a href="http://www.yoctoproject.org/docs/current/dev-manual/dev-manual.html#understanding-and-creating-layers" target="_blank">Read about layers in the documentation</a></li>
+          <li>Or type a layer name below</li>
+        </ul>
+      </div>
+
+      <form class="form-inline">
+        <div class="form-group">
+          <input id="layer-add-input" class="form-control" autocomplete="off" placeholder="Type a layer name" data-minlength="1" data-autocomplete="off" data-provide="typeahead" data-source="" type="text">
+        </div>
+        <button id="add-layer-btn" class="btn btn-default" disabled>Add layer</button>
+        <p class="form-link">
+          <a href="{% url 'projectlayers' project.id %}" id="view-compatible-layers">View compatible layers</a>
+          <span class="text-muted">|</span>
+          <a href="{% url 'importlayer' project.id %}">Import layer</a>
+        </p>
+      </form>
+
+      <ul class="list-unstyled lead" id="layers-in-project-list">
+      </ul>
+  </div>
+
+</div>
+{% endblock %}

bitbake/lib/toaster/toastergui/templates/project_specific_topbar.html

@@ -0,0 +1,80 @@
+{% load static %}
+<script src="{% static 'js/projecttopbar.js' %}"></script>
+<script>
+  $(document).ready(function () {
+    var ctx = {
+      numProjectLayers : {{project.get_project_layer_versions.count}},
+      machine : "{{project.get_current_machine_name|default_if_none:""}}",
+    }
+
+    try {
+      projectTopBarInit(ctx);
+    } catch (e) {
+      document.write("Sorry, An error has occurred loading this page (pstb):"+e);
+      console.warn(e);
+    }
+  });
+</script>
+
+<div class="col-md-12">
+  <div class="alert alert-success alert-dismissible change-notification" id="project-created-notification" style="display:none">
+    <button type="button" class="close" data-dismiss="alert">&times;</button>
+		<p>Your project <strong>{{project.name}}</strong> has been created. You can now <a class="alert-link" href="{% url 'projectmachines' project.id %}">select your target machine</a> and <a class="alert-link" href="{% url 'projectimagerecipes' project.id %}">choose image recipes</a> to build.</p>
+  </div>
+  <!-- project name -->
+  <div class="page-header">
+    <h1 id="project-name-container">
+      <span class="project-name">{{project.name}}</span>
+      {% if project.is_default %}
+      <span class="glyphicon glyphicon-question-sign get-help" title="This project shows information about the builds you start from the command line while Toaster is running"></span>
+      {% endif %}
+    </h1>
+    <form id="project-name-change-form" class="form-inline" style="display: none;">
+      <div class="form-group">
+        <input class="form-control input-lg" type="text" id="project-name-change-input" autocomplete="off" value="{{project.name}}">
+      </div>
+      <button id="project-name-change-btn" class="btn btn-default btn-lg" type="button">Save</button>
+      <a href="#" id="project-name-change-cancel" class="btn btn-lg btn-link">Cancel</a>
+    </form>
+  </div>
+
+  {% with mrb_type='project' %}
+    {% include "mrb_section.html" %}
+  {% endwith %}
+
+  {% if not project.is_default %}
+  <div id="project-topbar">
+    <ul class="nav nav-tabs">
+      <li id="topbar-configuration-tab">
+      <a href="{% url 'project_specific' project.id %}">
+        Configuration
+      </a>
+      </li>
+      <li>
+      <a href="{% url 'importlayer' project.id %}">
+        Import layer
+      </a>
+      </li>
+      <li>
+      <a href="{% url 'newcustomimage' project.id %}">
+        New custom image
+      </a>
+      </li>
+      <li class="pull-right">
+        <form class="form-inline">
+          <div class="form-group">
+            <span class="glyphicon glyphicon-question-sign get-help" data-placement="left" title="Type the name of one or more recipes you want to build, separated by a space. You can also specify a task by appending a colon and a task name to the recipe name, like so: <code>busybox:clean</code>"></span>
+                <input id="build-input" type="text" class="form-control input-lg" placeholder="Select the default image recipe" autocomplete="off" disabled value="{{project.get_default_image}}">
+          </div>
+          {% if project.get_is_new %}
+            <button id="update-project-button" class="btn btn-primary btn-lg" data-project-id="{{project.id}}">Prepare Project</button>
+          {% else %}
+            <button id="cancel-project-button" class="btn info btn-lg" data-project-id="{{project.id}}">Cancel</button>
+            <button id="update-project-button" class="btn btn-primary btn-lg" data-project-id="{{project.id}}">Update</button>
+          {% endif %}
+        </form>
+      </li>
+    </ul>
+  </div>
+  {% endif %}
+</div>

bitbake/lib/toaster/toastergui/templates/projectconf.html

@@ -1,4 +1,4 @@
-{% extends "baseprojectpage.html" %}
+{% extends project_specific|yesno:"baseprojectspecificpage.html,baseprojectpage.html" %}
 {% load projecttags %}
 {% load humanize %}
 
@@ -438,8 +438,11 @@ function onEditPageUpdate(data) {
         var_context='m';
       }
     }
+    if (configvars_sorted[i][0].startsWith("INTERNAL_")) {
+        var_context='m';
+    }
     if (var_context == undefined) {
-      orightml += '<dt><span id="config_var_entry_'+configvars_sorted[i][2]+'" class="js-config-var-name"></span><span class="glyphicon glyphicon-trash js-icon-trash-config_var" id="config_var_trash_'+configvars_sorted[i][2]+'" x-data="'+configvars_sorted[i][2]+'"></span> </dt>'
+        orightml += '<dt><span id="config_var_entry_'+configvars_sorted[i][2]+'" class="js-config-var-name"></span><span class="glyphicon glyphicon-trash js-icon-trash-config_var" id="config_var_trash_'+configvars_sorted[i][2]+'" x-data="'+configvars_sorted[i][2]+'"></span> </dt>'
         orightml += '<dd class="variable-list">'
         orightml += '    <span class="lead" id="config_var_value_'+configvars_sorted[i][2]+'"></span>'
         orightml += '    <span class="glyphicon glyphicon-edit js-icon-pencil-config_var" x-data="'+configvars_sorted[i][2]+'"></span>'

bitbake/lib/toaster/toastergui/templates/recipe.html

@@ -176,7 +176,7 @@
                     <td>{{task.get_executed_display}}</td>
 
                     <td>{{task.get_outcome_display}} 
-                        {% if task.outcome = task.OUTCOME_FAILED %}
+                        {% if task.outcome == task.OUTCOME_FAILED %}
                             <a href="{% url 'build_artifact' build.pk "tasklogfile" task.pk %}">
                                 <span class="glyphicon glyphicon-download-alt
                                     get-help" title="Download task log

bitbake/lib/toaster/toastergui/templates/recipe_add_btn.html

@@ -0,0 +1,23 @@
+<a data-recipe-name="{{data.name}}" class="btn btn-default btn-block layer-exists-{{data.layer_version.pk}} set-default-recipe-btn" style="margin-top: 5px;
+  {% if data.layer_version.pk not in extra.current_layers %}
+    display:none;
+  {% endif %}"
+ >
+  Set recipe
+</a>
+<a class="btn btn-default btn-block layerbtn layer-add-{{data.layer_version.pk}}"
+  data-layer='{
+  "id": {{data.layer_version.pk}},
+  "name":  "{{data.layer_version.layer.name}}",
+  "layerdetailurl": "{%url "layerdetails" extra.pid data.layer_version.pk%}",
+   "xhrLayerUrl": "{% url "xhr_layer" extra.pid data.layer_version.pk %}"
+   }' data-directive="add"
+    {% if data.layer_version.pk in extra.current_layers %}
+     style="display:none;"
+    {% endif %}
+>
+  <span class="glyphicon glyphicon-plus"></span>
+  Add layer
+  <span class="glyphicon glyphicon-question-sign get-help" title="To set this
+      recipe you must first add the {{data.layer_version.layer.name}} layer to your project"></i>
+</a>

bitbake/lib/toaster/toastergui/urls.py

@@ -116,6 +116,11 @@ urlpatterns = [
             tables.ProjectBuildsTable.as_view(template_name="projectbuilds-toastertable.html"),
             name='projectbuilds'),
 
+        url(r'^newproject_specific/(?P<pid>\d+)/$', views.newproject_specific, name='newproject_specific'),
+        url(r'^project_specific/(?P<pid>\d+)/$', views.project_specific, name='project_specific'),
+        url(r'^landing_specific/(?P<pid>\d+)/$', views.landing_specific, name='landing_specific'),
+        url(r'^landing_specific_cancel/(?P<pid>\d+)/$', views.landing_specific_cancel, name='landing_specific_cancel'),
+
         # the import layer is a project-specific functionality;
         url(r'^project/(?P<pid>\d+)/importlayer$', views.importlayer, name='importlayer'),
 
@@ -233,6 +238,14 @@ urlpatterns = [
             api.XhrBuildRequest.as_view(),
             name='xhr_buildrequest'),
 
+        url(r'^xhr_projectupdate/project/(?P<pid>\d+)$',
+            api.XhrProjectUpdate.as_view(),
+            name='xhr_projectupdate'),
+
+        url(r'^xhr_setdefaultimage/project/(?P<pid>\d+)$',
+            api.XhrSetDefaultImageUrl.as_view(),
+            name='xhr_setdefaultimage'),
+
         url(r'xhr_project/(?P<project_id>\d+)$',
             api.XhrProject.as_view(),
             name='xhr_project'),

bitbake/lib/toaster/toastergui/views.py

@@ -25,6 +25,7 @@ import re
 from django.db.models import F, Q, Sum
 from django.db import IntegrityError
 from django.shortcuts import render, redirect, get_object_or_404
+from django.utils.http import urlencode
 from orm.models import Build, Target, Task, Layer, Layer_Version, Recipe
 from orm.models import LogMessage, Variable, Package_Dependency, Package
 from orm.models import Task_Dependency, Package_File
@@ -51,6 +52,7 @@ logger = logging.getLogger("toaster")
 
 # Project creation and managed build enable
 project_enable = ('1' == os.environ.get('TOASTER_BUILDSERVER'))
+is_project_specific = ('1' == os.environ.get('TOASTER_PROJECTSPECIFIC'))
 
 class MimeTypeFinder(object):
     # setting this to False enables additional non-standard mimetypes
@@ -70,6 +72,7 @@ class MimeTypeFinder(object):
 # single point to add global values into the context before rendering
 def toaster_render(request, page, context):
     context['project_enable'] = project_enable
+    context['project_specific'] = is_project_specific
     return render(request, page, context)
 
 
@@ -1383,6 +1386,86 @@ if True:
             'releases': Release.objects.order_by("description"),
         }
 
+        try:
+            context['defaultbranch'] = ToasterSetting.objects.get(name = "DEFAULT_RELEASE").value
+        except ToasterSetting.DoesNotExist:
+            pass
+
+        if request.method == "GET":
+            # render new project page
+            return toaster_render(request, template, context)
+        elif request.method == "POST":
+            mandatory_fields = ['projectname', 'ptype']
+            try:
+                ptype = request.POST.get('ptype')
+                if ptype == "import":
+                    mandatory_fields.append('importdir')
+                else:
+                    mandatory_fields.append('projectversion')
+                # make sure we have values for all mandatory_fields
+                missing = [field for field in mandatory_fields if len(request.POST.get(field, '')) == 0]
+                if missing:
+                    # set alert for missing fields
+                    raise BadParameterException("Fields missing: %s" % ", ".join(missing))
+
+                if not request.user.is_authenticated():
+                    user = authenticate(username = request.POST.get('username', '_anonuser'), password = 'nopass')
+                    if user is None:
+                        user = User.objects.create_user(username = request.POST.get('username', '_anonuser'), email = request.POST.get('email', ''), password = "nopass")
+
+                        user = authenticate(username = user.username, password = 'nopass')
+                    login(request, user)
+
+                #  save the project
+                if ptype == "import":
+                    if not os.path.isdir('%s/conf' % request.POST['importdir']):
+                        raise BadParameterException("Bad path or missing 'conf' directory (%s)" % request.POST['importdir'])
+                    from django.core import management
+                    management.call_command('buildimport', '--command=import', '--name=%s' % request.POST['projectname'], '--path=%s' % request.POST['importdir'], interactive=False)
+                    prj = Project.objects.get(name = request.POST['projectname'])
+                    prj.merged_attr = True
+                    prj.save()
+                else:
+                    release = Release.objects.get(pk = request.POST.get('projectversion', None ))
+                    prj = Project.objects.create_project(name = request.POST['projectname'], release = release)
+                    prj.user_id = request.user.pk
+                    if 'mergeattr' == request.POST.get('mergeattr', ''):
+                        prj.merged_attr = True
+                    prj.save()
+
+                return redirect(reverse(project, args=(prj.pk,)) + "?notify=new-project")
+
+            except (IntegrityError, BadParameterException) as e:
+                # fill in page with previously submitted values
+                for field in mandatory_fields:
+                    context.__setitem__(field, request.POST.get(field, "-- missing"))
+                if isinstance(e, IntegrityError) and "username" in str(e):
+                    context['alert'] = "Your chosen username is already used"
+                else:
+                    context['alert'] = str(e)
+                return toaster_render(request, template, context)
+
+        raise Exception("Invalid HTTP method for this page")
+
+    # new project
+    def newproject_specific(request, pid):
+        if not project_enable:
+            return redirect( landing )
+
+        project = Project.objects.get(pk=pid)
+        template = "newproject_specific.html"
+        context = {
+            'email': request.user.email if request.user.is_authenticated() else '',
+            'username': request.user.username if request.user.is_authenticated() else '',
+            'releases': Release.objects.order_by("description"),
+            'projectname': project.name,
+            'project_pk': project.pk,
+        }
+
+        # WORKAROUND: if we already know release, redirect 'newproject_specific' to 'project_specific'
+        if '1' == project.get_variable('INTERNAL_PROJECT_SPECIFIC_SKIPRELEASE'):
+            return redirect(reverse(project_specific, args=(project.pk,)))
+
         try:
             context['defaultbranch'] = ToasterSetting.objects.get(name = "DEFAULT_RELEASE").value
         except ToasterSetting.DoesNotExist:
@@ -1417,10 +1500,10 @@ if True:
                 else:
                     release = Release.objects.get(pk = request.POST.get('projectversion', None ))
 
-                prj = Project.objects.create_project(name = request.POST['projectname'], release = release)
+                prj = Project.objects.create_project(name = request.POST['projectname'], release = release, existing_project = project)
                 prj.user_id = request.user.pk
                 prj.save()
-                return redirect(reverse(project, args=(prj.pk,)) + "?notify=new-project")
+                return redirect(reverse(project_specific, args=(prj.pk,)) + "?notify=new-project")
 
             except (IntegrityError, BadParameterException) as e:
                 # fill in page with previously submitted values
@@ -1437,9 +1520,87 @@ if True:
     # Shows the edit project page
     def project(request, pid):
         project = Project.objects.get(pk=pid)
+
+        if '1' == os.environ.get('TOASTER_PROJECTSPECIFIC'):
+            if request.GET:
+                #Example:request.GET=<QueryDict: {'setMachine': ['qemuarm']}>
+                params = urlencode(request.GET).replace('%5B%27','').replace('%27%5D','')
+                return redirect("%s?%s" % (reverse(project_specific, args=(project.pk,)),params))
+            else:
+                return redirect(reverse(project_specific, args=(project.pk,)))
         context = {"project": project}
         return toaster_render(request, "project.html", context)
 
+    # Shows the edit project-specific page
+    def project_specific(request, pid):
+        project = Project.objects.get(pk=pid)
+
+        # Are we refreshing from a successful project specific update clone?
+        if Project.PROJECT_SPECIFIC_CLONING_SUCCESS == project.get_variable(Project.PROJECT_SPECIFIC_STATUS):
+            return redirect(reverse(landing_specific,args=(project.pk,)))
+
+        context = {
+            "project": project,
+            "is_new" : project.get_variable(Project.PROJECT_SPECIFIC_ISNEW),
+            "default_image_recipe" : project.get_variable(Project.PROJECT_SPECIFIC_DEFAULTIMAGE),
+            "mru" : Build.objects.all().filter(project=project,outcome=Build.IN_PROGRESS),
+            }
+        if project.build_set.filter(outcome=Build.IN_PROGRESS).count() > 0:
+            context['build_in_progress_none_completed'] = True
+        else:
+            context['build_in_progress_none_completed'] = False
+        return toaster_render(request, "project.html", context)
+
+    # perform the final actions for the project specific page
+    def project_specific_finalize(cmnd, pid):
+        project = Project.objects.get(pk=pid)
+        callback = project.get_variable(Project.PROJECT_SPECIFIC_CALLBACK)
+        if "update" == cmnd:
+            # Delete all '_PROJECT_PREPARE_' builds
+            for b in Build.objects.all().filter(project=project):
+                delete_build = False
+                for t in b.target_set.all():
+                    if '_PROJECT_PREPARE_' == t.target:
+                        delete_build = True
+                if delete_build:
+                    from django.core import management
+                    management.call_command('builddelete', str(b.id), interactive=False)
+            # perform callback at this last moment if defined, in case Toaster gets shutdown next
+            default_target = project.get_variable(Project.PROJECT_SPECIFIC_DEFAULTIMAGE)
+            if callback:
+                callback = callback.replace("<IMAGE>",default_target)
+        if "cancel" == cmnd:
+            if callback:
+                callback = callback.replace("<IMAGE>","none")
+                callback = callback.replace("--update","--cancel")
+        # perform callback at this last moment if defined, in case this Toaster gets shutdown next
+        ret = ''
+        if callback:
+            ret = os.system('bash -c "%s"' % callback)
+            project.set_variable(Project.PROJECT_SPECIFIC_CALLBACK,'')
+        # Delete the temp project specific variables
+        project.set_variable(Project.PROJECT_SPECIFIC_ISNEW,'')
+        project.set_variable(Project.PROJECT_SPECIFIC_STATUS,Project.PROJECT_SPECIFIC_NONE)
+        # WORKAROUND: Release this workaround flag
+        project.set_variable('INTERNAL_PROJECT_SPECIFIC_SKIPRELEASE','')
+
+    # Shows the final landing page for project specific update
+    def landing_specific(request, pid):
+        project_specific_finalize("update", pid)
+        context = {
+            "install_dir": os.environ['TOASTER_DIR'],
+        }
+        return toaster_render(request, "landing_specific.html", context)
+
+    # Shows the related landing-specific page
+    def landing_specific_cancel(request, pid):
+        project_specific_finalize("cancel", pid)
+        context = {
+            "install_dir": os.environ['TOASTER_DIR'],
+            "status": "cancel",
+        }
+        return toaster_render(request, "landing_specific.html", context)
+
     def jsunittests(request):
         """ Provides a page for the js unit tests """
         bbv = BitbakeVersion.objects.filter(branch="master").first()

bitbake/lib/toaster/toastergui/widgets.py

@@ -89,6 +89,10 @@ class ToasterTable(TemplateView):
 
         # global variables
         context['project_enable'] = ('1' == os.environ.get('TOASTER_BUILDSERVER'))
+        try:
+            context['project_specific'] = ('1' == os.environ.get('TOASTER_PROJECTSPECIFIC'))
+        except:
+            context['project_specific'] = ''
 
         return context
 
@@ -511,13 +515,20 @@ class MostRecentBuildsView(View):
                 buildrequest_id = build_obj.buildrequest.pk
             build['buildrequest_id'] = buildrequest_id
 
-            build['recipes_parsed_percentage'] = \
-                int((build_obj.recipes_parsed /
-                     build_obj.recipes_to_parse) * 100)
+            if build_obj.recipes_to_parse > 0:
+                build['recipes_parsed_percentage'] = \
+                    int((build_obj.recipes_parsed /
+                         build_obj.recipes_to_parse) * 100)
+            else:
+                build['recipes_parsed_percentage'] = 0
+            if build_obj.repos_to_clone > 0:
+                build['repos_cloned_percentage'] = \
+                    int((build_obj.repos_cloned /
+                         build_obj.repos_to_clone) * 100)
+            else:
+                build['repos_cloned_percentage'] = 0
 
-            build['repos_cloned_percentage'] = \
-                int((build_obj.repos_cloned /
-                     build_obj.repos_to_clone) * 100)
+            build['progress_item'] = build_obj.progress_item
 
             tasks_complete_percentage = 0
             if build_obj.outcome in (Build.SUCCEEDED, Build.FAILED):

bitbake/lib/toaster/toastermain/management/commands/builddelete.py

@@ -10,8 +10,12 @@ class Command(BaseCommand):
     args    = '<buildID1 buildID2 .....>'
     help    = "Deletes selected build(s)"
 
+    def add_arguments(self, parser):
+        parser.add_argument('buildids', metavar='N', type=int, nargs='+',
+                    help="Build ID's to delete")
+
     def handle(self, *args, **options):
-        for bid in args:
+        for bid in options['buildids']:
             try:
                 b = Build.objects.get(pk = bid)
             except ObjectDoesNotExist:

bitbake/lib/toaster/toastermain/management/commands/buildimport.py

@@ -0,0 +1,583 @@
+#
+# ex:ts=4:sw=4:sts=4:et
+# -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*-
+#
+# BitBake Toaster Implementation
+#
+# Copyright (C) 2018        Wind River Systems
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License version 2 as
+# published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License along
+# with this program; if not, write to the Free Software Foundation, Inc.,
+# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+# buildimport: import a project for project specific configuration
+#
+# Usage:
+#  (a) Set up Toaster environent
+#
+#  (b) Call buildimport
+#      $ /path/to/bitbake/lib/toaster/manage.py buildimport \
+#        --name=$PROJECTNAME \
+#        --path=$BUILD_DIRECTORY \
+#        --callback="$CALLBACK_SCRIPT" \
+#        --command="configure|reconfigure|import"
+#
+#  (c) Return is "|Default_image=%s|Project_id=%d"
+#
+#  (d) Open Toaster to this project using for example:
+#      $ xdg-open http://localhost:$toaster_port/toastergui/project_specific/$project_id
+#
+#  (e) To delete a project:
+#      $ /path/to/bitbake/lib/toaster/manage.py buildimport \
+#        --name=$PROJECTNAME --delete-project
+#
+
+
+# ../bitbake/lib/toaster/manage.py buildimport --name=test --path=`pwd` --callback="" --command=import
+
+from django.core.management.base import BaseCommand, CommandError
+from django.core.exceptions import ObjectDoesNotExist
+from orm.models import ProjectManager, Project, Release, ProjectVariable
+from orm.models import Layer, Layer_Version, LayerSource, ProjectLayer
+from toastergui.api import scan_layer_content
+from django.db import OperationalError
+
+import os
+import re
+import os.path
+import subprocess
+
+# Toaster variable section delimiters
+TOASTER_PROLOG = '#=== TOASTER_CONFIG_PROLOG ==='
+TOASTER_EPILOG = '#=== TOASTER_CONFIG_EPILOG ==='
+
+# quick development/debugging support
+verbose = 2
+def _log(msg):
+    if 1 == verbose:
+        print(msg)
+    elif 2 == verbose:
+        f1=open('/tmp/toaster.log', 'a')
+        f1.write("|" + msg + "|\n" )
+        f1.close()
+
+
+__config_regexp__  = re.compile( r"""
+    ^
+    (?P<exp>export\s+)?
+    (?P<var>[a-zA-Z0-9\-_+.${}/~]+?)
+    (\[(?P<flag>[a-zA-Z0-9\-_+.]+)\])?
+
+    \s* (
+        (?P<colon>:=) |
+        (?P<lazyques>\?\?=) |
+        (?P<ques>\?=) |
+        (?P<append>\+=) |
+        (?P<prepend>=\+) |
+        (?P<predot>=\.) |
+        (?P<postdot>\.=) |
+        =
+    ) \s*
+
+    (?!'[^']*'[^']*'$)
+    (?!\"[^\"]*\"[^\"]*\"$)
+    (?P<apo>['\"])
+    (?P<value>.*)
+    (?P=apo)
+    $
+    """, re.X)
+
+class Command(BaseCommand):
+    args    = "<name> <path> <release>"
+    help    = "Import a command line build directory"
+    vars    = {}
+    toaster_vars = {}
+
+    def add_arguments(self, parser):
+        parser.add_argument(
+            '--name', dest='name', required=True,
+            help='name of the project',
+            )
+        parser.add_argument(
+            '--path', dest='path', required=True,
+            help='path to the project',
+            )
+        parser.add_argument(
+            '--release', dest='release', required=False,
+            help='release for the project',
+            )
+        parser.add_argument(
+            '--callback', dest='callback', required=False,
+            help='callback for project config update',
+            )
+        parser.add_argument(
+            '--delete-project', dest='delete_project', required=False,
+            help='delete this project from the database',
+            )
+        parser.add_argument(
+            '--command', dest='command', required=False,
+            help='command (configure,reconfigure,import)',
+            )
+
+    # Extract the bb variables from a conf file
+    def scan_conf(self,fn):
+        vars = self.vars
+        toaster_vars = self.toaster_vars
+
+        #_log("scan_conf:%s" % fn)
+        if not os.path.isfile(fn):
+            return
+        f = open(fn, 'r')
+
+        #statements = ast.StatementGroup()
+        lineno = 0
+        is_toaster_section = False
+        while True:
+            lineno = lineno + 1
+            s = f.readline()
+            if not s:
+                break
+            w = s.strip()
+            # skip empty lines
+            if not w:
+                continue
+            # evaluate Toaster sections
+            if w.startswith(TOASTER_PROLOG):
+                is_toaster_section = True
+                continue
+            if w.startswith(TOASTER_EPILOG):
+                is_toaster_section = False
+                continue
+            s = s.rstrip()
+            while s[-1] == '\\':
+                s2 = f.readline().strip()
+                lineno = lineno + 1
+                if (not s2 or s2 and s2[0] != "#") and s[0] == "#" :
+                    echo("There is a confusing multiline, partially commented expression on line %s of file %s (%s).\nPlease clarify whether this is all a comment or should be parsed." % (lineno, fn, s))
+                s = s[:-1] + s2
+            # skip comments
+            if s[0] == '#':
+                continue
+            # process the line for just assignments
+            m = __config_regexp__.match(s)
+            if m:
+                groupd = m.groupdict()
+                var = groupd['var']
+                value = groupd['value']
+
+                if groupd['lazyques']:
+                    if not var in vars:
+                        vars[var] = value
+                    continue
+                if groupd['ques']:
+                    if not var in vars:
+                        vars[var] = value
+                    continue
+                # preset empty blank for remaining operators
+                if not var in vars:
+                    vars[var] = ''
+                if groupd['append']:
+                    vars[var] += value
+                elif groupd['prepend']:
+                    vars[var] = "%s%s" % (value,vars[var])
+                elif groupd['predot']:
+                    vars[var] = "%s %s" % (value,vars[var])
+                elif groupd['postdot']:
+                    vars[var] = "%s %s" % (vars[var],value)
+                else:
+                    vars[var] = "%s" % (value)
+                # capture vars in a Toaster section
+                if is_toaster_section:
+                    toaster_vars[var] = vars[var]
+
+        # DONE WITH PARSING
+        f.close()
+        self.vars = vars
+        self.toaster_vars = toaster_vars
+
+    # Update the scanned project variables
+    def update_project_vars(self,project,name):
+        pv, create = ProjectVariable.objects.get_or_create(project = project, name = name)
+        if (not name in self.vars.keys()) or (not self.vars[name]):
+            self.vars[name] = pv.value
+        else:
+            if pv.value != self.vars[name]:
+                pv.value = self.vars[name]
+        pv.save()
+
+    # Find the git version of the installation
+    def find_layer_dir_version(self,path):
+        #  * rocko               ...
+
+        install_version = ''
+        cwd = os.getcwd()
+        os.chdir(path)
+        p = subprocess.Popen(['git', 'branch', '-av'], stdout=subprocess.PIPE,
+                                                stderr=subprocess.PIPE)
+        out, err = p.communicate()
+        out = out.decode("utf-8")
+        for branch in out.split('\n'):
+            if ('*' == branch[0:1]) and ('no branch' not in branch):
+                install_version = re.sub(' .*','',branch[2:])
+                break
+            if 'remotes/m/master' in branch:
+                install_version = re.sub('.*base/','',branch)
+                break
+        os.chdir(cwd)
+        return install_version
+
+    # Compute table of the installation's registered layer versions (branch or commit)
+    def find_layer_dir_versions(self,INSTALL_URL_PREFIX):
+        lv_dict = {}
+        layer_versions = Layer_Version.objects.all()
+        for lv in layer_versions:
+            layer = Layer.objects.filter(pk=lv.layer.pk)[0]
+            if layer.vcs_url:
+                url_short = layer.vcs_url.replace(INSTALL_URL_PREFIX,'')
+            else:
+                url_short = ''
+            # register the core, branch, and the version variations
+            lv_dict["%s,%s,%s" % (url_short,lv.dirpath,'')] = (lv.id,layer.name)
+            lv_dict["%s,%s,%s" % (url_short,lv.dirpath,lv.branch)] = (lv.id,layer.name)
+            lv_dict["%s,%s,%s" % (url_short,lv.dirpath,lv.commit)] = (lv.id,layer.name)
+            #_log("  (%s,%s,%s|%s) = (%s,%s)" % (url_short,lv.dirpath,lv.branch,lv.commit,lv.id,layer.name))
+        return lv_dict
+
+    # Apply table of all layer versions
+    def extract_bblayers(self):
+        # set up the constants
+        bblayer_str = self.vars['BBLAYERS']
+        TOASTER_DIR = os.environ.get('TOASTER_DIR')
+        INSTALL_CLONE_PREFIX = os.path.dirname(TOASTER_DIR) + "/"
+        TOASTER_CLONE_PREFIX = TOASTER_DIR + "/_toaster_clones/"
+        INSTALL_URL_PREFIX = ''
+        layers = Layer.objects.filter(name='openembedded-core')
+        for layer in layers:
+            if layer.vcs_url:
+                INSTALL_URL_PREFIX = layer.vcs_url
+                break
+        INSTALL_URL_PREFIX = INSTALL_URL_PREFIX.replace("/poky","/")
+        INSTALL_VERSION_DIR = TOASTER_DIR
+        INSTALL_URL_POSTFIX = INSTALL_URL_PREFIX.replace(':','_')
+        INSTALL_URL_POSTFIX = INSTALL_URL_POSTFIX.replace('/','_')
+        INSTALL_URL_POSTFIX = "%s_%s" % (TOASTER_CLONE_PREFIX,INSTALL_URL_POSTFIX)
+
+        # get the set of available layer:layer_versions
+        lv_dict = self.find_layer_dir_versions(INSTALL_URL_PREFIX)
+
+        # compute the layer matches
+        layers_list = []
+        for line in bblayer_str.split(' '):
+            if not line:
+                continue
+            if line.endswith('/local'):
+                continue
+
+            # isolate the repo
+            layer_path = line
+            line = line.replace(INSTALL_URL_POSTFIX,'').replace(INSTALL_CLONE_PREFIX,'').replace('/layers/','/').replace('/poky/','/')
+
+            # isolate the sub-path
+            path_index = line.rfind('/')
+            if path_index > 0:
+                sub_path = line[path_index+1:]
+                line = line[0:path_index]
+            else:
+                sub_path = ''
+
+            # isolate the version
+            if TOASTER_CLONE_PREFIX in layer_path:
+                is_toaster_clone = True
+                # extract version from name syntax
+                version_index = line.find('_')
+                if version_index > 0:
+                    version = line[version_index+1:]
+                    line = line[0:version_index]
+                else:
+                    version = ''
+                _log("TOASTER_CLONE(%s/%s), version=%s" % (line,sub_path,version))
+            else:
+                is_toaster_clone = False
+                # version is from the installation
+                version = self.find_layer_dir_version(layer_path)
+                _log("LOCAL_CLONE(%s/%s), version=%s" % (line,sub_path,version))
+
+            # capture the layer information into layers_list
+            layers_list.append( (line,sub_path,version,layer_path,is_toaster_clone) )
+        return layers_list,lv_dict
+
+    #
+    def find_import_release(self,layers_list,lv_dict,default_release):
+        #   poky,meta,rocko => 4;openembedded-core
+        release = default_release
+        for line,path,version,layer_path,is_toaster_clone in layers_list:
+            key = "%s,%s,%s" % (line,path,version)
+            if key in lv_dict:
+                lv_id = lv_dict[key]
+                if 'openembedded-core' == lv_id[1]:
+                    _log("Find_import_release(%s):version=%s,Toaster=%s" % (lv_id[1],version,is_toaster_clone))
+                    # only versions in Toaster managed layers are accepted
+                    if not is_toaster_clone:
+                        break
+                    try:
+                        release = Release.objects.get(name=version)
+                    except:
+                        pass
+                    break
+        _log("Find_import_release:RELEASE=%s" % release.name)
+        return release
+
+    # Apply the found conf layers
+    def apply_conf_bblayers(self,layers_list,lv_dict,project,release=None):
+        for line,path,version,layer_path,is_toaster_clone in layers_list:
+            # Assert release promote if present
+            if release:
+                version = release
+            # try to match the key to a layer_version
+            key = "%s,%s,%s" % (line,path,version)
+            key_short = "%s,%s,%s" % (line,path,'')
+            lv_id = ''
+            if key in lv_dict:
+                lv_id = lv_dict[key]
+                lv = Layer_Version.objects.get(pk=int(lv_id[0]))
+                pl,created = ProjectLayer.objects.get_or_create(project=project,
+                                                   layercommit=lv)
+                pl.optional=False
+                pl.save()
+                _log("  %s => %s;%s" % (key,lv_id[0],lv_id[1]))
+            elif key_short in lv_dict:
+                lv_id = lv_dict[key_short]
+                lv = Layer_Version.objects.get(pk=int(lv_id[0]))
+                pl,created = ProjectLayer.objects.get_or_create(project=project,
+                                                   layercommit=lv)
+                pl.optional=False
+                pl.save()
+                _log("  %s ?> %s" % (key,lv_dict[key_short]))
+            else:
+                _log("%s <= %s" % (key,layer_path))
+                found = False
+                # does local layer already exist in this project?
+                try:
+                    for pl in ProjectLayer.objects.filter(project=project):
+                        if pl.layercommit.layer.local_source_dir == layer_path:
+                            found = True
+                            _log("  Project Local Layer found!")
+                except Exception as e:
+                    _log("ERROR: Local Layer '%s'" % e)
+                    pass
+
+                if not found:
+                    # Does Layer name+path already exist?
+                    try:
+                        layer_name_base = os.path.basename(layer_path)
+                        _log("Layer_lookup: try '%s','%s'" % (layer_name_base,layer_path))
+                        layer = Layer.objects.get(name=layer_name_base,local_source_dir = layer_path)
+                        # Found! Attach layer_version and ProjectLayer
+                        layer_version = Layer_Version.objects.create(
+                            layer=layer,
+                            project=project,
+                            layer_source=LayerSource.TYPE_IMPORTED)
+                        layer_version.save()
+                        pl,created = ProjectLayer.objects.get_or_create(project=project,
+                                                           layercommit=layer_version)
+                        pl.optional=False
+                        pl.save()
+                        found = True
+                        # add layer contents to this layer version
+                        scan_layer_content(layer,layer_version)
+                        _log("  Parent Local Layer found in db!")
+                    except Exception as e:
+                        _log("Layer_exists_test_failed: Local Layer '%s'" % e)
+                        pass
+
+                if not found:
+                    # Insure that layer path exists, in case of user typo
+                    if not os.path.isdir(layer_path):
+                        _log("ERROR:Layer path '%s' not found" % layer_path)
+                        continue
+                    # Add layer to db and attach project to it
+                    layer_name_base = os.path.basename(layer_path)
+                    # generate a unique layer name
+                    layer_name_matches = {}
+                    for layer in Layer.objects.filter(name__contains=layer_name_base):
+                        layer_name_matches[layer.name] = '1'
+                    layer_name_idx = 0
+                    layer_name_test = layer_name_base
+                    while layer_name_test in layer_name_matches.keys():
+                        layer_name_idx += 1
+                        layer_name_test = "%s_%d" % (layer_name_base,layer_name_idx)
+                    # create the layer and layer_verion objects
+                    layer = Layer.objects.create(name=layer_name_test)
+                    layer.local_source_dir = layer_path
+                    layer_version = Layer_Version.objects.create(
+                        layer=layer,
+                        project=project,
+                        layer_source=LayerSource.TYPE_IMPORTED)
+                    layer.save()
+                    layer_version.save()
+                    pl,created = ProjectLayer.objects.get_or_create(project=project,
+                                                       layercommit=layer_version)
+                    pl.optional=False
+                    pl.save()
+                    # register the layer's content
+                    _log("  Local Layer Add content")
+                    scan_layer_content(layer,layer_version)
+                    _log("  Local Layer Added '%s'!" % layer_name_test)
+
+    # Scan the project's conf files (if any)
+    def scan_conf_variables(self,project_path):
+        # scan the project's settings, add any new layers or variables
+        if os.path.isfile("%s/conf/local.conf" % project_path):
+            self.scan_conf("%s/conf/local.conf" % project_path)
+            self.scan_conf("%s/conf/bblayers.conf" % project_path)
+            # Import then disable old style Toaster conf files (before 'merged_attr')
+            old_toaster_local = "%s/conf/toaster.conf" % project_path
+            if os.path.isfile(old_toaster_local):
+                self.scan_conf(old_toaster_local)
+                shutil.move(old_toaster_local, old_toaster_local+"_old")
+            old_toaster_layer = "%s/conf/toaster-bblayers.conf" % project_path
+            if os.path.isfile(old_toaster_layer):
+                self.scan_conf(old_toaster_layer)
+                shutil.move(old_toaster_layer, old_toaster_layer+"_old")
+
+    # Scan the found conf variables (if any)
+    def apply_conf_variables(self,project,layers_list,lv_dict,release=None):
+        if self.vars:
+            # Catch vars relevant to Toaster (in case no Toaster section)
+            self.update_project_vars(project,'DISTRO')
+            self.update_project_vars(project,'MACHINE')
+            self.update_project_vars(project,'IMAGE_INSTALL_append')
+            self.update_project_vars(project,'IMAGE_FSTYPES')
+            self.update_project_vars(project,'PACKAGE_CLASSES')
+            # These vars are typically only assigned by Toaster
+            #self.update_project_vars(project,'DL_DIR')
+            #self.update_project_vars(project,'SSTATE_DIR')
+
+            # Assert found Toaster vars
+            for var in self.toaster_vars.keys():
+                pv, create = ProjectVariable.objects.get_or_create(project = project, name = var)
+                pv.value = self.toaster_vars[var]
+                _log("* Add/update Toaster var '%s' = '%s'" % (pv.name,pv.value))
+                pv.save()
+
+            # Assert found BBLAYERS
+            if 0 < verbose:
+                for pl in ProjectLayer.objects.filter(project=project):
+                    release_name = 'None' if not pl.layercommit.release else pl.layercommit.release.name
+                    print(" BEFORE:ProjectLayer=%s,%s,%s,%s" % (pl.layercommit.layer.name,release_name,pl.layercommit.branch,pl.layercommit.commit))
+            self.apply_conf_bblayers(layers_list,lv_dict,project,release)
+            if 0 < verbose:
+                for pl in ProjectLayer.objects.filter(project=project):
+                    release_name = 'None' if not pl.layercommit.release else pl.layercommit.release.name
+                    print(" AFTER :ProjectLayer=%s,%s,%s,%s" % (pl.layercommit.layer.name,release_name,pl.layercommit.branch,pl.layercommit.commit))
+
+
+    def handle(self, *args, **options):
+        project_name = options['name']
+        project_path = options['path']
+        project_callback = options['callback'] if options['callback'] else ''
+        release_name = options['release'] if options['release'] else ''
+
+        #
+        # Delete project
+        #
+
+        if options['delete_project']:
+            try:
+                print("Project '%s' delete from Toaster database" % (project_name))
+                project = Project.objects.get(name=project_name)
+                # TODO: deep project delete
+                project.delete()
+                print("Project '%s' Deleted" % (project_name))
+                return
+            except Exception as e:
+                print("Project '%s' not found, not deleted (%s)" % (project_name,e))
+                return
+
+        #
+        # Create/Update/Import project
+        #
+
+        # See if project (by name) exists
+        project = None
+        try:
+            # Project already exists
+            project = Project.objects.get(name=project_name)
+        except Exception as e:
+            pass
+
+        # Find the installation's default release
+        default_release = Release.objects.get(id=1)
+
+        # SANITY: if 'reconfig' but project does not exist (deleted externally), switch to 'import'
+        if ("reconfigure" == options['command']) and (None == project):
+            options['command'] = 'import'
+
+        # 'Configure':
+        if "configure" == options['command']:
+            # Note: ignore any existing conf files
+            # create project, SANITY: reuse any project of same name
+            project = Project.objects.create_project(project_name,default_release,project)
+
+        # 'Re-configure':
+        if "reconfigure" == options['command']:
+            # Scan the directory's conf files
+            self.scan_conf_variables(project_path)
+            # Scan the layer list
+            layers_list,lv_dict = self.extract_bblayers()
+            # Apply any new layers or variables
+            self.apply_conf_variables(project,layers_list,lv_dict)
+
+        # 'Import':
+        if "import" == options['command']:
+            # Scan the directory's conf files
+            self.scan_conf_variables(project_path)
+            # Remove these Toaster controlled variables
+            for var in ('DL_DIR','SSTATE_DIR'):
+                self.vars.pop(var, None)
+                self.toaster_vars.pop(var, None)
+            # Scan the layer list
+            layers_list,lv_dict = self.extract_bblayers()
+            # Find the directory's release, and promote to default_release if local paths
+            release = self.find_import_release(layers_list,lv_dict,default_release)
+            # create project, SANITY: reuse any project of same name
+            project = Project.objects.create_project(project_name,release,project)
+            # Apply any new layers or variables
+            self.apply_conf_variables(project,layers_list,lv_dict,release)
+            # WORKAROUND: since we now derive the release, redirect 'newproject_specific' to 'project_specific'
+            project.set_variable('INTERNAL_PROJECT_SPECIFIC_SKIPRELEASE','1')
+
+        # Set up the project's meta data
+        project.builddir = project_path
+        project.merged_attr = True
+        project.set_variable(Project.PROJECT_SPECIFIC_CALLBACK,project_callback)
+        project.set_variable(Project.PROJECT_SPECIFIC_STATUS,Project.PROJECT_SPECIFIC_EDIT)
+        if ("configure" == options['command']) or ("import" == options['command']):
+            # preset the mode and default image recipe
+            project.set_variable(Project.PROJECT_SPECIFIC_ISNEW,Project.PROJECT_SPECIFIC_NEW)
+            project.set_variable(Project.PROJECT_SPECIFIC_DEFAULTIMAGE,"core-image-minimal")
+            # Assert any extended/custom actions or variables for new non-Toaster projects
+            if not len(self.toaster_vars):
+                pass
+        else:
+            project.set_variable(Project.PROJECT_SPECIFIC_ISNEW,Project.PROJECT_SPECIFIC_NONE)
+
+        # Save the updated Project
+        project.save()
+
+        _log("Buildimport:project='%s' at '%d'" % (project_name,project.id))
+
+        if ('DEFAULT_IMAGE' in self.vars) and (self.vars['DEFAULT_IMAGE']):
+            print("|Default_image=%s|Project_id=%d" % (self.vars['DEFAULT_IMAGE'],project.id))
+        else:
+            print("|Project_id=%d" % (project.id))
+

documentation/Makefile

@@ -165,6 +165,7 @@ TARFILES = dev-style.css dev-manual.html \
 TARFILES = dev-style.css dev-manual.html figures/buildhistory-web.png \
            figures/dev-title.png figures/buildhistory.png \
            figures/recipe-workflow.png figures/bitbake-build-flow.png \
+           figures/multiconfig_files.png \
            eclipse
 	endif
 
@@ -261,14 +262,14 @@ TARFILES = mega-manual.html mega-style.css \
         figures/image-generation.png figures/key-dev-elements.png\
 	figures/sdk-generation.png figures/recipe-workflow.png \
 	figures/build-workspace-directory.png figures/mega-title.png \
-	figures/toaster-title.png figures/hosted-service.png \
+	figures/toaster-title.png figures/hosted-service.png figures/multiconfig_files.png \
 	figures/simple-configuration.png figures/poky-reference-distribution.png \
 	figures/compatible-layers.png figures/import-layer.png figures/new-project.png \
 	figures/sdk-environment.png figures/sdk-installed-standard-sdk-directory.png \
 	figures/sdk-devtool-add-flow.png figures/sdk-installed-extensible-sdk-directory.png \
 	figures/sdk-devtool-modify-flow.png figures/sdk-eclipse-dev-flow.png \
 	figures/sdk-devtool-upgrade-flow.png figures/bitbake-build-flow.png figures/bypqs-title.png \
-	figures/overview-manual-title.png
+	figures/overview-manual-title.png figures/sdk-autotools-flow.png figures/sdk-makefile-flow.png
 	endif
 
 MANUALS = $(DOC)/$(DOC).html
@@ -295,7 +296,7 @@ TARFILES = sdk-manual.html sdk-style.css figures/sdk-title.png \
            figures/sdk-environment.png figures/sdk-installed-standard-sdk-directory.png \
 	   figures/sdk-installed-extensible-sdk-directory.png figures/sdk-devtool-add-flow.png \
 	   figures/sdk-devtool-modify-flow.png figures/sdk-eclipse-dev-flow.png \
-	   figures/sdk-devtool-upgrade-flow.png \
+	   figures/sdk-devtool-upgrade-flow.png figures/sdk-autotools-flow.png figures/sdk-makefile-flow.png \
            eclipse
 MANUALS = $(DOC)/$(DOC).html $(DOC)/eclipse
 FIGURES = figures

documentation/brief-yoctoprojectqs/brief-yoctoprojectqs.xml

@@ -110,7 +110,7 @@
             <note>
                 For host package requirements on all supported Linux
                 distributions, see the
-                "<ulink url='&YOCTO_DOCS_REF_URL;#required-packages-for-the-host-development-system'>Required Packages for the Host Development System</ulink>"
+                "<ulink url='&YOCTO_DOCS_REF_URL;#required-packages-for-the-build-host'>Required Packages for the Build Host</ulink>"
                 section in the Yocto Project Reference Manual.
             </note>
             <literallayout class='monospaced'>

documentation/bsp-guide/bsp.xml

@@ -187,7 +187,7 @@
                 <emphasis>Set Up the Build Environment:</emphasis>
                 Be sure you are set up to use BitBake in a shell.
                 See the
-                "<ulink url='&YOCTO_DOCS_DEV_URL;#setting-up-the-development-host-to-use-the-yocto-project'>Preparing the Build Host</ulink>"
+                "<ulink url='&YOCTO_DOCS_DEV_URL;#dev-preparing-the-build-host'>Preparing the Build Host</ulink>"
                 section in the Yocto Project Development Tasks Manual for information
                 on how to get a build host ready that is either a native
                 Linux machine or a machine that uses CROPS.
@@ -1012,7 +1012,7 @@
                 to Support Development Using the Yocto
                 Project</emphasis>:
                 See the
-                "<ulink url='&YOCTO_DOCS_DEV_URL;#setting-up-the-development-host-to-use-the-yocto-project'>Preparing the Build Host</ulink>"
+                "<ulink url='&YOCTO_DOCS_DEV_URL;#dev-preparing-the-build-host'>Preparing the Build Host</ulink>"
                 section in the Yocto Project Development Tasks
                 Manual for options on how to get a system ready
                 to use the Yocto Project.

documentation/dev-manual/dev-manual-common-tasks.xml

@@ -4264,7 +4264,7 @@
                         You need to be sure that your development host is
                         set up to use the Yocto Project.
                         For information on how to set up your host, see the
-                        "<link linkend='setting-up-the-development-host-to-use-the-yocto-project'>Preparing the Build Host</link>"
+                        "<link linkend='dev-preparing-the-build-host'>Preparing the Build Host</link>"
                         section.
                         </para></listitem>
                     <listitem><para>
@@ -5321,104 +5321,215 @@
             </para>
         </section>
 
-        <section id='platdev-building-targets-with-multiple-configurations'>
-            <title>Building Targets with Multiple Configurations</title>
+        <section id='dev-building-images-for-multiple-targets-using-multiple-configurations'>
+            <title>Building Images for Multiple Targets Using Multiple Configurations</title>
 
             <para>
-                Bitbake also has functionality that allows you to build
-                multiple targets at the same time, where each target uses
-                a different configuration.
+                You can use a single <filename>bitbake</filename> command
+                to build multiple images or packages for different targets
+                where each image or package requires a different configuration
+                (multiple configuration builds).
+                The builds, in this scenario, are sometimes referred to as
+                "multiconfigs", and this section uses that term throughout.
             </para>
 
             <para>
-                In order to accomplish this, you setup each of the configurations
-                you need to use in parallel by placing the configuration files in
-                your current build directory alongside the usual
-                <filename>local.conf</filename> file.
+                This section describes how to set up for multiple
+                configuration builds and how to account for cross-build
+                dependencies between the multiconfigs.
             </para>
 
-            <para>
-                Follow these guidelines to create an environment that supports
-                multiple configurations:
-                <itemizedlist>
-                    <listitem><para>
-                        <emphasis>Create Configuration Files</emphasis>:
-                        You need to create a single configuration file for each
-                        configuration for which you want to add support.
-                        These files would contain lines such as the following:
-                        <literallayout class='monospaced'>
-     MACHINE = "A"
-                        </literallayout>
-                        The files would contain any other variables that can
-                        be set and built in the same directory.
-                        <note>
-                            You can change the
-                            <ulink url='&YOCTO_DOCS_REF_URL;#var-TMPDIR'><filename>TMPDIR</filename></ulink>
-                            to not conflict.
-                        </note></para>
+            <section id='dev-setting-up-and-running-a-multiple-configuration-build'>
+                <title>Setting Up and Running a Multiple Configuration Build</title>
 
-                        <para>
-                        Furthermore, the configuration file must be located in the
-                        current build directory in a directory named
-                        <filename>multiconfig</filename> under the build's
-                        <filename>conf</filename> directory where
-                        <filename>local.conf</filename> resides.
-                        The reason for this restriction is because the
-                        <filename>BBPATH</filename> variable is not constructed
-                        until the layers are parsed.
-                        Consequently, using the configuration file as a
-                        pre-configuration file is not possible unless it is
-                        located in the current working directory.
-                        </para></listitem>
-                    <listitem><para>
-                        <emphasis>Add the BitBake Multi-Config Variable to you Local Configuration File</emphasis>:
-                        Use the
-                        <filename>BBMULTICONFIG</filename>
-                        variable in your <filename>conf/local.conf</filename>
-                        configuration file to specify each separate configuration.
-                        For example, the following line tells BitBake it should load
-                        <filename>conf/multiconfig/configA.conf</filename>,
-                        <filename>conf/multiconfig/configB.conf</filename>, and
-                        <filename>conf/multiconfig/configC.conf</filename>.
-                        <literallayout class='monospaced'>
-     BBMULTICONFIG = "configA configB configC"
-                        </literallayout>
-                        </para></listitem>
-                    <listitem><para>
-                        <emphasis>Launch BitBake</emphasis>:
-                        Use the following BitBake command form to launch the
-                        build:
-                        <literallayout class='monospaced'>
+                <para>
+                    To accomplish a multiple configuration build, you must
+                    define each target's configuration separately using
+                    a parallel configuration file in the
+                    <ulink url='&YOCTO_DOCS_REF_URL;#build-directory'>Build Directory</ulink>,
+                    and you must follow a required file hierarchy.
+                    Additionally, you must enable the multiple configuration
+                    builds in your <filename>local.conf</filename> file.
+                </para>
+
+                <para>
+                    Follow these steps to set up and execute multiple
+                    configuration builds:
+                    <itemizedlist>
+                        <listitem><para>
+                            <emphasis>Create Separate Configuration Files</emphasis>:
+                            You need to create a single configuration file for
+                            each build target (each multiconfig).
+                            Minimally, each configuration file must define the
+                            machine and the temporary directory BitBake uses
+                            for the build.
+                            Suggested practice dictates that you do not
+                            overlap the temporary directories
+                            used during the builds.
+                            However, it is possible that you can share the
+                            temporary directory
+                            (<ulink url='&YOCTO_DOCS_REF_URL;#var-TMPDIR'><filename>TMPDIR</filename></ulink>).
+                            For example, consider a scenario with two
+                            different multiconfigs for the same
+                            <ulink url='&YOCTO_DOCS_REF_URL;#var-MACHINE'><filename>MACHINE</filename></ulink>: "qemux86" built for
+                            two distributions such as "poky" and "poky-lsb".
+                            In this case, you might want to use the same
+                            <filename>TMPDIR</filename>.</para>
+
+                            <para>Here is an example showing the minimal
+                            statements needed in a configuration file for
+                            a "qemux86" target whose temporary build directory
+                            is <filename>tmpmultix86</filename>:
+                            <literallayout class='monospaced'>
+     MACHINE="qemux86"
+     TMPDIR="${TOPDIR}/tmpmultix86"
+                            </literallayout></para>
+
+                            <para>The location for these multiconfig
+                            configuration files is specific.
+                            They must reside in the current build directory in
+                            a sub-directory of <filename>conf</filename> named
+                            <filename>multiconfig</filename>.
+                            Following is an example that defines two
+                            configuration files for the "x86" and "arm"
+                            multiconfigs:
+                            <imagedata fileref="figures/multiconfig_files.png" align="center" width="4in" depth="3in" />
+                            </para>
+
+                            <para>The reason for this required file hierarchy
+                            is because the <filename>BBPATH</filename> variable
+                            is not constructed until the layers are parsed.
+                            Consequently, using the configuration file as a
+                            pre-configuration file is not possible unless it is
+                            located in the current working directory.
+                            </para></listitem>
+                        <listitem><para>
+                            <emphasis>Add the BitBake Multi-configuration Variable to the Local Configuration File</emphasis>:
+                            Use the
+                            <ulink url='&YOCTO_DOCS_REF_URL;#var-BBMULTICONFIG'><filename>BBMULTICONFIG</filename></ulink>
+                            variable in your
+                            <filename>conf/local.conf</filename> configuration
+                            file to specify each multiconfig.
+                            Continuing with the example from the previous
+                            figure, the <filename>BBMULTICONFIG</filename>
+                            variable needs to enable two multiconfigs: "x86"
+                            and "arm" by specifying each configuration file:
+                            <literallayout class='monospaced'>
+     BBMULTICONFIG = "x86 arm"
+                            </literallayout>
+                            </para></listitem>
+                        <listitem><para>
+                            <emphasis>Launch BitBake</emphasis>:
+                            Use the following BitBake command form to launch the
+                            multiple configuration build:
+                            <literallayout class='monospaced'>
      $ bitbake [multiconfig:<replaceable>multiconfigname</replaceable>:]<replaceable>target</replaceable> [[[multiconfig:<replaceable>multiconfigname</replaceable>:]<replaceable>target</replaceable>] ... ]
-                        </literallayout>
-                        Following is an example that supports building a minimal
-                        image for configuration A alongside a standard
-                        <filename>core-image-sato</filename>, which takes its
-                        configuration from <filename>local.conf</filename>:
-                        <literallayout class='monospaced'>
-     $ bitbake multiconfig:configA:core-image-minimal core-image-sato
-                        </literallayout>
-                        </para></listitem>
-                </itemizedlist>
-            </para>
-
-            <para>
-                Support for multiple configurations in this current release of
-                the Yocto Project (&DISTRO_NAME; &DISTRO;) has some known issues:
-                <itemizedlist>
-                    <listitem><para>
-                        No inter-multi-configuration dependencies exist.
-                        </para></listitem>
-                    <listitem><para>
-                        Shared State (sstate) optimizations do not exist.
-                        Consequently, if the build uses the same object twice
+                            </literallayout>
+                            For the example in this section, the following
+                            command applies:
+                            <literallayout class='monospaced'>
+     $ bitbake multiconfig:x86:core-image-minimal multiconfig:arm:core-image-sato
+                            </literallayout>
+                            The previous BitBake command builds a
+                            <filename>core-image-minimal</filename> image that
+                            is configured through the
+                            <filename>x86.conf</filename> configuration file
+                            and builds a <filename>core-image-sato</filename>
+                            image that is configured through the
+                            <filename>arm.conf</filename> configuration file.
+                            </para></listitem>
+                    </itemizedlist>
+                    <note>
+                        Support for multiple configuration builds in the
+                        Yocto Project &DISTRO; (&DISTRO_NAME;) Release does
+                        not include Shared State (sstate) optimizations.
+                        Consequently, if a build uses the same object twice
                         in, for example, two different
                         <filename>TMPDIR</filename> directories, the build
-                        will either load from an existing sstate cache at the
-                        start or build the object twice.
-                        </para></listitem>
-                </itemizedlist>
-            </para>
+                        either loads from an existing sstate cache for that
+                        build at the start or builds the object fresh.
+                    </note>
+                </para>
+            </section>
+
+            <section id='dev-enabling-multiple-configuration-build-dependencies'>
+                <title>Enabling Multiple Configuration Build Dependencies</title>
+
+                <para>
+                    Sometimes dependencies can exist between targets
+                    (multiconfigs) in a multiple configuration build.
+                    For example, suppose that in order to build a
+                    <filename>core-image-sato</filename> image for an "x86"
+                    multiconfig, the root filesystem of an "arm"
+                    multiconfig must exist.
+                    This dependency is essentially that the
+                    <ulink url='&YOCTO_DOCS_REF_URL;#ref-tasks-image'><filename>do_image</filename></ulink>
+                    task in the <filename>core-image-sato</filename> recipe
+                    depends on the completion of the
+                    <ulink url='&YOCTO_DOCS_REF_URL;#ref-tasks-rootfs'><filename>do_rootfs</filename></ulink>
+                    task of the <filename>core-image-minimal</filename>
+                    recipe.
+                </para>
+
+                <para>
+                    To enable dependencies in a multiple configuration
+                    build, you must declare the dependencies in the recipe
+                    using the following statement form:
+                    <literallayout class='monospaced'>
+     <replaceable>task_or_package</replaceable>[mcdepends] = "multiconfig:<replaceable>from_multiconfig</replaceable>:<replaceable>to_multiconfig</replaceable>:<replaceable>recipe_name</replaceable>:<replaceable>task_on_which_to_depend</replaceable>"
+                    </literallayout>
+                    To better show how to use this statement, consider the
+                    example scenario from the first paragraph of this section.
+                    The following statement needs to be added to the recipe
+                    that builds the <filename>core-image-sato</filename>
+                    image:
+                    <literallayout class='monospaced'>
+     do_image[mcdepends] = "multiconfig:x86:arm:core-image-minimal:do_rootfs"
+                    </literallayout>
+                    In this example, the
+                    <replaceable>from_multiconfig</replaceable> is "x86".
+                    The <replaceable>to_multiconfig</replaceable> is "arm".
+                    The task on which the <filename>do_image</filename> task
+                    in the recipe depends is the <filename>do_rootfs</filename>
+                    task from the <filename>core-image-minimal</filename>
+                    recipe associated with the "arm" multiconfig.
+               </para>
+
+               <para>
+                   Once you set up this dependency, you can build the
+                   "x86" multiconfig using a BitBake command as follows:
+                   <literallayout class='monospaced'>
+     $ bitbake multiconfig:x86:core-image-sato
+                   </literallayout>
+                   This command executes all the tasks needed to create
+                   the <filename>core-image-sato</filename> image for the
+                   "x86" multiconfig.
+                   Because of the dependency, BitBake also executes through
+                   the <filename>do_rootfs</filename> task for the "arm"
+                   multiconfig build.
+               </para>
+
+               <para>
+                   Having a recipe depend on the root filesystem of another
+                   build might not seem that useful.
+                   Consider this change to the statement in the
+                   <filename>core-image-sato</filename> recipe:
+                   <literallayout class='monospaced'>
+     do_image[mcdepends] = "multiconfig:x86:arm:core-image-minimal:do_image"
+                   </literallayout>
+                   In this case, BitBake must create the
+                   <filename>core-image-minimal</filename> image for the
+                   "arm" build since the "x86" build depends on it.
+               </para>
+
+               <para>
+                   Because "x86" and "arm" are enabled for multiple
+                   configuration builds and have separate configuration
+                   files, BitBake places the artifacts for each build in the
+                   respective temporary build directories (i.e.
+                   <ulink url='&YOCTO_DOCS_REF_URL;#var-TMPDIR'><filename>TMPDIR</filename></ulink>).
+               </para>
+            </section>
         </section>
 
         <section id='building-an-initramfs-image'>
@@ -11400,8 +11511,8 @@
                             within a separately started QEMU or any
                             other virtual machine manager.
                             </para></listitem>
-                        <listitem><para><emphasis>"Systemd-bootTarget":</emphasis>
-                            Choose "Systemd-bootTarget" if your hardware is
+                        <listitem><para><emphasis>"SystemdbootTarget":</emphasis>
+                            Choose "SystemdbootTarget" if your hardware is
                             an EFI-based machine with
                             <filename>systemd-boot</filename> as bootloader and
                             <filename>core-image-testmaster</filename>
@@ -11409,10 +11520,10 @@
                             Also, your hardware under test must be in a
                             DHCP-enabled network that gives it the same IP
                             address for each reboot.</para>
-                            <para>If you choose "Systemd-bootTarget", there are
+                            <para>If you choose "SystemdbootTarget", there are
                             additional requirements and considerations.
                             See the
-                            "<link linkend='selecting-systemd-boottarget'>Selecting Systemd-bootTarget</link>"
+                            "<link linkend='selecting-systemdboottarget'>Selecting SystemdbootTarget</link>"
                             section, which follows, for more information.
                             </para></listitem>
                         <listitem><para><emphasis>"BeagleBoneTarget":</emphasis>
@@ -11458,12 +11569,12 @@
                 </para>
             </section>
 
-            <section id='selecting-systemd-boottarget'>
-                <title>Selecting Systemd-bootTarget</title>
+            <section id='selecting-systemdboottarget'>
+                <title>Selecting SystemdbootTarget</title>
 
                 <para>
                     If you did not set <filename>TEST_TARGET</filename> to
-                    "Systemd-bootTarget", then you do not need any information
+                    "SystemdbootTarget", then you do not need any information
                     in this section.
                     You can skip down to the
                     "<link linkend='qemu-image-running-tests'>Running Tests</link>"
@@ -11472,7 +11583,7 @@
 
                 <para>
                     If you did set <filename>TEST_TARGET</filename> to
-                    "Systemd-bootTarget", you also need to perform a one-time
+                    "SystemdbootTarget", you also need to perform a one-time
                     setup of your master image by doing the following:
                     <orderedlist>
                         <listitem><para><emphasis>Set <filename>EFI_PROVIDER</filename>:</emphasis>
@@ -11543,7 +11654,7 @@
 
                 <para>
                     The final thing you need to do when setting
-                    <filename>TEST_TARGET</filename> to "Systemd-bootTarget" is
+                    <filename>TEST_TARGET</filename> to "SystemdbootTarget" is
                     to set up the test image:
                     <orderedlist>
                         <listitem><para><emphasis>Set up your <filename>local.conf</filename> file:</emphasis>
@@ -11552,7 +11663,7 @@
                             <literallayout class='monospaced'>
      IMAGE_FSTYPES += "tar.gz"
      INHERIT += "testimage"
-     TEST_TARGET = "Systemd-bootTarget"
+     TEST_TARGET = "SystemdbootTarget"
      TEST_TARGET_IP = "192.168.2.3"
                             </literallayout>
                             </para></listitem>
@@ -11969,7 +12080,7 @@
                                     The target controller object used to deploy
                                     and start an image on a particular target
                                     (e.g. QemuTarget, SimpleRemote, and
-                                    Systemd-bootTarget).
+                                    SystemdbootTarget).
                                     Tests usually use the following:
                                     <itemizedlist>
                                         <listitem><para><emphasis><filename>ip</filename>:</emphasis>
@@ -15072,6 +15183,31 @@
                 </para>
             </section>
         </section>
+
+        <section id='copying-licenses-that-do-not-exist'>
+            <title>Copying Licenses that Do Not Exist</title>
+
+            <para>
+                Some packages, such as the linux-firmware package, have many
+                licenses that are not in any way common.
+                You can avoid adding a lot of these types of common license
+                files, which are only applicable to a specific package, by using
+                the
+                <ulink url='&YOCTO_DOCS_REF_URL;#var-NO_GENERIC_LICENSE'><filename>NO_GENERIC_LICENSE</filename></ulink>
+                variable.
+                Using this variable also avoids QA errors when you use a
+                non-common, non-CLOSED license in a recipe.
+            </para>
+
+            <para>
+                The following is an example that uses the
+                <filename>LICENSE.Abilis.txt</filename>
+                file as the license from the fetched source:
+                <literallayout class='monospaced'>
+     NO_GENERIC_LICENSE[Firmware-Abilis] = "LICENSE.Abilis.txt"
+                </literallayout>
+            </para>
+        </section>
     </section>
 
     <section id='using-the-error-reporting-tool'>

documentation/dev-manual/dev-manual-intro.xml

@@ -19,7 +19,7 @@
         </para>
 
         <para>
-            The following list describes what you can get from this manual:
+            This manual provides the following:
             <itemizedlist>
                 <listitem><para>
                     Procedures that help you get going with the Yocto Project.
@@ -44,7 +44,7 @@
         </para>
 
         <para>
-            This manual will not give you the following:
+            This manual does not provide the following:
             <itemizedlist>
                 <listitem><para>
                     Redundant Step-by-step Instructions:

documentation/dev-manual/dev-manual-start.xml

@@ -10,8 +10,10 @@
     This chapter provides procedures related to getting set up to use the
     Yocto Project.
     You can learn about creating a team environment that develops using the
-    Yocto Project, how to set up a build host, how to locate Yocto Project
-    source repositories, and how to create local Git repositories.
+    Yocto Project, how to set up a
+    <ulink url='&YOCTO_DOCS_REF_URL;#hardware-build-system-term'>build host</ulink>,
+    how to locate Yocto Project source repositories, and how to create local
+    Git repositories.
 </para>
 
 <section id="usingpoky-changes-collaborate">
@@ -19,69 +21,71 @@
 
     <para>
         It might not be immediately clear how you can use the Yocto
-        Project in a team development environment, or scale it for a large
-        team of developers.
+        Project in a team development environment, or how to scale it for a
+        large team of developers.
         One of the strengths of the Yocto Project is that it is extremely
         flexible.
         Thus, you can adapt it to many different use cases and scenarios.
-        However, these characteristics can cause a struggle if you are trying
+        However, this flexibility could cause difficulties if you are trying
         to create a working setup that scales across a large team.
     </para>
 
     <para>
         To help you understand how to set up this type of environment,
-        this section presents a procedure that gives you the information
-        to learn how to get the results you want.
+        this section presents a procedure that gives you information
+        that can help you get the results you want.
         The procedure is high-level and presents some of the project's most
         successful experiences, practices, solutions, and available
-        technologies that work well.
+        technologies that have proved to work well in the past.
         Keep in mind, the procedure here is a starting point.
-        You can build off it and customize it to fit any
+        You can build off these steps and customize the procedure to fit any
         particular working environment and set of practices.
         <orderedlist>
             <listitem><para>
                 <emphasis>Determine Who is Going to be Developing:</emphasis>
                 You need to understand who is going to be doing anything
                 related to the Yocto Project and what their roles would be.
-                Making this determination is essential to completing the
+                Making this determination is essential to completing
                 steps two and three, which are to get your equipment together
                 and set up your development environment's hardware topology.
                 </para>
 
                 <para>The following roles exist:
-                <itemizedlist>
-                    <listitem><para>
-                        <emphasis>Application Development:</emphasis>
-                        These types of developers do application level work
-                        on top of an existing software stack.
-                        </para></listitem>
-                    <listitem><para>
-                        <emphasis>Core System Development:</emphasis>
-                        These types of developers work on the contents of the
-                        operating system image itself.
-                        </para></listitem>
-                    <listitem><para>
-                        <emphasis>Build Engineer:</emphasis>
-                        This type of developer manages Autobuilders and
-                        releases.
-                        Not all environments need a Build Engineer.
-                        </para></listitem>
-                    <listitem><para>
-                        <emphasis>Test Engineer:</emphasis>
-                        This type of developer creates and manages automated
-                        tests needed to ensure all application and core
-                        system development meets desired quality standards.
-                        </para></listitem>
-                </itemizedlist>
+                    <itemizedlist>
+                        <listitem><para>
+                            <emphasis>Application Developer:</emphasis>
+                            This type of developer does application level work
+                            on top of an existing software stack.
+                            </para></listitem>
+                        <listitem><para>
+                            <emphasis>Core System Developer:</emphasis>
+                            This type of developer works on the contents of the
+                            operating system image itself.
+                            </para></listitem>
+                        <listitem><para>
+                            <emphasis>Build Engineer:</emphasis>
+                            This type of developer manages Autobuilders and
+                            releases.
+                            Not all environments need a Build Engineer.
+                            </para></listitem>
+                        <listitem><para>
+                            <emphasis>Test Engineer:</emphasis>
+                            This type of developer creates and manages automated
+                            tests that are used to ensure all application and
+                            core system development meets desired quality
+                            standards.
+                            </para></listitem>
+                    </itemizedlist>
                 </para></listitem>
             <listitem><para>
                 <emphasis>Gather the Hardware:</emphasis>
                 Based on the size and make-up of the team, get the hardware
                 together.
-                Any development, build, or test engineer should be using
-                a system that is running a supported Linux distribution.
-                Systems, in general, should be high performance (e.g. dual,
-                six-core Xeons with 24 Gbytes of RAM and plenty of disk space).
+                Ideally, any development, build, or test engineer uses
+                a system that runs a supported Linux distribution.
+                These systems, in general, should be high performance
+                (e.g. dual, six-core Xeons with 24 Gbytes of RAM and plenty
+                of disk space).
                 You can help ensure efficiency by having any machines used
                 for testing or that run Autobuilders be as high performance
                 as possible.
@@ -107,11 +111,12 @@
                 <emphasis>Use Git as Your Source Control Manager (SCM):</emphasis>
                 Keeping your
                 <ulink url='&YOCTO_DOCS_REF_URL;#metadata'>Metadata</ulink>
-                and any software you are developing under the
-                control of an SCM system that is compatible
-                with the OpenEmbedded build system is advisable.
-                Of the SCMs BitBake supports, the
-                Yocto Project team strongly recommends using
+                (i.e. recipes, configuration files, classes, and so forth)
+                and any software you are developing under the control of an SCM
+                system that is compatible   with the OpenEmbedded build system
+                is advisable.
+                Of the SCMs BitBake supports, the Yocto Project team strongly
+                recommends using
                 <ulink url='&YOCTO_DOCS_OM_URL;#git'>Git</ulink>.
                 Git is a distributed system that is easy to backup,
                 allows you to work remotely, and then connects back to the
@@ -129,20 +134,19 @@
                 being used to generate the web interface that lets you view the
                 repositories.
                 The <filename>gitolite</filename> software identifies users
-                using SSH keys and allows branch-based
-                access controls to repositories that you can control as little
-                or as much as necessary.
-
+                using SSH keys and allows branch-based access controls to
+                repositories that you can control as little or as much as
+                necessary.
                 <note>
                    The setup of these services is beyond the scope of this
                    manual.
-                   However, sites such as these exist that describe how to
-                   perform setup:
+                   However, sites such as the following exist that describe
+                   how to perform setup:
                    <itemizedlist>
                        <listitem><para>
                            <ulink url='http://git-scm.com/book/ch4-8.html'>Git documentation</ulink>:
-                           Describes how to install <filename>gitolite</filename>
-                           on the server.
+                           Describes how to install
+                           <filename>gitolite</filename> on the server.
                            </para></listitem>
                        <listitem><para>
                            <ulink url='http://gitolite.com'>Gitolite</ulink>:
@@ -150,8 +154,8 @@
                             </para></listitem>
                         <listitem><para>
                             <ulink url='https://git.wiki.kernel.org/index.php/Interfaces,_frontends,_and_tools'>Interfaces, frontends, and tools</ulink>:
-                            Documentation on how to create interfaces and frontends
-                            for Git.
+                            Documentation on how to create interfaces and
+                            frontends for Git.
                             </para></listitem>
                     </itemizedlist>
                 </note>
@@ -161,23 +165,22 @@
                 As mentioned earlier, application developers are creating
                 applications on top of existing software stacks.
                 Following are some best practices for setting up machines
-                that do application development:
+                used for application development:
                 <itemizedlist>
                     <listitem><para>
-                        Use a pre-built toolchain that
-                        contains the software stack itself.
+                        Use a pre-built toolchain that contains the software
+                        stack itself.
                         Then, develop the application code on top of the
                         stack.
                         This method works well for small numbers of relatively
                         isolated applications.
                         </para></listitem>
                     <listitem><para>
-                        When possible, use the Yocto Project
-                        plug-in for the
+                        When possible, use the Yocto Project plug-in for the
                         <trademark class='trade'>Eclipse</trademark> IDE
                         and SDK development practices.
                         For more information, see the
-                        "<ulink url='&YOCTO_DOCS_SDK_URL;'>Yocto Project Application Development and the Extensible Software Development Kit (eSDK)</ulink>"
+                        <ulink url='&YOCTO_DOCS_SDK_URL;'>Yocto Project Application Development and the Extensible Software Development Kit (eSDK)</ulink>
                         manual.
                         </para></listitem>
                     <listitem><para>
@@ -186,27 +189,29 @@
                         toolchain downloads or as updates through a package
                         update mechanism using <filename>opkg</filename>
                         to provide updates to an existing toolchain.
-                        The exact mechanics of how and when to do this are a
-                        question for local policy.
+                        The exact mechanics of how and when to do this depend
+                        on local policy.
                         </para></listitem>
                     <listitem><para>
-                        Use multiple toolchains installed locally
-                        into different locations to allow development across
+                        Use multiple toolchains installed locally into
+                        different locations to allow development across
                         versions.
                         </para></listitem>
                 </itemizedlist>
                 </para></listitem>
             <listitem><para>
                 <emphasis>Set up the Core Development Machines:</emphasis>
-                As mentioned earlier, these types of developers work on the
-                contents of the operating system itself.
+                As mentioned earlier, core developers work on the contents of
+                the operating system itself.
                 Following are some best practices for setting up machines
                 used for developing images:
                 <itemizedlist>
                     <listitem><para>
-                        Have the Yocto Project build system itself available on
-                        the developer workstations so developers can run their own
-                        builds and directly rebuild the software stack.
+                        Have the
+                        <ulink url='&YOCTO_DOCS_REF_URL;#build-system-term'>OpenEmbedded build system</ulink>
+                        available on the developer workstations so developers
+                        can run their own builds and directly rebuild the
+                        software stack.
                         </para></listitem>
                     <listitem><para>
                         Keep the core system unchanged as much as
@@ -228,8 +233,9 @@
                 Autobuilders are often the core of the development
                 environment.
                 It is here that changes from individual developers are brought
-                together and centrally tested and subsequent decisions about
-                releases can be made.
+                together and centrally tested.
+                Based on this automated build and test environment, subsequent
+                decisions about releases can be made.
                 Autobuilders also allow for "continuous integration" style
                 testing of software components and regression identification
                 and tracking.</para>
@@ -239,22 +245,23 @@
                 The Yocto Project team has found this implementation
                 works well in this role.
                 A public example of this is the Yocto Project
-                Autobuilders, which we use to test the overall health of the
-                project.</para>
+                Autobuilders, which the Yocto Project team uses to test the
+                overall health of the project.</para>
 
                 <para>The features of this system are:
                 <itemizedlist>
                     <listitem><para>
-                         Highlights when commits break the build.
-                         </para></listitem>
-                    <listitem><para>
-                        Populates an sstate cache from which
-                        developers can pull rather than requiring local
-                        builds.
+                        Highlights when commits break the build.
                         </para></listitem>
                     <listitem><para>
-                        Allows commit hook triggers,
-                        which trigger builds when commits are made.
+                        Populates an
+                        <ulink url='&YOCTO_DOCS_OM_URL;#shared-state-cache'>sstate cache</ulink>
+                        from which developers can pull rather than requiring
+                        local builds.
+                        </para></listitem>
+                    <listitem><para>
+                        Allows commit hook triggers, which trigger builds when
+                        commits are made.
                         </para></listitem>
                     <listitem><para>
                         Allows triggering of automated image booting
@@ -275,19 +282,19 @@
                         Allows scheduling of builds so that resources
                         can be used efficiently.
                         </para></listitem>
-               </itemizedlist>
-               </para></listitem>
-           <listitem><para>
-               <emphasis>Set up Test Machines:</emphasis>
-               Use a small number of shared, high performance systems
-               for testing purposes.
-               Developers can use these systems for wider, more
-               extensive testing while they continue to develop
-               locally using their primary development system.
-               </para></listitem>
-           <listitem><para>
-               <emphasis>Document Policies and Change Flow:</emphasis>
-               The Yocto Project itself uses a hierarchical structure and a
+                </itemizedlist>
+                </para></listitem>
+            <listitem><para>
+                <emphasis>Set up Test Machines:</emphasis>
+                Use a small number of shared, high performance systems
+                for testing purposes.
+                Developers can use these systems for wider, more
+                extensive testing while they continue to develop
+                locally using their primary development system.
+                </para></listitem>
+            <listitem><para>
+                <emphasis>Document Policies and Change Flow:</emphasis>
+                The Yocto Project uses a hierarchical structure and a
                 pull model.
                 Scripts exist to create and send pull requests
                 (i.e. <filename>create-pull-request</filename> and
@@ -330,16 +337,20 @@
                     <listitem><para>
                         Maintain your Metadata in layers that make sense
                         for your situation.
-                        See the "<link linkend='understanding-and-creating-layers'>Understanding
-                        and Creating Layers</link>" section for more information on
-                        layers.
+                        See the
+                        "<ulink url='&YOCTO_DOCS_OM_URL;#the-yocto-project-layer-model'>The Yocto Project Layer Model</ulink>"
+                        section in the Yocto Project Overview and Concepts
+                        Manual and the
+                        "<link linkend='understanding-and-creating-layers'>Understanding and Creating Layers</link>"
+                        section for more information on layers.
                         </para></listitem>
                     <listitem><para>
                         Separate the project's Metadata and code by using
                         separate Git repositories.
                         See the
                         "<ulink url='&YOCTO_DOCS_OM_URL;#yocto-project-repositories'>Yocto Project Source Repositories</ulink>"
-                        section for information on these repositories.
+                        section in the Yocto Project Overview and Concepts
+                        Manual for information on these repositories.
                         See the
                         "<link linkend='locating-yocto-project-source-files'>Locating Yocto Project Source Files</link>"
                         section for information on how to set up local Git
@@ -360,7 +371,8 @@
                         </para></listitem>
                     <listitem><para>
                         The Yocto Project community encourages you
-                        to send patches to the project to fix bugs or add features.
+                        to send patches to the project to fix bugs or add
+                        features.
                         If you do submit patches, follow the project commit
                         guidelines for writing good commit messages.
                         See the "<link linkend='how-to-submit-a-change'>Submitting a Change to the Yocto Project</link>"
@@ -369,10 +381,12 @@
                     <listitem><para>
                         Send changes to the core sooner than later
                         as others are likely to run into the same issues.
-                        For some guidance on mailing lists to use, see the list in the
+                        For some guidance on mailing lists to use, see the list
+                        in the
                         "<link linkend='how-to-submit-a-change'>Submitting a Change to the Yocto Project</link>"
                         section.
-                        For a description of the available mailing lists, see the
+                        For a description of the available mailing lists, see
+                        the
                         "<ulink url='&YOCTO_DOCS_REF_URL;#resources-mailinglist'>Mailing Lists</ulink>"
                         section in the Yocto Project Reference Manual.
                         </para></listitem>
@@ -382,22 +396,22 @@
     </para>
 </section>
 
-<section id='setting-up-the-development-host-to-use-the-yocto-project'>
+<section id='dev-preparing-the-build-host'>
     <title>Preparing the Build Host</title>
 
     <para>
-        This section provides procedures to set up your development host to
-        use the Yocto Project.
-        You can use the Yocto Project on a native Linux development host or
-        you can use
+        This section provides procedures to set up a system to be used as your
+        <ulink url='&YOCTO_DOCS_REF_URL;#hardware-build-system-term'>build host</ulink>
+        for development using the Yocto Project.
+        Your build host can be a native Linux machine (recommended) or it can
+        be a machine (Linux, Mac, or Windows) that uses
         <ulink url='https://git.yoctoproject.org/cgit/cgit.cgi/crops/about/'>CROPS</ulink>,
         which leverages
-        <ulink url='https://www.docker.com/'>Docker Containers</ulink>,
-        to prepare any Linux, Mac, or Windows development host.
+        <ulink url='https://www.docker.com/'>Docker Containers</ulink>.
     </para>
 
     <para>
-        Once your development host is set up to use the Yocto Project,
+        Once your build host is set up to use the Yocto Project,
         further steps are necessary depending on what you want to
         accomplish.
         See the following references for information on how to prepare for
@@ -432,7 +446,7 @@
 
         <para>
             Follow these steps to prepare a native Linux machine as your
-            Yocto Project development host:
+            Yocto Project Build Host:
             <orderedlist>
                 <listitem><para>
                     <emphasis>Use a Supported Linux Distribution:</emphasis>
@@ -450,8 +464,8 @@
                     </para></listitem>
                 <listitem><para>
                     <emphasis>Have Enough Free Memory:</emphasis>
-                    You should have at least 50 Gbytes of free disk space
-                    for building images.
+                    Your system should have at least 50 Gbytes of free disk
+                    space for building images.
                     </para></listitem>
                 <listitem><para>
                     <emphasis>Meet Minimal Version Requirements:</emphasis>
@@ -480,14 +494,14 @@
                 <listitem><para>
                     <emphasis>Install Development Host Packages:</emphasis>
                     Required development host packages vary depending on your
-                    build machine and what you want to do with the Yocto
+                    build host and what you want to do with the Yocto
                     Project.
                     Collectively, the number of required packages is large
                     if you want to be able to cover all cases.</para>
 
                     <para>For lists of required packages for all scenarios,
                     see the
-                    "<ulink url='&YOCTO_DOCS_REF_URL;#required-packages-for-the-host-development-system'>Required Packages for the Host Development System</ulink>"
+                    "<ulink url='&YOCTO_DOCS_REF_URL;#required-packages-for-the-build-host'>Required Packages for the Build Host</ulink>"
                     section in the Yocto Project Reference Manual.
                     </para></listitem>
             </orderedlist>
@@ -514,7 +528,7 @@
 
         <para>
             With
-            <ulink url='https://git.yoctoproject.org/cgit/cgit.cgi/crops/about/'>CROPS</ulink>,
+            <ulink url='https://github.com/crops/crops/blob/master/README.md'>CROPS</ulink>,
             which leverages
             <ulink url='https://www.docker.com/'>Docker Containers</ulink>,
             you can create a Yocto Project development environment that
@@ -525,66 +539,101 @@
 
         <para>
             Follow these general steps to prepare a Windows, Mac, or Linux
-            machine as your Yocto Project development host:
+            machine as your Yocto Project build host:
             <orderedlist>
                 <listitem><para>
-                    <emphasis>Go to the Docker Installation Site:</emphasis>
+                    <emphasis>Determine What Your Build Host Needs:</emphasis>
                     <ulink url='https://www.docker.com/what-docker'>Docker</ulink>
                     is a software container platform that you need to install
-                    on the host development machine.
-                    To start the installation process, see the
-                    <ulink url='https://docs.docker.com/engine/installation/'>Docker Installation</ulink>
-                    site.
+                    on the build host.
+                    Depending on your build host, you might have to install
+                    different software to support Docker containers.
+                    Go to the Docker installation page and read about the
+                    platform requirements in
+                    "<ulink url='https://docs.docker.com/install/#supported-platforms'>Supported Platforms</ulink>"
+                    your build host needs to run containers.
                     </para></listitem>
                 <listitem><para>
-                    <emphasis>Choose Your Docker Edition:</emphasis>
-                    Docker comes in several editions.
-                    For the Yocto Project, the stable community edition
-                    (i.e. "Docker CE Stable") is adequate.
-                    You can learn more about the Docker editions from the
-                    site.
+                    <emphasis>Choose What To Install:</emphasis>
+                    Depending on whether or not your build host meets system
+                    requirements, you need to install "Docker CE Stable" or
+                    the "Docker Toolbox".
+                    Most situations call for Docker CE.
+                    However, if you have a build host that does not meet
+                    requirements (e.g. Pre-Windows 10 or Windows 10 "Home"
+                    version), you must install Docker Toolbox instead.
                     </para></listitem>
                 <listitem><para>
                     <emphasis>Go to the Install Site for Your Platform:</emphasis>
                     Click the link for the Docker edition associated with
-                    your development host machine's native software.
-                    For example, if your machine is running Microsoft
+                    your build host's native software.
+                    For example, if your build host is running Microsoft
                     Windows Version 10 and you want the Docker CE Stable
                     edition, click that link under "Supported Platforms".
                     </para></listitem>
-                <listitem><para>
-                    <emphasis>Understand What You Need:</emphasis>
-                    The install page has pre-requisites your machine must
-                    meet.
-                    Be sure you read through this page and make sure your
-                    machine meets the requirements to run Docker.
-                    If your machine does not meet the requirements, the page
-                    has instructions to handle exceptions.
-                    For example, to run Docker on Windows 10, you must have
-                    the pro version of the operating system.
-                    If you have the home version, you need to install the
-                    <ulink url='https://docs.docker.com/toolbox/overview/#ready-to-get-started'>Docker Toolbox</ulink>.
-                    </para>
-
-                    <para>Another example is that a Windows machine needs to
-                    have Microsoft Hyper-V.
-                    If you have a legacy version of the the Microsoft
-                    operating system or for any other reason you do not have
-                    Microsoft Hyper-V, you would have to enter the BIOS and
-                    enable virtualization.
-                    </para></listitem>
                 <listitem><para>
                     <emphasis>Install the Software:</emphasis>
                     Once you have understood all the pre-requisites, you can
                     download and install the appropriate software.
                     Follow the instructions for your specific machine and
-                    the type of the software you need to install.
+                    the type of the software you need to install:
+                    <itemizedlist>
+                        <listitem><para>
+                            Install
+                            <ulink url='https://docs.docker.com/docker-for-windows/install/#install-docker-for-windows-desktop-app'>Docker CE for Windows</ulink>
+                            for Windows build hosts that meet requirements.
+                            </para></listitem>
+                        <listitem><para>
+                            Install
+                            <ulink url='https://docs.docker.com/docker-for-mac/install/#install-and-run-docker-for-mac'>Docker CE for Macs</ulink>
+                            for Mac build hosts that meet requirements.
+                            </para></listitem>
+                        <listitem><para>
+                            Install
+                            <ulink url='https://docs.docker.com/toolbox/toolbox_install_windows/'>Docker Toolbox for Windows</ulink>
+                            for Windows build hosts that do not meet Docker
+                            requirements.
+                            </para></listitem>
+                        <listitem><para>
+                            Install
+                            <ulink url='https://docs.docker.com/toolbox/toolbox_install_mac/'>Docker Toolbox for MacOS</ulink>
+                            for Mac build hosts that do not meet Docker
+                            requirements.
+                            </para></listitem>
+                        <listitem><para>
+                            Install
+                            <ulink url='https://docs.docker.com/install/linux/docker-ce/centos/'>Docker CE for CentOS</ulink>
+                            for Linux build hosts running the CentOS
+                            distribution.
+                            </para></listitem>
+                        <listitem><para>
+                            Install
+                            <ulink url='https://docs.docker.com/install/linux/docker-ce/debian/'>Docker CE for Debian</ulink>
+                            for Linux build hosts running the Debian
+                            distribution.
+                            </para></listitem>
+                        <listitem><para>
+                            Install
+                            <ulink url='https://docs.docker.com/install/linux/docker-ce/fedora/'>Docker CE for Fedora</ulink>
+                            for Linux build hosts running the Fedora
+                            distribution.
+                            </para></listitem>
+                        <listitem><para>
+                            Install
+                            <ulink url='https://docs.docker.com/install/linux/docker-ce/ubuntu/'>Docker CE for Ubuntu</ulink>
+                            for Linux build hosts running the Ubuntu
+                            distribution.
+                            </para></listitem>
+                    </itemizedlist>
                     </para></listitem>
                 <listitem><para>
                     <emphasis>Optionally Orient Yourself With Docker:</emphasis>
                     If you are unfamiliar with Docker and the container
                     concept, you can learn more here -
                     <ulink url='https://docs.docker.com/get-started/'></ulink>.
+                    </para></listitem>
+                <listitem><para>
+                    <emphasis>Launch Docker or Docker Toolbox:</emphasis>
                     You should be able to launch Docker or the Docker Toolbox
                     and have a terminal shell on your development host.
                     </para></listitem>
@@ -593,7 +642,7 @@
                     Go to
                     <ulink url='https://github.com/crops/docker-win-mac-docs/wiki'></ulink>
                     and follow the directions for your particular
-                    development host (i.e. Linux, Mac, or Windows).</para>
+                    build host (i.e. Linux, Mac, or Windows).</para>
 
                     <para>Once you complete the setup instructions for your
                     machine, you have the Poky, Extensible SDK, and Toaster
@@ -622,8 +671,8 @@
     <title>Locating Yocto Project Source Files</title>
 
     <para>
-        This section contains procedures related to locating Yocto Project
-        files.
+        This section shows you how to locate and access the
+        source files that ship with the Yocto Project.
         You establish and use these local files to work on projects.
         <note><title>Notes</title>
             <itemizedlist>
@@ -670,7 +719,7 @@
                     </para></listitem>
                 <listitem><para>
                     <emphasis>Select the Repository:</emphasis>
-                    Click on the repository in which you are interested (i.e.
+                    Click on the repository in which you are interested (e.g.
                     <filename>poky</filename>).
                     </para></listitem>
                 <listitem><para>
@@ -704,6 +753,7 @@
                 The procedure in this section exists should you desire a
                 tarball snapshot of any given component.
             </note>
+            Follow these steps to locate and download a particular tarball:
             <orderedlist>
                 <listitem><para>
                     <emphasis>Access the Index of Releases:</emphasis>
@@ -753,7 +803,10 @@
             uses a "DOWNLOADS" page from which you can locate and download
             tarballs of any Yocto Project release.
             Rather than Git repositories, these files represent snapshot
-            tarballs.
+            tarballs similar to the tarballs located in the Index of Releases
+            described in the
+            "<link linkend='accessing-index-of-releases'>Accessing Index of Releases</link>"
+            section.
             <note><title>Tip</title>
                 The recommended method for accessing Yocto Project
                 components is to use Git to clone a repository and work from
@@ -771,18 +824,23 @@
                 <listitem><para>
                     <emphasis>Get to the Downloads Area:</emphasis>
                     Select the "DOWNLOADS" item from the pull-down
-                    "SOFTWARE" tab menu.
+                    "SOFTWARE" tab menu near the top of the page.
                     </para></listitem>
                 <listitem><para>
                     <emphasis>Select a Yocto Project Release:</emphasis>
                     Use the menu next to "RELEASE" to display and choose
-                    a Yocto Project release (e.g. sumo, rocko, pyro, and
-                    so forth.
-                    For a "map" of Yocto Project releases to version numbers,
-                    see the
-                    <ulink url='https://wiki.yoctoproject.org/wiki/Releases'>Releases</ulink>
-                    wiki page.
-                    </para></listitem>
+                    a recent or past supported Yocto Project release
+                    (e.g. &DISTRO_NAME_NO_CAP;,
+                    &DISTRO_NAME_NO_CAP_MINUS_ONE;, and so forth).
+                    <note><title>Tip</title>
+                        For a "map" of Yocto Project releases to version
+                        numbers, see the
+                        <ulink url='https://wiki.yoctoproject.org/wiki/Releases'>Releases</ulink>
+                        wiki page.
+                    </note>
+                    You can use the "RELEASE ARCHIVE" link to reveal a menu of
+                    all Yocto Project releases.
+                </para></listitem>
                 <listitem><para>
                     <emphasis>Download Tools or Board Support Packages (BSPs):</emphasis>
                     From the "DOWNLOADS" page, you can download tools or
@@ -799,8 +857,9 @@
         <para>
             Yocto Project maintains an area for nightly builds that contains
             tarball releases at <ulink url='&YOCTO_AB_NIGHTLY_URL;'/>.
-            These builds include Yocto Project releases, SDK installation
-            scripts, and experimental builds.
+            These builds include Yocto Project releases ("poky"),
+            toolchains, Yocto Project plugins for Eclipse, and builds for
+            supported machines.
         </para>
 
         <para>
@@ -808,14 +867,21 @@
             Yocto Project component, use the following procedure:
             <orderedlist>
                 <listitem><para>
-                    <emphasis>Access the Nightly Builds:</emphasis>
+                    <emphasis>Locate the Index of Nightly Builds:</emphasis>
                     Open a browser and go to
                     <ulink url='&YOCTO_AB_NIGHTLY_URL;'/> to access the
                     Nightly Builds.
                     </para></listitem>
+                <listitem><para>
+                    <emphasis>Select a Date:</emphasis>
+                    Click on the date in which you are interested.
+                    If you want the latest builds, use "CURRENT".
+                    </para></listitem>
                 <listitem><para>
                     <emphasis>Select a Build:</emphasis>
-                    Click on any build by date in which you are interested.
+                    Choose the area in which you are interested.
+                    For example, if you are looking for the most recent
+                    toolchains, select the "toolchain" link.
                     </para></listitem>
                 <listitem><para>
                     <emphasis>Find the Tarball:</emphasis>
@@ -831,27 +897,23 @@
     </section>
 </section>
 
-<section id='cloning-and-checking-out-branchs'>
+<section id='cloning-and-checking-out-branches'>
     <title>Cloning and Checking Out Branches</title>
 
     <para>
-        To use the Yocto Project, you need a release of the Yocto Project
-        locally installed on your development system.
-        The locally installed set of files is referred to as the
+        To use the Yocto Project for development, you need a release locally
+        installed on your development system.
+        This locally installed set of files is referred to as the
         <ulink url='&YOCTO_DOCS_REF_URL;#source-directory'>Source Directory</ulink>
         in the Yocto Project documentation.
     </para>
 
     <para>
-        You create your Source Directory by using
+        The preferred method of creating your Source Directory is by using
         <ulink url='&YOCTO_DOCS_OM_URL;#git'>Git</ulink> to clone a local
         copy of the upstream <filename>poky</filename> repository.
-        <note><title>Tip</title>
-            The preferred method of getting the Yocto Project Source
-            Directory set up is to clone the repository.
-        </note>
-        Working from a copy of the upstream repository allows you
-        to contribute back into the Yocto Project or simply work with
+        Working from a cloned copy of the upstream repository allows you
+        to contribute back into the Yocto Project or to simply work with
         the latest software on a development branch.
         Because Git maintains and creates an upstream repository with
         a complete history of changes and you are working with a local
@@ -871,21 +933,23 @@
             <orderedlist>
                 <listitem><para>
                     <emphasis>Set Your Directory:</emphasis>
-                    Be in the directory where you want to create your local
-                    copy of poky.
+                    Change your working directory to where you want to
+                    create your local copy of
+                    <filename>poky</filename>.
                     </para></listitem>
                 <listitem><para>
                     <emphasis>Clone the Repository:</emphasis>
-                    The following command clones the repository and uses
+                    The following example command clones the
+                    <filename>poky</filename> repository and uses
                     the default name "poky" for your local repository:
                     <literallayout class='monospaced'>
      $ git clone git://git.yoctoproject.org/poky
      Cloning into 'poky'...
-     remote: Counting objects: 367178, done.
-     remote: Compressing objects: 100% (88161/88161), done.
-     remote: Total 367178 (delta 272761), reused 366942 (delta 272525)
-     Receiving objects: 100% (367178/367178), 133.26 MiB | 6.40 MiB/s, done.
-     Resolving deltas: 100% (272761/272761), done.
+     remote: Counting objects: 416542, done.
+     remote: Compressing objects: 100% (98611/98611), done.
+     remote: Total 416542 (delta 311104), reused 416377 (delta 310939)
+     Receiving objects: 100% (416542/416542), 150.39 MiB | 15.77 MiB/s, done.
+     Resolving deltas: 100% (311104/311104), done.
      Checking connectivity... done.
                     </literallayout>
                     Unless you specify a specific development branch or
@@ -900,8 +964,8 @@
                     <link linkend='checkout-out-by-tag-in-poky'>Checking Out By Tag in Poky</link>"
                     sections, respectively.</para>
 
-                    <para>Once the repository is created, you can change to
-                    that directory and check its status.
+                    <para>Once the local repository is created, you can
+                    change to that directory and check its status.
                     Here, the single "master" branch exists on your system
                     and by default, it is checked out:
                     <literallayout class='monospaced'>
@@ -916,6 +980,10 @@
                     Your local repository of poky is identical to the
                     upstream poky repository at the time from which it was
                     cloned.
+                    As you work with the local branch, you can periodically
+                    use the <filename>git pull &dash;&dash;rebase</filename>
+                    command to be sure you are up-to-date with the upstream
+                    branch.
                     </para></listitem>
             </orderedlist>
         </para>

documentation/kernel-dev/kernel-dev-common.xml

@@ -25,7 +25,7 @@
             Before you can do any kernel development, you need to be
             sure your build host is set up to use the Yocto Project.
             For information on how to get set up, see the
-            "<ulink url='&YOCTO_DOCS_DEV_URL;#setting-up-the-development-host-to-use-the-yocto-project'>Preparing the Build Host</ulink>"
+            "<ulink url='&YOCTO_DOCS_DEV_URL;#dev-preparing-the-build-host'>Preparing the Build Host</ulink>"
             section in the Yocto Project Development Tasks Manual.
             Part of preparing the system is creating a local Git
             repository of the

documentation/overview-manual/overview-manual-yp-intro.xml

@@ -1296,7 +1296,7 @@
                     <para>It is worth noting that the term "package" can,
                     in general, have subtle meanings.
                     For example, the packages referred to in the
-                    "<ulink url='&YOCTO_DOCS_REF_URL;#required-packages-for-the-host-development-system'>Required Packages for the Host Development System</ulink>"
+                    "<ulink url='&YOCTO_DOCS_REF_URL;#required-packages-for-the-build-host'>Required Packages for the Build Host</ulink>"
                     section in the Yocto Project Reference Manual are compiled
                     binaries that, when installed, add functionality to your
                     Linux distribution.</para>

documentation/ref-manual/ref-classes.xml

@@ -3145,12 +3145,6 @@ This check was removed for YP 2.3 release
         and <filename><link linkend='var-SITEINFO_BITS'>SITEINFO_BITS</link></filename>
         that can be used elsewhere in the metadata.
     </para>
-
-    <para>
-        Because the
-        <link linkend='ref-classes-base'><filename>base</filename></link> class
-        includes the <filename>siteinfo</filename> class, it is always active.
-    </para>
 </section>
 
 <section id='ref-classes-spdx'>