oe-selftest: devtool: Support meta being a symbolic link

oe-selftest's devtool tests have been broken since commit 2457cd57
(oe-selftest: devtool: avoid parallel races by using temporary copy of
core) if meta is a symbolic link.

(From OE-Core rev: daba6c5a991b370709d17e51305334f55a3858ec)

Signed-off-by: Peter Kjellerstedt <peter.kjellerstedt@axis.com>
Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>

bitbake/bin/bitbake

@@ -38,7 +38,7 @@ from bb.main import bitbake_main, BitBakeConfigParameters, BBMainException
 if sys.getfilesystemencoding() != "utf-8":
     sys.exit("Please use a locale setting which supports UTF-8 (such as LANG=en_US.UTF-8).\nPython can't change the filesystem locale after loading so we need a UTF-8 when Python starts or things won't work.")
 
-__version__ = "1.39.1"
+__version__ = "1.40.0"
 
 if __name__ == "__main__":
     if __version__ != bb.__version__:

bitbake/doc/bitbake-user-manual/bitbake-user-manual-intro.xml

@@ -723,6 +723,163 @@
                     </literallayout>
                 </para>
             </section>
+
+            <section id='executing-a-multiple-configuration-build'>
+                <title>Executing a Multiple Configuration Build</title>
+
+                <para>
+                    BitBake is able to build multiple images or packages
+                    using a single command where the different targets
+                    require different configurations (multiple configuration
+                    builds).
+                    Each target, in this scenario, is referred to as a
+                    "multiconfig".
+                </para>
+
+                <para>
+                    To accomplish a multiple configuration build, you must
+                    define each target's configuration separately using
+                    a parallel configuration file in the build directory.
+                    The location for these multiconfig configuration files
+                    is specific.
+                    They must reside in the current build directory in
+                    a sub-directory of <filename>conf</filename> named
+                    <filename>multiconfig</filename>.
+                    Following is an example for two separate targets:
+                    <imagedata fileref="figures/bb_multiconfig_files.png" align="center" width="4in" depth="3in" />
+                </para>
+
+                <para>
+                    The reason for this required file hierarchy
+                    is because the <filename>BBPATH</filename> variable
+                    is not constructed until the layers are parsed.
+                    Consequently, using the configuration file as a
+                    pre-configuration file is not possible unless it is
+                    located in the current working directory.
+                </para>
+
+                <para>
+                    Minimally, each configuration file must define the
+                    machine and the temporary directory BitBake uses
+                    for the build.
+                    Suggested practice dictates that you do not
+                    overlap the temporary directories used during the
+                    builds.
+                </para>
+
+                <para>
+                    Aside from separate configuration files for each
+                    target, you must also enable BitBake to perform multiple
+                    configuration builds.
+                    Enabling is accomplished by setting the
+                    <link linkend='var-BBMULTICONFIG'><filename>BBMULTICONFIG</filename></link>
+                    variable in the <filename>local.conf</filename>
+                    configuration file.
+                    As an example, suppose you had configuration files
+                    for <filename>target1</filename> and
+                    <filename>target2</filename> defined in the build
+                    directory.
+                    The following statement in the
+                    <filename>local.conf</filename> file both enables
+                    BitBake to perform multiple configuration builds and
+                    specifies the two multiconfigs:
+                    <literallayout class='monospaced'>
+     BBMULTICONFIG = "target1 target2"
+                    </literallayout>
+                </para>
+
+                <para>
+                    Once the target configuration files are in place and
+                    BitBake has been enabled to perform multiple configuration
+                    builds, use the following command form to start the
+                    builds:
+                    <literallayout class='monospaced'>
+     $ bitbake [multiconfig:<replaceable>multiconfigname</replaceable>:]<replaceable>target</replaceable> [[[multiconfig:<replaceable>multiconfigname</replaceable>:]<replaceable>target</replaceable>] ... ]
+                    </literallayout>
+                    Here is an example for two multiconfigs:
+                    <filename>target1</filename> and
+                    <filename>target2</filename>:
+                    <literallayout class='monospaced'>
+     $ bitbake multiconfig:target1:<replaceable>target</replaceable> multiconfig:target2:<replaceable>target</replaceable>
+                    </literallayout>
+                </para>
+            </section>
+
+            <section id='bb-enabling-multiple-configuration-build-dependencies'>
+                <title>Enabling Multiple Configuration Build Dependencies</title>
+
+                <para>
+                    Sometimes dependencies can exist between targets
+                    (multiconfigs) in a multiple configuration build.
+                    For example, suppose that in order to build an image
+                    for a particular architecture, the root filesystem of
+                    another build for a different architecture needs to
+                    exist.
+                    In other words, the image for the first multiconfig depends
+                    on the root filesystem of the second multiconfig.
+                    This dependency is essentially that the task in the recipe
+                    that builds one multiconfig is dependent on the
+                    completion of the task in the recipe that builds
+                    another multiconfig.
+                </para>
+
+                <para>
+                    To enable dependencies in a multiple configuration
+                    build, you must declare the dependencies in the recipe
+                    using the following statement form:
+                    <literallayout class='monospaced'>
+     <replaceable>task_or_package</replaceable>[mcdepends] = "multiconfig:<replaceable>from_multiconfig</replaceable>:<replaceable>to_multiconfig</replaceable>:<replaceable>recipe_name</replaceable>:<replaceable>task_on_which_to_depend</replaceable>"
+                    </literallayout>
+                    To better show how to use this statement, consider an
+                    example with two multiconfigs: <filename>target1</filename>
+                    and <filename>target2</filename>:
+                    <literallayout class='monospaced'>
+     <replaceable>image_task</replaceable>[mcdepends] = "multiconfig:target1:target2:<replaceable>image2</replaceable>:<replaceable>rootfs_task</replaceable>"
+                    </literallayout>
+                    In this example, the
+                    <replaceable>from_multiconfig</replaceable> is "target1" and
+                    the <replaceable>to_multiconfig</replaceable> is "target2".
+                    The task on which the image whose recipe contains
+                    <replaceable>image_task</replaceable> depends on the
+                    completion of the <replaceable>rootfs_task</replaceable>
+                    used to build out <replaceable>image2</replaceable>, which
+                    is associated with the "target2" multiconfig.
+               </para>
+
+               <para>
+                   Once you set up this dependency, you can build the
+                   "target1" multiconfig using a BitBake command as follows:
+                   <literallayout class='monospaced'>
+     $ bitbake multiconfig:target1:<replaceable>image1</replaceable>
+                   </literallayout>
+                   This command executes all the tasks needed to create
+                   <replaceable>image1</replaceable> for the "target1"
+                   multiconfig.
+                   Because of the dependency, BitBake also executes through
+                   the <replaceable>rootfs_task</replaceable> for the "target2"
+                   multiconfig build.
+               </para>
+
+               <para>
+                   Having a recipe depend on the root filesystem of another
+                   build might not seem that useful.
+                   Consider this change to the statement in the
+                   <replaceable>image1</replaceable> recipe:
+                   <literallayout class='monospaced'>
+     <replaceable>image_task</replaceable>[mcdepends] = "multiconfig:target1:target2:<replaceable>image2</replaceable>:<replaceable>image_task</replaceable>"
+                   </literallayout>
+                   In this case, BitBake must create
+                   <replaceable>image2</replaceable> for the "target2"
+                   build since the "target1" build depends on it.
+               </para>
+
+               <para>
+                   Because "target1" and "target2" are enabled for multiple
+                   configuration builds and have separate configuration
+                   files, BitBake places the artifacts for each build in the
+                   respective temporary build directories.
+               </para>
+            </section>
         </section>
     </section>
 </chapter>

bitbake/doc/bitbake-user-manual/bitbake-user-manual-metadata.xml

@@ -342,7 +342,7 @@
 
             <para>
                 When you use this syntax, BitBake expects one or more strings.
-                Surrounding spaces are removed as well.
+                Surrounding spaces and spacing are preserved.
                 Here is an example:
                 <literallayout class='monospaced'>
      FOO = "123 456 789 123456 123 456 123 456"
@@ -352,8 +352,8 @@
      FOO2_remove = "abc def"
                 </literallayout>
                 The variable <filename>FOO</filename> becomes
-                "789 123456" and <filename>FOO2</filename> becomes
-                "ghi abcdef".
+                "  789 123456    " and <filename>FOO2</filename> becomes
+                "  ghi abcdef    ".
             </para>
 
             <para>

bitbake/doc/bitbake-user-manual/bitbake-user-manual-ref-variables.xml

@@ -646,10 +646,10 @@
             <glossdef>
                 <para>
                     Contains the name of the currently executing task.
-                    The value does not include the "do_" prefix.
+                    The value includes the "do_" prefix.
                     For example, if the currently executing task is
                     <filename>do_config</filename>, the value is
-                    "config".
+                    "do_config".
                 </para>
             </glossdef>
         </glossentry>
@@ -1205,6 +1205,45 @@
             </glossdef>
         </glossentry>
 
+        <glossentry id='var-BBMULTICONFIG'><glossterm>BBMULTICONFIG</glossterm>
+            <info>
+                BBMULTICONFIG[doc] = "Enables BitBake to perform multiple configuration builds and lists each separate configuration (multiconfig)."
+            </info>
+            <glossdef>
+                <para role="glossdeffirst">
+<!--                <para role="glossdeffirst"><imagedata fileref="figures/define-generic.png" /> -->
+                    Enables BitBake to perform multiple configuration builds
+                    and lists each separate configuration (multiconfig).
+                    You can use this variable to cause BitBake to build
+                    multiple targets where each target has a separate
+                    configuration.
+                    Define <filename>BBMULTICONFIG</filename> in your
+                    <filename>conf/local.conf</filename> configuration file.
+                </para>
+
+                <para>
+                    As an example, the following line specifies three
+                    multiconfigs, each having a separate configuration file:
+                    <literallayout class='monospaced'>
+     BBMULTIFONFIG = "configA configB configC"
+                    </literallayout>
+                    Each configuration file you use must reside in the
+                    build directory within a directory named
+                    <filename>conf/multiconfig</filename> (e.g.
+                    <replaceable>build_directory</replaceable><filename>/conf/multiconfig/configA.conf</filename>).
+                </para>
+
+                <para>
+                    For information on how to use
+                    <filename>BBMULTICONFIG</filename> in an environment that
+                    supports building targets with multiple configurations,
+                    see the
+                    "<link linkend='executing-a-multiple-configuration-build'>Executing a Multiple Configuration Build</link>"
+                    section.
+                </para>
+            </glossdef>
+        </glossentry>
+
         <glossentry id='var-BBPATH'><glossterm>BBPATH</glossterm>
             <glossdef>
                 <para>

bitbake/lib/bb/__init__.py

@@ -21,7 +21,7 @@
 # with this program; if not, write to the Free Software Foundation, Inc.,
 # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
 
-__version__ = "1.39.1"
+__version__ = "1.40.0"
 
 import sys
 if sys.version_info < (3, 4, 0):

bitbake/lib/bb/data.py

@@ -38,6 +38,7 @@ the speed is more critical here.
 # Based on functions from the base bb module, Copyright 2003 Holger Schurig
 
 import sys, os, re
+import hashlib
 if sys.argv[0][-5:] == "pydoc":
     path = os.path.dirname(os.path.dirname(sys.argv[1]))
 else:
@@ -283,14 +284,12 @@ def build_dependencies(key, keys, shelldeps, varflagsexcl, d):
     try:
         if key[-1] == ']':
             vf = key[:-1].split('[')
-            value = d.getVarFlag(vf[0], vf[1], False)
-            parser = d.expandWithRefs(value, key)
+            value, parser = d.getVarFlag(vf[0], vf[1], False, retparser=True)
             deps |= parser.references
             deps = deps | (keys & parser.execs)
             return deps, value
         varflags = d.getVarFlags(key, ["vardeps", "vardepvalue", "vardepsexclude", "exports", "postfuncs", "prefuncs", "lineno", "filename"]) or {}
         vardeps = varflags.get("vardeps")
-        value = d.getVarFlag(key, "_content", False)
 
         def handle_contains(value, contains, d):
             newvalue = ""
@@ -309,10 +308,19 @@ def build_dependencies(key, keys, shelldeps, varflagsexcl, d):
                 return newvalue
             return value + newvalue
 
+        def handle_remove(value, deps, removes, d):
+            for r in sorted(removes):
+                r2 = d.expandWithRefs(r, None)
+                value += "\n_remove of %s" % r
+                deps |= r2.references
+                deps = deps | (keys & r2.execs)
+            return value
+
         if "vardepvalue" in varflags:
-           value = varflags.get("vardepvalue")
+            value = varflags.get("vardepvalue")
         elif varflags.get("func"):
             if varflags.get("python"):
+                value = d.getVarFlag(key, "_content", False)
                 parser = bb.codeparser.PythonParser(key, logger)
                 if value and "\t" in value:
                     logger.warning("Variable %s contains tabs, please remove these (%s)" % (key, d.getVar("FILE")))
@@ -321,13 +329,15 @@ def build_dependencies(key, keys, shelldeps, varflagsexcl, d):
                 deps = deps | (keys & parser.execs)
                 value = handle_contains(value, parser.contains, d)
             else:
-                parsedvar = d.expandWithRefs(value, key)
+                value, parsedvar = d.getVarFlag(key, "_content", False, retparser=True)
                 parser = bb.codeparser.ShellParser(key, logger)
                 parser.parse_shell(parsedvar.value)
                 deps = deps | shelldeps
                 deps = deps | parsedvar.references
                 deps = deps | (keys & parser.execs) | (keys & parsedvar.execs)
                 value = handle_contains(value, parsedvar.contains, d)
+                if hasattr(parsedvar, "removes"):
+                    value = handle_remove(value, deps, parsedvar.removes, d)
             if vardeps is None:
                 parser.log.flush()
             if "prefuncs" in varflags:
@@ -337,10 +347,12 @@ def build_dependencies(key, keys, shelldeps, varflagsexcl, d):
             if "exports" in varflags:
                 deps = deps | set(varflags["exports"].split())
         else:
-            parser = d.expandWithRefs(value, key)
+            value, parser = d.getVarFlag(key, "_content", False, retparser=True)
             deps |= parser.references
             deps = deps | (keys & parser.execs)
             value = handle_contains(value, parser.contains, d)
+            if hasattr(parser, "removes"):
+                value = handle_remove(value, deps, parser.removes, d)
 
         if "vardepvalueexclude" in varflags:
             exclude = varflags.get("vardepvalueexclude")
@@ -394,6 +406,43 @@ def generate_dependencies(d):
         #print "For %s: %s" % (task, str(deps[task]))
     return tasklist, deps, values
 
+def generate_dependency_hash(tasklist, gendeps, lookupcache, whitelist, fn):
+    taskdeps = {}
+    basehash = {}
+
+    for task in tasklist:
+        data = lookupcache[task]
+
+        if data is None:
+            bb.error("Task %s from %s seems to be empty?!" % (task, fn))
+            data = ''
+
+        gendeps[task] -= whitelist
+        newdeps = gendeps[task]
+        seen = set()
+        while newdeps:
+            nextdeps = newdeps
+            seen |= nextdeps
+            newdeps = set()
+            for dep in nextdeps:
+                if dep in whitelist:
+                    continue
+                gendeps[dep] -= whitelist
+                newdeps |= gendeps[dep]
+            newdeps -= seen
+
+        alldeps = sorted(seen)
+        for dep in alldeps:
+            data = data + dep
+            var = lookupcache[dep]
+            if var is not None:
+                data = data + str(var)
+        k = fn + "." + task
+        basehash[k] = hashlib.md5(data.encode("utf-8")).hexdigest()
+        taskdeps[task] = alldeps
+
+    return taskdeps, basehash
+
 def inherits_class(klass, d):
     val = d.getVar('__inherit_cache', False) or []
     needle = os.path.join('classes', '%s.bbclass' % klass)

bitbake/lib/bb/data_smart.py

@@ -42,6 +42,7 @@ __setvar_keyword__ = ["_append", "_prepend", "_remove"]
 __setvar_regexp__ = re.compile('(?P<base>.*?)(?P<keyword>_append|_prepend|_remove)(_(?P<add>[^A-Z]*))?$')
 __expand_var_regexp__ = re.compile(r"\${[^{}@\n\t :]+}")
 __expand_python_regexp__ = re.compile(r"\${@.+?}")
+__whitespace_split__ = re.compile('(\s)')
 
 def infer_caller_details(loginfo, parent = False, varval = True):
     """Save the caller the trouble of specifying everything."""
@@ -104,11 +105,7 @@ class VariableParse:
             if self.varname and key:
                 if self.varname == key:
                     raise Exception("variable %s references itself!" % self.varname)
-            if key in self.d.expand_cache:
-                varparse = self.d.expand_cache[key]
-                var = varparse.value
-            else:
-                var = self.d.getVarFlag(key, "_content")
+            var = self.d.getVarFlag(key, "_content")
             self.references.add(key)
             if var is not None:
                 return var
@@ -267,6 +264,16 @@ class VariableHistory(object):
             return
         self.variables[var].append(loginfo.copy())
 
+    def rename_variable_hist(self, oldvar, newvar):
+        if not self.dataroot._tracking:
+            return
+        if oldvar not in self.variables:
+            return
+        if newvar not in self.variables:
+            self.variables[newvar] = []
+        for i in self.variables[oldvar]:
+            self.variables[newvar].append(i.copy())
+
     def variable(self, var):
         remote_connector = self.dataroot.getVar('_remote_data', False)
         if remote_connector:
@@ -401,9 +408,6 @@ class DataSmart(MutableMapping):
         if not isinstance(s, str): # sanity check
             return VariableParse(varname, self, s)
 
-        if varname and varname in self.expand_cache:
-            return self.expand_cache[varname]
-
         varparse = VariableParse(varname, self)
 
         while s.find('${') != -1:
@@ -427,9 +431,6 @@ class DataSmart(MutableMapping):
 
         varparse.value = s
 
-        if varname:
-            self.expand_cache[varname] = varparse
-
         return varparse
 
     def expand(self, s, varname = None):
@@ -498,6 +499,7 @@ class DataSmart(MutableMapping):
 
     def setVar(self, var, value, **loginfo):
         #print("var=" + str(var) + "  val=" + str(value))
+        self.expand_cache = {}
         parsing=False
         if 'parsing' in loginfo:
             parsing=True
@@ -510,7 +512,7 @@ class DataSmart(MutableMapping):
 
         if 'op' not in loginfo:
             loginfo['op'] = "set"
-        self.expand_cache = {}
+
         match  = __setvar_regexp__.match(var)
         if match and match.group("keyword") in __setvar_keyword__:
             base = match.group('base')
@@ -619,6 +621,7 @@ class DataSmart(MutableMapping):
 
         val = self.getVar(key, 0, parsing=True)
         if val is not None:
+            self.varhistory.rename_variable_hist(key, newkey)
             loginfo['variable'] = newkey
             loginfo['op'] = 'rename from %s' % key
             loginfo['detail'] = val
@@ -660,6 +663,7 @@ class DataSmart(MutableMapping):
         self.setVar(var + "_prepend", value, ignore=True, parsing=True)
 
     def delVar(self, var, **loginfo):
+        self.expand_cache = {}
         if '_remote_data' in self.dict:
             connector = self.dict["_remote_data"]["_content"]
             res = connector.delVar(var)
@@ -669,7 +673,6 @@ class DataSmart(MutableMapping):
         loginfo['detail'] = ""
         loginfo['op'] = 'del'
         self.varhistory.record(**loginfo)
-        self.expand_cache = {}
         self.dict[var] = {}
         if var in self.overridedata:
             del self.overridedata[var]
@@ -692,13 +695,13 @@ class DataSmart(MutableMapping):
                          override = None
 
     def setVarFlag(self, var, flag, value, **loginfo):
+        self.expand_cache = {}
         if '_remote_data' in self.dict:
             connector = self.dict["_remote_data"]["_content"]
             res = connector.setVarFlag(var, flag, value)
             if not res:
                 return
 
-        self.expand_cache = {}
         if 'op' not in loginfo:
             loginfo['op'] = "set"
         loginfo['flag'] = flag
@@ -719,9 +722,21 @@ class DataSmart(MutableMapping):
                 self.dict["__exportlist"]["_content"] = set()
             self.dict["__exportlist"]["_content"].add(var)
 
-    def getVarFlag(self, var, flag, expand=True, noweakdefault=False, parsing=False):
+    def getVarFlag(self, var, flag, expand=True, noweakdefault=False, parsing=False, retparser=False):
+        if flag == "_content":
+            cachename = var
+        else:
+            if not flag:
+                bb.warn("Calling getVarFlag with flag unset is invalid")
+                return None
+            cachename = var + "[" + flag + "]"
+
+        if expand and cachename in self.expand_cache:
+            return self.expand_cache[cachename].value
+
         local_var, overridedata = self._findVar(var)
         value = None
+        removes = set()
         if flag == "_content" and overridedata is not None and not parsing:
             match = False
             active = {}
@@ -748,7 +763,11 @@ class DataSmart(MutableMapping):
                             match = active[a]
                             del active[a]
             if match:
-                value = self.getVar(match, False)
+                value, subparser = self.getVarFlag(match, "_content", False, retparser=True)
+                if hasattr(subparser, "removes"):
+                    # We have to carry the removes from the overridden variable to apply at the
+                    # end of processing
+                    removes = subparser.removes
 
         if local_var is not None and value is None:
             if flag in local_var:
@@ -784,17 +803,13 @@ class DataSmart(MutableMapping):
                 if match:
                     value = r + value
 
-        if expand and value:
-            # Only getvar (flag == _content) hits the expand cache
-            cachename = None
-            if flag == "_content":
-                cachename = var
-            else:
-                cachename = var + "[" + flag + "]"
-            value = self.expand(value, cachename)
+        parser = None
+        if expand or retparser:
+            parser = self.expandWithRefs(value, cachename)
+        if expand:
+            value = parser.value
 
-        if value and flag == "_content" and local_var is not None and "_remove" in local_var:
-            removes = []
+        if value and flag == "_content" and local_var is not None and "_remove" in local_var and not parsing:
             self.need_overrides()
             for (r, o) in local_var["_remove"]:
                 match = True
@@ -803,26 +818,45 @@ class DataSmart(MutableMapping):
                         if not o2 in self.overrides:
                             match = False                            
                 if match:
-                    removes.extend(self.expand(r).split())
+                    removes.add(r)
+
+        if value and flag == "_content" and not parsing:
+            if removes and parser:
+                expanded_removes = {}
+                for r in removes:
+                    expanded_removes[r] = self.expand(r).split()
+
+                parser.removes = set()
+                val = ""
+                for v in __whitespace_split__.split(parser.value):
+                    skip = False
+                    for r in removes:
+                        if v in expanded_removes[r]:
+                            parser.removes.add(r)
+                            skip = True
+                    if skip:
+                        continue
+                    val = val + v
+                parser.value = val
+                if expand:
+                    value = parser.value
+
+        if parser:
+            self.expand_cache[cachename] = parser
+
+        if retparser:
+            return value, parser
 
-            if removes:
-                filtered = filter(lambda v: v not in removes,
-                                  value.split())
-                value = " ".join(filtered)
-                if expand and var in self.expand_cache:
-                    # We need to ensure the expand cache has the correct value
-                    # flag == "_content" here
-                    self.expand_cache[var].value = value
         return value
 
     def delVarFlag(self, var, flag, **loginfo):
+        self.expand_cache = {}
         if '_remote_data' in self.dict:
             connector = self.dict["_remote_data"]["_content"]
             res = connector.delVarFlag(var, flag)
             if not res:
                 return
 
-        self.expand_cache = {}
         local_var, _ = self._findVar(var)
         if not local_var:
             return

bitbake/lib/bb/fetch2/__init__.py

@@ -837,14 +837,16 @@ def runfetchcmd(cmd, d, quiet=False, cleanup=None, log=None, workdir=None):
     if not cleanup:
         cleanup = []
 
-    # If PATH contains WORKDIR which contains PV which contains SRCPV we
+    # If PATH contains WORKDIR which contains PV-PR which contains SRCPV we
     # can end up in circular recursion here so give the option of breaking it
     # in a data store copy.
     try:
         d.getVar("PV")
+        d.getVar("PR")
     except bb.data_smart.ExpansionError:
         d = bb.data.createCopy(d)
         d.setVar("PV", "fetcheravoidrecurse")
+        d.setVar("PR", "fetcheravoidrecurse")
 
     origenv = d.getVar("BB_ORIGENV", False)
     for var in exportvars:
@@ -1016,16 +1018,7 @@ def try_mirror_url(fetch, origud, ud, ld, check = False):
                     origud.method.build_mirror_data(origud, ld)
             return origud.localpath
         # Otherwise the result is a local file:// and we symlink to it
-        if not os.path.exists(origud.localpath):
-            if os.path.islink(origud.localpath):
-                # Broken symbolic link
-                os.unlink(origud.localpath)
-
-            # As per above, in case two tasks end up here simultaneously.
-            try:
-                os.symlink(ud.localpath, origud.localpath)
-            except FileExistsError:
-                pass
+        ensure_symlink(ud.localpath, origud.localpath)
         update_stamp(origud, ld)
         return ud.localpath
 
@@ -1059,6 +1052,22 @@ def try_mirror_url(fetch, origud, ud, ld, check = False):
             bb.utils.unlockfile(lf)
 
 
+def ensure_symlink(target, link_name):
+    if not os.path.exists(link_name):
+        if os.path.islink(link_name):
+            # Broken symbolic link
+            os.unlink(link_name)
+
+        # In case this is executing without any file locks held (as is
+        # the case for file:// URLs), two tasks may end up here at the
+        # same time, in which case we do not want the second task to
+        # fail when the link has already been created by the first task.
+        try:
+            os.symlink(target, link_name)
+        except FileExistsError:
+            pass
+
+
 def try_mirrors(fetch, d, origud, mirrors, check = False):
     """
     Try to use a mirrored version of the sources.
@@ -1088,7 +1097,9 @@ def trusted_network(d, url):
         return True
 
     pkgname = d.expand(d.getVar('PN', False))
-    trusted_hosts = d.getVarFlag('BB_ALLOWED_NETWORKS', pkgname, False)
+    trusted_hosts = None
+    if pkgname:
+        trusted_hosts = d.getVarFlag('BB_ALLOWED_NETWORKS', pkgname, False)
 
     if not trusted_hosts:
         trusted_hosts = d.getVar('BB_ALLOWED_NETWORKS')

bitbake/lib/bb/fetch2/git.py

@@ -299,17 +299,22 @@ class Git(FetchMethod):
         return ud.clonedir
 
     def need_update(self, ud, d):
+        return self.clonedir_need_update(ud, d) or self.shallow_tarball_need_update(ud) or self.tarball_need_update(ud)
+
+    def clonedir_need_update(self, ud, d):
         if not os.path.exists(ud.clonedir):
             return True
         for name in ud.names:
             if not self._contains_ref(ud, d, name, ud.clonedir):
                 return True
-        if ud.shallow and ud.write_shallow_tarballs and not os.path.exists(ud.fullshallow):
-            return True
-        if ud.write_tarballs and not os.path.exists(ud.fullmirror):
-            return True
         return False
 
+    def shallow_tarball_need_update(self, ud):
+        return ud.shallow and ud.write_shallow_tarballs and not os.path.exists(ud.fullshallow)
+
+    def tarball_need_update(self, ud):
+        return ud.write_tarballs and not os.path.exists(ud.fullmirror)
+
     def try_premirror(self, ud, d):
         # If we don't do this, updating an existing checkout with only premirrors
         # is not possible
@@ -471,11 +476,27 @@ class Git(FetchMethod):
         if os.path.exists(destdir):
             bb.utils.prunedir(destdir)
 
-        if ud.shallow and self.need_update(ud, d):
-            bb.utils.mkdirhier(destdir)
-            runfetchcmd("tar -xzf %s" % ud.fullshallow, d, workdir=destdir)
-        else:
-            runfetchcmd("%s clone %s %s/ %s" % (ud.basecmd, ud.cloneflags, ud.clonedir, destdir), d)
+        source_found = False
+        source_error = []
+
+        if not source_found:
+            clonedir_is_up_to_date = not self.clonedir_need_update(ud, d)
+            if clonedir_is_up_to_date:
+                runfetchcmd("%s clone %s %s/ %s" % (ud.basecmd, ud.cloneflags, ud.clonedir, destdir), d)
+                source_found = True
+            else:
+                source_error.append("clone directory not available or not up to date: " + ud.clonedir)
+
+        if not source_found:
+            if ud.shallow and os.path.exists(ud.fullshallow):
+                bb.utils.mkdirhier(destdir)
+                runfetchcmd("tar -xzf %s" % ud.fullshallow, d, workdir=destdir)
+                source_found = True
+            else:
+                source_error.append("shallow clone not enabled or not available: " + ud.fullshallow)
+
+        if not source_found:
+            raise bb.fetch2.UnpackError("No up to date source found: " + "; ".join(source_error), ud.url)
 
         repourl = self._get_repo_url(ud)
         runfetchcmd("%s remote set-url origin %s" % (ud.basecmd, repourl), d, workdir=destdir)

bitbake/lib/bb/fetch2/gitsm.py

@@ -31,9 +31,12 @@ NOTE: Switching a SRC_URI from "git://" to "gitsm://" requires a clean of your r
 
 import os
 import bb
+import copy
 from   bb.fetch2.git import Git
 from   bb.fetch2 import runfetchcmd
 from   bb.fetch2 import logger
+from   bb.fetch2 import Fetch
+from   bb.fetch2 import BBFetchException
 
 class GitSM(Git):
     def supports(self, ud, d):
@@ -42,96 +45,81 @@ class GitSM(Git):
         """
         return ud.type in ['gitsm']
 
-    def uses_submodules(self, ud, d, wd):
+    @staticmethod
+    def parse_gitmodules(gitmodules):
+        modules = {}
+        module = ""
+        for line in gitmodules.splitlines():
+            if line.startswith('[submodule'):
+                module = line.split('"')[1]
+                modules[module] = {}
+            elif module and line.strip().startswith('path'):
+                path = line.split('=')[1].strip()
+                modules[module]['path'] = path
+            elif module and line.strip().startswith('url'):
+                url = line.split('=')[1].strip()
+                modules[module]['url'] = url
+        return modules
+
+    def update_submodules(self, ud, d):
+        submodules = []
+        paths = {}
+        uris = {}
+        local_paths = {}
+
         for name in ud.names:
             try:
-                runfetchcmd("%s show %s:.gitmodules" % (ud.basecmd, ud.revisions[name]), d, quiet=True, workdir=wd)
-                return True
-            except bb.fetch.FetchError:
-                pass
-        return False
+                gitmodules = runfetchcmd("%s show %s:.gitmodules" % (ud.basecmd, ud.revisions[name]), d, quiet=True, workdir=ud.clonedir)
+            except:
+                # No submodules to update
+                continue
 
-    def _set_relative_paths(self, repopath):
-        """
-        Fix submodule paths to be relative instead of absolute,
-        so that when we move the repo it doesn't break
-        (In Git 1.7.10+ this is done automatically)
-        """
-        submodules = []
-        with open(os.path.join(repopath, '.gitmodules'), 'r') as f:
-            for line in f.readlines():
-                if line.startswith('[submodule'):
-                    submodules.append(line.split('"')[1])
+            for m, md in self.parse_gitmodules(gitmodules).items():
+                submodules.append(m)
+                paths[m] = md['path']
+                uris[m] = md['url']
+                if uris[m].startswith('..'):
+                    newud = copy.copy(ud)
+                    newud.path = os.path.realpath(os.path.join(newud.path, md['url']))
+                    uris[m] = Git._get_repo_url(self, newud)
 
         for module in submodules:
-            repo_conf = os.path.join(repopath, module, '.git')
-            if os.path.exists(repo_conf):
-                with open(repo_conf, 'r') as f:
-                    lines = f.readlines()
-                newpath = ''
-                for i, line in enumerate(lines):
-                    if line.startswith('gitdir:'):
-                        oldpath = line.split(': ')[-1].rstrip()
-                        if oldpath.startswith('/'):
-                            newpath = '../' * (module.count('/') + 1) + '.git/modules/' + module
-                            lines[i] = 'gitdir: %s\n' % newpath
-                            break
-                if newpath:
-                    with open(repo_conf, 'w') as f:
-                        for line in lines:
-                            f.write(line)
+            module_hash = runfetchcmd("%s ls-tree -z -d %s %s" % (ud.basecmd, ud.revisions[name], paths[module]), d, quiet=True, workdir=ud.clonedir)
+            module_hash = module_hash.split()[2]
 
-            repo_conf2 = os.path.join(repopath, '.git', 'modules', module, 'config')
-            if os.path.exists(repo_conf2):
-                with open(repo_conf2, 'r') as f:
-                    lines = f.readlines()
-                newpath = ''
-                for i, line in enumerate(lines):
-                    if line.lstrip().startswith('worktree = '):
-                        oldpath = line.split(' = ')[-1].rstrip()
-                        if oldpath.startswith('/'):
-                            newpath = '../' * (module.count('/') + 3) + module
-                            lines[i] = '\tworktree = %s\n' % newpath
-                            break
-                if newpath:
-                    with open(repo_conf2, 'w') as f:
-                        for line in lines:
-                            f.write(line)
+            # Build new SRC_URI
+            proto = uris[module].split(':', 1)[0]
+            url = uris[module].replace('%s:' % proto, 'gitsm:', 1)
+            url += ';protocol=%s' % proto
+            url += ";name=%s" % module
+            url += ";bareclone=1;nocheckout=1"
 
-    def update_submodules(self, ud, d, allow_network):
-        # We have to convert bare -> full repo, do the submodule bit, then convert back
-        tmpclonedir = ud.clonedir + ".tmp"
-        gitdir = tmpclonedir + os.sep + ".git"
-        bb.utils.remove(tmpclonedir, True)
-        os.mkdir(tmpclonedir)
-        os.rename(ud.clonedir, gitdir)
-        runfetchcmd("sed " + gitdir + "/config -i -e 's/bare.*=.*true/bare = false/'", d)
-        runfetchcmd(ud.basecmd + " reset --hard", d, workdir=tmpclonedir)
-        runfetchcmd(ud.basecmd + " checkout -f " + ud.revisions[ud.names[0]], d, workdir=tmpclonedir)
+            ld = d.createCopy()
+            # Not necessary to set SRC_URI, since we're passing the URI to
+            # Fetch.
+            #ld.setVar('SRC_URI', url)
+            ld.setVar('SRCREV_%s' % module, module_hash)
 
-        try:
-            if allow_network:
-                fetch_flags = ""
-            else:
-                fetch_flags = "--no-fetch"
+            # Workaround for issues with SRCPV/SRCREV_FORMAT errors
+            # error refer to 'multiple' repositories.  Only the repository
+            # in the original SRC_URI actually matters...
+            ld.setVar('SRCPV', d.getVar('SRCPV'))
+            ld.setVar('SRCREV_FORMAT', module)
 
-            # The 'git submodule sync' sandwiched between two successive 'git submodule update' commands is
-            # intentional. See the notes on the similar construction in download() for an explanation.
-            runfetchcmd("%(basecmd)s submodule update --init --recursive %(fetch_flags)s || (%(basecmd)s submodule sync --recursive && %(basecmd)s submodule update --init --recursive %(fetch_flags)s)" % {'basecmd': ud.basecmd, 'fetch_flags' : fetch_flags}, d, workdir=tmpclonedir)
-        except bb.fetch.FetchError:
-            if allow_network:
-                raise
-            else:
-                # This method was called as a probe to see whether the submodule history
-                # is complete enough to allow the current working copy to have its
-                # modules filled in. It's not, so swallow up the exception and report
-                # the negative result.
-                return False
-        finally:
-            self._set_relative_paths(tmpclonedir)
-            runfetchcmd("sed " + gitdir + "/config -i -e 's/bare.*=.*false/bare = true/'", d, workdir=tmpclonedir)
-            os.rename(gitdir, ud.clonedir,)
-            bb.utils.remove(tmpclonedir, True)
+            newfetch = Fetch([url], ld, cache=False)
+            newfetch.download()
+            local_paths[module] = newfetch.localpath(url)
+
+            # Correct the submodule references to the local download version...
+            runfetchcmd("%(basecmd)s config submodule.%(module)s.url %(url)s" % {'basecmd': ud.basecmd, 'module': module, 'url' : local_paths[module]}, d, workdir=ud.clonedir)
+
+            symlink_path = os.path.join(ud.clonedir, 'modules', paths[module])
+            if not os.path.exists(symlink_path):
+                try:
+                    os.makedirs(os.path.dirname(symlink_path), exist_ok=True)
+                except OSError:
+                    pass
+                os.symlink(local_paths[module], symlink_path)
 
         return True
 
@@ -147,56 +135,117 @@ class GitSM(Git):
         # Now check that the submodule histories are new enough. The git-submodule command doesn't have
         # any clean interface for doing this aside from just attempting the checkout (with network
         # fetched disabled).
-        return not self.update_submodules(ud, d, allow_network=False)
+        return not self.update_submodules(ud, d)
 
     def download(self, ud, d):
         Git.download(self, ud, d)
 
         if not ud.shallow or ud.localpath != ud.fullshallow:
-            submodules = self.uses_submodules(ud, d, ud.clonedir)
-            if submodules:
-                self.update_submodules(ud, d, allow_network=True)
+            self.update_submodules(ud, d)
+
+    def copy_submodules(self, submodules, ud, destdir, d):
+        if ud.bareclone:
+            repo_conf = destdir
+        else:
+            repo_conf = os.path.join(destdir, '.git')
+
+        if submodules and not os.path.exists(os.path.join(repo_conf, 'modules')):
+            os.mkdir(os.path.join(repo_conf, 'modules'))
+
+        for module in submodules:
+            srcpath = os.path.join(ud.clonedir, 'modules', module)
+            modpath = os.path.join(repo_conf, 'modules', module)
+
+            if os.path.exists(srcpath):
+                if os.path.exists(os.path.join(srcpath, '.git')):
+                    srcpath = os.path.join(srcpath, '.git')
+
+                target = modpath
+                if os.path.exists(modpath):
+                    target = os.path.dirname(modpath)
+
+                os.makedirs(os.path.dirname(target), exist_ok=True)
+                runfetchcmd("cp -fpLR %s %s" % (srcpath, target), d)
+            elif os.path.exists(modpath):
+                # Module already exists, likely unpacked from a shallow mirror clone
+                pass
+            else:
+                # This is fatal, as we do NOT want git-submodule to hit the network
+                raise bb.fetch2.FetchError('Submodule %s does not exist in %s or %s.' % (module, srcpath, modpath))
 
     def clone_shallow_local(self, ud, dest, d):
         super(GitSM, self).clone_shallow_local(ud, dest, d)
 
-        runfetchcmd('cp -fpPRH "%s/modules" "%s/"' % (ud.clonedir, os.path.join(dest, '.git')), d)
+        # Copy over the submodules' fetched histories too.
+        repo_conf = os.path.join(dest, '.git')
+
+        submodules = []
+        for name in ud.names:
+            try:
+                gitmodules = runfetchcmd("%s show %s:.gitmodules" % (ud.basecmd, ud.revision), d, quiet=True, workdir=dest)
+            except:
+                # No submodules to update
+                continue
+
+            submodules = list(self.parse_gitmodules(gitmodules).keys())
+
+        self.copy_submodules(submodules, ud, dest, d)
 
     def unpack(self, ud, destdir, d):
         Git.unpack(self, ud, destdir, d)
 
-        if self.uses_submodules(ud, d, ud.destdir):
-            runfetchcmd(ud.basecmd + " checkout " + ud.revisions[ud.names[0]], d, workdir=ud.destdir)
+        # Copy over the submodules' fetched histories too.
+        if ud.bareclone:
+            repo_conf = ud.destdir
+        else:
+            repo_conf = os.path.join(ud.destdir, '.git')
 
-            # Copy over the submodules' fetched histories too.
-            if ud.bareclone:
-                repo_conf = ud.destdir
-            else:
-                repo_conf = os.path.join(ud.destdir, '.git')
+        submodules = []
+        paths = {}
+        uris = {}
+        local_paths = {}
+        for name in ud.names:
+            try:
+                gitmodules = runfetchcmd("%s show HEAD:.gitmodules" % (ud.basecmd), d, quiet=True, workdir=ud.destdir)
+            except:
+                # No submodules to update
+                continue
 
-            if os.path.exists(ud.clonedir):
-                # This is not a copy unpacked from a shallow mirror clone. So
-                # the manual intervention to populate the .git/modules done
-                # in clone_shallow_local() won't have been done yet.
-                runfetchcmd("cp -fpPRH %s %s" % (os.path.join(ud.clonedir, 'modules'), repo_conf), d)
-                fetch_flags = "--no-fetch"
-            elif os.path.exists(os.path.join(repo_conf, 'modules')):
-                # Unpacked from a shallow mirror clone. Manual population of
-                # .git/modules is already done.
-                fetch_flags = "--no-fetch"
-            else:
-                # This isn't fatal; git-submodule will just fetch it
-                # during do_unpack().
-                fetch_flags = ""
-                bb.error("submodule history not retrieved during do_fetch()")
+            for m, md in self.parse_gitmodules(gitmodules).items():
+                submodules.append(m)
+                paths[m] = md['path']
+                uris[m] = md['url']
 
-            # Careful not to hit the network during unpacking; all history should already
-            # be fetched.
-            #
-            # The repeated attempts to do the submodule initialization sandwiched around a sync to
-            # install the correct remote URLs into the submodules' .git/config metadata are deliberate.
-            # Bad remote URLs are leftover in the modules' .git/config files from the unpack of bare
-            # clone tarballs and an initial 'git submodule update' is necessary to prod them back to
-            # enough life so that the 'git submodule sync' realizes the existing module .git/config
-            # files exist to be updated.
-            runfetchcmd("%(basecmd)s submodule update --init --recursive %(fetch_flags)s || (%(basecmd)s submodule sync --recursive && %(basecmd)s submodule update --init --recursive %(fetch_flags)s)" % {'basecmd': ud.basecmd, 'fetch_flags': fetch_flags}, d, workdir=ud.destdir)
+        self.copy_submodules(submodules, ud, ud.destdir, d)
+
+        submodules_queue = [(module, os.path.join(repo_conf, 'modules', module)) for module in submodules]
+        while len(submodules_queue) != 0:
+            module, modpath = submodules_queue.pop()
+
+            # add submodule children recursively
+            try:
+                gitmodules = runfetchcmd("%s show HEAD:.gitmodules" % (ud.basecmd), d, quiet=True, workdir=modpath)
+                for m, md in self.parse_gitmodules(gitmodules).items():
+                    submodules_queue.append([m, os.path.join(modpath, 'modules', m)])
+            except:
+                # no children
+                pass
+
+            # Determine (from the submodule) the correct url to reference
+            try:
+                output = runfetchcmd("%(basecmd)s config remote.origin.url" % {'basecmd': ud.basecmd}, d, workdir=modpath)
+            except bb.fetch2.FetchError as e:
+                # No remote url defined in this submodule
+                continue
+
+            local_paths[module] = output
+
+            # Setup the local URL properly (like git submodule init or sync would do...)
+            runfetchcmd("%(basecmd)s config submodule.%(module)s.url %(url)s" % {'basecmd': ud.basecmd, 'module': module, 'url' : local_paths[module]}, d, workdir=ud.destdir)
+
+            # Ensure the submodule repository is NOT set to bare, since we're checking it out...
+            runfetchcmd("%s config core.bare false" % (ud.basecmd), d, quiet=True, workdir=modpath)
+
+        if submodules:
+            # Run submodule update, this sets up the directories -- without touching the config
+            runfetchcmd("%s submodule update --recursive --no-fetch" % (ud.basecmd), d, quiet=True, workdir=ud.destdir)

bitbake/lib/bb/main.py

@@ -405,9 +405,6 @@ def setup_bitbake(configParams, configuration, extrafeatures=None):
         # In status only mode there are no logs and no UI
         logger.addHandler(handler)
 
-    # Clear away any spurious environment variables while we stoke up the cooker
-    cleanedvars = bb.utils.clean_environment()
-
     if configParams.server_only:
         featureset = []
         ui_module = None
@@ -423,6 +420,10 @@ def setup_bitbake(configParams, configuration, extrafeatures=None):
 
     server_connection = None
 
+    # Clear away any spurious environment variables while we stoke up the cooker
+    # (done after import_extension_module() above since for example import gi triggers env var usage)
+    cleanedvars = bb.utils.clean_environment()
+
     if configParams.remote_server:
         # Connect to a remote XMLRPC server
         server_connection = bb.server.xmlrpcclient.connectXMLRPC(configParams.remote_server, featureset,

bitbake/lib/bb/siggen.py

@@ -110,42 +110,13 @@ class SignatureGeneratorBasic(SignatureGenerator):
         ignore_mismatch = ((d.getVar("BB_HASH_IGNORE_MISMATCH") or '') == '1')
         tasklist, gendeps, lookupcache = bb.data.generate_dependencies(d)
 
-        taskdeps = {}
-        basehash = {}
+        taskdeps, basehash = bb.data.generate_dependency_hash(tasklist, gendeps, lookupcache, self.basewhitelist, fn)
 
         for task in tasklist:
-            data = lookupcache[task]
-
-            if data is None:
-                bb.error("Task %s from %s seems to be empty?!" % (task, fn))
-                data = ''
-
-            gendeps[task] -= self.basewhitelist
-            newdeps = gendeps[task]
-            seen = set()
-            while newdeps:
-                nextdeps = newdeps
-                seen |= nextdeps
-                newdeps = set()
-                for dep in nextdeps:
-                    if dep in self.basewhitelist:
-                        continue
-                    gendeps[dep] -= self.basewhitelist
-                    newdeps |= gendeps[dep]
-                newdeps -= seen
-
-            alldeps = sorted(seen)
-            for dep in alldeps:
-                data = data + dep
-                var = lookupcache[dep]
-                if var is not None:
-                    data = data + str(var)
-            datahash = hashlib.md5(data.encode("utf-8")).hexdigest()
             k = fn + "." + task
-            if not ignore_mismatch and k in self.basehash and self.basehash[k] != datahash:
-                bb.error("When reparsing %s, the basehash value changed from %s to %s. The metadata is not deterministic and this needs to be fixed." % (k, self.basehash[k], datahash))
-            self.basehash[k] = datahash
-            taskdeps[task] = alldeps
+            if not ignore_mismatch and k in self.basehash and self.basehash[k] != basehash[k]:
+                bb.error("When reparsing %s, the basehash value changed from %s to %s. The metadata is not deterministic and this needs to be fixed." % (k, self.basehash[k], basehash[k]))
+            self.basehash[k] = basehash[k]
 
         self.taskdeps[fn] = taskdeps
         self.gendeps[fn] = gendeps

bitbake/lib/bb/tests/data.py

@@ -281,7 +281,7 @@ class TestConcatOverride(unittest.TestCase):
     def test_remove(self):
         self.d.setVar("TEST", "${VAL} ${BAR}")
         self.d.setVar("TEST_remove", "val")
-        self.assertEqual(self.d.getVar("TEST"), "bar")
+        self.assertEqual(self.d.getVar("TEST"), " bar")
 
     def test_remove_cleared(self):
         self.d.setVar("TEST", "${VAL} ${BAR}")
@@ -300,7 +300,7 @@ class TestConcatOverride(unittest.TestCase):
         self.d.setVar("TEST", "${VAL} ${BAR}")
         self.d.setVar("TEST_remove", "val")
         self.d.setVar("TEST_TEST", "${TEST} ${TEST}")
-        self.assertEqual(self.d.getVar("TEST_TEST"), "bar bar")
+        self.assertEqual(self.d.getVar("TEST_TEST"), " bar  bar")
 
     def test_empty_remove(self):
         self.d.setVar("TEST", "")
@@ -311,13 +311,25 @@ class TestConcatOverride(unittest.TestCase):
         self.d.setVar("BAR", "Z")
         self.d.setVar("TEST", "${BAR}/X Y")
         self.d.setVar("TEST_remove", "${BAR}/X")
-        self.assertEqual(self.d.getVar("TEST"), "Y")
+        self.assertEqual(self.d.getVar("TEST"), " Y")
 
     def test_remove_expansion_items(self):
         self.d.setVar("TEST", "A B C D")
         self.d.setVar("BAR", "B D")
         self.d.setVar("TEST_remove", "${BAR}")
-        self.assertEqual(self.d.getVar("TEST"), "A C")
+        self.assertEqual(self.d.getVar("TEST"), "A  C ")
+
+    def test_remove_preserve_whitespace(self):
+        # When the removal isn't active, the original value should be preserved
+        self.d.setVar("TEST", " A B")
+        self.d.setVar("TEST_remove", "C")
+        self.assertEqual(self.d.getVar("TEST"), " A B")
+
+    def test_remove_preserve_whitespace2(self):
+        # When the removal is active preserve the whitespace
+        self.d.setVar("TEST", " A B")
+        self.d.setVar("TEST_remove", "B")
+        self.assertEqual(self.d.getVar("TEST"), " A ")
 
 class TestOverrides(unittest.TestCase):
     def setUp(self):
@@ -374,6 +386,15 @@ class TestOverrides(unittest.TestCase):
         self.d.setVar("OVERRIDES", "foo:bar:some_val")
         self.assertEqual(self.d.getVar("TEST"), "testvalue3")
 
+    def test_remove_with_override(self):
+        self.d.setVar("TEST_bar", "testvalue2")
+        self.d.setVar("TEST_some_val", "testvalue3 testvalue5")
+        self.d.setVar("TEST_some_val_remove", "testvalue3")
+        self.d.setVar("TEST_foo", "testvalue4")
+        self.d.setVar("OVERRIDES", "foo:bar:some_val")
+        self.assertEqual(self.d.getVar("TEST"), " testvalue5")
+
+
 class TestKeyExpansion(unittest.TestCase):
     def setUp(self):
         self.d = bb.data.init()
@@ -443,6 +464,54 @@ class Contains(unittest.TestCase):
         self.assertFalse(bb.utils.contains_any("SOMEFLAG", "x y z", True, False, self.d))
 
 
+class TaskHash(unittest.TestCase):
+    def test_taskhashes(self):
+        def gettask_bashhash(taskname, d):
+            tasklist, gendeps, lookupcache = bb.data.generate_dependencies(d)
+            taskdeps, basehash = bb.data.generate_dependency_hash(tasklist, gendeps, lookupcache, set(), "somefile")
+            bb.warn(str(lookupcache))
+            return basehash["somefile." + taskname]
+
+        d = bb.data.init()
+        d.setVar("__BBTASKS", ["mytask"])
+        d.setVar("__exportlist", [])
+        d.setVar("mytask", "${MYCOMMAND}")
+        d.setVar("MYCOMMAND", "${VAR}; foo; bar; exit 0")
+        d.setVar("VAR", "val")
+        orighash = gettask_bashhash("mytask", d)
+
+        # Changing a variable should change the hash
+        d.setVar("VAR", "val2")
+        nexthash = gettask_bashhash("mytask", d)
+        self.assertNotEqual(orighash, nexthash)
+
+        d.setVar("VAR", "val")
+        # Adding an inactive removal shouldn't change the hash
+        d.setVar("BAR", "notbar")
+        d.setVar("MYCOMMAND_remove", "${BAR}")
+        nexthash = gettask_bashhash("mytask", d)
+        self.assertEqual(orighash, nexthash)
+
+        # Adding an active removal should change the hash
+        d.setVar("BAR", "bar;")
+        nexthash = gettask_bashhash("mytask", d)
+        self.assertNotEqual(orighash, nexthash)
+
+        # Setup an inactive contains()
+        d.setVar("VAR", "${@bb.utils.contains('VAR2', 'A', 'val', '', d)}")
+        orighash = gettask_bashhash("mytask", d)
+
+        # Activate the contains() and the hash should change
+        d.setVar("VAR2", "A")
+        nexthash = gettask_bashhash("mytask", d)
+        self.assertNotEqual(orighash, nexthash)
+
+        # The contains should be inactive but even though VAR2 has a
+        # different value the hash should match the original
+        d.setVar("VAR2", "B")
+        nexthash = gettask_bashhash("mytask", d)
+        self.assertEqual(orighash, nexthash)
+
 class Serialize(unittest.TestCase):
 
     def test_serialize(self):

bitbake/lib/bb/tests/fetch.py

@@ -463,6 +463,124 @@ class MirrorUriTest(FetcherTest):
                                 'https://BBBB/B/B/B/bitbake/bitbake-1.0.tar.gz',
                                 'http://AAAA/A/A/A/B/B/bitbake/bitbake-1.0.tar.gz'])
 
+
+class GitDownloadDirectoryNamingTest(FetcherTest):
+    def setUp(self):
+        super(GitDownloadDirectoryNamingTest, self).setUp()
+        self.recipe_url = "git://git.openembedded.org/bitbake"
+        self.recipe_dir = "git.openembedded.org.bitbake"
+        self.mirror_url = "git://github.com/openembedded/bitbake.git"
+        self.mirror_dir = "github.com.openembedded.bitbake.git"
+
+        self.d.setVar('SRCREV', '82ea737a0b42a8b53e11c9cde141e9e9c0bd8c40')
+
+    def setup_mirror_rewrite(self):
+        self.d.setVar("PREMIRRORS", self.recipe_url + " " + self.mirror_url + " \n")
+
+    @skipIfNoNetwork()
+    def test_that_directory_is_named_after_recipe_url_when_no_mirroring_is_used(self):
+        self.setup_mirror_rewrite()
+        fetcher = bb.fetch.Fetch([self.recipe_url], self.d)
+
+        fetcher.download()
+
+        dir = os.listdir(self.dldir + "/git2")
+        self.assertIn(self.recipe_dir, dir)
+
+    @skipIfNoNetwork()
+    def test_that_directory_exists_for_mirrored_url_and_recipe_url_when_mirroring_is_used(self):
+        self.setup_mirror_rewrite()
+        fetcher = bb.fetch.Fetch([self.recipe_url], self.d)
+
+        fetcher.download()
+
+        dir = os.listdir(self.dldir + "/git2")
+        self.assertIn(self.mirror_dir, dir)
+        self.assertIn(self.recipe_dir, dir)
+
+    @skipIfNoNetwork()
+    def test_that_recipe_directory_and_mirrored_directory_exists_when_mirroring_is_used_and_the_mirrored_directory_already_exists(self):
+        self.setup_mirror_rewrite()
+        fetcher = bb.fetch.Fetch([self.mirror_url], self.d)
+        fetcher.download()
+        fetcher = bb.fetch.Fetch([self.recipe_url], self.d)
+
+        fetcher.download()
+
+        dir = os.listdir(self.dldir + "/git2")
+        self.assertIn(self.mirror_dir, dir)
+        self.assertIn(self.recipe_dir, dir)
+
+
+class TarballNamingTest(FetcherTest):
+    def setUp(self):
+        super(TarballNamingTest, self).setUp()
+        self.recipe_url = "git://git.openembedded.org/bitbake"
+        self.recipe_tarball = "git2_git.openembedded.org.bitbake.tar.gz"
+        self.mirror_url = "git://github.com/openembedded/bitbake.git"
+        self.mirror_tarball = "git2_github.com.openembedded.bitbake.git.tar.gz"
+
+        self.d.setVar('BB_GENERATE_MIRROR_TARBALLS', '1')
+        self.d.setVar('SRCREV', '82ea737a0b42a8b53e11c9cde141e9e9c0bd8c40')
+
+    def setup_mirror_rewrite(self):
+        self.d.setVar("PREMIRRORS", self.recipe_url + " " + self.mirror_url + " \n")
+
+    @skipIfNoNetwork()
+    def test_that_the_recipe_tarball_is_created_when_no_mirroring_is_used(self):
+        fetcher = bb.fetch.Fetch([self.recipe_url], self.d)
+
+        fetcher.download()
+
+        dir = os.listdir(self.dldir)
+        self.assertIn(self.recipe_tarball, dir)
+
+    @skipIfNoNetwork()
+    def test_that_the_mirror_tarball_is_created_when_mirroring_is_used(self):
+        self.setup_mirror_rewrite()
+        fetcher = bb.fetch.Fetch([self.recipe_url], self.d)
+
+        fetcher.download()
+
+        dir = os.listdir(self.dldir)
+        self.assertIn(self.mirror_tarball, dir)
+
+
+class GitShallowTarballNamingTest(FetcherTest):
+    def setUp(self):
+        super(GitShallowTarballNamingTest, self).setUp()
+        self.recipe_url = "git://git.openembedded.org/bitbake"
+        self.recipe_tarball = "gitshallow_git.openembedded.org.bitbake_82ea737-1_master.tar.gz"
+        self.mirror_url = "git://github.com/openembedded/bitbake.git"
+        self.mirror_tarball = "gitshallow_github.com.openembedded.bitbake.git_82ea737-1_master.tar.gz"
+
+        self.d.setVar('BB_GIT_SHALLOW', '1')
+        self.d.setVar('BB_GENERATE_SHALLOW_TARBALLS', '1')
+        self.d.setVar('SRCREV', '82ea737a0b42a8b53e11c9cde141e9e9c0bd8c40')
+
+    def setup_mirror_rewrite(self):
+        self.d.setVar("PREMIRRORS", self.recipe_url + " " + self.mirror_url + " \n")
+
+    @skipIfNoNetwork()
+    def test_that_the_tarball_is_named_after_recipe_url_when_no_mirroring_is_used(self):
+        fetcher = bb.fetch.Fetch([self.recipe_url], self.d)
+
+        fetcher.download()
+
+        dir = os.listdir(self.dldir)
+        self.assertIn(self.recipe_tarball, dir)
+
+    @skipIfNoNetwork()
+    def test_that_the_mirror_tarball_is_created_when_mirroring_is_used(self):
+        self.setup_mirror_rewrite()
+        fetcher = bb.fetch.Fetch([self.recipe_url], self.d)
+
+        fetcher.download()
+
+        dir = os.listdir(self.dldir)
+        self.assertIn(self.mirror_tarball, dir)
+
+
 class FetcherLocalTest(FetcherTest):
     def setUp(self):
         def touch(fn):
@@ -751,27 +869,27 @@ class FetcherNetworkTest(FetcherTest):
         self.assertRaises(bb.fetch.ParameterError, self.gitfetcher, url, url)
 
     @skipIfNoNetwork()
-    def test_gitfetch_premirror(self):
-        url1 = "git://git.openembedded.org/bitbake"
-        url2 = "git://someserver.org/bitbake"
+    def test_gitfetch_finds_local_tarball_for_mirrored_url_when_previous_downloaded_by_the_recipe_url(self):
+        recipeurl = "git://git.openembedded.org/bitbake"
+        mirrorurl = "git://someserver.org/bitbake"
         self.d.setVar("PREMIRRORS", "git://someserver.org/bitbake git://git.openembedded.org/bitbake \n")
-        self.gitfetcher(url1, url2)
+        self.gitfetcher(recipeurl, mirrorurl)
 
     @skipIfNoNetwork()
-    def test_gitfetch_premirror2(self):
-        url1 = url2 = "git://someserver.org/bitbake"
+    def test_gitfetch_finds_local_tarball_when_previous_downloaded_from_a_premirror(self):
+        recipeurl = "git://someserver.org/bitbake"
         self.d.setVar("PREMIRRORS", "git://someserver.org/bitbake git://git.openembedded.org/bitbake \n")
-        self.gitfetcher(url1, url2)
+        self.gitfetcher(recipeurl, recipeurl)
 
     @skipIfNoNetwork()
-    def test_gitfetch_premirror3(self):
+    def test_gitfetch_finds_local_repository_when_premirror_rewrites_the_recipe_url(self):
         realurl = "git://git.openembedded.org/bitbake"
-        dummyurl = "git://someserver.org/bitbake"
+        recipeurl = "git://someserver.org/bitbake"
         self.sourcedir = self.unpackdir.replace("unpacked", "sourcemirror.git")
         os.chdir(self.tempdir)
         bb.process.run("git clone %s %s 2> /dev/null" % (realurl, self.sourcedir), shell=True)
-        self.d.setVar("PREMIRRORS", "%s git://%s;protocol=file \n" % (dummyurl, self.sourcedir))
-        self.gitfetcher(dummyurl, dummyurl)
+        self.d.setVar("PREMIRRORS", "%s git://%s;protocol=file \n" % (recipeurl, self.sourcedir))
+        self.gitfetcher(recipeurl, recipeurl)
 
     @skipIfNoNetwork()
     def test_git_submodule(self):
@@ -867,12 +985,12 @@ class FetchLatestVersionTest(FetcherTest):
         ("dtc", "git://git.qemu.org/dtc.git", "65cc4d2748a2c2e6f27f1cf39e07a5dbabd80ebf", "")
             : "1.4.0",
         # combination version pattern
-        ("sysprof", "git://gitlab.gnome.org/GNOME/sysprof;protocol=https", "cd44ee6644c3641507fb53b8a2a69137f2971219", "")
+        ("sysprof", "git://gitlab.gnome.org/GNOME/sysprof.git;protocol=https", "cd44ee6644c3641507fb53b8a2a69137f2971219", "")
             : "1.2.0",
         ("u-boot-mkimage", "git://git.denx.de/u-boot.git;branch=master;protocol=git", "62c175fbb8a0f9a926c88294ea9f7e88eb898f6c", "")
             : "2014.01",
         # version pattern "yyyymmdd"
-        ("mobile-broadband-provider-info", "git://gitlab.gnome.org/GNOME/mobile-broadband-provider-info;protocol=https", "4ed19e11c2975105b71b956440acdb25d46a347d", "")
+        ("mobile-broadband-provider-info", "git://gitlab.gnome.org/GNOME/mobile-broadband-provider-info.git;protocol=https", "4ed19e11c2975105b71b956440acdb25d46a347d", "")
             : "20120614",
         # packages with a valid UPSTREAM_CHECK_GITTAGREGEX
         ("xf86-video-omap", "git://anongit.freedesktop.org/xorg/driver/xf86-video-omap", "ae0394e687f1a77e966cf72f895da91840dffb8f", "(?P<pver>(\d+\.(\d\.?)*))")
@@ -1344,6 +1462,9 @@ class GitShallowTest(FetcherTest):
         smdir = os.path.join(self.tempdir, 'gitsubmodule')
         bb.utils.mkdirhier(smdir)
         self.git('init', cwd=smdir)
+        # Make this look like it was cloned from a remote...
+        self.git('config --add remote.origin.url "%s"' % smdir, cwd=smdir)
+        self.git('config --add remote.origin.fetch "+refs/heads/*:refs/remotes/origin/*"', cwd=smdir)
         self.add_empty_file('asub', cwd=smdir)
 
         self.git('submodule init', cwd=self.srcdir)
@@ -1577,3 +1698,30 @@ class GitShallowTest(FetcherTest):
         self.assertNotEqual(orig_revs, revs)
         self.assertRefs(['master', 'origin/master'])
         self.assertRevCount(orig_revs - 1758)
+
+    def test_that_unpack_throws_an_error_when_the_git_clone_nor_shallow_tarball_exist(self):
+        self.add_empty_file('a')
+        fetcher, ud = self.fetch()
+        bb.utils.remove(self.gitdir, recurse=True)
+        bb.utils.remove(self.dldir, recurse=True)
+
+        with self.assertRaises(bb.fetch2.UnpackError) as context:
+            fetcher.unpack(self.d.getVar('WORKDIR'))
+
+        self.assertTrue("No up to date source found" in context.exception.msg)
+        self.assertTrue("clone directory not available or not up to date" in context.exception.msg)
+        self.assertTrue("shallow clone not enabled or not available" in context.exception.msg)
+
+    @skipIfNoNetwork()
+    def test_that_unpack_does_work_when_using_git_shallow_tarball_but_tarball_is_not_available(self):
+        self.d.setVar('SRCREV', 'e5939ff608b95cdd4d0ab0e1935781ab9a276ac0')
+        self.d.setVar('BB_GIT_SHALLOW', '1')
+        self.d.setVar('BB_GENERATE_SHALLOW_TARBALLS', '1')
+        fetcher = bb.fetch.Fetch(["git://git.yoctoproject.org/fstests"], self.d)
+        fetcher.download()
+
+        bb.utils.remove(self.dldir + "/*.tar.gz")
+        fetcher.unpack(self.unpackdir)
+
+        dir = os.listdir(self.unpackdir + "/git/")
+        self.assertIn("fstests.doap", dir)

bitbake/lib/bb/tests/parse.py

@@ -44,9 +44,13 @@ C = "3"
 """
 
     def setUp(self):
+        self.origdir = os.getcwd()
         self.d = bb.data.init()
         bb.parse.siggen = bb.siggen.init(self.d)
 
+    def tearDown(self):
+        os.chdir(self.origdir)
+
     def parsehelper(self, content, suffix = ".bb"):
 
         f = tempfile.NamedTemporaryFile(suffix = suffix)

bitbake/lib/bb/ui/buildinfohelper.py

@@ -1603,14 +1603,14 @@ class BuildInfoHelper(object):
         mockevent.lineno = -1
         self.store_log_event(mockevent)
 
-    def store_log_event(self, event):
+    def store_log_event(self, event,cli_backlog=True):
         self._ensure_build()
 
         if event.levelno < formatter.WARNING:
             return
 
         # early return for CLI builds
-        if self.brbe is None:
+        if cli_backlog and self.brbe is None:
             if not 'backlog' in self.internal_state:
                 self.internal_state['backlog'] = []
             self.internal_state['backlog'].append(event)
@@ -1622,7 +1622,7 @@ class BuildInfoHelper(object):
                 tempevent = self.internal_state['backlog'].pop()
                 logger.debug(1, "buildinfohelper: Saving stored event %s "
                              % tempevent)
-                self.store_log_event(tempevent)
+                self.store_log_event(tempevent,cli_backlog)
             else:
                 logger.info("buildinfohelper: All events saved")
                 del self.internal_state['backlog']
@@ -1987,7 +1987,8 @@ class BuildInfoHelper(object):
         if 'backlog' in self.internal_state:
             # we save missed events in the database for the current build
             tempevent = self.internal_state['backlog'].pop()
-            self.store_log_event(tempevent)
+            # Do not skip command line build events
+            self.store_log_event(tempevent,False)
 
         if not connection.features.autocommits_when_autocommit_is_off:
             transaction.set_autocommit(True)

bitbake/lib/bb/ui/taskexp.py

@@ -103,9 +103,16 @@ class DepExplorer(Gtk.Window):
         self.pkg_treeview.get_selection().connect("changed", self.on_cursor_changed)
         column = Gtk.TreeViewColumn("Package", Gtk.CellRendererText(), text=COL_PKG_NAME)
         self.pkg_treeview.append_column(column)
-        pane.add1(scrolled)
         scrolled.add(self.pkg_treeview)
 
+        self.search_entry = Gtk.SearchEntry.new()
+        self.pkg_treeview.set_search_entry(self.search_entry)
+
+        left_panel = Gtk.VPaned()
+        left_panel.add(self.search_entry)
+        left_panel.add(scrolled)
+        pane.add1(left_panel)
+
         box = Gtk.VBox(homogeneous=True, spacing=4)
 
         # Task Depends
@@ -129,6 +136,7 @@ class DepExplorer(Gtk.Window):
         pane.add2(box)
 
         self.show_all()
+        self.search_entry.grab_focus()
 
     def on_package_activated(self, treeview, path, column, data_col):
         model = treeview.get_model()

bitbake/lib/bb/utils.py

@@ -497,7 +497,11 @@ def lockfile(name, shared=False, retry=True, block=False):
                 if statinfo.st_ino == statinfo2.st_ino:
                     return lf
             lf.close()
-        except Exception:
+        except OSError as e:
+            if e.errno == errno.EACCES:
+                logger.error("Unable to acquire lock '%s', %s",
+                             e.strerror, name)
+                sys.exit(1)
             try:
                 lf.close()
             except Exception:

bitbake/lib/layerindexlib/__init__.py

@@ -448,7 +448,7 @@ layerBranches set.  If not, they are effectively blank.'''
 This function is used to implement debugging and provide the user info.
 '''
         for lix in self.indexes:
-            if object not in lix:
+            if not hasattr(lix, object):
                 continue
 
             logger.plain ('')
@@ -1046,15 +1046,15 @@ class LayerBranch(LayerIndexItemObj):
         self.id = id
         self.collection = collection
         self.version = version
-        if type(layer) != type(LayerItem):
-            self.layer_id = layer
-        else:
+        if isinstance(layer, LayerItem):
             self.layer = layer
-
-        if type(branch) != type(Branch):
-            self.branch_id = branch
         else:
+            self.layer_id = layer
+
+        if isinstance(branch, Branch):
             self.branch = branch
+        else:
+            self.branch_id = branch
 
         self.vcs_subdir = vcs_subdir
         self.vcs_last_fetch = vcs_last_fetch
@@ -1088,7 +1088,7 @@ class LayerBranch(LayerIndexItemObj):
 
     @layer.setter
     def layer(self, value):
-        if type(value) != type(LayerItem):
+        if not isinstance(value, LayerItem):
             raise TypeError('value is not a LayerItem')
         if self.index != value.index:
             raise AttributeError('Object and value do not share the same index and thus key set.')
@@ -1122,7 +1122,7 @@ class LayerBranch(LayerIndexItemObj):
 
     @branch.setter
     def branch(self, value):
-        if type(value) != type(LayerItem):
+        if not isinstance(value, LayerItem):
             raise TypeError('value is not a LayerItem')
         if self.index != value.index:
             raise AttributeError('Object and value do not share the same index and thus key set.')
@@ -1181,7 +1181,7 @@ class LayerIndexItemObj_LayerBranch(LayerIndexItemObj):
 
     @layerbranch.setter
     def layerbranch(self, value):
-        if type(value) != type(LayerBranch):
+        if not isinstance(value, LayerBranch):
             raise TypeError('value (%s) is not a layerBranch' % type(value))
         if self.index != value.index:
             raise AttributeError('Object and value do not share the same index and thus key set.')
@@ -1207,14 +1207,14 @@ class LayerIndexItemObj_LayerBranch(LayerIndexItemObj):
 class LayerDependency(LayerIndexItemObj_LayerBranch):
     def define_data(self, id, layerbranch, dependency, required=True):
         self.id = id
-        if type(layerbranch) != type(LayerBranch):
-            self.layerbranch_id = layerbranch
-        else:
+        if isinstance(layerbranch, LayerBranch):
             self.layerbranch = layerbranch
-        if type(dependency) != type(LayerDependency):
-            self.dependency_id = dependency
         else:
+            self.layerbranch_id = layerbranch
+        if isinstance(dependency, LayerDependency):
             self.dependency = dependency
+        else:
+            self.dependency_id = dependency
         self.required = required
 
     @property
@@ -1240,7 +1240,7 @@ class LayerDependency(LayerIndexItemObj_LayerBranch):
 
     @dependency.setter
     def dependency(self, value):
-        if type(value) != type(LayerDependency):
+        if not isinstance(value, LayerDependency):
             raise TypeError('value (%s) is not a dependency' % type(value))
         if self.index != value.index:
             raise AttributeError('Object and value do not share the same index and thus key set.')
@@ -1288,10 +1288,10 @@ class Recipe(LayerIndexItemObj_LayerBranch):
         self.inherits = inherits
         self.updated = updated or datetime.datetime.today().isoformat()
         self.blacklisted = blacklisted
-        if type(layerbranch) != type(LayerBranch):
-            self.layerbranch_id = layerbranch
-        else:
+        if isinstance(layerbranch, LayerBranch):
             self.layerbranch = layerbranch
+        else:
+            self.layerbranch_id = layerbranch
 
     @property
     def fullpath(self):
@@ -1324,10 +1324,10 @@ class Machine(LayerIndexItemObj_LayerBranch):
         self.id = id
         self.name = name
         self.description = description
-        if type(layerbranch) != type(LayerBranch):
-            self.layerbranch_id = layerbranch
-        else:
+        if isinstance(layerbranch, LayerBranch):
             self.layerbranch = layerbranch
+        else:
+            self.layerbranch_id = layerbranch
         self.updated = updated or datetime.datetime.today().isoformat()
 
 class Distro(LayerIndexItemObj_LayerBranch):
@@ -1337,13 +1337,12 @@ class Distro(LayerIndexItemObj_LayerBranch):
         self.id = id
         self.name = name
         self.description = description
-        if type(layerbranch) != type(LayerBranch):
-            self.layerbranch_id = layerbranch
-        else:
+        if isinstance(layerbranch, LayerBranch):
             self.layerbranch = layerbranch
+        else:
+            self.layerbranch_id = layerbranch
         self.updated = updated or datetime.datetime.today().isoformat()
 
-
 # When performing certain actions, we may need to sort the data.
 # This will allow us to keep it consistent from run to run.
 def sort_entry(item):

bitbake/lib/layerindexlib/cooker.py

@@ -136,10 +136,13 @@ class CookerPlugin(layerindexlib.plugin.IndexPlugin):
                 layerrev = self._run_command('git rev-parse HEAD', layerpath, default="<unknown>")
 
                 for remotes in self._run_command('git remote -v', layerpath, default="").split("\n"):
-                    remote = remotes.split("\t")[1].split(" ")[0]
-                    if "(fetch)" == remotes.split("\t")[1].split(" ")[1]:
-                        layerurl = self._handle_git_remote(remote)
-                        break
+                    if not remotes:
+                        layerurl = self._handle_git_remote(layerpath)
+                    else:
+                        remote = remotes.split("\t")[1].split(" ")[0]
+                        if "(fetch)" == remotes.split("\t")[1].split(" ")[1]:
+                            layerurl = self._handle_git_remote(remote)
+                            break
 
             layerItemId += 1
             index.layerItems[layerItemId] = layerindexlib.LayerItem(index, None)
@@ -297,7 +300,7 @@ class CookerPlugin(layerindexlib.plugin.IndexPlugin):
 
             for layerBranchId in index.layerBranches:
                 # load_bblayers uses the description to cache the actual path...
-                machine_path = index.layerBranches[layerBranchId].getDescription()
+                machine_path = index.layerBranches[layerBranchId].layer.description
                 machine_path = os.path.join(machine_path, 'conf/machine')
                 if os.path.isdir(machine_path):
                     for (dirpath, _, filenames) in os.walk(machine_path):
@@ -310,7 +313,7 @@ class CookerPlugin(layerindexlib.plugin.IndexPlugin):
                                 machine = layerindexlib.Machine(index, None)
                                 machine.define_data(id=machineId, name=fname[:-5],
                                                     description=fname[:-5],
-                                                    layerbranch=collection_layerbranch[entry])
+                                                    layerbranch=index.layerBranches[layerBranchId])
 
                                 index.add_element("machines", [machine])
 
@@ -321,7 +324,7 @@ class CookerPlugin(layerindexlib.plugin.IndexPlugin):
 
             for layerBranchId in index.layerBranches:
                 # load_bblayers uses the description to cache the actual path...
-                distro_path = index.layerBranches[layerBranchId].getDescription()
+                distro_path = index.layerBranches[layerBranchId].layer.description
                 distro_path = os.path.join(distro_path, 'conf/distro')
                 if os.path.isdir(distro_path):
                     for (dirpath, _, filenames) in os.walk(distro_path):
@@ -334,7 +337,7 @@ class CookerPlugin(layerindexlib.plugin.IndexPlugin):
                                 distro = layerindexlib.Distro(index, None)
                                 distro.define_data(id=distroId, name=fname[:-5],
                                                     description=fname[:-5],
-                                                    layerbranch=collection_layerbranch[entry])
+                                                    layerbranch=index.layerBranches[layerBranchId])
 
                                 index.add_element("distros", [distro])
 

bitbake/lib/layerindexlib/tests/cooker.py

@@ -32,7 +32,7 @@ class LayerIndexCookerTest(LayersTest):
         # configure the test data.  But we can emulate the basics of the layer.conf
         # files, so that is what we will do.
 
-        new_topdir = os.path.join(os.path.dirname(__file__), "testdata")
+        new_topdir = os.path.join(os.path.dirname(os.path.realpath(__file__)), "testdata")
         new_bbpath = os.path.join(new_topdir, "build")
 
         self.d.setVar('TOPDIR', new_topdir)

bitbake/lib/layerindexlib/tests/restapi.py

@@ -23,113 +23,122 @@ from layerindexlib.tests.common import LayersTest
 
 import logging
 
+def skipIfNoNetwork():
+    if os.environ.get("BB_SKIP_NETTESTS") == "yes":
+        return unittest.skip("Network tests being skipped")
+    return lambda f: f
+
 class LayerIndexWebRestApiTest(LayersTest):
 
-    if os.environ.get("BB_SKIP_NETTESTS") == "yes":
-        print("Unset BB_SKIP_NETTESTS to run network tests")
-    else:
-        def setUp(self):
-            LayersTest.setUp(self)
-            self.layerindex = layerindexlib.LayerIndex(self.d)
-            self.layerindex.load_layerindex('http://layers.openembedded.org/layerindex/api/;branch=sumo', load=['layerDependencies'])
+    @skipIfNoNetwork()
+    def setUp(self):
+        self.assertFalse(os.environ.get("BB_SKIP_NETTESTS") == "yes", msg="BB_SKIP_NETTESTS set, but we tried to test anyway")
+        LayersTest.setUp(self)
+        self.layerindex = layerindexlib.LayerIndex(self.d)
+        self.layerindex.load_layerindex('http://layers.openembedded.org/layerindex/api/;branch=sumo', load=['layerDependencies'])
 
-        def test_layerindex_is_empty(self):
-            self.assertFalse(self.layerindex.is_empty(), msg="Layerindex is empty")
+    @skipIfNoNetwork()
+    def test_layerindex_is_empty(self):
+        self.assertFalse(self.layerindex.is_empty(), msg="Layerindex is empty")
 
-        def test_layerindex_store_file(self):
-            self.layerindex.store_layerindex('file://%s/file.json' % self.tempdir, self.layerindex.indexes[0])
+    @skipIfNoNetwork()
+    def test_layerindex_store_file(self):
+        self.layerindex.store_layerindex('file://%s/file.json' % self.tempdir, self.layerindex.indexes[0])
 
-            self.assertTrue(os.path.isfile('%s/file.json' % self.tempdir), msg="Temporary file was not created by store_layerindex")
+        self.assertTrue(os.path.isfile('%s/file.json' % self.tempdir), msg="Temporary file was not created by store_layerindex")
 
-            reload = layerindexlib.LayerIndex(self.d)
-            reload.load_layerindex('file://%s/file.json' % self.tempdir)
+        reload = layerindexlib.LayerIndex(self.d)
+        reload.load_layerindex('file://%s/file.json' % self.tempdir)
 
-            self.assertFalse(reload.is_empty(), msg="Layerindex is empty")
+        self.assertFalse(reload.is_empty(), msg="Layerindex is empty")
 
-            # Calculate layerItems in original index that should NOT be in reload
-            layerItemNames = []
-            for itemId in self.layerindex.indexes[0].layerItems:
-                layerItemNames.append(self.layerindex.indexes[0].layerItems[itemId].name)
+        # Calculate layerItems in original index that should NOT be in reload
+        layerItemNames = []
+        for itemId in self.layerindex.indexes[0].layerItems:
+            layerItemNames.append(self.layerindex.indexes[0].layerItems[itemId].name)
 
-            for layerBranchId in self.layerindex.indexes[0].layerBranches:
-                layerItemNames.remove(self.layerindex.indexes[0].layerBranches[layerBranchId].layer.name)
+        for layerBranchId in self.layerindex.indexes[0].layerBranches:
+            layerItemNames.remove(self.layerindex.indexes[0].layerBranches[layerBranchId].layer.name)
 
-            for itemId in reload.indexes[0].layerItems:
-                self.assertFalse(reload.indexes[0].layerItems[itemId].name in layerItemNames, msg="Item reloaded when it shouldn't have been")
+        for itemId in reload.indexes[0].layerItems:
+            self.assertFalse(reload.indexes[0].layerItems[itemId].name in layerItemNames, msg="Item reloaded when it shouldn't have been")
 
-            # Compare the original to what we wrote...
-            for type in self.layerindex.indexes[0]._index:
-                if type == 'apilinks' or \
-                   type == 'layerItems' or \
-                   type in self.layerindex.indexes[0].config['local']:
-                    continue
-                for id in getattr(self.layerindex.indexes[0], type):
-                    self.logger.debug(1, "type %s" % (type))
+        # Compare the original to what we wrote...
+        for type in self.layerindex.indexes[0]._index:
+            if type == 'apilinks' or \
+               type == 'layerItems' or \
+               type in self.layerindex.indexes[0].config['local']:
+                continue
+            for id in getattr(self.layerindex.indexes[0], type):
+                self.logger.debug(1, "type %s" % (type))
 
-                    self.assertTrue(id in getattr(reload.indexes[0], type), msg="Id number not in reloaded index")
+                self.assertTrue(id in getattr(reload.indexes[0], type), msg="Id number not in reloaded index")
 
-                    self.logger.debug(1, "%s ? %s" % (getattr(self.layerindex.indexes[0], type)[id], getattr(reload.indexes[0], type)[id]))
+                self.logger.debug(1, "%s ? %s" % (getattr(self.layerindex.indexes[0], type)[id], getattr(reload.indexes[0], type)[id]))
 
-                    self.assertEqual(getattr(self.layerindex.indexes[0], type)[id], getattr(reload.indexes[0], type)[id], msg="Reloaded contents different")
+                self.assertEqual(getattr(self.layerindex.indexes[0], type)[id], getattr(reload.indexes[0], type)[id], msg="Reloaded contents different")
 
-        def test_layerindex_store_split(self):
-            self.layerindex.store_layerindex('file://%s' % self.tempdir, self.layerindex.indexes[0])
+    @skipIfNoNetwork()
+    def test_layerindex_store_split(self):
+        self.layerindex.store_layerindex('file://%s' % self.tempdir, self.layerindex.indexes[0])
 
-            reload = layerindexlib.LayerIndex(self.d)
-            reload.load_layerindex('file://%s' % self.tempdir)
+        reload = layerindexlib.LayerIndex(self.d)
+        reload.load_layerindex('file://%s' % self.tempdir)
 
-            self.assertFalse(reload.is_empty(), msg="Layer index is empty")
+        self.assertFalse(reload.is_empty(), msg="Layer index is empty")
 
-            for type in self.layerindex.indexes[0]._index:
-                if type == 'apilinks' or \
-                   type == 'layerItems' or \
-                   type in self.layerindex.indexes[0].config['local']:
-                    continue
-                for id in getattr(self.layerindex.indexes[0] ,type):
-                    self.logger.debug(1, "type %s" % (type))
+        for type in self.layerindex.indexes[0]._index:
+            if type == 'apilinks' or \
+               type == 'layerItems' or \
+               type in self.layerindex.indexes[0].config['local']:
+                continue
+            for id in getattr(self.layerindex.indexes[0] ,type):
+                self.logger.debug(1, "type %s" % (type))
 
-                    self.assertTrue(id in getattr(reload.indexes[0], type), msg="Id number missing from reloaded data")
+                self.assertTrue(id in getattr(reload.indexes[0], type), msg="Id number missing from reloaded data")
 
-                    self.logger.debug(1, "%s ? %s" % (getattr(self.layerindex.indexes[0] ,type)[id], getattr(reload.indexes[0], type)[id]))
+                self.logger.debug(1, "%s ? %s" % (getattr(self.layerindex.indexes[0] ,type)[id], getattr(reload.indexes[0], type)[id]))
 
-                    self.assertEqual(getattr(self.layerindex.indexes[0] ,type)[id], getattr(reload.indexes[0], type)[id], msg="reloaded data does not match original")
+                self.assertEqual(getattr(self.layerindex.indexes[0] ,type)[id], getattr(reload.indexes[0], type)[id], msg="reloaded data does not match original")
 
-        def test_dependency_resolution(self):
-            # Verify depth first searching...
-            (dependencies, invalidnames) = self.layerindex.find_dependencies(names=['meta-python'])
+    @skipIfNoNetwork()
+    def test_dependency_resolution(self):
+        # Verify depth first searching...
+        (dependencies, invalidnames) = self.layerindex.find_dependencies(names=['meta-python'])
 
-            first = True
-            for deplayerbranch in dependencies:
-                layerBranch = dependencies[deplayerbranch][0]
-                layerDeps = dependencies[deplayerbranch][1:]
+        first = True
+        for deplayerbranch in dependencies:
+            layerBranch = dependencies[deplayerbranch][0]
+            layerDeps = dependencies[deplayerbranch][1:]
 
-                if not first:
-                    continue
+            if not first:
+                continue
 
-                first = False
+            first = False
 
-                # Top of the deps should be openembedded-core, since everything depends on it.
-                self.assertEqual(layerBranch.layer.name, "openembedded-core", msg='OpenEmbedded-Core is no the first dependency')
+            # Top of the deps should be openembedded-core, since everything depends on it.
+            self.assertEqual(layerBranch.layer.name, "openembedded-core", msg='OpenEmbedded-Core is no the first dependency')
 
-                # meta-python should cause an openembedded-core dependency, if not assert!
-                for dep in layerDeps:
-                    if dep.layer.name == 'meta-python':
-                        break
-                else:
-                    self.logger.debug(1, "meta-python was not found")
-                    self.assetTrue(False)
-
-                # Only check the first element...
-                break
+            # meta-python should cause an openembedded-core dependency, if not assert!
+            for dep in layerDeps:
+                if dep.layer.name == 'meta-python':
+                    break
             else:
-                # Empty list, this is bad.
-                self.logger.debug(1, "Empty list of dependencies")
-                self.assertIsNotNone(first, msg="Empty list of dependencies")
+                self.logger.debug(1, "meta-python was not found")
+                self.assetTrue(False)
 
-                # Last dep should be the requested item
-                layerBranch = dependencies[deplayerbranch][0]
-                self.assertEqual(layerBranch.layer.name, "meta-python", msg="Last dependency not meta-python")
+            # Only check the first element...
+            break
+        else:
+            # Empty list, this is bad.
+            self.logger.debug(1, "Empty list of dependencies")
+            self.assertIsNotNone(first, msg="Empty list of dependencies")
 
+            # Last dep should be the requested item
+            layerBranch = dependencies[deplayerbranch][0]
+            self.assertEqual(layerBranch.layer.name, "meta-python", msg="Last dependency not meta-python")
+
+    @skipIfNoNetwork()
     def test_find_collection(self):
         def _check(collection, expected):
             self.logger.debug(1, "Looking for collection %s..." % collection)
@@ -149,6 +158,7 @@ class LayerIndexWebRestApiTest(LayersTest):
         for collection,result in tests:
             _check(collection, result)
 
+    @skipIfNoNetwork()
     def test_find_layerbranch(self):
         def _check(name, expected):
             self.logger.debug(1, "Looking for layerbranch %s..." % name)

bitbake/lib/toaster/bldcontrol/localhostbecontroller.py

@@ -177,8 +177,11 @@ class LocalhostBEController(BuildEnvironmentController):
                 try:
                     localremotes = self._shellcmd("git remote -v",
                                                   localdirname,env=git_env)
-                    if not giturl in localremotes and commit != 'HEAD':
-                        raise BuildSetupException("Existing git repository at %s, but with different remotes ('%s', expected '%s'). Toaster will not continue out of fear of damaging something." % (localdirname, ", ".join(localremotes.split("\n")), giturl))
+                    # NOTE: this nice-to-have check breaks when using git remaping to get past firewall
+                    #       Re-enable later with .gitconfig remapping checks
+                    #if not giturl in localremotes and commit != 'HEAD':
+                    #    raise BuildSetupException("Existing git repository at %s, but with different remotes ('%s', expected '%s'). Toaster will not continue out of fear of damaging something." % (localdirname, ", ".join(localremotes.split("\n")), giturl))
+                    pass
                 except ShellCmdException:
                     # our localdirname might not be a git repository
                     #- that's fine

bitbake/lib/toaster/orm/fixtures/oe-core.xml

@@ -23,14 +23,14 @@
     <field type="CharField" name="branch">master</field>
   </object>
   <object model="orm.bitbakeversion" pk="4">
-    <field type="CharField" name="name">rocko</field>
+    <field type="CharField" name="name">thud</field>
     <field type="CharField" name="giturl">git://git.openembedded.org/bitbake</field>
-    <field type="CharField" name="branch">1.36</field>
+    <field type="CharField" name="branch">1.40</field>
   </object>
 
   <!-- Releases available -->
   <object model="orm.release" pk="1">
-    <field type="CharField" name="name">rocko</field>
+    <field type="CharField" name="name">sumo</field>
     <field type="CharField" name="description">Openembedded Sumo</field>
     <field rel="ManyToOneRel" to="orm.bitbakeversion" name="bitbake_version">1</field>
     <field type="CharField" name="branch_name">sumo</field>
@@ -51,11 +51,11 @@
     <field type="TextField" name="helptext">Toaster will run your builds using the tip of the &lt;a href=\"http://cgit.openembedded.org/openembedded-core/log/\"&gt;OpenEmbedded master&lt;/a&gt; branch.</field>
   </object>
   <object model="orm.release" pk="4">
-    <field type="CharField" name="name">rocko</field>
+    <field type="CharField" name="name">thud</field>
     <field type="CharField" name="description">Openembedded Rocko</field>
     <field rel="ManyToOneRel" to="orm.bitbakeversion" name="bitbake_version">1</field>
-    <field type="CharField" name="branch_name">rocko</field>
-    <field type="TextField" name="helptext">Toaster will run your builds using the tip of the &lt;a href=\"http://cgit.openembedded.org/openembedded-core/log/?h=rocko\"&gt;OpenEmbedded Rocko&lt;/a&gt; branch.</field>
+    <field type="CharField" name="branch_name">thud</field>
+    <field type="TextField" name="helptext">Toaster will run your builds using the tip of the &lt;a href=\"http://cgit.openembedded.org/openembedded-core/log/?h=thud\"&gt;OpenEmbedded Thud&lt;/a&gt; branch.</field>
   </object>
 
   <!-- Default layers for each release -->

bitbake/lib/toaster/orm/fixtures/poky.xml

@@ -26,9 +26,9 @@
     <field type="CharField" name="dirpath">bitbake</field>
   </object>
   <object model="orm.bitbakeversion" pk="4">
-    <field type="CharField" name="name">rocko</field>
+    <field type="CharField" name="name">thud</field>
     <field type="CharField" name="giturl">git://git.yoctoproject.org/poky</field>
-    <field type="CharField" name="branch">rocko</field>
+    <field type="CharField" name="branch">thud</field>
     <field type="CharField" name="dirpath">bitbake</field>
   </object>
 
@@ -57,10 +57,10 @@
   </object>
   <object model="orm.release" pk="4">
     <field type="CharField" name="name">rocko</field>
-    <field type="CharField" name="description">Yocto Project 2.4 "Rocko"</field>
+    <field type="CharField" name="description">Yocto Project 2.6 "Thud"</field>
     <field rel="ManyToOneRel" to="orm.bitbakeversion" name="bitbake_version">1</field>
-    <field type="CharField" name="branch_name">rocko</field>
-    <field type="TextField" name="helptext">Toaster will run your builds using the tip of the &lt;a href="http://git.yoctoproject.org/cgit/cgit.cgi/poky/log/?h=rocko"&gt;Yocto Project Rocko branch&lt;/a&gt;.</field>
+    <field type="CharField" name="branch_name">thud</field>
+    <field type="TextField" name="helptext">Toaster will run your builds using the tip of the &lt;a href="http://git.yoctoproject.org/cgit/cgit.cgi/poky/log/?h=thud"&gt;Yocto Project Thud branch&lt;/a&gt;.</field>
   </object>
 
   <!-- Default project layers for each release -->

bitbake/lib/toaster/orm/models.py

@@ -1750,8 +1750,8 @@ class CustomImageRecipe(Recipe):
         if base_recipe_path:
             base_recipe = open(base_recipe_path, 'r').read()
         else:
-            raise IOError("Based on recipe file not found: %s" %
-                          base_recipe_path)
+            # Pass back None to trigger error message to user
+            return None
 
         # Add a special case for when the recipe we have based a custom image
         # recipe on requires another recipe.
@@ -1877,7 +1877,7 @@ class Distro(models.Model):
     description = models.CharField(max_length=255)
 
     def get_vcs_distro_file_link_url(self):
-        path = self.name+'.conf'
+        path = 'conf/distro/%s.conf' % self.name
         return self.layer_version.get_vcs_file_link_url(path)
 
     def __unicode__(self):

bitbake/lib/toaster/toastergui/api.py

@@ -677,7 +677,13 @@ class XhrCustomRecipe(View):
         recipe_path = os.path.join(layerpath, "recipes", "%s.bb" %
                                    recipe.name)
         with open(recipe_path, "w") as recipef:
-            recipef.write(recipe.generate_recipe_file_contents())
+            content = recipe.generate_recipe_file_contents()
+            if not content:
+                # Delete this incomplete image recipe object
+                recipe.delete()
+                return error_response("recipe-parent-not-exist")
+            else:
+                recipef.write(recipe.generate_recipe_file_contents())
 
         return JsonResponse(
             {"error": "ok",

bitbake/lib/toaster/toastergui/static/js/libtoaster.js

@@ -275,7 +275,8 @@ var libtoaster = (function () {
 
   function _addRmLayer(layerObj, add, doneCb){
     if (layerObj.xhrLayerUrl === undefined){
-      throw("xhrLayerUrl is undefined")
+      alert("ERROR: missing xhrLayerUrl object. Please file a bug.");
+      return;
     }
 
     if (add === true) {

bitbake/lib/toaster/toastergui/static/js/newcustomimage_modal.js

@@ -25,6 +25,8 @@ function newCustomImageModalInit(){
   var duplicateNameMsg = "An image with this name already exists. Image names must be unique.";
   var duplicateImageInProjectMsg = "An image with this name already exists in this project."
   var invalidBaseRecipeIdMsg = "Please select an image to customise.";
+  var missingParentRecipe = "The parent recipe file was not found. Cancel this action, build any target (like 'quilt-native') to force all new layers to clone, and try again";
+  var unknownError = "Unexpected error: ";
 
   // set button to "submit" state and enable text entry so user can
   // enter the custom recipe name
@@ -62,6 +64,7 @@ function newCustomImageModalInit(){
     if (nameInput.val().length > 0) {
       libtoaster.createCustomRecipe(nameInput.val(), baseRecipeId,
       function(ret) {
+        showSubmitState();
         if (ret.error !== "ok") {
           console.warn(ret.error);
           if (ret.error === "invalid-name") {
@@ -73,6 +76,10 @@ function newCustomImageModalInit(){
           } else if (ret.error === "image-already-exists") {
             showNameError(duplicateImageInProjectMsg);
             return;
+          } else if (ret.error === "recipe-parent-not-exist") {
+            showNameError(missingParentRecipe);
+          } else {
+            showNameError(unknownError + ret.error);
           }
         } else {
           imgCustomModal.modal('hide');

bitbake/lib/toaster/toastergui/tables.py

@@ -1615,14 +1615,12 @@ class DistrosTable(ToasterTable):
                         hidden=True,
                         field_name="layer_version__get_vcs_reference")
 
-        wrtemplate_file_template = '''<code>conf/machine/{{data.name}}.conf</code>
-        <a href="{{data.get_vcs_machine_file_link_url}}" target="_blank"><span class="glyphicon glyphicon-new-window"></i></a>'''
-
+        distro_file_template = '''<code>conf/distro/{{data.name}}.conf</code>
+        {% if 'None' not in data.get_vcs_distro_file_link_url %}<a href="{{data.get_vcs_distro_file_link_url}}" target="_blank"><span class="glyphicon glyphicon-new-window"></i></a>{% endif %}'''
         self.add_column(title="Distro file",
                         hidden=True,
                         static_data_name="templatefile",
-                        static_data_template=wrtemplate_file_template)
-
+                        static_data_template=distro_file_template)
 
         self.add_column(title="Select",
                         help_text="Sets the selected distro to the project",

bitbake/lib/toaster/toastergui/templates/customise_btn.html

@@ -5,7 +5,11 @@
   >
   Customise
 </button>
-<button class="btn btn-default btn-block layer-add-{{data.layer_version.pk}} layerbtn" data-layer='{ "id": {{data.layer_version.pk}}, "name":  "{{data.layer_version.layer.name}}", "layerdetailurl": "{%url 'layerdetails' extra.pid data.layer_version.pk%}"}' data-directive="add"
+<button class="btn btn-default btn-block layer-add-{{data.layer_version.pk}} layerbtn"
+    data-layer='{ "id": {{data.layer_version.pk}}, "name":  "{{data.layer_version.layer.name}}",
+      "layerdetailurl": "{%url 'layerdetails' extra.pid data.layer_version.pk%}",
+      "xhrLayerUrl": "{% url "xhr_layer" extra.pid data.layer_version.pk %}"}'
+    data-directive="add"
     {% if data.layer_version.pk in extra.current_layers %}
     style="display:none;"
     {% endif %}

bitbake/lib/toaster/toastermain/management/commands/buildimport.py

@@ -55,6 +55,7 @@ import os
 import re
 import os.path
 import subprocess
+import shutil
 
 # Toaster variable section delimiters
 TOASTER_PROLOG = '#=== TOASTER_CONFIG_PROLOG ==='

bitbake/toaster-requirements.txt

@@ -1,3 +1,3 @@
-Django>1.8,<1.11.9
+Django>1.8,<1.12
 beautifulsoup4>=4.4.0
 pytz

documentation/dev-manual/dev-manual-common-tasks.xml

@@ -189,8 +189,8 @@
                                 <ulink url='&YOCTO_WIKI_URL;/wiki/Releases'>Yocto Project</ulink>
                                 releases for which the current version is
                                 compatible.
-                                This variable is a good way to indicate how
-                                up-to-date your particular layer is.
+                                This variable is a good way to indicate if
+                                your particular layer is current.
                                 </para></listitem>
                         </itemizedlist>
                         </para></listitem>

documentation/poky.ent

@@ -8,7 +8,7 @@
 <!ENTITY YOCTO_DOC_VERSION_MINUS_ONE "2.5">
 <!ENTITY DISTRO_REL_TAG "yocto-2.6">
 <!ENTITY METAINTELVERSION "9.0">
-<!ENTITY REL_MONTH_YEAR "TBD">
+<!ENTITY REL_MONTH_YEAR "October 2018">
 <!ENTITY META_INTEL_REL_TAG "&METAINTELVERSION;-&DISTRO_NAME_NO_CAP;-&YOCTO_DOC_VERSION;">
 <!ENTITY POKYVERSION "21.0.0">
 <!ENTITY POKYVERSION_COMPRESSED "2100">

documentation/profile-manual/profile-manual-intro.xml

@@ -92,7 +92,9 @@
      EXTRA_IMAGE_FEATURES = "debug-tweaks tools-profile dbg-pkgs"
             </literallayout>
             Additionally, in order to generate the right type of
-            debuginfo, we also need to add the following to local.conf:
+            debuginfo, we also need to set
+            <ulink url='&YOCTO_DOCS_REF_URL;#var-PACKAGE_DEBUG_SPLIT_STYLE'><filename>PACKAGE_DEBUG_SPLIT_STYLE</filename></ulink>
+            in the <filename>local.conf</filename> file:
             <literallayout class='monospaced'>
      PACKAGE_DEBUG_SPLIT_STYLE = 'debug-file-directory'
             </literallayout>

documentation/profile-manual/profile-manual-usage.xml

@@ -375,7 +375,9 @@
      EXTRA_IMAGE_FEATURES = "debug-tweaks tools-profile dbg-pkgs"
                 </literallayout>
                 Additionally, in order to generate the type of debuginfo that
-                perf understands, we also need to add the following to local.conf:
+                perf understands, we also need to set
+                <ulink url='&YOCTO_DOCS_REF_URL;#var-PACKAGE_DEBUG_SPLIT_STYLE'><filename>PACKAGE_DEBUG_SPLIT_STYLE</filename></ulink>
+                in the <filename>local.conf</filename> file:
                 <literallayout class='monospaced'>
      PACKAGE_DEBUG_SPLIT_STYLE = 'debug-file-directory'
                 </literallayout>

documentation/ref-manual/ref-classes.xml

@@ -2173,8 +2173,9 @@ This check was removed for YP 2.3 release
 
     <para>
         The <filename>native</filename> class provides common
-        functionality for recipes that wish to build tools to run on the build
-        host (i.e. tools that use the compiler or other tools from the
+        functionality for recipes that build tools to run on the
+        <link linkend='hardware-build-system-term'>build host</link>
+        (i.e. tools that use the compiler or other tools from the
         build host).
     </para>
 
@@ -2182,30 +2183,35 @@ This check was removed for YP 2.3 release
         You can create a recipe that builds tools that run natively on the
         host a couple different ways:
         <itemizedlist>
-            <listitem><para>Create a <replaceable>myrecipe</replaceable><filename>-native.bb</filename>
-                that inherits the <filename>native</filename> class.
+            <listitem><para>
+                Create a
+                <replaceable>myrecipe</replaceable><filename>-native.bb</filename>
+                recipe that inherits the <filename>native</filename> class.
                 If you use this method, you must order the inherit statement
                 in the recipe after all other inherit statements so that the
                 <filename>native</filename> class is inherited last.
+                <note><title>Warning</title>
+                    When creating a recipe this way, the recipe name must
+                    follow this naming convention:
+                    <literallayout class='monospaced'>
+     <replaceable>myrecipe</replaceable>-native.bb
+                    </literallayout>
+                    Not using this naming convention can lead to subtle
+                    problems caused by existing code that depends on that
+                    naming convention.
+                </note>
                 </para></listitem>
-            <listitem><para>Create or modify a target recipe that contains
-                the following:
+            <listitem><para>
+                Create or modify a target recipe that contains the following:
                 <literallayout class='monospaced'>
      <link linkend='var-BBCLASSEXTEND'><filename>BBCLASSEXTEND</filename></link> = "native"
                 </literallayout>
                 Inside the recipe, use <filename>_class-native</filename> and
                 <filename>_class-target</filename> overrides to specify any
                 functionality specific to the respective native or target
-                case.</para></listitem>
+                case.
+                </para></listitem>
         </itemizedlist>
-        <note><title>Warning</title>
-            When creating a recipe, you must follow this naming convention:
-            <literallayout class='monospaced'>
-     native-<replaceable>myrecipe</replaceable>.bb
-            </literallayout>
-            Not doing so can lead to subtle problems because code exists
-            that depends on the naming convention.
-        </note>
     </para>
 
     <para>

documentation/ref-manual/ref-terms.xml

@@ -29,11 +29,31 @@
                 information in the similarly-named recipe file.
                 For an example of an append file in use, see the
                 "<ulink url='&YOCTO_DOCS_DEV_URL;#using-bbappend-files'>Using .bbappend Files in Your Layer</ulink>"
-                section in the Yocto Project Development Tasks Manual.
-                <note>
-                    Append files can also use wildcard patterns in their
-                    version numbers so they can be applied to more than one
-                    version of the underlying recipe file.
+                section in the Yocto Project Development Tasks Manual.</para>
+
+                <para>When you name an append file, you can use the
+                "<filename>%</filename>" wildcard character to allow for
+                matching recipe names.
+                For example, suppose you have an append file named as follows:
+                <literallayout class='monospaced'>
+     busybox_1.21.%.bbappend
+                </literallayout>
+                That append file would match any
+                <filename>busybox_1.21.</filename><replaceable>x</replaceable><filename>.bb</filename>
+                version of the recipe.
+                So, the append file would match the following recipe names:
+                <literallayout class='monospaced'>
+     busybox_1.21.1.bb
+     busybox_1.21.2.bb
+     busybox_1.21.3.bb
+                </literallayout>
+                <note><title>Important</title>
+                    The use of the "<filename>%</filename>" character
+                    is limited in that it only works directly in front of the
+                    <filename>.bbappend</filename> portion of the append file's
+                    name.
+                    You cannot use the wildcard character in any other
+                    location of the name.
                 </note>
                 </para></listitem>
             <listitem><para id='bitbake-term'>

documentation/ref-manual/ref-variables.xml

@@ -679,6 +679,13 @@
                             <literallayout class='monospaced'>
      BB_ALLOWED_NETWORKS = "*.gnu.org"
                             </literallayout>
+                            <note><title>Important</title>
+                                The use of the "<filename>*</filename>"
+                                character only works at the beginning of
+                                a host name.
+                                You cannot use the wildcard character in any
+                                other location of the name.
+                            </note>
                             </para></listitem>
                         <listitem><para>
                             Mirrors not in the host list are skipped and
@@ -1133,12 +1140,22 @@
 
         <glossentry id='var-BBFILES'><glossterm>BBFILES</glossterm>
             <info>
-                BBFILES[doc] = "List of recipe files used by BitBake to build software."
+                BBFILES[doc] = "A space-separated list of recipe files BitBake uses to build software."
             </info>
             <glossdef>
                 <para role="glossdeffirst">
 <!--                <para role="glossdeffirst"><imagedata fileref="figures/define-generic.png" /> -->
-                    List of recipe files used by BitBake to build software.
+                    A space-separated list of recipe files BitBake uses to
+                    build software.
+                </para>
+
+                <para>
+                    When specifying recipe files, you can pattern match using
+                    Python's
+                    <ulink url='https://docs.python.org/2/library/glob.html'><filename>glob</filename></ulink>
+                    syntax.
+                    For details on the syntax, see the documentation by
+                    following the previous link.
                 </para>
             </glossdef>
         </glossentry>
@@ -1267,15 +1284,19 @@
                     match any of the expressions.
                     It is as if BitBake does not see them at all.
                     Consequently, matching files are not parsed or otherwise
-                    used by BitBake.</para>
+                    used by BitBake.
+                </para>
+
                 <para>
                     The values you provide are passed to Python's regular
                     expression compiler.
+                    Consequently, the syntax follows Python's Regular
+                    Expression (re) syntax.
                     The expressions are compared against the full paths to
                     the files.
                     For complete syntax information, see Python's
-                    documentation for the appropriate release at
-                    <ulink url='http://docs.python.org/release/'></ulink>.
+                    documentation at
+                    <ulink url='http://docs.python.org/release/2.3/lib/re-syntax.html'></ulink>.
                 </para>
 
                 <para>
@@ -1443,6 +1464,17 @@
                     set up during compilation so that they are correct for
                     use when installed into the sysroot and called by the
                     build processes of other recipes.
+                    <note>
+                        The <filename>BINCONFIG_GLOB</filename> variable
+                        uses
+                        <ulink url='http://tldp.org/LDP/abs/html/globbingref.html'>shell globbing</ulink>,
+                        which is recognition and expansion of wildcards during
+                        pattern matching.
+                        Shell globbing is very similar to
+                        <ulink url='https://docs.python.org/2/library/fnmatch.html#module-fnmatch'><filename>fnmatch</filename></ulink>
+                        and
+                        <ulink url='https://docs.python.org/2/library/glob.html'><filename>glob</filename></ulink>.
+                    </note>
                 </para>
 
                 <para>
@@ -2366,6 +2398,14 @@
                     Defines wildcards to match when installing a list of
                     complementary packages for all the packages explicitly
                     (or implicitly) installed in an image.
+                    <note>
+                        The <filename>COMPLEMENTARY_GLOB</filename> variable
+                        uses Unix filename pattern matching
+                        (<ulink url='https://docs.python.org/2/library/fnmatch.html#module-fnmatch'><filename>fnmatch</filename></ulink>),
+                        which is similar to the Unix style pathname pattern
+                        expansion
+                        (<ulink url='https://docs.python.org/2/library/glob.html'><filename>glob</filename></ulink>).
+                    </note>
                     The resulting list of complementary packages is associated
                     with an item that can be added to
                     <link linkend='var-IMAGE_FEATURES'><filename>IMAGE_FEATURES</filename></link>.
@@ -4782,23 +4822,38 @@
                     <literallayout class='monospaced'>
      FILES_${PN} += "${bindir}/mydir1 ${bindir}/mydir2/myfile"
                     </literallayout>
+                    <note><title>Notes</title>
+                        <itemizedlist>
+                            <listitem><para>
+                                When specifying files or paths, you can pattern
+                                match using Python's
+                                <ulink url='https://docs.python.org/2/library/glob.html'><filename>glob</filename></ulink>
+                                syntax.
+                                For details on the syntax, see the
+                                documentation by following the previous link.
+                                </para></listitem>
+                            <listitem><para>
+                                When specifying paths as part of the
+                                <filename>FILES</filename> variable, it is
+                                good practice to use appropriate path
+                                variables.
+                                For example, use <filename>${sysconfdir}</filename>
+                                rather than <filename>/etc</filename>, or
+                                <filename>${bindir}</filename> rather than
+                                <filename>/usr/bin</filename>.
+                                You can find a list of these variables at the
+                                top of the
+                                <filename>meta/conf/bitbake.conf</filename>
+                                file in the
+                                <link linkend='source-directory'>Source Directory</link>.
+                                You will also find the default values of the
+                                various <filename>FILES_*</filename> variables
+                                in this file.
+                                </para></listitem>
+                        </itemizedlist>
+                    </note>
                 </para>
 
-                <note>
-                    When specifying paths as part of the
-                    <filename>FILES</filename> variable, it is good practice
-                    to use appropriate path variables.
-                    For example, use <filename>${sysconfdir}</filename> rather
-                    than <filename>/etc</filename>, or
-                    <filename>${bindir}</filename> rather than
-                    <filename>/usr/bin</filename>.
-                    You can find a list of these variables at the top of the
-                    <filename>meta/conf/bitbake.conf</filename> file in the
-                    <link linkend='source-directory'>Source Directory</link>.
-                    You will also find the default values of the various
-                    <filename>FILES_*</filename> variables in this file.
-                </note>
-
                 <para>
                     If some of the files you provide with the
                     <filename>FILES</filename> variable are editable and you
@@ -9646,6 +9701,12 @@ recipes-graphics/xorg-font/font-alias_1.0.3.bb:PR = "${INC_PR}.3"
                             ".debug" previously described with the exception
                             that no source files are installed.
                             </para></listitem>.
+                        <listitem><para>
+                            "debug-with-srcpkg": The same behavior as
+                            ".debug" previously described with the exception
+                            that all source files are placed in a separate
+                            <filename>*-src</filename> pkg.
+                            </para></listitem>
                     </itemizedlist>
                 </para>
 
@@ -10929,47 +10990,63 @@ recipes-graphics/xorg-font/font-alias_1.0.3.bb:PR = "${INC_PR}.3"
             <glossdef>
                 <para role="glossdeffirst">
 <!--                <para role="glossdeffirst"><imagedata fileref="figures/define-generic.png" /> -->
-                    If there are multiple versions of recipes available, this
-                    variable determines which recipe should be given preference.
+                    If multiple versions of recipes exist, this
+                    variable determines which version is given preference.
                     You must always suffix the variable with the
                     <link linkend='var-PN'><filename>PN</filename></link>
                     you want to select, and you should set the
                     <link linkend='var-PV'><filename>PV</filename></link>
                     accordingly for precedence.
-                    You can use the "<filename>%</filename>" character as a
-                    wildcard to match any number of characters, which can be
-                    useful when specifying versions that contain long revision
-                    numbers that could potentially change.
+                </para>
+
+                <para>
+                    The <filename>PREFERRED_VERSION</filename> variable
+                    supports limited wildcard use through the
+                    "<filename>%</filename>" character.
+                    You can use the character to match any number of
+                    characters, which can be useful when specifying versions
+                    that contain long revision numbers that potentially change.
                     Here are two examples:
                     <literallayout class='monospaced'>
      PREFERRED_VERSION_python = "3.4.0"
      PREFERRED_VERSION_linux-yocto = "4.12%"
                     </literallayout>
-                    <note>
-                        The specified version is matched against
-                        <link linkend='var-PV'><filename>PV</filename></link>,
-                        which does not necessarily match the version part of
-                        the recipe's filename.
-                        For example, consider two recipes
-                        <filename>foo_1.2.bb</filename> and
-                        <filename>foo_git.bb</filename> where
-                        <filename>foo_git.bb</filename> contains the following
-                        assignment:
-                        <literallayout class='monospaced'>
-     PV = "1.1+git${SRCPV}"
-                        </literallayout>
-                        In this case, the correct way to select
-                        <filename>foo_git.bb</filename> is by using an
-                        assignment such as the following:
-                        <literallayout class='monospaced'>
-     PREFERRED_VERSION_foo = "1.1+git%"
-                        </literallayout>
-                        Compare that previous example against the following
-                        incorrect example, which does not work:
-                        <literallayout class='monospaced'>
-     PREFERRED_VERSION_foo = "git"
-                        </literallayout>
+                    <note><title>Important</title>
+                        The use of the "<filename>%</filename>" character
+                        is limited in that it only works at the end of the
+                        string.
+                        You cannot use the wildcard character in any other
+                        location of the string.
                     </note>
+                </para>
+
+                <para>
+                    The specified version is matched against
+                    <link linkend='var-PV'><filename>PV</filename></link>,
+                    which does not necessarily match the version part of
+                    the recipe's filename.
+                    For example, consider two recipes
+                    <filename>foo_1.2.bb</filename> and
+                    <filename>foo_git.bb</filename> where
+                    <filename>foo_git.bb</filename> contains the following
+                    assignment:
+                    <literallayout class='monospaced'>
+     PV = "1.1+git${SRCPV}"
+                    </literallayout>
+                    In this case, the correct way to select
+                    <filename>foo_git.bb</filename> is by using an
+                    assignment such as the following:
+                    <literallayout class='monospaced'>
+     PREFERRED_VERSION_foo = "1.1+git%"
+                    </literallayout>
+                    Compare that previous example against the following
+                    incorrect example, which does not work:
+                    <literallayout class='monospaced'>
+     PREFERRED_VERSION_foo = "git"
+                    </literallayout>
+                </para>
+
+                <para>
                     Sometimes the <filename>PREFERRED_VERSION</filename>
                     variable can be set by configuration files in a way that
                     is hard to change.

meta-poky/conf/local.conf.sample.extended

@@ -59,6 +59,11 @@
 # NOTE: If removing en_US.UTF-8 you will also need to uncomment, and set
 # appropriate value for IMAGE_LINGUAS.
 # WARNING: this may break localisation!
+# WARNING: some recipes expect certain localizations to be enabled, e.g.
+# bash-ptest: fr-fr, de-de
+# glib-2.0-ptest: tr-tr, lt-lt, ja-jp.euc-jp, fa-ir, ru-ru, de-de, hr-hr, el-gr, fr-fr, es-es, en-gb
+# if you remove some of these and enable ptest, you'll get QA warning like:
+# ERROR: glib-2.0-1_2.58.0-r0 do_package_qa: QA Issue: glib-2.0-ptest rdepends on locale-base-de-de, but it isn't a build dependency? [build-deps]
 #GLIBC_GENERATE_LOCALES = "en_GB.UTF-8 en_US.UTF-8"
 #IMAGE_LINGUAS ?= "en-gb"
 

meta-selftest/conf/layer.conf

@@ -9,4 +9,4 @@ BBFILE_COLLECTIONS += "selftest"
 BBFILE_PATTERN_selftest = "^${LAYERDIR}/"
 BBFILE_PRIORITY_selftest = "5"
 
-LAYERSERIES_COMPAT_selftest = "sumo"
+LAYERSERIES_COMPAT_selftest = "thud"

meta-skeleton/conf/layer.conf

@@ -14,4 +14,4 @@ LAYERVERSION_skeleton = "1"
 
 LAYERDEPENDS_skeleton = "core"
 
-LAYERSERIES_COMPAT_skeleton = "sumo"
+LAYERSERIES_COMPAT_skeleton = "thud"

meta-yocto-bsp/recipes-kernel/linux/linux-yocto_4.14.bbappend

@@ -8,11 +8,11 @@ KMACHINE_genericx86 ?= "common-pc"
 KMACHINE_genericx86-64 ?= "common-pc-64"
 KMACHINE_beaglebone-yocto ?= "beaglebone"
 
-SRCREV_machine_genericx86    ?= "78a16a4d8cfd58f91be412797aac248e811d083b"
-SRCREV_machine_genericx86-64 ?= "78a16a4d8cfd58f91be412797aac248e811d083b"
-SRCREV_machine_edgerouter ?= "04043b48eb792c8baa310840307d480a28a43053"
-SRCREV_machine_beaglebone-yocto ?= "d62ddfa26a59c83f6128790b7aa2baadc04d2960"
-SRCREV_machine_mpc8315e-rdb ?= "80a106ecf737b69864e07b955eccb62f809cf120"
+SRCREV_machine_genericx86    ?= "2c5caa7e84311f2a0097974a697ac1f59030530e"
+SRCREV_machine_genericx86-64 ?= "2c5caa7e84311f2a0097974a697ac1f59030530e"
+SRCREV_machine_edgerouter ?= "e06bfa18c727bd0e6e10cf26d9f161e4c791f52b"
+SRCREV_machine_beaglebone-yocto ?= "b8805de77dcf8f59d8368fee4921c146c1300a6a"
+SRCREV_machine_mpc8315e-rdb ?= "f88e87360b10f8fbd853a7d412982e6620f3f96d"
 
 COMPATIBLE_MACHINE_genericx86 = "genericx86"
 COMPATIBLE_MACHINE_genericx86-64 = "genericx86-64"
@@ -20,8 +20,8 @@ COMPATIBLE_MACHINE_edgerouter = "edgerouter"
 COMPATIBLE_MACHINE_beaglebone-yocto = "beaglebone-yocto"
 COMPATIBLE_MACHINE_mpc8315e-rdb = "mpc8315e-rdb"
 
-LINUX_VERSION_genericx86 = "4.14.71"
-LINUX_VERSION_genericx86-64 = "4.14.71"
-LINUX_VERSION_edgerouter = "4.14.48"
-LINUX_VERSION_beaglebone-yocto = "4.14.48"
-LINUX_VERSION_mpc8315e-rdb = "4.14.48"
+LINUX_VERSION_genericx86 = "4.14.76"
+LINUX_VERSION_genericx86-64 = "4.14.76"
+LINUX_VERSION_edgerouter = "4.14.71"
+LINUX_VERSION_beaglebone-yocto = "4.14.71"
+LINUX_VERSION_mpc8315e-rdb = "4.14.71"

meta-yocto-bsp/recipes-kernel/linux/linux-yocto_4.18.bbappend

@@ -8,11 +8,11 @@ KMACHINE_genericx86 ?= "common-pc"
 KMACHINE_genericx86-64 ?= "common-pc-64"
 KMACHINE_beaglebone-yocto ?= "beaglebone"
 
-SRCREV_machine_genericx86    ?= "0cdc8564c61958a39704d97e008120bd7c762f60"
-SRCREV_machine_genericx86-64 ?= "0cdc8564c61958a39704d97e008120bd7c762f60"
-SRCREV_machine_edgerouter ?= "eba03655e8e436ef6090100423bcea43e4911478"
-SRCREV_machine_beaglebone-yocto ?= "eba03655e8e436ef6090100423bcea43e4911478"
-SRCREV_machine_mpc8315e-rdb ?= "193513391ec14cb5402fcc4ed7bb2b49193842bb"
+SRCREV_machine_genericx86    ?= "33859b3077c83d8a024946ba06a59990546fcbc7"
+SRCREV_machine_genericx86-64 ?= "33859b3077c83d8a024946ba06a59990546fcbc7"
+SRCREV_machine_edgerouter ?= "0cdc8564c61958a39704d97e008120bd7c762f60"
+SRCREV_machine_beaglebone-yocto ?= "0cdc8564c61958a39704d97e008120bd7c762f60"
+SRCREV_machine_mpc8315e-rdb ?= "20be2231bb047b02248d7f965c1c3af0c6be3a5e"
 
 COMPATIBLE_MACHINE_genericx86 = "genericx86"
 COMPATIBLE_MACHINE_genericx86-64 = "genericx86-64"
@@ -20,8 +20,8 @@ COMPATIBLE_MACHINE_edgerouter = "edgerouter"
 COMPATIBLE_MACHINE_beaglebone-yocto = "beaglebone-yocto"
 COMPATIBLE_MACHINE_mpc8315e-rdb = "mpc8315e-rdb"
 
-LINUX_VERSION_genericx86 = "4.18.9"
-LINUX_VERSION_genericx86-64 = "4.18.9"
-LINUX_VERSION_edgerouter = "4.18.3"
-LINUX_VERSION_beaglebone-yocto = "4.18.3"
-LINUX_VERSION_mpc8315e-rdb = "4.18.3"
+LINUX_VERSION_genericx86 = "4.18.14"
+LINUX_VERSION_genericx86-64 = "4.18.14"
+LINUX_VERSION_edgerouter = "4.18.9"
+LINUX_VERSION_beaglebone-yocto = "4.18.9"
+LINUX_VERSION_mpc8315e-rdb = "4.18.9"

meta/classes/archiver.bbclass

@@ -98,9 +98,12 @@ python () {
 
         # There is a corner case with "gcc-source-${PV}" recipes, they don't have
         # the "do_configure" task, so we need to use "do_preconfigure"
-        if pn.startswith("gcc-source-"):
+        def hasTask(task):
+            return bool(d.getVarFlag(task, "task", False)) and not bool(d.getVarFlag(task, "noexec", False))
+
+        if hasTask("do_preconfigure"):
             d.appendVarFlag('do_ar_configured', 'depends', ' %s:do_preconfigure' % pn)
-        else:
+        elif hasTask("do_configure"):
             d.appendVarFlag('do_ar_configured', 'depends', ' %s:do_configure' % pn)
         d.appendVarFlag('do_deploy_archives', 'depends', ' %s:do_ar_configured' % pn)
 
@@ -243,21 +246,26 @@ python do_ar_configured() {
         # do_configure, we archive the already configured ${S} to
         # instead of.
         elif pn != 'libtool-native':
+            def runTask(task):
+                prefuncs = d.getVarFlag(task, 'prefuncs') or ''
+                for func in prefuncs.split():
+                    if func != "sysroot_cleansstate":
+                        bb.build.exec_func(func, d)
+                bb.build.exec_func(task, d)
+                postfuncs = d.getVarFlag(task, 'postfuncs') or ''
+                for func in postfuncs.split():
+                    if func != 'do_qa_configure':
+                        bb.build.exec_func(func, d)
+
             # Change the WORKDIR to make do_configure run in another dir.
             d.setVar('WORKDIR', d.getVar('ARCHIVER_WORKDIR'))
-            if bb.data.inherits_class('kernel-yocto', d):
-                bb.build.exec_func('do_kernel_configme', d)
-            if bb.data.inherits_class('cmake', d):
-                bb.build.exec_func('do_generate_toolchain_file', d)
-            prefuncs = d.getVarFlag('do_configure', 'prefuncs')
-            for func in (prefuncs or '').split():
-                if func != "sysroot_cleansstate":
-                    bb.build.exec_func(func, d)
-            bb.build.exec_func('do_configure', d)
-            postfuncs = d.getVarFlag('do_configure', 'postfuncs')
-            for func in (postfuncs or '').split():
-                if func != "do_qa_configure":
-                    bb.build.exec_func(func, d)
+
+            preceeds = bb.build.preceedtask('do_configure', False, d)
+            for task in preceeds:
+                if task != 'do_patch' and task != 'do_prepare_recipe_sysroot':
+                    runTask(task)
+            runTask('do_configure')
+
         srcdir = d.getVar('S')
         builddir = d.getVar('B')
         if srcdir != builddir:

meta/classes/devtool-source.bbclass

@@ -183,14 +183,14 @@ python devtool_post_patch() {
 
     extra_overrides = d.getVar('DEVTOOL_EXTRA_OVERRIDES')
     if extra_overrides:
-        extra_override_list = extra_overrides.split(':')
+        extra_overrides = set(extra_overrides.split(':'))
         devbranch = d.getVar('DEVTOOL_DEVBRANCH')
         default_overrides = d.getVar('OVERRIDES').split(':')
         no_overrides = []
         # First, we may have some overrides that are referred to in the recipe set in
         # our configuration, so we need to make a branch that excludes those
         for override in default_overrides:
-            if override not in extra_override_list:
+            if override not in extra_overrides:
                 no_overrides.append(override)
         if default_overrides != no_overrides:
             # Some overrides are active in the current configuration, so
@@ -208,7 +208,7 @@ python devtool_post_patch() {
         else:
             bb.process.run('git checkout %s -b devtool-no-overrides' % devbranch, cwd=srcsubdir)
 
-        for override in extra_override_list:
+        for override in extra_overrides:
             localdata = bb.data.createCopy(d)
             if override in default_overrides:
                 bb.process.run('git branch devtool-override-%s %s' % (override, devbranch), cwd=srcsubdir)

meta/classes/externalsrc.bbclass

@@ -41,8 +41,8 @@ python () {
     # re-parsed so that the file-checksums function for do_compile is run every
     # time.
     bpn = d.getVar('BPN')
-    if bpn == d.getVar('PN'):
-        classextend = (d.getVar('BBCLASSEXTEND') or '').split()
+    classextend = (d.getVar('BBCLASSEXTEND') or '').split()
+    if bpn == d.getVar('PN') or not classextend:
         if (externalsrc or
                 ('native' in classextend and
                  d.getVar('EXTERNALSRC_pn-%s-native' % bpn)) or

meta/classes/go.bbclass

@@ -179,3 +179,13 @@ FILES_${PN}-staticdev = "${libdir}/go/pkg"
 
 INSANE_SKIP_${PN} += "ldflags"
 INSANE_SKIP_${PN}-ptest += "ldflags"
+
+# Add -buildmode=pie to GOBUILDFLAGS to satisfy "textrel" QA checking, but mips
+# doesn't support -buildmode=pie, so skip the QA checking for mips and its
+# variants.
+python() {
+    if 'mips' in d.getVar('TARGET_ARCH'):
+        d.appendVar('INSANE_SKIP_%s' % d.getVar('PN'), " textrel")
+    else:
+        d.appendVar('GOBUILDFLAGS', ' -buildmode=pie')
+}

meta/classes/gobject-introspection.bbclass

@@ -40,4 +40,4 @@ FILES_${PN}_append = " ${libdir}/girepository-*/*.typelib"
     
 # .gir files go to dev package, as they're needed for developing (but not for
 # running) things that depends on introspection.
-FILES_${PN}-dev_append = " ${datadir}/gir-*/*.gir"
+FILES_${PN}-dev_append = " ${datadir}/gir-*/*.gir ${libdir}/gir-*/*.gir"

meta/classes/image.bbclass

@@ -38,6 +38,8 @@ IMAGE_FEATURES[validitems] += "debug-tweaks read-only-rootfs empty-root-password
 # Generate companion debugfs?
 IMAGE_GEN_DEBUGFS ?= "0"
 
+# These pacackages will be installed as additional into debug rootfs
+IMAGE_INSTALL_DEBUGFS ?= ""
 
 # These packages will be removed from a read-only rootfs after all other
 # packages have been installed
@@ -125,7 +127,7 @@ def rootfs_variables(d):
                  'IMAGE_ROOTFS_MAXSIZE','IMAGE_NAME','IMAGE_LINK_NAME','IMAGE_MANIFEST','DEPLOY_DIR_IMAGE','IMAGE_FSTYPES','IMAGE_INSTALL_COMPLEMENTARY','IMAGE_LINGUAS',
                  'MULTILIBRE_ALLOW_REP','MULTILIB_TEMP_ROOTFS','MULTILIB_VARIANTS','MULTILIBS','ALL_MULTILIB_PACKAGE_ARCHS','MULTILIB_GLOBAL_VARIANTS','BAD_RECOMMENDATIONS','NO_RECOMMENDATIONS',
                  'PACKAGE_ARCHS','PACKAGE_CLASSES','TARGET_VENDOR','TARGET_ARCH','TARGET_OS','OVERRIDES','BBEXTENDVARIANT','FEED_DEPLOYDIR_BASE_URI','INTERCEPT_DIR','USE_DEVFS',
-                 'CONVERSIONTYPES', 'IMAGE_GEN_DEBUGFS', 'ROOTFS_RO_UNNEEDED', 'IMGDEPLOYDIR', 'PACKAGE_EXCLUDE_COMPLEMENTARY', 'REPRODUCIBLE_TIMESTAMP_ROOTFS']
+                 'CONVERSIONTYPES', 'IMAGE_GEN_DEBUGFS', 'ROOTFS_RO_UNNEEDED', 'IMGDEPLOYDIR', 'PACKAGE_EXCLUDE_COMPLEMENTARY', 'REPRODUCIBLE_TIMESTAMP_ROOTFS', 'IMAGE_INSTALL_DEBUGFS']
     variables.extend(rootfs_command_variables(d))
     variables.extend(variable_depends(d))
     return " ".join(variables)

meta/classes/image_types.bbclass

@@ -154,7 +154,7 @@ multiubi_mkfs() {
 	local ubinize_args="$2"
     
         # Added prompt error message for ubi and ubifs image creation.
-        if [ -z "$mkubifs_args"] || [ -z "$ubinize_args" ]; then
+        if [ -z "$mkubifs_args" ] || [ -z "$ubinize_args" ]; then
             bbfatal "MKUBIFS_ARGS and UBINIZE_ARGS have to be set, see http://www.linux-mtd.infradead.org/faq/ubifs.html for details"
         fi
     

meta/classes/kernel-arch.bbclass

@@ -29,6 +29,7 @@ def map_kernel_arch(a, d):
     elif re.match('aarch64_ilp32$', a):         return 'arm64'
     elif re.match('aarch64_be_ilp32$', a):      return 'arm64'
     elif re.match('mips(isa|)(32|64|)(r6|)(el|)$', a):      return 'mips'
+    elif re.match('mcf', a):                    return 'm68k'
     elif re.match('riscv(32|64|)(eb|)$', a):    return 'riscv'
     elif re.match('p(pc|owerpc)(|64)', a):      return 'powerpc'
     elif re.match('sh(3|4)$', a):               return 'sh'

meta/classes/kernel-yocto.bbclass

@@ -337,8 +337,8 @@ python do_kernel_configcheck() {
 
     try:
         configs = subprocess.check_output(['scc', '--configs', '-o', s + '/.kernel-meta'], env=env).decode('utf-8')
-    except subprocess.CalledProcessError:
-        bb.fatal( "Cannot gather config fragments for audit: %s" % configs)
+    except subprocess.CalledProcessError as e:
+        bb.fatal( "Cannot gather config fragments for audit: %s" % e.output.decode("utf-8") )
 
     try:
         subprocess.check_call(['kconf_check', '--report', '-o',

meta/classes/meson.bbclass

@@ -1,4 +1,4 @@
-inherit python3native
+inherit siteinfo python3native
 
 DEPENDS_append = " meson-native ninja-native"
 
@@ -24,17 +24,17 @@ MESONOPTS = " --prefix ${prefix} \
               --infodir ${@noprefix('infodir', d)} \
               --sysconfdir ${sysconfdir} \
               --localstatedir ${localstatedir} \
-              --sharedstatedir ${sharedstatedir}"
+              --sharedstatedir ${sharedstatedir} \
+              -Dc_args='${BUILD_CPPFLAGS} ${BUILD_CFLAGS}' \
+              -Dc_link_args='${BUILD_LDFLAGS}' \
+              -Dcpp_args='${BUILD_CPPFLAGS} ${BUILD_CXXFLAGS}' \
+              -Dcpp_link_args='${BUILD_LDFLAGS}'"
 
 MESON_TOOLCHAIN_ARGS = "${HOST_CC_ARCH}${TOOLCHAIN_OPTIONS}"
 MESON_C_ARGS = "${MESON_TOOLCHAIN_ARGS} ${CFLAGS}"
 MESON_CPP_ARGS = "${MESON_TOOLCHAIN_ARGS} ${CXXFLAGS}"
 MESON_LINK_ARGS = "${MESON_TOOLCHAIN_ARGS} ${LDFLAGS}"
 
-# both are required but not used by meson
-MESON_HOST_ENDIAN = "bogus-endian"
-MESON_TARGET_ENDIAN = "bogus-endian"
-
 EXTRA_OEMESON_append = " ${PACKAGECONFIG_CONFARGS}"
 
 MESON_CROSS_FILE = ""
@@ -59,6 +59,16 @@ def meson_cpu_family(var, d):
     else:
         return arch
 
+def meson_endian(prefix, d):
+    arch, os = d.getVar(prefix + "_ARCH"), d.getVar(prefix + "_OS")
+    sitedata = siteinfo_data_for_machine(arch, os, d)
+    if "endian-little" in sitedata:
+        return "little"
+    elif "endian-big" in sitedata:
+        return "big"
+    else:
+        bb.fatal("Cannot determine endianism for %s-%s" % (arch, os))
+
 addtask write_config before do_configure
 do_write_config[vardeps] += "MESON_C_ARGS MESON_CPP_ARGS MESON_LINK_ARGS CC CXX LD AR NM STRIP READELF"
 do_write_config() {
@@ -86,19 +96,22 @@ gtkdoc_exe_wrapper = '${B}/gtkdoc-qemuwrapper'
 system = '${HOST_OS}'
 cpu_family = '${@meson_cpu_family('HOST_ARCH', d)}'
 cpu = '${HOST_ARCH}'
-endian = '${MESON_HOST_ENDIAN}'
+endian = '${@meson_endian('HOST', d)}'
 
 [target_machine]
 system = '${TARGET_OS}'
 cpu_family = '${@meson_cpu_family('TARGET_ARCH', d)}'
 cpu = '${TARGET_ARCH}'
-endian = '${MESON_TARGET_ENDIAN}'
+endian = '${@meson_endian('TARGET', d)}'
 EOF
 }
 
 CONFIGURE_FILES = "meson.build"
 
 meson_do_configure() {
+    # Work around "Meson fails if /tmp is mounted with noexec #2972"
+    mkdir -p "${B}/meson-private/tmp"
+    export TMPDIR="${B}/meson-private/tmp"
     bbnote Executing meson ${EXTRA_OEMESON}...
     if ! meson ${MESONOPTS} "${MESON_SOURCEPATH}" "${B}" ${MESON_CROSS_FILE} ${EXTRA_OEMESON}; then
         bbfatal_log meson failed
@@ -108,14 +121,15 @@ meson_do_configure() {
 override_native_tools() {
     # Set these so that meson uses the native tools for its build sanity tests,
     # which require executables to be runnable. The cross file will still
-    # override these for the target build. Note that we do *not* set CFLAGS,
-    # LDFLAGS, etc. as they will be slurped in by meson and applied to the
-    # target build, causing errors.
+    # override these for the target build.
     export CC="${BUILD_CC}"
     export CXX="${BUILD_CXX}"
     export LD="${BUILD_LD}"
     export AR="${BUILD_AR}"
-
+    # These contain *target* flags but will be used as *native* flags.  The
+    # correct native flags will be passed via -Dc_args and so on, unset them so
+    # they don't interfere with tools invoked by Meson (such as g-ir-scanner)
+    unset CPPFLAGS CFLAGS CXXFLAGS LDFLAGS
 }
 
 meson_do_configure_prepend_class-target() {

meta/classes/mime.bbclass

@@ -53,5 +53,5 @@ python populate_packages_append () {
             postrm += d.getVar('mime_postrm')
             d.setVar('pkg_postrm_%s' % pkg, postrm)
             bb.note("adding shared-mime-info-data dependency to %s" % pkg)
-            d.appendVar('RDEPENDS_' + pkg, " shared-mime-info-data")
+            d.appendVar('RDEPENDS_' + pkg, " " + d.getVar('MLPREFIX')+"shared-mime-info-data")
 }

meta/classes/multilib.bbclass

@@ -11,8 +11,12 @@ python multilib_virtclass_handler () {
     # There should only be one kernel in multilib configs
     # We also skip multilib setup for module packages.
     provides = (e.data.getVar("PROVIDES") or "").split()
-    if "virtual/kernel" in provides or bb.data.inherits_class('module-base', e.data) or "make-mod-scripts" in e.data.getVar("PN"):
-        raise bb.parse.SkipRecipe("We shouldn't have multilib variants for the kernel")
+    non_ml_recipes = d.getVar('NON_MULTILIB_RECIPES').split()
+    bpn = e.data.getVar("BPN")
+    if "virtual/kernel" in provides or \
+            bb.data.inherits_class('module-base', e.data) or \
+            bpn in non_ml_recipes:
+        raise bb.parse.SkipRecipe("We shouldn't have multilib variants for %s" % bpn)
 
     save_var_name=e.data.getVar("MULTILIB_SAVE_VARNAME") or ""
     for name in save_var_name.split():
@@ -135,7 +139,8 @@ python do_package_qa_multilib() {
                 i = i[len('virtual/'):]
             if (not i.startswith('kernel-module')) and (not i.startswith(mlprefix)) and \
                 (not 'cross-canadian' in i) and (not i.startswith("nativesdk-")) and \
-                (not i.startswith("rtld")) and (not i.startswith('kernel-vmlinux')):
+                (not i.startswith("rtld")) and (not i.startswith('kernel-vmlinux')) \
+                and (not i.startswith("kernel-image")):
                 candidates.append(i)
         if len(candidates) > 0:
             msg = "%s package %s - suspicious values '%s' in %s" \

meta/classes/multilib_global.bbclass

@@ -164,8 +164,11 @@ python multilib_virtclass_handler_global () {
     if variant:
         return
 
+    non_ml_recipes = d.getVar('NON_MULTILIB_RECIPES').split()
+
     if bb.data.inherits_class('kernel', e.data) or \
-            bb.data.inherits_class('module-base', e.data):
+            bb.data.inherits_class('module-base', e.data) or \
+            d.getVar('BPN') in non_ml_recipes:
             variants = (e.data.getVar("MULTILIB_VARIANTS") or "").split()
 
             import oe.classextend

meta/classes/nativesdk.bbclass

@@ -12,6 +12,11 @@ MACHINEOVERRIDES = ""
 
 MULTILIBS = ""
 
+# we need consistent staging dir whether or not multilib is enabled
+STAGING_DIR_HOST = "${WORKDIR}/recipe-sysroot"
+STAGING_DIR_TARGET = "${WORKDIR}/recipe-sysroot"
+RECIPE_SYSROOT = "${WORKDIR}/recipe-sysroot"
+
 #
 # Update PACKAGE_ARCH and PACKAGE_ARCHS
 #

meta/classes/npm.bbclass

@@ -10,7 +10,7 @@ def node_pkgname(d):
 
 NPMPN ?= "${@node_pkgname(d)}"
 
-NPM_INSTALLDIR = "${D}${libdir}/node_modules/${NPMPN}"
+NPM_INSTALLDIR = "${D}${libdir}/node/${NPMPN}"
 
 # function maps arch names to npm arch names
 def npm_oe_arch_map(target_arch, d):

meta/classes/populate_sdk_base.bbclass

@@ -228,14 +228,17 @@ fakeroot tar_sdk() {
 	tar ${SDKTAROPTS} -cf - . | xz -T 0 > ${SDKDEPLOYDIR}/${TOOLCHAIN_OUTPUTNAME}.tar.xz
 }
 
+TOOLCHAIN_SHAR_EXT_TMPL ?= "${COREBASE}/meta/files/toolchain-shar-extract.sh"
+TOOLCHAIN_SHAR_REL_TMPL ?= "${COREBASE}/meta/files/toolchain-shar-relocate.sh"
+
 fakeroot create_shar() {
 	# copy in the template shar extractor script
-	cp ${COREBASE}/meta/files/toolchain-shar-extract.sh ${SDKDEPLOYDIR}/${TOOLCHAIN_OUTPUTNAME}.sh
+	cp ${TOOLCHAIN_SHAR_EXT_TMPL} ${SDKDEPLOYDIR}/${TOOLCHAIN_OUTPUTNAME}.sh
 
 	rm -f ${T}/pre_install_command ${T}/post_install_command
 
 	if [ ${SDK_RELOCATE_AFTER_INSTALL} -eq 1 ] ; then
-		cp ${COREBASE}/meta/files/toolchain-shar-relocate.sh ${T}/post_install_command
+		cp ${TOOLCHAIN_SHAR_REL_TMPL} ${T}/post_install_command
 	fi
 	cat << "EOF" >> ${T}/pre_install_command
 ${SDK_PRE_INSTALL_COMMAND}
@@ -293,14 +296,14 @@ def sdk_command_variables(d):
 def sdk_variables(d):
     variables = ['BUILD_IMAGES_FROM_FEEDS','SDK_OS','SDK_OUTPUT','SDKPATHNATIVE','SDKTARGETSYSROOT','SDK_DIR','SDK_VENDOR','SDKIMAGE_INSTALL_COMPLEMENTARY','SDK_PACKAGE_ARCHS','SDK_OUTPUT',
                  'SDKTARGETSYSROOT','MULTILIB_VARIANTS','MULTILIBS','ALL_MULTILIB_PACKAGE_ARCHS','MULTILIB_GLOBAL_VARIANTS','BAD_RECOMMENDATIONS','NO_RECOMMENDATIONS','PACKAGE_ARCHS',
-                 'PACKAGE_CLASSES','TARGET_VENDOR','TARGET_VENDOR','TARGET_ARCH','TARGET_OS','BBEXTENDVARIANT','FEED_DEPLOYDIR_BASE_URI', 'PACKAGE_EXCLUDE_COMPLEMENTARY']
+                 'PACKAGE_CLASSES','TARGET_VENDOR','TARGET_VENDOR','TARGET_ARCH','TARGET_OS','BBEXTENDVARIANT','FEED_DEPLOYDIR_BASE_URI', 'PACKAGE_EXCLUDE_COMPLEMENTARY', 'IMAGE_INSTALL_DEBUGFS']
     variables.extend(sdk_command_variables(d))
     return " ".join(variables)
 
 do_populate_sdk[vardeps] += "${@sdk_variables(d)}"
 
-do_populate_sdk[file-checksums] += "${COREBASE}/meta/files/toolchain-shar-relocate.sh:True \
-                                    ${COREBASE}/meta/files/toolchain-shar-extract.sh:True"
+do_populate_sdk[file-checksums] += "${TOOLCHAIN_SHAR_REL_TMPL}:True \
+                                    ${TOOLCHAIN_SHAR_EXT_TMPL}:True"
 
 do_populate_sdk[dirs] = "${PKGDATA_DIR} ${TOPDIR}"
 do_populate_sdk[depends] += "${@' '.join([x + ':do_populate_sysroot' for x in d.getVar('SDK_DEPENDS').split()])}  ${@d.getVarFlag('do_rootfs', 'depends', False)}"

meta/classes/populate_sdk_ext.bbclass

@@ -531,7 +531,8 @@ install_tools() {
 	scripts="devtool recipetool oe-find-native-sysroot runqemu* wic"
 	for script in $scripts; do
 		for scriptfn in `find ${SDK_OUTPUT}/${SDKPATH}/${scriptrelpath} -maxdepth 1 -executable -name "$script"`; do
-			lnr ${scriptfn} ${SDK_OUTPUT}/${SDKPATHNATIVE}${bindir_nativesdk}/`basename $scriptfn`
+			targetscriptfn="${SDK_OUTPUT}/${SDKPATHNATIVE}${bindir_nativesdk}/$(basename $scriptfn)"
+			test -e ${targetscriptfn} || lnr ${scriptfn} ${targetscriptfn}
 		done
 	done
 	# We can't use the same method as above because files in the sysroot won't exist at this point

meta/classes/rootfs-postcommands.bbclass

@@ -253,7 +253,6 @@ python write_image_manifest () {
     pkgs = image_list_installed_packages(d)
     with open(manifest_name, 'w+') as image_manifest:
         image_manifest.write(format_pkg_list(pkgs, "ver"))
-        image_manifest.write("\n")
 
     if os.path.exists(manifest_name):
         manifest_link = deploy_dir + "/" + link_name + ".manifest"

meta/classes/rootfs_deb.bbclass

@@ -6,7 +6,7 @@ ROOTFS_PKGMANAGE = "dpkg apt"
 
 do_rootfs[depends] += "dpkg-native:do_populate_sysroot apt-native:do_populate_sysroot"
 do_populate_sdk[depends] += "dpkg-native:do_populate_sysroot apt-native:do_populate_sysroot bzip2-native:do_populate_sysroot"
-do_rootfs[recrdeptask] += "do_package_write_deb"
+do_rootfs[recrdeptask] += "do_package_write_deb do_package_qa"
 do_rootfs[vardeps] += "PACKAGE_FEED_URIS"
 
 do_rootfs[lockfiles] += "${DEPLOY_DIR_DEB}/deb.lock"

meta/classes/rootfs_ipk.bbclass

@@ -10,7 +10,7 @@ ROOTFS_PKGMANAGE = "opkg ${EXTRAOPKGCONFIG}"
 
 do_rootfs[depends] += "opkg-native:do_populate_sysroot opkg-utils-native:do_populate_sysroot"
 do_populate_sdk[depends] += "opkg-native:do_populate_sysroot opkg-utils-native:do_populate_sysroot"
-do_rootfs[recrdeptask] += "do_package_write_ipk"
+do_rootfs[recrdeptask] += "do_package_write_ipk do_package_qa"
 do_rootfs[vardeps] += "PACKAGE_FEED_URIS"
 
 do_rootfs[lockfiles] += "${WORKDIR}/ipk.lock"

meta/classes/rootfs_rpm.bbclass

@@ -23,7 +23,7 @@ RPMROOTFSDEPENDS = "rpm-native:do_populate_sysroot \
 do_rootfs[depends] += "${RPMROOTFSDEPENDS}"
 do_populate_sdk[depends] += "${RPMROOTFSDEPENDS}"
 
-do_rootfs[recrdeptask] += "do_package_write_rpm"
+do_rootfs[recrdeptask] += "do_package_write_rpm do_package_qa"
 do_rootfs[vardeps] += "PACKAGE_FEED_URIS"
 
 python () {

meta/classes/siteinfo.bbclass

@@ -15,7 +15,7 @@
 # It is an error for the target not to exist.
 # If 'what' doesn't exist then an empty value is returned
 #
-def siteinfo_data(d):
+def siteinfo_data_for_machine(arch, os, d):
     archinfo = {
         "allarch": "endian-little bit-32", # bogus, but better than special-casing the checks below for allarch
         "aarch64": "endian-little bit-64 arm-common arm-64",
@@ -33,6 +33,7 @@ def siteinfo_data(d):
         "i686": "endian-little bit-32 ix86-common",
         "ia64": "endian-little bit-64",
         "lm32": "endian-big bit-32",
+        "m68k": "endian-big bit-32",
         "microblaze": "endian-big bit-32 microblaze-common",
         "microblazeeb": "endian-big bit-32 microblaze-common",
         "microblazeel": "endian-little bit-32 microblaze-common",
@@ -131,15 +132,13 @@ def siteinfo_data(d):
         locs = { "archinfo" : archinfo, "osinfo" : osinfo, "targetinfo" : targetinfo, "d" : d}
         archinfo, osinfo, targetinfo = bb.utils.better_eval(call, locs)
 
-    hostarch = d.getVar("HOST_ARCH")
-    hostos = d.getVar("HOST_OS")
-    target = "%s-%s" % (hostarch, hostos)
+    target = "%s-%s" % (arch, os)
 
     sitedata = []
-    if hostarch in archinfo:
-        sitedata.extend(archinfo[hostarch].split())
-    if hostos in osinfo:
-        sitedata.extend(osinfo[hostos].split())
+    if arch in archinfo:
+        sitedata.extend(archinfo[arch].split())
+    if os in osinfo:
+        sitedata.extend(osinfo[os].split())
     if target in targetinfo:
         sitedata.extend(targetinfo[target].split())
     sitedata.append(target)
@@ -148,6 +147,9 @@ def siteinfo_data(d):
     bb.debug(1, "SITE files %s" % sitedata);
     return sitedata
 
+def siteinfo_data(d):
+    return siteinfo_data_for_machine(d.getVar("HOST_ARCH"), d.getVar("HOST_OS"), d)
+
 python () {
     sitedata = set(siteinfo_data(d))
     if "endian-little" in sitedata:

meta/classes/testexport.bbclass

@@ -131,6 +131,11 @@ def copy_needed_files(d, tc):
     shutil.copy2(image_manifest, os.path.join(test_data_path, 'manifest'))
     shutil.copy2(tdname, os.path.join(test_data_path, 'testdata.json'))
 
+    for subdir, dirs, files in os.walk(export_path):
+        for dir in dirs:
+            if dir == '__pycache__':
+                shutil.rmtree(os.path.join(subdir, dir))
+
     # Create tar file for common parts of testexport
     create_tarball(d, "testexport.tar.gz", d.getVar("TEST_EXPORT_DIR"))
 

meta/classes/testimage.bbclass

@@ -40,7 +40,11 @@ TEST_NEEDED_PACKAGES_DIR ?= "${WORKDIR}/testimage/packages"
 TEST_EXTRACTED_DIR ?= "${TEST_NEEDED_PACKAGES_DIR}/extracted"
 TEST_PACKAGED_DIR ?= "${TEST_NEEDED_PACKAGES_DIR}/packaged"
 
-RPMTESTSUITE = "${@bb.utils.contains('IMAGE_PKGTYPE', 'rpm', 'dnf rpm', '', d)}"
+PKGMANTESTSUITE = "\
+    ${@bb.utils.contains('IMAGE_PKGTYPE', 'rpm', 'dnf rpm', '', d)} \
+    ${@bb.utils.contains('IMAGE_PKGTYPE', 'ipk', 'opkg', '', d)} \
+    ${@bb.utils.contains('IMAGE_PKGTYPE', 'deb', 'apt', '', d)} \
+    "
 SYSTEMDSUITE = "${@bb.utils.filter('DISTRO_FEATURES', 'systemd', d)}"
 MINTESTSUITE = "ping"
 NETTESTSUITE = "${MINTESTSUITE} ssh df date scp oe_syslog ${SYSTEMDSUITE}"
@@ -51,14 +55,14 @@ DEFAULT_TEST_SUITES_pn-core-image-minimal = "${MINTESTSUITE}"
 DEFAULT_TEST_SUITES_pn-core-image-minimal-dev = "${MINTESTSUITE}"
 DEFAULT_TEST_SUITES_pn-core-image-full-cmdline = "${NETTESTSUITE} perl python logrotate ptest"
 DEFAULT_TEST_SUITES_pn-core-image-x11 = "${MINTESTSUITE}"
-DEFAULT_TEST_SUITES_pn-core-image-lsb = "${NETTESTSUITE} pam parselogs ${RPMTESTSUITE} ptest"
-DEFAULT_TEST_SUITES_pn-core-image-sato = "${NETTESTSUITE} connman xorg parselogs ${RPMTESTSUITE} \
+DEFAULT_TEST_SUITES_pn-core-image-lsb = "${NETTESTSUITE} pam parselogs ${PKGMANTESTSUITE} ptest"
+DEFAULT_TEST_SUITES_pn-core-image-sato = "${NETTESTSUITE} connman xorg parselogs ${PKGMANTESTSUITE} \
     ${@bb.utils.contains('IMAGE_PKGTYPE', 'rpm', 'python', '', d)} ptest gi"
 DEFAULT_TEST_SUITES_pn-core-image-sato-sdk = "${NETTESTSUITE} buildcpio buildlzip buildgalculator \
-    connman ${DEVTESTSUITE} logrotate perl parselogs python ${RPMTESTSUITE} xorg ptest gi stap"
-DEFAULT_TEST_SUITES_pn-core-image-lsb-dev = "${NETTESTSUITE} pam perl python parselogs ${RPMTESTSUITE} ptest gi"
+    connman ${DEVTESTSUITE} logrotate perl parselogs python ${PKGMANTESTSUITE} xorg ptest gi stap"
+DEFAULT_TEST_SUITES_pn-core-image-lsb-dev = "${NETTESTSUITE} pam perl python parselogs ${PKGMANTESTSUITE} ptest gi"
 DEFAULT_TEST_SUITES_pn-core-image-lsb-sdk = "${NETTESTSUITE} buildcpio buildlzip buildgalculator \
-    connman ${DEVTESTSUITE} logrotate pam parselogs perl python ${RPMTESTSUITE} ptest gi stap"
+    connman ${DEVTESTSUITE} logrotate pam parselogs perl python ${PKGMANTESTSUITE} ptest gi stap"
 DEFAULT_TEST_SUITES_pn-meta-toolchain = "auto"
 
 # aarch64 has no graphics
@@ -78,13 +82,13 @@ TEST_QEMUBOOT_TIMEOUT ?= "1000"
 TEST_TARGET ?= "qemu"
 
 TESTIMAGEDEPENDS = ""
-TESTIMAGEDEPENDS_qemuall = "qemu-native:do_populate_sysroot qemu-helper-native:do_populate_sysroot qemu-helper-native:do_addto_recipe_sysroot"
+TESTIMAGEDEPENDS_append_qemuall = " qemu-native:do_populate_sysroot qemu-helper-native:do_populate_sysroot qemu-helper-native:do_addto_recipe_sysroot"
 TESTIMAGEDEPENDS += "${@bb.utils.contains('IMAGE_PKGTYPE', 'rpm', 'cpio-native:do_populate_sysroot', '', d)}"
-TESTIMAGEDEPENDS_qemuall += "${@bb.utils.contains('IMAGE_PKGTYPE', 'rpm', 'cpio-native:do_populate_sysroot', '', d)}"
-TESTIMAGEDEPENDS_qemuall += "${@bb.utils.contains('IMAGE_PKGTYPE', 'rpm', 'createrepo-c-native:do_populate_sysroot', '', d)}"
+TESTIMAGEDEPENDS_append_qemuall = " ${@bb.utils.contains('IMAGE_PKGTYPE', 'rpm', 'cpio-native:do_populate_sysroot', '', d)}"
+TESTIMAGEDEPENDS_append_qemuall = " ${@bb.utils.contains('IMAGE_PKGTYPE', 'rpm', 'createrepo-c-native:do_populate_sysroot', '', d)}"
 TESTIMAGEDEPENDS += "${@bb.utils.contains('IMAGE_PKGTYPE', 'rpm', 'dnf-native:do_populate_sysroot', '', d)}"
-TESTIMAGEDEPENDS += "${@bb.utils.contains('IMAGE_PKGTYPE', 'ipk', 'opkg-utils-native:do_populate_sysroot', '', d)}"
-TESTIMAGEDEPENDS += "${@bb.utils.contains('IMAGE_PKGTYPE', 'deb', 'apt-native:do_populate_sysroot', '', d)}"
+TESTIMAGEDEPENDS += "${@bb.utils.contains('IMAGE_PKGTYPE', 'ipk', 'opkg-utils-native:do_populate_sysroot package-index:do_package_index', '', d)}"
+TESTIMAGEDEPENDS += "${@bb.utils.contains('IMAGE_PKGTYPE', 'deb', 'apt-native:do_populate_sysroot  package-index:do_package_index', '', d)}"
 TESTIMAGEDEPENDS += "${@bb.utils.contains('IMAGE_PKGTYPE', 'rpm', 'createrepo-c-native:do_populate_sysroot', '', d)}"
 
 TESTIMAGELOCK = "${TMPDIR}/testimage.lock"

meta/classes/update-rc.d.bbclass

@@ -39,9 +39,9 @@ PACKAGE_WRITE_DEPS += "update-rc.d-native"
 updatercd_postinst() {
 if ${@use_updatercd(d)} && type update-rc.d >/dev/null 2>/dev/null; then
 	if [ -n "$D" ]; then
-		OPT="-r $D"
+		OPT="-f -r $D"
 	else
-		OPT="-s"
+		OPT="-f -s"
 	fi
 	update-rc.d $OPT ${INITSCRIPT_NAME} ${INITSCRIPT_PARAMS}
 fi

meta/classes/useradd_base.bbclass

@@ -51,10 +51,10 @@ perform_groupmems () {
 	local groupname=`echo "$opts" | awk '{ for (i = 1; i < NF; i++) if ($i == "-g" || $i == "--group") print $(i+1) }'`
 	local username=`echo "$opts" | awk '{ for (i = 1; i < NF; i++) if ($i == "-a" || $i == "--add") print $(i+1) }'`
 	bbnote "${PN}: Running groupmems command with group $groupname and user $username"
-	local mem_exists="`grep "^$groupname:[^:]*:[^:]*:\([^,]*,\)*$username\(,[^,]*\)*" $rootdir/etc/group || true`"
+	local mem_exists="`grep "^$groupname:[^:]*:[^:]*:\([^,]*,\)*$username\(,[^,]*\)*$" $rootdir/etc/group || true`"
 	if test "x$mem_exists" = "x"; then
 		eval flock -x $rootdir${sysconfdir} -c \"$PSEUDO groupmems \$opts\" || true
-		mem_exists="`grep "^$groupname:[^:]*:[^:]*:\([^,]*,\)*$username\(,[^,]*\)*" $rootdir/etc/group || true`"
+		mem_exists="`grep "^$groupname:[^:]*:[^:]*:\([^,]*,\)*$username\(,[^,]*\)*$" $rootdir/etc/group || true`"
 		if test "x$mem_exists" = "x"; then
 			bbfatal "${PN}: groupmems command did not succeed."
 		fi

meta/conf/distro/include/maintainers.inc

@@ -96,8 +96,8 @@ RECIPE_MAINTAINER_pn-chrpath = "Yi Zhao <yi.zhao@windriver.com>"
 RECIPE_MAINTAINER_pn-clutter-1.0 = "Ross Burton <ross.burton@intel.com>"
 RECIPE_MAINTAINER_pn-clutter-gst-3.0 = "Ross Burton <ross.burton@intel.com>"
 RECIPE_MAINTAINER_pn-clutter-gtk-1.0 = "Ross Burton <ross.burton@intel.com>"
-RECIPE_MAINTAINER_pn-cmake = "Otavio Salvador <otavio.salvador@ossystems.com.br>"
-RECIPE_MAINTAINER_pn-cmake-native = "Otavio Salvador <otavio.salvador@ossystems.com.br>"
+RECIPE_MAINTAINER_pn-cmake = "Pascal Bach <pascal.bach@siemens.com>"
+RECIPE_MAINTAINER_pn-cmake-native = "Pascal Bach <pascal.bach@siemens.com>"
 RECIPE_MAINTAINER_pn-cogl-1.0 = "Ross Burton <ross.burton@intel.com>"
 RECIPE_MAINTAINER_pn-connman = "Changhyeok Bae <changhyeok.bae@lge.com>"
 RECIPE_MAINTAINER_pn-connman-conf = "Ross Burton <ross.burton@intel.com>"
@@ -629,6 +629,7 @@ RECIPE_MAINTAINER_pn-systemd = "Chen Qi <Qi.Chen@windriver.com>"
 RECIPE_MAINTAINER_pn-systemd-boot = "Chen Qi <Qi.Chen@windriver.com>"
 RECIPE_MAINTAINER_pn-systemd-bootchart = "Chen Qi <Qi.Chen@windriver.com>"
 RECIPE_MAINTAINER_pn-systemd-bootconf = "Chen Qi <Qi.Chen@windriver.com>"
+RECIPE_MAINTAINER_pn-systemd-conf = "Chen Qi <Qi.Chen@windriver.com>"
 RECIPE_MAINTAINER_pn-systemd-compat-units = "Chen Qi <Qi.Chen@windriver.com>"
 RECIPE_MAINTAINER_pn-systemd-serialgetty = "Chen Qi <Qi.Chen@windriver.com>"
 RECIPE_MAINTAINER_pn-systemd-systemctl-native = "Chen Qi <Qi.Chen@windriver.com>"

meta/conf/distro/include/tcmode-default.inc

@@ -29,6 +29,7 @@ GDBVERSION ?= "8.2%"
 GLIBCVERSION ?= "2.28%"
 LINUXLIBCVERSION ?= "4.18%"
 QEMUVERSION ?= "3.0%"
+GOVERSION ?= "1.11%"
 
 PREFERRED_VERSION_gcc ?= "${GCCVERSION}"
 PREFERRED_VERSION_gcc-cross-${TARGET_ARCH} ?= "${GCCVERSION}"
@@ -69,7 +70,6 @@ PREFERRED_VERSION_qemu ?= "${QEMUVERSION}"
 PREFERRED_VERSION_qemu-native ?= "${QEMUVERSION}"
 PREFERRED_VERSION_nativesdk-qemu ?= "${QEMUVERSION}"
 
-GOVERSION ?= "1.9%"
 PREFERRED_VERSION_virtual/${TARGET_PREFIX}go ?= "${GOVERSION}"
 PREFERRED_VERSION_go-cross-${TARGET_ARCH} ?= "${GOVERSION}"
 PREFERRED_VERSION_go-crosssdk-${SDK_ARCH} ?= "${GOVERSION}"

meta/conf/layer.conf

@@ -7,7 +7,7 @@ BBFILE_COLLECTIONS += "core"
 BBFILE_PATTERN_core = "^${LAYERDIR}/"
 BBFILE_PRIORITY_core = "5"
 
-LAYERSERIES_CORENAMES = "sumo thud"
+LAYERSERIES_CORENAMES = "thud"
 
 # This should only be incremented on significant changes that will
 # cause compatibility issues with other layers
@@ -74,6 +74,7 @@ SIGGEN_EXCLUDE_SAFE_RECIPE_DEPS += " \
   gnome-icon-theme->librsvg \
   font-alias->font-util \
   systemd-boot->systemd-bootconf \
+  systemd->systemd-conf \
   weston-init->weston \
   weston-init->kbd \
 "

meta/conf/machine/include/m68k/arch-m68k.inc

@@ -0,0 +1,8 @@
+# m68k Architecture definitions
+
+DEFAULTTUNE ?= "m68k"
+
+# Architecture name
+TUNE_ARCH = "m68k"
+
+TUNE_PKGARCH = "${TUNE_PKGARCH_tune-${DEFAULTTUNE}}"

meta/conf/machine/include/tune-mcf5441x.inc

@@ -0,0 +1,13 @@
+DEFAULTTUNE ?= "mcf5441x"
+
+require conf/machine/include/m68k/arch-m68k.inc
+
+TUNEVALID[mcf5441x] = "Enable coldfire specific processor optimizations"
+TUNE_CCARGS .= "${@bb.utils.contains('TUNE_FEATURES', 'mcf5441x', ' -mcpu=54418', '', d)}"
+
+AVAILTUNES += "mcf5441x"
+TUNE_FEATURES_tune-mcf5441x = "mcf5441x"
+TUNE_ARCH_tune-mcf5441x = "mcf5441x"
+TUNE_PKGARCH_tune-mcf5441x = "mcf5441x"
+PACKAGE_EXTRA_ARCHS_tune-mcf5441x = "m68k mcf5441x"
+

meta/conf/machine/qemux86-64.conf

@@ -31,6 +31,8 @@ MACHINE_FEATURES += "x86 pci"
 
 MACHINE_ESSENTIAL_EXTRA_RDEPENDS += "v86d"
 
+MACHINE_EXTRA_RRECOMMENDS = "kernel-module-snd-ens1370 kernel-module-snd-rawmidi"
+
 KERNEL_MODULE_AUTOLOAD += "uvesafb"
 KERNEL_MODULE_PROBECONF += "uvesafb"
 module_conf_uvesafb = "options uvesafb mode_option=${UVESA_MODE}"

meta/conf/machine/qemux86.conf

@@ -30,6 +30,8 @@ MACHINE_FEATURES += "x86 pci"
 
 MACHINE_ESSENTIAL_EXTRA_RDEPENDS += "v86d"
 
+MACHINE_EXTRA_RRECOMMENDS = "kernel-module-snd-ens1370 kernel-module-snd-rawmidi"
+
 KERNEL_MODULE_AUTOLOAD += "uvesafb"
 KERNEL_MODULE_PROBECONF += "uvesafb"
 module_conf_uvesafb = "options uvesafb mode_option=${UVESA_MODE}"

meta/conf/multilib.conf

@@ -26,3 +26,7 @@ OPKG_ARGS_append = " --force-maintainer --force-overwrite"
 PKG_CONFIG_PATH .= ":${WORKDIR}/recipe-sysroot/${datadir}/pkgconfig"
 PKG_CONFIG_PATH[vardepsexclude] = "datadir WORKDIR"
 PKG_CONFIG_PATH[vardepvalueexclude] = ":${WORKDIR}/recipe-sysroot/${datadir}/pkgconfig"
+
+# These recipes don't need multilib variants, the ${BPN} PROVDES/RPROVDES
+# ${MLPREFIX}${BPN}
+NON_MULTILIB_RECIPES = "grub grub-efi make-mod-scripts"

meta/lib/oe/package_manager.py

@@ -864,8 +864,9 @@ class RpmPM(PackageManager):
             failed_postinsts_abort(list(failed_scriptlets_pkgnames.keys()), self.d.expand("${T}/log.do_${BB_CURRENTTASK}"))
 
     def remove(self, pkgs, with_dependencies = True):
-        if len(pkgs) == 0:
+        if not pkgs:
             return
+
         self._prepare_pkg_transaction()
 
         if with_dependencies:
@@ -1322,7 +1323,11 @@ class OpkgPM(OpkgDpkgPM):
         if not pkgs:
             return
 
-        cmd = "%s %s install %s" % (self.opkg_cmd, self.opkg_args, ' '.join(pkgs))
+        cmd = "%s %s" % (self.opkg_cmd, self.opkg_args)
+        for exclude in (self.d.getVar("PACKAGE_EXCLUDE") or "").split():
+            cmd += " --add-exclude %s" % exclude
+        cmd += " install "
+        cmd += " ".join(pkgs)
 
         os.environ['D'] = self.target_rootfs
         os.environ['OFFLINE_ROOT'] = self.target_rootfs
@@ -1349,6 +1354,9 @@ class OpkgPM(OpkgDpkgPM):
                                               (cmd, e.returncode, e.output.decode("utf-8")))
 
     def remove(self, pkgs, with_dependencies=True):
+        if not pkgs:
+            return
+
         if with_dependencies:
             cmd = "%s %s --force-remove --force-removal-of-dependent-packages remove %s" % \
                 (self.opkg_cmd, self.opkg_args, ' '.join(pkgs))
@@ -1659,6 +1667,9 @@ class DpkgPM(OpkgDpkgPM):
 
 
     def remove(self, pkgs, with_dependencies=True):
+        if not pkgs:
+            return
+
         if with_dependencies:
             os.environ['APT_CONFIG'] = self.apt_conf_file
             cmd = "%s purge %s" % (self.apt_get_cmd, ' '.join(pkgs))

meta/lib/oe/rootfs.py

@@ -144,6 +144,20 @@ class Rootfs(object, metaclass=ABCMeta):
         bb.note("  Install complementary '*-dbg' packages...")
         self.pm.install_complementary('*-dbg')
 
+        if self.d.getVar('PACKAGE_DEBUG_SPLIT_STYLE') == 'debug-with-srcpkg':
+            bb.note("  Install complementary '*-src' packages...")
+            self.pm.install_complementary('*-src')
+
+        """
+        Install additional debug packages. Possibility to install additional packages,
+        which are not automatically installed as complementary package of
+        standard one, e.g. debug package of static libraries.
+        """
+        extra_debug_pkgs = self.d.getVar('IMAGE_INSTALL_DEBUGFS')
+        if extra_debug_pkgs:
+            bb.note("  Install extra debug packages...")
+            self.pm.install(extra_debug_pkgs.split(), True)
+
         bb.note("  Rename debug rootfs...")
         try:
             shutil.rmtree(self.image_rootfs + '-dbg')
@@ -853,9 +867,8 @@ class OpkgRootfs(DpkgOpkgRootfs):
         opkg_pre_process_cmds = self.d.getVar('OPKG_PREPROCESS_COMMANDS')
         opkg_post_process_cmds = self.d.getVar('OPKG_POSTPROCESS_COMMANDS')
 
-        # update PM index files, unless users provide their own feeds
-        if (self.d.getVar('BUILD_IMAGES_FROM_FEEDS') or "") != "1":
-            self.pm.write_index()
+        # update PM index files
+        self.pm.write_index()
 
         execute_pre_post_process(self.d, opkg_pre_process_cmds)
 

meta/lib/oe/utils.py

@@ -347,7 +347,13 @@ def format_pkg_list(pkg_dict, ret_format=None):
         for pkg in sorted(pkg_dict):
             output.append(pkg)
 
-    return '\n'.join(output)
+    output_str = '\n'.join(output)
+
+    if output_str:
+        # make sure last line is newline terminated
+        output_str += '\n'
+
+    return output_str
 
 def host_gcc_version(d, taskcontextonly=False):
     import re, subprocess

meta/lib/oeqa/core/runner.py

@@ -76,40 +76,43 @@ class OETestResult(_TestResult):
         else:
             msg = "%s - FAIL - Required tests failed" % component
         skipped = len(self.skipped)
-        if skipped: 
+        if skipped:
             msg += " (skipped=%d)" % skipped
         self.tc.logger.info(msg)
 
-    def _getDetailsNotPassed(self, case, type, desc):
-        found = False
+    def _getTestResultDetails(self, case):
+        result_types = {'failures': 'FAILED', 'errors': 'ERROR', 'skipped': 'SKIPPED',
+                        'expectedFailures': 'EXPECTEDFAIL', 'successes': 'PASSED'}
 
-        for (scase, msg) in getattr(self, type):
-            if case.id() == scase.id():
-                found = True
-                break
-            scase_str = str(scase.id())
-
-            # When fails at module or class level the class name is passed as string
-            # so figure out to see if match
-            m = re.search("^setUpModule \((?P<module_name>.*)\)$", scase_str)
-            if m:
-                if case.__class__.__module__ == m.group('module_name'):
+        for rtype in result_types:
+            found = False
+            for (scase, msg) in getattr(self, rtype):
+                if case.id() == scase.id():
                     found = True
                     break
+                scase_str = str(scase.id())
 
-            m = re.search("^setUpClass \((?P<class_name>.*)\)$", scase_str)
-            if m:
-                class_name = "%s.%s" % (case.__class__.__module__,
-                        case.__class__.__name__)
+                # When fails at module or class level the class name is passed as string
+                # so figure out to see if match
+                m = re.search("^setUpModule \((?P<module_name>.*)\)$", scase_str)
+                if m:
+                    if case.__class__.__module__ == m.group('module_name'):
+                        found = True
+                        break
 
-                if class_name == m.group('class_name'):
-                    found = True
-                    break
+                m = re.search("^setUpClass \((?P<class_name>.*)\)$", scase_str)
+                if m:
+                    class_name = "%s.%s" % (case.__class__.__module__,
+                                            case.__class__.__name__)
 
-        if found:
-            return (found, msg)
+                    if class_name == m.group('class_name'):
+                        found = True
+                        break
 
-        return (found, None)
+            if found:
+                return result_types[rtype], msg
+
+        return 'UNKNOWN', None
 
     def addSuccess(self, test):
         #Added so we can keep track of successes too
@@ -121,17 +124,7 @@ class OETestResult(_TestResult):
         for case_name in self.tc._registry['cases']:
             case = self.tc._registry['cases'][case_name]
 
-            result_types = ['failures', 'errors', 'skipped', 'expectedFailures', 'successes']
-            result_desc = ['FAILED', 'ERROR', 'SKIPPED', 'EXPECTEDFAIL', 'PASSED']
-
-            fail = False
-            desc = None
-            for idx, name in enumerate(result_types):
-                (fail, msg) = self._getDetailsNotPassed(case, result_types[idx],
-                        result_desc[idx])
-                if fail:
-                    desc = result_desc[idx]
-                    break
+            (status, log) = self._getTestResultDetails(case)
 
             oeid = -1
             if hasattr(case, 'decorators'):
@@ -143,12 +136,7 @@ class OETestResult(_TestResult):
             if case.id() in self.starttime and case.id() in self.endtime:
                 t = " (" + "{0:.2f}".format(self.endtime[case.id()] - self.starttime[case.id()]) + "s)"
 
-            if fail:
-                self.tc.logger.info("RESULTS - %s - Testcase %s: %s%s" % (case.id(),
-                    oeid, desc, t))
-            else:
-                self.tc.logger.info("RESULTS - %s - Testcase %s: %s%s" % (case.id(),
-                    oeid, 'UNKNOWN', t))
+            self.tc.logger.info("RESULTS - %s - Testcase %s: %s%s" % (case.id(), oeid, status, t))
 
 class OEListTestsResult(object):
     def wasSuccessful(self):

meta/lib/oeqa/manual/abat.patch

@@ -0,0 +1,64 @@
+########
+diff --git a/glxgears_check.sh b/glxgears_check.sh
+index 17622b8..c4d3b97 100755
+--- a/glxgears_check.sh
++++ b/glxgears_check.sh
+@@ -31,7 +31,7 @@ else
+ 
+     sleep 6
+ 
+-    XPID=$( ps ax | awk '{print $1, $5}' | grep glxgears | awk '{print $1}')
++    XPID=$( ps | awk '{print $1, $5}' | grep glxgears | awk '{print $1}')
+     if [ ! -z "$XPID" ]; then
+         kill -9 $XPID >/dev/null 2>&1
+         echo "glxgears can run, PASS!"
+diff --git a/x_close.sh b/x_close.sh
+index e287be1..3429f1a 100755
+--- a/x_close.sh
++++ b/x_close.sh
+@@ -22,7 +22,7 @@
+ #
+ function close_proc(){
+ echo "kill process Xorg"
+-XPID=$(  ps ax | awk '{print $1, $5}' | egrep "X$|Xorg$" | awk '{print $1}')
++XPID=$(  ps | awk '{print $1, $6}' | egrep "X$|Xorg$" | awk '{print $1}')
+ if [ ! -z "$XPID" ]; then
+     kill $XPID
+     sleep 4
+diff --git a/x_start.sh b/x_start.sh
+index 9cf6eab..2305796 100755
+--- a/x_start.sh
++++ b/x_start.sh
+@@ -24,7 +24,7 @@
+ X_ERROR=0
+ 
+ #test whether X has started
+-PXID=$(ps ax |awk '{print $1,$5}' |egrep "Xorg$|X$" |grep -v grep | awk '{print $1}')
++PXID=$(ps |awk '{print $1,$6}' |egrep "Xorg$|X$" |grep -v grep | awk '{print $1}')
+ if [ ! -z "$PXID" ]; then
+     echo "[WARNING] Xorg has started!"
+     XORG_STATUS="started"
+@@ -35,9 +35,11 @@ else
+     #start up the x server
+     echo "Start up the X server for test in display $DISPLAY................"
+ 
+-    $XORG_DIR/bin/X >/dev/null 2>&1 &
++    #$XORG_DIR/bin/X >/dev/null 2>&1 &
++    #sleep 8
++    #xterm &
++    /etc/init.d/xserver-nodm start &
+     sleep 8
+-    xterm &
+ fi
+     XLOG_FILE=/var/log/Xorg.0.log
+     [ -f $XORG_DIR/var/log/Xorg.0.log ] && XLOG_FILE=$XORG_DIR/var/log/Xorg.0.log
+@@ -54,7 +56,7 @@ fi
+         X_ERROR=1
+     fi
+ 
+-    XPID=$( ps ax | awk '{print $1, $5}' | egrep "X$|Xorg$" |grep -v grep| awk '{print $1}')
++    XPID=$( ps | awk '{print $1, $6}' | egrep "X$|Xorg$" |grep -v grep| awk '{print $1}')
+     if [ -z "$XPID" ]; then
+         echo "Start up X server FAIL!"
+  echo
+########

meta/lib/oeqa/manual/bsp-qemu.json

@@ -0,0 +1,222 @@
+[
+  {
+    "test": {
+      "@alias": "bsps-qemu.bsps-tools.qemu_can_be_started_with_KVM_enabled",
+      "author": [
+        {
+          "email": "alexandru.c.georgescu@intel.com",
+          "name": "alexandru.c.georgescu@intel.com"
+        }
+      ],
+      "execution": {
+        "1": {
+          "action": "Build a kernel with KVM enabled  \n\nIn Local.conf add  \n\nQEMU_USE_KVM = \"${@ 'intel-corei7-64 intel-core2-32 qemux86 qemux86-64' if os.access('/dev/kvm', os.R_OK|os.W_OK) else '' }\"  \n\n ",
+          "expected_results": ""
+        },
+        "2": {
+          "action": "Start qemu with option \"kvm\" with runqemu \n    a. If you start qemu with kvm failed, maybe it is because host not install kvm and vhost_net module. Follow below link to install them. \n    b. vhost_test refer:  https://wiki.yoctoproject.org/wiki/Running_an_x86_Yocto_Linux_image_under_QEMU_KVM \n    c. kvm refer: https://wiki.yoctoproject.org/wiki/How_to_enable_KVM_for_Poky_qemu",
+          "expected_results": ""
+        },
+        "3": {
+          "action": "Check if qemu starts up and if kvm_intel module is used",
+          "expected_results": ""
+        },
+        "4": {
+          "action": "If kvm_intel module is not used when starting qemu, it will show 0 in \"Used by\" column when you run \"lsmod | grep kvm_intel\" ",
+          "expected_results": "KVM enabled with qemu \nExecute \"lsmod | grep kvm_intel\" from your host twice, before and after you \nstart the qemu with kvm option. Before start, the number should be 0, \nafter start, the number should bigger than 0."
+        }
+      },
+      "summary": "qemu_can_be_started_with_KVM_enabled"
+    }
+  },
+  {
+    "test": {
+      "@alias": "bsps-qemu.bsps-tools.Post-installation_logging",
+      "author": [
+        {
+          "email": "yi.zhao@windriver.com",
+          "name": "yi.zhao@windriver.com"
+        }
+      ],
+      "execution": {
+        "1": {
+          "action": "Download the poky source and set environment \n",
+          "expected_results": "The /var/log/postinstall.log should exist in the first boot. The content of this log is like below:  \n\nRunning postinst /etc/rpm-postinsts/man... \nList directory to check the output log \nbin \nboot \ndev \netc \nhome \nlib \nlost+found \nmedia \nmnt \nproc \nrun \nsbin \nsys \ntmp \nusr \nvar \nList nonexist directory to check the stderr redirection log \nls: /nonexist: No such file or directory "
+        },
+        "2": {
+          "action": "Add the following lines to a .bb file. For expample, meta/recipes-connectivity/openssh/openssh_6.2p2.bb:   \n\npkg_postinst_ontarget_${PN} () {  \n       #!/bin/sh -e  \n       if [ x\"$D\" = \"x\" ]; then  \n       echo \"List directory to check the output log\"  \n       ls /  \n       echo \"List nonexist directory to check the stderr redirection log\"  \n       ls /nonexist  \n       else  \n       exit 1  \n       fi  \n}  \n\nMake sure the feature \"debug-tweaks\" is added in conf/local.conf \n",
+          "expected_results": ""
+        },
+        "3": {
+          "action": "Add ssh-server-openssh to EXTRA_IMAGE_FEATURES in local.conf \n",
+          "expected_results": ""
+        },
+        "4": {
+          "action": "Build core-image-minimal \n",
+          "expected_results": ""
+        },
+        "5": {
+          "action": "Boot up the image and check the /var/log/postinstall.log  ",
+          "expected_results": ""
+        }
+      },
+      "summary": "Post-installation_logging"
+    }
+  },
+  {
+    "test": {
+      "@alias": "bsps-qemu.bsps-tools.Add_user_with_cleartext_type_password_during_filesystem_construction",
+      "author": [
+        {
+          "email": "ke.zou@windriver.com",
+          "name": "ke.zou@windriver.com"
+        }
+      ],
+      "execution": {
+        "1": {
+          "action": "Download the poky source and set the environment  \n\n",
+          "expected_results": "No error during image building procedure. \n"
+        },
+        "2": {
+          "action": "Add the following lines in conf/local.conf  \n\nINHERIT += \"extrausers\"  \n\nEXTRA_USERS_PARAMS = \"\\ \nuseradd -s /bin/sh -P 'tester3' tester3;\\ \n\"  \n\nThe above settings do the following things: \na. Add a user tester3 with cleartext password 'tester3' ",
+          "expected_results": "Image can boot up \n"
+        },
+        "3": {
+          "action": "Build the image\n ",
+          "expected_results": "Login with user name \"tester3\" and password \"tester3\" "
+        }
+      },
+      "summary": "Add_user_with_cleartext_type_password_during_filesystem_construction"
+    }
+  },
+  {
+    "test": {
+      "@alias": "bsps-qemu.bsps-tools.rpm_-__install_dependency_package",
+      "author": [
+        {
+          "email": "alexandru.c.georgescu@intel.com",
+          "name": "alexandru.c.georgescu@intel.com"
+        }
+      ],
+      "execution": {
+        "1": {
+          "action": "Get a not previously installed RPM package or build one on local machine, which should have run-time dependency.For example, \"mc\" (Midnight Commander, which is a visual file manager) should depend on \"ncurses-terminfo\".   \n\n$ bitbake mc  \n\n\n",
+          "expected_results": ""
+        },
+        "2": {
+          "action": "Copy the package into a system folder (for example /home/root/rpm_packages).  \n\n\n",
+          "expected_results": ""
+        },
+        "3": {
+          "action": "Run \"rpm -ivh package_name\" and check the output, for example \"rpm -ivh mc.rpm*\" should report the dependency on \"ncurses-terminfo\".\n\n\n\n",
+          "expected_results": "3 . rpm command should report message when some RPM installation depends on other packages."
+        }
+      },
+      "summary": "rpm_-__install_dependency_package"
+    }
+  },
+  {
+    "test": {
+      "@alias": "bsps-qemu.bsps-tools.Check_rpm_install/removal_log_file_size(auto)",
+      "author": [
+        {
+          "email": "alexandru.c.georgescu@intel.com",
+          "name": "alexandru.c.georgescu@intel.com"
+        }
+      ],
+      "execution": {
+        "1": {
+          "action": "Get some rpm or other kind of installation packages.  \n\n",
+          "expected_results": "Steps 1- 4 (more than 2.3) \nEach file will occupy around 10MB, and there should be some method to keep rpm log in a small size. (the size of the db of RPMs must not be taking so much space)  \nStep 5 (less than or equal to 2.3)\nThe size on /var/lib/rpm/ must keep around 30MB"
+        },
+        "2": {
+          "action": "After system is up, check the size of log file named as \"log.xxxxxx\" on  /var/lib/rpm/log  \n\n",
+          "expected_results": ""
+        },
+        "3": {
+          "action": "After several install/removal of packages, with either of the install/removal commands   (rpm/smart/zypper/dnf install/removal), check again the size of log file.  \n\n",
+          "expected_results": ""
+        },
+        "4": {
+          "action": "For packages installation, there will be some database files under /var/lib/rpm/, named as \"__db.xxx\" and there will be some log files   \nunder /var/lib/rpm/log, named as \"\"log.xxxxxx\"\".   \n\nNote: You will only see the log.xxxx on /var/lib/rpm/log mentioned above if the poky version is minor than 2.3.For poky 2.3 or major versions this has been modified and the package RPM4 does not show the logs.xxxx. if major, follow the next step.  \n\n",
+          "expected_results": ""
+        },
+        "5": {
+          "action": "Repeat steps (1 and 3)  and check the size of /var/lib/rpm/  \n\nMore info: https://bugzilla.yoctoproject.org/show_bug.cgi?id=9259",
+          "expected_results": ""
+        }
+      },
+      "summary": "Check_rpm_install/removal_log_file_size"
+    }
+  },
+  {
+    "test": {
+      "@alias": "bsps-qemu.bsps-runtime.only_one_connmand_in_background(auto)",
+      "author": [
+        {
+          "email": "alexandru.c.georgescu@intel.com",
+          "name": "alexandru.c.georgescu@intel.com"
+        }
+      ],
+      "execution": {
+        "1": {
+          "action": "Boot system",
+          "expected_results": ""
+        },
+        "2": {
+          "action": "Run \"ps aux |grep connmand\" or \"ps -ef | grep connmand\" or \"ps | grep connmand\"",
+          "expected_results": "Connmand (connection manager, used to manage internet connections)  should be shown as an active process \n\n"
+        },
+        "3": {
+          "action": "Run command \"connmand\" to try to launch to a second connmand process",
+          "expected_results": ""
+        },
+        "4": {
+          "action": "Check, with \"ps\" connmand  if a second connmand can be generated ",
+          "expected_results": "There should be only one connmand process instance in background ."
+        }
+      },
+      "summary": "only_one_connmand_in_background"
+    }
+  },
+  {
+    "test": {
+      "@alias": "bsps-qemu.bsps-runtime.X_server_can_start_up_with_runlevel_5_boot",
+      "author": [
+        {
+          "email": "alexandru.c.georgescu@intel.com",
+          "name": "alexandru.c.georgescu@intel.com"
+        }
+      ],
+      "execution": {
+        "1": {
+          "action": "boot up system with default runlevel  \n\n",
+          "expected_results": "X server can start up well and desktop display has no problem .  \n\n"
+        },
+        "2": {
+          "action": "type runlevel at command prompt",
+          "expected_results": "Output:N 5"
+        }
+      },
+      "summary": "X_server_can_start_up_with_runlevel_5_boot"
+    }
+  },
+  {
+    "test": {
+      "@alias": "bsps-qemu.bsps-runtime.check_bash_in_image",
+      "author": [
+        {
+          "email": "alexandru.c.georgescu@intel.com",
+          "name": "alexandru.c.georgescu@intel.com"
+        }
+      ],
+      "execution": {
+        "1": {
+          "action": "After system is up, check if bash command exists with command \"which bash\"",
+          "expected_results": "bash command should exist in image giving something as below  \"/bin/bash\""
+        }
+      },
+      "summary": "check_bash_in_image"
+    }
+  }
+]

meta/lib/oeqa/manual/build-appliance.json

@@ -0,0 +1,122 @@
+[
+    {
+        "test": {
+            "@alias": "build-appliance.build-appliance.Bitbake_build-appliance-image",
+            "author": [
+                {
+                    "email": "alexandru.c.georgescu@intel.com",
+                    "name": "alexandru.c.georgescu@intel.com"
+                }
+            ],
+            "execution": {
+                "1": {
+                    "action": "Get poky source code and prepare the build environment",
+                    "expected_results": "bitbake build-appliance-image is successful  "
+                },
+                "2": {
+                    "action": "Set MACHINE to qemux86 and add the following line to conf/local.conf :  SRCREV_pn-build-appliance-image = \"${AUTOREV}\"",
+                    "expected_results": ""
+                },
+                "3": {
+                    "action": "Run \"bitbake build-appliance-image\" \n \n",
+                    "expected_results": ""
+                }
+            },
+            "summary": "Bitbake_build-appliance-image"
+        }
+    },
+    {
+        "test": {
+            "@alias": "build-appliance.build-appliance.Build_core-image-minimal_with_build-appliance-image",
+            "author": [
+                {
+                    "email": "corneliux.stoicescu@intel.com",
+                    "name": "corneliux.stoicescu@intel.com"
+                }
+            ],
+            "execution": {
+                "1": {
+                    "action": "Build with AUTOREV or download from Autobuilder an image for Yocto Build Appliance.  ",
+                    "expected_results": ""
+                },
+                "2": {
+                    "action": "Boot the image under VMWare Player.  ",
+                    "expected_results": ""
+                },
+                "3": {
+                    "action": "Build qemux86 core-image-minimal using bitbake command line in the build-appliance-image  ",
+                    "expected_results": ""
+                },
+                "4": {
+                    "action": "Launch the image just built using runqemu.  ",
+                    "expected_results": "core-image-minimal should build and boot. "
+                }
+            },
+            "summary": "Build_core-image-minimal_with_build-appliance-image"
+        }
+    },
+    {
+        "test": {
+            "@alias": "build-appliance.build-appliance.Build_a_image_without_error_(added_recipe)",
+            "author": [
+                {
+                    "email": "sstncr@gmail.com",
+                    "name": "sstncr@gmail.com"
+                }
+            ],
+            "execution": {
+                "1": {
+                    "action": "Launch Build Appliance",
+                    "expected_results": "User could build a image without error and the added package is in the image"
+                },
+                "2": {
+                    "action": "Set \"Machine\" in conf/local.conf, for example, qemuarm",
+                    "expected_results": ""
+                },
+                "3": {
+                    "action": "Install a new package to the image, for example, acpid. Set the following line in conf/local.conf: IMAGE_INSTALL_append = \" acpid\"",
+                    "expected_results": ""
+                },
+                "4": {
+                    "action": "Build a image using bitbake command line, for example, bitbake core-image-minimal",
+                    "expected_results": ""
+                },
+                "5": {
+                    "action": "After build finished, launch the image and check if the added package built into image",
+                    "expected_results": ""
+                }
+            },
+            "summary": "Build_a_image_without_error_(added_recipe)."
+        }
+    },
+    {
+        "test": {
+            "@alias": "build-appliance.build-appliance.Create_core-image-sato-sdk_using_build_appliance",
+            "author": [
+                {
+                    "email": "ee.peng.yeoh@intel.com",
+                    "name": "ee.peng.yeoh@intel.com"
+                }
+            ],
+            "execution": {
+                "1": {
+                    "action": "Build with AUTOREV or download from Autobuilder an image for Yocto Build Appliance.  ",
+                    "expected_results": ""
+                },
+                "2": {
+                    "action": "Boot the image under VMWare Player.  ",
+                    "expected_results": ""
+                },
+                "3": {
+                    "action": "Build qemux86 core-image-sato-sdk using bitbake command line in the build-appliance-image  ",
+                    "expected_results": ""
+                },
+                "4": {
+                    "action": "Launch the image just built using runqemu. ",
+                    "expected_results": ""
+                }
+            },
+            "summary": "Create_core-image-sato-sdk_using_build_appliance"
+        }
+    }
+]

meta/lib/oeqa/manual/crops.json

@@ -0,0 +1,294 @@
+[
+    {
+        "test": {
+            "@alias": "crops-default.crops-default.sdkext_eSDK_devtool_build_make",
+            "author": [
+                {
+                    "email": "francisco.j.pedraza.gonzalez@intel.com",
+                    "name": "francisco.j.pedraza.gonzalez@intel.com"
+                }
+            ],
+            "execution": {
+                "1": {
+                    "action": "IMPORTANT NOTE: The firsts 5 steps refer to configuration of the environment to run the rest of the steps. These only apply for CROPS-eSDK.    \n\n\n\n",
+                    "expected_results": ""
+                },
+                "2": {
+                    "action": " Initiate your Crops-esdk environment as it says in wiki https://github.com/crops/docker-win-mac-docs/wiki   \n\n",
+                    "expected_results": ""
+                },
+                "3": {
+                    "action": "Create the following tree of files  <crops-esdk-workdir-workspace>/sdkext/files/myapp <crops-esdk-workdir-workspace>/sdkext/files/myapp_cmake   \n\n\n",
+                    "expected_results": ""
+                },
+                "4": {
+                    "action": " Create the following files withing the myapp directory  myapp.c and the Makefile. Write the following inside of each file: \n---------------------------------------- \nMakefile should contain  \n\nall: myapp  \n\nmyapp: myapp.o \n\t$(CC) $(LDFLAGS) $< -o $@  \n\nmyapp.o: myapp.c  \n\t$(CC) $(CFLAGS) -c $< -o $@ \n\nclean: \n\trm -rf myapp.o myapp  \n\n-----------------------------  \nmyapp.c  shold contain  \n\n\n#include <stdio.h>    \n\nint \nmain(int argc, char *argv[]) \n{ \n\tprintf(\"Hello world\\n\"); \n \n\treturn 0; \n} \n------------------------------------  \n\n",
+                    "expected_results": "be sure that the indentations on the makefile are tabs not spaces.  \n\n"
+                },
+                "5": {
+                    "action": " Create the following files within the myapp_cmake directory CMakeLists.txt and myapp.c. Write the following inside each file:  \n\n------------------------------------ \nCMakeLists.txt should contain:  \n\ncmake_minimum_required (VERSION 2.6) \nproject (myapp) \n# The version number. \nset (myapp_VERSION_MAJOR 1) \nset (myapp_VERSION_MINOR 0)  \n\n# add the executable \nadd_executable (myapp myapp.c)  \n\ninstall(TARGETS myapp \nRUNTIME DESTINATION bin)  \n\n------------------------------------------ \nmyapp.c should contain:  \n\n#include <stdio.h>     \n\nint  \nmain(int argc, char *argv[])  \n{ \n\tprintf(\"Hello world\\n\");  \n\n\treturn 0; \n} \n-------------------------------------------------  \n\n",
+                    "expected_results": "Be sure that the indentations on CMakeLists.txt is tabs not spaces."
+                },
+                "6": {
+                    "action": " source environment-setup-i586-poky-linux  \n\n",
+                    "expected_results": "This should output a message that says SDK environment now set up; additionally you may now run devtool to perform development tasks etc etc ...   \n\n"
+                },
+                "7": {
+                    "action": " run command which devtool  \n\n",
+                    "expected_results": "this should output the directory of the devtool script and it should be within the sdk workdir you are working in.  \n\n  "
+                },
+                "8": {
+                    "action": "devtool add myapp <directory>(this is myapp dir)    \n\n\n",
+                    "expected_results": "The directory you should input is the myapp directory. This should automatically create the recipe myapp.bb under <crops-esdk-workdir-workspace>/recipes/myapp/myapp.bb"
+                },
+                "9": {
+                    "action": " devtool build myapp  \n\n",
+                    "expected_results": "This should compile an image"
+                },
+                "10": {
+                    "action": " devtool reset myapp  ",
+                    "expected_results": "This cleans sysroot of the myapp recipe, but it leaves the source tree intact. meaning it does not erase."
+                }
+            },
+            "summary": "sdkext_eSDK_devtool_build_make"
+        }
+    },
+    {
+        "test": {
+            "@alias": "crops-default.crops-default.sdkext_devtool_build_esdk_package",
+            "author": [
+                {
+                    "email": "francisco.j.pedraza.gonzalez@intel.com",
+                    "name": "francisco.j.pedraza.gonzalez@intel.com"
+                }
+            ],
+            "execution": {
+                "1": {
+                    "action": "IMPORTANT NOTE: The firsts 5 steps refer to configuration of the environment to run the rest of the steps. These only apply for CROPS-eSDK.    \n\n\n\n",
+                    "expected_results": ""
+                },
+                "2": {
+                    "action": " Initiate your Crops-esdk environment as it says in wiki https://github.com/crops/docker-win-mac-docs/wiki   \n\n",
+                    "expected_results": ""
+                },
+                "3": {
+                    "action": " Create the following tree of files    <crops-esdk-workdir-workspace>/sdkext/files/myapp/ \n <crops-esdk-workdir-workspace>/sdkext/files/myapp_cmake  \n\n",
+                    "expected_results": ""
+                },
+                "4": {
+                    "action": " Create the following files withing the myapp directory  myapp.c and the Makefile. Write the following inside of each file: \n---------------------------------------- \nMakefile should contain  \n\nall: myapp  \n\nmyapp: myapp.o \n\t$(CC) $(LDFLAGS) $< -o $@ \n\nmyapp.o: myapp.c  \n\t$(CC) $(CFLAGS) -c $< -o $@ \n\nclean: \n\trm -rf myapp.o myapp  \n\n-----------------------------  \nmyapp.c  shold contain  \n\n#include <stdio.h>   \n\nint \nmain(int argc, char *argv[]) \n{ \n\tprintf(\"Hello world\\n\"); \n \n\treturn 0; \n} \n------------------------------------  \n\n",
+                    "expected_results": "be sure that the indentations on the makefile are tabs not spaces.  \n\n"
+                },
+                "5": {
+                    "action": " Create the following files within the myapp_cmake directory CMakeLists.txt and myapp.c. Write the following inside each file:  \n\n------------------------------------ \nCMakeLists.txt should contain:  \n\ncmake_minimum_required (VERSION 2.6) \nproject (myapp) \n# The version number. \nset (myapp_VERSION_MAJOR 1) \nset (myapp_VERSION_MINOR 0)  \n\n# add the executable \nadd_executable (myapp myapp.c)  \n\ninstall(TARGETS myapp \nRUNTIME DESTINATION bin)  \n\n------------------------------------------ \nmyapp.c should contain:  \n\n#include<stdio.h>     \n\nint  \nmain(int argc, char *argv[])  \n{ \n\tprintf(\"Hello world\\n\");  \n\n\treturn 0; \n} \n-------------------------------------------------  \n\n",
+                    "expected_results": "Be sure that the indentations on CMakeLists.txt is tabs not spaces.  \n\n"
+                },
+                "6": {
+                    "action": " source environment-setup-i586-poky-linux  \n\n",
+                    "expected_results": "This should output a message that says SDK environment now set up; additionally you may now run devtool to perform development tasks etc etc ...  \n\n"
+                },
+                "7": {
+                    "action": " run command which devtool  \n\n",
+                    "expected_results": " this should output the directory of the devtool script and it should be within the sdk workdir you are working in.  \n\n"
+                },
+                "8": {
+                    "action": " devtool add myapp <directory>  (this is myapp dir)  \n\n",
+                    "expected_results": " The directory you should input is the myapp directory. This should automatically create the recipe myapp.bb under <crops-esdk-workdir-workspace>/recipes/myapp/myapp.bb  \n\n"
+                },
+                "9": {
+                    "action": " devtool package myapp  \n\n",
+                    "expected_results": " you should expect a package creation of myapp and it should be under the  /tmp/deploy/  \n\n"
+                },
+                "10": {
+                    "action": " devtool reset myapp  ",
+                    "expected_results": "This cleans sysroot of the myapp recipe, but it leaves the source tree intact. meaning it does not erase.\n</package_format>"
+                }
+            },
+            "summary": "sdkext_devtool_build_esdk_package"
+        }
+    },
+    {
+        "test": {
+            "@alias": "crops-default.crops-default.sdkext_devtool_build_cmake",
+            "author": [
+                {
+                    "email": "francisco.j.pedraza.gonzalez@intel.com",
+                    "name": "francisco.j.pedraza.gonzalez@intel.com"
+                }
+            ],
+            "execution": {
+                "1": {
+                    "action": "IMPORTANT NOTE: The firsts 5 steps refer to configuration of the environment to run the rest of the steps. These only apply for CROPS-eSDK.    \n\n\n\n",
+                    "expected_results": ""
+                },
+                "2": {
+                    "action": " Initiate your Crops-esdk environment as it says in wiki https://github.com/crops/docker-win-mac-docs/wiki   \n\n",
+                    "expected_results": ""
+                },
+                "3": {
+                    "action": " Create the following tree of files    <crops-esdk-workdir-workspace>/sdkext/files/myapp \n <crops-esdk-workdir-workspace>/sdkext/files/myapp_cmake  \n\n",
+                    "expected_results": ""
+                },
+                "4": {
+                    "action": " Create the following files withing the myapp directory  myapp.c and the Makefile. Write the following inside of each file: \n---------------------------------------- \nMakefile should contain  \n\nall: myapp  \n\nmyapp: myapp.o \n\t$(CC) $(LDFLAGS) $< -o $@ \n\nmyapp.o: myapp.c  \n\t$(CC) $(CFLAGS) -c $< -o $@ \n\nclean: \n\trm -rf myapp.o myapp  \n\n-----------------------------  \nmyapp.c  shold contain  \n\n#include <stdio.h>   \n\nint \nmain(int argc, char *argv[]) \n{ \n\tprintf(\"Hello world\\n\"); \n \n\treturn 0; \n} \n------------------------------------  \n\n",
+                    "expected_results": "be sure that the indentations on the makefile are tabs not spaces.   \n\n"
+                },
+                "5": {
+                    "action": " Create the following files within the myapp_cmake directory CMakeLists.txt and myapp.c. Write the following inside each file:  \n\n------------------------------------ \nCMakeLists.txt should contain:  \n\ncmake_minimum_required (VERSION 2.6) \nproject (myapp) \n# The version number. \nset (myapp_VERSION_MAJOR 1) \nset (myapp_VERSION_MINOR 0)  \n\n# add the executable \nadd_executable (myapp myapp.c)  \n\ninstall(TARGETS myapp \nRUNTIME DESTINATION bin)  \n\n------------------------------------------ \nmyapp.c should contain:  \n\n#include    \n\nint  \nmain(int argc, char *argv[])  \n{ \n\tprintf(\"Hello world\\n\");  \n\n\treturn 0; \n} \n-------------------------------------------------  \n\n",
+                    "expected_results": "Be sure that the indentations on CMakeLists.txt is tabs not spaces.  \n\n"
+                },
+                "6": {
+                    "action": " source environment-setup-i586-poky-linux  \n\n",
+                    "expected_results": "This should output a message that says SDK environment now set up; additionally you may now run devtool to perform development tasks etc etc ...  \n\n"
+                },
+                "7": {
+                    "action": " run command which devtool  \n\n",
+                    "expected_results": "this should output the directory of the devtool script and it should be within the sdk workdir you are working in.  \n\n"
+                },
+                "8": {
+                    "action": " devtool add myapp <directory>   (this is myapp_cmake dir)  \n\n",
+                    "expected_results": "The directory you should input is the myapp_cmake directory. This should automatically create the recipe myapp.bb under <crops-esdk-workdir-workspace>/recipes/myapp/myapp.bb  \n\n"
+                },
+                "9": {
+                    "action": " devtool build myapp  \n\n",
+                    "expected_results": "This should compile an image  \n\n"
+                },
+                "10": {
+                    "action": " devtool reset myapp  ",
+                    "expected_results": "This cleans sysroot of the myapp recipe, but it leaves the source tree intact. meaning it does not erase.  "
+                }
+            },
+            "summary": "sdkext_devtool_build_cmake"
+        }
+    },
+    {
+        "test": {
+            "@alias": "crops-default.crops-default.sdkext_extend_autotools_recipe_creation",
+            "author": [
+                {
+                    "email": "francisco.j.pedraza.gonzalez@intel.com",
+                    "name": "francisco.j.pedraza.gonzalez@intel.com"
+                }
+            ],
+            "execution": {
+                "1": {
+                    "action": "IMPORTANT NOTE: The firsts 2 steps refer to configuration of the environment to run the rest of the steps. These only apply for CROPS-eSDK.    \n\n\n\n",
+                    "expected_results": ""
+                },
+                "2": {
+                    "action": "Initiate your Crops-esdk environment as it says in wiki https://github.com/crops/docker-win-mac-docs/wiki   \n\n",
+                    "expected_results": ""
+                },
+                "3": {
+                    "action": " source environment-setup-i586-poky-linux  \n\n",
+                    "expected_results": " This should output a message that says SDK environment now set up; additionally you may now run devtool to perform development tasks etc etc ...   \n\n"
+                },
+                "4": {
+                    "action": "run command which devtool  \n\n",
+                    "expected_results": "this should output the directory of the devtool script and it should be within the sdk workdir you are working in.   \n\n"
+                },
+                "5": {
+                    "action": "devtool sdk-install -s libxml2  \n\n",
+                    "expected_results": "this should install libxml2   \n\n"
+                },
+                "6": {
+                    "action": "devtool add librdfa  https://github.com/rdfa/librdfa  \n\n",
+                    "expected_results": "This should automatically create the recipe librdfa.bb under /recipes/librdfa/librdfa.bb   \n\n"
+                },
+                "7": {
+                    "action": "devtool build librdfa  \n\n",
+                    "expected_results": "This should compile   \n\n"
+                },
+                "8": {
+                    "action": "devtool reset librdfa  ",
+                    "expected_results": "This cleans sysroot of the librdfa recipe, but it leaves the source tree intact. meaning it does not erase."
+                }
+            },
+            "summary": "sdkext_extend_autotools_recipe_creation"
+        }
+    },
+    {
+        "test": {
+            "@alias": "crops-default.crops-default.sdkext_devtool_kernelmodule",
+            "author": [
+                {
+                    "email": "francisco.j.pedraza.gonzalez@intel.com",
+                    "name": "francisco.j.pedraza.gonzalez@intel.com"
+                }
+            ],
+            "execution": {
+                "1": {
+                    "action": "IMPORTANT NOTE: The firsts 2 steps refer to configuration of the environment to run the rest of the steps. These only apply for CROPS-eSDK.    \n\n\n",
+                    "expected_results": ""
+                },
+                "2": {
+                    "action": " Initiate your Crops-esdk environment as it says in wiki https://github.com/crops/docker-win-mac-docs/wiki   \n\n",
+                    "expected_results": ""
+                },
+                "3": {
+                    "action": "source environment-setup-i586-poky-linux   \n\n",
+                    "expected_results": "This should output a message that says SDK environment now set up; additionally you may now run devtool to perform development tasks etc etc ...   \n \n"
+                },
+                "4": {
+                    "action": "run command which devtool  \n\n",
+                    "expected_results": "this should output the directory of the devtool script and it should be within the sdk workdir you are working in.   \n\n"
+                },
+                "5": {
+                    "action": "devtool add v4l2loopback-driver  https://github.com/umlaeute/v4l2loopback.git   \n\n",
+                    "expected_results": "This should automatically create the recipe v4l2loopback-driver.bb under <crops-esdk-workdir-workspace>/recipes/v4l2loopback-driver/v4l2loopback-driver.bb "
+                },
+                "6": {
+                    "action": "devtool build v4l2loopback-driver   \n\n",
+                    "expected_results": "This should compile an image   \n\n"
+                },
+                "7": {
+                    "action": "devtool reset v4l2loopback-driver   ",
+                    "expected_results": "This cleans sysroot of the v4l2loopback-driver recipe, but it leaves the source tree intact. meaning it does not erase."
+                }
+            },
+            "summary": "sdkext_devtool_kernelmodule"
+        }
+    },
+    {
+        "test": {
+            "@alias": "crops-default.crops-default.sdkext_recipes_for_nodejs",
+            "author": [
+                {
+                    "email": "francisco.j.pedraza.gonzalez@intel.com",
+                    "name": "francisco.j.pedraza.gonzalez@intel.com"
+                }
+            ],
+            "execution": {
+                "1": {
+                    "action": "IMPORTANT NOTE: The firsts 2 steps refer to configuration of the environment to run the rest of the steps. These only apply for CROPS-eSDK.    \n\n\nlets say variable npm = npm://registry.npmjs.org;name=winston;version=2.2.0   \n\n",
+                    "expected_results": ""
+                },
+                "2": {
+                    "action": "Initiate your Crops-esdk environment as it says in wiki https://github.com/crops/docker-win-mac-docs/wiki   \n\n",
+                    "expected_results": ""
+                },
+                "3": {
+                    "action": "source environment-setup-i586-poky-linux   \n\n",
+                    "expected_results": "This should output a message that says SDK environment now set up; additionally you may now run devtool to perform development tasks etc etc ...   \n\n"
+                },
+                "4": {
+                    "action": "run command which devtool   \n\n",
+                    "expected_results": "this should output the directory of the devtool script and it should be within the sdk workdir you are working in.  \n\n"
+                },
+                "5": {
+                    "action": " 4a) git clone git://git.openembedded.org/meta-openembedded in  layers/build  directory   \n \n4b) Add meta-openembedded/meta-oe in bblayer.conf as mentioned below: ${SDKBASEMETAPATH}/layers/build/meta-openembedded/meta-oe \\   \n\n4c) devtool add \"npm://registry.npmjs.org;name=npm;version=2.2.0\"   \n\n",
+                    "expected_results": " This should automatically create the recipe npm.bb under /recipes/npm/npm.bb  \n\n"
+                },
+                "6": {
+                    "action": "devtool build npm   \n\n",
+                    "expected_results": "This should compile an image  \n\n"
+                },
+                "7": {
+                    "action": " devtool reset npm",
+                    "expected_results": "This cleans sysroot of the npm recipe, but it leaves the source tree intact. meaning it does not erase."
+                }
+            },
+            "summary": "sdkext_recipes_for_nodejs"
+        }
+    }
+]

meta/lib/oeqa/manual/eclipse-plugin.json

@@ -0,0 +1,322 @@
+[
+    {
+        "test": {
+            "@alias": "eclipse-plugin.eclipse-plugin.support_SSH_connection_to_Target",
+            "author": [
+                {
+                    "email": "ee.peng.yeoh@intel.com",
+                    "name": "ee.peng.yeoh@intel.com"
+                }
+            ],
+            "execution": {
+                "1": {
+                    "action": "In Eclipse, swich to Remote System Explorer to create a connention baseed on SSH, input the remote target IP address as the Host name, make sure disable the proxy in Window->Preferences->General->Network Connection, set Direct as Active Provider field.  ",
+                    "expected_results": "the connection based on SSH could be set up."
+                },
+                "2": {
+                    "action": "Configure connection from Eclipse: Run->Run Configurations->C/C++ Remote Application\\ ->New Connection->General->SSH Only  ",
+                    "expected_results": ""
+                },
+                "3": {
+                    "action": "Then right click to connect, input the user ID and password.  ",
+                    "expected_results": ""
+                },
+                "4": {
+                    "action": "expand the connection, it will show the Sftp Files etc.   \nNOTE. Might need to change dropbear to openssh and add the packagegroup-core-eclipse-debug recipe",
+                    "expected_results": ""
+                }
+            },
+            "summary": "support_SSH_connection_to_Target"
+        }
+    },
+    {
+        "test": {
+            "@alias": "eclipse-plugin.eclipse-plugin.Launch_QEMU_from_Eclipse",
+            "author": [
+                {
+                    "email": "ee.peng.yeoh@intel.com",
+                    "name": "ee.peng.yeoh@intel.com"
+                }
+            ],
+            "execution": {
+                "1": {
+                    "action": "Set the Yocto ADT's toolchain root location, sysroot location and kernel, in the menu Window -> Preferences -> Yocto ADT.   \n \n",
+                    "expected_results": ""
+                },
+                "2": {
+                    "action": "wget  autobuilder.yoctoproject.org/pub/releases//machines/qemu/qemux86/qemu (ex:core-image-sato-sdk-qemux86-date-rootfs-tar-bz2) \nsource /opt/poky/version/environment-setup-i585-poky-linux  \n\nExtract qemu with runqemu-extract-sdk /home/user/file(ex.core-image-sato-sdk-qemux86.bz2) \n/home/user/qemux86-sato-sdk  \n\n",
+                    "expected_results": " Qemu can be lauched normally."
+                },
+                "3": {
+                    "action": "(a)Point to the Toolchain:  \n \nIf you are using a stand-alone pre-built toolchain, you should be pointing to the /opt/poky/{test-version} directory as Toolchain Root Location. This is the default location for toolchains installed by the ADT Installer or by hand. If ADT is installed in other location, use that location as Toolchain location.\nIf you are using a system-derived toolchain, the path you provide for the Toolchain Root Location field is the Yocto Project's build directory.  \n \n           E.g:/home/user/yocto/poky/build \n",
+                    "expected_results": ""
+                },
+                "4": {
+                    "action": "(b)Specify the Sysroot Location: \nSysroot Location is the location where the root filesystem for the target hardware is created on the development system by the ADT Installer (SYSROOT in step 2 of the case ADT installer Installation).  \n   \n         Local :     e.g: /home/user/qemux86-sato-sdk \nUsing ADT : e.g :/home/user/test-yocto/qemux86  \n\n",
+                    "expected_results": ""
+                },
+                "5": {
+                    "action": "(c)Select the Target Architecture:  \n \nThe target architecture is the type of hardware you are going to use or emulate. Use the pull-down Target Architecture menu to make your selection.  \n \n\n",
+                    "expected_results": ""
+                },
+                "6": {
+                    "action": "(d) QEMU: \nSelect this option if you will be using the QEMU emulator. Specify the Kernel matching the QEMU architecture you are using. \n      wget  autobuilder.yoctoproject.org/pub/releases//machines/qemu/qemux86/bzImage-qemux86.bin \n      e.g: /home/$USER/yocto/adt-installer/download_image/bzImage-qemux86.bin  \n\n",
+                    "expected_results": ""
+                },	
+                "7": {
+                    "action": "(e) select OK to save the settings.  \n\n\n1: In the Eclipse toolbar, expose the Run -> External Tools menu. Your image should appear as a selectable menu item.  \n2: Select your image in the navigation pane to launch the emulator in a new window. \n3: If needed, enter your host root password in the shell window at the prompt. This sets up a Tap 0 connection needed for running in user-space NFS mode. \n",
+                    "expected_results": ""
+                }				
+            },
+            "summary": "Launch_QEMU_from_Eclipse"
+        }
+    },
+    {
+        "test": {
+            "@alias": "eclipse-plugin.eclipse-plugin.Relocatable_SDK_-_C_-_Build_Hello_World_ANSI_C_Autotools_Project",
+            "author": [
+                {
+                    "email": "ee.peng.yeoh@intel.com",
+                    "name": "ee.peng.yeoh@intel.com"
+                }
+            ],
+            "execution": {
+                "1": {
+                    "action": "Launch a QEMU of target enviroment.(Reference to case \"ADT - Launch qemu by eclipse\") ",
+                    "expected_results": ""
+                },
+                "2": {
+                    "action": "Select File -> New -> Project.",
+                    "expected_results": ""
+                },
+                "3": {
+                    "action": "Double click C/C++.",
+                    "expected_results": ""
+                },
+                "4": {
+                    "action": "Click C or C++ Project to create the project.",
+                    "expected_results": ""
+                },
+                "5": {
+                    "action": "Expand Yocto ADT Project.",
+                    "expected_results": ""
+                },
+                "6": {
+                    "action": "Select Hello World ANSI C Autotools Project.",
+                    "expected_results": ""
+                },
+                "7": {
+                    "action": "Put a name in the Project name. Do not use hyphens as part of the name. \n \n",
+                    "expected_results": ""
+                },
+                "8": {
+                    "action": "Click Next.",
+                    "expected_results": ""
+                },
+                "9": {
+                    "action": "Add information in the Author and Copyright notice fields. \n1",
+                    "expected_results": ""
+                },
+                "10": {
+                    "action": "Click Finish. \n1",
+                    "expected_results": ""
+                },
+                "11": {
+                    "action": "If the \"open perspective\" prompt appears, click \"Yes\" so that you open the C/C++ perspective. \n1",
+                    "expected_results": ""
+                },
+                "12": {
+                    "action": "In the Project Explorer window, right click the project -> Reconfigure project. \n1",
+                    "expected_results": ""
+                },
+                "13": {
+                    "action": "In the Project Explorer window, right click the project -> Build project. \n1",
+                    "expected_results": "Under the Project files, a new folder appears called Binaries. This indicates that the compilation have been successful and the project binary have been created. \n"
+                },
+                "14": {
+                    "action": "Right click it again and Run as -> Run Configurations.  \n\t\t\tUnder Run Configurations expand \"C/C++ Remote Application\". A configuration for the current project should appear. Clicking it will display the configuration settings. \n\t\t\tin \"C/C++ Application\" field input Remote Absolute File path for C/C++ Application. e.g.: /home/root/myapplication \n\t\t\tIn \"Connection\" drop-down list make sure a TCF connection is set up for your target. If not, create a new one by clicking the New button. \n1",
+                    "expected_results": "step 14 to step 16 -> Build succeed and the console outputs Hello world, you can also check the output on target."
+                },
+                "15": {
+                    "action": "After all settings are done, select the Run button on the bottom right corner \n\n1",
+                    "expected_results": ""
+                },
+                "16": {
+                    "action": "Repeat the steps 14-15, but instead of using Run Configurations use Debug Configurations: \nRight click it again and Debug as -> Debug Configurations \nUnder Debug Configurations expand \"C/C++ Remote Application\". A configuration for the current project should appear. Clicking it will display the configuration settings. \nin \"C/C++ Application\" field input Remote Absolute File path for C/C++ Application.\ne.g.: /home/root/myapplication \nIn \"Connection\" drop-down list make sure a TCF connection is set up for your target. If not, create a new one by clicking the New button \n1",
+                    "expected_results": ""
+                },
+                "17": {
+                    "action": "After all settings are done, select the Debug button on the bottom right corner",
+                    "expected_results": ""
+                }
+            },
+            "summary": "Relocatable_SDK_-_C_-_Build_Hello_World_ANSI_C_Autotools_Project"
+        }
+    },
+    {
+        "test": {
+            "@alias": "eclipse-plugin.eclipse-plugin.Relocatable_SDK_-_C++_-_Build_Hello_World_C++_Autotools_project",
+            "author": [
+                {
+                    "email": "ee.peng.yeoh@intel.com",
+                    "name": "ee.peng.yeoh@intel.com"
+                }
+            ],
+            "execution": {
+                "1": {
+                    "action": "Launch a QEMU of target enviroment.(Reference to case \"ADT - Launch qemu by eclipse\") ",
+                    "expected_results": ""
+                },
+                "2": {
+                    "action": "Select File -> New -> Project. ",
+                    "expected_results": ""
+                },
+                "3": {
+                    "action": "Double click C/C++. ",
+                    "expected_results": ""
+                },
+                "4": {
+                    "action": "Click C or C++ Project to create the project. ",
+                    "expected_results": ""
+                },
+                "5": {
+                    "action": "Expand Yocto ADT Project. ",
+                    "expected_results": ""
+                },
+                "6": {
+                    "action": "Select Hello World ANSI C++ Autotools Project. ",
+                    "expected_results": ""
+                },
+                "7": {
+                    "action": "Put a name in the Project name. Do not use hyphens as part of the name.  \n \n",
+                    "expected_results": ""
+                },
+                "8": {
+                    "action": "Click Next.",
+                    "expected_results": ""
+                },
+                "9": {
+                    "action": "Add information in the Author and Copyright notice fields.",
+                    "expected_results": ""
+                },
+                "10": {
+                    "action": "Click Finish. \n1",
+                    "expected_results": ""
+                },
+                "11": {
+                    "action": "If the \"open perspective\" prompt appears, click \"Yes\" so that you open the C/C++ perspective. \n1",
+                    "expected_results": ""
+                },
+                "12": {
+                    "action": "In the Project Explorer window, right click the project -> Reconfigure project. \n1",
+                    "expected_results": ""
+                },
+                "13": {
+                    "action": "In the Project Explorer window, right click the project -> Build project. \n\n1",
+                    "expected_results": "under the Project files, a new folder appears called Binaries. This indicates that the compilation have been successful and the project binary have been created.  \n"
+                },
+                "14": {
+                    "action": "Right click it again and Run as -> Run Configurations. \n\t\t\tUnder Run Configurations expand \"C/C++ Remote Application\". A configuration for the current project should appear. Clicking it will display the configuration settings. \n\t\t\tin \"C/C++ Application\" field input Remote Absolute File path for C/C++ Application. e.g.: /home/root/myapplication \n\t\t\tIn \"Connection\" drop-down list make sure a TCF connection is set up for your target. If not, create a new one by clicking the New button. \n1",
+                    "expected_results": "step 14 to step 16 -> Build succeed and the console outputs Hello world, you can also check the output on target."
+                },
+                "15": {
+                    "action": "After all settings are done, select the Run button on the bottom right corner \n\n1",
+                    "expected_results": ""
+                },
+                "16": {
+                    "action": "Repeat the steps 14-15, but instead of using Run Configurations use Debug Configurations: \n\t\tRight click it again and Debug as -> Debug Configurations \n\t\tUnder Debug Configurations expand \"C/C++ Remote Application\". A configuration for the current project should appear. Clicking it will display the configuration settings. \n\t\tin \"C/C++ Application\" field input Remote Absolute File path for C/C++ Application. \n\t\te.g.: /home/root/myapplication \n\t\tIn \"Connection\" drop-down list make sure a TCF connection is set up for your target. If not, create a new one by clicking the New button \n1",
+                    "expected_results": ""
+                },
+                "17": {
+                    "action": "After all settings are done, select the Debug button on the bottom right corner",
+                    "expected_results": ""
+                }
+            },
+            "summary": "Relocatable_SDK_-_C++_-_Build_Hello_World_C++_Autotools_project"
+        }
+    },
+    {
+        "test": {
+            "@alias": "eclipse-plugin.eclipse-plugin.Build_Eclipse_Plugin_from_source",
+            "author": [
+                {
+                    "email": "laurentiu.serban@intel.com",
+                    "name": "laurentiu.serban@intel.com"
+                }
+            ],
+            "execution": {
+                "1": {
+                    "action": "Clone eclipse-poky source.   \n    \n    - git clone git://git.yoctoproject.org/eclipse-poky  \n\n",
+                    "expected_results": "Eclipse plugin is successfully installed  \n\nDocumentation is there. For example if you have release yocto-2.0.1 you will found on   http://autobuilder.yoctoproject.org/pub/releases/yocto-2.0.1/eclipse-plugin/mars/  archive with documentation like org.yocto.doc-development-$date.zip  \n  \n"
+                },
+                "2": {
+                    "action": "Checkout correct tag.  \n\n    - git checkout <eclipse-version>/<yocto-version> \n\n",
+                    "expected_results": "After plugin is build you must have  4 archive in foder scripts from eclipse-poky:  \n     - org.yocto.bc - mars-master-$date.zip  \n     - org.yocto.doc - mars-master-$date.zip    --> documentation  \n     - org.yocto.sdk - mars-master-$date.zip       \n     - org.yocto.sdk - mars-master-$date.-archive.zip  --> plugin  "
+                },
+                "3": {
+                    "action": "Move to scripts/ folder.  \n\n",
+                    "expected_results": ""
+                },
+                "4": {
+                    "action": "Run ./setup.sh  \n\n",
+                    "expected_results": ""
+                },
+                "5": {
+                    "action": "When the script finishes, it prompts a command to issue to build the plugin. It should look similar to the following:  \n\n$ ECLIPSE_HOME=/eclipse-poky/scripts/eclipse ./build.sh /&1 | tee -a build.log  \n\nHere, the three arguments to the build script are tag name, branch for documentation and release name.  \n\n",
+                    "expected_results": ""
+                },
+                "6": {
+                    "action": "On an eclipse without the Yocto Plugin, select \"Install New Software\" from Help pull-down menu  \n\n",
+                    "expected_results": ""
+                },
+                "7": {
+                    "action": "Select Add and from the dialog choose Archive...  Look for the *archive.zip file that was built previously with the build.sh script. Click OK.  \n\n",
+                    "expected_results": ""
+                },
+                "8": {
+                    "action": "Select all components and proceed with Installation of plugin. Restarting eclipse might be required.\n",
+                    "expected_results": ""
+                }
+            },
+            "summary": "Build_Eclipse_Plugin_from_source"
+        }
+    },
+    {
+        "test": {
+            "@alias": "eclipse-plugin.eclipse-plugin.Eclipse_Poky_installation_and_setup",
+            "author": [
+                {
+                    "email": "ee.peng.yeoh@intel.com",
+                    "name": "ee.peng.yeoh@intel.com"
+                }
+            ],
+            "execution": {
+                "1": {
+                    "action": "Install SDK  \n\ta)Download https://autobuilder.yocto.io/pub/releases//toolchain/x86_64/poky-glibc-x86_64-core-\timage-sato-i586-toolchain-.sh  \n\tb)Run the SDK installer and accept the default installation directory ",
+                    "expected_results": ""
+                },
+                "2": {
+                    "action": "Install \"Eclipse IDE for C/C++ Developers\" Oxygen release (4.7.0)  \n\ta) Go to https://www.eclipse.org/downloads/packages/all, click \"Oxygen R\"  \n\tb) Click to download the build for your OS  \n\tc) Click \"Download\" button to download from a mirror  \n\td) Run \"tar xf\" to extract the downloaded archive  ",
+                    "expected_result": ""
+                },
+                "3": {
+                    "action": "Install \"Eclipse IDE for C/C++ Developers\" Oxygen release (4.7.0) (Continue)  \n\te) Run \"eclipse/eclipse\" to start Eclipse  \n\tf) Optional step for host machine within Intel network: In Eclipse workbench window, go to \"Window\" menu -> \"Preferences...\".  \n\tg) In \"Preferences\" dialog, go to \"General\" -> \"Network Connections\", set \"Active Provider\" to \"Manual\". In \"Proxy \tentries\" table, select HTTP and click \"Edit\" and enter host \"proxy-chain.intel.com\" port 911, click OK. Repeat for HTTPS with port 912 \nClick OK to close \"Preferences\" dialog.  \n\th) Go to \"File\" menu -> \"Restart\" to restart Eclipse for proxy settings to take effect. ",
+                    "expected_result": ""
+                },
+                "4": {
+                    "action": "Install Eclipse Poky plugins  \n\ta) Download https://autobuilder.yocto.io/pub/releases/<yocto-version>/eclipse-plugin/<eclipse-version>/org.yocto.sdk-development-<date>-archive.zip   \n\tb) In Eclipse workbench window, go to \"Help\" menu -> \"Install New Software...\"  \n\tc) In \"Install\" dialog, click \"Add...\" button  \n\td) In \"Add Repository\" dialog, enter \"Eclipse Poky\" for (repository) Name, click \"Archive...\"  ",
+                    "expected_results": ""
+                },
+                "5": {
+                    "action": "Install Eclipse Poky plugins (continue)  \n\te) In \"Repository archive\" browse dialog, select the downloaded Eclipse Poky repository archive  \n\tf) Back in \"Add Repository\" dialog, click \"OK\"  \n\tg) Back in \"Install\" dialog, make sure \"Work with:\" is set to \"Eclipse Poky\" repository, tick \"Yocto Project \tDocumentation Plug-in\" and \"Yocto Project SDK Plug-in\", click \"Next >\" and verify plugins/features name/version, \tclick \"Next >\" and accept license agreement, click \"Finish\"  \n\th) If \"Security Warning\" dialog appears, click \"OK\" to install unsigned content.  \n\ti) In \"Software Updates\" dialog, click \"Yes\" to restart Eclipse to complete Eclipse Poky plugins installation. ",
+                    "expected_results": ""
+                },
+                "6": {
+                    "action": "Setup Eclipse Poky to use SDK  \n\ta) In Eclipse workbench window, go to \"Window\" menu -> \"Preferences\".  \n\tb) In \"Preferences\" window, go to \"Yocto Project SDK\", in \"Cross Compiler Options\" frame, select \"Standalone pre-\tbuilt toolchain\".  ",
+                    "expected_results": "Eclipse Poky plugins installed and running successfully, e.g. observe that \"Yocto Project Tools\" menu is available on Eclipse workbench window."
+                }
+            },
+            "summary": "Eclipse_Poky_installation_and_setup"
+        }
+    }
+]

meta/lib/oeqa/manual/kernel-dev.json

@@ -0,0 +1,200 @@
+[
+    {
+        "test": {
+            "@alias": "kernel-configuration.kernel-configuration.TCTEMP_2.3_MANUAL_Kernel_dev_defconfig",
+            "author": [
+                {
+                    "email": "ee.peng.yeoh@intel.com",
+                    "name": "ee.peng.yeoh@intel.com"
+                }
+            ],
+            "execution": {
+                "1": {
+                    "action": "Follow the Set Up procedure to complete the common and specific prerequisites for this test case. https://wiki.yoctoproject.org/wiki/Kernel_Development_Test_Cases#Setup https://wiki.yoctoproject.org/wiki/Kernel_Development_Test_Cases#Prerequisites_7  ",
+                    "expected_results": ""
+                },
+                "2": {
+                    "action": "Execute the test case steps asdocumented on the \"Kernel Development Test Cases\" wiki. https://wiki.yoctoproject.org/wiki/Kernel_Development_Test_Cases#Steps_7",
+                    "expected_results": "Review expected results on thethe \"Kernel Development Test Cases\"wiki. https://wiki.yoctoproject.org/wiki/Kernel_Development_Test_Cases#Expected_Results_7"
+                }
+            },
+            "summary": "TCTEMP_2.3_MANUAL_Kernel_dev_defconfig"
+        }
+    },
+    {
+        "test": {
+            "@alias": "kernel-configuration.kernel-configuration.TCTEMP_2.3_MANUAL_Kernel_dev_defconfig+fragments",
+            "author": [
+                {
+                    "email": "ee.peng.yeoh@intel.com",
+                    "name": "ee.peng.yeoh@intel.com"
+                }
+            ],
+            "execution": {
+                "1": {
+                    "action": "Follow the Set Up procedure to complete the common and specific prerequisites for this test case. https://wiki.yoctoproject.org/wiki/Kernel_Development_Test_Cases#Setup https://wiki.yoctoproject.org/wiki/Kernel_Development_Test_Cases#Prerequisites_8  ",
+                    "expected_results": ""
+                },
+                "2": {
+                    "action": "Execute the test case steps asdocumented on the \"Kernel Development Test Cases\" wiki. https://wiki.yoctoproject.org/wiki/Kernel_Development_Test_Cases#Steps_8",
+                    "expected_results": "Review expected results on thethe \"Kernel Development Test Cases\"wiki. https://wiki.yoctoproject.org/wiki/Kernel_Development_Test_Cases#Expected_Results_8"
+                }
+            },
+            "summary": "TCTEMP_2.3_MANUAL_Kernel_dev_defconfig+fragments"
+        }
+    },
+    {
+        "test": {
+            "@alias": "kernel-configuration.kernel-configuration.TCTEMP_2.3_MANUAL_Kernel_dev_Applying_patches",
+            "author": [
+                {
+                    "email": "ee.peng.yeoh@intel.com",
+                    "name": "ee.peng.yeoh@intel.com"
+                }
+            ],
+            "execution": {
+                "1": {
+                    "action": "Follow the Set Up procedure to complete the common and specific prerequisites for this test case. https://wiki.yoctoproject.org/wiki/Kernel_Development_Test_Cases#Setup https://wiki.yoctoproject.org/wiki/Kernel_Development_Test_Cases#Prerequisites  ",
+                    "expected_results": ""
+                },
+                "2": {
+                    "action": "Execute the test case steps asdocumented on the \"Kernel Development Test Cases\" wiki. https://wiki.yoctoproject.org/wiki/Kernel_Development_Test_Cases#Steps",
+                    "expected_results": "Review expected results on thethe \"Kernel Development Test Cases\"wiki. https://wiki.yoctoproject.org/wiki/Kernel_Development_Test_Cases#Expected_Results"
+                }
+            },
+            "summary": "TCTEMP_2.3_MANUAL_Kernel_dev_Applying_patches"
+        }
+    },
+    {
+        "test": {
+            "@alias": "kernel-configuration.kernel-configuration.TCTEMP_2.3_MANUAL_Kernel_dev_linux-yocto-local-source",
+            "author": [
+                {
+                    "email": "ee.peng.yeoh@intel.com",
+                    "name": "ee.peng.yeoh@intel.com"
+                }
+            ],
+            "execution": {
+                "1": {
+                    "action": "Follow the Set Up procedure to complete the common and specific prerequisites for this test case. https://wiki.yoctoproject.org/wiki/Kernel_Development_Test_Cases#Setup https://wiki.yoctoproject.org/wiki/Kernel_Development_Test_Cases#Prerequisites_2  ",
+                    "expected_results": ""
+                },
+                "2": {
+                    "action": "Execute the test case steps asdocumented on the \"Kernel Development Test Cases\" wiki. https://wiki.yoctoproject.org/wiki/Kernel_Development_Test_Cases#Steps_2",
+                    "expected_results": "Review expected results on thethe \"Kernel Development Test Cases\"wiki. https://wiki.yoctoproject.org/wiki/Kernel_Development_Test_Cases#Expected_Results_2"
+                }
+            },
+            "summary": "TCTEMP_2.3_MANUAL_Kernel_dev_linux-yocto-local-source"
+        }
+    },
+    {
+        "test": {
+            "@alias": "kernel-configuration.kernel-configuration.TCTEMP_2.3_MANUAL_Kernel_dev_linux-yocto-custom-local-source",
+            "author": [
+                {
+                    "email": "ee.peng.yeoh@intel.com",
+                    "name": "ee.peng.yeoh@intel.com"
+                }
+            ],
+            "execution": {
+                "1": {
+                    "action": "Follow the Set Up procedure to complete the common and specific prerequisites for this test case. https://wiki.yoctoproject.org/wiki/Kernel_Development_Test_Cases#Setup https://wiki.yoctoproject.org/wiki/Kernel_Development_Test_Cases#Prerequisites_3  ",
+                    "expected_results": ""
+                },
+                "2": {
+                    "action": "Execute the test case steps asdocumented on the \"Kernel Development Test Cases\" wiki. https://wiki.yoctoproject.org/wiki/Kernel_Development_Test_Cases#Steps_3",
+                    "expected_results": "Review expected results on thethe \"Kernel Development Test Cases\"wiki. https://wiki.yoctoproject.org/wiki/Kernel_Development_Test_Cases#Expected_Results_3"
+                }
+            },
+            "summary": "TCTEMP_2.3_MANUAL_Kernel_dev_linux-yocto-custom-local-source"
+        }
+    },
+    {
+        "test": {
+            "@alias": "kernel-configuration.kernel-configuration.TCTEMP_2.3_MANUAL_Kernel_dev_recipe-space_meta",
+            "author": [
+                {
+                    "email": "ee.peng.yeoh@intel.com",
+                    "name": "ee.peng.yeoh@intel.com"
+                }
+            ],
+            "execution": {
+                "1": {
+                    "action": "Follow the Set Up procedure to complete the common and specific prerequisites for this test case. https://wiki.yoctoproject.org/wiki/Kernel_Development_Test_Cases#Setup https://wiki.yoctoproject.org/wiki/Kernel_Development_Test_Cases#Prerequisites_5  ",
+                    "expected_results": ""
+                },
+                "2": {
+                    "action": "Execute the test case steps asdocumented on the \"Kernel Development Test Cases\" wiki. https://wiki.yoctoproject.org/wiki/Kernel_Development_Test_Cases#Steps_5",
+                    "expected_results": "Review expected results on thethe \"Kernel Development Test Cases\"wiki. https://wiki.yoctoproject.org/wiki/Kernel_Development_Test_Cases#Expected_Results_5"
+                }
+            },
+            "summary": "TCTEMP_2.3_MANUAL_Kernel_dev_recipe-space_meta"
+        }
+    },
+    {
+        "test": {
+            "@alias": "kernel-configuration.kernel-configuration.TCTEMP_2.3_MANUAL_Kernel_dev_External_source",
+            "author": [
+                {
+                    "email": "ee.peng.yeoh@intel.com",
+                    "name": "ee.peng.yeoh@intel.com"
+                }
+            ],
+            "execution": {
+                "1": {
+                    "action": "Follow the Set Up procedure to complete the common and specific prerequisites for this test case. https://wiki.yoctoproject.org/wiki/Kernel_Development_Test_Cases#Setup https://wiki.yoctoproject.org/wiki/Kernel_Development_Test_Cases#Prerequisites_6  ",
+                    "expected_results": ""
+                },
+                "2": {
+                    "action": "Execute the test case steps asdocumented on the \"Kernel Development Test Cases\" wiki. https://wiki.yoctoproject.org/wiki/Kernel_Development_Test_Cases#Steps_6",
+                    "expected_results": "Review expected results on thethe \"Kernel Development Test Cases\"wiki. https://wiki.yoctoproject.org/wiki/Kernel_Development_Test_Cases#Expected_Results_6"
+                }
+            },
+            "summary": "TCTEMP_2.3_MANUAL_Kernel_dev_External_source"
+        }
+    },
+    {
+        "test": {
+            "@alias": "kernel-configuration.kernel-configuration.TCTEMP_2.3_MANUAL_Kernel_dev_building_external_modules(hello-mod)",
+            "author": [
+                {
+                    "email": "ee.peng.yeoh@intel.com",
+                    "name": "ee.peng.yeoh@intel.com"
+                }
+            ],
+            "execution": {
+                "1": {
+                    "action": "Follow the Set Up procedure to complete the common and specific prerequisites for this test case. https://wiki.yoctoproject.org/wiki/Kernel_Development_Test_Cases#Setup https://wiki.yoctoproject.org/wiki/Kernel_Development_Test_Cases#Prerequisites_10  ",
+                    "expected_results": ""
+                },
+                "2": {
+                    "action": "Execute the test case steps asdocumented on the \"Kernel Development Test Cases\" wiki. https://wiki.yoctoproject.org/wiki/Kernel_Development_Test_Cases#Setup_10",
+                    "expected_results": "Review expected results on thethe \"Kernel Development Test Cases\"wiki. https://wiki.yoctoproject.org/wiki/Kernel_Development_Test_Cases#Expected_Results_10"
+                }
+            },
+            "summary": "TCTEMP_2.3_MANUAL_Kernel_dev_building_external_modules(hello-mod)"
+        }
+    },
+    {
+        "test": {
+            "@alias": "kernel-configuration.kernel-configuration.TCTEMP_2.3_MANUAL_Kernel_dev_local_parallel_meta",
+            "author": [
+                {
+                    "email": "ee.peng.yeoh@intel.com",
+                    "name": "ee.peng.yeoh@intel.com"
+                }
+            ],
+            "execution": {
+                "1": {
+                    "action": "Follow the Set Up procedure to complete the common and specific prerequisites for this test case. https://wiki.yoctoproject.org/wiki/Kernel_Development_Test_Cases#Setup https://wikioproject.org/wiki/Kernel_Development_Test_Cases#Prerequisites_4  ",
+                    "expected_results": ""
+                },
+                "2": {
+                    "action": "Execute the test case steps asdocumented on the \"Kernel Development Test Cases\" wiki. https://wiki.yoctoproject.org/wiki/Kernel_Development_Test_Cases#Steps_4",
+                    "expected_results": "Review expected results on thethe \"Kernel Development Test Cases\"wiki. https://wiki.yoctoproject.org/wiki/Kernel_Development_Test_Cases#Expected_Results_4"
+                }
+            },
+            "summary": "TCTEMP_2.3_MANUAL_Kernel_dev_local_parallel_meta"
+        }
+    }
+]

meta/lib/oeqa/manual/oe-core.json

@@ -0,0 +1,260 @@
+[
+  {
+    "test": {
+      "@alias": "oe-core.scripts.Use_scripts/pybootchartgui/pybootchartgui.py_to_generate_build_profiles",
+      "author": [
+        {
+          "email": "alexandru.c.georgescu@intel.com",
+          "name": "alexandru.c.georgescu@intel.com"
+        }
+      ],
+      "execution": {
+        "1": {
+          "action": "Run a build for a recipe (e.g. core-image-minimal)",
+          "expected_results": ""
+        },
+        "2": {
+          "action": "Run the profiling script, ../scripts/pybootchartgui/pybootchartgui.py  tmp/buildstats/    \n\n",
+          "expected_results": ""
+        },
+        "3": {
+          "action": "Verify the results ",
+          "expected_results": "The scripts generates svg files with the profiling results . "
+        }
+      },
+      "summary": "Use_scripts/pybootchartgui/pybootchartgui.py_to_generate_build_profiles"
+    }
+  },
+  {
+    "test": {
+      "@alias": "oe-core.scripts.Crosstap_script_check",
+      "author": [
+        {
+          "email": "alexandru.c.georgescu@intel.com",
+          "name": "alexandru.c.georgescu@intel.com"
+        }
+      ],
+      "execution": {
+        "1": {
+          "action": "Create the trace_open.stp script as follows in the host machine:  \n\n\nprobe syscall.open     \n\n{  \n\n\n        printf (\"%s(%d) open (%s)\\n\", execname(), pid(), argstr)  \n\n}  \n\n\n\nif the above failed, then create the below instead.  \n\nprobe syscall.open \n{ \n  printf (\"%s(%d) open\\n\", execname(), pid()) \n\n}  \n \n",
+          "expected_results": ""
+        },
+        "2": {
+          "action": "Add 'tools-profile' and 'ssh-server-openssh' to EXTRA_IMAGE_FEATURES in local.conf \n\n\n",
+          "expected_results": ""
+        },
+        "3": {
+          "action": "Build a core-image-minimal image, build systemtap-native. Start the image under qemu. \n\n",
+          "expected_results": ""
+        },
+        "4": {
+          "action": "Make sure that the ssh service is started on the Qemu machine. \n\n",
+          "expected_results": ""
+        },
+        "5": {
+          "action": "From the host machine poky build_dir, run \"crosstap root@192.168.7.2 trace_open.stp\".",
+          "expected_results": ""
+        },
+        "6": {
+          "action": "In QEMU, try to open some applications, such as open a terminal, input some command,  \n\n",
+          "expected_results": ""
+        },
+        "7": {
+          "action": "Check the host machine, \"crosstap\" has related output. \n\n\n\nNOTE:  Do not build the kernel from shared state(sstate-cache) for this to work.",
+          "expected_results": "The script should successfully connect to the qemu machine and there \nshould be presented a list of services(pid, process name) which run on \nthe qemu machine. "
+        }
+      },
+      "summary": "Crosstap_script_check"
+    }
+  },
+  {
+    "test": {
+      "@alias": "oe-core.scripts.List_all_the_PACKAGECONFIG's_flags",
+      "author": [
+        {
+          "email": "yi.zhao@windriver.com",
+          "name": "yi.zhao@windriver.com"
+        }
+      ],
+      "execution": {
+        "1": {
+          "action": " Download the poky source and setup the environment. ",
+          "expected_results": ""
+        },
+        "2": {
+          "action": "Run \"../scripts/contrib/list-packageconfig-flags.py\" ",
+          "expected_results": "In step 2, will list available pkgs which have PACKAGECONFIG flags:  \nPACKAGE NAME (or RECIPE NAME)           PACKAGECONFIG FLAGS  \n==============================================================  \nalsa-tools-1.0.26.1                                         defaultval gtk+  \navahi-ui-0.6.31                                                defaultval python  \nbluez4-4.101                                                alsa defaultval pie  \n"
+        },
+        "3": {
+          "action": "Run \"../scripts/contrib/list-packageconfig-flags.py -f\" ",
+          "expected_results": "In step 3, will list available PACKAGECONFIG flags and all affected pkgs  \nPACKAGECONFIG FLAG       PACKAGE NAMES (or RECIPE NAMES)  \n====================================  \n3g                             connman-1.16  \n        \navahi                        cups-1.6.3  pulseaudio-4.0  \nbeecrypt                   rpm-5.4.9  rpm-native-5.4.9  \n"
+        },
+        "4": {
+          "action": "Run \"../scripts/contrib/list-packageconfig-flags.py -a\" ",
+          "expected_results": "In step 4, will list all pkgs and PACKAGECONFIG information:  \n==================================================  \ngtk+-2.24.18  \n/home/jiahongxu/yocto/poky/meta/recipes-gnome/gtk+/gtk+_2.24.18.bb  \nPACKAGECONFIG x11  \nPACKAGECONFIG[x11] --with-x=yes --with-gdktarget=x11,--with-x=no,${X11DEPENDS}  \nxf86-video-intel-2.21.9  \n/home/jiahongxu/yocto/poky/meta/recipes-graphics/xorg-driver/xf86-video-intel_2.21.9.bb  \nPACKAGECONFIG None  \nPACKAGECONFIG[xvmc] --enable-xvmc,--disable-xvmc,libxvmc  \nPACKAGECONFIG[sna] --enable-sna,--disable-sna  \n"
+        },
+        "5": {
+          "action": "Run \"../scripts/contrib/list-packageconfig-flags.py -p\"   ",
+          "expected_results": "In step 5, will list pkgs with preferred version:  \nPACKAGE NAME (or RECIPE NAME)              PACKAGECONFIG FLAGS  \n===================================================  \nalsa-tools-1.0.26.1                                           defaultval gtk+  \navahi-ui-0.6.31                                                   defaultval python  \nbluez4-4.101                                                       alsa defaultval pie  \nbluez5-5.7                                                            alsa defaultval obex-profiles  \n\n\n\n "
+        }
+      },
+      "summary": "List_all_the_PACKAGECONFIG's_flags"
+    }
+  },
+  {
+    "test": {
+      "@alias": "oe-core.bitbake.Test_bitbake_menuconfig",
+      "author": [
+        {
+          "email": "jose.perez.carranza@intel.com",
+          "name": "jose.perez.carranza@intel.com"
+        }
+      ],
+      "execution": {
+        "1": {
+          "action": "clone poky  \n \n",
+          "expected_results": ""
+        },
+        "2": {
+          "action": "cd poky   \n\n",
+          "expected_results": ""
+        },
+        "3": {
+          "action": "source oe-init-build-env && cd build    \n\n",
+          "expected_results": ""
+        },
+        "4": {
+          "action": "set below in local.conf   \n\n  \tMACHINE = \"qemux86\"  \n\n",
+          "expected_results": ""
+        },
+        "5": {
+          "action": "bitbake linux-yocto -c kernel_configme -f   \n\n",
+          "expected_results": ""
+        },
+        "6": {
+          "action": "bitbake linux-yocto -c menuconfig   \n\n",
+          "expected_results": ""
+        },
+        "7": {
+          "action": "Once menuconfig launches, use the interface to navigate through the selections and \n   enable option \"64-bit kernel\"  \n\n",
+          "expected_results": ""
+        },
+        "8": {
+          "action": "Save changes and set name of the file as \"test.config\"  ",
+          "expected_results": "Open file:  \n \npoky/build//tmp/work/qemux86-poky-linux/linux-yocto/4.X.X+*/linux-qemux86-standard-build/test.config   \n  \n \n\nand verify that changes are present in the file as follows:   \n \nCONFIG_64BIT=y  \n \nCONFIG_X86_64=y"
+        }
+      },
+      "summary": "Test_bitbake_menuconfig"
+    }
+  },
+  {
+    "test": {
+      "@alias": "oe-core.bitbake.test_bitbake_devshell",
+      "author": [
+        {
+          "email": "jose.perez.carranza@intel.com",
+          "name": "jose.perez.carranza@intel.com"
+        }
+      ],
+      "execution": {
+        "1": {
+          "action": "clone poky ",
+          "expected_results": ""
+        },
+        "2": {
+          "action": "cd poky  ",
+          "expected_results": ""
+        },
+        "3": {
+          "action": "source oe-init-build-env && cd build ",
+          "expected_results": ""
+        },
+        "4": {
+          "action": "bitbake matchbox-desktop ",
+          "expected_results": "Package was build correctly "
+        },
+        "5": {
+          "action": "bitbake matchbox-desktop -c devshell ",
+          "expected_results": "A terminal with a shell prompt within the OpenEmbedded build environment is opened "
+        },
+        "6": {
+          "action": "Verify that \"matchbox-desktop\" binary file is not created under\"src\" directory ",
+          "expected_results": ""
+        },
+        "7": {
+          "action": "Run command:./configure && make ",
+          "expected_results": "Verify that \"matchbox-desktop\" binary file was created successfully under \"src/\" directory "
+        },
+        "8": {
+          "action": "Exit fromthe devshell terminal,exit ",
+          "expected_results": "Terminal back to the build directory"
+        }
+      },
+      "summary": "test_bitbake_devshell"
+    }
+  },
+  {
+    "test": {
+      "@alias": "oe-core.bitbake.test_dependency_explorer_is_launched",
+      "author": [
+        {
+          "email": "jose.perez.carranza@intel.com",
+          "name": "jose.perez.carranza@intel.com"
+        }
+      ],
+      "execution": {
+        "1": {
+          "action": "clone poky ",
+          "expected_results": ""
+        },
+        "2": {
+          "action": "cd poky ",
+          "expected_results": ""
+        },
+        "3": {
+          "action": "source oe-init-build-env ",
+          "expected_results": ""
+        },
+        "4": {
+          "action": "bitbake -u taskexp -g core-image-full-cmdline    \n\nNOTE: To execute the last command of this test, it's necessary that the machine is executing an X11 server, or if that's not the case (for example, if running the test on a headless server), it is required to enable ssh X11 forwarding on both, the server and the client, and have the X11 server running on the client.    \n\nThe instructions to enable X11 forwarding vary between distributions. But for example, these are the steps to enable it between a server running openSUSE Leap 42.1 and a client with Fedora 24:  \nA. On the server, make sure /home//.ssh/config contains the line:  \n     ForwardX11 yes  \nB. On the server, make sure xauth is installed by running:  \n      which xauth  \nC. On the client, connect to the server, enabling X11 forwarding, for example by using:  \n      ssh -X user@server    \nNOTE 2: depexp was renamed to taskexp on 2.3 M4",
+          "expected_results": "Verify that a \"dependency explorer\" is opened and file \n   dependencies are listed  "
+        }
+      },
+      "summary": "test_dependency_explorer_is_launched"
+    }
+  },
+  {
+    "test": {
+      "@alias": "oe-core.bitbake.test_bitbake_sane_error_for_invalid_layer",
+      "author": [
+        {
+          "email": "jose.perez.carranza@intel.com",
+          "name": "jose.perez.carranza@intel.com"
+        }
+      ],
+      "execution": {
+        "1": {
+          "action": "clone poky  \n",
+          "expected_results": ""
+        },
+        "2": {
+          "action": "cd poky  \n \n",
+          "expected_results": ""
+        },
+        "3": {
+          "action": "source oe-init-build-env && cd build  \n \n",
+          "expected_results": ""
+        },
+        "4": {
+          "action": "Add a invalid layer to conf/bblayers.conf  \"<poky dir>/my-invalid-layer\" \n\t\n",
+          "expected_results": ""
+        },
+        "5": {
+          "action": "bitbake core-image-minimal",
+          "expected_results": "Below error should be displayed:\n\"ERROR: Layer directory  does not exist! Please check BBLAYERS in <poky dir>/<build dir>/conf/bblayers.conf\""
+        }
+      },
+      "summary": "test_bitbake_sane_error_for_invalid_layer"
+    }
+  }
+]

meta/lib/oeqa/manual/sdk.json

@@ -0,0 +1,32 @@
+[
+    {
+        "test": {
+            "@alias": "sdk.sdk_runqemu.test_sdk_toolchain_can_run_multiple_QEMU_machines_under_UNFS",
+            "author": [
+                {
+                    "email": "ee.peng.yeoh@intel.com",
+                    "name": "ee.peng.yeoh@intel.com"
+                }
+            ],
+            "execution": {
+                "1": {
+                    "action": "Prepare kernel, rootfs tar.bz2 image, and qemu configuration  \n    \ta. Download kernel, rootfs tar.bz2 image and qemu configuration from public autobuilder webpage  \n    \tb. Goto https://autobuilder.yocto.io/pub/releases/<target_release>/machines/qemu/qemux86/ \n    \tc. Download  \n    \t  \ti. rootfs tar.bz2: core-image-sato-sdk-qemux86.tar.bz2 \n      \t\tii. kernel: bzImage-qemux86.bin \n      \t\tiii. qemu configuration: core-image-sato-sdk-qemux86.qemuboot.conf ",
+                    "expected_results": ""
+                },
+                "2": {
+                    "action": "Download & install sdk toolchain from public autobuilder \n    \ta. Goto https://autobuilder.yocto.io/pub/releases/<target_release>/toolchain/x86_64/ \n    \tb. Download poky-glibc-x86_64-core-image-sato-sdk-<type-arch>-toolchain-<release-version>.sh \n    \tc. Run command: poky-glibc-x86_64-core-image-sato-sdk-<type-arch>-toolchain-<release-version>.sh",
+                    "expected_results": ""
+                },
+                "3": {
+                    "action": "Extract rootfs twice into two images \n    \ta. Run 2 commands below:  \n                  runqemu-extract-sdk core-image-sato-sdk-qemux86.tar.bz2 qemux86_rootfs_image1  \n                  runqemu-extract-sdk core-image-sato-sdk-qemux86.tar.bz2 qemux86_rootfs_image2",
+                    "expected_results": ""
+                },
+                "4": {
+                    "action": " From the 2 terminals, start qemu to boot up both two images \n    \ta. Run 2 commands below:  \n                  runqemu core-image-sato-sdk-qemux86.qemuboot.conf qemux86_rootfs_image1  \n                  runqemu core-image-sato-sdk-qemux86.qemuboot.conf qemux86_rootfs_image2 ",
+                    "expected_results": "Expect both qemu to boot up successfully."
+                }
+            },
+            "summary": "test_sdk_toolchain_can_run_multiple_QEMU_machines_under_UNFS"
+        }
+    }
+]

meta/lib/oeqa/runtime/cases/ksample.py

@@ -34,7 +34,7 @@ class KSample(OERuntimeTestCase):
         cmd = "ls " + "/lib/modules/" + output + "/kernel/samples/" + path + module_name
         status, output = self.target.run(cmd)
         if status != 0:
-            error_info = module_name + "doesn't exist"
+            error_info = module_name + " doesn't exist"
             self.skipTest(error_info)
 
     def kfifo_func(self, name=''):

meta/lib/oeqa/runtime/cases/opkg.py

@@ -17,7 +17,10 @@ class OpkgRepoTest(OpkgTest):
 
     @classmethod
     def setUpClass(cls):
-        service_repo = os.path.join(cls.tc.td['DEPLOY_DIR_IPK'], 'all')
+        allarchfeed = 'all'
+        if cls.tc.td["MULTILIB_VARIANTS"]:
+            allarchfeed = cls.tc.td["TUNE_PKGARCH"]
+        service_repo = os.path.join(cls.tc.td['DEPLOY_DIR_IPK'], allarchfeed)
         cls.repo_server = HTTPService(service_repo, cls.tc.target.server_ip)
         cls.repo_server.start()
 

meta/lib/oeqa/selftest/cases/devtool.py

@@ -32,8 +32,9 @@ def setUpModule():
             # This helps us match exactly when we're using this path later
             canonical_layerpath += '/'
         if not edited_layers and canonical_layerpath.endswith('/meta/'):
+            canonical_layerpath = os.path.realpath(canonical_layerpath) + '/'
             edited_layers.append(layerpath)
-            oldmetapath = layerpath
+            oldmetapath = os.path.realpath(layerpath)
             result = runCmd('git rev-parse --show-toplevel', cwd=canonical_layerpath)
             oldreporoot = result.output.rstrip()
             newmetapath = os.path.join(corecopydir, os.path.relpath(oldmetapath, oldreporoot))

meta/lib/oeqa/selftest/cases/runqemu.py

@@ -166,14 +166,17 @@ class QemuTest(OESelftestTestCase):
         # when qemu was shutdown by the above shutdown command
         qemu.runner.stop_thread()
         time_track = 0
-        while True:
-            is_alive = qemu.check()
-            if not is_alive:
-                return True
-            if time_track > timeout:
-                return False
-            time.sleep(1)
-            time_track += 1
+        try:
+            while True:
+                is_alive = qemu.check()
+                if not is_alive:
+                    return True
+                if time_track > timeout:
+                    return False
+                time.sleep(1)
+                time_track += 1
+        except SystemExit:
+            return True
 
     def test_qemu_can_shutdown(self):
         self.assertExists(self.qemuboot_conf)

meta/lib/oeqa/selftest/cases/sstatetests.py

@@ -262,6 +262,7 @@ class SStateTests(SStateBase):
         self.write_config("""
 MACHINE = "qemux86"
 TMPDIR = "${TOPDIR}/tmp-sstatesamehash"
+TCLIBCAPPEND = ""
 BUILD_ARCH = "x86_64"
 BUILD_OS = "linux"
 SDKMACHINE = "x86_64"
@@ -272,6 +273,7 @@ PACKAGE_CLASSES = "package_rpm package_ipk package_deb"
         self.write_config("""
 MACHINE = "qemux86"
 TMPDIR = "${TOPDIR}/tmp-sstatesamehash2"
+TCLIBCAPPEND = ""
 BUILD_ARCH = "i686"
 BUILD_OS = "linux"
 SDKMACHINE = "i686"
@@ -307,12 +309,14 @@ PACKAGE_CLASSES = "package_rpm package_ipk package_deb"
 
         self.write_config("""
 TMPDIR = \"${TOPDIR}/tmp-sstatesamehash\"
+TCLIBCAPPEND = \"\"
 NATIVELSBSTRING = \"DistroA\"
 """)
         self.track_for_cleanup(self.topdir + "/tmp-sstatesamehash")
         bitbake("core-image-sato -S none")
         self.write_config("""
 TMPDIR = \"${TOPDIR}/tmp-sstatesamehash2\"
+TCLIBCAPPEND = \"\"
 NATIVELSBSTRING = \"DistroB\"
 """)
         self.track_for_cleanup(self.topdir + "/tmp-sstatesamehash2")
@@ -340,10 +344,12 @@ NATIVELSBSTRING = \"DistroB\"
 
         configA = """
 TMPDIR = \"${TOPDIR}/tmp-sstatesamehash\"
+TCLIBCAPPEND = \"\"
 MACHINE = \"qemux86-64\"
 """
         configB = """
 TMPDIR = \"${TOPDIR}/tmp-sstatesamehash2\"
+TCLIBCAPPEND = \"\"
 MACHINE = \"qemuarm\"
 """
         self.sstate_allarch_samesigs(configA, configB)
@@ -356,6 +362,7 @@ MACHINE = \"qemuarm\"
 
         configA = """
 TMPDIR = \"${TOPDIR}/tmp-sstatesamehash\"
+TCLIBCAPPEND = \"\"
 MACHINE = \"qemux86-64\"
 require conf/multilib.conf
 MULTILIBS = \"multilib:lib32\"
@@ -363,6 +370,7 @@ DEFAULTTUNE_virtclass-multilib-lib32 = \"x86\"
 """
         configB = """
 TMPDIR = \"${TOPDIR}/tmp-sstatesamehash2\"
+TCLIBCAPPEND = \"\"
 MACHINE = \"qemuarm\"
 require conf/multilib.conf
 MULTILIBS = \"\"
@@ -407,6 +415,7 @@ MULTILIBS = \"\"
 
         self.write_config("""
 TMPDIR = \"${TOPDIR}/tmp-sstatesamehash\"
+TCLIBCAPPEND = \"\"
 MACHINE = \"qemux86\"
 require conf/multilib.conf
 MULTILIBS = "multilib:lib32"
@@ -416,6 +425,7 @@ DEFAULTTUNE_virtclass-multilib-lib32 = "x86"
         bitbake("world meta-toolchain -S none")
         self.write_config("""
 TMPDIR = \"${TOPDIR}/tmp-sstatesamehash2\"
+TCLIBCAPPEND = \"\"
 MACHINE = \"qemux86copy\"
 require conf/multilib.conf
 MULTILIBS = "multilib:lib32"
@@ -451,6 +461,7 @@ DEFAULTTUNE_virtclass-multilib-lib32 = "x86"
 
         self.write_config("""
 TMPDIR = "${TOPDIR}/tmp-sstatesamehash"
+TCLIBCAPPEND = ""
 BB_NUMBER_THREADS = "${@oe.utils.cpu_count()}"
 PARALLEL_MAKE = "-j 1"
 DL_DIR = "${TOPDIR}/download1"
@@ -464,6 +475,7 @@ http_proxy = ""
         bitbake("world meta-toolchain -S none")
         self.write_config("""
 TMPDIR = "${TOPDIR}/tmp-sstatesamehash2"
+TCLIBCAPPEND = ""
 BB_NUMBER_THREADS = "${@oe.utils.cpu_count()+1}"
 PARALLEL_MAKE = "-j 2"
 DL_DIR = "${TOPDIR}/download2"

meta/lib/oeqa/utils/qemurunner.py

@@ -350,10 +350,10 @@ class QemuRunner:
         return True
 
     def stop(self):
-        self.stop_thread()
-        self.stop_qemu_system()
         if hasattr(self, "origchldhandler"):
             signal.signal(signal.SIGCHLD, self.origchldhandler)
+        self.stop_thread()
+        self.stop_qemu_system()
         if self.runqemu:
             if hasattr(self, "monitorpid"):
                 os.kill(self.monitorpid, signal.SIGKILL)

meta/recipes-connectivity/bind/bind/0001-avoid-start-failure-with-bind-user.patch

@@ -0,0 +1,27 @@
+From 31dde3562f287429eea94b77250d184818b49063 Mon Sep 17 00:00:00 2001
+From: Chen Qi <Qi.Chen@windriver.com>
+Date: Mon, 15 Oct 2018 16:55:09 +0800
+Subject: [PATCH] avoid start failure with bind user
+
+Upstream-Status: Pending
+
+Signed-off-by: Chen Qi <Qi.Chen@windriver.com>
+---
+ init.d | 1 +
+ 1 file changed, 1 insertion(+)
+
+diff --git a/init.d b/init.d
+index b2eec60..6e03936 100644
+--- a/init.d
++++ b/init.d
+@@ -57,6 +57,7 @@ case "$1" in
+ 	modprobe capability >/dev/null 2>&1 || true
+ 	if [ ! -f /etc/bind/rndc.key ]; then
+ 	    /usr/sbin/rndc-confgen -a -b 512 -r /dev/urandom
++	    chown root:bind /etc/bind/rndc.key >/dev/null 2>&1 || true
+ 	    chmod 0640 /etc/bind/rndc.key
+ 	fi
+ 	if [ -f /var/run/named/named.pid ]; then
+-- 
+2.7.4
+